存活检查

kubeService/service.js

@@ -183,28 +183,35 @@ const getPodStatus = (podInfo) => {
     return 'Pending'
   }
   if (podInfo.status.conditions) {
-    if (podInfo.status.conditions[2].status === 'False') {
-      return 'PodScheduling'
-    }
-    if (podInfo.status.conditions[0].status === 'False') {
-      return 'Initializing'
-    }
-    if (podInfo.status.conditions[1].status === 'False') {
-      return 'Waiting'
+    for (const item of podInfo.status.conditions) {
+      if (item.type === 'Initialized' && item.status === 'False') {
+        return 'Initializing'
+      }
+      if (item.type === 'PodScheduled' && item.status === 'False') {
+        return 'PodScheduling'
+      }
+      if (item.type === 'ContainersReady' && item.status === 'False') {
+        return 'ContainerCreating'
+      }
+      if (item.type === 'Ready' && item.status === 'False') {
+        return 'Waiting'
+      }
+      if (item.type === 'Ready' && item.status === 'True') {
+        return 'Normal'
+      }
     }
   }
   if (podInfo.metadata.deletionTimestamp) {
     return 'Terminating'
   }
-  return 'Normal'
 }
 
 const formatPodInfo = (podInfo) => {
-  // if (podInfo.metadata.name.indexOf('xyqb') !== -1) {
-  //   console.log(1, JSON.stringify(podInfo), podInfo.metadata.labels['qcloud-app'])
-  // }
   const podStatus = getPodStatus(podInfo)
-
+  // for debug
+  // if (podInfo.metadata.name.indexOf('xyqb-user2') !== -1) {
+  //   console.log(1, podStatus, JSON.stringify(podInfo.status.conditions))
+  // }
   const imageID = _.get(podInfo.status.containerStatuses, '[0].imageID', '')
   const image = _.get(podInfo.spec.containers, '[0].image', '')
 

middleware/client/index.js

@@ -5,7 +5,14 @@ const config = require('kubernetes-client').config
 module.exports = function client() {
   const client = {}
   return async function (ctx, next) {
-    const exclude = ['/service/listEnvVarsNew', '/service/listEnvVars', '/cluster']
+    const exclude = [
+      '/service/listEnvVarsNew',
+      '/service/listEnvVars',
+      '/cluster',
+      '/repository',
+      '/repository/getTags',
+      '/repository/getNamespaceInfo',
+    ]
     if (exclude.includes(ctx.path)) {
       await next()
     } else {

yamls/java.yaml

@@ -84,7 +84,7 @@ spec:
           initialDelaySeconds: 30
           timeoutSeconds: 2
           # 默认值10
-          periodSeconds: 10
+          periodSeconds: 5
       volumes:
         - name: filelog
           hostPath:

yamls/mongo.pvc.yaml

@@ -51,13 +51,17 @@ spec:
             value: qa
           - name: MONGO_INITDB_ROOT_PASSWORD
             value: qatest
+        livenessProbe:
+          exec:
+            command: ["mongo", "-uqa", "-pqatest","admin"]
+          initialDelaySeconds: 100
+          successThreshold: 1
         readinessProbe:
           exec:
             command: ["mongo", "-uqa", "-pqatest","admin"]
           initialDelaySeconds: 5
-          periodSeconds: 2
+          periodSeconds: 5
           timeoutSeconds: 1
-          failureThreshold: 40
         volumeMounts:
           - mountPath: "/var/lib/mongo"
             name: {{serviceName}}

yamls/mysql.pvc.yaml

@@ -72,13 +72,17 @@ spec:
               value: Quantgroup2017
             - name: MYSQL_USER
               value: qa
+          livenessProbe:
+            exec:
+              command: ["mysql", "-uqa", "-pqatest", "-e", "SELECT 1"]
+            initialDelaySeconds: 100
+            successThreshold: 1
           readinessProbe:
             exec:
               command: ["mysql", "-uqa", "-pqatest", "-e", "SELECT 1"]
             initialDelaySeconds: 5
-            periodSeconds: 2
+            periodSeconds: 5
             timeoutSeconds: 1
-            failureThreshold: 40
           volumeMounts:
             - mountPath: "/var/lib/mysql"
               name: {{serviceName}}

yamls/rabbitmq.pvc.yaml

@@ -53,6 +53,12 @@ spec:
             value: qa
           - name: RABBITMQ_DEFAULT_PASS
             value: qatest
+        livenessProbe:
+          exec:
+            command:
+              - /opt/readyCheck.sh
+          initialDelaySeconds: 100
+          successThreshold: 1
         readinessProbe:
           exec:
             command:
@@ -61,12 +67,10 @@ spec:
           timeoutSeconds: 2
           periodSeconds: 5
           successThreshold: 1
-          failureThreshold: 40
         # 数据持久化
         volumeMounts:
           - name: {{serviceName}}
             mountPath: "/var/lib/rabbitmq"
-
       volumes:
         - name: {{serviceName}}
           persistentVolumeClaim:

yamls/redis.yaml

@@ -32,13 +32,17 @@ spec:
             limits:
               cpu: {{resources.cpu.limit}}m
               memory: {{resources.memory.limit}}Mi
+          livenessProbe:
+            exec:
+              command: ["redis-cli", "-p", "6379","info"]
+            initialDelaySeconds: 100
+            successThreshold: 1
           readinessProbe:
             exec:
               command: ["redis-cli", "-p", "6379","info"]
             initialDelaySeconds: 5
-            periodSeconds: 2
             timeoutSeconds: 1
-            failureThreshold: 40
+            periodSeconds: 5
         - image: ccr.ccs.tencentyun.com/{{image}}
           imagePullPolicy: Always
           name: {{serviceName}}-6380
@@ -50,13 +54,17 @@ spec:
             limits:
               cpu: {{resources.cpu.limit}}m
               memory: 200Mi
+          livenessProbe:
+            exec:
+              command: ["redis-cli", "-p", "6380","info"]
+            initialDelaySeconds: 100
+            successThreshold: 1
           readinessProbe:
             exec:
               command: ["redis-cli", "-p", "6380","info"]
             initialDelaySeconds: 5
-            periodSeconds: 2
             timeoutSeconds: 1
-            failureThreshold: 40
+            periodSeconds: 5
         - image: ccr.ccs.tencentyun.com/{{image}}
           imagePullPolicy: Always
           name: {{serviceName}}-6381
@@ -68,13 +76,17 @@ spec:
             limits:
               cpu: {{resources.cpu.limit}}m
               memory: {{resources.memory.limit}}Mi
+          livenessProbe:
+            exec:
+              command: ["redis-cli", "-p", "6381","info"]
+            initialDelaySeconds: 100
+            successThreshold: 1
           readinessProbe:
             exec:
               command: ["redis-cli", "-p", "6381","info"]
             initialDelaySeconds: 5
-            periodSeconds: 2
             timeoutSeconds: 1
-            failureThreshold: 40
+            periodSeconds: 5
         - image: ccr.ccs.tencentyun.com/{{image}}
           imagePullPolicy: Always
           name: {{serviceName}}-6382
@@ -86,13 +98,17 @@ spec:
             limits:
               cpu: {{resources.cpu.limit}}m
               memory: {{resources.memory.limit}}Mi
+          livenessProbe:
+            exec:
+              command: ["redis-cli", "-p", "6382","info"]
+            initialDelaySeconds: 100
+            successThreshold: 1
           readinessProbe:
             exec:
               command: ["redis-cli", "-p", "6382","info"]
             initialDelaySeconds: 5
-            periodSeconds: 2
             timeoutSeconds: 1
-            failureThreshold: 40
+            periodSeconds: 5
         - image: ccr.ccs.tencentyun.com/{{image}}
           imagePullPolicy: Always
           name: {{serviceName}}-6383
@@ -104,13 +120,17 @@ spec:
             limits:
               cpu: {{resources.cpu.limit}}m
               memory: {{resources.memory.limit}}Mi
+          livenessProbe:
+            exec:
+              command: ["redis-cli", "-p", "6383","info"]
+            initialDelaySeconds: 100
+            successThreshold: 1
           readinessProbe:
             exec:
               command: ["redis-cli", "-p", "6383","info"]
             initialDelaySeconds: 5
-            periodSeconds: 2
             timeoutSeconds: 1
-            failureThreshold: 40
+            periodSeconds: 5
       dnsPolicy: ClusterFirst
       imagePullSecrets:
         - name: qcloudregistrykey

yamls/ui.node.yaml

@@ -63,15 +63,20 @@ spec:
           limits:
             cpu: {{resources.cpu.limit}}m
             memory: {{resources.memory.limit}}Mi
+        livenessProbe:
+          exec:
+            command:
+              - /home/quant_group/readyCheck.sh
+          initialDelaySeconds: 100
+          successThreshold: 1
         readinessProbe:
           exec:
             command:
               - /home/quant_group/readyCheck.sh
           initialDelaySeconds: 2
-          timeoutSeconds: 2
-          periodSeconds: 3
+          timeoutSeconds: 1
+          periodSeconds: 5
           successThreshold: 1
-          failureThreshold: 40
       volumes:
         - name: filelog
           hostPath:

yamls/zookeeper.pvc.yaml

@@ -35,8 +35,6 @@ spec:
         qcloud-app: {{serviceName}}
         type: base
     spec:
-      nodeSelector:
-        zone: "3"
       containers:
       - image: ccr.ccs.tencentyun.com/{{image}}
         imagePullPolicy: Always
@@ -48,14 +46,21 @@ spec:
           limits:
             cpu: {{resources.cpu.limit}}m
             memory: {{resources.memory.limit}}Mi
+        livenessProbe:
+          exec:
+            command:
+              - /zookeeper-3.4.13/readyCheck.sh
+          initialDelaySeconds: 100
+          successThreshold: 1
+          # 默认值3
+          failureThreshold: 3
         readinessProbe:
           exec:
             command:
               - /zookeeper-3.4.13/readyCheck.sh
           initialDelaySeconds: 5
-          periodSeconds: 2
           timeoutSeconds: 1
-          failureThreshold: 40
+          periodSeconds: 5
         volumeMounts:
           - mountPath: "/var/lib/zookeeper"
             name: {{serviceName}}