修改原有Redis 不删除key的问题,新增恢复通知

app/index.js

@@ -2,7 +2,7 @@ const Koa = require('koa')
 const Router = require('koa-router')
 const bodyParser = require('koa-bodyparser')
 const log4js = require('koa-log4')
-const schedule = require('./schedule')
+const { flushAll, checkError, checkRecover } = require('./schedule')
 const logConf = require('../config/logger')
 const error = require('../middleware/error')
 const result = require('../middleware/result')
@@ -50,8 +50,9 @@ exports.start = function () {
   // 加载各种服务
   const app = new Koa()
   const router = new Router()
-  // 开启定时任务 每分钟检查一次
-  schedule('0 */1 * * * ?')
+  checkError('0 */2 * * * ?')
+  checkRecover('0 */1 * * * ?')
+  flushAll('0 0 9 * * ?')
   // 加载所有路由
   loadRoutes(router)
   deploy()

app/schedule.js

@@ -2,9 +2,9 @@ const schedule = require('node-schedule')
 const request = require('request')
 const moment = require('moment')
 const Redis = require('ioredis')
-const logger = require('koa-log4').getLogger()
-const { podGetstatus } = require('../kubeService/service')
-
+const logger = require('koa-log4')
+  .getLogger()
+const { podGetstatus, getPods } = require('../kubeService/service')
 
 const redis = new Redis(6380, '172.30.220.22')
 const awaitRequest = function (options) {
@@ -18,27 +18,52 @@ const awaitRequest = function (options) {
     })
   })
 }
-const dingTalkPush = async function (item) {
-  const status = Object.keys(item.status.containerStatuses[0].state)[0]
-  let message = ''
+const dingTalkPush = async function (item, is_recover) {
+  // const key = `${item.metadata.namespace}:${item.metadata.name}#${item.metadata.labels['qcloud-app']}`
+  let message
+  let status
+  switch (item.status.conditions.length) {
+    case 1:
+      message = item.status.conditions[0].reason
+      break;
+    case 2:
+      break
+    case 3:
+      status = Object.keys(item.status.containerStatuses[0].state)[0]
       if (status === 'running') {
         message = '服务启动可能出现错误,请及时查看'
       } else {
         message = item.status.containerStatuses[0].state[status].reason
       }
-  const dingData = {
+      break;
+    default:
+      break
+  }
+  const dingData = is_recover ? {
     msgtype: 'markdown',
     markdown: {
       title: 'pipeline项目添加信息如下',
-      text: '> 描述信息 : 腾讯云服务异常提醒通知\n\n'
+      text: '> 描述信息 : 腾讯云服务---恢复正常通知\n\n'
+        + `> 项目名称 : ${item.metadata.labels['qcloud-app']}\n\n`
+        + `> 项目类型 : ${item.metadata.labels.type}\n\n`
+        + `> 命名空间 : ${item.metadata.namespace}\n\n`
+        + `> 恢复时间 : ${moment()
+          .format('YYYY-MM-DD HH:mm:ss')}\n\n`,
+    },
+  } : {
+    msgtype: 'markdown',
+    markdown: {
+      title: 'pipeline项目添加信息如下',
+      text: '> 描述信息 : 腾讯云服务---异常提醒通知\n\n'
         + `> 项目名称 : ${item.metadata.labels['qcloud-app']}\n\n`
         + `> 项目类型 : ${item.metadata.labels.type}\n\n`
         + `> 命名空间 : ${item.metadata.namespace}\n\n`
         + `> 异常原因 : ${message}\n\n`
-        + `> 异常时间 : ${moment().format('YYYY-MM-DD HH:mm:ss')}\n\n`,
+        + `> 异常时间 : ${moment()
+          .format('YYYY-MM-DD HH:mm:ss')}\n\n`,
     },
-  }
-  const res = await awaitRequest({
+  };
+  await awaitRequest({
     url: 'https://oapi.dingtalk.com/robot/send?access_token=473e49a1b6d4952e2306e0a8e530573384a6340052c40365a141b30757fd0997',
     method: 'POST',
     headers: {
@@ -46,43 +71,98 @@ const dingTalkPush = async function (item) {
     },
     body: JSON.stringify(dingData),
   })
-  if (JSON.parse(res).errcode === 0) {
-    await redis.set(`${item.metadata.namespace}:${item.metadata.name}`, 'send')
+  // if (JSON.parse(res).errcode === 0 && is_recover) {
+  //   await redis.set(key, 'send')
+  // } else {
+  //   logger.error(res.errmsg)
+  // }
+}
+const checkRecoverPod = async () => {
+  let stream = redis.scanStream({
+    match: '*#*',
+  });
+  stream.on('data', async (resKeys) => {
+    for (let i = 0; i < resKeys.length; i += 1) {
+      const namespace = resKeys[i].split(':')[0]
+      const podname = resKeys[i].split('#')[1]
+      // 获取新的状态
+      if (podname.indexOf(resKeys[i].split('#')[1]) > -1) {
+        const Pod = await getPods(namespace)
+        Pod.body.items.forEach(async (item) => {
+          if (item.metadata.name.indexOf(podname) > -1) {
+            if (item.status.conditions.length === 3) {
+              const c1 = item.status.conditions[0].status === 'True'
+              const c2 = item.status.conditions[1].status === 'True'
+              const c3 = item.status.conditions[2].status === 'True'
+              if (c1 && c2 && c3) {
+                const result = await redis.get(`copy${namespace}&${resKeys[i].split('#')[1]}`)
+                const afterKey = await redis.get(resKeys[i])
+                // 位置不可变 否则无法发送钉钉推送
+                const flag = result || afterKey
+                if (flag === 'send') {
+                  dingTalkPush(item, true)
+                  await redis.del(resKeys[i])
+                  await redis.del(`copy${namespace}&${resKeys[i].split('#')[1]}`)
                 } else {
-    logger.error(res.errmsg)
+                  await redis.del(resKeys[i])
+                  await redis.del(`copy${namespace}&${resKeys[i].split('#')[1]}`)
+                }
+              }
+            }
           }
+        })
+      }
+    }
+  })
+  stream.on('end', () => {
+    stream = null
+  });
 }
-const job = async () => {
+const checkErrorPod = async () => {
   const listPods = await podGetstatus()
   listPods.body.items.forEach((item) => {
+    const key = `${item.metadata.namespace}:${item.metadata.name}#${item.metadata.labels['qcloud-app']}`
     item.status.conditions.forEach(async (value) => {
       if (value.status !== 'True') {
-        const res = await redis.get(`${item.metadata.namespace}:${item.metadata.name}`)
-
+        const res = await redis.get(key)
         if (res != null) {
           if (res === 'send') {
             logger.info(item.metadata.name, ':已到达阈值并已发送钉钉提醒服务')
           } else {
             const counter = Number(res)
             if (counter > 5) {
-              await redis.set(`${item.metadata.namespace}:${item.metadata.name}`, counter + 1)
-              // const res = await podlog(item.metadata.namespace, item.metadata.name)
-              dingTalkPush(item, counter)
+              await redis.set(key, 'send')
+              dingTalkPush(item, false)
             } else {
-              await redis.set(`${item.metadata.namespace}:${item.metadata.name}`, counter + 1)
+              await redis.set(key, counter + 1)
             }
           }
         } else {
-          await redis.set(`${item.metadata.namespace}:${item.metadata.name}`, 1)
-          // 提醒的推送通知一天重置一次,避免多次提醒。
-          await redis.expire(`${item.metadata.namespace}:${item.metadata.name}`, 3600 * 24)
+          await redis.set(key, 1)
         }
       }
     })
   })
 }
-module.exports = (cron) => {
+// checkRecoverPod()
+// checkErrorPod()
+const checkError = (cron) => {
   schedule.scheduleJob(cron, () => {
-    job()
-  });
+    checkErrorPod()
+  })
+}
+const checkRecover = (cron) => {
+  schedule.scheduleJob(cron, () => {
+    checkRecoverPod()
+  })
+}
+const flushAll = (cron) => {
+  schedule.scheduleJob(cron, async () => {
+    await redis.flushall()
+  })
+}
+module.exports = {
+  flushAll,
+  checkError,
+  checkRecover,
 }

app/service.js

 const Router = require('koa-router')
 const logger = require('koa-log4').getLogger()
 const _ = require('lodash')
+const Redis = require('ioredis')
 
+const redis = new Redis(6380, '172.30.220.22')
 const { ingressCreate, ingressDelete } = require('../kubeService/ingress')
 const { projectConfig, defaultConfig } = require('../serviceTemplate/resourceLimit')
 const {
@@ -67,10 +69,15 @@ router.post('/details', async (ctx) => {
   const data = await getServiceDetail(ctx.request.body.namespace, ctx.request.body.serviceName, ctx.request.body.type)
   ctx.body = ctx.ok(data)
 })
-
 router.post('/delete', async (ctx) => {
-  const { namespace, serviceName } = ctx.request.body
-
+  const { namespace, serviceName, podName } = ctx.request.body
+  const key = `${namespace}:${podName}#${serviceName}`
+  console.log(key)
+  const res = await redis.get(key)
+  await redis.del(key)
+  if (res) {
+    await redis.set(`copy${namespace}&${serviceName}`, res)
+  }
   await serviceDelete(namespace, serviceName)
   await replicaSetDelete(namespace, serviceName)
   await pvcDelete(namespace, serviceName)
@@ -111,6 +118,12 @@ router.post('/modifyImage', async (ctx) => {
 })
 
 router.post('/redeploy', async (ctx) => {
+  const key = `${ctx.request.body.namespace}:${ctx.request.body.podName}#${ctx.request.body.serviceName}`
+  const res = await redis.get(key)
+  await redis.del(key)
+  if (res) {
+    await redis.set(`copy${ctx.request.body.namespace}&${ctx.request.body.serviceName}`, res)
+  }
   await serviceRestart(ctx.request.body.namespace, ctx.request.body.podName)
   ctx.body = ctx.ok('重置服务成功')
 })

kubeService/service.js

@@ -6,7 +6,7 @@ const moment = require('moment')
 const yaml = require('js-yaml')
 const logger = require('koa-log4').getLogger('kubeService')
 const yamls = require('../yamls')
-const APP_CONFIG = require('../config')
+const dict = require('./dictService')
 
 const client = new Client({
   config: config.fromKubeconfig(
@@ -52,13 +52,8 @@ const serviceCreate = async (data) => {
         break;
 
       case 'Deployment':
-        let obj = jsonObj
-        if (APP_CONFIG.noHealthCheckApp.includes(serviceName)) {
-          obj = _.omit(jsonObj, ['spec.template.spec.containers[0].readinessProbe'])
-        }
-        logger.info('创建deploy', serviceName, JSON.stringify(obj))
-
-        await client.apis.apps.v1beta1.namespaces(namespace).deployments.post({ body: obj })
+        logger.info('创建deploy', JSON.stringify(jsonObj))
+        await client.apis.apps.v1beta1.namespaces(namespace).deployments.post({ body: jsonObj })
         break;
 
       case 'PersistentVolumeClaim':
@@ -93,7 +88,7 @@ const serviceUpdate = async (data) => {
   for (const item of yamlArray) {
     const jsonObj = yaml.load(item);
     if (jsonObj.kind === 'Deployment') {
-      logger.info('更新deploy:', JSON.stringify(jsonObj))
+      logger.info('Deployment:', JSON.stringify(jsonObj))
       await client.apis.apps.v1beta1.namespaces(namespace).deployments(serviceName).put({ body: jsonObj })
     }
   }
@@ -101,11 +96,9 @@ const serviceUpdate = async (data) => {
 
 const formatServiceInfo = (obj) => {
   const portObj = {}
-  if (obj.spec.type === 'NodePort') {
   obj.spec.ports.forEach((i) => {
-      portObj[`port_${i.port}`] = i.nodePort
+    portObj[`${i.name}_port`] = i.nodePort
   })
-  }
 
   return _.assign(portObj, {
     clusterIp: obj.spec.clusterIP,
@@ -139,8 +132,8 @@ const formatPodInfo = (podInfo) => {
   // }
   const podStatus = getPodStatus(podInfo)
 
-  const imageID = _.get(podInfo.status.containerStatuses, '[0].imageID', '')
-  const image = _.get(podInfo.spec.containers, '[0].image', '')
+  // const containerImage = _.get(podInfo.status.containerStatuses, '[0].image', '')
+  const containerImage = _.get(podInfo.spec.containers, '[0].image', '')
 
   const ret = {
     serviceName: (podInfo.metadata.labels && podInfo.metadata.labels['qcloud-app']) || podInfo.metadata.name,
@@ -150,13 +143,14 @@ const formatPodInfo = (podInfo) => {
     lanIp: podInfo.status.hostIP,
     startTime: podInfo.status.startTime,
     createdAt: moment(new Date(podInfo.status.startTime)).startOf('minute').fromNow(),
-    image,
-    imageID,
+    image: containerImage,
     labels: podInfo.metadata.labels,
   }
 
-  if (image !== '') {
-    ret.branch = image.split(':')[1]
+  if (containerImage !== '') {
+    ret.branch = containerImage.split(':')[1]
+    ret.branch_no_time = containerImage.split(':')[1].split('-')[0]
+    ret.branch_time = containerImage.split(':')[1].split('-')[1]
   }
 
   return ret
@@ -227,7 +221,6 @@ const getServiceDetail = async (namespace, name, type) => {
 }
 
 const serviceRestart = async (namespace, name) => {
-  logger.info('重置服务', namespace, name)
   await client.api.v1.namespaces(namespace).pods(name).delete()
 }
 
@@ -238,7 +231,7 @@ const serviceDelete = async (namespace, name) => {
     logger.info('删除svc', namespace, name)
     await client.api.v1.namespaces(namespace).services(name).delete()
   } catch (error) {
-    logger.warn(error.toString())
+    logger.error(error)
   }
 }
 
@@ -253,15 +246,6 @@ const replicaSetDelete = async (namespace, name) => {
   // await client.apis.apps.v1.namespaces(namespace).replicasets(rsName).delete()
 }
 
-const pvcDelete = async (namespace, name) => {
-  try {
-    logger.info('删除pvc', namespace, name)
-    await client.api.v1.namespaces(namespace).persistentvolumeclaim(`${name}-${namespace}`).delete()
-  } catch (error) {
-    logger.warn(error.toString())
-  }
-}
-
 const getServices = async (namespace) => {
   const data = await client.api.v1.namespaces(namespace).services.get()
   return data
@@ -280,5 +264,4 @@ module.exports = {
   imageUpdate,
   getReplicaSet,
   replicaSetDelete,
-  pvcDelete,
 }