资源限制

.eslintrc.js

@@ -14,6 +14,7 @@ module.exports = {
     "no-await-in-loop": "off",
     "consistent-return": "off",
     "no-shadow" :"off",
+    "no-unused-expressions":"off",
     "no-useless-escape": "off",
     "max-len": ["error", {"code": 140}]
   }

app/controller/resource.js

+const Router = require('koa-router')
+
+
+const { CpuResource, MemoryResource } = require('../service/mongoService')
+
+const router = new Router()
+module.exports = router
+const getCpu = async (ctx) => {
+  const res = await CpuResource.getCpu(ctx.request.query)
+  ctx.body = ctx.ok(res)
+}
+
+const getMemory = async (ctx) => {
+  const res = await MemoryResource.getMemory(ctx.request.query)
+  ctx.body = ctx.ok(res)
+}
+
+
+router
+  .get('/cpu', getCpu)
+  .get('/memory', getMemory)

app/index.js

@@ -26,6 +26,7 @@ const repo = require('./controller/repo')
 const envTemplate = require('./controller/envTemplate')
 const env = require('./controller/env')
 const pipeline = require('./controller/pipeline')
+const resource = require('./controller/resource')
 // todo: 引入auth中间件
 router
   .use(error())
@@ -49,6 +50,7 @@ router
   .use('/envTemplate', envTemplate.routes())
   .use('/env', env.routes())
   .use('/pipeline', pipeline.routes())
+  .use('/resource', resource.routes())
 
 
 module.exports = router

app/jobs/cpuRate.js

+const schedule = require('node-schedule')
+const logger = require('koa-log4').getLogger('jobs:calculation cpu')
+const moment = require('moment')
+const { CpuResource } = require('../service/mongoService')
+const quick = require('../utils/quickSort')
+const awaitRequest = require('../utils/awaitRequest')
+
+const config = require(global.configPath)
+
+
+const endTime = Math.round(moment().valueOf() / 1000)
+const startTime = endTime - (60 * 60 * 12)
+async function job() {
+  const cpuLimitRes = await awaitRequest({
+    url: 'http://172.17.5.17:30002/api/v1/query?query=avg (kube_pod_container_resource_limits_cpu_cores{}) by (container)',
+    method: 'GET',
+  })
+  /**
+   *将cpulimit数据整合成{k:v,k:v}
+   */
+  const finalCpuLimit = {}
+  cpuLimitRes.data.result.forEach((item) => {
+    finalCpuLimit[item.metric.container] = item.value[1]
+  })
+  const cpuUsageRes = await awaitRequest({
+    url: `http://172.17.5.17:30002/api/v1/query_range?query=sum (rate (container_cpu_usage_seconds_total
+    {image!="",name=~"^k8s_.*",io_kubernetes_container_name!="POD",pod_name=~"^()()().*$"}[1m])) by
+     (pod_name)&start=${startTime}&end=${endTime}&step=300`,
+    method: 'GET',
+  })
+  /**
+   * 筛选不重复pod的最大值
+   *  格式:[{ k: 'account-center-85f4d56d9b-qwr9r',v: '0.002291927622220808' },{}]
+   */
+  const finalCpuUsage = []
+  cpuUsageRes.data.result.forEach((item) => {
+    const sortArr = quick(item.values)
+    const k = item.metric.pod_name
+    const v = sortArr[sortArr.length - 1][1]
+    finalCpuUsage.push({ k, v })
+  })
+  /**
+   * 再次筛选数据,去掉重复的pod，并获取最大值
+   * 格式:{k:v,k:v}
+   */
+  const finalRealData = {}
+  Object.keys(finalCpuLimit).forEach((item) => {
+    let num = 0
+    finalCpuUsage.forEach((value) => {
+      if (value.k.indexOf(item) > -1) {
+        if (value.v > num) {
+          num = value.v
+        }
+      }
+    })
+    finalRealData[item] = num
+  })
+  /**
+   * 整合成最后数据[]finalSendData
+   */
+  const finalSendData = []
+  Object.keys(finalCpuLimit).forEach((item) => {
+    finalSendData.push({
+      name: item,
+      cpuLimit: finalCpuLimit[item],
+      cpuMaxUsage: finalRealData[item],
+      Percentage: `${(finalRealData[item] / finalCpuLimit[item]).toFixed(3)}`,
+    })
+  })
+  function quickSort(arr) {
+    if (arr.length <= 1) {
+      return arr
+    }
+    const middleIndex = Math.floor(arr.length / 2);
+    const middleValue = arr.splice(middleIndex, 1);
+    const arrLeft = [];
+    const arrRight = [];
+    for (let i = 0; i < arr.length; i += 1) {
+      arr[i].Percentage < middleValue[0].Percentage ? arrLeft.push(arr[i]) : arrRight.push(arr[i]);
+    }
+    return quickSort(arrLeft).concat(middleValue, quickSort(arrRight))
+  }
+
+  const saveData = quickSort(finalSendData)
+  const res = await CpuResource.findOne({ create_time: moment().format('YYYY-MM-DD') })
+  if (!res) {
+    await CpuResource.saveCpu({ create_time: moment().format('YYYY-MM-DD'), cpu: saveData })
+  }
+  const dingdingMessage = {
+    msgtype: 'link',
+    link: {
+      text: `cpu统计告知:${moment().format('YYYY-MM-DD HH-mm-ss')}`,
+      title: 'cpu统计已同步更新到qa-home下的resource目录，点击可查看详情',
+      messageUrl: `${config.dingTalk}/resources/cpu`,
+    },
+  }
+  await awaitRequest({
+    url: 'https://oapi.dingtalk.com/robot/send?access_token=08fc72710502b4fec2aeab456ae1ff87e1d75d07f111b067bd1fa85bbedd5473',
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json;charset=utf-8',
+    },
+    body: JSON.stringify(dingdingMessage),
+  })
+}
+module.exports = schedule.scheduleJob('0 0 22 * * ?', async () => {
+  try {
+    job()
+  } catch (e) {
+    logger.info('calculation cpu', e)
+  }
+})

app/jobs/memoryRate.js

+const schedule = require('node-schedule')
+const logger = require('koa-log4').getLogger('jobs:calculation memory')
+const moment = require('moment')
+const awaitRequest = require('../utils/awaitRequest')
+const { MemoryResource } = require('../service/mongoService')
+
+const config = require(global.configPath)
+
+async function job() {
+  const memoryLimitRes = await awaitRequest({
+    url: 'http://172.17.5.17:30002/api/v1/query?query=avg (kube_pod_container_resource_limits_memory_bytes{pod=~"^()()'
+      + '().*$",namespace!~"kube-system|monitor"} / (1024 * 1024 )) by (container)',
+    method: 'GET',
+  })
+  const finalMemoryLimit = {}
+  memoryLimitRes.data.result.forEach((item) => {
+    finalMemoryLimit[item.metric.container] = item.value[1]
+  })
+  const memoryUsageRes = await awaitRequest({
+    url: 'http://172.17.5.17:30002/api/v1/query?query=sum (container_memory_max_usage_bytes'
+      + '{id!="/",pod_name=~"^()()().*$"}/(1024*1024)) by (pod_name)',
+    method: 'GET',
+  })
+  const finalMemoryUsage = []
+  memoryUsageRes.data.result.forEach((item) => {
+    const k = item.metric.pod_name || '不存在的值'
+    const v = item.value[1]
+    finalMemoryUsage.push({ k, v })
+  })
+  const finalRealData = {}
+  Object.keys(finalMemoryLimit).forEach((item) => {
+    let num = 0
+    finalMemoryUsage.forEach((value) => {
+      try {
+        if (value.k.indexOf(item) > -1) {
+          if (value.v > num) {
+            num = value.v
+          }
+        }
+      } catch (e) {
+        console.log(value)
+      }
+    })
+    finalRealData[item] = num
+  })
+  const finalSendData = []
+  Object.keys(finalMemoryLimit).forEach((item) => {
+    finalSendData.push({
+      name: item,
+      memoryLimit: `${Math.round(finalMemoryLimit[item])}Mib`,
+      memoryMaxUsage: `${Math.round(finalRealData[item])}Mib`,
+      Percentage: `${(finalRealData[item] / finalMemoryLimit[item]).toFixed(3)}`,
+    })
+  })
+  function quick(arr) {
+    if (arr.length <= 1) {
+      return arr
+    }
+    const middleIndex = Math.floor(arr.length / 2);
+    const middleValue = arr.splice(middleIndex, 1);
+    const arrLeft = [];
+    const arrRight = [];
+    for (let i = 0; i < arr.length; i += 1) {
+      arr[i].Percentage < middleValue[0].Percentage ? arrLeft.push(arr[i]) : arrRight.push(arr[i]);
+    }
+    return quick(arrLeft).concat(middleValue, quick(arrRight))
+  }
+  const saveData = quick(finalSendData)
+  const res = await MemoryResource.findOne({ create_time: moment().format('YYYY-MM-DD') })
+  if (!res) {
+    await MemoryResource.saveMemory({ create_time: moment().format('YYYY-MM-DD'), memory: saveData })
+  }
+
+  const dingdingMessage = {
+    msgtype: 'link',
+    link: {
+      text: `内存统计告知:${moment().format('YYYY-MM-DD HH-mm-ss')}`,
+      title: '内存统计已同步更新到qa-home下的resource目录，点击可查看详情',
+      messageUrl: `${config.dingTalk}/resources/memory`,
+    },
+  }
+  await awaitRequest({
+    url: 'https://oapi.dingtalk.com/robot/send?access_token=7cab1f948d67c3565a30323c58773c0ba77c7bd1a3eb1c11f8b2d1b6b4a395df',
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json;charset=utf-8',
+    },
+    body: JSON.stringify(dingdingMessage),
+  })
+}
+module.exports = schedule.scheduleJob('0 0 22 * * ?', async () => {
+  try {
+    job()
+  } catch (e) {
+    logger.info('calculation memory', e)
+  }
+})

app/jobs/recovery.js

+/**
+ * 根据不同维度来判断环境是否需要回收
+ *
+ * * */
+const schedule = require('node-schedule')
+const logger = require('koa-log4').getLogger('jobs:calculation cpu')
+const request = require('request')
+const moment = require('moment');
+const config = require('../../src');
+
+const elasticsearch = function (namespace) {
+  const now = new Date();
+  const aWeekAgo = moment().subtract(7, 'days').format()
+  return {
+    query: {
+      bool: {
+        must: [
+          {
+            range: {
+              logtime: {
+                format: 'strict_date_optional_time',
+                gte: aWeekAgo,
+                lte: now,
+              },
+            },
+          },
+        ],
+        filter: [
+          {
+            bool: {
+              filter: [
+                {
+                  bool: {
+                    should: [
+                      {
+                        match: {
+                          type: 'tke-eos',
+                        },
+                      },
+                    ],
+                    minimum_should_match: 1,
+                  },
+                },
+                {
+                  bool: {
+                    should: [
+                      {
+                        match_phrase: {
+                          message: `/service/listEnvVars?namespace=${namespace}`,
+                        },
+                      },
+                    ],
+                    minimum_should_match: 1,
+                  },
+                },
+              ],
+            },
+          },
+        ],
+        should: [],
+        must_not: [],
+      },
+    },
+  }
+}
+
+const awaitRequest = function (options) {
+  return new Promise((resolve, reject) => {
+    request(options, (error, res, body) => {
+      if (error) {
+        reject(error)
+      } else {
+        resolve(JSON.parse(body))
+      }
+    })
+  })
+}
+/**
+ * 发送钉钉通知
+ * @param msg
+ */
+const dingtalk = async function (namespace, msg) {
+  const dingdingMessage = {
+    msgtype: 'markdown',
+    markdown: {
+      title: '服务环境清理通知',
+      text: '> 描述信息 : 以下环境长时间无人使用建议清理\n\n'
+        + `> namespace : ${namespace}\n\n`
+        + `> 建议原因 : ${msg}\n\n`
+        + `> 时间 : ${moment()
+          .format('YYYY-MM-DD HH:mm:ss')}\n\n`,
+    },
+  }
+  await awaitRequest({
+    url: 'https://oapi.dingtalk.com/robot/send?access_token=fda5fe4616b73ab85c087cdea55dec94d4655705ebaf68e0d0ab330f1e1de00b',
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json;charset=utf-8',
+    },
+    body: JSON.stringify(dingdingMessage),
+  })
+}
+/**
+ * 获取所有的服务
+ * 一会去掉monitor
+ * @returns {Promise<any[]>}
+ */
+const allServiceList = async function () {
+  const namespaceList = await awaitRequest({
+    url: `${config.api.tke_api}/namespace`,
+    method: 'GET',
+  })
+  const task = []
+  namespaceList.data.namespaces.forEach((item) => {
+    const serviceList = awaitRequest({
+      url: `${config.api.tke_api}/service?namespace=${item.name}`,
+      method: 'GET',
+    })
+    task.push(serviceList)
+  })
+  const result = await Promise.all(task)
+  return result
+}
+
+/**
+ * 判断某个命名空间下所有的服务是否都创建于7天前
+ * @param servicesList
+ */
+function filterServicesCreateTime(servicesList) {
+  // 所有的服务都是7天前创建的 就返回true
+  let flag = false
+  const now = moment(new Date()).valueOf()
+  for (let i = 0; i < servicesList.length; i += 1) {
+    const startTime = moment(servicesList.startTime).valueOf()
+    const dayNumber = Math.floor((now - startTime) / (1000 * 60 * 60 * 24))
+    if (dayNumber < 7) {
+      flag = false
+      break
+    } else {
+      flag = true
+    }
+  }
+  return flag
+}
+
+const job = async function () {
+  const result = await allServiceList()
+  result.forEach(async (item) => {
+    const namespace = item.data.namespace
+    if (item.data.services.length === 0) {
+      dingtalk(namespace, '此命名空间下没有创建的服务')
+    } else {
+      const all7 = filterServicesCreateTime(item)
+      if (all7) {
+        // 去获取日志 看看日志是否有调用量
+        const res = await awaitRequest({
+          url: 'http://172.21.1.13:9200/logstash-indexer-tke-eos-2019.07/_search',
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json;charset=utf-8',
+          },
+          body: JSON.stringify(elasticsearch(namespace)),
+        })
+        // 如果不存在日志     缺少trarfik 条件
+        if (res.hits.total.value === 0) {
+          dingtalk(namespace, '此环境在一周时间内无服务使用')
+        }
+      }
+    }
+  })
+}
+
+module.exports = schedule.scheduleJob('0 0 10 * * ?', async () => {
+  try {
+    job()
+  } catch (e) {
+    logger.info('calculation cpu', e)
+  }
+})

app/middleware/auth/index.js

@@ -17,6 +17,8 @@ module.exports = function () {
     '/k8s/service/modifyImage',
     '/pipeline/webhooks',
     '/pipeline/dingTalk',
+    '/resource/cpu',
+    '/resource/memory',
   ]
   return async function (ctx, next) {
     // 从header中获取信息

app/model/mongo/qa_api/CpuResource.js

+
+const { Schema } = require('mongoose')
+
+// 表结构
+const schema = new Schema({
+  cpu: { type: Array },
+  create_time: { type: String },
+}, {
+  versionKey: false,
+  collection: 'cpuResource', // 表名
+  timestamps: false,
+})
+schema.statics.saveCpu = function (doc) {
+  return this.create(doc)
+}
+schema.statics.updateCpu = function (doc) {
+  return this.update(doc)
+}
+schema.statics.getCpu = function (data) {
+  return this.find(data)
+}
+module.exports = schema

app/model/mongo/qa_api/MemoryResource.js

+
+const { Schema } = require('mongoose')
+
+
+// 表结构
+const schema = new Schema({
+  memory: { type: Array },
+  create_time: { type: String },
+}, {
+  versionKey: false,
+  collection: 'memoryResource', // 表名
+  timestamps: false,
+})
+schema.statics.saveMemory = function (doc) {
+  return this.create(doc)
+}
+schema.statics.updateMemory = function (doc) {
+  return this.update(doc)
+}
+schema.statics.getMemory = function (data) {
+  return this.find(data)
+}
+module.exports = schema

app/utils/awaitRequest.js

+const request = require('request')
+
+module.exports = function (options) {
+  return new Promise((resolve, reject) => {
+    request(options, (error, res, body) => {
+      if (error) {
+        reject(error)
+      } else {
+        resolve(JSON.parse(body))
+      }
+    })
+  })
+}

app/utils/quickSort.js

+/**
+ * 此数组为 array [[xx,yy],[xx,yy],[xx,yy]] 二重数组，对yy进行比较
+ * @param array
+ */
+module.exports = function quick(arr) {
+  if (arr.length <= 1) {
+    return arr
+  }
+  const middleIndex = Math.floor(arr.length / 2);
+  const middleValue = arr.splice(middleIndex, 1);
+  const arrLeft = [];
+  const arrRight = [];
+  for (let i = 0; i < arr.length; i += 1) {
+    arr[i][1] < middleValue[0][1] ? arrLeft.push(arr[i]) : arrRight.push(arr[i]);
+  }
+  return quick(arrLeft).concat(middleValue, quick(arrRight))
+}

bin/server.js

@@ -12,6 +12,9 @@ const config = require(global.configPath)
 global.config = config
 const redis = require('../app/middleware/redis')
 const auth = require('../app/middleware/auth')
+require('../app/jobs/cpuRate')
+require('../app/jobs/memoryRate')
+require('../app/jobs/recovery')
 
 log4js.configure(config.log4js)
 const logger = log4js.getLogger()
@@ -20,6 +23,7 @@ require('../app/service/mongoService')
 require('../app/service/redisService')
 require('../app/jobs')
 
+
 const logFormat = ':req[x-real-ip] :req[x-forwarded-for] - -'
     + ' ":method :url HTTP/:http-version"'
     + ' :status :content-length ":referrer"'

src/dev.config.js

@@ -77,5 +77,5 @@ module.exports = {
   CiCD: '',
 
   binlogDir: '/Users/zhiyong/IdeaProjects/github/binlog2sql/binlog2sql',
-
+  dingTalk: 'http://localhost:9000',
 }

src/prod.config.js

@@ -74,5 +74,5 @@ module.exports = {
   eos_host: 'eos.quantgroups.com',
 
   binlogDir: '/home/quant_group/binlog2sql/binlog2sql',
-
+  dingTalk: 'http://qa2.liangkebang.com',
 }