优化资源使用率接口

app/controller/resource.js

 const Router = require('koa-router')
-
-
 const { CpuResource, MemoryResource } = require('../service/mongoService')
 
 const router = new Router()
 module.exports = router
+
+const formatData = (list) => {
+  const data = list.reduce((a, b) => {
+    b.name = b.name.split('-').slice(0, -2).join('-')
+    if (!a[b.name]) {
+      a[b.name] = b
+    }
+    if (a[b.name].Percentage < b.Percentage) {
+      a[b.name] = b
+    }
+    return a
+  }, {})
+
+  const res = []
+  // eslint-disable-next-line guard-for-in
+  for (const i in data) {
+    res.push(data[i])
+  }
+
+  res.sort((a, b) => (a.Percentage - b.Percentage))
+  return res
+}
+
+const handleQuery = obj => ({
+  name: obj.name,
+  createdAt: {
+    $gte: new Date(new Date(obj.startTime).toISOString()),
+    $lt: new Date(new Date(obj.endTime).toISOString()),
+  },
+})
+
 const getCpu = async (ctx) => {
-  const res = await CpuResource.getCpu(ctx.request.query.startTime, ctx.request.query.endTime)
+  const query = handleQuery(ctx.request.query)
+  const list = await CpuResource.getCpu2(query, { _id: 0 })
+  const res = formatData(list)
   ctx.body = ctx.ok(res)
 }
 
+const getCpuInfo = async (ctx) => {
+  const query = handleQuery(ctx.request.query)
+  const list = await CpuResource.getCpu2(query, { _id: 0 }, { Percentage: 1 })
+  ctx.body = ctx.ok(list)
+}
+
 const getMemory = async (ctx) => {
-  const res = await MemoryResource.getMemory(ctx.request.query.startTime, ctx.request.query.endTime)
+  const query = handleQuery(ctx.request.query)
+  const list = await MemoryResource.getMemory2(query, { _id: 0 })
+  const res = formatData(list)
   ctx.body = ctx.ok(res)
 }
 
+const getMemoryInfo = async (ctx) => {
+  const query = handleQuery(ctx.request.query)
+  const list = await MemoryResource.getMemory2(query, { _id: 0 }, { Percentage: 1 })
+  ctx.body = ctx.ok(list)
+}
 
 router
   .get('/cpu', getCpu)
+  .get('/cpuInfo', getCpuInfo)
   .get('/memory', getMemory)
+  .get('/memoryInfo', getMemoryInfo)

app/jobs/cpuRate.js

@@ -2,112 +2,77 @@ const schedule = require('node-schedule')
 const logger = require('koa-log4').getLogger('jobs:calculation cpu')
 const moment = require('moment')
 const { CpuResource } = require('../service/mongoService')
-const quick = require('../utils/quickSort')
 const awaitRequest = require('../utils/awaitRequest')
 
 const config = require(global.configPath)
 
-
 const endTime = Math.round(moment().valueOf() / 1000)
 const startTime = endTime - (60 * 60 * 12)
 async function job() {
+  logger.info('cpu rate job start')
+  const finalCpuLimit = {}
   const cpuLimitRes = await awaitRequest({
-    url: `${config.prometheus}/api/v1/query?query=avg (kube_pod_container_resource_limits_cpu_cores{}) by (container)`,
+    url: `${config.prometheus}/api/v1/query?query=sum%20(container_spec_cpu_quota%20
+      {namespace!~%22kube-system|monitor|rc-test%22}/(100*1000))%20by%20(pod_name)`,
     method: 'GET',
   })
-  /**
-   *将cpulimit数据整合成{k:v,k:v}
-   */
-  const finalCpuLimit = {}
+
   cpuLimitRes.data.result.forEach((item) => {
-    finalCpuLimit[item.metric.container] = item.value[1]
+    if (item.metric.pod_name) {
+      finalCpuLimit[item.metric.pod_name] = item.value[1]
+    }
   })
+
+  const finalCpuUsage = {}
   const cpuUsageRes = await awaitRequest({
     url: `${config.prometheus}/api/v1/query_range?query=sum (rate (container_cpu_usage_seconds_total
     {image!="",name=~"^k8s_.*",io_kubernetes_container_name!="POD",pod_name=~"^()()().*$"}[1m])) by
-     (pod_name)&start=${startTime}&end=${endTime}&step=300`,
+     (pod_name)&start=${startTime}&end=${endTime}&step=90`,
     method: 'GET',
   })
-  /**
-   * 筛选不重复pod的最大值
-   *  格式:[{ k: 'account-center-85f4d56d9b-qwr9r',v: '0.002291927622220808' },{}]
-   */
-  const finalCpuUsage = []
+
   cpuUsageRes.data.result.forEach((item) => {
-    const sortArr = quick(item.values)
-    const k = item.metric.pod_name
-    const v = sortArr[sortArr.length - 1][1]
-    finalCpuUsage.push({ k, v })
-  })
-  /**
-   * 再次筛选数据,去掉重复的pod，并获取最大值
-   * 格式:{k:v,k:v}
-   */
-  const finalRealData = {}
-  Object.keys(finalCpuLimit).forEach((item) => {
-    let num = 0
-    finalCpuUsage.forEach((value) => {
-      if (value.k.indexOf(item) > -1) {
-        if (value.v > num) {
-          num = value.v
+    if (item.metric.pod_name) {
+      finalCpuUsage[item.metric.pod_name] = parseFloat(item.values[0][1])
+      item.values.forEach((i) => {
+        const cpuUsage = parseFloat(i[1])
+        if (cpuUsage > finalCpuUsage[item.metric.pod_name]) {
+          finalCpuUsage[item.metric.pod_name] = cpuUsage
         }
-      }
-    })
-    finalRealData[item] = num
+      })
+    }
   })
-  /**
-   * 整合成最后数据[]finalSendData
-   */
-  const finalSendData = []
+
   Object.keys(finalCpuLimit).forEach((item) => {
-    finalSendData.push({
+    const data = {
       name: item,
       cpuLimit: finalCpuLimit[item],
-      cpuMaxUsage: finalRealData[item],
-      Percentage: `${(finalRealData[item] / finalCpuLimit[item]).toFixed(3)}`,
-    })
-  })
-  function quickSort(arr) {
-    if (arr.length <= 1) {
-      return arr
+      cpuMaxUsage: finalCpuUsage[item],
+      Percentage: `${(finalCpuUsage[item] / finalCpuLimit[item]).toFixed(3)}`,
     }
-    const middleIndex = Math.floor(arr.length / 2);
-    const middleValue = arr.splice(middleIndex, 1);
-    const arrLeft = [];
-    const arrRight = [];
-    for (let i = 0; i < arr.length; i += 1) {
-      arr[i].Percentage < middleValue[0].Percentage ? arrLeft.push(arr[i]) : arrRight.push(arr[i]);
-    }
-    return quickSort(arrLeft).concat(middleValue, quickSort(arrRight))
-  }
-
-  const saveData = quickSort(finalSendData)
-  saveData.forEach((item) => {
-    CpuResource.saveCpu({
-      name: item.name,
-      cpuLimit: item.cpuLimit,
-      cpuMaxUsage: item.cpuMaxUsage,
-      Percentage: item.Percentage,
-    })
+    CpuResource.saveCpu(data)
   })
+
   const dingdingMessage = {
     msgtype: 'link',
     link: {
-      text: `cpu统计告知:${moment().format('YYYY-MM-DD HH-mm-ss')}`,
       title: 'cpu统计已同步更新到qa-home下的resource目录，点击可查看详情',
-      messageUrl: `${config.dingTalk}/resources/cpu`,
+      text: `cpu统计告知:${moment().format('YYYY-MM-DD HH:mm:ss')}`,
+      messageUrl: `${config.qahomeHost}/resources/cpu`,
     },
   }
   await awaitRequest({
-    url: config.dingdingCpu,
+    url: config.dingdingResouce,
     method: 'POST',
     headers: {
       'Content-Type': 'application/json;charset=utf-8',
     },
     body: JSON.stringify(dingdingMessage),
   })
+  logger.info('cpu rate job end')
 }
-module.exports = schedule.scheduleJob('0 0 22 * * ?', async () => {
+// module.exports = schedule.scheduleJob('*/2 * * * ?', async () => {
+module.exports = schedule.scheduleJob('30 0 20 * * ?', async () => {
   try {
     job()
   } catch (e) {

app/jobs/memoryRate.js

@@ -7,93 +7,61 @@ const { MemoryResource } = require('../service/mongoService')
 const config = require(global.configPath)
 
 async function job() {
+  logger.info('memory rate job start')
+  const finalMemoryLimit = {}
   const memoryLimitRes = await awaitRequest({
-    url: `${config.prometheus}/api/v1/query?query=avg (kube_pod_container_resource_limits_memory_bytes
-    {pod=~"^()()().*$",namespace!~"kube-system|monitor"} / (1024 * 1024 )) by (container)`,
+    url: `${config.prometheus}/api/v1/query?query=sum%20(container_spec_memory_limit_bytes%20{pod=~%22^()()().*$%22,
+      namespace!~%22kube-system|monitor|rc-test%22}/(1024*1024))%20by%20(pod_name)`,
     method: 'GET',
   })
-  const finalMemoryLimit = {}
   memoryLimitRes.data.result.forEach((item) => {
-    finalMemoryLimit[item.metric.container] = item.value[1]
+    if (item.metric.pod_name) {
+      finalMemoryLimit[item.metric.pod_name] = item.value[1]
+    }
   })
+
+  const finalMemoryUsage = {}
   const memoryUsageRes = await awaitRequest({
     url: `${config.prometheus}/api/v1/query?query=sum (container_memory_max_usage_bytes
       {id!="/",pod_name=~"^()()().*$"}/(1024*1024)) by (pod_name)`,
     method: 'GET',
   })
-  const finalMemoryUsage = []
   memoryUsageRes.data.result.forEach((item) => {
-    const k = item.metric.pod_name || '不存在的值'
-    const v = item.value[1]
-    finalMemoryUsage.push({ k, v })
-  })
-  const finalRealData = {}
-  Object.keys(finalMemoryLimit).forEach((item) => {
-    let num = 0
-    finalMemoryUsage.forEach((value) => {
-      try {
-        if (value.k.indexOf(item) > -1) {
-          if (value.v > num) {
-            num = value.v
-          }
-        }
-      } catch (e) {
-        logger.info(e)
-      }
-    })
-    finalRealData[item] = num
+    if (item.metric.pod_name) {
+      finalMemoryUsage[item.metric.pod_name] = item.value[1]
+    }
   })
-  const finalSendData = []
+
   Object.keys(finalMemoryLimit).forEach((item) => {
-    finalSendData.push({
+    const data = {
       name: item,
       memoryLimit: Math.round(finalMemoryLimit[item]),
-      memoryMaxUsage: Math.round(finalRealData[item]),
-      Percentage: `${(finalRealData[item] / finalMemoryLimit[item]).toFixed(3)}`,
-    })
-  })
-  function quick(arr) {
-    if (arr.length <= 1) {
-      return arr
+      memoryMaxUsage: Math.round(finalMemoryUsage[item]),
+      Percentage: `${(finalMemoryUsage[item] / finalMemoryLimit[item]).toFixed(3)}`,
     }
-    const middleIndex = Math.floor(arr.length / 2);
-    const middleValue = arr.splice(middleIndex, 1);
-    const arrLeft = [];
-    const arrRight = [];
-    for (let i = 0; i < arr.length; i += 1) {
-      arr[i].Percentage < middleValue[0].Percentage ? arrLeft.push(arr[i]) : arrRight.push(arr[i]);
-    }
-    return quick(arrLeft).concat(middleValue, quick(arrRight))
-  }
-  const saveData = quick(finalSendData)
-  saveData.forEach((item) => {
-    MemoryResource.saveMemory({
-      name: item.name,
-      memoryLimit: item.memoryLimit,
-      memoryMaxUsage: item.memoryMaxUsage,
-      Percentage: item.Percentage,
-    })
+    MemoryResource.saveMemory(data)
   })
 
-
   const dingdingMessage = {
     msgtype: 'link',
     link: {
-      text: `内存统计告知:${moment().format('YYYY-MM-DD HH-mm-ss')}`,
       title: '内存统计已同步更新到qa-home下的resource目录，点击可查看详情',
-      messageUrl: `${config.dingTalk}/resources/memory`,
+      text: `内存统计告知:${moment().format('YYYY-MM-DD HH:mm:ss')}`,
+      messageUrl: `${config.qahomeHost}/resources/memory`,
     },
   }
   await awaitRequest({
-    url: config.dingdingMemory,
+    url: config.dingdingResouce,
     method: 'POST',
     headers: {
       'Content-Type': 'application/json;charset=utf-8',
     },
     body: JSON.stringify(dingdingMessage),
   })
+  logger.info('memory rate job end')
 }
-module.exports = schedule.scheduleJob('0 0 22 * * ?', async () => {
+// module.exports = schedule.scheduleJob('*/2 * * * ?', async () => {
+module.exports = schedule.scheduleJob('0 0 20 * * ?', async () => {
   try {
     job()
   } catch (e) {

app/middleware/auth/index.js

@@ -17,6 +17,8 @@ module.exports = function () {
     '/pipeline/dingTalk',
     '/resource/cpu',
     '/resource/memory',
+    '/cluster/list',
+    '/proconfig/get_project_for_jenkins',
   ]
   return async function (ctx, next) {
     // 从header中获取信息

app/model/mongo/qa_api/CpuResource.js

@@ -38,4 +38,14 @@ schema.statics.getCpu = function (start, end) {
       },
     }])
 }
+
+schema.statics.getCpu2 = function (query, column, sort) {
+  if (query.name) {
+    query.name = new RegExp(query.name)
+  } else {
+    delete query.name
+  }
+  return this.find(query, column).sort(sort).lean()
+}
+
 module.exports = schema

app/model/mongo/qa_api/MemoryResource.js

@@ -39,10 +39,14 @@ schema.statics.getMemory = function (start, end) {
       },
     }])
 }
-// schema.statics.getType = function () {
-//   return this.aggregate([
-//     { $match: {} },
-//     { $group: { _id: '$type', total: { $sum: 1 } } },
-//   ]).sort({ total: -1 })
-// }
+
+schema.statics.getMemory2 = function (query, column, sort) {
+  if (query.name) {
+    query.name = new RegExp(query.name)
+  } else {
+    delete query.name
+  }
+  return this.find(query, column).sort(sort).lean()
+}
+
 module.exports = schema

src/app.config.js

@@ -52,10 +52,8 @@ module.exports = {
     pass: 'Quantgroup123',
   },
   prometheus: 'http://172.17.5.17:30002',
-  dingdingCpu: 'https://oapi.dingtalk.com/robot/send?access_token'
-    + '=08fc72710502b4fec2aeab456ae1ff87e1d75d07f111b067bd1fa85bbedd5473',
-  dingdingMemory: 'https://oapi.dingtalk.com/robot/send?access_token'
-    + '=7cab1f948d67c3565a30323c58773c0ba77c7bd1a3eb1c11f8b2d1b6b4a395df',
+  dingdingResouce: 'https://oapi.dingtalk.com/robot/send?access_token'
+    + '=fae6b46742f9f261997a7f4f07bb6ba21f274dc3e602065fff5c4f76cf1dbf53',
   sonarHost: 'http://sonar.quantgroups.com',
   qahomeHost: 'http://qa2.liangkebang.com',
   snorDingRobotAddr: 'https://oapi.dingtalk.com/robot/send?access_token=0da7a1066b42bb23f079dacec3b7975e8cc12cee7efc5316a9b5b1900779a407',