Commit 7c73cb7d authored by kewei.jia's avatar kewei.jia

资源限制

parent 0a07ef65
...@@ -14,6 +14,7 @@ module.exports = { ...@@ -14,6 +14,7 @@ module.exports = {
"no-await-in-loop": "off", "no-await-in-loop": "off",
"consistent-return": "off", "consistent-return": "off",
"no-shadow" :"off", "no-shadow" :"off",
"no-unused-expressions":"off",
"no-useless-escape": "off", "no-useless-escape": "off",
"max-len": ["error", {"code": 140}] "max-len": ["error", {"code": 140}]
} }
......
const Router = require('koa-router')
const { CpuResource, MemoryResource } = require('../service/mongoService')
const router = new Router()
module.exports = router
const getCpu = async (ctx) => {
const res = await CpuResource.getCpu(ctx.request.query)
ctx.body = ctx.ok(res)
}
const getMemory = async (ctx) => {
const res = await MemoryResource.getMemory(ctx.request.query)
ctx.body = ctx.ok(res)
}
router
.get('/cpu', getCpu)
.get('/memory', getMemory)
...@@ -26,6 +26,7 @@ const repo = require('./controller/repo') ...@@ -26,6 +26,7 @@ const repo = require('./controller/repo')
const envTemplate = require('./controller/envTemplate') const envTemplate = require('./controller/envTemplate')
const env = require('./controller/env') const env = require('./controller/env')
const pipeline = require('./controller/pipeline') const pipeline = require('./controller/pipeline')
const resource = require('./controller/resource')
// todo: 引入auth中间件 // todo: 引入auth中间件
router router
.use(error()) .use(error())
...@@ -49,6 +50,7 @@ router ...@@ -49,6 +50,7 @@ router
.use('/envTemplate', envTemplate.routes()) .use('/envTemplate', envTemplate.routes())
.use('/env', env.routes()) .use('/env', env.routes())
.use('/pipeline', pipeline.routes()) .use('/pipeline', pipeline.routes())
.use('/resource', resource.routes())
module.exports = router module.exports = router
const schedule = require('node-schedule')
const logger = require('koa-log4').getLogger('jobs:calculation cpu')
const moment = require('moment')
const { CpuResource } = require('../service/mongoService')
const quick = require('../utils/quickSort')
const awaitRequest = require('../utils/awaitRequest')
const config = require(global.configPath)
const endTime = Math.round(moment().valueOf() / 1000)
const startTime = endTime - (60 * 60 * 12)
async function job() {
const cpuLimitRes = await awaitRequest({
url: 'http://172.17.5.17:30002/api/v1/query?query=avg (kube_pod_container_resource_limits_cpu_cores{}) by (container)',
method: 'GET',
})
/**
*将cpulimit数据整合成{k:v,k:v}
*/
const finalCpuLimit = {}
cpuLimitRes.data.result.forEach((item) => {
finalCpuLimit[item.metric.container] = item.value[1]
})
const cpuUsageRes = await awaitRequest({
url: `http://172.17.5.17:30002/api/v1/query_range?query=sum (rate (container_cpu_usage_seconds_total
{image!="",name=~"^k8s_.*",io_kubernetes_container_name!="POD",pod_name=~"^()()().*$"}[1m])) by
(pod_name)&start=${startTime}&end=${endTime}&step=300`,
method: 'GET',
})
/**
* 筛选不重复pod的最大值
* 格式:[{ k: 'account-center-85f4d56d9b-qwr9r',v: '0.002291927622220808' },{}]
*/
const finalCpuUsage = []
cpuUsageRes.data.result.forEach((item) => {
const sortArr = quick(item.values)
const k = item.metric.pod_name
const v = sortArr[sortArr.length - 1][1]
finalCpuUsage.push({ k, v })
})
/**
* 再次筛选数据,去掉重复的pod,并获取最大值
* 格式:{k:v,k:v}
*/
const finalRealData = {}
Object.keys(finalCpuLimit).forEach((item) => {
let num = 0
finalCpuUsage.forEach((value) => {
if (value.k.indexOf(item) > -1) {
if (value.v > num) {
num = value.v
}
}
})
finalRealData[item] = num
})
/**
* 整合成最后数据[]finalSendData
*/
const finalSendData = []
Object.keys(finalCpuLimit).forEach((item) => {
finalSendData.push({
name: item,
cpuLimit: finalCpuLimit[item],
cpuMaxUsage: finalRealData[item],
Percentage: `${(finalRealData[item] / finalCpuLimit[item]).toFixed(3)}`,
})
})
function quickSort(arr) {
if (arr.length <= 1) {
return arr
}
const middleIndex = Math.floor(arr.length / 2);
const middleValue = arr.splice(middleIndex, 1);
const arrLeft = [];
const arrRight = [];
for (let i = 0; i < arr.length; i += 1) {
arr[i].Percentage < middleValue[0].Percentage ? arrLeft.push(arr[i]) : arrRight.push(arr[i]);
}
return quickSort(arrLeft).concat(middleValue, quickSort(arrRight))
}
const saveData = quickSort(finalSendData)
const res = await CpuResource.findOne({ create_time: moment().format('YYYY-MM-DD') })
if (!res) {
await CpuResource.saveCpu({ create_time: moment().format('YYYY-MM-DD'), cpu: saveData })
}
const dingdingMessage = {
msgtype: 'link',
link: {
text: `cpu统计告知:${moment().format('YYYY-MM-DD HH-mm-ss')}`,
title: 'cpu统计已同步更新到qa-home下的resource目录,点击可查看详情',
messageUrl: `${config.dingTalk}/resources/cpu`,
},
}
await awaitRequest({
url: 'https://oapi.dingtalk.com/robot/send?access_token=08fc72710502b4fec2aeab456ae1ff87e1d75d07f111b067bd1fa85bbedd5473',
method: 'POST',
headers: {
'Content-Type': 'application/json;charset=utf-8',
},
body: JSON.stringify(dingdingMessage),
})
}
module.exports = schedule.scheduleJob('0 0 22 * * ?', async () => {
try {
job()
} catch (e) {
logger.info('calculation cpu', e)
}
})
const schedule = require('node-schedule')
const logger = require('koa-log4').getLogger('jobs:calculation memory')
const moment = require('moment')
const awaitRequest = require('../utils/awaitRequest')
const { MemoryResource } = require('../service/mongoService')
const config = require(global.configPath)
async function job() {
const memoryLimitRes = await awaitRequest({
url: 'http://172.17.5.17:30002/api/v1/query?query=avg (kube_pod_container_resource_limits_memory_bytes{pod=~"^()()'
+ '().*$",namespace!~"kube-system|monitor"} / (1024 * 1024 )) by (container)',
method: 'GET',
})
const finalMemoryLimit = {}
memoryLimitRes.data.result.forEach((item) => {
finalMemoryLimit[item.metric.container] = item.value[1]
})
const memoryUsageRes = await awaitRequest({
url: 'http://172.17.5.17:30002/api/v1/query?query=sum (container_memory_max_usage_bytes'
+ '{id!="/",pod_name=~"^()()().*$"}/(1024*1024)) by (pod_name)',
method: 'GET',
})
const finalMemoryUsage = []
memoryUsageRes.data.result.forEach((item) => {
const k = item.metric.pod_name || '不存在的值'
const v = item.value[1]
finalMemoryUsage.push({ k, v })
})
const finalRealData = {}
Object.keys(finalMemoryLimit).forEach((item) => {
let num = 0
finalMemoryUsage.forEach((value) => {
try {
if (value.k.indexOf(item) > -1) {
if (value.v > num) {
num = value.v
}
}
} catch (e) {
console.log(value)
}
})
finalRealData[item] = num
})
const finalSendData = []
Object.keys(finalMemoryLimit).forEach((item) => {
finalSendData.push({
name: item,
memoryLimit: `${Math.round(finalMemoryLimit[item])}Mib`,
memoryMaxUsage: `${Math.round(finalRealData[item])}Mib`,
Percentage: `${(finalRealData[item] / finalMemoryLimit[item]).toFixed(3)}`,
})
})
function quick(arr) {
if (arr.length <= 1) {
return arr
}
const middleIndex = Math.floor(arr.length / 2);
const middleValue = arr.splice(middleIndex, 1);
const arrLeft = [];
const arrRight = [];
for (let i = 0; i < arr.length; i += 1) {
arr[i].Percentage < middleValue[0].Percentage ? arrLeft.push(arr[i]) : arrRight.push(arr[i]);
}
return quick(arrLeft).concat(middleValue, quick(arrRight))
}
const saveData = quick(finalSendData)
const res = await MemoryResource.findOne({ create_time: moment().format('YYYY-MM-DD') })
if (!res) {
await MemoryResource.saveMemory({ create_time: moment().format('YYYY-MM-DD'), memory: saveData })
}
const dingdingMessage = {
msgtype: 'link',
link: {
text: `内存统计告知:${moment().format('YYYY-MM-DD HH-mm-ss')}`,
title: '内存统计已同步更新到qa-home下的resource目录,点击可查看详情',
messageUrl: `${config.dingTalk}/resources/memory`,
},
}
await awaitRequest({
url: 'https://oapi.dingtalk.com/robot/send?access_token=7cab1f948d67c3565a30323c58773c0ba77c7bd1a3eb1c11f8b2d1b6b4a395df',
method: 'POST',
headers: {
'Content-Type': 'application/json;charset=utf-8',
},
body: JSON.stringify(dingdingMessage),
})
}
module.exports = schedule.scheduleJob('0 0 22 * * ?', async () => {
try {
job()
} catch (e) {
logger.info('calculation memory', e)
}
})
/**
* 根据不同维度来判断环境是否需要回收
*
* * */
const schedule = require('node-schedule')
const logger = require('koa-log4').getLogger('jobs:calculation cpu')
const request = require('request')
const moment = require('moment');
const config = require('../../src');
const elasticsearch = function (namespace) {
const now = new Date();
const aWeekAgo = moment().subtract(7, 'days').format()
return {
query: {
bool: {
must: [
{
range: {
logtime: {
format: 'strict_date_optional_time',
gte: aWeekAgo,
lte: now,
},
},
},
],
filter: [
{
bool: {
filter: [
{
bool: {
should: [
{
match: {
type: 'tke-eos',
},
},
],
minimum_should_match: 1,
},
},
{
bool: {
should: [
{
match_phrase: {
message: `/service/listEnvVars?namespace=${namespace}`,
},
},
],
minimum_should_match: 1,
},
},
],
},
},
],
should: [],
must_not: [],
},
},
}
}
const awaitRequest = function (options) {
return new Promise((resolve, reject) => {
request(options, (error, res, body) => {
if (error) {
reject(error)
} else {
resolve(JSON.parse(body))
}
})
})
}
/**
* 发送钉钉通知
* @param msg
*/
const dingtalk = async function (namespace, msg) {
const dingdingMessage = {
msgtype: 'markdown',
markdown: {
title: '服务环境清理通知',
text: '> 描述信息 : 以下环境长时间无人使用建议清理\n\n'
+ `> namespace : ${namespace}\n\n`
+ `> 建议原因 : ${msg}\n\n`
+ `> 时间 : ${moment()
.format('YYYY-MM-DD HH:mm:ss')}\n\n`,
},
}
await awaitRequest({
url: 'https://oapi.dingtalk.com/robot/send?access_token=fda5fe4616b73ab85c087cdea55dec94d4655705ebaf68e0d0ab330f1e1de00b',
method: 'POST',
headers: {
'Content-Type': 'application/json;charset=utf-8',
},
body: JSON.stringify(dingdingMessage),
})
}
/**
* 获取所有的服务
* 一会去掉monitor
* @returns {Promise<any[]>}
*/
const allServiceList = async function () {
const namespaceList = await awaitRequest({
url: `${config.api.tke_api}/namespace`,
method: 'GET',
})
const task = []
namespaceList.data.namespaces.forEach((item) => {
const serviceList = awaitRequest({
url: `${config.api.tke_api}/service?namespace=${item.name}`,
method: 'GET',
})
task.push(serviceList)
})
const result = await Promise.all(task)
return result
}
/**
* 判断某个命名空间下所有的服务是否都创建于7天前
* @param servicesList
*/
function filterServicesCreateTime(servicesList) {
// 所有的服务都是7天前创建的 就返回true
let flag = false
const now = moment(new Date()).valueOf()
for (let i = 0; i < servicesList.length; i += 1) {
const startTime = moment(servicesList.startTime).valueOf()
const dayNumber = Math.floor((now - startTime) / (1000 * 60 * 60 * 24))
if (dayNumber < 7) {
flag = false
break
} else {
flag = true
}
}
return flag
}
const job = async function () {
const result = await allServiceList()
result.forEach(async (item) => {
const namespace = item.data.namespace
if (item.data.services.length === 0) {
dingtalk(namespace, '此命名空间下没有创建的服务')
} else {
const all7 = filterServicesCreateTime(item)
if (all7) {
// 去获取日志 看看日志是否有调用量
const res = await awaitRequest({
url: 'http://172.21.1.13:9200/logstash-indexer-tke-eos-2019.07/_search',
method: 'POST',
headers: {
'Content-Type': 'application/json;charset=utf-8',
},
body: JSON.stringify(elasticsearch(namespace)),
})
// 如果不存在日志 缺少trarfik 条件
if (res.hits.total.value === 0) {
dingtalk(namespace, '此环境在一周时间内无服务使用')
}
}
}
})
}
module.exports = schedule.scheduleJob('0 0 10 * * ?', async () => {
try {
job()
} catch (e) {
logger.info('calculation cpu', e)
}
})
...@@ -17,6 +17,8 @@ module.exports = function () { ...@@ -17,6 +17,8 @@ module.exports = function () {
'/k8s/service/modifyImage', '/k8s/service/modifyImage',
'/pipeline/webhooks', '/pipeline/webhooks',
'/pipeline/dingTalk', '/pipeline/dingTalk',
'/resource/cpu',
'/resource/memory',
] ]
return async function (ctx, next) { return async function (ctx, next) {
// 从header中获取信息 // 从header中获取信息
......
const { Schema } = require('mongoose')
// 表结构
const schema = new Schema({
cpu: { type: Array },
create_time: { type: String },
}, {
versionKey: false,
collection: 'cpuResource', // 表名
timestamps: false,
})
schema.statics.saveCpu = function (doc) {
return this.create(doc)
}
schema.statics.updateCpu = function (doc) {
return this.update(doc)
}
schema.statics.getCpu = function (data) {
return this.find(data)
}
module.exports = schema
const { Schema } = require('mongoose')
// 表结构
const schema = new Schema({
memory: { type: Array },
create_time: { type: String },
}, {
versionKey: false,
collection: 'memoryResource', // 表名
timestamps: false,
})
schema.statics.saveMemory = function (doc) {
return this.create(doc)
}
schema.statics.updateMemory = function (doc) {
return this.update(doc)
}
schema.statics.getMemory = function (data) {
return this.find(data)
}
module.exports = schema
const request = require('request')
module.exports = function (options) {
return new Promise((resolve, reject) => {
request(options, (error, res, body) => {
if (error) {
reject(error)
} else {
resolve(JSON.parse(body))
}
})
})
}
/**
* 此数组为 array [[xx,yy],[xx,yy],[xx,yy]] 二重数组,对yy进行比较
* @param array
*/
module.exports = function quick(arr) {
if (arr.length <= 1) {
return arr
}
const middleIndex = Math.floor(arr.length / 2);
const middleValue = arr.splice(middleIndex, 1);
const arrLeft = [];
const arrRight = [];
for (let i = 0; i < arr.length; i += 1) {
arr[i][1] < middleValue[0][1] ? arrLeft.push(arr[i]) : arrRight.push(arr[i]);
}
return quick(arrLeft).concat(middleValue, quick(arrRight))
}
...@@ -12,6 +12,9 @@ const config = require(global.configPath) ...@@ -12,6 +12,9 @@ const config = require(global.configPath)
global.config = config global.config = config
const redis = require('../app/middleware/redis') const redis = require('../app/middleware/redis')
const auth = require('../app/middleware/auth') const auth = require('../app/middleware/auth')
require('../app/jobs/cpuRate')
require('../app/jobs/memoryRate')
require('../app/jobs/recovery')
log4js.configure(config.log4js) log4js.configure(config.log4js)
const logger = log4js.getLogger() const logger = log4js.getLogger()
...@@ -20,6 +23,7 @@ require('../app/service/mongoService') ...@@ -20,6 +23,7 @@ require('../app/service/mongoService')
require('../app/service/redisService') require('../app/service/redisService')
require('../app/jobs') require('../app/jobs')
const logFormat = ':req[x-real-ip] :req[x-forwarded-for] - -' const logFormat = ':req[x-real-ip] :req[x-forwarded-for] - -'
+ ' ":method :url HTTP/:http-version"' + ' ":method :url HTTP/:http-version"'
+ ' :status :content-length ":referrer"' + ' :status :content-length ":referrer"'
......
...@@ -77,5 +77,5 @@ module.exports = { ...@@ -77,5 +77,5 @@ module.exports = {
CiCD: '', CiCD: '',
binlogDir: '/Users/zhiyong/IdeaProjects/github/binlog2sql/binlog2sql', binlogDir: '/Users/zhiyong/IdeaProjects/github/binlog2sql/binlog2sql',
dingTalk: 'http://localhost:9000',
} }
...@@ -74,5 +74,5 @@ module.exports = { ...@@ -74,5 +74,5 @@ module.exports = {
eos_host: 'eos.quantgroups.com', eos_host: 'eos.quantgroups.com',
binlogDir: '/home/quant_group/binlog2sql/binlog2sql', binlogDir: '/home/quant_group/binlog2sql/binlog2sql',
dingTalk: 'http://qa2.liangkebang.com',
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment