Skip to content

E
enoch
Commit 2d93ae6e authored by jingbo.wang's avatar jingbo.wang
Browse files
Options

service-path ok

parent 99f2d204
Hide whitespace changes
Inline Side-by-side
Showing with 125 additions and 655 deletions
pkg/report-form/node.go 0 → 100644
View file @ 2d93ae6e
package report_form
pkg/report-form/path.go
View file @ 2d93ae6e
......@@ -51,7 +51,7 @@ func (p *Path) initCount() {
const sqlGetCount = `SELECT count("traceId") AS count FROM trace_info ` +
`WHERE sys_name = '%s' AND path = '%s' AND time >= %d AND time < %d;`
sql := fmt.Sprintf(sqlGetCount, p.serviceName, p.path, p.startTime.UnixNano(), p.endTime.UnixNano())
glog.Debug(sql)
// glog.Debug(sql)
count, ok := queryOneAndToInt(sql)
if !ok {
glog.Error("init count query one")
......@@ -65,7 +65,7 @@ func (p *Path) initSumDuration() {
const sqlGetAverageDuration = `SELECT sum("duration") AS average FROM trace_info ` +
`WHERE sys_name = '%s' AND path = '%s' AND time >= %d AND time < %d;`
sql := fmt.Sprintf(sqlGetAverageDuration, p.serviceName, p.path, p.startTime.UnixNano(), p.endTime.UnixNano())
glog.Debug(sql)
// glog.Debug(sql)
sum, ok := queryOneAndToInt(sql)
if !ok {
glog.Error("init average query one")
......@@ -90,7 +90,7 @@ func (p *Path) initMedianDuration() {
const sqlGetMedianDuration = `SELECT median("duration") AS media FROM trace_info ` +
`WHERE sys_name = '%s' AND path = '%s' AND time >= %d AND time < %d;`
sql := fmt.Sprintf(sqlGetMedianDuration, p.serviceName, p.path, p.startTime.UnixNano(), p.endTime.UnixNano())
glog.Debug(sql)
// glog.Debug(sql)
median, ok := queryOneAndToInt(sql)
if !ok {
glog.Error("init median query one")
......@@ -104,7 +104,7 @@ func (p *Path) initMaxDurationTracePoint() {
const sqlGetMaxDurationTracePoint = `SELECT top("duration", "traceId", 1) AS max FROM trace_info ` +
`WHERE sys_name = '%s' AND path = '%s' AND time >= %d AND time < %d;`
sql := fmt.Sprintf(sqlGetMaxDurationTracePoint, p.serviceName, p.path, p.startTime.UnixNano(), p.endTime.UnixNano())
glog.Debug(sql)
// glog.Debug(sql)
req, err := query(sql)
if err != nil {
glog.Error("init max duration trace point:", err)
......
pkg/report-form/report_form.go deleted 100644 → 0
View file @ 99f2d204
package report_form
/*
import (
"encoding/json"
"fmt"
"git.quantgroup.cn/DevOps/enoch/pkg/global"
"git.quantgroup.cn/DevOps/enoch/pkg/glog"
"github.com/influxdata/influxdb/client/v2"
"github.com/influxdata/influxdb/models"
"strings"
"time"
)
const (
getServiceInfoSql = `SELECT * FROM %s WHERE sys_name = '%s' AND time >= %d AND time < %d;`
tableTraceInfo = "trace_info"
minTrace = 50 //小于50的调用不计入统计
minDuration = time.Millisecond * 10
getServiceListSql = `show tag values from trace_info with key = sys_name`
healthCheck = `/tech/health/check`
)
//获取sql
func getSql(sysName string, table string, start time.Time, end time.Time) string {
return fmt.Sprintf(getServiceInfoSql, table, sysName, start.UnixNano(), end.UnixNano())
}
//获取row的index
func getModRowKeyIdx(key string, row models.Row) (int, bool) {
for i, v := range row.Columns {
if v == key {
return i, true
}
}
return -1, false
}
//查询
func Query(sql string) (*client.Response, error) {
config := client.HTTPConfig{
Addr: global.InfluxDbAddress,
}
connect, err := client.NewHTTPClient(config)
defer func() { _ = connect.Close() }()
if err != nil {
return nil, err
}
rtn, err := connect.Query(client.NewQuery(sql, global.InfluxDbName, ""))
if err != nil {
return nil, err
}
return rtn, nil
}
//从db获取服务列表
func GetServiceNameListFromDb() []string {
resp, err := Query(getServiceListSql)
if err != nil {
return []string{}
}
rtn := make([]string, 0)
for _, v := range resp.Results {
for _, row := range v.Series {
index, ok := getModRowKeyIdx("value", row)
if !ok {
continue
}
for _, value := range row.Values {
serviceName, ok := value[index].(string)
if !ok {
continue
}
rtn = append(rtn, serviceName)
}
}
}
return rtn
}
//从db拉取数据
func GetDataFromDb(startTime time.Time, endTime time.Time) ServiceMap {
serviceNameList := GetServiceNameListFromDb()
serviceMap := NewServiceMap()
for _, serviceName := range serviceNameList {
sql := getSql(serviceName, tableTraceInfo, startTime, endTime)
resp, err := Query(sql)
if err != nil {
glog.Error("get service trace errror:", err)
continue
}
for _, v := range resp.Results {
for _, row := range v.Series {
traceList := rowToTraceList(serviceName, row)
for _, trace := range traceList {
serviceMap.AddTrace(serviceName, trace.Path, trace)
}
}
}
}
return serviceMap
}
func rowToTraceList(serviceName string, row models.Row) []*Trace {
//[time duration host msg path sys_name traceId]
rtn := make([]*Trace, 0)
durationIndex, ok := getModRowKeyIdx("duration", row)
if !ok {
return rtn
}
pathIndex, ok := getModRowKeyIdx("path", row)
if !ok {
return rtn
}
timeIndex, ok := getModRowKeyIdx("time", row)
if !ok {
return rtn
}
idIndex, ok := getModRowKeyIdx("traceId", row)
if !ok {
return rtn
}
hostIndex, ok := getModRowKeyIdx("host", row)
if !ok {
return rtn
}
for _, value := range row.Values {
durationRaw, ok := value[durationIndex].(json.Number)
if !ok {
continue
}
durationInt64, err := durationRaw.Int64()
if err != nil {
continue
}
duration := time.Duration(durationInt64) * 1000000 //精度为毫秒，*1000后为纳秒
path, ok := value[pathIndex].(string)
if !ok {
continue
}
pathSplit := strings.Split(path, " ")
if len(pathSplit) != 2 || pathSplit[1] == healthCheck {
continue
}
timeRaw, ok := value[timeIndex].(string)
if !ok {
continue
}
timePoint, err := time.Parse(time.RFC3339, timeRaw)
if err != nil {
continue
}
id, ok := value[idIndex].(string)
if !ok {
continue
}
host := value[hostIndex].(string)
if !ok {
continue
}
rtn = append(rtn, NewTrace(id, serviceName, host, path, duration, timePoint))
}
return rtn
}
func GetGeneralTable(startTime time.Time, endTime time.Time) {
rtn := ""
sm := GetDataFromDb(startTime, endTime)
rtn += fmt.Sprintf("API访问平均时间：%v, API访问量:%d\n", sm.GetAverageTime(), sm.GetCount())
//删除掉小于50访问量的节点
for _, service := range sm {
for path, traceList := range service.GetPathMap() {
if traceList.GetCount() < minTrace {
service.DelPath(path)
}
}
}
//生成
traceList := NewTraceList("")
for _, v := range sm {
if trace, ok := v.GetMaxMediaTrace(); ok && trace.Duration > minDuration {
traceList.AddTrace(trace)
}
}
for _, v := range traceList.GetTraceList() {
rtn += fmt.Sprintln(v.Duration, v.ServiceName, v.TimePoint, v.Id, v.Path)
}
fmt.Print(rtn)
}
*/
pkg/report-form/report_form_test.go deleted 100644 → 0
View file @ 99f2d204
package report_form
/*
import (
"fmt"
"testing"
"time"
)
func TestGetServiceListFromDb(t *testing.T) {
t.Log(GetServiceNameListFromDb())
}
func TestGetDataFromDb(t *testing.T) {
start := time.Now().Add(time.Second*-86400 - 300)
end := time.Now().Add(time.Second * -300)
GetGeneralTable(start, end)
}
//获取path列表
func TestPath(t *testing.T) {
fmt.Println("111")
resp, err := Query(`show series`)
if err != nil {
t.Error(err)
}
fmt.Println("222")
//f, err := os.Create("series_list")
if err != nil {
t.Error(err)
}
sum := 0
sqlList := make([]string, 0, 5120)
for _, v := range resp.Results {
for _, row := range v.Series {
for _, series := range row.Values {
msg := series[0].(string)
path := getPath(msg)
sum++
if isDel(path) {
fmt.Println(path)
sql := fmt.Sprintf("DELETE FROM trace_info WHERE path = '%s';", path)
sqlList = append(sqlList, sql)
}
}
}
}
fmt.Println(sum)
}
*/
/*
resp, err = Query(strings.Join(sqlList, " "))
if err != nil {
t.Error(err)
}
if resp.Err != "" {
t.Error(resp.Err)
}
*/
/*
sqlList := make([]string, 0, 5120)
for _, v := range resp.Results {
for _, row := range v.Series {
for _, series := range row.Values {
msg := series[0].(string)
seriesSplit := strings.Split(series[0].(string), ",")
if len(seriesSplit) > 3 {
path := getPath(msg)
if isDel(path) {
//if len(path) > 7 && strings.ToLower(path[0:7]) == "options" {
path = strings.ReplaceAll(path, "\\", "")
sql := fmt.Sprintf("DELETE FROM trace_info WHERE path = '%s';", path)
sqlList = append(sqlList, sql)
if len(sqlList) == 5120 {
fmt.Println("query start")
resp, err = Query(strings.Join(sqlList, " "))
if err != nil {
t.Error(err)
}
if resp.Err != "" {
t.Error(resp.Err)
}
sqlList = make([]string, 0, 5120)
}
}
}
}
}
}
resp, err = Query(strings.Join(sqlList, " "))
if err != nil {
t.Error(err)
}
if resp.Err != "" {
t.Error(resp.Err)
}
*/
/*
func getPath(s string) string {
start := strings.Index(s, "path=")
ss := s[start+len("path="):]
end := strings.Index(ss, "sys_name=")
rtn := ss
if end != -1 {
rtn = ss[0 : end-1]
}
rtn = strings.ReplaceAll(rtn, "\\", "")
// rtn = strings.ToLower(rtn)
return rtn
}
var delList = []string{
`get /contractno=`,
`options /variable/`,
`POST /ex/api/qiancheng/`,
`OPTIONS /bill/0/`,
`OPTIONS /installment/`,
`OPTIONS /user/`,
`options /bill/`,
`options /debt/`,
`options /installment/`,
`options /maxreliefamount/`,
`options /user/`,
`options /ex/report/list/`,
`GET /ex/train/order_detail/`,
`get /static/js/`,
`get /clear-proof/condition/`,
}
func isDel(s string) bool {
if strings.HasPrefix(s, "options") {
return true
}
if strings.HasPrefix(s, "OPTIONS") {
return true
}
for _, prefix := range delList {
if strings.HasPrefix(s, prefix) {
return true
}
}
return false
}
*/
pkg/report-form/service.go
View file @ 2d93ae6e
package report_form
/*
import "time"
import (
"fmt"
"git.quantgroup.cn/DevOps/enoch/pkg/glog"
"time"
)
type Service struct {
name string
pathMap map[string]*TraceList
name string //服务名称
startTime time.Time //开始时间
endTime time.Time //结束时间
pathMap map[string]*Path //path列表
sumDuration time.Duration //总响应时间
count int //总访问量
averageDuration time.Duration //平均响应时间
maxMedianPath *Path //最大中位响应时间path
maxAveragePath *Path //最大平均响应时间path
}
func NewService(name string) *Service {
return &Service{
name: name,
pathMap: make(map[string]*TraceList),
func NewService(name string, startTime time.Time, endTime time.Time) *Service {
rtn := Service{
name: name,
startTime: startTime,
endTime: endTime,
pathMap: make(map[string]*Path),
}
}
//添加trace
func (s *Service) AddTrace(path string, trace *Trace) {
if _, ok := s.pathMap[path]; !ok {
s.pathMap[path] = NewTraceList(path)
//初始化path
pathList := rtn.getPathList()
for _, path := range pathList {
pathObj, ok := NewPath(rtn.name, path, rtn.startTime, rtn.endTime)
if !ok {
continue
}
rtn.pathMap[path] = pathObj
}
s.pathMap[path].AddTrace(trace)
}
//初始化访问时间相关的参数
rtn.initDuration()
func (s *Service) GetPathMap() map[string]*TraceList {
return s.pathMap
return &rtn
}
//删除path
func (s *Service) DelPath(path string) {
delete(s.pathMap, path)
}
func (s *Service) getPathList() []string {
const sqlGetPathList = `SHOW TAG VALUES FROM trace_info WITH key = "path" WHERE sys_name = '%s';`
sql := fmt.Sprintf(sqlGetPathList, s.name)
resp, err := query(sql)
if err != nil {
glog.Error("query sql:", err)
return []string{}
}
if len(resp.Results) != 1 {
return []string{}
}
res := resp.Results[0]
if len(res.Series) != 1 {
return []string{}
}
row := res.Series[0]
idx, ok := getModRowKeyIdx("value", row)
if !ok {
return []string{}
}
//获取最大中位值的trace
func (s *Service) GetMaxMediaTrace() (*Trace, bool) {
tl := NewTraceList("")
for _, path := range s.pathMap {
if trace, ok := path.GetMediaTrace(); ok {
tl.AddTrace(trace)
rtn := make([]string, 0)
for _, v := range row.Values {
pathInterface := v[idx]
path, ok := pathInterface.(string)
if !ok {
continue
}
rtn = append(rtn, path)
}
return tl.GetMediaTrace()
return rtn
}
//求服务的平均响应时间
func (s *Service) GetAverageTime() time.Duration {
count := s.GetCount()
if count == 0 {
return 0
//初始化访问时间
func (s *Service) initDuration() {
sum := time.Duration(0)
count := 0
for _, pathObj := range s.pathMap {
sum += pathObj.GetSumDuration()
count += pathObj.GetCount()
}
s.count = count
s.sumDuration = sum
sum := time.Duration(0)
for _, v := range s.pathMap {
for _, vv := range v.traceList {
sum += vv.Duration
//平均
if s.count == 0 {
return
}
s.averageDuration = s.sumDuration / time.Duration(s.count)
if len(s.pathMap) != 0 {
medianMax := time.Duration(-99999)
averageMax := time.Duration(-99999)
for _, pathObj := range s.pathMap {
if pathObj.GetMedianDuration() > medianMax {
s.maxMedianPath = pathObj
}
if pathObj.GetAverageDuration() > averageMax {
s.maxAveragePath = pathObj
}
}
}
}
return sum / time.Duration(count)
//获取总访问时间
func (s *Service) GetSumDuration() time.Duration {
return s.sumDuration
}
//求服务的访问量
//获取总访问量
func (s *Service) GetCount() int {
sum := 0
for _, v := range s.pathMap {
sum += v.GetCount()
}
return sum
return s.count
}
//获取平均访问时间
func (s *Service) GetAverageDuration() time.Duration {
return s.averageDuration
}
//获取中位访问时间最大的path
func (s *Service) GetMaxMedianDurationPath() (*Path, bool) {
return s.maxMedianPath, s.maxMedianPath != nil
}
//获取平均访问时间最大的path
func (s *Service) GetMaxAverageDurationPath() (*Path, bool) {
return s.maxAveragePath, s.maxAveragePath != nil
}
*/
pkg/report-form/service_map.go
View file @ 2d93ae6e
package report_form
/*
import "time"
type ServiceMap map[string]*Service
func NewServiceMap() ServiceMap {
return make(map[string]*Service)
}
//添加trace
func (sm ServiceMap) AddTrace(service string, path string, trace *Trace) {
if _, ok := sm[service]; !ok {
sm[service] = NewService(service)
}
sm[service].AddTrace(path, trace)
}
//获取service
func (sm ServiceMap) GetService(service string) (*Service, bool) {
s, ok := sm[service]
return s, ok
}
//求服务的平均响应时间
func (sm ServiceMap) GetAverageTime() time.Duration {
count := sm.GetCount()
if count == 0 {
return 0
}
sum := time.Duration(0)
for _, v := range sm {
for _, vv := range v.pathMap {
for _, vvv := range vv.traceList {
sum += vvv.Duration
}
}
}
return sum / time.Duration(count)
}
//求服务的访问量
func (sm ServiceMap) GetCount() int {
sum := 0
for _, v := range sm {
sum += v.GetCount()
}
return sum
}
*/
pkg/report-form/service_map_test.go
View file @ 2d93ae6e
package report_form
/*
import (
"testing"
"time"
)
func TestNewServiceMap(t *testing.T) {
const (
service = "123"
path = "/hello"
host = "1.1.1.1"
)
sm := NewServiceMap()
sm.AddTrace(service, path, NewTrace("1", service, host, path, 1, time.Now()))
s, ok := sm.GetService(service)
if !ok {
t.Error("can not found service:", service)
}
t.Log(s.GetCount())
}
*/
pkg/report-form/service_test.go
View file @ 2d93ae6e
package report_form
/*
import (
"strconv"
"testing"
"time"
)
func TestNewService(t *testing.T) {
const (
path = "/hello"
host = "1.1.1.1"
serviceName = "aaaa"
serviceName = "cuishou"
)
s := NewService(serviceName)
for i := 1; i <= 10; i++ {
s.AddTrace(path, NewTrace(strconv.Itoa(i), serviceName, host, path, time.Duration(i), time.Now().Add(time.Duration(i))))
s := NewService(serviceName, time.Now().Add(time.Hour*-100), time.Now())
t.Log(s.GetCount())
t.Log(s.GetSumDuration())
t.Log(s.GetAverageDuration())
median, ok := s.GetMaxMedianDurationPath()
if ok {
t.Log("median_max_path:", median.GetPath())
}
t.Log(s.GetAverageTime())
t.Log(s.GetCount())
average, ok := s.GetMaxAverageDurationPath()
if ok {
t.Log("average_max_path:", average.GetPath())
}
}
*/
pkg/report-form/trace.go deleted 100644 → 0
View file @ 99f2d204
package report_form
/*
import (
"math"
"sort"
"time"
)
type Trace struct {
Id string
ServiceName string
Host string
Path string
Duration time.Duration
TimePoint time.Time
}
func NewTrace(id string, serviceName string, host string, path string, duration time.Duration, timePoint time.Time) *Trace {
return &Trace{
Id: id,
ServiceName: serviceName,
Host: host,
Path: path,
Duration: duration,
TimePoint: timePoint,
}
}
type TraceList struct {
Path string
traceList []*Trace
isSort bool
}
func NewTraceList(path string) *TraceList {
return &TraceList{
Path: path,
traceList: make([]*Trace, 0),
}
}
func (tl *TraceList) AddTrace(trace *Trace) {
tl.traceList = append(tl.traceList, trace)
tl.isSort = false
}
func (tl *TraceList) GetTraceList() []*Trace {
tl.sortTime()
return tl.traceList
}
func (tl *TraceList) GetPath() string {
return tl.Path
}
//排序
func (tl *TraceList) sortTime() {
if tl.isSort {
return
}
sort.Slice(tl.traceList, func(i, j int) bool {
if tl.traceList[i].Duration < tl.traceList[j].Duration {
return false
}
return true
})
tl.isSort = true
}
//获取平均值
func (tl *TraceList) GetAverageTime() time.Duration {
if len(tl.traceList) == 0 {
return 0
}
sum := time.Duration(0)
for _, v := range tl.traceList {
sum += v.Duration
}
return sum / time.Duration(len(tl.traceList))
}
//获取中位trace
func (tl *TraceList) GetMediaTrace() (*Trace, bool) {
if len(tl.traceList) == 0 {
return nil, false
}
tl.sortTime()
return tl.traceList[len(tl.traceList)/2], true
}
//获取最大trace
func (tl *TraceList) GetMaxTrace() (*Trace, bool) {
if len(tl.traceList) == 0 {
return nil, false
}
tl.sortTime()
return tl.traceList[0], true
}
//获取最小trace
func (tl *TraceList) GetMinTrace() (*Trace, bool) {
if len(tl.traceList) == 0 {
return nil, false
}
tl.sortTime()
return tl.traceList[len(tl.traceList)-1], true
}
//获取请求次数
func (tl *TraceList) GetCount() int {
return len(tl.traceList)
}
//标准差
//注：标准差越大说明分布越不稳定
func (tl *TraceList) GetStandardDeviation() float64 {
if len(tl.traceList) == 0 {
return 0
}
average := tl.GetAverageTime()
sum := time.Duration(0)
for _, v := range tl.traceList {
sum += (average - v.Duration) * (average - v.Duration)
}
return math.Sqrt(float64(sum))
}
*/
pkg/report-form/trace_test.go deleted 100644 → 0
View file @ 99f2d204
package report_form
/*
import (
"strconv"
"testing"
"time"
)
func TestNewTraceList(t *testing.T) {
const (
path = "/hello"
host = "1.1.1.1"
serviceName = "aaaa"
)
tl := NewTraceList(path)
for i := 1; i <= 10; i++ {
tl.AddTrace(NewTrace(strconv.Itoa(i), serviceName, host, path, time.Duration(i), time.Now().Add(time.Duration(i))))
}
t.Log(tl.GetCount())
t.Log(tl.GetAverageTime())
t.Log(tl.GetMaxTrace())
t.Log(tl.GetMinTrace())
t.Log(tl.GetPath())
t.Log(tl.GetStandardDeviation())
t.Log(tl.GetMediaTrace())
t.Log(tl.GetTraceList())
}
*/
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment