merge restructure

.gitignore

@@ -2,4 +2,5 @@
 *.log
 enoch
 go.sum
-test.go
+test.go
\ No newline at end of file
+cache*
\ No newline at end of file

go.mod

@@ -4,15 +4,17 @@ go 1.12
 
 require (
 	github.com/Shopify/sarama v1.23.1
-	github.com/bsm/sarama-cluster v2.1.15+incompatible
-	github.com/gomodule/redigo v2.0.0+incompatible
-	github.com/hashicorp/consul/api v1.2.0
-	github.com/influxdata/influxdb v1.7.8
-	github.com/json-iterator/go v1.1.7
-	github.com/onsi/ginkgo v1.10.1 // indirect
-	github.com/onsi/gomega v1.7.0 // indirect
-	github.com/robfig/cron v1.2.0
-	github.com/vrg0/go-common v0.0.0-20190925101101-e6595edace1b
+	github.com/buaazp/fasthttprouter v0.1.1
+	github.com/google/go-cmp v0.3.1 // indirect
+	github.com/gorilla/websocket v1.4.1 // indirect
+	github.com/influxdata/influxdb v1.7.9
+	github.com/mkevac/debugcharts v0.0.0-20180124214838-d3203a8fa926
+	github.com/shirou/gopsutil v2.19.11+incompatible // indirect
+	github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect
+	github.com/valyala/fasthttp v1.6.0
+	github.com/vrg0/go-common v0.0.0-20191213082238-e4e6080702f1
+	go.uber.org/zap v1.13.0
+	golang.org/x/sys v0.0.0-20200219091948-cb0a6d8edb6c // indirect
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
 	gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
 )

main.go

 package main
 
 import (
-	"flag"
-	"git.quantgroup.cn/DevOps/enoch/service"
-	"git.quantgroup.cn/DevOps/enoch/service/conf"
-	"git.quantgroup.cn/DevOps/enoch/service/consumer"
-	"git.quantgroup.cn/DevOps/enoch/service/continuous_queries"
-	"git.quantgroup.cn/DevOps/enoch/service/data"
-	"git.quantgroup.cn/DevOps/enoch/service/file_cache"
-	"git.quantgroup.cn/DevOps/enoch/service/job"
-	"git.quantgroup.cn/DevOps/enoch/service/node-check"
-	"github.com/vrg0/go-common/logger"
+	"fmt"
+	"git.quantgroup.cn/DevOps/enoch/pkg/api-server"
+	"git.quantgroup.cn/DevOps/enoch/pkg/dao"
+	"git.quantgroup.cn/DevOps/enoch/pkg/global"
+	"git.quantgroup.cn/DevOps/enoch/pkg/glog"
+	node_check "git.quantgroup.cn/DevOps/enoch/pkg/node-check"
+	"git.quantgroup.cn/DevOps/enoch/pkg/points"
+	"git.quantgroup.cn/DevOps/enoch/pkg/report-form"
+	"github.com/Shopify/sarama"
+	_ "github.com/mkevac/debugcharts"
+	"github.com/valyala/fasthttp"
 	"net/http"
 	_ "net/http/pprof"
 	"os"
-	"strconv"
+	"os/signal"
+	"strings"
+	"syscall"
+	"time"
 )
 
-var quartz bool
-var denv string
-var didc string
+func handlerKafkaMsg() {
+	fmt.Println(global.InfluxDbAddress)
+	db := dao.New(global.BatchSize, time.Second*60, global.InfluxDbAddress, global.DaoFileCacheDir)
+
+	//处理调用链条信息
+	braveTopic := global.Config.GetOrDefault(global.NamespaceTechSleuth, "tech.brave.kafkaTopic", "")
+
+	braveConsumer, err := global.KafkaRecver.NewConsumer("enoch-group", []string{braveTopic}, func(msg *sarama.ConsumerMessage) {
+		pointList, err := points.TraceBaseInfoToPoint(msg.Value)
+		if err != nil {
+			glog.Errorf("can not make trace point:", err)
+			return
+		}
+		for _, point := range pointList {
+			db.MsgProcess(point)
+		}
+	})
+	if err != nil {
+		glog.Fatal("监听kafka recver失败!")
+	}
 
-func init() {
-	flag.BoolVar(&quartz, "quartz", false, "quartz")
-	flag.StringVar(&denv, "env", "dev", "环境")
-	flag.StringVar(&didc, "idc", "local", "机房")
-	flag.Parse()
-
-	logPath := "./enoch.log"
-	if denv == "pro" {
-		logPath = "/home/quant_group/enoch/enoch.log"
+	//处理基本信息
+	healthTopic := global.Config.GetOrDefault(global.NamespaceTechSleuth, "tech.enoch.kafkaTopic", "")
+	healthConsumer, err := global.KafkaRecver.NewConsumer("quantGroup-enoch-agent", []string{healthTopic}, func(msg *sarama.ConsumerMessage) {
+		pointList, err := points.HostBaseInfoToPoint(msg.Value)
+		if err != nil {
+			glog.Errorf("can not make health point:", err)
+			return
+		}
+		for _, point := range pointList {
+			db.MsgProcess(point)
+		}
+	})
+	if err != nil {
+		glog.Fatal("监听kafka recver失败!")
 	}
 
-	if e := logger.Init(denv == "pro", logPath); e != nil {
-		panic(e)
+	glog.Info(global.AppName + "启动")
+	//平滑退出
+	sigterm := make(chan os.Signal, 1)
+	signal.Notify(sigterm, syscall.SIGINT, syscall.SIGTERM)
+	select {
+	case <-sigterm:
+		if braveConsumer != nil {
+			braveConsumer.Close()
+			glog.Info("braceConsumer平滑退出")
+		}
+		if healthConsumer != nil {
+			healthConsumer.Close()
+			glog.Info("healthConsumer平滑退出")
+		}
+		if db != nil {
+			db.Close()
+			glog.Info("dao平滑退出")
+		}
+		glog.Info(global.AppName + "平滑退出")
 	}
 }
 
-func main() {
-	conf.Load(denv, didc)
-
-	file, err := os.OpenFile("quantgroup.log", os.O_RDWR|os.O_CREATE, 0666)
-	defer func() { _ = file.Close() }()
-	if err != nil {
-		logger.Fatal("create file error", err)
-	}
-	file_cache.Load(conf.GlobalConfig.FileCachePath)
-	file_cache.RegisterJob(consumer.ReSubmit)
-	go file_cache.Delete()
-	port := conf.GlobalConfig.Port
-
-	logger.Info(conf.GlobalConfig.AppName + "项目启动, port:" + port + ",环境:" + conf.GlobalConfig.Env)
-	defer logger.Info("项目结束")
-
-	//初始化redis连接池
-	data.RedisPoolInit()
-	go consumer.AgentClusterConsumer(conf.HealthTopic(), consumer.HealthMessageHandler{})
-	go consumer.AgentClusterConsumer(conf.BraveTopic(), consumer.BraveMessageHandler{})
-	intPort, _ := strconv.Atoi(port)
-
-	if quartz {
-		logger.Info("启动定时任务")
-		job.AutoEmailPerformInfo()
+//TODO 可优化成Raft算法，目前是固定节点的
+func isMaster() bool {
+	//开发环境，但InfluxDb的Ip是生产环境ip的时候，不执行初始化操作
+	if global.IsDev() && strings.Contains(global.InfluxDbAddress, "172.16") {
+		return false
 	}
 
-	//开启服务状态监控，当服务状态异常时，调用web-hook函数
-	if denv == "dev" {		//开发环境
-		go node_check.NodeCheck()
-		job.AutoAlarm()
-	} else if job.CheckIp("172.30.12.22") {  //生产环境，只有172.30.12.22执行
-		go node_check.NodeCheck()
-		job.AutoAlarm()
-		continuous_queries.Load()		//连续查询设置只在生产环境上执行
+	//生产环境，但LocalIp不是"172.30.12.22"，不执行初始化操作
+	if !global.IsDev() && global.LocalIp != "172.30.12.22" {
+		return false
 	}
 
+	return true
+}
+
+func main() {
+	//性能监控
 	go func() {
-		_ = http.ListenAndServe("0.0.0.0:"+strconv.Itoa(intPort+1), nil)
+		defer func() {
+			if err := recover(); err != nil {
+				glog.Error(err)
+			}
+		}()
+		if err := http.ListenAndServe(":9999", nil); err != nil {
+			glog.Error(err)
+		}
 	}()
 
-	http.HandleFunc("/duration", service.DurationInterface)
-	http.HandleFunc("/tech/health/check", func(writer http.ResponseWriter, request *http.Request) {
-		writer.WriteHeader(http.StatusOK)
-	})
-	http.HandleFunc("/counter", service.CounterInterface)
+	//主节点：刷新连续查询、健康状态报表、告警策略
+	if isMaster() {
+		//初始化数据库（创建DB，创建连续查询）
+		dao.DbInit()
+
+		//健康状态报表
+		//每天0点生成报表
+		report_form.RegularReport(global.ReportFormDir)
+		//每周1早10点发送邮件
+		report_form.RegularMail(global.ReportFormDir)
+		//节点健康状态检查
+		node_check.NodeHealthCheckAndNotify()
+		//TODO 告警策略
 
-	err = http.ListenAndServe(":"+port, nil)
-	if err != nil {
-		logger.Fatal("服务启动失败", err)
 	}
 
+	//对外api
+	api_server.ListenAndServe()
+
+	//处理消息（阻塞）
+	handlerKafkaMsg()
 }
 
+func init() {
+	api_server.HandlerFunc(api_server.GET, "/tech/health/check", healthCheck)
+	api_server.HandlerFunc(api_server.HEAD, "/tech/health/check", healthCheck)
+}
 
+func healthCheck(ctx *fasthttp.RequestCtx) {
+	ctx.Response.SetStatusCode(fasthttp.StatusOK)
+	ctx.Response.SetBodyString("ok")
+}

pkg/api-server/api_server.go

+package api_server
+
+import (
+	"fmt"
+	"git.quantgroup.cn/DevOps/enoch/pkg/global"
+	"git.quantgroup.cn/DevOps/enoch/pkg/glog"
+	"github.com/buaazp/fasthttprouter"
+	"github.com/valyala/fasthttp"
+	"github.com/vrg0/go-common/logger"
+	"github.com/vrg0/go-common/util"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+var (
+	router = fasthttprouter.New()
+	once   = new(sync.Once)
+)
+
+const (
+	GET     = "GET"
+	PUT     = "PUT"
+	DELETE  = "DELETE"
+	POST    = "POST"
+	PATCH   = "PATCH"
+	HEAD    = "HEAD"
+	OPTIONS = "OPTIONS"
+)
+
+//跨域
+func wrap(handler fasthttp.RequestHandler) fasthttp.RequestHandler {
+	return func(ctx *fasthttp.RequestCtx) {
+		handler(ctx)
+		ctx.Response.Header.Add("Access-Control-Allow-Origin", "*")
+	}
+}
+
+//注册函数
+//HandlerFunc只能在init()中执行
+func HandlerFunc(method string, path string, handle fasthttp.RequestHandler) {
+	if pc, _, _, ok := runtime.Caller(1); ok && strings.Contains(runtime.FuncForPC(pc).Name(), ".init") {
+		router.Handle(method, path, handle)
+	} else {
+		glog.Error("HandlerFunc must be execute in init()")
+	}
+}
+
+//程序对外API的http-server
+//启动服务
+func ListenAndServe() {
+	once.Do(listenAndServe)
+}
+
+func listenAndServe() {
+	//handlerFunc 发生 panic 后返回的内容
+	router.PanicHandler = func(ctx *fasthttp.RequestCtx, i interface{}) {
+		rtn := "panic: " + ctx.String() + " "
+		switch x := i.(type) {
+		case error:
+			rtn += x.Error()
+		case string:
+			rtn += x
+		default:
+			rtn += "未识别的panic类型"
+		}
+		rtn += "\n"
+		rtn += ctx.Request.String()
+
+		glog.Error(rtn)
+		ctx.Response.SetStatusCode(500)
+		ctx.Response.SetBodyString(rtn)
+	}
+
+	//启动http server
+	go func() {
+		//server panic后自动重启
+		defer func() {
+			if err := recover(); err != nil {
+				errStr := fmt.Sprint("api fasthttp panic:", err)
+				glog.Error(errStr)
+				time.Sleep(time.Second * 1)
+				listenAndServe()
+			}
+		}()
+
+		//创建http-server对象
+		l := global.Logger.GetStandardLogger()
+		l.SetPrefix("_API_SERVER_ ")
+		hook := logger.NewHookWriter(l.Writer())
+		hook.AddHookFunc(func(data []byte) bool {
+			body := util.BytesString(data)
+			if strings.Contains(body, "error") {
+			}
+			return true
+		})
+		l.SetOutput(hook)
+		server := fasthttp.Server{
+			Logger:  l,
+			Handler: wrap(router.Handler),
+		}
+
+		//启动http-server
+		firstRunFlag := true
+		for {
+			if firstRunFlag {
+				firstRunFlag = false
+			} else {
+				time.Sleep(time.Second * 1) //二次重启延时1秒
+			}
+
+			if err := server.ListenAndServe("0.0.0.0:" + strconv.Itoa(global.HttpPort)); err != nil {
+				glog.Error("api fasthttp err", err)
+			}
+		}
+	}()
+	//等待http-server启动成功
+	time.Sleep(time.Millisecond * 100)
+}

pkg/api-server/api_server_test.go

+package api_server
+
+import (
+	"github.com/valyala/fasthttp"
+	"strings"
+	"testing"
+)
+
+func init() {
+	//正确执行的测试场景
+	HandlerFunc(GET, "/hello", func(ctx *fasthttp.RequestCtx) {
+		ctx.Response.SetBodyString("hello!")
+	})
+
+	//panic测试场景
+	HandlerFunc(GET, "/panic", func(ctx *fasthttp.RequestCtx) {
+		panic("panic_test")
+	})
+}
+
+//测试panic回调函数是否好用
+func TestHandlerFuncPanic(t *testing.T) {
+	ListenAndServe()
+
+	if _, body, err := fasthttp.Get(nil, "http://127.0.0.1:5555/panic"); err != nil {
+		t.Error(err)
+	} else if !strings.Contains(string(body), "panic_test") {
+		t.Error("recover panic fatal")
+	}
+}
+
+//测试http-server是否启动
+func TestListenAndServe(t *testing.T) {
+	ListenAndServe()
+
+	if _, _, err := fasthttp.Get(nil, "http://127.0.0.1:5555/hello"); err != nil {
+		t.Error(err)
+	}
+}
+
+//不在init()中执行会报panic
+func TestHandlerFunc(t *testing.T) {
+	defer func() {
+		if err := recover(); err != nil {
+			if errStr, ok := err.(string); !ok || errStr != "HandlerFunc must be execute in init()" {
+				t.Error("HandlerFunc Err")
+			}
+		}
+	}()
+	HandlerFunc(GET, "/test", func(ctx *fasthttp.RequestCtx) {
+		ctx.Response.SetBodyString("test!")
+	})
+}

pkg/dao/cache_point.go

+package dao
+
+import (
+	"github.com/influxdata/influxdb/client/v2"
+	"time"
+)
+
+type CachePoint struct {
+	Name   string                 `json:"name"`
+	Tags   map[string]string      `json:"tags"`
+	Fields map[string]interface{} `json:"fields"`
+	Time   time.Time              `json:"time"`
+}
+
+func NewPoint(point *client.Point) CachePoint {
+	cp := CachePoint{}
+	cp.Name = point.Name()
+	cp.Time = point.Time()
+	cp.Fields, _ = point.Fields()
+	cp.Tags = point.Tags()
+	return cp
+}

pkg/dao/dao.go

+package dao
+
+import (
+	"bytes"
+	"context"
+	"encoding/gob"
+	"git.quantgroup.cn/DevOps/enoch/pkg/global"
+	"git.quantgroup.cn/DevOps/enoch/pkg/glog"
+	"github.com/influxdata/influxdb/client/v2"
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+	"sync"
+	"time"
+)
+
+const (
+	ChannelSize       = 1024
+	filePrefixWriting = "writing"
+	filePrefixCache   = "cache"
+)
+
+type Dao struct {
+	batchSize    int
+	size         int
+	channel      chan *client.Point
+	flashTime    time.Duration
+	dbAddress    string
+	cacheFileDir string
+	isClose      bool //平滑退出标记
+	ctx          context.Context
+	ctxCancel    context.CancelFunc
+	wg           *sync.WaitGroup
+}
+
+func New(batchSize int, flashTime time.Duration, dbAddress string, cacheFileDir string) *Dao {
+	ctx, cancel := context.WithCancel(context.Background())
+	rtn := &Dao{
+		batchSize:    batchSize,
+		size:         0,
+		channel:      make(chan *client.Point, ChannelSize),
+		flashTime:    flashTime,
+		dbAddress:    dbAddress,
+		cacheFileDir: cacheFileDir,
+		isClose:      false,
+		ctx:          ctx,
+		ctxCancel:    cancel,
+		wg:           new(sync.WaitGroup),
+	}
+
+	if stat, err := os.Stat(cacheFileDir); err != nil || !stat.IsDir() {
+		glog.Error("cacheFileDir:", err)
+	}
+
+	//数据入库
+	go rtn.run()
+
+	//清空文件缓存
+	go rtn.flashFileCache()
+
+	return rtn
+}
+
+//平滑退出Dao，会flash缓存
+func (d *Dao) Close() {
+	d.isClose = true
+	d.ctxCancel()
+	d.wg.Wait()
+}
+
+func (d *Dao) flashFileCache() {
+	d.wg.Add(1)
+	timer := time.NewTimer(0)
+	for {
+		select {
+		case <-timer.C:
+			fileList, err := ioutil.ReadDir(d.cacheFileDir)
+			if err != nil || len(fileList) == 0 {
+				continue
+			}
+			for _, file := range fileList {
+				sl := strings.Split(file.Name(), ":")
+				if len(sl) == 0 || sl[0] != filePrefixCache {
+					continue
+				}
+
+				//读取文件
+				filePath := path.Join(d.cacheFileDir, file.Name())
+				data, err := ioutil.ReadFile(filePath)
+				if err != nil {
+					glog.Error("can not read file:", filePath, err)
+					continue
+				}
+
+				cachePointList := make([]CachePoint, 0)
+				buf := bytes.NewBuffer(data)
+				dec := gob.NewDecoder(buf)
+				if err := dec.Decode(&cachePointList); err != nil {
+					glog.Error("can not decode file:", filePath, err)
+					continue
+				}
+
+				pointList := make([]*client.Point, 0)
+				for _, cachePoint := range cachePointList {
+					point, err := client.NewPoint(cachePoint.Name, cachePoint.Tags, cachePoint.Fields, cachePoint.Time)
+					if err != nil {
+						continue
+					}
+					pointList = append(pointList, point)
+				}
+
+				if err := d.writeDb(pointList); err != nil {
+					glog.Warn("flash file cache: can not write db", err)
+					continue
+				} else {
+					glog.Info("文件缓存写入influxdb成功：", filePath)
+				}
+
+				if err := os.Remove(filePath); err != nil {
+					glog.Error("删除文件失败：", filePath)
+				}
+			}
+			timer.Reset(d.flashTime)
+		case <-d.ctx.Done():
+			glog.Info("flash file cache 平滑退出")
+			timer.Stop()
+			d.wg.Done()
+			return
+		}
+	}
+}
+
+func (d *Dao) MsgProcess(point *client.Point) {
+	if point == nil {
+		return
+	}
+
+	if d.isClose {
+		return
+	}
+
+	d.channel <- point
+}
+
+//list满或者超时，则数据入库
+func (d *Dao) run() {
+	d.wg.Add(1)
+	defer func() {
+		if err := recover(); err != nil {
+			glog.Error(err)
+		}
+	}()
+
+	pointList := make([]*client.Point, 0, d.batchSize)
+	timer := time.NewTimer(d.flashTime)
+	for {
+		select {
+		case point := <-d.channel:
+			if len(pointList) == d.batchSize {
+				go d.batchWrite(pointList)
+				pointList = make([]*client.Point, 0, d.batchSize)
+				timer.Reset(d.flashTime)
+			}
+			pointList = append(pointList, point)
+		case <-timer.C:
+			//保存文件
+			go d.batchWrite(pointList)
+			pointList = make([]*client.Point, 0, d.batchSize)
+			timer.Reset(d.flashTime)
+		case <-d.ctx.Done():
+			go d.batchWrite(pointList)
+			glog.Info("存入influx平滑退出")
+			timer.Stop()
+			d.wg.Done()
+			return
+		}
+	}
+}
+
+func (d *Dao) writeDb(pointList []*client.Point) error {
+	defer func() {
+		if err := recover(); err != nil {
+			glog.Error("pointList panic!", err)
+			return
+		}
+	}()
+
+	config := client.HTTPConfig{
+		Addr:    d.dbAddress,
+		Timeout: time.Second * 10,
+	}
+
+	//建立连接
+	connect, err := client.NewHTTPClient(config)
+	defer func() { _ = connect.Close() }()
+	if err != nil {
+		return err
+	}
+
+	points, err := client.NewBatchPoints(client.BatchPointsConfig{
+		Database: global.InfluxDbName,
+	})
+	if err != nil {
+		return err
+	}
+
+	points.AddPoints(pointList)
+	err = connect.Write(points)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (d *Dao) writeFile(pointList []*client.Point) error {
+	defer func() {
+		if err := recover(); err != nil {
+			glog.Error("writeFile panic!", err)
+			return
+		}
+	}()
+
+	cachePointList := make([]CachePoint, 0)
+	for _, point := range pointList {
+		cachePointList = append(cachePointList, NewPoint(point))
+	}
+
+	buf := new(bytes.Buffer)
+	enc := gob.NewEncoder(buf)
+	if err := enc.Encode(cachePointList); err != nil {
+		return err
+	}
+	data := buf.Bytes()
+
+	fileName := time.Now().Format("2006-01-02T15:04:05.999999999")
+	writingFileName := path.Join(d.cacheFileDir, filePrefixWriting+":"+fileName)
+	if err := ioutil.WriteFile(writingFileName, data, 0644); err != nil {
+		return err
+	}
+
+	cacheFileName := path.Join(d.cacheFileDir, filePrefixCache+":"+fileName)
+	if err := os.Rename(writingFileName, cacheFileName); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (d *Dao) batchWrite(pointList []*client.Point) {
+	defer func() {
+		if err := recover(); err != nil {
+			glog.Error("batch write panic!", err)
+			return
+		}
+	}()
+
+	d.wg.Add(1)
+	defer d.wg.Done()
+
+	if len(pointList) == 0 {
+		return
+	}
+
+	if err := d.writeDb(pointList); err != nil {
+		glog.Warn("写入influxDB失败:", err)
+	} else {
+		glog.Infof("成功写入influxDB%d条", len(pointList))
+		return
+	}
+
+	if err := d.writeFile(pointList); err != nil {
+		glog.Error("写入文件缓存失败，数据丢失:", err)
+	} else {
+		glog.Infof("成功写入文件缓存%d条", len(pointList))
+		return
+	}
+}

pkg/dao/dao_test.go

+package dao
+
+import (
+	"fmt"
+	"github.com/influxdata/influxdb/client/v2"
+	"testing"
+	"time"
+)
+
+func TestNew(t *testing.T) {
+	d := New(1024, time.Second*60, "http://", ".")
+	if d == nil {
+		t.Error("d != nil")
+	}
+}
+
+func TestDao_MsgProcess(t *testing.T) {
+	d := New(1024, time.Second*5, "http://", ".")
+	fmt.Println("111222")
+	for i := 0; i < 11000; i++ {
+		point, _ := client.NewPoint("123", map[string]string{"123": "123"}, map[string]interface{}{"123": "123"})
+		d.MsgProcess(point)
+	}
+	fmt.Println("1111aaa")
+	time.Sleep(time.Second * 10)
+
+	for i := 0; i < 11000; i++ {
+		point, _ := client.NewPoint("123", map[string]string{"123": "123"}, map[string]interface{}{"123": "123"})
+		d.MsgProcess(point)
+	}
+
+	if d == nil {
+		t.Error("d != nil")
+	}
+	select {}
+}
+
+func TestDao_MsgProcess2(t *testing.T) {
+	d := New(1024, time.Second*5, "http://", "/Users/fengjunkai/llog")
+	time.Sleep(time.Second * 4)
+	fmt.Println("111222")
+	for i := 0; i < 11000; i++ {
+		point, _ := client.NewPoint("123", map[string]string{"123": "123"}, map[string]interface{}{"123": "123"})
+		d.MsgProcess(point)
+	}
+	fmt.Println("1111aaa")
+
+	for i := 0; i < 11000; i++ {
+		point, _ := client.NewPoint("123", map[string]string{"123": "123"}, map[string]interface{}{"123": "123"})
+		d.MsgProcess(point)
+	}
+
+	if d == nil {
+		t.Error("d != nil")
+	}
+	select {}
+}
+
+func TestChannel(t *testing.T) {
+	a := make(chan int, 10)
+
+	go func() {
+		defer func() {
+			if e := recover(); e != nil {
+				t.Log("平滑退出chan")
+			}
+		}()
+		for {
+			a <- 1
+			fmt.Println("111")
+			time.Sleep(time.Second)
+		}
+	}()
+
+	time.Sleep(time.Second * 5)
+
+	close(a)
+
+	time.Sleep(time.Second * 5)
+
+	for x := range a {
+		fmt.Println("print a:", x)
+	}
+}

pkg/dao/dbinit.go

+package dao
+
+import (
+	"encoding/json"
+	"fmt"
+	"git.quantgroup.cn/DevOps/enoch/pkg/global"
+	"git.quantgroup.cn/DevOps/enoch/pkg/glog"
+	"github.com/influxdata/influxdb/client/v2"
+	"strings"
+	"time"
+)
+
+const (
+	//apdex公式：apdex = (满意样本 + 可容忍样本/2) / 样本总数
+
+	//样本总数：request的总量
+	ApdexCqAll = `CREATE CONTINUOUS QUERY cq_apdex_all ON monitor RESAMPLE FOR 1h BEGIN ` +
+		`SELECT count(traceId) AS ct_all INTO monitor.autogen.apdex ` +
+		`FROM monitor.autogen.trace_info GROUP BY sys_name, time(1m) fill(1) END;`
+
+	//满意样本：request的响应时间[0, m)秒
+	ApdexCqSat = `CREATE CONTINUOUS QUERY cq_apdex_sat_%s ON monitor RESAMPLE FOR 1h BEGIN ` +
+		`SELECT count(traceId) AS sat INTO monitor.autogen.apdex FROM monitor.autogen.trace_info ` +
+		`WHERE sys_name = '%s' AND "duration" < %d ` +
+		`GROUP BY sys_name, time(1m) fill(0) END;`
+
+	//可容忍样本：request的响应时间[m, n)秒
+	ApdexCqTol = `CREATE CONTINUOUS QUERY cq_apdex_tol_%s ON monitor RESAMPLE FOR 1h BEGIN ` +
+		`SELECT count(traceId) AS tol INTO monitor.autogen.apdex FROM monitor.autogen.trace_info ` +
+		`WHERE sys_name = '%s' AND "duration" >= %d AND "duration" < %d ` +
+		`GROUP BY sys_name, time(1m) fill(0) END;`
+
+	//获取全部服务
+	SysNameSql = "show tag values from trace_info with key = sys_name;"
+
+	//显示连续查询
+	ShowCqSql = "SHOW CONTINUOUS QUERIES;"
+
+	//删除连续查询
+	DropCqSql = "DROP CONTINUOUS QUERY %s ON %s;"
+)
+
+var (
+	ApdexDefaultThreshold = Threshold{
+		Good:      100,
+		Available: 400,
+	}
+)
+
+type Threshold struct {
+	Good      int `json:"good"`      //良好门限
+	Available int `json:"available"` //可用门限
+}
+
+func init() {
+	//获取默认门限
+	if data, ok := global.Config.Get("threshold", "threshold.default"); ok {
+		if err := json.Unmarshal([]byte(data), &ApdexDefaultThreshold); err != nil {
+			glog.Warn("apdex default threshold json unmarshal")
+		}
+	}
+}
+
+//获取门限，如果获取不到则返回default
+func getApdexThreshold(sysName string) *Threshold {
+	rtn := Threshold{}
+	if data, ok := global.Config.Get("threshold", "threshold."+sysName); ok {
+		if err := json.Unmarshal([]byte(data), &rtn); err == nil {
+			return &rtn
+		}
+	}
+	return &ApdexDefaultThreshold
+}
+
+func apdexSatSql(sysName string, m int) string {
+	cqName := strings.Replace(sysName, "-", "_", 10)
+	return fmt.Sprintf(ApdexCqSat, cqName, sysName, m)
+}
+
+func apdexTolSql(sysName string, m int, n int) string {
+	cqName := strings.Replace(sysName, "-", "_", 10)
+	return fmt.Sprintf(ApdexCqTol, cqName, sysName, m, n)
+}
+
+func DbInit() {
+	config := client.HTTPConfig{
+		Addr:    global.InfluxDbAddress,
+		Timeout: time.Second * 4,
+	}
+	c, err := client.NewHTTPClient(config)
+	if err != nil {
+		glog.Error("can not new influxdb http client")
+		return
+	}
+
+	//创建数据库
+	if _, err := c.Query(client.Query{Command: "create database " + global.InfluxDbName}); err != nil {
+		glog.Error(err)
+		return
+	}
+
+	//删除连续查询
+	cqList, err := c.Query(client.Query{Command: ShowCqSql, Database: global.InfluxDbName})
+	if err != nil {
+		glog.Error(err)
+		return
+	}
+	cqNameList := make([]string, 0)
+	if len(cqList.Results) > 0 {
+		for _, row := range cqList.Results[0].Series {
+			if row.Name == global.InfluxDbName {
+				for _, v := range row.Values {
+					cqNameList = append(cqNameList, v[0].(string))
+				}
+			}
+		}
+	}
+	delCqSqls := strings.Builder{}
+	for _, cqName := range cqNameList {
+		delCqSqls.WriteString(fmt.Sprintf(DropCqSql, cqName, global.InfluxDbName))
+	}
+	glog.Info(delCqSqls.String())
+	if _, err := c.Query(client.Query{Command: delCqSqls.String(), Database: global.InfluxDbName}); err != nil {
+		glog.Error(err)
+	}
+
+	//添加连续查询(sat & tol)
+	serviceList, err := c.Query(client.Query{Command: SysNameSql, Database: global.InfluxDbName})
+	if err != nil {
+		glog.Error(err)
+		return
+	}
+	sysNameList := make([]string, 0)
+	if len(serviceList.Results) > 0 &&
+		len(serviceList.Results[0].Series) > 0 &&
+		len(serviceList.Results[0].Series[0].Values) > 1 {
+		for _, v := range serviceList.Results[0].Series[0].Values {
+			sysNameList = append(sysNameList, v[1].(string))
+		}
+	}
+	glog.Debug(sysNameList)
+	newCqSqls := strings.Builder{}
+	for _, sysName := range sysNameList {
+		//根据sys_name获取threshold
+		apdexThreshold := getApdexThreshold(sysName)
+		newCqSqls.WriteString(apdexSatSql(sysName, apdexThreshold.Good))
+		newCqSqls.WriteString(apdexTolSql(sysName, apdexThreshold.Good, apdexThreshold.Available))
+	}
+	glog.Info(newCqSqls.String())
+	if _, err := c.Query(client.Query{Command: newCqSqls.String(), Database: global.InfluxDbName}); err != nil {
+		glog.Error(err)
+	}
+
+	//创建连续查询(ALL)
+	if _, err := c.Query(client.Query{Command: ApdexCqAll, Database: global.InfluxDbName}); err != nil {
+		glog.Error(err)
+		return
+	}
+}

service/util/sendemail.go → pkg/email/email.go

-package util
-
+package email
 
 import (
 	"crypto/tls"
 	"gopkg.in/gomail.v2"
 )
 
-
 type HostInfo struct {
 	address string
 	port    int
 }
 
-func SendEmail(title string, content string, receiver ... string) {
+func SendEmail(title string, content string, receiver ...string) {
 	hostInfo := HostInfo{"mail.quantgroup.cn", 587}
 
 	m := gomail.NewMessage()
@@ -20,7 +18,7 @@ func SendEmail(title string, content string, receiver ... string) {
 	//m.SetHeader("To", []string{receiver})
 	m.SetHeader("To", receiver...)
 	m.SetHeader("Subject", title)
-	m.SetBody("text/plain", content)
+	m.SetBody("text/html", content)
 
 	userName := "program@quantgroup.cn"
 	pwd := "Fuck147999!!!"

pkg/global/global.go

+package global
+
+import (
+	"encoding/json"
+	"github.com/Shopify/sarama"
+	"github.com/valyala/fasthttp"
+	"github.com/vrg0/go-common/args"
+	"github.com/vrg0/go-common/conf"
+	"github.com/vrg0/go-common/kafka"
+	"github.com/vrg0/go-common/logger"
+	"github.com/vrg0/go-common/registry"
+	"github.com/vrg0/go-common/util"
+	"go.uber.org/zap/zapcore"
+	"os"
+	"strconv"
+	"strings"
+)
+
+const (
+	AppName = "enoch"
+
+	//NAMESPACE
+	NamespaceApplication = "application"
+	NamespaceTechDeploy  = "tech.deploy"
+	NamespaceTechSleuth  = "tech.sleuth"
+
+	//kafka group
+	KafkaGroup = AppName
+
+	//influxDB的数据库名称
+	InfluxDbName = "monitor"
+
+	BatchSize = 4096
+)
+
+var (
+	Env                            = args.GetOrDefault("env", "dev")
+	Idc                            = args.GetOrDefault("idc", "k8s")
+	EosNamespace                   = args.GetOrDefault("eos_namespace", "arch")
+	EosHost                        = args.GetOrDefault("eos_host", "http://eos.quantgroups.com/")
+	LocalIp, _                     = util.LocalIp()
+	Config          *conf.Conf     = nil
+	Logger          *logger.Logger = nil
+	KafkaVersion                   = sarama.V1_0_0_0
+	KafkaRecver     *kafka.Recver  = nil
+	InfluxDbAddress                = ""
+	DaoFileCacheDir                = ""
+	ConsulDc                       = ""
+	ConsulAddress                  = ""
+	ReportFormDir                  = ""
+	LogPath                        = ""
+	LogLevel                       = zapcore.DebugLevel
+	HttpPort                       = 9091
+)
+
+type EosResult struct {
+	Success bool                   `json:"success"`
+	Details map[string]interface{} `json:"details"`
+}
+
+func init() {
+	//初始化配置
+	url := "apollo-" + Env + ".quantgroups.com/"
+	if config, err := conf.New(url, AppName, Idc, os.Args[0]+".cache_file", nil); err != nil {
+		panic(err)
+	} else {
+		Config = config
+	}
+
+	//测试环境添加kv替换
+	if IsDev() {
+		statusCode, body, err := fasthttp.Get(nil, EosHost+"api/apollo/env_vars?namespace="+EosNamespace)
+		if err != nil {
+			panic(err)
+		}
+		if statusCode != 200 {
+			panic("初始化eos数据出错:" + string(body))
+		}
+		var result = EosResult{}
+		err = json.Unmarshal(body, &result)
+		if !result.Success {
+			panic("eos数据解析出错" + string(body))
+		}
+		kvMap := make(map[string]string)
+		for k, v := range result.Details {
+			if vString, ok := v.(string); ok {
+				kvMap[k] = vString
+			}
+			if vFloat64, ok := v.(float64); ok {
+				kvMap[k] = strconv.Itoa(int(vFloat64))
+			}
+		}
+		Config.RefreshKvMap(kvMap)
+	}
+
+	//初始化日志
+	LogPath = Config.GetOrDefault(NamespaceApplication, "log.path", "/dev/stdout")
+	LogLevel = getLoggerLevel(Config.GetOrDefault(NamespaceApplication, "log.level", "info"))
+	Logger = logger.New(LogPath, LogLevel)
+
+	//初始化kafka
+	kafkaLogger := Logger.GetStandardLogger()
+	kafkaLogger.SetPrefix("_KAFKA_RECVER_ ")
+
+	kafkaAddress, ok := Config.Get(NamespaceTechSleuth, "tech.enoch.kafkaHost")
+	if !ok {
+		Logger.Fatal("can not get kafka address")
+	} else {
+		Logger.Info("kafkaAddress:", kafkaAddress)
+	}
+	KafkaRecver = kafka.NewRecver(KafkaVersion, strings.Split(kafkaAddress, ","), kafkaLogger)
+
+	InfluxDbAddress = Config.GetOrDefault(NamespaceApplication, "influxdb.address", "")
+	//InfluxDbAddress = "http://172.20.6.33:8086"
+	DaoFileCacheDir = Config.GetOrDefault(NamespaceApplication, "dao.file.cache.dir", "/var")
+	ReportFormDir = Config.GetOrDefault(NamespaceApplication, "report.form.dir", "/var")
+	httpPortStr := Config.GetOrDefault(NamespaceTechDeploy, "http_port", "9091")
+	port, err := strconv.Atoi(httpPortStr)
+	if err != nil {
+		Logger.Error("a to i http_port err:", httpPortStr)
+	} else {
+		HttpPort = port
+	}
+
+	//初始化registry
+	if consulDc, ok := Config.Get(NamespaceApplication, "consul.datacenter"); !ok {
+		Logger.Error("get must conf error: application consul.datacenter")
+	} else {
+		ConsulDc = consulDc
+		Logger.Debug("consul dc", ConsulDc)
+	}
+	if consulAddress, ok := Config.Get(NamespaceApplication, "consul.address"); !ok {
+		Logger.Error("get must conf error: application consul.address")
+	} else {
+		ConsulAddress = consulAddress
+		Logger.Debug("consul address", ConsulAddress)
+	}
+	consulCluster := strings.Split(ConsulAddress, ",")
+	if e := registry.Init("consul", map[string]interface{}{"dc": ConsulDc, "cluster": consulCluster}); e != nil {
+		Logger.Error("consul初始化失败", e)
+	}
+}
+
+func getLoggerLevel(level string) zapcore.Level {
+	switch strings.ToLower(level) {
+	case "info":
+		return zapcore.InfoLevel
+	case "debug":
+		return zapcore.DebugLevel
+	case "panic":
+		return zapcore.PanicLevel
+	case "warning":
+		return zapcore.WarnLevel
+	case "fatal":
+		return zapcore.FatalLevel
+	case "error":
+		return zapcore.ErrorLevel
+	default:
+		return zapcore.InfoLevel
+	}
+}
+
+func IsDev() bool {
+	return strings.ToUpper(Env) == "DEV"
+}

pkg/global/global_test.go

+package global
+
+import "testing"
+
+func TestGlobal(t *testing.T) {
+	t.Log(Env)
+	t.Log(Idc)
+	t.Log(EosNamespace)
+	t.Log(EosHost)
+	t.Log(LocalIp)
+	t.Log(AppName)
+	t.Log(Config.GetOrDefault(NamespaceApplication, "log.level", "unknown"))
+	Logger.Info("log test")
+}

pkg/glog/glog.go

+package glog
+
+import "git.quantgroup.cn/DevOps/enoch/pkg/global"
+
+var sugar = global.Logger
+
+func Debug(args ...interface{}) {
+	sugar.Debug(args...)
+}
+
+func Info(args ...interface{}) {
+	sugar.Info(args...)
+}
+
+func Warn(args ...interface{}) {
+	sugar.Warn(args...)
+}
+
+func Error(args ...interface{}) {
+	sugar.Error(args...)
+}
+
+func DPanic(args ...interface{}) {
+	sugar.DPanic(args...)
+}
+
+func Panic(args ...interface{}) {
+	sugar.Panic(args...)
+}
+
+func Fatal(args ...interface{}) {
+	sugar.Fatal(args...)
+}
+
+func Debugf(template string, args ...interface{}) {
+	sugar.Debugf(template, args...)
+}
+
+func Infof(template string, args ...interface{}) {
+	sugar.Infof(template, args...)
+}
+
+func Warnf(template string, args ...interface{}) {
+	sugar.Warnf(template, args...)
+}
+
+func Errorf(template string, args ...interface{}) {
+	sugar.Errorf(template, args...)
+}
+
+func DPanicf(template string, args ...interface{}) {
+	sugar.DPanicf(template, args...)
+}
+
+func Panicf(template string, args ...interface{}) {
+	sugar.Panicf(template, args...)
+}
+
+func Fatalf(template string, args ...interface{}) {
+	sugar.Fatalf(template, args...)
+}
+
+func Debugw(msg string, keysAndValues ...interface{}) {
+	sugar.Debugw(msg, keysAndValues...)
+}
+
+func Infow(msg string, keysAndValues ...interface{}) {
+	sugar.Infow(msg, keysAndValues...)
+}
+
+func Warnw(msg string, keysAndValues ...interface{}) {
+	sugar.Warnw(msg, keysAndValues...)
+}
+
+func Errorw(msg string, keysAndValues ...interface{}) {
+	sugar.Errorw(msg, keysAndValues...)
+}
+
+func DPanicw(msg string, keysAndValues ...interface{}) {
+	sugar.DPanicw(msg, keysAndValues...)
+}
+
+func Panicw(msg string, keysAndValues ...interface{}) {
+	sugar.Panicw(msg, keysAndValues...)
+}
+
+func Fatalw(msg string, keysAndValues ...interface{}) {
+	sugar.Fatalw(msg, keysAndValues...)
+}

pkg/glog/glog_test.go

+package glog
+
+import "testing"
+
+func TestGlog(t *testing.T) {
+	Debug("hello")
+	Info("hello")
+	Warn("hello")
+	Error("hello")
+	DPanic("hello")
+	//Panic("hello")
+	//Fatal("hello")
+	Debugf("hello")
+	Infof("hello")
+	Warnf("hello")
+	Errorf("hello")
+	DPanicf("hello")
+	//Panicf("hello")
+	//Fatalf("hello")
+	Debugw("hello")
+	Infow("hello")
+	Warnw("hello")
+	Errorw("hello")
+	DPanicw("hello")
+	//Panicw("hello")
+	//Fatalw("hello")
+}

pkg/mod/cache/vcs/1bb0f6aea2776dd67270c7cb096c102f9d61a1a91114cf9c8eff7a222a21173c.info

-git2:https://github.com/DataDog/zstd
\ No newline at end of file

pkg/mod/cache/vcs/1bb0f6aea2776dd67270c7cb096c102f9d61a1a91114cf9c8eff7a222a21173c/FETCH_HEAD

-aebefd9fcb99f22cd691ef778a12ed68f0e6a1ab		tag 'v1.3.4' of https://github.com/DataDog/zstd

pkg/mod/cache/vcs/1bb0f6aea2776dd67270c7cb096c102f9d61a1a91114cf9c8eff7a222a21173c/HEAD

-ref: refs/heads/master

pkg/mod/cache/vcs/1bb0f6aea2776dd67270c7cb096c102f9d61a1a91114cf9c8eff7a222a21173c/config

-[core]
-	repositoryformatversion = 0
-	filemode = true
-	bare = true
-	ignorecase = true
-	precomposeunicode = true
-[remote "origin"]
-	url = https://github.com/DataDog/zstd
-	fetch = +refs/heads/*:refs/remotes/origin/*

pkg/mod/cache/vcs/1bb0f6aea2776dd67270c7cb096c102f9d61a1a91114cf9c8eff7a222a21173c/description

-Unnamed repository; edit this file 'description' to name the repository.

pkg/mod/cache/vcs/1bb0f6aea2776dd67270c7cb096c102f9d61a1a91114cf9c8eff7a222a21173c/hooks/applypatch-msg.sample

-#!/bin/sh
-#
-# An example hook script to check the commit log message taken by
-# applypatch from an e-mail message.
-#
-# The hook should exit with non-zero status after issuing an
-# appropriate message if it wants to stop the commit.  The hook is
-# allowed to edit the commit message file.
-#
-# To enable this hook, rename this file to "applypatch-msg".
-
-. git-sh-setup
-commitmsg="$(git rev-parse --git-path hooks/commit-msg)"
-test -x "$commitmsg" && exec "$commitmsg" ${1+"$@"}
-:

pkg/mod/cache/vcs/1bb0f6aea2776dd67270c7cb096c102f9d61a1a91114cf9c8eff7a222a21173c/hooks/commit-msg.sample

-#!/bin/sh
-#
-# An example hook script to check the commit log message.
-# Called by "git commit" with one argument, the name of the file
-# that has the commit message.  The hook should exit with non-zero
-# status after issuing an appropriate message if it wants to stop the
-# commit.  The hook is allowed to edit the commit message file.
-#
-# To enable this hook, rename this file to "commit-msg".
-
-# Uncomment the below to add a Signed-off-by line to the message.
-# Doing this in a hook is a bad idea in general, but the prepare-commit-msg
-# hook is more suited to it.
-#
-# SOB=$(git var GIT_AUTHOR_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p')
-# grep -qs "^$SOB" "$1" || echo "$SOB" >> "$1"
-
-# This example catches duplicate Signed-off-by lines.
-
-test "" = "$(grep '^Signed-off-by: ' "$1" |
-	 sort | uniq -c | sed -e '/^[ 	]*1[ 	]/d')" || {
-	echo >&2 Duplicate Signed-off-by lines.
-	exit 1
-}

pkg/mod/cache/vcs/1bb0f6aea2776dd67270c7cb096c102f9d61a1a91114cf9c8eff7a222a21173c/hooks/fsmonitor-watchman.sample

-#!/usr/bin/perl
-
-use strict;
-use warnings;
-use IPC::Open2;
-
-# An example hook script to integrate Watchman
-# (https://facebook.github.io/watchman/) with git to speed up detecting
-# new and modified files.
-#
-# The hook is passed a version (currently 1) and a time in nanoseconds
-# formatted as a string and outputs to stdout all files that have been
-# modified since the given time. Paths must be relative to the root of
-# the working tree and separated by a single NUL.
-#
-# To enable this hook, rename this file to "query-watchman" and set
-# 'git config core.fsmonitor .git/hooks/query-watchman'
-#
-my ($version, $time) = @ARGV;
-
-# Check the hook interface version
-
-if ($version == 1) {
-	# convert nanoseconds to seconds
-	$time = int $time / 1000000000;
-} else {
-	die "Unsupported query-fsmonitor hook version '$version'.\n" .
-	    "Falling back to scanning...\n";
-}
-
-my $git_work_tree;
-if ($^O =~ 'msys' || $^O =~ 'cygwin') {
-	$git_work_tree = Win32::GetCwd();
-	$git_work_tree =~ tr/\\/\//;
-} else {
-	require Cwd;
-	$git_work_tree = Cwd::cwd();
-}
-
-my $retry = 1;
-
-launch_watchman();
-
-sub launch_watchman {
-
-	my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j --no-pretty')
-	    or die "open2() failed: $!\n" .
-	    "Falling back to scanning...\n";
-
-	# In the query expression below we're asking for names of files that
-	# changed since $time but were not transient (ie created after
-	# $time but no longer exist).
-	#
-	# To accomplish this, we're using the "since" generator to use the
-	# recency index to select candidate nodes and "fields" to limit the
-	# output to file names only. Then we're using the "expression" term to
-	# further constrain the results.
-	#
-	# The category of transient files that we want to ignore will have a
-	# creation clock (cclock) newer than $time_t value and will also not
-	# currently exist.
-
-	my $query = <<"	END";
-		["query", "$git_work_tree", {
-			"since": $time,
-			"fields": ["name"],
-			"expression": ["not", ["allof", ["since", $time, "cclock"], ["not", "exists"]]]
-		}]
-	END
-
-	print CHLD_IN $query;
-	close CHLD_IN;
-	my $response = do {local $/; <CHLD_OUT>};
-
-	die "Watchman: command returned no output.\n" .
-	    "Falling back to scanning...\n" if $response eq "";
-	die "Watchman: command returned invalid output: $response\n" .
-	    "Falling back to scanning...\n" unless $response =~ /^\{/;
-
-	my $json_pkg;
-	eval {
-		require JSON::XS;
-		$json_pkg = "JSON::XS";
-		1;
-	} or do {
-		require JSON::PP;
-		$json_pkg = "JSON::PP";
-	};
-
-	my $o = $json_pkg->new->utf8->decode($response);
-
-	if ($retry > 0 and $o->{error} and $o->{error} =~ m/unable to resolve root .* directory (.*) is not watched/) {
-		print STDERR "Adding '$git_work_tree' to watchman's watch list.\n";
-		$retry--;
-		qx/watchman watch "$git_work_tree"/;
-		die "Failed to make watchman watch '$git_work_tree'.\n" .
-		    "Falling back to scanning...\n" if $? != 0;
-
-		# Watchman will always return all files on the first query so
-		# return the fast "everything is dirty" flag to git and do the
-		# Watchman query just to get it over with now so we won't pay
-		# the cost in git to look up each individual file.
-		print "/\0";
-		eval { launch_watchman() };
-		exit 0;
-	}
-
-	die "Watchman: $o->{error}.\n" .
-	    "Falling back to scanning...\n" if $o->{error};
-
-	binmode STDOUT, ":utf8";
-	local $, = "\0";
-	print @{$o->{files}};
-}

pkg/mod/cache/vcs/1bb0f6aea2776dd67270c7cb096c102f9d61a1a91114cf9c8eff7a222a21173c/hooks/post-update.sample

-#!/bin/sh
-#
-# An example hook script to prepare a packed repository for use over
-# dumb transports.
-#
-# To enable this hook, rename this file to "post-update".
-
-exec git update-server-info

pkg/mod/cache/vcs/1bb0f6aea2776dd67270c7cb096c102f9d61a1a91114cf9c8eff7a222a21173c/hooks/pre-applypatch.sample

-#!/bin/sh
-#
-# An example hook script to verify what is about to be committed
-# by applypatch from an e-mail message.
-#
-# The hook should exit with non-zero status after issuing an
-# appropriate message if it wants to stop the commit.
-#
-# To enable this hook, rename this file to "pre-applypatch".
-
-. git-sh-setup
-precommit="$(git rev-parse --git-path hooks/pre-commit)"
-test -x "$precommit" && exec "$precommit" ${1+"$@"}
-:

pkg/mod/cache/vcs/1bb0f6aea2776dd67270c7cb096c102f9d61a1a91114cf9c8eff7a222a21173c/hooks/pre-commit.sample

-#!/bin/sh
-#
-# An example hook script to verify what is about to be committed.
-# Called by "git commit" with no arguments.  The hook should
-# exit with non-zero status after issuing an appropriate message if
-# it wants to stop the commit.
-#
-# To enable this hook, rename this file to "pre-commit".
-
-if git rev-parse --verify HEAD >/dev/null 2>&1
-then
-	against=HEAD
-else
-	# Initial commit: diff against an empty tree object
-	against=4b825dc642cb6eb9a060e54bf8d69288fbee4904
-fi
-
-# If you want to allow non-ASCII filenames set this variable to true.
-allownonascii=$(git config --bool hooks.allownonascii)
-
-# Redirect output to stderr.
-exec 1>&2
-
-# Cross platform projects tend to avoid non-ASCII filenames; prevent
-# them from being added to the repository. We exploit the fact that the
-# printable range starts at the space character and ends with tilde.
-if [ "$allownonascii" != "true" ] &&
-	# Note that the use of brackets around a tr range is ok here, (it's
-	# even required, for portability to Solaris 10's /usr/bin/tr), since
-	# the square bracket bytes happen to fall in the designated range.
-	test $(git diff --cached --name-only --diff-filter=A -z $against |
-	  LC_ALL=C tr -d '[ -~]\0' | wc -c) != 0
-then
-	cat <<\EOF
-Error: Attempt to add a non-ASCII file name.
-
-This can cause problems if you want to work with people on other platforms.
-
-To be portable it is advisable to rename the file.
-
-If you know what you are doing you can disable this check using:
-
-  git config hooks.allownonascii true
-EOF
-	exit 1
-fi
-
-# If there are whitespace errors, print the offending file names and fail.
-exec git diff-index --check --cached $against --

pkg/mod/cache/vcs/1bb0f6aea2776dd67270c7cb096c102f9d61a1a91114cf9c8eff7a222a21173c/hooks/pre-push.sample

-#!/bin/sh
-
-# An example hook script to verify what is about to be pushed.  Called by "git
-# push" after it has checked the remote status, but before anything has been
-# pushed.  If this script exits with a non-zero status nothing will be pushed.
-#
-# This hook is called with the following parameters:
-#
-# $1 -- Name of the remote to which the push is being done
-# $2 -- URL to which the push is being done
-#
-# If pushing without using a named remote those arguments will be equal.
-#
-# Information about the commits which are being pushed is supplied as lines to
-# the standard input in the form:
-#
-#   <local ref> <local sha1> <remote ref> <remote sha1>
-#
-# This sample shows how to prevent push of commits where the log message starts
-# with "WIP" (work in progress).
-
-remote="$1"
-url="$2"
-
-z40=0000000000000000000000000000000000000000
-
-while read local_ref local_sha remote_ref remote_sha
-do
-	if [ "$local_sha" = $z40 ]
-	then
-		# Handle delete
-		:
-	else
-		if [ "$remote_sha" = $z40 ]
-		then
-			# New branch, examine all commits
-			range="$local_sha"
-		else
-			# Update to existing branch, examine new commits
-			range="$remote_sha..$local_sha"
-		fi
-
-		# Check for WIP commit
-		commit=`git rev-list -n 1 --grep '^WIP' "$range"`
-		if [ -n "$commit" ]
-		then
-			echo >&2 "Found WIP commit in $local_ref, not pushing"
-			exit 1
-		fi
-	fi
-done
-
-exit 0

pkg/mod/cache/vcs/1bb0f6aea2776dd67270c7cb096c102f9d61a1a91114cf9c8eff7a222a21173c/hooks/pre-rebase.sample

-#!/bin/sh
-#
-# Copyright (c) 2006, 2008 Junio C Hamano
-#
-# The "pre-rebase" hook is run just before "git rebase" starts doing
-# its job, and can prevent the command from running by exiting with
-# non-zero status.
-#
-# The hook is called with the following parameters:
-#
-# $1 -- the upstream the series was forked from.
-# $2 -- the branch being rebased (or empty when rebasing the current branch).
-#
-# This sample shows how to prevent topic branches that are already
-# merged to 'next' branch from getting rebased, because allowing it
-# would result in rebasing already published history.
-
-publish=next
-basebranch="$1"
-if test "$#" = 2
-then
-	topic="refs/heads/$2"
-else
-	topic=`git symbolic-ref HEAD` ||
-	exit 0 ;# we do not interrupt rebasing detached HEAD
-fi
-
-case "$topic" in
-refs/heads/??/*)
-	;;
-*)
-	exit 0 ;# we do not interrupt others.
-	;;
-esac
-
-# Now we are dealing with a topic branch being rebased
-# on top of master.  Is it OK to rebase it?
-
-# Does the topic really exist?
-git show-ref -q "$topic" || {
-	echo >&2 "No such branch $topic"
-	exit 1
-}
-
-# Is topic fully merged to master?
-not_in_master=`git rev-list --pretty=oneline ^master "$topic"`
-if test -z "$not_in_master"
-then
-	echo >&2 "$topic is fully merged to master; better remove it."
-	exit 1 ;# we could allow it, but there is no point.
-fi
-
-# Is topic ever merged to next?  If so you should not be rebasing it.
-only_next_1=`git rev-list ^master "^$topic" ${publish} | sort`
-only_next_2=`git rev-list ^master           ${publish} | sort`
-if test "$only_next_1" = "$only_next_2"
-then
-	not_in_topic=`git rev-list "^$topic" master`
-	if test -z "$not_in_topic"
-	then
-		echo >&2 "$topic is already up to date with master"
-		exit 1 ;# we could allow it, but there is no point.
-	else
-		exit 0
-	fi
-else
-	not_in_next=`git rev-list --pretty=oneline ^${publish} "$topic"`
-	/usr/bin/perl -e '
-		my $topic = $ARGV[0];
-		my $msg = "* $topic has commits already merged to public branch:\n";
-		my (%not_in_next) = map {
-			/^([0-9a-f]+) /;
-			($1 => 1);
-		} split(/\n/, $ARGV[1]);
-		for my $elem (map {
-				/^([0-9a-f]+) (.*)$/;
-				[$1 => $2];
-			} split(/\n/, $ARGV[2])) {
-			if (!exists $not_in_next{$elem->[0]}) {
-				if ($msg) {
-					print STDERR $msg;
-					undef $msg;
-				}
-				print STDERR " $elem->[1]\n";
-			}
-		}
-	' "$topic" "$not_in_next" "$not_in_master"
-	exit 1
-fi
-
-<<\DOC_END
-
-This sample hook safeguards topic branches that have been
-published from being rewound.
-
-The workflow assumed here is:
-
- * Once a topic branch forks from "master", "master" is never
-   merged into it again (either directly or indirectly).
-
- * Once a topic branch is fully cooked and merged into "master",
-   it is deleted.  If you need to build on top of it to correct
-   earlier mistakes, a new topic branch is created by forking at
-   the tip of the "master".  This is not strictly necessary, but
-   it makes it easier to keep your history simple.
-
- * Whenever you need to test or publish your changes to topic
-   branches, merge them into "next" branch.
-
-The script, being an example, hardcodes the publish branch name
-to be "next", but it is trivial to make it configurable via
-$GIT_DIR/config mechanism.
-
-With this workflow, you would want to know:
-
-(1) ... if a topic branch has ever been merged to "next".  Young
-    topic branches can have stupid mistakes you would rather
-    clean up before publishing, and things that have not been
-    merged into other branches can be easily rebased without
-    affecting other people.  But once it is published, you would
-    not want to rewind it.
-
-(2) ... if a topic branch has been fully merged to "master".
-    Then you can delete it.  More importantly, you should not
-    build on top of it -- other people may already want to
-    change things related to the topic as patches against your
-    "master", so if you need further changes, it is better to
-    fork the topic (perhaps with the same name) afresh from the
-    tip of "master".
-
-Let's look at this example:
-
-		   o---o---o---o---o---o---o---o---o---o "next"
-		  /       /           /           /
-		 /   a---a---b A     /           /
-		/   /               /           /
-	       /   /   c---c---c---c B         /
-	      /   /   /             \         /
-	     /   /   /   b---b C     \       /
-	    /   /   /   /             \     /
-    ---o---o---o---o---o---o---o---o---o---o---o "master"
-
-
-A, B and C are topic branches.
-
- * A has one fix since it was merged up to "next".
-
- * B has finished.  It has been fully merged up to "master" and "next",
-   and is ready to be deleted.
-
- * C has not merged to "next" at all.
-
-We would want to allow C to be rebased, refuse A, and encourage
-B to be deleted.
-
-To compute (1):
-
-	git rev-list ^master ^topic next
-	git rev-list ^master        next
-
-	if these match, topic has not merged in next at all.
-
-To compute (2):
-
-	git rev-list master..topic
-
-	if this is empty, it is fully merged to "master".
-
-DOC_END

pkg/mod/cache/vcs/1bb0f6aea2776dd67270c7cb096c102f9d61a1a91114cf9c8eff7a222a21173c/hooks/pre-receive.sample

-#!/bin/sh
-#
-# An example hook script to make use of push options.
-# The example simply echoes all push options that start with 'echoback='
-# and rejects all pushes when the "reject" push option is used.
-#
-# To enable this hook, rename this file to "pre-receive".
-
-if test -n "$GIT_PUSH_OPTION_COUNT"
-then
-	i=0
-	while test "$i" -lt "$GIT_PUSH_OPTION_COUNT"
-	do
-		eval "value=\$GIT_PUSH_OPTION_$i"
-		case "$value" in
-		echoback=*)
-			echo "echo from the pre-receive-hook: ${value#*=}" >&2
-			;;
-		reject)
-			exit 1
-		esac
-		i=$((i + 1))
-	done
-fi

pkg/mod/cache/vcs/1bb0f6aea2776dd67270c7cb096c102f9d61a1a91114cf9c8eff7a222a21173c/hooks/prepare-commit-msg.sample

-#!/bin/sh
-#
-# An example hook script to prepare the commit log message.
-# Called by "git commit" with the name of the file that has the
-# commit message, followed by the description of the commit
-# message's source.  The hook's purpose is to edit the commit
-# message file.  If the hook fails with a non-zero status,
-# the commit is aborted.
-#
-# To enable this hook, rename this file to "prepare-commit-msg".
-
-# This hook includes three examples. The first one removes the
-# "# Please enter the commit message..." help message.
-#
-# The second includes the output of "git diff --name-status -r"
-# into the message, just before the "git status" output.  It is
-# commented because it doesn't cope with --amend or with squashed
-# commits.
-#
-# The third example adds a Signed-off-by line to the message, that can
-# still be edited.  This is rarely a good idea.
-
-COMMIT_MSG_FILE=$1
-COMMIT_SOURCE=$2
-SHA1=$3
-
-/usr/bin/perl -i.bak -ne 'print unless(m/^. Please enter the commit message/..m/^#$/)' "$COMMIT_MSG_FILE"
-
-# case "$COMMIT_SOURCE,$SHA1" in
-#  ,|template,)
-#    /usr/bin/perl -i.bak -pe '
-#       print "\n" . `git diff --cached --name-status -r`
-# 	 if /^#/ && $first++ == 0' "$COMMIT_MSG_FILE" ;;
-#  *) ;;
-# esac
-
-# SOB=$(git var GIT_COMMITTER_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p')
-# git interpret-trailers --in-place --trailer "$SOB" "$COMMIT_MSG_FILE"
-# if test -z "$COMMIT_SOURCE"
-# then
-#   /usr/bin/perl -i.bak -pe 'print "\n" if !$first_line++' "$COMMIT_MSG_FILE"
-# fi

pkg/mod/cache/vcs/1bb0f6aea2776dd67270c7cb096c102f9d61a1a91114cf9c8eff7a222a21173c/hooks/update.sample

-#!/bin/sh
-#
-# An example hook script to block unannotated tags from entering.
-# Called by "git receive-pack" with arguments: refname sha1-old sha1-new
-#
-# To enable this hook, rename this file to "update".
-#
-# Config
-# ------
-# hooks.allowunannotated
-#   This boolean sets whether unannotated tags will be allowed into the
-#   repository.  By default they won't be.
-# hooks.allowdeletetag
-#   This boolean sets whether deleting tags will be allowed in the
-#   repository.  By default they won't be.
-# hooks.allowmodifytag
-#   This boolean sets whether a tag may be modified after creation. By default
-#   it won't be.
-# hooks.allowdeletebranch
-#   This boolean sets whether deleting branches will be allowed in the
-#   repository.  By default they won't be.
-# hooks.denycreatebranch
-#   This boolean sets whether remotely creating branches will be denied
-#   in the repository.  By default this is allowed.
-#
-
-# --- Command line
-refname="$1"
-oldrev="$2"
-newrev="$3"
-
-# --- Safety check
-if [ -z "$GIT_DIR" ]; then
-	echo "Don't run this script from the command line." >&2
-	echo " (if you want, you could supply GIT_DIR then run" >&2
-	echo "  $0 <ref> <oldrev> <newrev>)" >&2
-	exit 1
-fi
-
-if [ -z "$refname" -o -z "$oldrev" -o -z "$newrev" ]; then
-	echo "usage: $0 <ref> <oldrev> <newrev>" >&2
-	exit 1
-fi
-
-# --- Config
-allowunannotated=$(git config --bool hooks.allowunannotated)
-allowdeletebranch=$(git config --bool hooks.allowdeletebranch)
-denycreatebranch=$(git config --bool hooks.denycreatebranch)
-allowdeletetag=$(git config --bool hooks.allowdeletetag)
-allowmodifytag=$(git config --bool hooks.allowmodifytag)
-
-# check for no description
-projectdesc=$(sed -e '1q' "$GIT_DIR/description")
-case "$projectdesc" in
-"Unnamed repository"* | "")
-	echo "*** Project description file hasn't been set" >&2
-	exit 1
-	;;
-esac
-
-# --- Check types
-# if $newrev is 0000...0000, it's a commit to delete a ref.
-zero="0000000000000000000000000000000000000000"
-if [ "$newrev" = "$zero" ]; then
-	newrev_type=delete
-else
-	newrev_type=$(git cat-file -t $newrev)
-fi
-
-case "$refname","$newrev_type" in
-	refs/tags/*,commit)
-		# un-annotated tag
-		short_refname=${refname##refs/tags/}
-		if [ "$allowunannotated" != "true" ]; then
-			echo "*** The un-annotated tag, $short_refname, is not allowed in this repository" >&2
-			echo "*** Use 'git tag [ -a | -s ]' for tags you want to propagate." >&2
-			exit 1
-		fi
-		;;
-	refs/tags/*,delete)
-		# delete tag
-		if [ "$allowdeletetag" != "true" ]; then
-			echo "*** Deleting a tag is not allowed in this repository" >&2
-			exit 1
-		fi
-		;;
-	refs/tags/*,tag)
-		# annotated tag
-		if [ "$allowmodifytag" != "true" ] && git rev-parse $refname > /dev/null 2>&1
-		then
-			echo "*** Tag '$refname' already exists." >&2
-			echo "*** Modifying a tag is not allowed in this repository." >&2
-			exit 1
-		fi
-		;;
-	refs/heads/*,commit)
-		# branch
-		if [ "$oldrev" = "$zero" -a "$denycreatebranch" = "true" ]; then
-			echo "*** Creating a branch is not allowed in this repository" >&2
-			exit 1
-		fi
-		;;
-	refs/heads/*,delete)
-		# delete branch
-		if [ "$allowdeletebranch" != "true" ]; then
-			echo "*** Deleting a branch is not allowed in this repository" >&2
-			exit 1
-		fi
-		;;
-	refs/remotes/*,commit)
-		# tracking branch
-		;;
-	refs/remotes/*,delete)
-		# delete tracking branch
-		if [ "$allowdeletebranch" != "true" ]; then
-			echo "*** Deleting a tracking branch is not allowed in this repository" >&2
-			exit 1
-		fi
-		;;
-	*)
-		# Anything else (is there anything else?)
-		echo "*** Update hook: unknown type of update to ref $refname of type $newrev_type" >&2
-		exit 1
-		;;
-esac
-
-# --- Finished
-exit 0

pkg/mod/cache/vcs/1bb0f6aea2776dd67270c7cb096c102f9d61a1a91114cf9c8eff7a222a21173c/info/exclude

-# git ls-files --others --exclude-from=.git/info/exclude
-# Lines that start with '#' are comments.
-# For a project mostly in C, the following would be a good set of
-# exclude patterns (uncomment them if you want to use them):
-# *.[oa]
-# *~

pkg/mod/cache/vcs/1bb0f6aea2776dd67270c7cb096c102f9d61a1a91114cf9c8eff7a222a21173c/objects/1d/e939f76e5f4f13287e040998ed792d34cdeb2d

-x
M gS	$d?PHhEO?3;}~@E}I9HJ4M
-@s}z12$(LJvVl	G(HT4"Ŝ)+o8iL:KP6RB(	hhG65@Rȼl >dZHQCK{9!vBr6^*^:ݍ`+M4
Hv; qioEZ^^MvO{_NC&*׫[rxr\|uUwlvōW"7!|)g!@O1h
\ No newline at end of file

pkg/mod/cache/vcs/1bb0f6aea2776dd67270c7cb096c102f9d61a1a91114cf9c8eff7a222a21173c/objects/66/1b9b0e18c5541fbd0f172d3da11c52bdc4ece8

-x
Sn0Y_1M/aN
-ОibPJ"Jޥnάo&Clun2øuɄVhM|XI?GXyZ?k#utJmt mIVєP 4۱[M*(~9?<Q)F
(#V;48Y҈zOW6b#Y"`~a۠i"v46߂O$,slRK:#&LwH\s֭[:貹8*ق;iq}<10
G1<e
owk{Aii*[Π:]u4^Ն<PyB89~r͟#E5^҃AVlJ
-|pԨݠFLG5詊"5Cipý=uk4Y
\ No newline at end of file

pkg/mod/cache/vcs/1bb0f6aea2776dd67270c7cb096c102f9d61a1a91114cf9c8eff7a222a21173c/objects/bc/cc948892d8461e1763cb79b3e2de05e4a1753c

-x
Vo6޳kčZ`,HN
-d/$db4鑔;;R%M]tzH,wRSFЅsQ,tN^ZTRd %˻$TzpcvCJìqu.M9J8R@lzqdFbm
\
w}u>OG
-`v03tNCA!!>zP2^qGa}e
 qxAЪZazGRXGW*XGG}p+$T&F8M?:*%]ͥHo?. zUsҍ}PŎ.-4Fʈuxq8B΄rhT"yk>%Q]*+
-E9¸\h:k-1|uQh2z?nFSxL+ [$2uȳ(	%G`/XNITRjBw_zNqփC)-:u"K_Z)
v3҄J%h2~(yd
k1G>za!7z	y2$n-TZK	92Ǵ,
-5m;fn(AGP?=bIii[Qe=\2a&+sfRW)Z_Į}S
-?=mqqT9$^^%WV}n6:MD,\}H;\Piw<e/_BaGt!FHE{ѐ*'Pu&4TwBӂr+_q5N2ub&U{ҪW%Ƨ޻l
-a_!Ŭihl˕ݑ|bQO)R sAr^ꁗ EjET@j=	1JY9ímaՎ[IL!'W}
-~:u2inyK*db8ބTƂ6UlDtB=җ]6No9
\ No newline at end of file