添加配置文件

go.mod

@@ -4,6 +4,7 @@ require (
 	github.com/DataDog/zstd v1.3.4 // indirect
 	github.com/Shopify/sarama v1.20.0
 	github.com/Shopify/toxiproxy v2.1.3+incompatible // indirect
+	github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619
 	github.com/bsm/sarama-cluster v2.1.15+incompatible
 	github.com/eapache/go-resiliency v1.1.0 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
@@ -15,6 +16,7 @@ require (
 	github.com/onsi/gomega v1.4.3 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a // indirect
 	github.com/robfig/cron v0.0.0-20180505203441-b41be1df6967
+	github.com/timest/env v0.0.0-20180717050204-5fce78d35255
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
 	gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
 )

go.sum

@@ -18,6 +18,8 @@ github.com/Shopify/sarama v1.20.0 h1:wAMHhl1lGRlobeoV/xOKpbqD2OQsOvY4A/vIOGroIe8
 github.com/Shopify/sarama v1.20.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
 github.com/Shopify/toxiproxy v2.1.3+incompatible h1:awiJqUYH4q4OmoBiRccJykjd7B+w0loJi2keSna4X/M=
 github.com/Shopify/toxiproxy v2.1.3+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
+github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619 h1:6X8iB881g299aNEv6KXrcjL31iLOH7yA6NXoQX+MbDg=
+github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
 github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -102,6 +104,7 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/goreleaser/goreleaser v0.94.0/go.mod h1:OjbYR2NhOI6AEUWCowMSBzo9nP1aRif3sYtx+rhp+Zo=
 github.com/goreleaser/nfpm v0.9.7/go.mod h1:F2yzin6cBAL9gb+mSiReuXdsfTrOQwDMsuSpULof+y4=
@@ -152,6 +155,7 @@ github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJS
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jsternberg/zap-logfmt v1.2.0/go.mod h1:kz+1CUmCutPWABnNkOu9hOHKdT2q3TDYCcsFy9hpqb0=
+github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE=
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0=
@@ -209,6 +213,7 @@ github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.0.0-20171201122222-661e31bf844d/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@@ -226,7 +231,9 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg
 github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a h1:JSvGDIbmil4Ui/dDdFBExb7/cmkNjyX5F97oglmvCDo=
 github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
@@ -241,9 +248,12 @@ github.com/stevvooe/resumable v0.0.0-20180830230917-22b14a53ba50/go.mod h1:1pdIZ
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/tcnksm/go-input v0.0.0-20180404061846-548a7d7a8ee8/go.mod h1:IlWNj9v/13q7xFbaK4mbyzMNwrZLaWSHx/aibKIZuIg=
 github.com/testcontainers/testcontainer-go v0.0.0-20181115231424-8e868ca12c0f/go.mod h1:SrG3IY071gtmZJjGbKO+POJ57a/MMESerYNWt6ZRtKs=
+github.com/timest/env v0.0.0-20180717050204-5fce78d35255 h1:dDesTEcEuYIc8RNMm5dWWxamlr7m4x4MhCryWDBVaik=
+github.com/timest/env v0.0.0-20180717050204-5fce78d35255/go.mod h1:PKIuG7CPWS0/fq1dphTqvt0RZm0w4fp4eZxBQDJhJxs=
 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/tylerb/graceful v1.2.15/go.mod h1:LPYTbOYmUTdabwRt0TGhLllQ0MUNbs0Y5q1WXJOI9II=

main.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"git.quantgroup.cn/DevOps/enoch/service"
 	"git.quantgroup.cn/DevOps/enoch/service/conf"
+	"git.quantgroup.cn/DevOps/enoch/service/data"
 	"git.quantgroup.cn/DevOps/enoch/service/job"
 	"log"
 	"net/http"
@@ -13,10 +14,10 @@ import (
 )
 
 func main() {
-
+	conf.Init()
 	var port string
 	var quartz bool
-	flag.StringVar(&port, "port", "9091", "端口")
+	flag.StringVar(&port, "port", conf.GlobalConfig.Port, "端口")
 	flag.BoolVar(&quartz, "quartz", false, "quartz")
 	flag.Parse()
 
@@ -32,10 +33,10 @@ func main() {
 	defer logger.Println("项目结束")
 
 	//初始化redis连接池
-	conf.RedisPoolInit()
+	data.RedisPoolInit()
 
-	go service.AgentClusterConsumer()
-	go service.ClusterConsumer()
+	//go service.AgentClusterConsumer()
+	//go service.ClusterConsumer()
 	intPort, _ := strconv.Atoi(port)
 
 	if quartz {

service/conf/config.go

+package conf
+
+import "time"
+
+type Config struct {
+	AppName string `json:"app_name"`
+	Port string `json:"port"`
+	Redis Redis `json:"redis"`
+	Kafka Kafka `json:"kafka"`
+	InfluxDb InfluxDb `json:"influx_db"`
+}
+
+type Redis struct {
+	Host string `json:"host"`
+	Port string `json:"port"`
+	MaxIdle int `json:"max_idle"`
+	MaxActive int `json:"max_active"`
+	IdleTimeout time.Duration `json:"idle_timeout"`
+}
+
+type Kafka struct {
+	Brokers []string `json:"brokers"`
+}
+
+type InfluxDb struct {
+	 Host string `json:"host"`
+	 Port string `json:"port"`
+}
+

service/conf/dev/conf.json

+{
+  "app_name": "enoch",
+  "port": "9091",
+  "redis": {
+    "host": "http://localhost",
+    "port": "3007",
+    "max_idle": 0,
+    "max_active": 5000,
+    "idle_timeout": 120
+  },
+  "kafka": {
+    "brokers": ["192.168.4.100:15091", "192.168.4.100:15092", "192.168.4.100:15093"]
+  },
+  "influx_db": {
+    "host": "http://localhost",
+    "port": "9001"
+  }
+}

service/conf/load_conf.go

+package conf
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+)
+
+
+var GlobalConfig = Config{}
+
+func Init() {
+	denv := os.Getenv("Denv")
+	if "" == denv {
+		denv = "dev"
+	}
+	var filename = "service/conf/" + denv + "/conf.json"
+	bytes, err := ioutil.ReadFile(filename)
+	if nil != err {
+		fmt.Println("启动报错：未找到对应的环境变量配置。env: " + filename)
+	}
+	err = json.Unmarshal(bytes, &GlobalConfig)
+	if nil != err {
+		fmt.Println(err)
+	}
+}

service/conf/pro/conf.json

+{
+  "app_name": "enoch",
+  "port": "9091",
+  "redis": {
+    "host": "172.30.12.2",
+    "port": "6379",
+    "max_idle": 0,
+    "max_active": 5000,
+    "idle_timeout": 120
+  },
+  "kafka": {
+    "brokers": ["172.30.12.19:9092", "172.30.12.20:9092", "172.30.12.21:9092"]
+  },
+  "influx_db": {
+    "host": "http://172.20.6.33",
+    "port": "8086"
+  }
+}

service/conf/influxdb_conf.go → service/data/influxdb_conf.go

-package conf
+package data
 
 import (
+	"git.quantgroup.cn/DevOps/enoch/service/conf"
 	"github.com/influxdata/influxdb/client/v2"
 	"log"
 )
 
 func NewClient() client.Client {
-
-	conf := client.HTTPConfig{
-		Addr: "http://172.20.6.33:8086",
+	var httpConfig = client.HTTPConfig{
+		Addr: conf.GlobalConfig.InfluxDb.Host + conf.GlobalConfig.InfluxDb.Port,
 	}
-
-	con, err := client.NewHTTPClient(conf)
+	con, err := client.NewHTTPClient(httpConfig)
 
 	if err != nil {
 		log.Fatal(err)

service/conf/redis_conf.go → service/data/redis_conf.go

-package conf
+package data
 
 import (
 	"flag"
+	"git.quantgroup.cn/DevOps/enoch/service/conf"
 	"github.com/gomodule/redigo/redis"
 	"time"
 )
 
 func newPool(server, password string) *redis.Pool {
-
+	var redisConf = conf.GlobalConfig.Redis
 	return &redis.Pool{
-		MaxIdle:     0,
-		MaxActive:   5000,
-		IdleTimeout: 120 * time.Second,
+		MaxIdle:     redisConf.MaxIdle,
+		MaxActive:   redisConf.MaxActive,
+		IdleTimeout: redisConf.IdleTimeout * time.Second,
 		Dial: func() (redis.Conn, error) {
 			conn, err := redis.Dial("tcp", server)
 			if err != nil {

service/kafka_agent_sarama.go

@@ -3,6 +3,7 @@ package service
 import (
 	"context"
 	"fmt"
+	"git.quantgroup.cn/DevOps/enoch/service/conf"
 	"github.com/Shopify/sarama"
 	"github.com/bsm/sarama-cluster"
 	"log"
@@ -12,8 +13,7 @@ import (
 )
 
 var (
-	//agentBrokers    = []string{"192.168.4.100:15091", "192.168.4.100:15092", "192.168.4.100:15093"}
-	agentBrokers    = []string{"172.30.12.19:9092", "172.30.12.20:9092", "172.30.12.21:9092"}
+	agentBrokers    = conf.GlobalConfig.Kafka.Brokers
 	agentKafkaTopic = "quantGroup.tech.enoch.pro"
 	agentKafkaGroup = "quantGroup-enoch-agent"
 )

service/kafka_sarama.go

@@ -3,6 +3,7 @@ package service
 import (
 	"context"
 	"fmt"
+	"git.quantgroup.cn/DevOps/enoch/service/conf"
 	"github.com/Shopify/sarama"
 	"github.com/bsm/sarama-cluster"
 	"log"
@@ -12,7 +13,7 @@ import (
 )
 
 var (
-	brokers    = []string{"172.30.12.19:9092", "172.30.12.20:9092", "172.30.12.21:9092"}
+	brokers    = conf.GlobalConfig.Kafka.Brokers
 	kafkaTopic = "quantGroup.tech.brave.pro"
 	kafkaGroup = "enoch-group"
 )

service/msg_process.go

@@ -3,7 +3,7 @@ package service
 import (
 	"encoding/json"
 	"fmt"
-	"git.quantgroup.cn/DevOps/enoch/service/conf"
+	"git.quantgroup.cn/DevOps/enoch/service/data"
 	"github.com/gomodule/redigo/redis"
 	"github.com/influxdata/influxdb/client/v2"
 	"log"
@@ -81,7 +81,7 @@ func msgInfluxProcess(traceMsgs []TraceMsg) {
 }
 
 func batchWrite(pointArray []*client.Point) {
-	c := conf.NewClient()
+	c := data.NewClient()
 	defer func() { _ = c.Close() }()
 
 	points, err := client.NewBatchPoints(client.BatchPointsConfig{
@@ -101,7 +101,7 @@ func batchWrite(pointArray []*client.Point) {
 }
 
 func msgRedisProcess(traceMsg []TraceMsg) {
-	conn := conf.Pool.Get()
+	conn := data.Pool.Get()
 	defer func() { _ = conn.Close() }()
 	for _, v := range traceMsg {
 		if v.Kind != "SERVER" {
@@ -171,7 +171,7 @@ func casScoreAdd(conn redis.Conn, durationKey string, member string, duration in
 
 func Duration(day string, fun func(sysName string, durations map[string]string)) {
 
-	conn := conf.Pool.Get()
+	conn := data.Pool.Get()
 	defer func() { _ = conn.Close() }()
 	matchPattern := "duration*" + day
 	reply, err := redis.Values(conn.Do("scan", 0, "match", matchPattern, "count", 10000))
@@ -192,7 +192,7 @@ func Duration(day string, fun func(sysName string, durations map[string]string))
 }
 
 func Counter(day string, fun func(sysName string, durations map[string]string)) {
-	conn := conf.Pool.Get()
+	conn := data.Pool.Get()
 	defer func() { _ = conn.Close() }()
 	matchPattern := "counter*" + day
 	reply, err := redis.Values(conn.Do("scan", 0, "match", matchPattern, "count", 10000))

service/test.go

@@ -2,11 +2,11 @@ package service
 
 import (
 	"fmt"
-	"git.quantgroup.cn/DevOps/enoch/service/conf"
+	"git.quantgroup.cn/DevOps/enoch/service/data"
 )
 
 func Test() {
-	conn := conf.Pool.Get()
+	conn := data.Pool.Get()
 	defer conn.Close()
 
 	_, err := conn.Do("hincrby", "key", "field", 1)