Merge branch 'master' of http://git.quantgroup.cn/QA/qa-deploy-utils

db-utils/config/db_config

-database_list=mall xyqb account_center cash_loan_flow clotho contract display dujiangyan financial_system financial_system_api fund_manage gyxd mall_settlement merchant notify offline payment_center qinqiong quartz reconciliation redpacket risk_control score silk_road spider_center traffic_distribution urge_dispatcher user_credit xyqb_app xyqb_btzt xyqb_user
-source_db_pwd=Uiwb@o^fR&nHOa5t
\ No newline at end of file
+database_list=mall xyqb account_center cash_loan_flow clotho contract display dujiangyan financial_system financial_system_api fund_manage gyxd mall_settlement merchant notify offline payment_center qinqiong quartz reconciliation redpacket risk_control score silk_road spider_center traffic_distribution urge_dispatcher user_credit xyqb_app xyqb_btzt xyqb_user
\ No newline at end of file

db-utils/daily_db_backup.sh

 #!/bin/bash
 set -u
+source ~/.profile
 work_dir=/home/qa-deploy-utils/db-utils
 sync_config_folder=/home/db_sync_configs
 db_backup_folder=/home/daily_db_backup
 
 source_db_host=$1
-source_db_user=$2
-source_db_pwd=`grep source_db_pwd $work_dir/config/db_config | cut -d "=" -f 2`
+source_db_user=`curl -s http://192.168.4.3:10088/config_server/get_db_config/xyqb/user`
+source_db_pwd=`curl -s http://192.168.4.3:10088/config_server/get_db_config/xyqb/pwd`
 
-ip_value=$4
+target_db_host=192.168.6.2
+target_db_port=3306
 
 target_db_user=$5
 target_db_pwd=$6
@@ -25,13 +27,22 @@ function db_schema_sync()
 	echo "mkdir -p $db_backup_sub_folders"
 	mkdir -p $db_backup_sub_folders
 	
-	# echo "--- Step 1: generate_config_file: $config_file_name ---"
+	echo "--- Step 1: generate_config_file: $config_file_name ---"
 	# echo "source infos: $source_db_host $source_db_user"
 	# echo "target infos: $target_db_host $target_db_user"
 	# echo "database_name is : $sync_db_name"
 
-	echo "sh $work_dir/db_gen_config.sh $source_db_host $source_db_user $source_db_pwd 192.168.4.12 $target_db_user $target_db_pwd $database $config_file_name $db_backup_sub_folders false 3306"
-	sh $work_dir/db_gen_config.sh $source_db_host $source_db_user $source_db_pwd 192.168.4.12 $target_db_user $target_db_pwd $database $config_file_name $db_backup_sub_folders false 3306
+	echo "sh $work_dir/db_gen_config.sh $source_db_host $source_db_user $source_db_pwd $target_db_host $target_db_user $target_db_pwd $database $config_file_name $db_backup_sub_folders false 3306"
+	sh $work_dir/db_gen_config.sh $source_db_host $source_db_user $source_db_pwd $target_db_host $target_db_user $target_db_pwd $database $config_file_name $db_backup_sub_folders false 3306
+
+	echo "--- Step 3: sync schema: $database $target_db_host ---"
+	#sync_cmd="mss -conf ${config_file_name} -sync"
+	echo "mss -conf ${config_file_name} -sync > /dev/null"
+	mss -conf ${config_file_name} -sync > /dev/null
+	echo "mysql-schema-sync result:$?"
+
+	echo "--- Step 4: import basic data from backup file: $database $db_backup_sub_folders ---"
+	sh $work_dir/db_import.sh $database $target_db_host $db_backup_sub_folders $target_db_port
 }
 
 database_list=`grep database_list $work_dir/config/db_config | cut -d "=" -f 2`

db-utils/db_import.sh

@@ -10,8 +10,7 @@
 database_name=$1
 mysql_host=$2
 db_sub_backup_folder=$3
-delete_business_data=$4
-target_db_port=$5
+target_db_port=$4
 
 echo "cd $db_sub_backup_folder"
 cd $db_sub_backup_folder

db-utils/db_replace_ip.sh

@@ -99,21 +99,3 @@ else
 		#done
 	fi
 fi
-
-# sed -i "s/\/m.xyqb.com/\/xyqb-ui-$ip_last.q-gp.com/g" $entry
-# sed -i  "s/\/bt.xyqb.com/\/${server_host}:7050/g" $entry
-# sed -i "s/\/api.xyqb.com/\/${server_host}:7003/g" $entry
-# sed -i "s/\/r.xyqb.com/\/${server_host}:7004/g" $entry
-# sed -i "s/\/rapi.xyqb.com/\/${server_host}:7005/g" $entry
-# sed -i "s/\/payapi.xyqb.com/\/${server_host}:7006/g" $entry
-# sed -i "s/\/pay.xyqb.com/\/${server_host}:7007/g" $entry
-# sed -i "s/\/off.xyqb.com/\/${server_host}:7008/g" $entry
-# sed -i "s/\/offapi.xyqb.com/\/${server_host}:7010/g" $entry
-# sed -i "s/\/financial.xyqb.com/\/${server_host}:7011/g" $entry
-# sed -i "s/\/fsapi.xyqb.com/\/${server_host}:7012/g" $entry
-# sed -i "s/\/auth.xyqb.com/\/${server_host}:7013/g" $entry
-# sed -i "s/\/qapi.xyqb.com/\/${server_host}:7014/g" $entry
-# sed -i "s/\/mallapi.xyqb.com/\/${server_host}:7019/g" $entry
-# sed -i "s/\/scapi.xyqb.com/\/${server_host}:7021/g" $entry
-# sed -i "s/\/gyxdapi.xyqb.com/\/${server_host}:7029/g" $entry
-# sed -i "s/\/btapi.xyqb.com/\/${server_host}:7025/g" $entry

db-utils/db_schema_sync.sh

@@ -30,8 +30,8 @@ sync_config_folder=/home/db_sync_configs
 db_backup_folder=/home/db_backups
 
 source_db_host=$1
-source_db_user=$2
-source_db_pwd=`grep source_db_pwd $work_dir/config/db_config | cut -d "=" -f 2`
+source_db_user=`curl -s http://192.168.4.3:10088/config_server/get_db_config/xyqb/user`
+source_db_pwd=`curl -s http://192.168.4.3:10088/config_server/get_db_config/xyqb/pwd`
 
 ip_value=$4
 
@@ -106,7 +106,7 @@ function db_schema_sync()
 	echo "mysql-schema-sync result:$?"
 
 	echo "--- Step 4: import basic data from backup file: $database $db_backup_sub_folders ---"
-	sh $work_dir/db_import.sh $database $target_db_host $db_backup_sub_folders $delete_business_data $target_db_port
+	sh $work_dir/db_import.sh $database $target_db_host $db_backup_sub_folders $target_db_port
 	# if [ $? != 0 ];then echo "db_import.sh $database error";exit -1; fi
 	
 }
@@ -118,7 +118,8 @@ function run_sql_file()
 	sh $work_dir/db_execute_sql.sh $target_db_host $sync_db_name $target_db_port
 }
 
-database_list=`grep database_list $work_dir/config/db_config | cut -d "=" -f 2`
+# database_list=`grep database_list $work_dir/config/db_config | cut -d "=" -f 2`
+database_list=`curl -s http://192.168.4.3:10088/config_server/get_db_config/all/name`
 
 if [[ "$sync_db_name" = "all" ]];then
 	echo "sync all database schema"

db-utils/dump.sh

+#!/bin/bash
+dbs=`curl http://192.168.4.3:10088/config_server/get_db_config/all/name`
+for db in $dbs
+do
+	sql_command=`curl http://192.168.4.3:10088/config_server/get_db_dump_command/${db}/opt`
+  table_opt="mysqldump --opt $sql_command > /tmp/${db}_opt.sql"
+	echo "$table_opt"
+	eval $table_opt
+	mysql -h 192.168.6.2 -P 3306 -uqa -pqatest db < /tmp/${db}_opt.sql
+	
+		
+	sql_tables=`curl http://192.168.4.3:10088/config_server/get_db_dump_command/${db}/table`
+	if [[ sql_tables == '' ]];then
+		echo "这个数据库没有基础数据需要同步"
+	else
+		table_data="mysqldump $sql_command --skip-lock-tables --tables  $sql_tables --skip-add-locks --no-create-info --complete-insert > /tmp/${db}_db.sql"
+		echo "$table_data"
+		eval $table_data
+		
+		mysql -h 192.168.6.2 -P 3306 -uqa -pqatest db < /tmp/${db}_db.sql
+	fi
+done
\ No newline at end of file

qa_shell_script/config_server/config/db.js

+var db = function( host, port, user, pwd, sys_table_list){
+	this.sys_table_list = sys_table_list;
+	this.host = host || '192.168.4.39';
+	this.port = port || '3306';
+	this.user = user || 'xyqb_query';
+	this.pwd = pwd || 'Uiwb@o^fR&nHOa5t';
+}
+module.exports = {
+	db_config:function(){
+		this.dbs = {
+		mall : new db('192.168.3.19','','','','activity_3c_product ad_detail admin_banner advertisement_type album app_channel app_channel_ad app_name app_name_ad category category_banner category_nature commodity_slogan commodity_slogan_product configuration edx_city faster_entrance faster_entrance_image goods_after_sale_config goods_after_sale_step home_config home_page hot_category jd_address jd_after_sale_config jd_brand jd_category jd_product_sku_image jd_property jd_sku jd_sku_detail_image jd_sku_specs life_faster_entrance life_service mall_category mall_config merchant nature_value new_3c_product product product_category product_nature product_nature_value_bak product_sku product_sku_image product_specification product_spu recommend_brand recommend_product sale_product_detail sale_product_theme spu_image supplier tab_navigate title_banner loan_config virtual_brand virtual_category virtual_product_discount virtual_product_hot virtual_product_sku virtual_product_spu virtual_product_template'),
+
+		xyqb : new db('','','','','baitiao_merchant baitiao_secure_key channel funding_contract funding_corp loan_product merchant_channel payeco_return_url_info red_packet t_phonenum zmxy_return_url configuration secure_proxy funding_corp_config funding_info_map baitiao_preferential_rule funding_corp_policy'),
+
+		account_center : new db('192.168.4.116','6001','','',''),
+
+		cash_loan_flow : new db('192.168.4.80','8001','','','order_callback callback_configuration channel_configuration filter_configuration'),
+
+		clotho : new db('192.168.3.19','3308','','',''),
+
+		contract : new db('','3311','','','contract_template_new'),
+
+		display : new db('192.168.3.19','3307','','','discovery_config'),
+
+		dujiangyan : new db('','3310','','','external_coopr'),
+
+		financial_system : new db('','6603','','','collection_user sys_dictionary sys_status_edge sys_status_vertice'),
+
+		financial_system_api : new db('','6603','','','mailinfo'),
+
+		fund_manage : new db('','3310','','','business_info email_info fund_info merchant'),
+
+		gyxd : new db('192.168.4.116','6602','','','administrative_districts templet user'),
+
+		mall_settlement : new db('192.168.4.253','3315','','','role role_user user'),
+
+		merchant : new db('','3314','','','authority product_nature role role_authority role_supplier role_user user'),
+		
+		notify : new db('192.168.4.6','7306','','','merchant merchant_config'),
+
+		offline : new db('','3311','','','goods_detail merchant operator_user'),
+
+		payment_center : new db('','3308','','','card_switcher merchant merchant_config pay_approach_config merchant_pay_approach_relation merchant_bank_config quota'),
+
+		qinqiong : new db('','3310','','','secure_key secure_proxy'),
+
+		quartz : new db('','3310','','','certain_time_job'),
+	
+		reconciliation : new db('192.168.3.19','3308','','',''),
+
+		redpacket : new db('192.168.4.116','6001','','','tb_activity_channel tb_dict_channel tb_dict_out_channel'),
+	
+		risk_control : new db('','3309','','','company product_strategy_dispatcher quantify_product red_packet_info user'),
+	
+		score : new db('192.168.4.249','3322','','',''),
+	
+		silk_road : new db('192.168.4.80','9999','','',''),
+	
+		spider_center : new db('192.168.4.253','3317','','','merchant merchant_config'),
+
+		traffic_distribution : new db('192.168.3.36','3313','','','merchant merchant_carousel merchant_credit merchant_detail merchant_section merchant_sort ppdai_exclude_channel t_module t_module_role t_role t_user t_user_role traffic_channel_banner_ids traffic_channel_ids traffic_channel_sort traffic_infos'),
+
+		urge_dispatcher : new db('192.168.4.116','9901','','',''),
+
+		user_credit : new db('','9001','','',''),
+
+		xyqb_app : new db('','3307','','','activities app_banners app_config app_versions banners blanknote_entries credit_questions hot_games loan_banners loan_questions migrations sharings widgets js_patch upgrade_version'),
+
+		xyqb_btzt : new db('','9002','','','zt_channel zt_fund zt_fund_channel zt_fund_credit zt_fund_term'),
+
+		xyqb_user : new db('','3313','','','merchant_config merchant'),	
+		}
+		return this.dbs
+	},
+	get_mysql_dump_opt:function(key){
+		let item = this.dbs[key];
+		return `--host=${item.host} --port=${item.port} --user=${item.user} --password='${item.pwd}' -d ${key} `		
+	},
+	get_mysql_dump_table:function(key){
+		let item = this.dbs[key];
+		return `${item.sys_table_list}`
+	},
+	get_target_mysql_command:function(key){
+		
+	}
+}
\ No newline at end of file

qa_shell_script/config_server/get_project_config_server.js

 var express = require('express');
 var cproc = require('child_process');
 var bodyParser = require('body-parser');
+var  db_config = require('./config/db');
+// return
 var execSync = cproc.execSync;
 var options = {
   inflate: true,
@@ -38,7 +40,7 @@ var attrComsArr = [];
 for(let i of attrComs){
 	attrComsArr.push(attrs[i]);
 }
-// ------------UTILS----------
+// ------------UTILS------------
 function parseStr2Json(str){
 	var a = {};
 	var strs = str.split(" ");
@@ -70,8 +72,41 @@ function parseJson2Str(json){
 	return a.join(" ")
 }
 // ------------UTILS----------
-
+// DB 维护数据
+let db_conf = db_config.db_config();
+let db_conf_keys = Object.keys(db_conf);
+// 
+admin.get('/get_db_config/:name/:attr', function (req, res) {
+	let str = ""
+	if(req.params.name == "all"){
+		if(req.params.attr == "name"){
+			res.send(db_conf_keys.join(" "));
+		}
+	}else{
+		if(req.params.attr != "all"){
+				res.send(db_conf[req.params.name][req.params.attr])
+		}else{
+			let item = db_conf[req.params.name]
+			res.send(`${item.host};${item.port};${item.user};${item.pwd};${item.sys_table_list};`)
+		}
+	}
+});
+admin.get('/get_db_dump_command/:name/:opt', function (req, res) {
+	if(req.params.opt!="opt"){
+			res.send(db_config.get_mysql_dump_table(req.params.name));
+	}else{
+			res.send(db_config.get_mysql_dump_opt(req.params.name));
+	}
+});
+admin.get('/get_db_config/refersh',function(){
+ 	res.setHeader("Content-Type","application/json");
+	db_conf = db_config.db_config();
+	db_conf_keys = Object.keys(db_conf);
+	res.send({code:"OK"})
+})
 // 获取节点命令
+
+
 admin.get('/get_node_command/:name', function (req, res) {
  	res.setHeader("Content-Type","application/json");
 	var str = execSync(`node ./get_project_config.js  -name ${req.params.name} -attr config_file/command/gitPathHead`)
@@ -219,6 +254,35 @@ admin.get('/get_proxy', function (req, res) {
 	res.send({namespace:String(str)});
 });
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 app.use('/config_server', admin)
 console.log("http://192.168.4.3:10088/config_server/get_node_command") 
 app.listen(10088)

qa_shell_script/ngrok/make_frp_ini_V2.sh

@@ -28,7 +28,8 @@ function createNginxHttpAndSSLConf()
 	echo "    listen 80;" >> $file_name
 	echo "    server_name  $cus_domain.q-gp.com;" >> $file_name
 	echo "    	underscores_in_headers on;" >> $file_name
-
+	echo "    proxy_set_header  REMOTE_ADDR        \$remote_addr; " >> $file_name
+	echo "    proxy_set_header  REMOTE_PORT        \$remote_port; " >> $file_name
 	echo "    proxy_set_header Host  \$host;"  >> $file_name
 	echo "    proxy_set_header QT_TOKEN  \$http_QT_TOKEN;"  >> $file_name
 
@@ -51,6 +52,8 @@ function createNginxHttpAndSSLConf()
 	echo "" >> $file_name
 	echo "    ssl_ciphers  HIGH:!aNULL:!MD5;" >> $file_name
 	echo "    ssl_prefer_server_ciphers  on;" >> $file_name
+	echo "    proxy_set_header  REMOTE_ADDR        \$remote_addr; " >> $file_name
+	echo "    proxy_set_header  REMOTE_PORT        \$remote_port; " >> $file_name
 	echo "    proxy_set_header Host  \$host;"  >> $file_name
 	echo "" >> $file_name
 	echo "    location / {" >> $file_name
@@ -76,8 +79,11 @@ createK8sNginxConf(){
 	echo "    listen 80 default; " >> $file_name
 	echo "    server_name _; " >> $file_name
 	echo "    underscores_in_headers on; " >> $file_name
-	echo "    proxy_set_header Host  \$host; " >> $file_name
 	echo "    proxy_set_header QT_TOKEN  \$http_QT_TOKEN; " >> $file_name
+	
+	echo "    proxy_set_header Host  \$host; " >> $file_name
+	echo "    proxy_set_header  REMOTE_ADDR        \$remote_addr; " >> $file_name
+	echo "    proxy_set_header  REMOTE_PORT        \$remote_port; " >> $file_name
 	echo "		 " >> $file_name
 	echo "    set \$ip \"\"; " >> $file_name
 	echo "    location / { " >> $file_name
@@ -97,6 +103,8 @@ createK8sNginxConf(){
 	echo "    ssl_ciphers  HIGH:!aNULL:!MD5; " >> $file_name
 	echo "    ssl_prefer_server_ciphers  on; " >> $file_name
 	echo "    proxy_set_header Host  \$host; " >> $file_name
+	echo "    proxy_set_header  REMOTE_ADDR        \$remote_addr; " >> $file_name
+	echo "    proxy_set_header  REMOTE_PORT        \$remote_port; " >> $file_name
 	echo " " >> $file_name
 	echo "		set \$ip \"\"; " >> $file_name
 	echo "    location / { " >> $file_name