Skip to content

B
bigdata
Commit 1be075e7 authored by 桂秋月's avatar 桂秋月
Browse files
Options

优化部分方法

parent 5f524fac
Show whitespace changes
Inline Side-by-side
Showing with 5647 additions and 52 deletions
bigData.iml
View file @ 1be075e7
......@@ -2,9 +2,7 @@
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$" isTestSource="false" />
</content>
<content url="file://$MODULE_DIR$" />
<orderEntry type="jdk" jdkName="Python 3.7" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
......
dataFile/out.txt 0 → 100644
View file @ 1be075e7
This source diff could not be displayed because it is too large. You can view the blob instead.
databaseConn/__init__.py
View file @ 1be075e7
......@@ -9,6 +9,13 @@ timenow = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
#sys.path.append(os.path.abspath(__file__))
channel = [214, 217, 1]
mysql_info={
"tobonline":{
"name":"qiuyue.gui",
"pwd":"bJ3q6DqrbUxL0YnL",
"host":"10.250.2.212",
"port":"13312"
},
"119":{
#'root', 'quantGroup#com', '172.24.17.119', 15307
"name":"root",
......@@ -91,3 +98,51 @@ mysql_info={
mongo_info={
}
select_delivery_sql="""
select a.sn, time(a.create_date) as time,
a.operater_id,
a.op_name,
case when a.after_delivery_status=0 then '初始化'
when a.after_delivery_status=10 then '待收餐员收餐'
when a.after_delivery_status=11 then '取餐员已到店'
when a.after_delivery_status=20 then '待配送员接单'
when a.after_delivery_status=21 then '配送员已接单，配送中'
when a.after_delivery_status=30 then '配送员已接单，配送中'
when a.after_delivery_status=40 then '已送达自提点，待用户取餐'
when a.after_delivery_status=41 then '订单完成，用户已取餐'
when a.after_delivery_status=99 then '用户已取消订单'
when a.after_delivery_status=25 then '配送员已收餐已接单，配送中'
when a.after_delivery_status=27 then '配送员已收餐已到店'
else '其他-非已知状态' END as '当前状态',
case when a.role_type=10 then '收餐员'
when a.role_type=20 then '配送员'
when a.role_type=30 then '分拣员'
else '系统' END as 'role'
from delivery.wd_delivery_order_flow a
join (select * from delivery.wd_delivery_order where user_phone ='{}' order by create_date desc limit 1) b on a.delivery_order_id=b.id
"""
select_delivery_order_sql="""
select sn,
op_name,create_date,
case when after_delivery_status=0 then '初始化'
when after_delivery_status=10 then '待收餐员收餐'
when after_delivery_status=11 then '取餐员已到店'
when after_delivery_status=20 then '待配送员接单'
when after_delivery_status=21 then '配送员已接单，配送中'
when after_delivery_status=30 then '配送员已接单，配送中'
when after_delivery_status=40 then '已送达自提点，待用户取餐'
when after_delivery_status=41 then '订单完成，用户已取餐'
when after_delivery_status=99 then '用户已取消订单'
when after_delivery_status=25 then '配送员已接单，配送中'
when after_delivery_status=27 then '配送员已到店'
else '其他-非已知状态' END as '当前状态',
case when role_type=10 then '收餐员'
when role_type=20 then '配送员'
when role_type=30 then '分拣员'
else '系统' END as 'role'
from delivery.wd_delivery_order_flow where sn in {}
and create_date>=DATE(NOW())
"""
\ No newline at end of file
databaseConn/mysqlOperation.py
View file @ 1be075e7
......@@ -24,7 +24,7 @@ def mysqlInfo(namespace,dbtype='mysql'):
try:
resp=requests.get(url)
#print(resp.json(),url)
result['host']=jsonpath.jsonpath(resp.json(),'$..nodeIp')[0]
result['host']='middleware.liangkebang.net' or jsonpath.jsonpath(resp.json(),'$..nodeIp')[0]
result['port']=jsonpath.jsonpath(resp.json(),'$..nodePort')[0]
result['pwd']='qatest'
result['name']='qa'
......@@ -38,14 +38,14 @@ def mysql_universal(namespace):
mysql_info=mysqlInfo(namespace)
#print(mysql_info)
temp=create_engine(mysql_universal.format(**mysql_info))
#print("mysql连接信息:",temp)
print("mysql连接信息:",temp,mysql_info)
return temp
def execmysl(namespace,sql):
if 'select' not in sql.strip().lower() and 'alter' not in sql.strip().lower() and 'update' not in sql.lower():
raise Exception('在mysql看来不是有效的sql',sql)
try:
print('执行的sql如下：',sql)
#print('执行的sql如下：',sql)
df=pd.read_sql(sql,con=mysql_universal(namespace))
return df
except Exception as e:
......@@ -58,7 +58,7 @@ def concatSql(sql,**kwargs):
'''
:param sql:
:param kwargs:
:return: 只针对select的sql拼接，拼接后where是and|or的关系
:return: 只针对select的sql拼接，拼接后where是and的关系
'''
if not kwargs:
return sql
......@@ -84,5 +84,20 @@ def concatSql(sql,**kwargs):
return sql+' where '+temp[:-4]
if __name__ == '__main__':
print(mysqlInfo('saas4'))
#print(execmysl('yxm',"select 1 "))
\ No newline at end of file
from tools.publicFun import time
time="2021-10-12"
temp='202210270001500000'
temp='13683241439'
if len(temp)==11:
sql=select_delivery_sql.format(temp)
else:
t=temp.split(',')
sql=select_delivery_order_sql.format(tuple(t if len(t)>1 else t+['0']))
#print(sql)
#print(mysqlInfo('feature_aliyun'))
#print(execmysl('tobonline',sql))
#print(execmysl('tobonline',sql))
# sql="""
# SELECT * from xyqb_user.user where phone_no='17718523326'
# """
print(execmysl('yxm','select 1'))
databaseConn/redisOperation.py
View file @ 1be075e7
......@@ -49,7 +49,7 @@ def get_simple_redis_instance(namespace):
#reids_universal="reids://{name}:{pwd}@{host}:{port}"
from databaseConn.mysqlOperation import mysqlInfo
redis_info=mysqlInfo(namespace,dbtype='redis')
#print(redis_info)
print('redis==',redis_info)
pool = redis.ConnectionPool(host=redis_info.get('host'), port=redis_info.get('port'),decode_responses=True)
redis_instance = redis.Redis(connection_pool=pool)
return redis_instance
......@@ -132,13 +132,17 @@ if __name__ == '__main__':
redis_key="appstart:1d0cdee1-85af-4e5b-856e-4a2e453befb6-2"
redis_key="price-tracker:stock:9"
redis_key="nearline_recommend:real-time:user_feature:54ad92ae-d1f1-4dad-a001-38904248fba5"
redis_key="*yxm-05719a86-5684-401d-a006-a6127eabeabe*"
redis_key="user:session:yxm-f461d0e0-fa38-4354-93fa-a20c5e0b415a"
redis_key="user:session:yxm-0c6d8ea3-a7fc-4d58-aabb-cac9bb26bdf1*"
#redis_key="nearline_recommend:real-time:user_feature:*"
#redis_key="nearline_recommend:offline:feed_sort_sku_cid3_feature3"
#print(getRedisValue('yxm',redis_key,oper='delete1'))
#print(getRedisValue(redis_key='89',key=redis_key))
conn=redis_conn('89')
print(conn.type(redis_key))
#print(getRedisValue('89',key=redis_key))
# conn=redis_conn('89')
# print(conn.type(redis_key))
print(getRedisValue('yxm',key=redis_key))
print(1290743/(3600*24))
......
search/abSearch.py
View file @ 1be075e7
......@@ -2,7 +2,7 @@ import os
import random
import pandas as pd
from search import *
def requestsend(uuid,deviceid,searchContent,selectedActivitie,page=1,searchtype='txt',env='pro'):
def requestsend(uuid,deviceid,searchContent,selectedActivitie,page=1,searchtype='txt',env='test'):
"""
:param uuid: 用户uuid
:param deviceid: 设备号id
......@@ -18,7 +18,8 @@ def requestsend(uuid,deviceid,searchContent,selectedActivitie,page=1,searchtype=
if env=='online':
baseurl="http://caesar-gateway.q-gp.com"
elif env=="test":
baseurl="http://172.16.9.90:8091"
#baseurl="http://172.16.9.90:8091"
baseurl="https://caesar-gateway-bd.liangkebang.net"
else:
baseurl="http://open-search-engine.ec-{env}.qg-rc.net".format(env=env) if env=='test' \
else "http://open-search-engine.qg-{env}.qgwebservice.com".format(env=env)
......@@ -33,7 +34,7 @@ def requestsend(uuid,deviceid,searchContent,selectedActivitie,page=1,searchtype=
"userUuid": uuid,
"deviceId": deviceid,
"extraParams": {
"appVersion": "8.6.00",
"appVersion": "8.7.00",
"terminal": "ios"
},
##文本搜索
......@@ -239,24 +240,14 @@ def analysis(filename=None):
';耗时大于150ms的总数',len(fist1_1000_2),
';耗时大于200ms的总数',len(fist1_1000_3))
#return result
if __name__ == '__main__':
# t=hotWord('线上环境搜索热词.xlsx')['search_words'].apply(lambda x:str(x).strip())
# print(t)
#print(totalrun(isFirstPage=1))
isexec=0
isexec=1
ishotwordrand=0
if isexec:
allRun(1,isFirstPage=1,sheetname=3,ishotwordrand=ishotwordrand)
allRun(200,isFirstPage=1,sheetname=0,ishotwordrand=ishotwordrand)
else:
print(analysis())
# a=[['236473224274432', '7188063'],['236473224274432', '7188063']]
......
tools/__init__.py
View file @ 1be075e7
from databaseConn.mysqlOperation import *
from recommend.publicSql import *
from databaseConn import *
from dateutil.relativedelta import relativedelta
lkb_git_url="http://git.quantgroup.cn"
tjzm_git_url="https://gitlab.tjzimu.com"
git_token='zubux2fMyyp8s8Cys3T6'
......
tools/build_git.py
View file @ 1be075e7
......@@ -4,7 +4,7 @@ import pandas as pd
base_url = "http://jenkins.quantgroups.cn"
# java_url="/TKE/job/tke-java/"
jk = jenkins.Jenkins(base_url, username="qiuyue.gui", password="1234567890Yue@")
jk = jenkins.Jenkins(base_url, username="qiuyue.gui", password="24fG-fHPUwOc1mP")
header = {"Host": "jenkins.quantgroups.cn"}
......@@ -38,7 +38,7 @@ def buildJob(project, branch, env,urltype ,jobname="java",prefix_job='ACK'):
# jk.get_build_info()
# jk.build_job(jobname,branch,env)
#job_config = jk.get_job_config(jobname)
params = {"GIT_REPO": project, "DEPLOY": 'true', 'NAMESPACE': env, "BRANCH_NAME": branch}
params = {"GIT_REPO": project.lower(), "DEPLOY": 'true', 'NAMESPACE': env, "BRANCH_NAME": branch}
# branchs_url=requests.get("")
t = jk.build_job(jobname, params)
print("project({0}-{1}-{2}) build succ".format(project, branch, env), t)
......@@ -66,7 +66,7 @@ def get_project_branch(project,urltype):
#brand_url=git_url+"/api/v4/projects/{project_id}/repository/branches?per_page=100".format(project_id=project_info.get(project))
brand_url=git_url+"/api/v4/projects/{project_id}/repository/branches?per_page=100&page={num}".format(project_id=project_info.get(project),num=i)
temp=requests.get(brand_url,headers=git_headers)
#print(temp.json())
#print('分支数据---',brand_url,temp.json())
brands_name=jsonpath.jsonpath(temp.json(),'$[*].name')
if brands_name:
temp_branchs+=brands_name
......@@ -83,8 +83,12 @@ def main(project,type="",prefix_job='ACK'):
#="nearline-recommend-parent"
project_branch={
"kdsp":{
"branch":"feature-mix-test-yxm-202207201023",
"env":'yxm',
"branch":"feature-20220830-maxmember-label",
"env":'yxm2',
"jobname":"java"},
"kdsp-operation":{
"branch":"feature-20220830-maxmember-label",
"env":'yxm2',
"jobname":"java"},
"nearline-recommend-parent":{
......@@ -103,8 +107,8 @@ def main(project,type="",prefix_job='ACK'):
"jobname":"ui"
},
"new-op-optimized-ui":{
"branch":"feat/memberDayS3",
"env":'yxm',
"branch":"feat/maxMemberWhiteList",
"env":'yxm2',
"jobname":"ui"
},
"op-api":{
......@@ -123,7 +127,82 @@ def main(project,type="",prefix_job='ACK'):
"env":'toc',
"jobname":"java",
"urltype":"tjzm"
}
},
"marketModel-new":{
"branch":"feature-20221012-dev-config",
"env":'tob1',
"jobname":"java"
},
"cartModel-new":{
"branch":"feature-20221027-changeConfig",
"env":'tob1',
"jobname":"java"
},
"dataModel-new":{
"branch":"feature-20221012-dev-config",
"env":'tob1',
"jobname":"java"
},
"distributionModel-new":{
"branch":"master",
"env":'tob1',
"jobname":"java"
},
"wdshop-new":{
"branch":"feature-20220928-access-leaf",
"env":'saas3',
"jobname":"java"
},
"authModel-new":{
"branch":"feature-20221009-dev-config",
"env":'tob1',
"jobname":"java"
},
"wd-distribution":{
"branch":"master",
"env":'tob1',
"jobname":"java"
},
"elsModel":{
"branch":"feature-20220722-map2.0",
"env":'saas4',
"jobname":"java"
},
"openModel":{
"branch":"feature-20220929-arrivalTime",
"env":'saas4',
"jobname":"java"
},
"horkos-ui":{
"branch":"feature-20220922-addDeliveryType",
"env":'saas4',
"jobname":"ui"
}, "wd-delivery":{
"branch":"master",
"env":'saas3',
"jobname":"java"
},
"qg-arch-leaf":{
"branch":"init",
"env":'saas3',
"jobname":"java"
},
"wd-order":{
"branch":"master",
"env":'tob1',
"jobname":"java"
},
"commonModel-new":{
"branch":"feature-20221012-dev-config",
"env":'tob1',
"jobname":"java"
},
"DBProxyManager-new":{
"branch":"feature-20221012-dev-config",
"env":'tob1',
"jobname":"java"
},
}
urltype=project_branch.get(project).get('urltype')
branch=project_branch.get(project).get('branch')
......@@ -135,5 +214,30 @@ def main(project,type="",prefix_job='ACK'):
if __name__ == '__main__':
project="kdsp"
main(project)
print(timenow)
project=["wd-delivery"]#,"horkos-ui"]
project=['DBProxyManager-new','dataModel-new','authModel-new','marketModel-new','cartModel-new'] \
or ['wdshop-new'] or ["qg-arch-leaf"]
project=['cartModel-new']
for i in project:
main(i)
# project="marketModel-new"
# #main(project)
# project="cartModel-new"
# #main(project)
# project="dataModel-new"
# #main(project)
# project="distributionModel-new"
# #main(project)
# project="wd-distribution"
# #main(project)
# project="wdshop-new"
# #main(project)
# project='elsModel'
# #main(project)
# project="openModel"
# #main(project)
# project="horkos-ui"
# #main(project)
# project="wd-delivery"
# #main(project)
tools/publicFun.py
View file @ 1be075e7
from tools import *
import hashlib
import faker
import faker,base64
def brand_card(name='建设银行'):
bank = []
if name == '建设银行':
for i in range(6227007201360049787, 6227007201360049987, 1):
bank.append(i)
if len(bank)>1:
break
return bank
if name == '招商银行':
for i in range(6215593700011926793, 6215593700011926993, 1):
bank.append(i)
if len(bank)>1:
break
return bank
if name == '工商银行':
for i in range(6212261001080775106, 6212261001080799999, 1):
bank.append(i)
if len(bank)>1:
break
return bank
if name == '中国银行':
for i in range(6217856200015168391, 6217856200015168591, 1):
bank.append(i)
if len(bank)>1:
break
return bank
def gen_title(ftype):
'''
......@@ -9,10 +40,22 @@ def gen_title(ftype):
:return:
'''
t=faker.Faker(locale='zh_CN')
if ftype in('address','name'):
return eval('t.{}()'.format(ftype))
if ftype in('address','name','ssn','bank_country','bban','credit_card_security_code','credit_card'):
if ftype=='ssn':
while True:
idno=eval('t.{}()'.format(ftype))
birth_year=str(idno[6:10])
if birth_year>'1980' and birth_year<'1999':
birth_year=str(idno[6:10])
birth_month=str(idno[10:12])
birth_day=str(idno[12:14])
print(birth_year,birth_month,birth_day)
break
return idno
if ftype=='phone':
return t.phone_number()
if ftype=='brandcard':
return brand_card()[0]
return t.name()
def md5jiami(mess):
......@@ -50,10 +93,50 @@ def listTodict(k):
temp[k1]=v1
return dict(temp)
def date_oper(num,unit='days'):
unit_format={
"day":"days",
"second":"seconds",
"minute":"minutes",
"hour":"hours",
"week":"weeks",
"year":"years",
"month":"months"
}
unit_temp=unit_format.get(unit) or unit if unit.endswith('s') else 'days'
temp= datetime.datetime.now()+relativedelta(**{unit_temp:num})
return temp.strftime('%Y-%m-%d %H:%M:%S')
def file_base64(inputfilename,outputfilename,type=1):
inputfilename=os.path.join(project_dir,'dataFile',inputfilename)
outputfilename=os.path.join(project_dir,'dataFile',outputfilename)
#filename="/Users/dm/Downloads/量化派/下载的文件/image2021-5-17_12-3-35.png"
with open(inputfilename,'rb') as inputfile:
if type==1:
#base64加密
outfile=open(outputfilename,'wb')
base64.encode(inputfile,outfile)
print("加密文件写到:",outputfilename)
outfile.close()
else:
#base64解密
inputfile=open(inputfilename,'rb')
outputfile=open(outputfilename,'wb')
base64.decode(inputfile,outputfile)
print("解密文件写到:",outputfilename)
inputfile.close()
outputfile.close()
def timenow():
return time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
def gen_card_pic():
from idcardgenerator import gui
gui.run()
if __name__ == '__main__':
t=[{'15':[1]},{'1590': [1]}, {'1592': [0]},{'1592':[1]}]
print(listTodict(t))
\ No newline at end of file
#t=[{'15':[1]},{'1590': [1]}, {'1592': [0]},{'1592':[1]}]
#print(listTodict(t))
#print(file_base64('8B0456AD-D1CC-446E-B098-0B2C661D581F.png','out.txt'))
print(gen_title('ssn'))
gen_card_pic()
\ No newline at end of file
tools/selectOnline.py 0 → 100644
View file @ 1be075e7
from tools import *
def selectDelivery(temp):
'''
:param temp: 支持手机号查询和订单号查询，手机号只能查一个，查询最近一次，订单查询可以查询多笔，以逗号分割
:return:
'''
# temp='202210270001500000'
# temp='13683241439'
if len(temp)==11:
sql=select_delivery_sql.format(temp)
else:
t=temp.split(',')
sql=select_delivery_order_sql.format(tuple(t if len(t)>1 else t+['0']))
print(execmysl('tobonline',sql))
if __name__ == '__main__':
temp='13683241439'
temp='15201517606'
#temp='15011473728'
#temp="202211150000200000"
#temp='18801043265'
selectDelivery(temp)
\ No newline at end of file
tools/temp.py
View file @ 1be075e7
import base64
import pandas as pd
import requests
from tools import *
......@@ -33,6 +35,7 @@ def get_token(numlist):
for i in numlist:
url="https://xfg.mumcooking.com//weidaoyun/shop/member/getkey.jhtml?w=eAxzXoqV"
temp=requests.get(url,headers=map_header)
print('{}===='.format(i),temp.json())
token=jsonpath.jsonpath(temp.json(),'$..token')
temps.append(token[0] if token else 0)
return temps
......@@ -69,12 +72,14 @@ def main(threadnum=3):
from tools.mulThread import MyThread
from tools.listOperation import split_by_num
from tools.fileOperation import readFile,readRenameColums
listsss=[i for i in range(10)]
listsss_df=readFile("/Users/dm/Downloads/量化派/下载的文件/file_172a5f4d4baf858bf49e1ecb7c713238.csv")
listsss=listsss_df['token'].to_list()
divlists=split_by_num(listsss,threadnum,)
#listsss=[i for i in range(1000)]
#listsss_df=readFile("/Users/dm/Downloads/量化派/下载的文件/file_172a5f4d4baf858bf49e1ecb7c713238.csv")
#listsss=listsss_df['token'].to_list()
listsss=[i for i in range(1000)]
divlists=split_by_num(listsss,threadnum)
mylist=[]
final_result=[]
#return 0
for i in range(threadnum):
t=MyThread(get_token,divlists[i])
mylist.append(t)
......@@ -85,18 +90,69 @@ def main(threadnum=3):
final_result+=i.get_result()
return final_result
def get_scence_list(keyword):
'''
根据关键词搜索测试场景
:return:
'''
url="https://thor.quantgroups.cn/scene/list?pageNum=1&pageSize=100&name={}&label=FINANCE".format(keyword)
t=requests.get(url)
temp=jsonpath.jsonpath(t.json(),"$..list[*].name")
#print(temp)
for i in temp:
print(i)
def file_base64(inputfilename,outputfilename,type=1):
filename="/Users/dm/Downloads/量化派/下载的文件/image2021-5-17_12-3-35.png"
with open(inputfilename,'rb') as inputfile:
if type==1:
#base64加密
outfile=open(outputfilename,'wb')
base64.encode(inputfile,outfile)
print("加密文件写到:",outputfilename)
outfile.close()
else:
#base64解密
inputfile=open(inputfilename,'rb')
outputfile=open(outputfilename,'wb')
base64.decode(inputfile,outputfile)
print("解密文件写到:",outputfilename)
inputfile.close()
outputfile.close()
temp=open(filename,'rb+')
temp1=open('ces2.txt','rb+')
temp2=open('ces3.png','wb+')
#t=base64.encode(temp,temp2)
t=base64.decode(temp1,temp2)
print(t)
if __name__ == '__main__':
t=main(3)
print(t)
t.to_csv('/Users/dm/Downloads/apache-jmeter-4.0/bin/token1.csv',index=0,encoding = 'utf-8')
# from idcardgenerator import gui
# gui.run()
# import faker
# t=faker.Faker(locale='zh_CN')
# print(t.phone_number())
# temp=t.ssn()
# print(temp)
# print(temp[6:10],temp[10:12],temp[12:14])
# print(t.address()[:20])
#print(get_scence_list('vcc'))
# from tools.fileOperation import readFile
# token_len=readFile('/Users/dm/Downloads/apache-jmeter-4.0/bin/phoneno.csv')
# tttemp=[]
#tttemp=[]
#
# for i in range(10000):
# for i in range(1):
# temp=get_token()
# tttemp.append(tttemp)
# print("i",i,temp)
# df=pd.DataFrame(tttemp)
# df.to_csv('/Users/dm/Downloads/apache-jmeter-4.0/bin/token.csv',index=0,encoding = 'utf-8')
#df=pd.DataFrame(tttemp)
#df.to_csv('/Users/dm/Downloads/apache-jmeter-4.0/bin/token.csv',index=0,encoding = 'utf-8')
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment