新增可控制台执行的ab耗时脚本

databaseConn/__init__.py

@@ -33,10 +33,16 @@ mysql_info={
         "host":"172.29.5.16",
         "port":"3306"
     },
-    "27":{
+    "8066_27":{
         "name":"real_data_test",
         "pwd":"qpRBSarm$Tv*YO!n",
         "host":"172.30.5.27",
         "port":"8066"
+    },
+    "7438_27":{
+        "name":"xyqb_recommender_system_w",
+        "pwd":"Qa6pxybKE8KgsUGV",
+        "host":"172.30.5.27",
+        "port":"7438"
     }
 }

databaseConn/mongoOperation.py

+from databaseConn import *
+

databaseConn/mysqlOperation.py

@@ -32,7 +32,9 @@ def mysql_universal(namespace):
     mysql_universal="mysql://{name}:{pwd}@{host}:{port}"
     mysql_info=mysqlInfo(namespace)
     #print(mysql_info)
-    return create_engine(mysql_universal.format(**mysql_info))
+    temp=create_engine(mysql_universal.format(**mysql_info))
+    print("mysql连接信息:",temp)
+    return temp
 
 def execmysl(namespace,sql):
     if 'select' not in sql or 'from' not in sql:
@@ -68,4 +70,4 @@ def concatSql(sql,**kwargs):
     return sql+' where '+temp[:-4]
 
 if __name__ == '__main__':
-    print(execmysl(119,'update 1'))
+    print(execmysl('qa',"select sku_name from kdsp.t_sku_info where  sku_name like '%%口红%%'"))
\ No newline at end of file

databaseConn/redisOperation.py

@@ -84,7 +84,7 @@ def getRedisValue(key,conntype=None,oper='select'):
 
 if __name__ == '__main__':
     userUuid='00003a93-2a32-4501-b338-755b6cb1ec49'
-    t=getRedisValue("search_fm_offline_feature:cd11201d3e789c63_query_offline")
+    t=getRedisValue("same_product_batch_uuid")
     print(t)
 
     ## search_fm_offline_feature:{md5(搜索词)}_query_offline

recommend/__init__.py

@@ -6,5 +6,6 @@ from tools.listOperation import *
 from databaseConn.mysqlOperation import *
 from databaseConn.redisOperation import *
 from recommend.publicSql import *
+
 cur_dir=os.path.dirname(os.path.abspath(__file__))
 file_path=os.path.join(cur_dir,'tempFile')

recommend/orderTopic.py

+from recommend import *
+from recommend.publicFunc import skuinfo,dapanBu
+
+def preconditions(uuid):
+    '''
+    :param uuid:
+    :return: 判断上批次是否为加购topic,是则不触发，不是才触发
+    '''
+    back_batch_redis=""
+    return True
+
+def recall_J1(skuno):
+    '''
+    :param skuno:
+    :return: 将所有的数据都召回，然后进行去重并选取100
+    '''
+    # graph_change_sql=concatSql(graph_relation_sql,**{"sku1_id":skuno})
+    # graph_df=execmysl('7438_27',graph_change_sql)
+    # temp= graph_df.sort_values(by='rank')['sku2_id'].to_list()[:100]
+    # if len(set(100))>100:
+    #     return removeRepeat(temp)[:100]
+    # als_change_sql1=concatSql(als_sql,**{"product_id1":skuno})
+    # als_df1=execmysl('7438_27',als_change_sql1)
+    # temp+=als_df1.sort_values(by='rank')['product_id2'].to_list()
+    # als_change_sql2=concatSql(als_sql,**{"product_id2":skuno})
+    # als_df2=execmysl('7438_27',als_change_sql1)
+    # temp+=als_df2.sort_values(by='rank')['product_id1'].to_list()
+    # return removeRepeat(temp)[:100]
+    temp=[]
+    graph_change_redis=graph_redis.format(sku_no=skuno)
+    graph_date=getRedisValue(graph_change_redis).get(graph_change_redis)
+    als_change_redis=als_redis.format(sku_no=skuno)
+    als_date=getRedisValue(als_change_redis).get(als_change_redis)
+    temp+=graph_date
+    temp+=als_date
+    temp+=supply_J1(skuno)
+    return temp
+
+
+
+def supply_J1(skuno):
+    sku_info=skuinfo(skuno)
+    dapan_change_sql=concatSql(dapan_sql,**{"cid3":sku_info.get('cid3')})
+    dapan_df=execmysl('7438_27',dapan_change_sql)
+    temp=dapan_df[dapan_df['brand_name']!=sku_info.get('brand_name')]['sku_no'].to_list()
+    temp=removeRepeat(temp)[:50]
+    dapan_change1_sql=concatSql(dapan_sql,**{"brand_name":sku_info.get('brand_name')})
+    dapan1_df=execmysl('7438_27',dapan_change1_sql)
+    temp+=dapan_df[dapan_df['cid3']!=sku_info.get('cid3')]['sku_no'].to_list()
+    return removeRepeat(temp)[:100]
+
+def recall_J2():
+    pass
+
+def recall_J3():
+    return dapanBu()
+
+
+if __name__ == '__main__':
+    print(recall_J1('12312'))

recommend/publicFunc.py

@@ -206,7 +206,14 @@ def concatCondition(seedData,contype='jg'):
         conditions.append({'condition':condition7,'by':'score_brand'})
         return conditions
 
-
+def dapanBu():
+    '''
+    :return: 获取当前大盘批次，然后拼接redis，拿到大盘数据
+    '''
+    cur_dapan_redis="same_product_batch_uuid"
+    batch=getRedisValue(cur_dapan_redis).get(cur_dapan_redis)
+    dapan_redis=cur_dapan_redis+":"+str(batch)
+    return getRedisValue(dapan_redis).get(dapan_redis)
 
 
 def jgDapanBu():
@@ -238,7 +245,7 @@ if __name__ == '__main__':
     # #change_sql=concatSql(jd_hot_sql,)
     # t=normalClickTopic(sku_no,change_sql)
     # print(t)
-    print(concatCondition(skus))
+    print(dapanBu())
 
 
 

recommend/publicSql.py

@@ -48,3 +48,26 @@ jg_purpost_all_sql="""
 select * from   where cid3={} or cid2={} or cid1={} or brand_name='{}'
 """
 
+#图关联表
+graph_redis="""
+product_graph:{sku_no}
+"""
+graph_relation_sql="""
+select * from xyqb_recommender_system.mix_products_graph 
+"""
+##ALS相关表
+als_redis="""
+product_correlation:{sku_no}
+"""
+als_sql="""
+select * from xyqb_recommender_system.mix_products_correlation 
+"""
+
+dapan_hot_sql="""
+select * from 
+xyqb_recommender_system.kdsp_same_ranked_list
+"""
+##获取上一批次记录后，再去mysql批次表里拿数据
+back_batch_redis="""
+recommend_recall:{uuid}:{sku_no}
+"""

search/abSearch_script.py

+import os,itertools,requests,jsonpath,time,traceback,datetime
+import random,sys
+import pandas as pd
+import uuid as codeuuid
+#from search import *
+channel=[214,1,217]
+cur_dir=os.path.dirname(os.path.abspath(__file__))
+file_path=os.path.join(cur_dir,'tempFile')
+
+project_dir=os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+data_file_path=os.path.join(project_dir,'dataFile')
+def readFile(filename,sheetname=0):
+    '''
+    :param filename:
+    :return:读取csx或xlsx文件
+    '''
+    houzhui=filename.split('.')[-1]
+    if houzhui=='csv':
+        df=pd.read_csv(filename, index_col=0,sheetname=sheetname)
+    else:
+        df=pd.read_excel(filename,sheet_name=sheetname)
+    return df
+def mergelist(a):
+    '''
+    :param a:list
+    :return: 嵌套list合并成一个
+    '''
+    t=itertools.chain.from_iterable(a)
+    return list(t)
+def filePath(filepath,name=''):
+    filename=genReportName(name)
+    return os.path.join(filepath,filename)
+def genReportName(name=''):
+    '''
+    :param name:
+    :return:生成文件名
+    '''
+    t=round(datetime.datetime.now().timestamp())
+    return name+'_'+str(t)+'.xlsx'
+def requestsend(uuid,deviceid,searchContent,selectedActivitie,page=1,searchtype='txt',env='online'):
+    """
+    :param uuid: 用户uuid
+    :param deviceid: 设备号id
+    :param searchContent: 文本搜索
+    :param page: 默认1，搜索页
+    :param selectedActivitie: 专题id
+    :param type: txt表示文本搜索，否则为专题搜索
+    :param env: 环境
+    :return:
+    """
+    if env not in ['test','pro','online']:
+        raise Exception('env[{}]只能:test|pro|online'.format(env))
+    if env=='online':
+        baseurl="http://caesar-gateway.q-gp.com"
+    else:
+        baseurl="http://open-search-engine.ec-{env}.qg-rc.net".format(env=env) if env=='test'  \
+                        else "http://open-search-engine.qg-{env}.qgwebservice.com".format(env=env)
+    url=baseurl+"/search"
+    header = {
+        "XQG-USER-UUID": uuid,
+        "content-type": "application/json",
+        "XQG-DEVICE-ID": deviceid
+    }
+    body = {
+        "channel": random.sample(channel, 1)[0],
+        "userUuid": uuid,
+        "deviceId": deviceid,
+        "extraParams": {
+            "appVersion": "8.6.00",
+            "terminal": "ios"
+        },
+        ##文本搜索
+        # "searchContent":searchContent ,
+        ##专题搜索
+        # "selectedActivities":[{
+        #     "id": ''.join(random.sample([str(i) for i in range(40)],1)),
+        #     "type":2
+        # }],
+        "flipInfo": {
+            "pageNo": page,
+            "pageSize": 20
+        }
+    }
+    if searchtype == 'txt': ##文本
+        body['searchContent'] = searchContent
+    elif searchtype == 'label': ##
+        body['selectedActivities'] = [{
+            "id": selectedActivitie,  # ''.join(random.sample(','.join('215,214,34,33,32,31,30'),1)),
+            "type": 2
+        }]
+    elif searchtype == 'capt':
+        body['selectedActivities'] = [{
+            "id": selectedActivitie,  # ''.join(random.sample(','.join('215,214,34,33,32,31,30'),1)),
+            "type": 3
+        }]
+    elif searchtype == 'activity':
+        body['selectedActivities'] = [{
+            "id": selectedActivitie,  # ''.join(random.sample(','.join('215,214,34,33,32,31,30'),1)),
+            "type": 1
+        }]
+    t = requests.post(url, json=body, headers=header)
+    print('搜索词:',searchContent)
+    try:
+        spu_nos = jsonpath.jsonpath(t.json(), '$..skuNo') or ['null']
+        sku_names = jsonpath.jsonpath(t.json(), '$..skuName') or ['null']
+        page_size = jsonpath.jsonpath(t.json(), '$..totalPage')[0]
+        search_id = jsonpath.jsonpath(t.json(), '$..searchId')[0]
+        total_skus=jsonpath.jsonpath(t.json(), '$..totalCount')[0]
+        request_time = t.elapsed.total_seconds()
+        tttt=','.join(spu_nos)
+        #print(tttt,type(tttt))
+        # with open('sku_result_3.txt', 'a+') as file:
+        #     file.write('搜索词:' + searchContent + ";result:" + ','.join(sku_names) + "\n")
+        return page_size, ','.join(spu_nos), str(request_time), search_id,','.join(sku_names),searchContent,total_skus
+    except:
+        print('搜索接口报错:',searchContent,t.text)
+
+
+def hotWord(filename,sheetname=0):
+    #from tools.fileOperation import readRenameColums
+    filepath=os.path.join(data_file_path,filename)
+    sheetname=int(sheetname)
+    df=readRenameColums(filepath,fcolums=['search_words'],sheetname=sheetname)
+    return df
+
+def readRenameColums(filename,fcolums,sheetname=0):
+    '''
+    :param filename:
+    :param fcolums:list，重命名的数据
+    :return:列名重命名
+    '''
+    df=readFile(filename,sheetname)
+    try:
+        df=df.rename(columns=dict(zip(df.columns,fcolums[0:len(df.columns)])))
+    except Exception as e:
+        print('重命名的列名数[{}]小于文档中的列名数[{}]，导致无法重命名'.format(len(fcolums),len(df.columns)))
+        #print('===接口',traceback.print_exc())
+        ##获取错误的堆栈信息
+        print("异常的堆栈信息:")
+        traceback.format_exc(limit=2)
+    return df
+def genUuidDeviceid():
+    """
+    deviced必须随机生成，不然无法统计spu总数。故就算uuid是相同的，deviceid也不能相同
+    :return:
+    """
+    uuid,deviced=codeuuid.uuid4().urn.split(':')[-1],codeuuid.uuid4().urn.split(':')[-1]
+    uuid=sys.argv[1]#'c0c27ffb-f2da-4650-8855-2da2191ce772'
+    deviced='guiqiuyue_'+str(deviced)
+    return uuid,deviced
+def totalrun(selectedActivitie=1, type='txt', num=0,isFirstPage=False,sheetname=0,ishotwordrand=1):
+    '''
+    :param selectedActivitie: 选填
+    :param type: 跑文本搜索，还是专题搜索。目前支持：'txt'｜'label'｜'capt'｜'activity'
+    :param num: 和ishotwordrand组合，如果ishotwordrand=1，则随机获取搜索词；如果为0，则按照顺序获取搜索词
+    :param isFirstPage: 如果只跑第一页，则需要设置为true
+    :param sheetname:获取搜索词的sheetname，默认0，第一个工作薄
+    :return:
+    '''
+    timenow=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
+    temp=[]
+    result_temp={}
+    searchContents=hotWord('线上环境搜索热词.xlsx',sheetname=sheetname)['search_words'].apply(lambda x:str(x).strip()).to_list()
+    if ishotwordrand:
+        random_value=random.Random().randint(0,len(searchContents)-1)
+    else:
+        random_value=num
+    #from tools.publicFun import genUuidDeviceid
+    uuid,deviceid=genUuidDeviceid()
+    selectedActivitie='2'
+    flag=1
+    try:
+        page_size, spu_nos, \
+        request_time, search_id,sku_names,searchContent,total_skus=\
+            requestsend(uuid,deviceid,searchContents[random_value],selectedActivitie,flag,type)
+        result_temp["nowtime"]=str(timenow)
+        result_temp["deviceid"]=deviceid
+        result_temp["uuid"]=uuid
+        result_temp["searchcontent"]=searchContent
+        result_temp["pageno"]=flag
+        result_temp["costtime"]=request_time
+        result_temp["skunos"]=spu_nos
+        result_temp['totalSkus']=total_skus
+        temp.append(result_temp)
+        #file_template=["nowtime","searchcontent","pageno","costtime","deviceid","uuid","skunos"]
+        flag+=1
+        while  flag<=page_size and not isFirstPage:
+            result_temp={}
+            page_size, spu_nos, \
+            request_time, search_id,sku_names,searchContent= \
+                requestsend(uuid,deviceid,searchContents[random_value],selectedActivitie,flag,type)
+            result_temp["nowtime"]=timenow
+            result_temp["searchcontent"]=searchContent
+            result_temp["pageno"]=flag
+            result_temp["costtime"]=request_time
+            result_temp["deviceid"]=deviceid
+            result_temp["uuid"]=uuid
+            result_temp["skunos"]=spu_nos
+            result_temp['totalSkus']=total_skus
+            temp.append(result_temp)
+            flag+=1
+    #filename=genReportName('searchResult')
+        print(temp)
+        return temp
+    except:
+        traceback.print_exc(limit=2)
+
+
+def allRun(num,isFirstPage=0,sheetname=0,ishotwordrand=1):
+    ttemp=[]
+    for i in range(num):
+        print('第[{}]次搜索'.format(str(i)))
+        try:
+            temp=totalrun(num=i,isFirstPage=isFirstPage,sheetname=sheetname,ishotwordrand=ishotwordrand)
+            ttemp+=temp
+            #print(temp)
+        except:
+            traceback.print_exc(limit=2)
+    df=pd.DataFrame(ttemp)
+    reportname=filePath(file_path,'searchResult')
+    #df['skunos']=df['skunos'].astype('string')
+    df.to_excel(reportname,index=0,encoding = 'utf-8',float_format = str)
+
+
+
+def analysis(filename=None):
+    #temp=defaultdict()
+    result=[]
+    filename=filename if filename \
+        else os.popen("cd {};ls -t |grep -v 'init' |head -1".format(file_path)).read(100)
+    filename_path=os.path.join(file_path,filename.strip())
+    print('解析文件名：',filename_path)
+    df=readFile(filename_path)
+    df['searchcontent'].fillna('无效',inplace=True)
+    mean_costtime=str(round(df['costtime'].mean(),3)*1000)+'ms'
+    count_by=['uuid','deviceid','searchcontent']
+    uds=df.groupby(count_by)#['skunos']
+    #count_by_skus=count_by.append('skunos')
+    #df1=pd.DataFrame(spus,columns=count_by_skus)#['skunos']
+    for uuid,deviceid,searchcontent in uds.groups:
+        temp={}
+        temp['uuid']=uuid
+        temp['deviceid']=deviceid
+        temp['searchcontent']=searchcontent
+
+        tempskus=df[(df['uuid']==uuid) & (df['deviceid']==deviceid) \
+                            &(df['searchcontent']==searchcontent)]['skunos'].to_list()#.__str__()
+        temp['costtime']=df[(df['uuid']==uuid) & (df['deviceid']==deviceid) \
+                            &(df['searchcontent']==searchcontent)]['costtime'].to_list()
+        temp['totalSkus']=mergelist([str(i).split(',') for i in tempskus])
+        #print(tempskus)
+
+        temp['isrepeat']=0 if len(temp['totalSkus']) == len(list(set(temp['totalSkus']))) else 1
+
+        temp['curpage_countskus']=len(temp['totalSkus'])
+
+        #print(uuid,deviceid,searchcontent)
+
+        #temppp=1
+        temp['countskus']=df[(df['uuid']==uuid) & (df['deviceid']==deviceid) \
+                             &(df['searchcontent']==searchcontent)]['totalSkus'].to_list()[0]
+
+        result.append(temp)
+
+    startans=pd.DataFrame(result)
+
+    fist_200=startans[(startans['countskus']<=200)]
+    fist_500=startans[(startans['countskus']>200) & (startans['countskus']<=500)]
+    fist_1000=startans[(startans['countskus']>500) & (startans['countskus']<=1000)]
+
+    fist1_200_1=[i for i in mergelist(fist_200['costtime'].to_list()) if str(i)>='0.1']
+    fist1_200_2=[i for i in mergelist(fist_200['costtime'].to_list()) if str(i)>='0.15']
+    fist1_200_3=[i for i in mergelist(fist_200['costtime'].to_list()) if str(i)>='0.2']
+
+
+    fist1_500_1=[i for i in mergelist(fist_500['costtime'].to_list()) if str(i)>='0.1']
+    fist1_500_2=[i for i in mergelist(fist_500['costtime'].to_list()) if str(i)>='0.15']
+    fist1_500_3=[i for i in mergelist(fist_500['costtime'].to_list()) if str(i)>='0.2']
+
+    fist1_1000_1=[i for i in mergelist(fist_1000['costtime'].to_list()) if str(i)>='0.1']
+    fist1_1000_2=[i for i in mergelist(fist_1000['costtime'].to_list()) if str(i)>='0.15']
+    fist1_1000_3=[i for i in mergelist(fist_1000['costtime'].to_list()) if str(i)>='0.2']
+
+    print('总请求数:',startans.shape[0],';平均耗时:',mean_costtime)
+    print("spu是否有重复数据:",1 if startans[startans['isrepeat']==1].shape[0] else 0)
+    print("搜索词spus总数<=200时-->",'耗时大于100ms的总数',len(fist1_200_1),\
+          ';耗时大于150ms的总数',len(fist1_200_2),
+          ';耗时大于200ms的总数',len(fist1_200_3))
+
+    print("搜索词spus总数<=500&>200时-->",'耗时大于100ms的总数',len(fist1_500_1), \
+          ';耗时大于150ms的总数',len(fist1_500_2),
+          ';耗时大于200ms的总数',len(fist1_500_3))
+
+    print("搜索词spus总数<=1000&>500时-->",'耗时大于100ms的总数',len(fist1_1000_1), \
+          ';耗时大于150ms的总数',len(fist1_1000_2),
+          ';耗时大于200ms的总数',len(fist1_1000_3))
+
+
+
+    #return result
+
+
+
+
+
+
+
+if __name__ == '__main__':
+    # t=hotWord('线上环境搜索热词.xlsx')['search_words'].apply(lambda x:str(x).strip())
+    # print(t)
+    #print(totalrun(isFirstPage=1))
+    isexec=sys.argv[2]
+    ishotwordrand=0
+    print(sys.argv)
+    if isexec and isexec!='0':
+        allRun(1000,isFirstPage=1,sheetname=sys.argv[3],ishotwordrand=ishotwordrand)
+    else:
+        print(analysis())
+    # a=[['236473224274432', '7188063'],['236473224274432', '7188063']]
+    # print(list(itertools.chain.from_iterable(a)))
+
+
+
+
+
+

tools/__init__.py

 from databaseConn.mysqlOperation import *
 from recommend.publicSql import *
 from databaseConn import *
+
+dapan_sql
\ No newline at end of file

tools/httprequest.py

 import requests
-from search.abSearch import totalrun
+from search.abSearch_script import totalrun
 from tools import *
 from tools.publicFun import genUuidDeviceid
 from tools.fileOperation import *

tools/publicFun.py

@@ -5,7 +5,7 @@ def genUuidDeviceid():
     :return:
     """
     uuid,deviced=codeuuid.uuid4().urn.split(':')[-1],codeuuid.uuid4().urn.split(':')[-1]
-    uuid='c1d7ff4e-ee78-48de-8b8d-50c2af29c3ff'
+    uuid='c0c27ffb-f2da-4650-8855-2da2191ce772'
     deviced='guiqiuyue_'+str(deviced)
     return uuid,deviced