Skip to content

M
model_monitoring_monthly
Commit 9753fa69 authored by 舒皓月's avatar 舒皓月
Browse files
Options

20191009

parent 2bb3bf7d
Pipeline #193 failed with stages
      Show whitespace changes
      Inline Side-by-side
      Showing with 29 additions and 216 deletions
      AUC_time.py
      View file @ 9753fa69
      ......@@ -31,7 +31,7 @@ class AUCMonitor:
      passdue_day=15, save_path='./auc/',
      min_user_group=500, interval_days=15, min_auc=0.55,
      date_list=('2019-03-01', '2019-03-15', '2019-03-31', '2019-04-15'),
      if_save=True, if_read=True):
      if_read=True):
      # 考虑到数据库配置基本不变, 所以不设置创建对象时对应输入变量.
      self.mysql_engine = pymysql.connect(host='172.20.6.9',
      ......@@ -44,21 +44,14 @@ class AUCMonitor:
      self.mongo_client = pymongo.MongoClient(
      "mongodb://haoyue.shu:x2egwRHk7WhQ4So1@172.18.3.22:27017/?authSource=rc_mgo_feature_dp")
      self.mongo_db = self.mongo_client['rc_mgo_feature_dp']
      self.mongo_table = self.mongo_db['wf_audit_log_with_feature']
      self.mongo_table = self.mongo_db['wf_audit_log_with_feature_online']
      # 读取整理在Excel中的模型相关信息.
      self.field_info_df = pd.read_excel(excel_path, sheet_name=sheet_name)
      self.field_name_list = self.field_info_df.field_name.tolist()
      self.field_query_list = self.field_info_df.field_query.tolist()
      self.field_app_type_list = self.field_info_df.app_type.tolist()
      self.field_app_type_list = [str(x) for x in self.field_app_type_list]
      self.field_DB_list = self.field_info_df.DB.tolist()
      self.field_query_name_dict = dict(zip(self.field_query_list, self.field_name_list))
      self.field_query_app_type_dict = dict(zip(self.field_query_list, self.field_app_type_list))
      ## 空跑信息.
      self.na_time = self.field_info_df.na_time.tolist() # 空跑时间段
      self.na_app_type = self.field_info_df.na_app_type.tolist() # 空跑申请类型
      self.na_app_chan = self.field_info_df.na_app_chan.tolist() # 空跑渠道
      # 一些定义的常量
      self.passdue_day = passdue_day # 逾期天数, 默认15.
      ......@@ -86,7 +79,6 @@ class AUCMonitor:
      self.auc_info_df = pd.DataFrame(columns=auc_cols)
      # 程序数据读写模式.
      self.if_save = if_save # 是否保存从数据库抽取的数据.
      self.if_read = if_read # 是否从从数据库读取.
      # 创建文件夹.
      ......@@ -134,54 +126,6 @@ class AUCMonitor:
      else:
      return '0' + str(x)
      def filter_data(self, df, field):
      '''
      过滤空跑数据.
      :param df: df.
      :param field: str, 字段名.
      :return: df, 过滤后的数据.
      '''
      df = df[~((df['applied_type'] == 1) & (df['applied_channel'].apply(lambda x: 'Android' in x)))]
      field_idx = self.field_query_list.index(field)
      na_time = self.na_time[field_idx]
      na_type = self.na_app_type[field_idx]
      na_chan = self.na_app_chan[field_idx]
      if pd.isnull(na_time): # 没有空跑时间, 则不记录.
      return df
      # 时间.
      t_s, t_e = na_time.split('~')
      if len(t_e) == 0: # 若还在空跑, 则不记录.
      return pd.DataFrame()
      else:
      na_df = df[
      (df['applied_at'].apply(lambda x: x[:10] >= t_s)) & (df['applied_at'].apply(lambda x: x[:10] <= t_e))]
      if na_df.shape[0] == 0:
      return df
      # 申请类型.
      if pd.isnull(na_type):
      return df[~df.index.isin(na_df.index.values)]
      else:
      tmp_df = pd.DataFrame()
      for i in str(int(na_type)):
      tmp_df = tmp_df.append(na_df[na_df['applied_type'] == int(i)])
      na_df = tmp_df
      if na_df.shape[0] == 0:
      return df
      # 申请渠道.
      if pd.isnull(na_chan):
      return df[~df.index.isin(na_df.index.values)]
      else:
      tmp_df = pd.DataFrame()
      for i in na_chan.split(','):
      tmp_df = tmp_df.append(na_df[na_df['applied_channel'].apply(lambda x: i in x)])
      na_df = tmp_df
      if na_df.shape[0] == 0:
      return df
      return df[~df.index.isin(na_df.index.values)]
      def helper_auc(self, user_group_name=None, df=None, info_dict=None, field=None):
      '''
      信息提取函数.
      ......@@ -226,13 +170,6 @@ class AUCMonitor:
      [field, 'date_label', 'applied_type', 'applied_channel', 'overdue', 'passdue_day', 'applied_at']].copy()
      # 筛选出放款, 且逾期表现的数据.
      df_copy = df_copy[(df_copy['overdue'].notna()) & (df_copy[field].notna())]
      # 筛选正确申请类型的数据.
      tmp_df = pd.DataFrame()
      for i in self.field_query_app_type_dict[field]:
      tmp_df = tmp_df.append(df_copy[df_copy['applied_type'] == int(i)])
      df_copy = tmp_df
      # 过滤空跑数据.
      df_copy = self.filter_data(df_copy, field)
      if df_copy.shape[0] == 0:
      print('仍在空跑.')
      return None
      ......@@ -374,14 +311,7 @@ class AUCMonitor:
      print('在MySQL中找到该字段: %s' % self.field_query_name_dict[field])
      else:
      print('在MySQL中不存在该字段: %s' % self.field_query_name_dict[field])
      # 删除该字段.
      idx = self.field_query_list.index(field)
      self.field_query_list.pop(idx)
      self.field_DB_list.pop(idx)
      self.field_app_type_list.pop(idx)
      self.field_name_list.pop(idx)
      del self.field_query_name_dict[field]
      del self.field_query_app_type_dict[field]
      if len(real_mysql_field) > 0:
      self.mysql_df = self.query_mysql('''SELECT order_no, applied_at, %s, \
      ......@@ -398,8 +328,8 @@ class AUCMonitor:
      AND applied_at <= "%s 00:00:00"'''
      % (self.date_list[0], self.date_list[-1]))
      print('MySQL数据获取成功.')
      if self.if_save:
      self.mysql_df.to_csv(self.save_path + 'data/mysql_data.csv', index=False)
      self.mysql_df = pd.read_csv(self.save_path + 'data/mysql_data.csv')
      else:
      self.mysql_df = pd.read_csv(self.save_path + 'data/mysql_data.csv')
      ......@@ -420,8 +350,8 @@ class AUCMonitor:
      self.mongo_df['applied_type'] = self.mongo_df['wf_loan_type'].apply(func_0)
      del self.mongo_df['wf_loan_type']
      print('MongoDB数据获取成功.')
      if self.if_save:
      self.mongo_df.to_csv(self.save_path + 'data/mongo_data.csv', index=False)
      self.mongo_df = pd.read_csv(self.save_path + 'data/mongo_data.csv')
      else:
      self.mongo_df = pd.read_csv(self.save_path + 'data/mongo_data.csv')
      self.mongo_df = self.mongo_df.loc[self.mongo_df['applied_type'].notna()]
      ......@@ -453,7 +383,11 @@ class AUCMonitor:
      # 清洗数据.
      def clean_data(data):
      if pd.isnull(data):
      return np.nan
      try:
      if data <= 0 or data > 999999:
      return np.nan
      return float(data)
      except:
      return np.nan
      ......@@ -469,8 +403,10 @@ class AUCMonitor:
      print('不包含以下字段:')
      for field in na_field_list:
      print(self.field_query_name_dict[field])
      self.field_query_list.remove(field)
      self.field_name_list.remove(self.field_query_name_dict[field])
      idx = self.field_query_list.index(field)
      self.field_query_list.pop(idx)
      self.field_DB_list.pop(idx)
      self.field_name_list.pop(idx)
      del self.field_query_name_dict[field]
      # 数据按时间划分.
      ......
      PSI_time.py
      View file @ 9753fa69
      ......@@ -45,21 +45,14 @@ class PSIMonitor:
      self.mongo_client = pymongo.MongoClient(
      "mongodb://haoyue.shu:x2egwRHk7WhQ4So1@172.18.3.22:27017/?authSource=rc_mgo_feature_dp")
      self.mongo_db = self.mongo_client['rc_mgo_feature_dp']
      self.mongo_table = self.mongo_db['wf_audit_log_with_feature']
      self.mongo_table = self.mongo_db['wf_audit_log_with_feature_online']
      # 读取整理在Excel中的模型相关信息.
      self.field_info_df = pd.read_excel(excel_path, sheet_name=sheet_name)
      self.field_name_list = self.field_info_df.field_name.tolist()
      self.field_query_list = self.field_info_df.field_query.tolist()
      self.field_app_type_list = self.field_info_df.app_type.tolist()
      self.field_app_type_list = [str(x) for x in self.field_app_type_list]
      self.field_DB_list = self.field_info_df.DB.tolist()
      self.field_query_name_dict = dict(zip(self.field_query_list, self.field_name_list))
      self.field_query_app_type_dict = dict(zip(self.field_query_list, self.field_app_type_list))
      ## 空跑信息.
      self.na_time = self.field_info_df.na_time.tolist() # 空跑时间段
      self.na_app_type = self.field_info_df.na_app_type.tolist() # 空跑申请类型
      self.na_app_chan = self.field_info_df.na_app_chan.tolist() # 空跑渠道
      # 一些定义的常量
      self.min_user_group = min_user_group * interval_days / 30 # 最小客群数量.
      ......@@ -166,54 +159,6 @@ class PSIMonitor:
      except:
      return None
      def filter_data(self, df, field):
      '''
      过滤空跑数据.
      :param df: df.
      :param field: str, 字段名.
      :return: df, 过滤后的数据.
      '''
      df = df[~((df['applied_type'] == 1) & (df['applied_channel'].apply(lambda x: 'Android' in x)))]
      field_idx = self.field_query_list.index(field)
      na_time = self.na_time[field_idx]
      na_type = self.na_app_type[field_idx]
      na_chan = self.na_app_chan[field_idx]
      if pd.isnull(na_time): # 没有空跑时间, 则不记录.
      return df
      # 时间.
      t_s, t_e = na_time.split('~')
      if len(t_e) == 0: # 若还在空跑, 则不记录.
      return pd.DataFrame()
      else:
      na_df = df[
      (df['applied_at'].apply(lambda x: x[:10] >= t_s)) & (df['applied_at'].apply(lambda x: x[:10] <= t_e))]
      if na_df.shape[0] == 0:
      return df
      # 申请类型.
      if pd.isnull(na_type):
      return df[~df.index.isin(na_df.index.values)]
      else:
      tmp_df = pd.DataFrame()
      for i in str(int(na_type)):
      tmp_df = tmp_df.append(na_df[na_df['applied_type'] == int(i)])
      na_df = tmp_df
      if na_df.shape[0] == 0:
      return df
      # 申请渠道.
      if pd.isnull(na_chan):
      return df[~df.index.isin(na_df.index.values)]
      else:
      tmp_df = pd.DataFrame()
      for i in na_chan.split(','):
      tmp_df = tmp_df.append(na_df[na_df['applied_channel'].apply(lambda x: i in x)])
      na_df = tmp_df
      if na_df.shape[0] == 0:
      return df
      return df[~df.index.isin(na_df.index.values)]
      def helper_psi(self, user_group_name=None, df=None, info_dict=None, field=None):
      '''
      信息提取函数.
      ......@@ -265,14 +210,7 @@ class PSIMonitor:
      # 分离数据.
      df_copy = self.merge_data[[field, 'date_label', 'applied_type', 'applied_channel', 'applied_at']].copy()
      # 选择包含正确申请类型的数据.
      tmp_df = pd.DataFrame()
      for i in self.field_query_app_type_dict[field]:
      tmp_df = tmp_df.append(df_copy[df_copy['applied_type'] == int(i)])
      df_copy = tmp_df
      df_copy = df_copy[df_copy[field].notna()]
      # 过滤空跑数据.
      df_copy = self.filter_data(df_copy, field)
      if df_copy.shape[0] == 0:
      print('仍在空跑.')
      return None
      ......@@ -399,10 +337,8 @@ class PSIMonitor:
      idx = self.field_query_list.index(field)
      self.field_query_list.pop(idx)
      self.field_DB_list.pop(idx)
      self.field_app_type_list.pop(idx)
      self.field_name_list.pop(idx)
      del self.field_query_name_dict[field]
      del self.field_query_app_type_dict[field]
      if len(real_mysql_field) > 0:
      self.mysql_df = self.query_mysql('''SELECT order_no, applied_at, %s, \
      applied_from, applied_channel, transacted, passdue_day \
      ......@@ -480,8 +416,10 @@ class PSIMonitor:
      print('不包含以下字段:')
      for field in na_field_list:
      print(self.field_query_name_dict[field])
      self.field_query_list.remove(field)
      self.field_name_list.remove(self.field_query_name_dict[field])
      idx = self.field_query_list.index(field)
      self.field_query_list.pop(idx)
      self.field_DB_list.pop(idx)
      self.field_name_list.pop(idx)
      del self.field_query_name_dict[field]
      # 数据按时间划分.
      ......
      VLM_time.py
      View file @ 9753fa69
      ......@@ -48,21 +48,14 @@ class VLMMonitor:
      self.mongo_client = pymongo.MongoClient(
      "mongodb://haoyue.shu:x2egwRHk7WhQ4So1@172.18.3.22:27017/?authSource=rc_mgo_feature_dp")
      self.mongo_db = self.mongo_client['rc_mgo_feature_dp']
      self.mongo_table = self.mongo_db['wf_audit_log_with_feature']
      self.mongo_table = self.mongo_db['wf_audit_log_with_feature_online']
      # 读取整理在Excel中的模型相关信息.
      self.field_info_df = pd.read_excel(excel_path, sheet_name=sheet_name)
      self.field_name_list = self.field_info_df.field_name.tolist()
      self.field_query_list = self.field_info_df.field_query.tolist()
      self.field_app_type_list = self.field_info_df.app_type.tolist()
      self.field_app_type_list = [str(x) for x in self.field_app_type_list]
      self.field_DB_list = self.field_info_df.DB.tolist()
      self.field_query_name_dict = dict(zip(self.field_query_list, self.field_name_list))
      self.field_query_app_type_dict = dict(zip(self.field_query_list, self.field_app_type_list))
      ## 空跑信息.
      self.na_time = self.field_info_df.na_time.tolist() # 空跑时间段
      self.na_app_type = self.field_info_df.na_app_type.tolist() # 空跑申请类型
      self.na_app_chan = self.field_info_df.na_app_chan.tolist() # 空跑渠道
      # 文件存储路径.
      self.fig_save_path = save_path + 'image/'
      ......@@ -143,53 +136,6 @@ class VLMMonitor:
      pass
      def filter_data(self, df, field):
      '''
      过滤空跑数据.
      :param df: df.
      :param field: str, 字段名.
      :return: df, 过滤后的数据.
      '''
      df = df[~((df['applied_type'] == 1) & (df['applied_channel'].apply(lambda x: 'Android' in x)))]
      field_idx = self.field_query_list.index(field)
      na_time = self.na_time[field_idx]
      na_type = self.na_app_type[field_idx]
      na_chan = self.na_app_chan[field_idx]
      if pd.isnull(na_time): # 没有空跑时间, 则不记录.
      return df
      # 时间.
      t_s, t_e = na_time.split('~')
      if len(t_e) == 0: # 若还在空跑, 则不记录.
      return pd.DataFrame()
      else:
      na_df = df[
      (df['applied_at'].apply(lambda x: x[:10] >= t_s)) & (df['applied_at'].apply(lambda x: x[:10] <= t_e))]
      if na_df.shape[0] == 0:
      return df
      # 申请类型.
      if pd.isnull(na_type):
      return df[~df.index.isin(na_df.index.values)]
      else:
      tmp_df = pd.DataFrame()
      for i in str(int(na_type)):
      tmp_df = tmp_df.append(na_df[na_df['applied_type'] == int(i)])
      na_df = tmp_df
      if na_df.shape[0] == 0:
      return df
      # 申请渠道.
      if pd.isnull(na_chan):
      return df[~df.index.isin(na_df.index.values)]
      else:
      tmp_df = pd.DataFrame()
      for i in na_chan.split(','):
      tmp_df = tmp_df.append(na_df[na_df['applied_channel'].apply(lambda x: i in x)])
      na_df = tmp_df
      if na_df.shape[0] == 0:
      return df
      return df[~df.index.isin(na_df.index.values)]
      def mk_test(self, x, alpha=0.01):
      '''
      MK test.
      ......@@ -327,16 +273,9 @@ class VLMMonitor:
      '''
      # 过滤空跑数据.
      df_copy = self.merge_data[['applied_at', 'applied_type', 'applied_channel', field]].copy()
      df_copy = self.filter_data(df_copy, field)
      if df_copy.shape[0] == 0:
      print('%s还在空跑.' % self.field_query_name_dict[field])
      return None
      # 申请类型过滤.
      tmp_df = pd.DataFrame()
      for i in self.field_query_app_type_dict[field]:
      tmp_df = tmp_df.append(df_copy[df_copy['applied_type'] == int(i)])
      df_copy = tmp_df
      # 收集覆盖客群.
      user_group_dict = {} # user_group_dict = {'首申-融360': (1, 融360)}
      main_user_group_dict = {}
      ......@@ -501,10 +440,8 @@ class VLMMonitor:
      idx = self.field_query_list.index(field)
      self.field_query_list.pop(idx)
      self.field_DB_list.pop(idx)
      self.field_app_type_list.pop(idx)
      self.field_name_list.pop(idx)
      del self.field_query_name_dict[field]
      del self.field_query_app_type_dict[field]
      if real_mysql_field:
      self.mysql_df = self.query_mysql('''SELECT order_no, applied_at, \
      applied_from, applied_channel, passdue_day, %s \
      ......@@ -600,8 +537,10 @@ class VLMMonitor:
      print('不包含以下字段:')
      for field in na_field_list:
      print(self.field_query_name_dict[field])
      self.field_query_list.remove(field)
      self.field_name_list.remove(self.field_query_name_dict[field])
      idx = self.field_query_list.index(field)
      self.field_query_list.pop(idx)
      self.field_DB_list.pop(idx)
      self.field_name_list.pop(idx)
      del self.field_query_name_dict[field]
      # 处理数据.
      ......
      model_score.xlsx
      View file @ 9753fa69
      No preview for this file type
      Markdown is supported
      0% or
      You are about to add 0 people to the discussion. Proceed with caution.
      Finish editing this message first!
      Please register or to comment