20191009

AUC_time.py

@@ -31,7 +31,7 @@ class AUCMonitor:
                  passdue_day=15, save_path='./auc/',
                  min_user_group=500, interval_days=15, min_auc=0.55,
                  date_list=('2019-03-01', '2019-03-15', '2019-03-31', '2019-04-15'),
-                 if_save=True, if_read=True):
+                 if_read=True):
 
         # 考虑到数据库配置基本不变, 所以不设置创建对象时对应输入变量.
         self.mysql_engine = pymysql.connect(host='172.20.6.9',
@@ -44,21 +44,14 @@ class AUCMonitor:
         self.mongo_client = pymongo.MongoClient(
             "mongodb://haoyue.shu:x2egwRHk7WhQ4So1@172.18.3.22:27017/?authSource=rc_mgo_feature_dp")
         self.mongo_db = self.mongo_client['rc_mgo_feature_dp']
-        self.mongo_table = self.mongo_db['wf_audit_log_with_feature']
+        self.mongo_table = self.mongo_db['wf_audit_log_with_feature_online']
 
         # 读取整理在Excel中的模型相关信息.
         self.field_info_df = pd.read_excel(excel_path, sheet_name=sheet_name)
         self.field_name_list = self.field_info_df.field_name.tolist()
         self.field_query_list = self.field_info_df.field_query.tolist()
-        self.field_app_type_list = self.field_info_df.app_type.tolist()
-        self.field_app_type_list = [str(x) for x in self.field_app_type_list]
         self.field_DB_list = self.field_info_df.DB.tolist()
         self.field_query_name_dict = dict(zip(self.field_query_list, self.field_name_list))
-        self.field_query_app_type_dict = dict(zip(self.field_query_list, self.field_app_type_list))
-        ## 空跑信息.
-        self.na_time = self.field_info_df.na_time.tolist()  # 空跑时间段
-        self.na_app_type = self.field_info_df.na_app_type.tolist()  # 空跑申请类型
-        self.na_app_chan = self.field_info_df.na_app_chan.tolist()  # 空跑渠道
 
         # 一些定义的常量
         self.passdue_day = passdue_day  # 逾期天数, 默认15.
@@ -86,7 +79,6 @@ class AUCMonitor:
         self.auc_info_df = pd.DataFrame(columns=auc_cols)
 
         # 程序数据读写模式.
-        self.if_save = if_save  # 是否保存从数据库抽取的数据.
         self.if_read = if_read  # 是否从从数据库读取.
 
         # 创建文件夹.
@@ -134,54 +126,6 @@ class AUCMonitor:
         else:
             return '0' + str(x)
 
-    def filter_data(self, df, field):
-        '''
-        过滤空跑数据.
-        :param df: df.
-        :param field: str, 字段名.
-        :return: df, 过滤后的数据.
-        '''
-        df = df[~((df['applied_type'] == 1) & (df['applied_channel'].apply(lambda x: 'Android' in x)))]
-
-        field_idx = self.field_query_list.index(field)
-        na_time = self.na_time[field_idx]
-        na_type = self.na_app_type[field_idx]
-        na_chan = self.na_app_chan[field_idx]
-        if pd.isnull(na_time):  # 没有空跑时间, 则不记录.
-            return df
-        # 时间.
-        t_s, t_e = na_time.split('~')
-        if len(t_e) == 0:  # 若还在空跑, 则不记录.
-            return pd.DataFrame()
-        else:
-            na_df = df[
-                (df['applied_at'].apply(lambda x: x[:10] >= t_s)) & (df['applied_at'].apply(lambda x: x[:10] <= t_e))]
-            if na_df.shape[0] == 0:
-                return df
-        # 申请类型.
-        if pd.isnull(na_type):
-            return df[~df.index.isin(na_df.index.values)]
-        else:
-            tmp_df = pd.DataFrame()
-
-            for i in str(int(na_type)):
-                tmp_df = tmp_df.append(na_df[na_df['applied_type'] == int(i)])
-            na_df = tmp_df
-            if na_df.shape[0] == 0:
-                return df
-        # 申请渠道.
-        if pd.isnull(na_chan):
-            return df[~df.index.isin(na_df.index.values)]
-        else:
-            tmp_df = pd.DataFrame()
-            for i in na_chan.split(','):
-                tmp_df = tmp_df.append(na_df[na_df['applied_channel'].apply(lambda x: i in x)])
-            na_df = tmp_df
-            if na_df.shape[0] == 0:
-                return df
-
-        return df[~df.index.isin(na_df.index.values)]
-
     def helper_auc(self, user_group_name=None, df=None, info_dict=None, field=None):
         '''
         信息提取函数.
@@ -226,13 +170,6 @@ class AUCMonitor:
             [field, 'date_label', 'applied_type', 'applied_channel', 'overdue', 'passdue_day', 'applied_at']].copy()
         # 筛选出放款, 且逾期表现的数据.
         df_copy = df_copy[(df_copy['overdue'].notna()) & (df_copy[field].notna())]
-        # 筛选正确申请类型的数据.
-        tmp_df = pd.DataFrame()
-        for i in self.field_query_app_type_dict[field]:
-            tmp_df = tmp_df.append(df_copy[df_copy['applied_type'] == int(i)])
-        df_copy = tmp_df
-        # 过滤空跑数据.
-        df_copy = self.filter_data(df_copy, field)
         if df_copy.shape[0] == 0:
             print('仍在空跑.')
             return None
@@ -374,14 +311,7 @@ class AUCMonitor:
                     print('在MySQL中找到该字段: %s' % self.field_query_name_dict[field])
                 else:
                     print('在MySQL中不存在该字段: %s' % self.field_query_name_dict[field])
-                    # 删除该字段.
-                    idx = self.field_query_list.index(field)
-                    self.field_query_list.pop(idx)
-                    self.field_DB_list.pop(idx)
-                    self.field_app_type_list.pop(idx)
-                    self.field_name_list.pop(idx)
-                    del self.field_query_name_dict[field]
-                    del self.field_query_app_type_dict[field]
+
             if len(real_mysql_field) > 0:
 
                 self.mysql_df = self.query_mysql('''SELECT order_no, applied_at, %s, \
@@ -392,14 +322,14 @@ class AUCMonitor:
                                                  % (','.join(real_mysql_field), self.date_list[0], self.date_list[-1]))
             else:
                 self.mysql_df = self.query_mysql('''SELECT order_no, applied_at, \
-                                                                applied_from, applied_channel, transacted, passdue_day \
-                                                                FROM risk_analysis \
-                                                                WHERE applied_at >= "%s 00:00:00" \
-                                                                AND applied_at <= "%s 00:00:00"'''
+                                            applied_from, applied_channel, transacted, passdue_day \
+                                            FROM risk_analysis \
+                                            WHERE applied_at >= "%s 00:00:00" \
+                                            AND applied_at <= "%s 00:00:00"'''
                                                  % (self.date_list[0], self.date_list[-1]))
             print('MySQL数据获取成功.')
-            if self.if_save:
-                self.mysql_df.to_csv(self.save_path + 'data/mysql_data.csv', index=False)
+            self.mysql_df.to_csv(self.save_path + 'data/mysql_data.csv', index=False)
+            self.mysql_df = pd.read_csv(self.save_path + 'data/mysql_data.csv')
         else:
             self.mysql_df = pd.read_csv(self.save_path + 'data/mysql_data.csv')
 
@@ -420,8 +350,8 @@ class AUCMonitor:
             self.mongo_df['applied_type'] = self.mongo_df['wf_loan_type'].apply(func_0)
             del self.mongo_df['wf_loan_type']
             print('MongoDB数据获取成功.')
-            if self.if_save:
-                self.mongo_df.to_csv(self.save_path + 'data/mongo_data.csv', index=False)
+            self.mongo_df.to_csv(self.save_path + 'data/mongo_data.csv', index=False)
+            self.mongo_df = pd.read_csv(self.save_path + 'data/mongo_data.csv')
         else:
             self.mongo_df = pd.read_csv(self.save_path + 'data/mongo_data.csv')
         self.mongo_df = self.mongo_df.loc[self.mongo_df['applied_type'].notna()]
@@ -453,7 +383,11 @@ class AUCMonitor:
 
         # 清洗数据.
         def clean_data(data):
+            if pd.isnull(data):
+                return np.nan
             try:
+                if data <= 0 or data > 999999:
+                    return np.nan
                 return float(data)
             except:
                 return np.nan
@@ -469,8 +403,10 @@ class AUCMonitor:
         print('不包含以下字段:')
         for field in na_field_list:
             print(self.field_query_name_dict[field])
-            self.field_query_list.remove(field)
-            self.field_name_list.remove(self.field_query_name_dict[field])
+            idx = self.field_query_list.index(field)
+            self.field_query_list.pop(idx)
+            self.field_DB_list.pop(idx)
+            self.field_name_list.pop(idx)
             del self.field_query_name_dict[field]
 
         # 数据按时间划分.

PSI_time.py

@@ -45,21 +45,14 @@ class PSIMonitor:
         self.mongo_client = pymongo.MongoClient(
             "mongodb://haoyue.shu:x2egwRHk7WhQ4So1@172.18.3.22:27017/?authSource=rc_mgo_feature_dp")
         self.mongo_db = self.mongo_client['rc_mgo_feature_dp']
-        self.mongo_table = self.mongo_db['wf_audit_log_with_feature']
+        self.mongo_table = self.mongo_db['wf_audit_log_with_feature_online']
 
         # 读取整理在Excel中的模型相关信息.
         self.field_info_df = pd.read_excel(excel_path, sheet_name=sheet_name)
         self.field_name_list = self.field_info_df.field_name.tolist()
         self.field_query_list = self.field_info_df.field_query.tolist()
-        self.field_app_type_list = self.field_info_df.app_type.tolist()
-        self.field_app_type_list = [str(x) for x in self.field_app_type_list]
         self.field_DB_list = self.field_info_df.DB.tolist()
         self.field_query_name_dict = dict(zip(self.field_query_list, self.field_name_list))
-        self.field_query_app_type_dict = dict(zip(self.field_query_list, self.field_app_type_list))
-        ## 空跑信息.
-        self.na_time = self.field_info_df.na_time.tolist()  # 空跑时间段
-        self.na_app_type = self.field_info_df.na_app_type.tolist()  # 空跑申请类型
-        self.na_app_chan = self.field_info_df.na_app_chan.tolist()  # 空跑渠道
 
         # 一些定义的常量
         self.min_user_group = min_user_group * interval_days / 30  # 最小客群数量.
@@ -166,54 +159,6 @@ class PSIMonitor:
         except:
             return None
 
-    def filter_data(self, df, field):
-        '''
-        过滤空跑数据.
-        :param df: df.
-        :param field: str, 字段名.
-        :return: df, 过滤后的数据.
-        '''
-        df = df[~((df['applied_type'] == 1) & (df['applied_channel'].apply(lambda x: 'Android' in x)))]
-
-        field_idx = self.field_query_list.index(field)
-        na_time = self.na_time[field_idx]
-        na_type = self.na_app_type[field_idx]
-        na_chan = self.na_app_chan[field_idx]
-        if pd.isnull(na_time):  # 没有空跑时间, 则不记录.
-            return df
-        # 时间.
-        t_s, t_e = na_time.split('~')
-        if len(t_e) == 0:  # 若还在空跑, 则不记录.
-            return pd.DataFrame()
-        else:
-            na_df = df[
-                (df['applied_at'].apply(lambda x: x[:10] >= t_s)) & (df['applied_at'].apply(lambda x: x[:10] <= t_e))]
-            if na_df.shape[0] == 0:
-                return df
-        # 申请类型.
-        if pd.isnull(na_type):
-            return df[~df.index.isin(na_df.index.values)]
-        else:
-            tmp_df = pd.DataFrame()
-
-            for i in str(int(na_type)):
-                tmp_df = tmp_df.append(na_df[na_df['applied_type'] == int(i)])
-            na_df = tmp_df
-            if na_df.shape[0] == 0:
-                return df
-        # 申请渠道.
-        if pd.isnull(na_chan):
-            return df[~df.index.isin(na_df.index.values)]
-        else:
-            tmp_df = pd.DataFrame()
-            for i in na_chan.split(','):
-                tmp_df = tmp_df.append(na_df[na_df['applied_channel'].apply(lambda x: i in x)])
-            na_df = tmp_df
-            if na_df.shape[0] == 0:
-                return df
-
-        return df[~df.index.isin(na_df.index.values)]
-
     def helper_psi(self, user_group_name=None, df=None, info_dict=None, field=None):
         '''
         信息提取函数.
@@ -265,14 +210,7 @@ class PSIMonitor:
 
         # 分离数据.
         df_copy = self.merge_data[[field, 'date_label', 'applied_type', 'applied_channel', 'applied_at']].copy()
-        # 选择包含正确申请类型的数据.
-        tmp_df = pd.DataFrame()
-        for i in self.field_query_app_type_dict[field]:
-            tmp_df = tmp_df.append(df_copy[df_copy['applied_type'] == int(i)])
-        df_copy = tmp_df
         df_copy = df_copy[df_copy[field].notna()]
-        # 过滤空跑数据.
-        df_copy = self.filter_data(df_copy, field)
         if df_copy.shape[0] == 0:
             print('仍在空跑.')
             return None
@@ -399,10 +337,8 @@ class PSIMonitor:
                     idx = self.field_query_list.index(field)
                     self.field_query_list.pop(idx)
                     self.field_DB_list.pop(idx)
-                    self.field_app_type_list.pop(idx)
                     self.field_name_list.pop(idx)
                     del self.field_query_name_dict[field]
-                    del self.field_query_app_type_dict[field]
             if len(real_mysql_field) > 0:
                 self.mysql_df = self.query_mysql('''SELECT order_no, applied_at, %s, \
                                                 applied_from, applied_channel, transacted, passdue_day \
@@ -480,8 +416,10 @@ class PSIMonitor:
         print('不包含以下字段:')
         for field in na_field_list:
             print(self.field_query_name_dict[field])
-            self.field_query_list.remove(field)
-            self.field_name_list.remove(self.field_query_name_dict[field])
+            idx = self.field_query_list.index(field)
+            self.field_query_list.pop(idx)
+            self.field_DB_list.pop(idx)
+            self.field_name_list.pop(idx)
             del self.field_query_name_dict[field]
 
         # 数据按时间划分.

VLM_time.py

@@ -48,21 +48,14 @@ class VLMMonitor:
         self.mongo_client = pymongo.MongoClient(
             "mongodb://haoyue.shu:x2egwRHk7WhQ4So1@172.18.3.22:27017/?authSource=rc_mgo_feature_dp")
         self.mongo_db = self.mongo_client['rc_mgo_feature_dp']
-        self.mongo_table = self.mongo_db['wf_audit_log_with_feature']
+        self.mongo_table = self.mongo_db['wf_audit_log_with_feature_online']
 
         # 读取整理在Excel中的模型相关信息.
         self.field_info_df = pd.read_excel(excel_path, sheet_name=sheet_name)
         self.field_name_list = self.field_info_df.field_name.tolist()
         self.field_query_list = self.field_info_df.field_query.tolist()
-        self.field_app_type_list = self.field_info_df.app_type.tolist()
-        self.field_app_type_list = [str(x) for x in self.field_app_type_list]
         self.field_DB_list = self.field_info_df.DB.tolist()
         self.field_query_name_dict = dict(zip(self.field_query_list, self.field_name_list))
-        self.field_query_app_type_dict = dict(zip(self.field_query_list, self.field_app_type_list))
-        ## 空跑信息.
-        self.na_time = self.field_info_df.na_time.tolist()  # 空跑时间段
-        self.na_app_type = self.field_info_df.na_app_type.tolist()  # 空跑申请类型
-        self.na_app_chan = self.field_info_df.na_app_chan.tolist()  # 空跑渠道
 
         # 文件存储路径.
         self.fig_save_path = save_path + 'image/'
@@ -143,53 +136,6 @@ class VLMMonitor:
 
         pass
 
-    def filter_data(self, df, field):
-        '''
-        过滤空跑数据.
-        :param df: df.
-        :param field: str, 字段名.
-        :return: df, 过滤后的数据.
-        '''
-        df = df[~((df['applied_type'] == 1) & (df['applied_channel'].apply(lambda x: 'Android' in x)))]
-        field_idx = self.field_query_list.index(field)
-        na_time = self.na_time[field_idx]
-        na_type = self.na_app_type[field_idx]
-        na_chan = self.na_app_chan[field_idx]
-        if pd.isnull(na_time):  # 没有空跑时间, 则不记录.
-            return df
-        # 时间.
-        t_s, t_e = na_time.split('~')
-        if len(t_e) == 0:  # 若还在空跑, 则不记录.
-            return pd.DataFrame()
-        else:
-            na_df = df[
-                (df['applied_at'].apply(lambda x: x[:10] >= t_s)) & (df['applied_at'].apply(lambda x: x[:10] <= t_e))]
-            if na_df.shape[0] == 0:
-                return df
-        # 申请类型.
-        if pd.isnull(na_type):
-            return df[~df.index.isin(na_df.index.values)]
-        else:
-            tmp_df = pd.DataFrame()
-
-            for i in str(int(na_type)):
-                tmp_df = tmp_df.append(na_df[na_df['applied_type'] == int(i)])
-            na_df = tmp_df
-            if na_df.shape[0] == 0:
-                return df
-        # 申请渠道.
-        if pd.isnull(na_chan):
-            return df[~df.index.isin(na_df.index.values)]
-        else:
-            tmp_df = pd.DataFrame()
-            for i in na_chan.split(','):
-                tmp_df = tmp_df.append(na_df[na_df['applied_channel'].apply(lambda x: i in x)])
-            na_df = tmp_df
-            if na_df.shape[0] == 0:
-                return df
-
-        return df[~df.index.isin(na_df.index.values)]
-
     def mk_test(self, x, alpha=0.01):
         '''
         MK test.
@@ -327,16 +273,9 @@ class VLMMonitor:
         '''
         # 过滤空跑数据.
         df_copy = self.merge_data[['applied_at', 'applied_type', 'applied_channel', field]].copy()
-        df_copy = self.filter_data(df_copy, field)
         if df_copy.shape[0] == 0:
             print('%s还在空跑.' % self.field_query_name_dict[field])
             return None
-        # 申请类型过滤.
-        tmp_df = pd.DataFrame()
-
-        for i in self.field_query_app_type_dict[field]:
-            tmp_df = tmp_df.append(df_copy[df_copy['applied_type'] == int(i)])
-        df_copy = tmp_df
         # 收集覆盖客群.
         user_group_dict = {}  # user_group_dict = {'首申-融360': (1, 融360)}
         main_user_group_dict = {}
@@ -501,10 +440,8 @@ class VLMMonitor:
                     idx = self.field_query_list.index(field)
                     self.field_query_list.pop(idx)
                     self.field_DB_list.pop(idx)
-                    self.field_app_type_list.pop(idx)
                     self.field_name_list.pop(idx)
                     del self.field_query_name_dict[field]
-                    del self.field_query_app_type_dict[field]
             if real_mysql_field:
                 self.mysql_df = self.query_mysql('''SELECT order_no, applied_at, \
                                                    applied_from, applied_channel, passdue_day, %s \
@@ -600,8 +537,10 @@ class VLMMonitor:
         print('不包含以下字段:')
         for field in na_field_list:
             print(self.field_query_name_dict[field])
-            self.field_query_list.remove(field)
-            self.field_name_list.remove(self.field_query_name_dict[field])
+            idx = self.field_query_list.index(field)
+            self.field_query_list.pop(idx)
+            self.field_DB_list.pop(idx)
+            self.field_name_list.pop(idx)
             del self.field_query_name_dict[field]
 
         # 处理数据.