反射优化

J_feature/test.ipynb

@@ -74,8 +74,8 @@
       ]
      },
      "execution_count": 17,
-     "metadata": {},
-     "output_type": "execute_result"
+     "output_type": "execute_result",
+     "metadata": {}
     }
    ],
    "source": [
@@ -111,8 +111,8 @@
       ]
      },
      "execution_count": 14,
-     "metadata": {},
-     "output_type": "execute_result"
+     "output_type": "execute_result",
+     "metadata": {}
     }
    ],
    "source": [
@@ -131,8 +131,8 @@
       ]
      },
      "execution_count": 20,
-     "metadata": {},
-     "output_type": "execute_result"
+     "output_type": "execute_result",
+     "metadata": {}
     }
    ],
    "source": [
@@ -149,7 +149,7 @@
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 3
+    "version": 3.0
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
@@ -161,4 +161,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 2
-}
+}
\ No newline at end of file

config/settings.py

@@ -11,7 +11,7 @@ url_new_audit = '/features/new_audit'
 
 url_calc_features = '/calc/features' # 基础特征
 url_complex_order = '/features/complex_order' # 多订单
+# url_collection = '/calc/loanPostFeatures' # 催收模型
 url_collection = '/calc/loanPostFeatures' # 催收模型
-url_collection_test = '/calc/loanPostFeatures_test' # 催收模型
 
 

features/Base_Features.py

@@ -5,31 +5,31 @@ import pandas as pd
 from collections import defaultdict
 
 class BaseFeatures():
+    df_subclasses = pd.DataFrame(CONFIG_FEATURES.subclasses)
+    df_subclasses['package_filename'] = df_subclasses['package'] +'.'+ df_subclasses['filename']
 
-    def __init__(self):
-        self.df_subclasses = pd.DataFrame(CONFIG_FEATURES.subclasses)  # 反射关系表
-        self.df_subclasses['package_filename'] = self.df_subclasses['package'] +'.'+ self.df_subclasses['filename']
-        self._result = defaultdict(dict)
+    @classmethod
+    def get_subclasses(cls,featureName,mapping):
+        ''' 类反射 '''
 
-    def feature(self):
-        """子类实现具体特征逻辑"""
-        pass
+        BaseFeatures._result = defaultdict(dict)
+        codes = ','.join(cls.codes).split(',')
 
-    def get_subclasses(self,featureName,mapping):
-        ''' 类反射 '''
-        codes = ','.join(self.codes).split(',')
+        # df_subclasses = pd.DataFrame(CONFIG_FEATURES.subclasses)  # 反射关系表
+        # df_subclasses['package_filename'] = df_subclasses['package'] +'.'+ df_subclasses['filename']
 
         path = featureName.split('.')  # 获取文件路径
-        codes = self.get_mappingTosubclasses(path,codes,mapping,self._result)
-        not_codes = self.df_subclasses.loc[(self.df_subclasses['package'] ==featureName)]['class'].tolist()
-        self.df_subclasses = self.df_subclasses.loc[(self.df_subclasses['package'] ==featureName)& (self.df_subclasses['class'].isin(codes))]
+        codes = cls.get_mappingTosubclasses(path,codes,mapping,cls._result)
+        not_codes = cls.df_subclasses.loc[(cls.df_subclasses['package'] ==featureName)]['class'].tolist()
+        df_subclasses = cls.df_subclasses.loc[(cls.df_subclasses['package'] ==featureName)& (cls.df_subclasses['class'].isin(codes))]
 
         try :
-            _package = self.df_subclasses['package_filename'].tolist()
-            _class = self.df_subclasses['class'].tolist()
+            _package = df_subclasses['package_filename'].tolist()
+            _class = df_subclasses['class'].tolist()
             def get_attr(package,code,result):
                 """ 递归引入特征库,返回输出结果 """
                 lens = len(code)
+                print(lens)
                 if lens:
                     try:
                         m_py = importlib.import_module(package[0])
@@ -38,30 +38,33 @@ class BaseFeatures():
                     else:
                         if hasattr(m_py,code[0]):
                             obj_class = getattr(m_py,code[0])
-                            result.update(dict(obj_class.feature(self)))
+                            result.update(dict(obj_class.feature(BaseFeatures)))
                         else:
                             result.update({code[0]:{'value':'code 值不存在'}})
                     del code[0]
                     del package[0]
                     get_attr(package,code,result)
-            get_attr(_package,_class,self._result)
-            self._result = dict(self._result)
+                else:
+                    return
+
+            get_attr(_package,_class,BaseFeatures._result)
+            BaseFeatures._result = dict(BaseFeatures._result)
 
             diff_codes = list(set(list(codes)).difference(set(not_codes)))
             if diff_codes:
                 for code in diff_codes:
-                    self._result.update({code:{'value':'code 值不存在'}})
+                    BaseFeatures._result.update({code:{'value':'code 值不存在'}})
             # for n in codes:
             #     if n not in not_codes:
             #         self._result.update({n:{'value':'code 值不存在'}})
 
         except Exception as e:
-            return self._result.update({'错误信息: ','检查文件路径: {0}  和codes {1}'.format(path[0],path[1])})
-
-        self._result = self.mapping._set_Mapping_rename(mapping,self._result)
-        return self._result
+            return BaseFeatures._result.update({'错误信息: ','检查文件路径: {0}  和codes {1}'.format(path[0],path[1])})
 
-    def get_mappingTosubclasses(self,featureName,codes,mapping,result):
+        BaseFeatures._result = cls.mapping._set_Mapping_rename(mapping,BaseFeatures._result)
+        return BaseFeatures._result
+    @classmethod
+    def get_mappingTosubclasses(cls,featureName,codes,mapping,result):
         """将传入的特征名称转为代码存储的类名"""
 
         for key,value in mapping.items():
@@ -93,9 +96,10 @@ class BaseFeatures():
     def set_mapping(self,mapping):
         self.mapping = mapping
 
-    def set_private_var(self,**kwargs):
+    @classmethod
+    def set_private_var(cls,**kwargs):
         """动态生成变量并赋值"""
         for key ,value in kwargs.items():
-            setattr(self,key,value)
+            setattr(cls,key,value)
 
 

features/CONFIG_FEATURES.py

@@ -8,11 +8,7 @@ subclasses = [
         'class':'user_apply_refuse_cnt'
     },
 
-    {
-        'package':'features.calc_features',
-        'filename':'complexOrder',
-        'class':'current_loan_order_nums'
-    },
+
 
     {
         'package':'features.calc_features',
@@ -220,6 +216,20 @@ subclasses = [
         'class':'ind_curr_max_sumDelqAmt_rct_6mon'
     },
 
+    {
+        'package':'features.calc_features',
+        'filename':'complexOrder',
+        'class':'current_loan_order_nums'
+    },
 
-
+    {
+        'package':'features.calc_features',
+        'filename':'complexOrder',
+        'class':'current_loan_max_term'
+    },
+    {
+        'package':'features.calc_features',
+        'filename':'complexOrder',
+        'class':'current_loan_term_nums'
+    },
 ]
\ No newline at end of file

features/calc_features/complexOrder.py

+from features.Base_Features import BaseFeatures as BF
+from service.Sql_server import *
+from functools import reduce
+import datetime
+from dateutil.relativedelta import relativedelta
+
+
+class __INIT__(BF):
+    def __init__(self):
+        BF.df = get_sql_user_in_loan(self.user_id)
+        pass
+
+__INIT__()
+
+
+class current_loan_order_nums():
+    """当前在贷订单数"""
+    def feature(self):
+        current_loan_order_nums = len(self.df.loc[self.df.duplicated('loan_id',keep='first')== False])
+        self._result["current_loan_order_nums"]['value'] = current_loan_order_nums
+        return self._result
+
+class current_loan_term_nums():
+    """当前在贷总期数"""
+    def feature(self):
+
+        current_loan_term_nums = self.df.loc[self.df['is_zaidai'] == 1]['is_zaidai'].count()
+        self._result["current_loan_term_nums"]['value'] = int(current_loan_term_nums)
+        return self._result
+
+class current_loan_max_term():
+    """当前在贷最大期数"""
+    def feature(self):
+        current_loan_max_term = self.df.groupby('loan_id')[['is_zaidai']].agg(lambda x: reduce(lambda x,y: x+y,x)).reset_index()['is_zaidai'].max()
+        self._result["current_loan_max_term"]['value'] = int(current_loan_max_term)
+        return self._result
+
+
+
+
+
+
+
+
+
+

features/calc_features/primary.py

@@ -3,13 +3,18 @@ from features.Base_Features import BaseFeatures
 from collections import defaultdict
 from service.Sql_server import *
 
+from features.Base_Features import BaseFeatures as BF
+from service.Sql_server import *
+
+
+class __INIT__(BF):
+    def __init__(self):
+        pass
+__INIT__()
 
-class user_apply_refuse_cnt(BaseFeatures):
+class user_apply_refuse_cnt():
     def feature(self):
 
-        if self.user_id == '':
-            return None
-        _result = defaultdict(dict)
 
         #历史最大逾期天数 : 用户申请拒绝次数(一单一审放款拒绝和循环额度申请拒绝)
 
@@ -17,8 +22,8 @@ class user_apply_refuse_cnt(BaseFeatures):
         _df_loan_refuse_cnt_new = get_sql_loan_refuse_cnt_new(self.user_id)
 
         user_apply_refuse_cnt = _df_loan_refuse_cnt_old['count'].values[0] + _df_loan_refuse_cnt_new['count'].values[0]
-        _result['user_apply_refuse_cnt']['value'] = user_apply_refuse_cnt
-        return dict(_result)
+        self._result['user_apply_refuse_cnt']['value'] = user_apply_refuse_cnt
+        return self._result
 
 
 

handler/Base_Handler.py

@@ -50,16 +50,16 @@ class BaseHandler(tornado.web.RequestHandler):
             self.flush()
             self.finish()
 
-
-        self.feature_init = BaseFeatures()
-        self.feature_init.set_business_type(self._business_type)
-        self.feature_init.set_user_uuid(self._uuid)
-        self.feature_init.set_user_id(self._user_id)
-        self.feature_init.set_loan_id(self._loanId)
-        self.feature_init.set_term_no(self._termNo)
-        self.feature_init.set_apply_time(self._apply_time)
-        self.feature_init.set_codes(self._codes)
-        self.feature_init.set_mapping(self.mapping)
+        #
+        # self.feature_init = BaseFeatures()
+        # self.feature_init.set_business_type(self._business_type)
+        # self.feature_init.set_user_uuid(self._uuid)
+        # self.feature_init.set_user_id(self._user_id)
+        # self.feature_init.set_loan_id(self._loanId)
+        # self.feature_init.set_term_no(self._termNo)
+        # self.feature_init.set_apply_time(self._apply_time)
+        # self.feature_init.set_codes(self._codes)
+        # self.feature_init.set_mapping(self.mapping)
 
 
 

handler/Calc_Features_Handler.py

 # -*- coding:utf-8 -*-
 from handler.Base_Handler import BaseHandler
-
 from utils import JsonUtil
-
-
 import importlib
+from features.Base_Features import BaseFeatures
+
 
 class CalcFeatures(BaseHandler): #feature_基础特征
     def post(self):
         try:
 
-            _result = self.feature_init.get_subclasses('features.calc_features',self.mapping.Mapping_Basics)
+
+            BaseFeatures.set_private_var(mapping = self.mapping,
+                                         codes = self._codes,
+                                         business_type = self._business_type,
+                                         uuid=self._uuid,
+                                         user_id = self._user_id,
+                                         loanId = self._loanId,
+                                         orderId = self._orderId,
+                                         apply_time = self._apply_time,
+                                         termNo = self._termNo
+                                         )
+
+
+            _result = BaseFeatures.get_subclasses('features.calc_features',self.mapping.mapping_all)
 
             self.write(JsonUtil.build_json_feature(businessType=self._business_type, orderId=self._orderId, loanId=self._loanId,
                                                    userUuid = self._uuid,user_id=self._user_id,term_no = self._termNo,

handler/ComplexOrder_Handler.py

@@ -12,16 +12,27 @@ class ComplexOrder(BaseHandler): # 多订单的新特征接口
             _uuid = self.get_argument('user_uuid', default=None)
             _orderId = self.get_argument('order_id', default=None)
             _loanId = self.get_argument('loan_id', default=None)
-            _user_id = int(get_sql_user_id(_uuid))
+            df_user = get_sql_user_id(_uuid)
+            if df_user.empty == False:
+                self._user_id = int(df_user['user_id'].values[0])
+            else :
+                self.write(JsonUtil.build_json_feature(businessType=int(self._business_type), orderId=self._orderId,
+                                                       loanId=int(self._loanId),
+                                                       userUuid = self._uuid,term_no = int(self._termNo),
+                                                       massage = "没有找到对应的用户ID",
+                                                       code=JsonUtil.Constants.Code_Error_Value))
+                self.flush()
+
 
             columns = {}
             for key,value in self.mapping.Mapping_Complex_order.items():
                 columns[key] = value[1]
 
-            _result = complex_order_Features.f_complexOrder(_user_id, columns)
+            _result = complex_order_Features.f_complexOrder(self._user_id, columns)
             _result = self.mapping._set_Mapping_rename(self.mapping.Mapping_Complex_order,_result)
 
-            self.write(JsonUtil.build_json_feature(businessType=int(_business_type), orderId=_orderId, loanId=_loanId,userUuid = _uuid,user_id=_user_id,
+            self.write(JsonUtil.build_json_feature(businessType=int(_business_type), orderId=_orderId,
+                                                   loanId=_loanId,userUuid = _uuid,user_id=self._user_id,
                                                    features=_result, flag=True,code=JsonUtil.Constants.Code_Success))
 
             self.flush()
@@ -39,7 +50,11 @@ class ComplexOrder(BaseHandler): # 多订单的新特征接口
 
 
 
-
+# class ComplexOrder(BaseHandler):
+#     def post(self):
+#         self.df = get_sql_user_in_loan(self._user_id)
+#
+#     pass
 
 
 

handler/LoanPostFeatures_Handler.py

@@ -4,55 +4,11 @@ from handler.Base_Handler import BaseHandler
 from features import loanPost_Features
 from utils import JsonUtil
 from service.Sql_server import *
+from features.Base_Features import BaseFeatures
 
 
 
 class loanPostFeatures(BaseHandler): #催收模型特征_v1 接口
-    def post(self):
-        try:
-            # _business_type = self.get_argument('business_type', default=1)
-            # _uuid = self.get_argument('user_uuid', default=None)
-            # _orderId = self.get_argument('order_id', default=None)
-            # _loanId = self.get_argument('loan_id', default=None)
-            # _termNo = self.get_argument('term_no',default=None)
-            # _user_id = int(get_sql_user_id(self._uuid))
-
-
-            if self._loanId == '' or self._termNo == '':
-                self.write(JsonUtil.build_json_feature(businessType=int(self._business_type), orderId=self._orderId,
-                                                       loanId=int(self._loanId),
-                                                       userUuid = self._uuid,user_id=int(self._user_id),term_no = int(self._termNo),
-                                                       massage = "loanid and term not find deadline",
-                                                       code=JsonUtil.Constants.Code_Error_Value))
-                self.flush()
-
-            _df = get_sql_user_loan_all(self._user_id) # 获取用户订单信息
-            _result = loanPost_Features.f_loanPostFeatures(_df, self._loanId, self._termNo, self._user_id, self._orderId)
-            _result = self.mapping._set_Mapping_rename(self.mapping.Mapping_Collection,_result)
-
-            self.write(JsonUtil.build_json_feature(mssage="",
-                                                   businessType=int(self._business_type),
-                                                   orderId=self._orderId,
-                                                   loanId=int(self._loanId),
-                                                   userUuid = self._uuid,
-                                                   user_id=self._user_id,
-                                                   term_no = int(self._termNo),
-                                                   features=_result,
-                                                   flag=True,
-                                                   code=JsonUtil.Constants.Code_Success))
-
-            self.flush()
-            self.finish()
-
-
-        except Exception as e:
-            # == 报异常，查询失败
-            self.write(JsonUtil.build_json(JsonUtil.Constants.Code_Error, JsonUtil.Constants.Msg_Error))
-            self.flush()
-        pass
-
-
-class loanPostFeatures_test(BaseHandler): #催收模型特征_v1 接口
     def post(self):
 
         try:
@@ -76,21 +32,18 @@ class loanPostFeatures_test(BaseHandler): #催收模型特征_v1 接口
                     self.flush()
 
 
-                self.df,self.df_current_loan,self.planId,self.deadLine = public_base(self._user_id,self._loanId,self._termNo)
-
-                self.df_apply = get_sql_apply(self._user_id) # 获取用户申请次数
-                self.df_apply = self.df_apply .loc[self.df_apply['apply_time'] <= self.deadLine]
-
-
-                self.feature_init.set_private_var(_df = self.df,
-                                                  user_id = self._user_id,
-                                                  df_current_loan = self.df_current_loan,
-                                                  planId = self.planId,
-                                                  deadLine = self.deadLine,
-                                                  df_apply = self.df_apply)
+                BaseFeatures.set_private_var(mapping = self.mapping,
+                                             codes = self._codes,
+                                             business_type = self._business_type,
+                                             uuid=self._uuid,
+                                             user_id = self._user_id,
+                                             loanId = self._loanId,
+                                             orderId = self._orderId,
+                                             apply_time = self._apply_time,
+                                             termNo = self._termNo
+                                             )
 
-
-                _result = self.feature_init.get_subclasses('features.calc_loanPostFeatures',self.mapping.Mapping_Collection)
+                _result = BaseFeatures.get_subclasses('features.calc_loanPostFeatures',self.mapping.mapping_all)
 
                 self.write(JsonUtil.build_json_feature(mssage="",
                                                        businessType=int(self._business_type),
@@ -113,40 +66,3 @@ class loanPostFeatures_test(BaseHandler): #催收模型特征_v1 接口
             self.write(JsonUtil.build_json(JsonUtil.Constants.Code_Error, JsonUtil.Constants.Msg_Error))
             self.flush()
         pass
-
-import datetime
-def public_base(user_id,loanId = '',termNo = ''):
-
-
-    if loanId == '' or termNo == '':
-        return None
-
-    df = get_sql_user_loan_all(user_id)
-    df['deadline_new'] = df['deadline_new'].apply(lambda x : datetime.datetime.strftime(x,'%Y-%m-%d'))
-    df['deadline'] = df['deadline'].apply(lambda x : datetime.datetime.strftime(x,'%Y-%m-%d'))
-    df['repaid_at'] = df['repaid_at'].apply(lambda x : datetime.datetime.strftime(x,'%Y-%m-%d'))
-    df['repaid_at'] = pd.to_datetime(df['repaid_at'])
-    df['deadline_new'] = pd.to_datetime(df['deadline_new'])
-    df['deadline'] = pd.to_datetime(df['deadline'])
-    df['updated_at'] = pd.to_datetime(df['updated_at'])
-
-    df_current_loan = df.loc[(df['loan_id'] == loanId)&(df['term_no'] == termNo)]
-    planId = df_current_loan['plan_id'].values[0]
-    deadLine = df_current_loan['deadline_new'].tolist()[0]
-
-    df.loc[(df['repayment_status']==3)&(df['repaid_at'] < deadLine),'passdue_day'] = (df['repaid_at'] - df['deadline']).dt.days
-    df.loc[(df['repayment_status']==3)&(df['repaid_at'] > deadLine)&(df['deadline'] < deadLine),'passdue_day'] = (deadLine - df['deadline']).dt.days
-    df.loc[(df['repayment_status']!=3)&(df['repaid_at'] > deadLine)&(df['deadline'] < deadLine),'passdue_day'] = (deadLine - df['deadline']).dt.days
-
-    df.loc[df['passdue_day']>0,'is_overdue'] = 1
-    df.loc[df['repaid_at'] > deadLine,'repayment_status'] = 0
-
-    df_one = df.loc[(df['plan_id'] <= planId)
-                      &(( df['updated_at'] < deadLine)
-                        & ( df['progress'] == 16))] # 截取
-
-    df_two = df.loc[(df['loan_id'].isin([loanId]))&(df['plan_id'] <= planId)] # 截取
-
-    df = pd.concat([df_one,df_two],axis=0)
-
-    return df,df_current_loan,planId,deadLine

service/Mapping.py

@@ -22,6 +22,11 @@ class Mapping():
 
         _columns_Collection = pd.read_excel(settings.FILE_PATH+_dataPath+'feautre_催收模型特征_v1.xlsx',sheet_name='字段对应关系')
         self.Mapping_Collection = self._get_needs(_columns_Collection)
+
+        _columns = pd.read_excel(settings.FILE_PATH+_dataPath+'feature_mapping.xlsx',sheet_name='字段对应关系')
+        self.mapping_all = self._get_needs(_columns)
+
+
         print('完成初始化 Mapping ......')
 
     def _get_needs(self,df):

startup.py

@@ -34,10 +34,7 @@ def apps():
         (URL.url_new_audit,NewAudit_Handler.New_Audit),
         (URL.url_calc_features, Calc_Features_Handler.CalcFeatures),
         (URL.url_complex_order,ComplexOrder_Handler.ComplexOrder),
-
-
         (URL.url_collection, LoanPostFeatures_Handler.loanPostFeatures),
-        (URL.url_collection_test, LoanPostFeatures_Handler.loanPostFeatures_test),
     ])
 
 if __name__ == "__main__":

零时文件-可删除.ipynb

@@ -725,6 +725,54 @@
     "    print('2')"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class A (object):\n",
+    "    a = 0\n",
+    "    @classmethod\n",
+    "    def sum(cls):\n",
+    "        print(A.a)\n",
+    "class B (A):\n",
+    "    def __init__(self):\n",
+    "        print(A.a)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "a = [1,2]\n",
+    "b = [3,4,3]\n",
+    "aa = dict(zip(a,b))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "KeyError",
+     "evalue": "0",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-5-f10e8fb487cc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0maa\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;31mKeyError\u001b[0m: 0"
+     ]
+    }
+   ],
+   "source": [
+    "aa[0]"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,