11月份录入对账数据

现金贷/数据准备/1_1_1_现金贷跨月放款退款.py

+# -*- coding: utf-8 -*-
+# author WuKong
+import pandas as pd
+from sqlalchemy import create_engine
+import numpy as np
+import openpyxl as px
+from os import listdir, sys
+from os.path import isfile, join, splitext, basename
+
+reload(sys)
+sys.setdefaultencoding('utf8')
+
+engine_audit = create_engine('mysql+mysqldb://bo.wang:#mn89453YU9io80@172.16.3.201:3306/audit?charset=utf8', echo=True)
+engine_xyqb = create_engine('mysql+mysqldb://bowang:in78u6ytgbhj56r@192.168.4.6:8066/xyqb?charset=utf8', echo=True)
+engine_paycenter = create_engine('mysql+mysqldb://bowang:in78u6ytgbhj56r@192.168.4.6:8066/payment_center?charset=utf8',
+                                 echo=True)
+fund_loan_account_dict = {0: 1, 1: 3, 2: 1, 3: 1, 4: 1, 5: 1, 29: 29, 50: 1, 70: 1, 80: 1, 90: 90, 100: 100, 110: 1,
+                          120: 1, 130: 1, 135: 2, 140: 140, 141: 1, 150: 150, 160: 160, 170: 2, 180: 180, 190: 2,
+                          200: 200, 210: 210, 230: 230, 240: 240, 250: 2, 251: 2, 260: 2, 270: 6, 290: 290, 300: 300,
+                          310: 310, 320: 311, 330: 330, 340: 340, 350: 350}
+
+fund_source_fund_dict = {330: 210}
+
+sql_query_loan_manifest = """
+SELECT * FROM loan_manifest WHERE ref_id in (%s);
+"""
+
+sql_payment_center = """
+SELECT order_id, merchant_order_no loan_id FROM pay_order WHERE  order_id IN %s
+"""
+
+
+sql_audit_query_max_ref_id = """select max(ref_id)+1 'max_ref_id' from loan_manifest;"""
+
+
+ref_id_list = ["6165354","6165355","6165356","6165357","6165358","6165359","6165360","6165361","6165362","6165363","6165364","6165365","6165366","6165367","6165368","6165369","6165370","6165371","6165372","6165373","6165374","6165375","6165376","6165377","6165378","6165379","6165380","6165381","6165382","6165383","6165384","6165385","6165386","6165387","6165388","6165389","6165390","6165391","6165392","6165393","6165394","6165395","6165396","6165397","6165398","6165399","6165400","6165401","6165402","6165403","6165404","6165405","6165406","6165407","6165408","6165409","6165410","6165411","6165412","6165413","6165414","6165415","6165416","6165417","6165418","6165419","6165420","6165421","6165422","6165423","6165424","6165425"]
+
+
+path = u"""C:\\Users\\wangbo\\Desktop\\11月份对账\\放款数据\\九月份跨月退款.xlsx"""
+df_refund_data = pd.read_excel(path)
+order_id_list = df_refund_data.order_id.astype(str).tolist()
+df_pay_center = pd.read_sql(sql=sql_payment_center % str(tuple(order_id_list)),con=engine_paycenter)
+df_pay_center.order_id = df_pay_center.order_id.astype(str)
+df_refund_data.order_id = df_refund_data.order_id.astype(str)
+df_merge = pd.merge(df_pay_center,df_refund_data,on=['order_id'],how='inner')
+
+
+df_manifest = pd.read_sql(sql=sql_query_loan_manifest % ("'" + "','".join(ref_id_list) + "'"), con=engine_audit)
+df_manifest.drop(['ref_id','loan_paid_at','id'], inplace=True, axis=1)
+df_manifest['contract_loan_amount']= -df_manifest['contract_loan_amount']
+df_manifest['real_loan_amount']= -df_manifest['real_loan_amount']
+df_manifest['per_term_service_fee']= -df_manifest['per_term_service_fee']
+df_manifest['qg_one_time_service_fee']= -df_manifest['qg_one_time_service_fee']
+df_manifest['funding_one_time_service_fee']= -df_manifest['funding_one_time_service_fee']
+df_manifest['monthly_required_repayment_amount']= -df_manifest['monthly_required_repayment_amount']
+
+df_manifest = pd.merge(df_manifest,df_merge[['loan_id','loan_paid_at']],on=['loan_id'],how='inner')
+df_audit_max_ref_id = pd.read_sql(sql=sql_audit_query_max_ref_id, con=engine_audit)
+max_ref_id = df_audit_max_ref_id['max_ref_id'][0]
+if max_ref_id == None:
+    max_ref_id = 1
+df_manifest.insert(1, 'ref_id', range(max_ref_id, max_ref_id + len(df_manifest)))
+df_manifest.to_sql(name='loan_manifest', con=engine_audit, if_exists='append', index=None,chunksize=10000)
+print "finish insert loan_manifest"

现金贷/数据准备/1_1_现金贷放款导入.py

@@ -10,7 +10,7 @@ from os.path import isfile, join, splitext, basename
 reload(sys)
 sys.setdefaultencoding('utf8')
 
-engine_audit = create_engine('mysql+mysqldb://yulong:ohVU1nPITynSZwR2@172.16.3.201:3306/audit?charset=utf8', echo=True)
+engine_audit = create_engine('mysql+mysqldb://bo.wang:#mn89453YU9io80@172.16.3.201:3306/audit?charset=utf8', echo=True)
 engine_xyqb = create_engine('mysql+mysqldb://bowang:in78u6ytgbhj56r@192.168.4.6:8066/xyqb?charset=utf8', echo=True)
 engine_paycenter = create_engine('mysql+mysqldb://bowang:in78u6ytgbhj56r@192.168.4.6:8066/payment_center?charset=utf8',
                                  echo=True)
@@ -21,11 +21,14 @@ fund_loan_account_dict = {0: 1, 1: 3, 2: 1, 3: 1, 4: 1, 5: 1, 29: 29, 50: 1, 70:
 
 fund_source_fund_dict = {330: 210}
 '''
-17406567
+18240103
 SELECT MAX(id) FROM loan_manifest ;
-5919163
+6335931
 SELECT MAX(ref_id) FROM loan_manifest;
 '''
+# UPDATE loan_manifest SET loan_account_id=309 WHERE funding_code=270 AND loan_account_id=270;
+# UPDATE loan_manifest SET loan_account_id=311 WHERE funding_code=320 AND loan_account_id=320;
+# transaction_status in (2,5) AND
 sql_xyqb_loan_manifest = """SELECT loan_application_history_id "loan_id",funding_corp_id "funding_code",contract_loan_amount "contract_loan_amount", real_loan_amount "real_loan_amount", contract_term "contract_term",
 	monthly_interest_rate "monthly_interest_rate",
 	service_fee_per_term "per_term_service_fee",
@@ -39,7 +42,7 @@ sql_xyqb_loan_manifest = """SELECT loan_application_history_id "loan_id",funding
 	company_income "funding_one_time_service_fee"
 FROM
 	loan_application_manifest_history
-WHERE transaction_status in (2,5) AND
+WHERE
 	loan_application_history_id IN (%s)"""
 
 sql_xyqb_waiting_funding = """
@@ -55,17 +58,17 @@ SELECT order_id, merchant_order_no loan_id FROM pay_order WHERE  order_id IN ("1
 """
 # 跨月退款 不考虑放款状态
 sql_payment_center_ignore = """
-SELECT order_id, merchant_order_no loan_id FROM pay_order WHERE  order_id IN (%s)
+SELECT order_id, merchant_order_no loan_id FROM pay_order WHERE pay_status!=3 AND order_id IN (%s)
 """
 
 sql_audit_query_max_ref_id = """select max(ref_id)+1 'max_ref_id' from loan_manifest;"""
 
 sql_audit_query_loan_id = """SELECT id,loan_id FROM loan_manifest WHERE loan_id in (%s)"""
 
-# dir_path = "E:/loan_test/"
-dir_path = u"C:\\Users\\wangbo\\Desktop\\十月份对账\\放款数据\\处理数据"
+dir_path = u"C:\\Users\\wangbo\\Desktop\\11月份对账\\放款数据\\handle_fundcorp\\"
 # 晋商2 300 company_income
 # 债转
+# 6684643 SELECT max(ref_id) FROM loan_manifest;
 allfiles = [f for f in listdir(dir_path) if isfile(join(dir_path, f))]
 for file_name in allfiles:
     print "start handle file_name=%s" % file_name
@@ -92,8 +95,8 @@ for file_name in allfiles:
             df_pay_center = pd.read_sql(sql_payment_center_ignore % order_id_query, con=engine_paycenter)
         df.order_id = df.order_id.astype(str)
         df_pay_center.order_id = df_pay_center.order_id.astype(str)
-        df = pd.merge(df, df_pay_center, how='left', on='order_id')
-        # df = pd.merge(df, df_pay_center, how='inner', on='order_id')
+        # df = pd.merge(df, df_pay_center, how='left', on='order_id')
+        df = pd.merge(df, df_pay_center, how='inner', on='order_id')
         df.fillna(0, inplace=True)
         df.loan_id = df.loan_id.astype(int)
         print "放款未成功的订单： " + '","'.join(df.ix[df.loan_id == 0, 'order_id'].astype(str).tolist())
@@ -105,18 +108,23 @@ for file_name in allfiles:
                   inplace=True)
 
     loan_id_query = "'" + "','".join(df['loan_id'].astype('str').tolist()) + "'"
+    df['loan_id'] = df['loan_id'].astype(str)
     sql_xyqb_query = sql_xyqb_loan_manifest % loan_id_query
     # 检查loan_id是否已经录入进去过
     df_audit_exist_loan = pd.read_sql(sql=sql_audit_query_loan_id % loan_id_query, con=engine_audit)
     if len(df_audit_exist_loan) > 0:
-        print "exist loan_id: " + ",".join(df_audit_exist_loan['loan_id'].astype('str').tolist())
-        continue
+        exist_loan_id_list = df_audit_exist_loan['loan_id'].astype('str').tolist()
+        print "存在 exist loan_id: " + ",".join(exist_loan_id_list)
+        df = df.ix[~df.loan_id.isin(exist_loan_id_list)]
+        # continue
     df_xyqb_loan_manifest = pd.read_sql(sql=sql_xyqb_query, con=engine_xyqb)
-    df_audit_loan_manifest = df_xyqb_loan_manifest.merge(df, on='loan_id', how='left')
+    df_xyqb_loan_manifest.loan_id = df_xyqb_loan_manifest.loan_id.astype(str)
+    df_audit_loan_manifest = df_xyqb_loan_manifest.merge(df, on='loan_id', how='inner')
     df_audit_loan_manifest.fillna(0, inplace=True)
     # 如果是多个资金方的合并文件 则不做处理赋值
     if funding_code > 0:
         df_audit_loan_manifest['funding_code'] = funding_code
+
     df_audit_loan_manifest['contract_loan_amount_detail'] = df_audit_loan_manifest[
         'contract_loan_amount_detail'].astype(int)
     df_audit_loan_manifest['contract_loan_amount'] = df_audit_loan_manifest['contract_loan_amount'].astype(int)

现金贷/数据准备/1_2_录入恒丰银行还款数据.py

@@ -13,16 +13,16 @@ reload(sys)
 sys.setdefaultencoding('utf8')
 
 engine_new_transaction = create_engine(
-    'mysql+mysqldb://yulong:ohVU1nPITynSZwR2@172.16.3.201:3306/new_transaction?charset=utf8',
+    'mysql+mysqldb://bo.wang:#mn89453YU9io80@172.16.3.201:3306/new_transaction?charset=utf8',
     echo=True)
 
 engine_payment_center = create_engine(
     'mysql+mysqldb://bowang:in78u6ytgbhj56r@192.168.4.6:8066/payment_center?charset=utf8',
     echo=False)
 sql_payment_center = """
-SELECT id repay_order_id,order_id from repay_order WHERE id in %s
+SELECT id repay_order_id,order_id,repaid_at from repay_order WHERE id in %s
 """
-# 20139595
+# 23297553
 # SELECT MAX(id) FROM repay_record_online;
 # 0178000022434904
 def gen_repay_order_id(hengfeng_id):
@@ -31,7 +31,7 @@ def gen_repay_order_id(hengfeng_id):
 abd = "0178000022434904"
 abd = abd[4:]
 print int(abd)
-path = u"""C:\\Users\\wangbo\\Desktop\\十月份对账\\还款数据\\MTF_20171001072103_6684\\恒丰还款数据.xlsx"""
+path = u"""C:\\Users\\wangbo\\Desktop\\11月份对账\\还款数据\\补录恒丰还款数据.xlsx"""
 df_hengfeng = pd.read_excel(path)
 df_hengfeng = df_hengfeng[[u'订单号',u'交易金额',u'核心清算日期',u'交易账户',u'交易时间']]
 df_hengfeng.rename(columns={u"订单号":"hengfeng_id",u"核心清算日期":"bill_time",u"交易账户":"third_order_id",u"交易金额":"amount",u"交易时间":"feiqi_transfer_time"},inplace=True)
@@ -48,5 +48,6 @@ df_repay_order['repay_order_id'] = df_repay_order['repay_order_id'].astype(int)
 df_hengfeng = pd.merge(df_hengfeng,df_repay_order,how='inner',on='repay_order_id')
 del df_hengfeng['repay_order_id']
 del df_hengfeng['hengfeng_id']
-df_hengfeng.to_sql(name='repay_record_online', con=engine_new_transaction, if_exists='append', index=None,
-                    chunksize=1000)
+del df_hengfeng['repaid_at']
+# df_hengfeng.to_sql(name='repay_record_online', con=engine_new_transaction, if_exists='append', index=None,
\ No newline at end of file
+#                     chunksize=1000)
\ No newline at end of file

现金贷/数据准备/1_3_录入阳光徽商还款信息.py

@@ -13,7 +13,7 @@ reload(sys)
 sys.setdefaultencoding('utf8')
 
 engine_new_transaction = create_engine(
-    'mysql+mysqldb://yulong:ohVU1nPITynSZwR2@172.16.3.201:3306/new_transaction?charset=utf8',
+    'mysql+mysqldb://bo.wang:#mn89453YU9io80@172.16.3.201:3306/new_transaction?charset=utf8',
     echo=True)
 
 # 红包数据
@@ -86,8 +86,7 @@ def split_money(df):
     return df
 
 
-path = u'C:\\Users\\wangbo\\Desktop\\十月份对账\\还款数据\\阳光徽商\\量化派-徽商项目9月系统还款数据.xlsx'
-# path = u'C:/Users/wangbo/Desktop/7月份对账数据/还款数据/6月阳光徽商量化派还款数据提取.xlsx'
+path = u'C:\\Users\\wangbo\\Desktop\\11月份对账\\还款数据\\量化派-徽商项目10月系统还款数据.xlsx'
 
 
 df_yangguang = pd.read_excel(path)
@@ -102,7 +101,8 @@ df_yangguang.rename(
     inplace=True)
 
 # 查询该笔还款计划
-sql_query_plan = """
+sql_query_plan = """  rowcount = cursor.executemany(statement, parameters)
+
 SELECT ref_id , id 'plan_id', term_no,real_loan_id 'loan_id', service_fee 'qg_service_fee',should_punish 'qg_overdue_interest' FROM new_transaction.user_repayment_plan WHERE real_loan_id IN %s
 """
 

现金贷/数据准备/2_1_红包导入.py

@@ -15,7 +15,7 @@ reload(sys)
 sys.setdefaultencoding('utf8')
 
 engine_new_transaction = engine_new_transaction = create_engine(
-    'mysql+mysqldb://yulong:ohVU1nPITynSZwR2@172.16.3.201:3306/new_transaction?charset=utf8',
+    'mysql+mysqldb://bo.wang:#mn89453YU9io80@172.16.3.201:3306/new_transaction?charset=utf8',
     echo=True)
 
 engine_xyqb = create_engine(
@@ -27,13 +27,13 @@ engine_xyqb = create_engine(
 2、红包导入red_packet_detail
 3、如果是备份库，则红包表名字要改动；不同的库名字不一样，相应的sql 中涉及的表名查看对应的要改动
 '''
-# -- 2700745
+# -- 4124885
 # SELECT MAX(id) FROM red_packet_detail ;
 sql_xyqb='''
 select t1.plan_id ,t1.coupon_amount,t2.loan_application_history_id,term_no xyqb_term_no
 from repayment_plan_coupon t1
 join repayment_plan t2 on t1.plan_id = t2.id
-where t1.status = 1 and t1.updated_at >= '2017-09-01 00:00:00' and t1.updated_at < '2017-10-01 00:00:00'
+where t1.status = 1 and t1.updated_at >= '2017-10-01 00:00:00' and t1.updated_at < '2017-11-01 00:00:00'
 '''
 
 sql_plan='''

现金贷/数据准备/2_2_红包策略导入.py

@@ -16,7 +16,7 @@ reload(sys)
 sys.setdefaultencoding('utf8')
 
 engine_new_transaction = engine_new_transaction = create_engine(
-    'mysql+mysqldb://yulong:ohVU1nPITynSZwR2@172.16.3.201:3306/new_transaction?charset=utf8',
+    'mysql+mysqldb://bo.wang:#mn89453YU9io80@172.16.3.201:3306/new_transaction?charset=utf8',
     echo=True)
 
 engine_financial = create_engine(
@@ -38,7 +38,7 @@ engine_xyqb = create_engine(
 sql_xyqb = '''
 SELECT t1.loan_application_history_id,t2.id plan_id, t1.curr_term_no xyqb_term_no,discount
 FROM repay_xyqb_detail t1 JOIN  repayment_plan t2 ON t1.loan_application_history_id=t2.loan_application_history_id AND t1.curr_term_no=t2.term_no
- WHERE pay_center_repay_at>='2017-09-01' AND pay_center_repay_at<'2017-10-01' AND discount>0
+ WHERE pay_center_repay_at>='2017-10-01' AND pay_center_repay_at<'2017-11-01' AND discount>0
 '''
 
 sql_plan = '''

现金贷/数据准备/3_2_现金贷罚息公式_V3.py

@@ -17,14 +17,15 @@ sys.setdefaultencoding('utf8')
 2、罚息计算，是还款成功（即还款状态为 3）才进行计算，如果还款未完成，则不应该计算罚息。
 '''
 
+# t2.funding_code in (290,380,350,420) AND
 # ----------------------配置变量开始---------------------------
 # TODO 查询计算罚息的还款计划，如果要更新哪个放款，则指定具体的放款id或还款时间段等其他你需要的条件，但是请别删除查询项
 read_user_sql = '''
 select t1.id,t1.fund_code,t1.term_no,t1.deadline,t1.xyqb_repaid_at repaid_at,t1.principle cur_principle,t1.interest cur_interest,t2.monthly_interest_rate rate,t2.contract_term,t2.contract_loan_amount contract_amount
 from new_transaction.user_repayment_plan t1
 left join audit.loan_manifest t2 on t1.ref_id = t2.ref_id
-where t2.funding_code in (290,380,350,420) AND t1.repay_status = 3 and date(t1.deadline) < date(t1.xyqb_repaid_at) and
-t1.xyqb_repaid_at >= '2017-09-01 00:00:00' and t1.xyqb_repaid_at < '2017-10-10 00:00:00'
+where t1.repay_status = 3 and date(t1.deadline) < date(t1.xyqb_repaid_at) and
+t1.xyqb_repaid_at >= '2017-10-01 00:00:00' and t1.xyqb_repaid_at < '2017-11-10 00:00:00'
 '''
 
 # read_user_sql='''
@@ -38,7 +39,7 @@ t1.xyqb_repaid_at >= '2017-09-01 00:00:00' and t1.xyqb_repaid_at < '2017-10-10 0
 # TODO False 表示输出到控制台
 update_flag = True
 # TODO 数据源
-engine_audit = create_engine('mysql+mysqldb://yulong:ohVU1nPITynSZwR2@172.16.3.201:3306/new_transaction?charset=utf8',
+engine_audit = create_engine('mysql+mysqldb://bo.wang:#mn89453YU9io80@172.16.3.201:3306/new_transaction?charset=utf8',
                              echo=True)
 # ----------------------配置变量结束---------------------------
 

现金贷/数据准备/5_1_update_xjd_repay_plan_repay_record_ref.py

@@ -16,33 +16,26 @@ pd.options.mode.chained_assignment = None
 '''
 
 engine_audit = create_engine(
-    'mysql+mysqldb://yulong:ohVU1nPITynSZwR2@172.16.3.201:3306/new_transaction?charset=utf8', echo=True)
+    'mysql+mysqldb://bo.wang:#mn89453YU9io80@172.16.3.201:3306/new_transaction?charset=utf8', echo=True)
 
 chunksize = 5000
 
 
 # TODO '根据具体情况选择plan_id'
 # sql_plan_id = '''
-# SELECT t2.id plan_id FROM red_packet_detail t1 LEFT JOIN user_repayment_plan t2 ON t1.ref_id=t2.ref_id AND t1.term_no=t2.term_no WHERE xyqb_repaid_at>="2017-07-01" AND xyqb_repaid_at<"2017-08-01"
+# SELECT t2.id plan_id FROM red_packet_detail t1 LEFT JOIN user_repayment_plan t2 ON t1.ref_id=t2.ref_id AND t1.term_no=t2.term_no WHERE xyqb_repaid_at>="2017-10-01" AND xyqb_repaid_at<"2017-11-10"
 # '''
 
 # sql_plan_id = '''
 # SELECT t1.plan_id  FROM `xjd_repay_plan_repay_record_ref` t1
 # JOIN `user_repayment_plan` t2 ON t1.plan_id = t2.id
-# WHERE t2.fund_code in (290,350,380,420) AND  t2.xyqb_repaid_at >= '2017-09-01 00:00:00' AND t2.xyqb_repaid_at < '2017-10-01 00:00:00' AND t1.ref_amount > 0
+# WHERE   t2.xyqb_repaid_at >= '2017-10-01 00:00:00' AND t2.xyqb_repaid_at < '2017-11-05 00:00:00' AND t1.ref_amount > 0
 # '''
 
-# sql_plan_id = '''
-# SELECT t2.id plan_id FROM  user_repayment_plan t2 WHERE  id IN (6368687,
-# 6368687,
-# 6368689,
-# 4787737,
-# 4787737,
-# 195729,
-# 195799,
-# 195831
-# )
-# '''
+
+sql_plan_id = '''
+SELECT t2.id plan_id FROM  user_repayment_plan t2 WHERE  id IN (17818535,14951401,13183085,16775621,16023121,18481649,18478739,18239129,16135315,14459209,15815947,16793909,17317973,16647695,16779713,16960183,16862699,20435587,17356583,16947229,15048301,18538103,16853759,16723967,17990615,17992793,17992907,17995541,17996099,17999693,17917967,18001511,17939003,17939735,19348913)
+'''
 
 #
 # sql_plan_id = '''
@@ -232,6 +225,18 @@ def split_money_0(df):
 # 注意退款的情况
 sql_xjd_ref = '''select id, plan_id, ref_amount from xjd_repay_plan_repay_record_ref where plan_id in %s and ref_amount > 0 '''
 
+# sql_xjd_ref = '''SELECT
+# 	id,
+# 	plan_id,
+# 	ref_amount
+# FROM
+# 	xjd_repay_plan_repay_record_ref t1
+# JOIN new_transaction.repay_record_online t2 ON t1.record_id = t2.id
+# AND t1.repay_channel = t2.pay_approach
+# WHERE
+# 	plan_id IN % s
+# AND ref_amount > 0  '''
+
 sql_user_repayment_plan = '''
 select id plan_id, ref_id,term_no,principle, interest, service_fee, should_punish ,repay_status,xyqb_repaid_at from user_repayment_plan where id in %s
 '''

现金贷/数据准备/5_1_录入批量结清数据.py

@@ -12,11 +12,11 @@ sys.setdefaultencoding('utf8')
 
 
 engine_new_test_wangbo = create_engine(
-    'mysql+mysqldb://yulong:ohVU1nPITynSZwR2@172.16.3.201:3306/audit?charset=utf8',
+    'mysql+mysqldb://bo.wang:#mn89453YU9io80@172.16.3.201:3306/audit?charset=utf8',
     echo=True)
 
 
-dir_path = u"""C:\\Users\\wangbo\\Desktop\\十月份对账\\settle_data\\"""
+dir_path = u"""C:\\Users\\wangbo\\Desktop\\11月份对账\\提前结清数据\\"""
 
 sql_update_settle = """
 UPDATE  audit.funding_repayment_record SET settle_status=1 , settle_date=%s WHERE loan_id=%s AND term_no=%s limit 1;

现金贷/数据准备/5_2_update_red_money_user_repayment_plan.py

@@ -16,7 +16,7 @@ console = logging.StreamHandler()
 console.setFormatter(formatter)
 logging.getLogger('sqlalchemy.engine').addHandler(console)
 
-engine_audit = create_engine('mysql+mysqldb://yulong:ohVU1nPITynSZwR2@172.16.3.201:3306/new_transaction?charset=utf8',
+engine_audit = create_engine('mysql+mysqldb://bo.wang:#mn89453YU9io80@172.16.3.201:3306/new_transaction?charset=utf8',
                              echo=True)
 
 '''
@@ -31,22 +31,22 @@ engine_audit = create_engine('mysql+mysqldb://yulong:ohVU1nPITynSZwR2@172.16.3.2
 # ) group by plan_id
 # '''
 #AND t2.current_repayment = 0
-sql_plan_ref = """
-SELECT
-	sum(ref_amount) repay_amount,
-	plan_id
-FROM
-	xjd_repay_plan_repay_record_ref t1
-JOIN user_repayment_plan t2 ON t1.plan_id = t2.id
-WHERE
-t2.fund_code in (290,350,380,420)
-AND	t2.xyqb_repaid_at >= "2017-09-01"
-AND t2.xyqb_repaid_at < "2017-10-10"
-AND t2.repay_status = 3
-
-GROUP BY
-	t1.plan_id
-"""
+# sql_plan_ref = """
+# SELECT
+# 	sum(ref_amount) repay_amount,
+# 	plan_id
+# FROM
+# 	xjd_repay_plan_repay_record_ref t1
+# JOIN user_repayment_plan t2 ON t1.plan_id = t2.id
+# WHERE
+#
+# t2.xyqb_repaid_at >= "2017-10-01"
+# AND t2.xyqb_repaid_at < "2017-11-10"
+# AND t2.repay_status = 3
+#
+# GROUP BY
+# 	t1.plan_id
+# """
 #
 # sql_1 = """
 #
@@ -73,9 +73,9 @@ GROUP BY
 # '''
 # 89647
 # 9892093
-# sql_plan_ref='''
-# select sum(ref_amount) repay_amount,plan_id from xjd_repay_plan_repay_record_ref  where plan_id in (19306061) GROUP BY plan_id
-# '''
+sql_plan_ref='''
+select sum(ref_amount) repay_amount,plan_id from xjd_repay_plan_repay_record_ref  where plan_id in (17818535,14951401,13183085,16775621,16023121,18481649,18478739,18239129,16135315,14459209,15815947,16793909,17317973,16647695,16779713,16960183,16862699,20435587,17356583,16947229,15048301,18538103,16853759,16723967,17990615,17992793,17992907,17995541,17996099,17999693,17917967,18001511,17939003,17939735,19348913) GROUP BY plan_id
+'''
 
 
 sql_user_plan = """select id plan_id,ref_id,term_no, principle, interest, service_fee, should_punish,repay_status from user_repayment_plan where id in %s """

现金贷/数据准备/5_3_检查ref关联.py

@@ -10,7 +10,7 @@ pd.options.mode.chained_assignment = None
 '''
 
 engine_audit = create_engine(
-    'mysql+mysqldb://yulong:ohVU1nPITynSZwR2@172.16.3.201:3306/new_transaction?charset=utf8', echo=True)
+    'mysql+mysqldb://bo.wang:#mn89453YU9io80@172.16.3.201:3306/audit_temp?charset=utf8', echo=True)
 
 
 # 检查不一致的xjd_ref记录
@@ -99,7 +99,7 @@ SELECT
 	) total_amount
 FROM
 	xjd_repay_plan_repay_record_ref t1
-JOIN offline_alipay_record t2 ON t1.record_id = t2.id
+JOIN new_transaction.offline_alipay_record t2 ON t1.record_id = t2.id
 AND t1.repay_channel = 13
 WHERE
 	t1.ref_amount != (
@@ -184,4 +184,4 @@ print "ref table is OK"
 #         print str(tuple(df_check_plan.plan_id))
 #         print sql_check_plan
 #
-# print "ref table is OK"
+print "ref table is OK"

现金贷/数据准备/audit_temp_update_red_money_user_repayment_plan.py

+# -*- encoding: utf8 -*-
+
+import logging
+import pandas as pd
+from sqlalchemy import create_engine
+import numpy as np
+
+logging.basicConfig(level=logging.INFO,
+                    format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
+                    datefmt='%a, %d %b %Y %H:%M:%S',
+                    filename='log_update_repayment.log',
+                    filemode='a+')
+logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
+formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
+console = logging.StreamHandler()
+console.setFormatter(formatter)
+logging.getLogger('sqlalchemy.engine').addHandler(console)
+
+engine_audit = create_engine(
+    'mysql+mysqldb://bo.wang:#mn89453YU9io80@172.16.3.201:3306/audit_temp?charset=utf8', echo=True)
+
+'''
+注意：1、更新完后，需要校验拆分正确否；
+2、注意是否有未使用的红包，如果有，则检查原因
+3、如果红包确实不使用，则需要删除 red_packet_detail 中 对应的未使用的红包，注意是该笔有还款且还款状态完成；如果有还款状态未还款，但是有红包，询问审计或财务或玉龙如何处理
+'''
+
+
+sql_plan_ref = """
+SELECT
+	sum(ref_amount) repay_amount,
+	plan_id
+FROM
+	audit_temp.xjd_repay_plan_repay_record_ref t1
+JOIN audit_temp.user_repayment_plan t2 ON t1.plan_id = t2.id
+WHERE
+t1.plan_id in %s
+GROUP BY
+	t1.plan_id
+"""
+
+df_plan_id = pd.read_csv(u"""C:\\Users\\wangbo\\Desktop\\十月份对账\\new_ref.csv""")
+
+sql_user_plan = """select id plan_id,ref_id,term_no, principle, interest, service_fee, should_punish,repay_status from audit_temp.user_repayment_plan where id in %s """
+
+sql_red_package = '''
+select ref_id,term_no,amount red_amount from audit_temp.red_packet_detail where ref_id in %s
+'''
+
+sql_update = """update audit_temp.user_repayment_plan set real_principle = %s, real_interest = %s, real_service_fee = %s, \
+ real_punish = %s, current_repayment = %s, collection_relief = %s, remain = %s, repay_msg = %s,red_package = %s,mitigate_all=%s where id = %s;"""
+
+
+def cal_msg(x):
+    repay_amount = x['repay_amount']
+    principle = x['principle']
+    interest = x['interest']
+    service_fee = x['service_fee']
+    should_punish = x['should_punish']
+    res_msg = ''
+    remain = repay_amount - principle
+    remain = round(remain, 2)
+    if remain < 0:
+        res_msg += '减免本金%s元' % (-remain,)
+        if interest > 0:
+            res_msg += '，减免利息%s元' % (interest,)
+        if service_fee > 0:
+            res_msg += '，减免服务费%s元' % (service_fee,)
+        if should_punish > 0:
+            res_msg += '，减免罚息%s元' % (should_punish,)
+        return res_msg + '。'
+    remain -= interest
+    remain = round(remain, 2)
+    if remain < 0:
+        res_msg += '减免利息%s元' % (-remain,)
+        if service_fee > 0:
+            res_msg += '，减免服务费%s元' % (service_fee,)
+        if should_punish > 0:
+            res_msg += '，减免罚息%s元' % (should_punish,)
+        return res_msg + '。'
+    remain -= service_fee
+    remain = round(remain, 2)
+    if remain < 0:
+        res_msg += '减免服务费%s元' % (-remain,)
+        if should_punish > 0:
+            res_msg += '，减免罚息%s元' % (should_punish,)
+        return res_msg + '。'
+    remain -= should_punish
+    remain = round(remain, 2)
+    if remain < 0:
+        return '减免罚息%s元。' % -remain
+    return ''
+
+
+step = 5000
+df1_gen = pd.read_sql(sql=sql_plan_ref % str(tuple(df_plan_id.plan_id.astype(int).tolist())), con=engine_audit, chunksize=step)
+i = 0
+for df1 in df1_gen:
+    logging.info('-----------------------------> i: %s <--------------------------' % i)
+    i += 1
+    logging.info('----------------------------> len(df1): %s  <------------------------------' % len(df1))
+    ids = df1['plan_id'].astype(str).tolist()
+    df2 = pd.read_sql(sql=sql_user_plan % str(tuple(ids)).replace(',)', ')'), con=engine_audit)
+    ref_ids = df2.ref_id.astype(str).tolist()
+    df3 = pd.read_sql(sql_red_package % str(tuple(ref_ids)).replace(',)', ')'), engine_audit)
+    df2['ref_id'] = df2['ref_id'].astype(int)
+    df2['term_no'] = df2['term_no'].astype(int)
+    df3['ref_id'] = df3['ref_id'].astype(int)
+    df3['term_no'] = df3['term_no'].astype(int)
+
+    df = pd.merge(df1, df2, on='plan_id')
+    df = pd.merge(df, df3, on=['ref_id', 'term_no'], how='left')
+    df.fillna(0, inplace=True)
+    df['repay_amount'] = df['repay_amount'].apply(lambda x: round(x, 2))
+    df['principle'] = df['principle'].apply(lambda x: round(x, 2))
+    df['interest'] = df['interest'].apply(lambda x: round(x, 2))
+    df['service_fee'] = df['service_fee'].apply(lambda x: round(x, 2))
+    df['should_punish'] = df['should_punish'].apply(lambda x: np.round(x, 2))
+    df['red_package_fee'] = df['red_amount'].apply(lambda x: round(x, 2))
+    df.repay_status = df.repay_status.astype(int)
+    # repay_status == 3 可以做减免，其他的状态不进行减免
+    df.ix[df.repay_status != 3, 'principle'] = df.ix[df.repay_status != 3].apply(
+        lambda x: min(x.repay_amount, x.principle), axis=1)
+    df.ix[df.repay_status != 3, 'interest'] = df.ix[df.repay_status != 3].apply(
+        lambda x: min(x.interest, round(x.repay_amount - x.principle, 2)), axis=1)
+    df.ix[df.repay_status != 3, 'service_fee'] = df.ix[df.repay_status != 3].apply(
+        lambda x: min(x.service_fee, round(x.repay_amount - x.principle - x.interest, 2)), axis=1)
+    df.ix[df.repay_status != 3, 'should_punish'] = df.ix[df.repay_status != 3].apply(
+        lambda x: min(x.should_punish, round(x.repay_amount - x.principle - x.interest - x.service_fee, 2)), axis=1)
+    # 非还款完成状态不使用红包
+    df.ix[df.repay_status != 3, 'red_package_fee'] = 0
+
+
+    df['remain'] = df['repay_amount'] - df['principle'] - df['interest'] - df['service_fee'] - df['should_punish']
+    df['remain'] = df['remain'].apply(lambda x: round(x, 2))
+
+    # remain 小于零 变成减免
+    df['msg'] = df.apply(lambda x: cal_msg(x), axis=1)
+    df['collection_relife'] = 0
+    df.loc[df['remain'] < 0, 'collection_relife'] = -df.loc[df['remain'] < 0, 'remain']
+    df.loc[df['remain'] <= 0, 'remain'] = 0
+    # 如果有减免 < 红包，不使用红包 collection_relife是总减免
+    df.ix[(df.repay_status == 3) & (df.collection_relife < df.red_package_fee),'red_package_fee'] = 0
+
+    # mitigate_all是催收减免
+    df['mitigate_all'] = np.round(df['collection_relife'] - df['red_package_fee'], 2)
+    df_list = df[
+        ['principle', 'interest', 'service_fee', 'should_punish', 'repay_amount', 'collection_relife', 'remain', 'msg',
+         'red_package_fee', 'mitigate_all', 'plan_id']].values.tolist()
+    logging.info('----------------------------> len(df_list): %s  <------------------------------' % len(df_list))
+    engine_audit.execute(sql_update, df_list)
+
+print '-------------------------------------------done-----------------------------------'