Merge remote-tracking branch 'origin/master'

现金贷&去哪儿&白条--在用代码维护/普通白条/白条直投每日赔付（用实还表出）.py

+# coding: utf-8
+# Author: Perry_Zhu
+
+import datetime
+import sys
+import pandas as pd
+import pyexcelerate as pe
+from sqlalchemy import create_engine
+
+"""
+注意：ref_id=5403743，order_no='1709161050021394'线上放款重复，
+本版本已经修正，但是2017-10月份与财务对不上
+"""
+reload(sys)
+sys.setdefaultencoding('utf8')
+
+engine_new_transaction = create_engine(
+    'mysql+mysqldb://internal_r:ArbNgtvlJzZHXsEu@172.16.4.60:3306/new_transaction?charset=utf8', echo=True)
+
+start_time = '2017-08-01'
+end_time = '2017-11-01'
+end_y = 2017
+end_m = 11
+end_date = datetime.date(end_y, end_m, 1)
+tx_amount = 1
+sr_amount = 2
+zbj_amount = 3
+path = u'E:/数据汇总/白条/每日赔付/2017-10/'
+
+loan_info = """
+    SELECT r.order_no,r.term_no,date(o.loan_paid_at) loan_paid_at
+    FROM baitiao_audit.baitiao_order o
+    JOIN new_transaction.bt_funding_repayment_record r ON o.ref_id=r.ref_id
+    group by 1,2
+"""
+
+loan_plan = """
+ SELECT 
+      t.order_no,t.term_no,t.plan_day,SUM(t.principle+t.interest) plan_all,SUM(t.principle) principle
+    FROM(
+        SELECT r.order_no,r.term_no,DATE(r.repaid_at) plan_day,
+          sum(ifnull(r.principal,0))principle,0.0 interest
+        FROM  bt_funding_repayment_record r
+        JOIN user_bt_repayment_plan t1 on r.repay_plan_id=t1.repayment_plan_id AND t1.sys_tag='XYQB_BILL_BT' AND r.principal_account=%s and r.refund_status!=3
+        JOIN baitiao_audit.baitiao_order t2 ON t1.ref_id=t2.ref_id
+        WHERE 
+          r.repaid_at >= '%s' AND r.repaid_at < '%s'  
+        GROUP BY 1,2,3
+
+         UNION ALL 
+          SELECT r.order_no,r.term_no,DATE(r.repaid_at) plan_day,
+           0.0 principle,SUM(ifnull(r.interest,0)) interest
+        FROM  bt_funding_repayment_record r
+        JOIN user_bt_repayment_plan t1 on r.repay_plan_id=t1.repayment_plan_id AND t1.sys_tag='XYQB_BILL_BT'AND r.interest_account=%s and r.interest>0
+        JOIN baitiao_audit.baitiao_order t2 ON t1.ref_id=t2.ref_id
+        WHERE 
+          r.repaid_at >= '%s' AND r.repaid_at < '%s'  
+        GROUP BY 1,2,3
+    )t 
+    GROUP BY 1,2,3
+"""
+repay_online = '''
+SELECT plan.order_no,plan.term_no,DATE(t2.bill_time) repaid_at,DATE(plan.deadline) deadline ,
+      sum(ifnull(t1.principle,0)-ifnull(t1.mitigate_principle,0)) real_principle,
+      sum(ifnull(t1.interest,0)-ifnull(t1.mitigate_interest,0)) real_interest
+FROM baitiao_repay_plan_repay_record_ref t1
+JOIN repay_record_online t2 ON t1.record_id = t2.id AND t2.pay_approach in(22,23,24)
+JOIN user_bt_repayment_plan t3 ON t1.plan_id = t3.id AND t3.sys_tag='XYQB_BILL_BT' 
+JOIN bt_funding_repayment_plan plan on plan.repay_plan_id=t3.repayment_plan_id  AND plan.refund_status!=3
+JOIN baitiao_audit.baitiao_order t4 ON t4.ref_id=t3.ref_id AND t4.is_active = 1
+WHERE   
+ t2.bill_time >= '%s' 
+and t2.bill_time < '%s'
+GROUP BY 1,2,3,4;
+'''
+repay_alipay = '''
+SELECT plan.order_no,plan.term_no,DATE(t2.transfer_time) repaid_at,DATE(plan.deadline) deadline ,
+      sum(ifnull(t1.principle,0)-ifnull(t1.mitigate_principle,0)) real_principle,
+      sum(ifnull(t1.interest,0)-ifnull(t1.mitigate_interest,0)) real_interest
+FROM baitiao_repay_plan_repay_record_ref t1
+JOIN offline_alipay_record t2 ON  t1.record_id = t2.id AND t1.repay_channel = 13 
+JOIN user_bt_repayment_plan t3 ON t1.plan_id = t3.id AND t3.sys_tag='XYQB_BILL_BT'
+JOIN bt_funding_repayment_plan plan on plan.repay_plan_id=t3.repayment_plan_id  AND plan.refund_status!=3
+JOIN baitiao_audit.baitiao_order t4 ON t4.ref_id=t3.ref_id AND t4.is_active = 1
+WHERE 
+ t2.transfer_time >= '%s' 
+and  t2.transfer_time < '%s'
+GROUP BY 1,2,3,4
+'''
+
+
+def split_repay(df_repay, df_plan):
+    df_plan.order_no = df_plan.order_no.astype(str)
+    df_plan.term_no = df_plan.term_no.astype(int)
+    df_repay.order_no = df_repay.order_no.astype(str)
+    df_repay.term_no = df_repay.term_no.astype(int)
+    df_repay['repaid_at'] = pd.to_datetime(df_repay['repaid_at']).apply(lambda x: x.date())
+    df_repay['deadline'] = pd.to_datetime(df_repay['deadline']).apply(lambda x: x.date())
+    # 正常还款
+    df_zc = df_repay.ix[df_repay['repaid_at'] == df_repay['deadline']]
+    df_zc = df_zc.groupby(by=['order_no', 'term_no', 'repaid_at'])['real_principle'].sum()
+    df_zc = df_zc.to_frame()
+    df_zc.reset_index(inplace=True)
+    df_zc.rename(columns={'repaid_at': 'real_hk', 'real_principle': 'zc_prin_inst_total'}, inplace=True)
+    # ===========逾期还款==============
+    df_yq = df_repay.ix[df_repay['deadline'] < df_repay['repaid_at']]
+    df_yq = df_yq.groupby(by=['order_no', 'term_no', 'repaid_at'])['real_principle'].sum()
+    df_yq = df_yq.to_frame()
+    df_yq.reset_index(inplace=True)
+    df_yq.rename(columns={'repaid_at': 'real_hk', 'real_principle': 'yq_prin_inst_total'}, inplace=True)
+    # ============提前还款==============
+    df_tq = df_repay.ix[df_repay['deadline'] > df_repay['repaid_at']]
+    # 实还可用筛选
+    df_tq = df_tq.ix[df_tq['deadline'] < end_date]
+    # 提前还款的可用日期变更
+    df_tq = df_tq.groupby(by=['order_no', 'term_no', 'deadline'])['real_principle'].sum()
+    df_tq = df_tq.to_frame()
+    df_tq.reset_index(inplace=True)
+    df_tq.rename(columns={'deadline': 'real_hk', 'real_principle': 'tq_prin_inst_total'}, inplace=True)
+    # 合并
+    if len(df_repay) > 0:
+        df_hk_new = pd.merge(df_zc, df_yq, how='outer', on=['order_no', 'term_no', 'real_hk'])
+        df_hk_new = pd.merge(df_hk_new, df_tq, how='outer', on=['order_no', 'term_no', 'real_hk'])
+        df_hk_new.fillna(0, inplace=True)
+        df_h = df_hk_new.groupby(by=['order_no', 'term_no', 'real_hk'])[
+            'zc_prin_inst_total', 'yq_prin_inst_total', 'tq_prin_inst_total'].sum().reset_index()
+        df_plan_t = df_plan.loc[df_plan['principle'] > 0]
+        df_h.rename(columns={'real_hk': 'plan_day'}, inplace=True)
+        df_t = df_plan_t[['order_no', 'term_no', 'plan_day']]
+        df_t.rename(columns={'plan_day': 't_day'}, inplace=True)
+        df_hk_new = pd.merge(df_plan_t, df_h, how='outer', on=['order_no', 'term_no', 'plan_day'])
+        df_hk_new['hk_day'] = df_hk_new['plan_day']
+        df_hk_new = pd.merge(df_t, df_hk_new, how='left', on=['order_no', 'term_no'])
+        df_hk_new.loc[df_hk_new['hk_day'].isnull(), 'hk_day'] = df_hk_new.loc[df_hk_new['hk_day'].isnull(), 't_day']
+        df_hk_new.fillna(0, inplace=True)
+        df_hk_new = df_hk_new[['order_no', 'term_no', 'hk_day', 'plan_all', 'zc_prin_inst_total', 'yq_prin_inst_total',
+                               'tq_prin_inst_total']]
+    else:
+        df_hk_new = df_plan
+        df_hk_new['zc_prin_inst_total'] = 0
+        df_hk_new['yq_prin_inst_total'] = 0
+        df_hk_new['tq_prin_inst_total'] = 0
+    return df_hk_new
+
+
+def forward_repay_time(df):
+    # 2017-9月还资金方时间，线上给的有问题，必须将打款时间提前一天
+    df.loc[df.plan_day == datetime.date(2017, 9, 9), 'plan_day'] = datetime.date(2017, 9, 10)
+    df.loc[df.plan_day == datetime.date(2017, 9, 11), 'plan_day'] = datetime.date(2017, 9, 10)
+    df.loc[df.plan_day == datetime.date(2017, 9, 21), 'plan_day'] = datetime.date(2017, 9, 20)
+    df.loc[df.plan_day == datetime.date(2017, 9, 29), 'plan_day'] = datetime.date(2017, 9, 28)
+    return df
+
+
+def spread_loan_id():
+    df_info = pd.read_sql(loan_info, engine_new_transaction)
+    df_info.order_no = df_info.order_no.astype(str)
+    df_info.term_no = df_info.term_no.astype(int)
+    df_tx = pd.read_sql(loan_plan % (tx_amount, start_time, end_time, tx_amount, start_time, end_time),
+                        engine_new_transaction)
+    df_sr = pd.read_sql(loan_plan % (sr_amount, start_time, end_time, sr_amount, start_time, end_time),
+                        engine_new_transaction)
+    df_zbj = pd.read_sql(loan_plan % (zbj_amount, start_time, end_time, zbj_amount, start_time, end_time),
+                         engine_new_transaction)
+
+    # 2017-9月还资金方时间，线上给的有问题，必须将打款时间提前一天
+    # 提现账户
+    df_tx = forward_repay_time(df_tx)
+    # 收入账户
+    df_sr = forward_repay_time(df_sr)
+    # 质保金账户
+    df_zbj = forward_repay_time(df_zbj)
+    df_online = pd.read_sql(repay_online % (start_time, end_time), engine_new_transaction)
+    df_ali = pd.read_sql(repay_alipay % (start_time, end_time), engine_new_transaction)
+    df_repay = pd.concat([df_online, df_ali], axis=0, ignore_index=True)
+    df_repay_tx = split_repay(df_repay, df_tx)
+    df_repay_sr = split_repay(df_repay, df_sr)
+    df_repay_zbj = split_repay(df_repay, df_zbj)
+    df_repay_tx.rename(columns={'plan_all': 'tx_plan_all'}, inplace=True)
+    df_repay_sr.rename(columns={'plan_all': 'sr_plan_all'}, inplace=True)
+    df_repay_zbj.rename(columns={'plan_all': 'zbj_plan_all'}, inplace=True)
+    df_repay_tx.rename(columns={'zc_prin_inst_total': 'tx_zc_principle', 'yq_prin_inst_total': 'tx_yq_principle',
+                                'tq_prin_inst_total': 'tx_tq_principle'}, inplace=True)
+    df_repay_sr.rename(columns={'zc_prin_inst_total': 'sr_zc_principle', 'yq_prin_inst_total': 'sr_yq_principle',
+                                'tq_prin_inst_total': 'sr_tq_principle'}, inplace=True)
+    df_repay_zbj.rename(columns={'zc_prin_inst_total': 'zbj_zc_principle', 'yq_prin_inst_total': 'zbj_yq_principle',
+                                 'tq_prin_inst_total': 'zbj_tq_principle'}, inplace=True)
+    df_hk_new = pd.merge(df_repay_tx, df_repay_sr, how='outer', on=['order_no', 'term_no', 'hk_day'])
+    df_hk_new = pd.merge(df_hk_new, df_repay_zbj, how='outer', on=['order_no', 'term_no', 'hk_day'])
+    df_hk_new.order_no = df_hk_new.order_no.astype(str)
+    df_hk_new.term_no = df_hk_new.term_no.astype(int)
+    df_hk_new = pd.merge(df_hk_new, df_info, on=['order_no', 'term_no'], how='left')
+    df_hk_new.fillna(0, inplace=True)
+    df_hk_new = df_hk_new[
+        ['order_no', 'term_no', 'loan_paid_at', 'hk_day', 'tx_plan_all', 'tx_zc_principle', 'tx_yq_principle',
+         'tx_tq_principle', 'sr_plan_all', 'sr_zc_principle', 'sr_yq_principle', 'sr_tq_principle', 'zbj_plan_all',
+         'zbj_zc_principle', 'zbj_yq_principle', 'zbj_tq_principle']]
+    # 按日期出
+    df_out = df_hk_new[['hk_day', 'tx_plan_all', 'tx_zc_principle', 'tx_yq_principle', 'tx_tq_principle', 'sr_plan_all',
+                        'sr_zc_principle', 'sr_yq_principle', 'sr_tq_principle', 'zbj_plan_all', 'zbj_zc_principle',
+                        'zbj_yq_principle', 'zbj_tq_principle']]
+    df_out = df_out.groupby(by=['hk_day'])[
+        'tx_plan_all', 'tx_zc_principle', 'tx_yq_principle', 'tx_tq_principle', 'sr_plan_all', 'sr_zc_principle',
+        'sr_yq_principle', 'sr_tq_principle', 'zbj_plan_all', 'zbj_zc_principle', 'zbj_yq_principle',
+        'zbj_tq_principle'].sum().reset_index()
+    df_hk_new.rename(columns={'hk_day': '还款日期',
+                              'tq_prin_total': '客户提前可用本金', 'tx_zc_principle': '提现账户客户正常还款本金',
+                              'tx_yq_principle': '提现账户客户逾期还款本金', 'tx_tq_principle': '提现账户客户提前可用本金',
+                              'sr_zc_principle': '收入账户客户正常还款本金', 'sr_yq_principle': '收入账户客户逾期还款本金',
+                              'sr_tq_principle': '收入账户客户提前可用本金', 'zbj_zc_principle': '质保金账户客户正常还款本金',
+                              'tx_plan_all': '提现账户应还资方本息', 'sr_plan_all': '收入账户应还资方本息',
+                              'zbj_plan_all': '质保金账户应还资方本息', 'zbj_yq_principle': '质保金账户客户逾期还款本金',
+                              'zbj_tq_principle': '质保金账户客户提前可用本金'}, inplace=True)
+    df_out.rename(columns={'hk_day': '还款日期',
+                           'tq_prin_total': '客户提前可用本金', 'tx_zc_principle': '提现账户客户正常还款本金',
+                           'tx_yq_principle': '提现账户客户逾期还款本金', 'tx_tq_principle': '提现账户客户提前可用本金',
+                           'sr_zc_principle': '收入账户客户正常还款本金', 'sr_yq_principle': '收入账户客户逾期还款本金',
+                           'sr_tq_principle': '收入账户客户提前可用本金', 'zbj_zc_principle': '质保金账户客户正常还款本金',
+                           'tx_plan_all': '提现账户应还资方本息', 'sr_plan_all': '收入账户应还资方本息',
+                           'zbj_plan_all': '质保金账户应还资方本息', 'zbj_yq_principle': '质保金账户客户逾期还款本金',
+                           'zbj_tq_principle': '质保金账户客户提前可用本金'}, inplace=True)
+    df_hk_new.to_excel(path + u'5_loan_id维度展开2017-10.xlsx', index=None, columns=df_hk_new.columns, encoding='utf8')
+    df_out.to_excel(path + u'5_按日期展开2017-10.xlsx', index=None, columns=df_out.columns, encoding='utf8')
+
+
+spread_loan_id()