# -*- coding: utf-8 -*-
import os
import sys
import datetime
import itertools
import pandas as pd
from sqlalchemy import create_engine
import multiprocessing as mtp

reload(sys)
sys.setdefaultencoding('utf8')

start_time_deadline = '2017-01-01 00:00:00'

end_time_deadline = '2017-06-01 00:00:00'

start_time_loanPaidAt = '2016-01-01 00:00:00'
end_time_loanPaidAt = '2017-01-01 00:00:00'

start_repay_time = '2015-01-01 00:00:00'
start_time_repay_huarong_settle = '2017-05-01 00:00:00'
end_repay_time = '2017-06-01 00:00:00'

path = u'E:/peifu_LoanClass_merage/'
engine_financial = create_engine(
    'mysql+mysqldb://yulong:ohVU1nPITynSZwR2@172.16.3.201:3306/all_back_0531?charset=utf8',
    echo=True)

sql_fund = '''
select id fund_code,name fund_name from new_transaction.funding_corp WHERE id in (150)
'''
sql_repay_channel = '''
select id repay_channel , approach_name channel_name FROM new_transaction.`repay_channel`
'''
df_fund = pd.read_sql(sql_fund, con=engine_financial)
df_repay_channel = pd.read_sql(sql_repay_channel, con=engine_financial)

# (160,50,80,130,270, 310, 320, 210)
sql_fund_hk = '''
SELECT DATE(t1.deadline) day_hk,t1.funding_code fund_code,SUM(t1.`principle`) plan_principle,SUM(t1.`interest`) plan_interest
FROM all_back_0531.`funding_repayment_record`t1 JOIN all_back_0531.loan_manifest t2  ON t1.loan_id=t2.loan_id JOIN  temp.loan_class t3 on t2.ref_id=t3.ref_id  WHERE  t1.is_active = 1 AND  t3.class='%s' AND t1.funding_code=%s AND t2.is_active in (1,-2)
AND
 t2.loan_paid_at >='%s' and t2.loan_paid_at<'%s'
and t1.deadline >= '%s' AND t1.deadline < '%s'
GROUP BY DATE(t1.deadline)
'''

sql_fund_hk_user_plan = '''
SELECT DATE(urp.deadline) day_hk,lm.funding_code fund_code,SUM(urp.`principle`) plan_principle,SUM(urp.`interest`) plan_interest
FROM all_back_0531.`user_repayment_plan` urp LEFT JOIN all_back_0531.loan_manifest lm ON urp.loan_id=lm.loan_id LEFT JOIN temp.loan_class lc ON lm.ref_id = lc.ref_id WHERE  lm.is_active = 1 AND lc.class='%s' AND lm.funding_code=%s
AND lm.is_active in (1,-2) AND  lm.loan_paid_at >='%s' and lm.loan_paid_at<'%s'
and urp.deadline >= '%s' AND urp.deadline < '%s'
GROUP BY DATE(urp.deadline)
'''

# (270, 310, 320, 210)
sql_plan_hk = '''
SELECT DATE(urp.deadline) day_hk,SUM(urp.principle+urp.interest) plan_all FROM  all_back_0531.`user_repayment_plan` urp JOIN all_back_0531.loan_manifest lm ON urp.ref_id=lm.ref_id JOIN temp.loan_class lc ON lm.ref_id=lc.ref_id
WHERE lc.class='%s' AND urp.fund_code=%s AND lm.is_active in (1,-2) AND  lm.loan_paid_at >='%s' and lm.loan_paid_at<'%s' and urp.deadline >= '%s' AND urp.deadline < '%s'  GROUP BY DATE(urp.deadline)
'''

sql_A = '''
SELECT DATE(transfer_time) day_hk,transfer_amount plan_all FROM all_back_0531.lkb_to_fund WHERE  fund_corp='张初阳'
'''
sql_C = '''
SELECT DATE(transfer_time) day_hk,transfer_amount plan_all FROM all_back_0531.lkb_to_fund WHERE  fund_corp='李然'
'''
sql_D = '''
SELECT DATE(transfer_time) day_hk,transfer_amount plan_all FROM all_back_0531.lkb_to_fund WHERE  fund_corp='曾思成'
'''

sql_hk_online = '''
SELECT DATE(t3.xyqb_repaid_at) repaid_at,DATE(t3.deadline) deadline ,t1.repay_channel,t3.fund_code,
  t1.`principle` real_principle,t1.`mitigate_principle` mitigate_principle,t1.`interest` real_interest,t1.`mitigate_interest` mitigate_interest
FROM all_back_0531.`xjd_repay_plan_repay_record_ref` t1
JOIN all_back_0531.`repay_record_online` t2 ON t1.repay_channel < 13 AND t1.record_id = t2.id
JOIN all_back_0531.`user_repayment_plan` t3 ON t1.plan_id = t3.id
JOIN all_back_0531.loan_manifest t4 ON t3.ref_id=t4.ref_id
JOIN  temp.loan_class t5 ON t4.ref_id = t5.ref_id
WHERE t5.class = '%s' AND  t3.fund_code = %s AND t4.is_active in (1,-2) AND t4.loan_paid_at>='%s' AND t4.loan_paid_at<'%s'  and t3.xyqb_repaid_at >= '%s' and t3.xyqb_repaid_at < '%s'
'''

sql_hk_alipay = '''
SELECT DATE(t2.transfer_time) repaid_at,DATE(t3.deadline) deadline ,t1.repay_channel,t3.fund_code,t1.`principle` real_principle,t1.`mitigate_principle` mitigate_principle,t1.`interest` real_interest,t1.`mitigate_interest` mitigate_interest
FROM all_back_0531.`xjd_repay_plan_repay_record_ref` t1
JOIN all_back_0531.`offline_alipay_record` t2 ON t1.repay_channel = 13 AND t1.record_id = t2.id
JOIN all_back_0531.`user_repayment_plan` t3 ON t1.plan_id = t3.id
JOIN all_back_0531.loan_manifest t4 ON t3.ref_id=t4.ref_id
JOIN temp.loan_class t5 ON t4.ref_id = t5.ref_id
WHERE t5.class='%s' AND  t3.fund_code = %s AND t4.is_active in (1,-2) AND t4.loan_paid_at>='%s' AND t4.loan_paid_at<'%s' and t2.transfer_time >= '%s' and  t2.transfer_time < '%s'
'''

sql_hk_bank = '''
SELECT DATE(t2.transfer_time) repaid_at,DATE(t3.deadline) deadline ,t1.repay_channel,t3.fund_code,t1.`principle` real_principle,t1.`mitigate_principle` mitigate_principle,t1.`interest` real_interest,t1.`mitigate_interest` mitigate_interest
FROM all_back_0531.`xjd_repay_plan_repay_record_ref` t1
JOIN all_back_0531.`offline_bank_repay_record` t2 ON t1.repay_channel in (14,15,16) AND t1.record_id = t2.id
JOIN all_back_0531.`user_repayment_plan` t3 ON t1.plan_id = t3.id
JOIN all_back_0531.loan_manifest t4 ON t3.ref_id=t4.ref_id
JOIN temp.loan_class t5 ON t4.ref_id = t5.ref_id
WHERE t5.class='%s' AND  t3.fund_code = %s AND t4.is_active in (1,-2) AND t4.loan_paid_at>='%s' AND t4.loan_paid_at<'%s' and t2.transfer_time >= '%s' and  t2.transfer_time < '%s'
'''

sql_hk_fund_deadline_online = """SELECT
	DATE(t3.xyqb_repaid_at) repaid_at,
	DATE(t4.deadline) deadline,
	 t1.repay_channel,
	t3.fund_code,
	t1.`principle` real_principle,
	t1.`mitigate_principle` mitigate_principle,
	t1.`interest` real_interest,
	t1.`mitigate_interest` mitigate_interest
FROM
	all_back_0531.`xjd_repay_plan_repay_record_ref` t1
JOIN all_back_0531.`repay_record_online` t2 ON t1.repay_channel < 13
AND t1.record_id = t2.id
JOIN all_back_0531.`user_repayment_plan` t3 ON t1.plan_id = t3.id
JOIN all_back_0531.funding_repayment_record t4 ON t4.ref_id = t3.ref_id AND t4.term_no = t3.term_no
JOIN all_back_0531.loan_manifest t5 ON t4.ref_id = t5.ref_id
JOIN  temp.loan_class t6 ON t5.ref_id = t6.ref_id
WHERE t6.class = '%s' AND t3.fund_code =  %s  and t5.is_active in (1,-2) AND t5.loan_paid_at>='%s' AND t5.loan_paid_at<'%s' AND  t3.xyqb_repaid_at >= '%s' and t3.xyqb_repaid_at < '%s'
	"""

sql_hk_fund_deadline_alipay = '''
SELECT DATE(t2.transfer_time) repaid_at,DATE(t4.deadline) deadline ,t1.repay_channel,t3.fund_code,t1.`principle` real_principle,t1.`mitigate_principle` mitigate_principle,t1.`interest` real_interest,t1.`mitigate_interest` mitigate_interest
FROM all_back_0531.`xjd_repay_plan_repay_record_ref` t1
JOIN all_back_0531.`offline_alipay_record` t2 ON t1.repay_channel = 13 AND t1.record_id = t2.id
JOIN all_back_0531.`user_repayment_plan` t3 ON t1.plan_id = t3.id
JOIN all_back_0531.funding_repayment_record t4 ON t4.ref_id = t3.ref_id AND t4.term_no = t3.term_no
JOIN all_back_0531.loan_manifest t5 ON t4.ref_id = t5.ref_id
JOIN  temp.loan_class t6 ON t5.ref_id = t6.ref_id
WHERE t6.class = '%s' AND t3.fund_code=%s and t5.is_active in (1,-2) AND t5.loan_paid_at>='%s' AND t5.loan_paid_at<'%s' and t2.transfer_time >= '%s' and  t2.transfer_time < '%s'
'''

sql_hk_fund_deadline_bank = '''
SELECT DATE(t2.transfer_time) repaid_at,DATE(t4.deadline) deadline ,t1.repay_channel,t3.fund_code,t1.`principle` real_principle,t1.`mitigate_principle` mitigate_principle,t1.`interest` real_interest,t1.`mitigate_interest` mitigate_interest
FROM all_back_0531.`xjd_repay_plan_repay_record_ref` t1
JOIN all_back_0531.`offline_bank_repay_record` t2 ON t1.repay_channel in (14,15,16) AND t1.record_id = t2.id
JOIN all_back_0531.`user_repayment_plan` t3 ON t1.plan_id = t3.id
JOIN all_back_0531.funding_repayment_record t4 ON t4.ref_id = t3.ref_id AND t4.term_no = t3.term_no
JOIN all_back_0531.loan_manifest t5 ON t4.ref_id = t5.ref_id
JOIN  temp.loan_class t6 ON t5.ref_id = t6.ref_id
WHERE t6.class = '%s' AND  t3.fund_code=%s and t5.is_active in (1,-2) AND t5.loan_paid_at>='%s' AND t5.loan_paid_at<'%s' and t2.transfer_time >= '%s' and  t2.transfer_time < '%s'
'''

sql_fund_hk_hr = '''
SELECT DATE(t1.repaid_at) day_hk,t1.funding_code fund_code, SUM(t1.`principle`)+SUM(t1.`interest`) plan_all
FROM all_back_0531.`funding_repayment_record` t1 JOIN all_back_0531.loan_manifest t2 ON t1.loan_id=t2.loan_id JOIN temp.loan_class t3 ON t2.ref_id=t3.ref_id  WHERE
  t1.is_active = 1 AND t3.class='%s' AND t1.funding_code=%s
AND t2.is_active in (1,-2) AND  t2.loan_paid_at >='%s' and t2.loan_paid_at<'%s'
and t1.repaid_at >= '%s' AND t1.repaid_at < '%s'
GROUP BY DATE(t1.repaid_at)
'''

sql_query_advance_settle_plan_id = """
SELECT
	t3.id plan_id
FROM
	all_back_0531.`xjd_repay_plan_repay_record_ref` t1
JOIN all_back_0531.`user_repayment_plan` t3 ON t1.plan_id = t3.id
JOIN all_back_0531.loan_manifest t5 ON t3.ref_id = t5.ref_id
JOIN temp.loan_class t6 ON t5.ref_id = t6.ref_id
WHERE
    t6.class='%s' AND t3.fund_code = %s
AND t5.loan_paid_at>='%s' AND t5.loan_paid_at<'%s'
AND t3.xyqb_repaid_at >= '%s'
AND t3.xyqb_repaid_at < '%s'
AND t3.xyqb_repaid_at <= t3.deadline AND t3.deadline>='%s'
AND t3.ref_id IN (
	SELECT
		t3.ref_id
	FROM
		all_back_0531.`xjd_repay_plan_repay_record_ref` t1
	JOIN all_back_0531.`user_repayment_plan` t3 ON t1.plan_id = t3.id
	JOIN all_back_0531.loan_manifest t5 ON t3.ref_id = t5.ref_id
	AND t3.term_no = t5.contract_term
	WHERE
		t3.fund_code = %s
	AND t5.loan_paid_at>='%s' AND t5.loan_paid_at<'%s'
	AND t3.xyqb_repaid_at < '%s'
	AND t3.xyqb_repaid_at <= t3.deadline
);
"""

sql_query_advance_settle = """
SELECT
	t3.id plan_id,
	t3.ref_id,
	DATE(t3.xyqb_repaid_at) repaid_at,
	DATE(t4.deadline) deadline,
	t1.repay_channel,
	t3.term_no current_term,
	t5.contract_term,
	t3.fund_code,
	(
		t1.`principle` - t1.`mitigate_principle` + t1.`interest` - t1.`mitigate_interest`
	) real_principle
FROM
	all_back_0531.`xjd_repay_plan_repay_record_ref` t1
JOIN all_back_0531.`repay_record_online` t2 ON t1.repay_channel < 13
AND t1.record_id = t2.id
JOIN all_back_0531.`user_repayment_plan` t3 ON t1.plan_id = t3.id
JOIN all_back_0531.funding_repayment_record t4 ON t4.ref_id = t3.ref_id
AND t4.term_no = t3.term_no
JOIN all_back_0531.loan_manifest t5 ON t3.ref_id = t5.ref_id
JOIN temp.loan_class t6 ON t5.ref_id = t6.ref_id
WHERE
    t6.class='%s' AND 	t3.fund_code = %s  AND t5.loan_paid_at>='%s' AND t5.loan_paid_at<'%s' AND 	t3.id IN %s
"""


def ky_table(fund_code, fund_name, loan_class):
    if fund_code == 200:
        df_plan = pd.read_sql(
            sql_fund_hk % (loan_class, fund_code, start_time_loanPaidAt, end_time_loanPaidAt, start_time, end_time),
            con=engine_financial)
    else:
        df_plan = pd.read_sql(sql_fund_hk_user_plan % (
            loan_class, fund_code, start_time_loanPaidAt, end_time_loanPaidAt, start_time, end_time),
                              con=engine_financial)
    df_plan.fillna(0, inplace=True)
    df_plan['plan_all'] = df_plan['plan_interest'] + df_plan['plan_principle']
    df_plan.drop(['plan_principle', 'plan_interest'], inplace=True, axis=1)
    df_plan['day_hk'] = pd.to_datetime(df_plan['day_hk']).apply(lambda x: x.date())
    df_plan['repay_channel'] = 2
    df_online_hk = pd.read_sql(
        sql_hk_online % (loan_class, fund_code, start_time_loanPaidAt, end_time_loanPaidAt, start_time, end_time),
        con=engine_financial)
    df_alipay_hk = pd.read_sql(
        sql_hk_alipay % (loan_class, fund_code, start_time_loanPaidAt, end_time_loanPaidAt, start_time, end_time),
        con=engine_financial)
    df_hk = pd.concat([df_alipay_hk, df_online_hk], axis=0, ignore_index=True)
    df_hk['repaid_at'] = pd.to_datetime(df_hk['repaid_at']).apply(lambda x: x.date())
    df_hk['deadline'] = pd.to_datetime(df_hk['deadline']).apply(lambda x: x.date())
    df_hk.fillna(0, inplace=True)
    df_hk['real_principle'] = df_hk['real_principle'] - df_hk['mitigate_principle']
    df_hk['real_interest'] = df_hk['real_interest'] - df_hk['mitigate_interest']
    df_hk['real_principle'] = df_hk['real_principle'] + df_hk['real_interest']
    df_hk.drop(['mitigate_principle', 'mitigate_interest'], axis=1,
               inplace=True)

    df_zc = df_hk.ix[df_hk['repaid_at'] == df_hk['deadline']]
    df_zc = df_zc.groupby(by=['repaid_at', 'repay_channel'])['real_principle'].sum()
    df_zc = df_zc.to_frame()
    df_zc.reset_index(inplace=True)
    df_zc.rename(columns={'repaid_at': 'day_hk', 'real_principle': 'zc_prin_inst_total'}, inplace=True)
    df_zc['day_hk'] = pd.to_datetime(df_zc['day_hk']).apply(lambda x: x.date())
    df_yq = df_hk.ix[df_hk['deadline'] < df_hk['repaid_at']]
    df_yq = df_yq.groupby(by=['repaid_at', 'repay_channel'])['real_principle'].sum()
    df_yq = df_yq.to_frame()
    df_yq.reset_index(inplace=True)
    df_yq.rename(columns={'repaid_at': 'day_hk', 'real_principle': 'yq_prin_inst_total'}, inplace=True)
    df_yq['day_hk'] = pd.to_datetime(df_yq['day_hk']).apply(lambda x: x.date())
    df_tq = df_hk.ix[df_hk['deadline'] > df_hk['repaid_at']]

    df_tq_1 = df_tq.groupby(by=['deadline', 'repay_channel'])['real_principle'].sum()
    df_tq_1 = df_tq_1.to_frame()
    df_tq_1.reset_index(inplace=True)
    df_tq_1.rename(columns={'deadline': 'day_hk', 'real_principle': 'plan_tq_prin_inst_total'}, inplace=True)
    df_tq_1['day_hk'] = pd.to_datetime(df_tq_1['day_hk']).apply(lambda x: x.date())
    df_tq = df_tq.groupby(by=['repaid_at', 'repay_channel'])['real_principle'].sum()
    df_tq = df_tq.to_frame()
    df_tq.reset_index(inplace=True)
    df_tq.rename(columns={'repaid_at': 'day_hk', 'real_principle': 'tq_prin_inst_total'}, inplace=True)
    df_tq['day_hk'] = pd.to_datetime(df_tq['day_hk']).apply(lambda x: x.date())
    df_hk_new = pd.merge(df_zc, df_yq, how='outer', on=['day_hk', 'repay_channel'])
    df_hk_new = pd.merge(df_hk_new, df_tq, how='outer', on=['day_hk', 'repay_channel'])
    df_hk_new = pd.merge(df_plan, df_hk_new, how='outer', on=['day_hk', 'repay_channel'])
    df_hk_new = pd.merge(df_hk_new, df_tq_1, how='outer', on=['day_hk', 'repay_channel'])
    df_hk_new.fillna(0, inplace=True)
    df_hk_new = pd.merge(df_hk_new, df_repay_channel, on=['repay_channel'], how='left')
    df_hk_new['real_all'] = df_hk_new['zc_prin_inst_total'] + df_hk_new['tq_prin_inst_total'] + df_hk_new[
        'yq_prin_inst_total']
    df_hk_new['fund_name'] = fund_name
    df_hk_new['loan_class'] = loan_class
    df_hk_new = df_hk_new[
        ['fund_name', 'day_hk', 'repay_channel', 'channel_name', 'plan_all', 'real_all', 'zc_prin_inst_total',
         'tq_prin_inst_total', 'yq_prin_inst_total', 'real_service_fee', 'real_punish', 'plan_tq_prin_inst_total',
         'loan_class']]
    df_hk_new.rename(columns={'day_hk': '还款日期', 'repay_channel': '还款账户', 'plan_all': '资金方应还本息', 'real_all': '客户实还本息',
                              'zc_prin_inst_total': '客户正常还款本息', 'yq_prin_inst_total': '客户逾期还款本息',
                              'tq_prin_inst_total': '客户提前还款本息', 'loan_class': '客户等级',
                              'plan_tq_prin_inst_total': '客户提前应还款'
                              },
                     inplace=True)
    df_hk_new.to_excel(path + str(fund_code) + '_' + loan_class + '.xlsx', index=None, columns=df_hk_new.columns,
                       encoding='utf8')


def hr_table(fund_code, fund_name, loan_class):
    df_plan = pd.read_sql(sql_fund_hk % (
        loan_class, fund_code, start_time_loanPaidAt, end_time_loanPaidAt, start_time_deadline, end_time_deadline),
                          con=engine_financial)
    df_plan.fillna(0, inplace=True)
    df_plan['plan_all'] = df_plan['plan_interest'] + df_plan['plan_principle']
    df_plan.drop(['plan_principle', 'plan_interest'], inplace=True, axis=1)
    df_plan['day_hk'] = pd.to_datetime(df_plan['day_hk']).apply(lambda x: x.date())
    df_plan['repay_channel'] = 2
    df_online_hk = pd.read_sql(sql_hk_fund_deadline_online % (
        loan_class, fund_code, start_time_loanPaidAt, end_time_loanPaidAt, start_repay_time, end_repay_time),
                               con=engine_financial)
    df_alipay_hk = pd.read_sql(sql_hk_fund_deadline_alipay % (
        loan_class, fund_code, start_time_loanPaidAt, end_time_loanPaidAt, start_repay_time, end_repay_time),
                               con=engine_financial)
    df_hk = pd.concat([df_alipay_hk, df_online_hk], axis=0, ignore_index=True)
    df_hk['repaid_at'] = pd.to_datetime(df_hk['repaid_at']).apply(lambda x: x.date())
    df_hk['deadline'] = pd.to_datetime(df_hk['deadline']).apply(lambda x: x.date())
    df_hk['fund_code'] = df_hk['fund_code'].astype(int)
    df_hk['fund_name'] = fund_name
    df_hk.fillna(0, inplace=True)
    df_hk['real_principle'] = df_hk['real_principle'] - df_hk['mitigate_principle']
    df_hk['real_interest'] = df_hk['real_interest'] - df_hk['mitigate_interest']
    df_hk['real_principle'] = df_hk['real_principle'] + df_hk['real_interest']
    df_hk.drop(['mitigate_principle', 'mitigate_interest'], axis=1,
               inplace=True)

    df_zc = df_hk.ix[df_hk['repaid_at'] == df_hk['deadline']]
    df_zc = df_zc.groupby(by=['repaid_at', 'repay_channel'])['real_principle'].sum()
    df_zc = df_zc.to_frame()
    df_zc.reset_index(inplace=True)
    df_zc.rename(columns={'repaid_at': 'day_hk', 'real_principle': 'zc_prin_inst_total'}, inplace=True)
    df_zc['day_hk'] = pd.to_datetime(df_zc['day_hk']).apply(lambda x: x.date())
    df_yq = df_hk.ix[df_hk['deadline'] < df_hk['repaid_at']]
    df_yq = df_yq.groupby(by=['repaid_at', 'repay_channel'])['real_principle'].sum()
    df_yq = df_yq.to_frame()
    df_yq.reset_index(inplace=True)
    df_yq.rename(columns={'repaid_at': 'day_hk', 'real_principle': 'yq_prin_inst_total'}, inplace=True)
    df_yq['day_hk'] = pd.to_datetime(df_yq['day_hk']).apply(lambda x: x.date())

    df_tq = df_hk.ix[df_hk['deadline'] > df_hk['repaid_at']]

    df_tq = df_tq.groupby(by=['deadline', 'repay_channel'])['real_principle'].sum()
    df_tq = df_tq.to_frame()
    df_tq.reset_index(inplace=True)
    df_tq['deadline'] = pd.to_datetime(df_tq['deadline']).apply(lambda x: x.date())

    df_tq_5 = df_tq.ix[df_tq.repaid_at >= datetime.date(2017, 5, 1)]
    df_tq = df_tq.ix[df_tq.repaid_at < datetime.date(2017, 5, 1)]
    # ==================================================================================
    df_advance_plan_id = pd.read_sql(
        sql_query_advance_settle_plan_id % (loan_class,
                                            fund_code, start_time_loanPaidAt, end_time_loanPaidAt, start_time_repay_huarong_settle,
                                            end_repay_time,
                                            start_time_deadline, fund_code, start_time_loanPaidAt, end_time_loanPaidAt,
                                            end_repay_time),
        con=engine_financial)
    advance_planid_list = df_advance_plan_id['plan_id'].tolist()
    advance_planid_set = set(advance_planid_list)

    advance_planid_list = map(int, advance_planid_list)
    df_tq_settle = pd.read_sql(sql_query_advance_settle % (loan_class,
                                                           fund_code, start_time_loanPaidAt, end_time_loanPaidAt,
                                                           str(tuple(advance_planid_list))),
                               con=engine_financial)
    df_tq_settle_max_deadline = df_tq_settle.groupby(by=['ref_id'])['repaid_at'].agg(['max']).reset_index().rename(
        columns={'max': 'max_repaid_at'})
    df_tq_settle_max_deadline.reset_index(inplace=True)
    df_tq_settle_max_deadline['max_repaid_at'] = pd.to_datetime(df_tq_settle_max_deadline['max_repaid_at']).apply(
        lambda x: x.date())

    df_tq_settle = pd.merge(df_tq_settle, df_tq_settle_max_deadline, how='left', on=['ref_id'])
    df_tq_settle_leak = df_tq_settle.ix[df_tq_settle.max_repaid_at < datetime.date(2017, 5, 1)]
    df_tq_settle_leak.drop(['max_repaid_at'], inplace=True, axis=1)

    df_tq_settle = df_tq_settle.ix[df_tq_settle.max_repaid_at >= datetime.date(2017, 5, 1)]
    df_tq_settle.drop(['repaid_at'], inplace=True, axis=1)
    df_tq_settle.rename(columns={'deadline': 'repaid_at'}, inplace=True)
    df_tq_settle.rename(columns={'max_repaid_at': 'deadline'}, inplace=True)

    df_tq_no_settle = df_tq_5[~df_tq_5.plan_id.isin(advance_planid_set)]
    df_tq = pd.concat([df_tq_no_settle, df_tq_settle, df_tq_settle_leak, df_tq], ignore_index=True)
    df_tq = df_tq.groupby(by=['deadline', 'repay_channel'])['real_principle'].sum()
    df_tq = df_tq.to_frame()
    df_tq.reset_index(inplace=True)
    df_tq['deadline'] = pd.to_datetime(df_tq['deadline']).apply(lambda x: x.date())
    df_tq.rename(columns={'deadline': 'day_hk', 'real_principle': 'tq_prin_inst_total'}, inplace=True)
    df_tq['repay_channel'] = df_tq['repay_channel'].astype(int)
    # ==========================================================

    df_hk_new = pd.merge(df_zc, df_yq, how='outer', on=['day_hk', 'repay_channel'])
    df_hk_new = pd.merge(df_hk_new, df_tq, how='outer', on=['day_hk', 'repay_channel'])
    df_hk_new = pd.merge(df_plan, df_hk_new, how='outer', on=['day_hk', 'repay_channel'])
    df_hk_new.fillna(0, inplace=True)
    df_hk_new = pd.merge(df_hk_new, df_repay_channel, on=['repay_channel'], how='left')
    df_hk_new['real_all'] = df_hk_new['zc_prin_inst_total'] + df_hk_new['tq_prin_inst_total'] + df_hk_new[
        'yq_prin_inst_total']
    df_hk_new['fund_name'] = fund_name
    df_hk_new['loan_class'] = loan_class
    df_hk_new = df_hk_new[
        ['fund_name', 'day_hk', 'repay_channel', 'channel_name', 'plan_all', 'real_all', 'zc_prin_inst_total',
         'tq_prin_inst_total', 'yq_prin_inst_total', 'loan_class']]
    df_hk_new.rename(
        columns={'day_hk': u'还款日期', 'repay_channel': u'还款账户', 'plan_all': u'应还资金方本息', 'real_all': u'客户实还本息',
                 'zc_prin_inst_total': u'客户正常还款本息', 'yq_prin_inst_total': u'客户逾期还款本息',
                 'tq_prin_inst_total': u'客户提前还款本息', 'loan_class': u'客户等级'},
        inplace=True)
    df_hk_new.to_excel(path + str(fund_code) + '_' + loan_class + '.xlsx', index=None, columns=df_hk_new.columns,
                       encoding='utf8')


def other_table(fund_code, fund_name, loan_class):
    if fund_code == 200:
        df_plan = pd.read_sql(sql_plan_hk % (
            loan_class, fund_code, start_time_loanPaidAt, end_time_loanPaidAt, start_time_deadline, end_time_deadline),
                              con=engine_financial)
    elif fund_code == 50:
        df_plan = pd.read_sql(sql_A, engine_financial)
    elif fund_code == 80:
        df_plan = pd.read_sql(sql_C, engine_financial)
    elif fund_code == 130:
        df_plan = pd.read_sql(sql_D, engine_financial)
    elif fund_code in (270, 310, 320, 210):
        df_plan = pd.read_sql(sql_fund_hk_user_plan % (
            loan_class, fund_code, start_time_loanPaidAt, end_time_loanPaidAt, start_time_deadline, end_time_deadline),
                              con=engine_financial)
        df_plan.fillna(0, inplace=True)
        df_plan['plan_all'] = df_plan['plan_interest'] + df_plan['plan_principle']
        df_plan.drop(['plan_principle', 'plan_interest'], inplace=True, axis=1)
    else:
        df_plan = pd.read_sql(
            sql_fund_hk % (loan_class, fund_code, start_time_loanPaidAt, end_time_loanPaidAt, start_time, end_time),
            con=engine_financial)
        df_plan.fillna(0, inplace=True)
        df_plan['plan_all'] = df_plan['plan_interest'] + df_plan['plan_principle']
        df_plan.drop(['plan_principle', 'plan_interest'], inplace=True, axis=1)

    df_plan['day_hk'] = pd.to_datetime(df_plan['day_hk']).apply(lambda x: x.date())
    df_plan['repay_channel'] = 2

    if fund_code in [2, 3, 4, 5]:
        df_online_hk = pd.read_sql(sql_hk_fund_deadline_online % (
            loan_class, fund_code, start_time_loanPaidAt, end_time_loanPaidAt, start_time, end_time),
                                   con=engine_financial)
        df_alipay_hk = pd.read_sql(sql_hk_fund_deadline_alipay % (
            loan_class, fund_code, start_time_loanPaidAt, end_time_loanPaidAt, start_time, end_time),
                                   con=engine_financial)
        df_bank_hk = pd.read_sql(sql_hk_fund_deadline_bank % (
            loan_class, fund_code, start_time_loanPaidAt, end_time_loanPaidAt, start_time, end_time),
                                 con=engine_financial)
    else:
        df_online_hk = pd.read_sql(
            sql_hk_online % (loan_class, fund_code, start_time_loanPaidAt, end_time_loanPaidAt, start_time, end_time),
            con=engine_financial)
        df_alipay_hk = pd.read_sql(
            sql_hk_alipay % (loan_class, fund_code, start_time_loanPaidAt, end_time_loanPaidAt, start_time, end_time),
            con=engine_financial)
        df_bank_hk = pd.read_sql(
            sql_hk_bank % (loan_class, fund_code, start_time_loanPaidAt, end_time_loanPaidAt, start_time, end_time),
            con=engine_financial)
    df_hk = pd.concat([df_alipay_hk, df_online_hk, df_bank_hk], axis=0, ignore_index=True)
    df_hk['repaid_at'] = pd.to_datetime(df_hk['repaid_at']).apply(lambda x: x.date())
    df_hk['deadline'] = pd.to_datetime(df_hk['deadline']).apply(lambda x: x.date())
    df_hk.fillna(0, inplace=True)
    df_hk['real_principle'] = df_hk['real_principle'] - df_hk['mitigate_principle']
    df_hk['real_interest'] = df_hk['real_interest'] - df_hk['mitigate_interest']
    df_hk['real_principle'] = df_hk['real_principle'] + df_hk['real_interest']
    df_hk.drop(['mitigate_principle', 'mitigate_interest'], axis=1,
               inplace=True)
    if len(df_hk) < 1:
        return

    df_zc = df_hk.ix[df_hk['repaid_at'] == df_hk['deadline']]
    df_zc = df_zc.groupby(by=['repaid_at', 'repay_channel'])['real_principle'].sum()
    df_zc = df_zc.to_frame()
    df_zc.reset_index(inplace=True)
    df_zc.rename(columns={'repaid_at': 'day_hk', 'real_principle': 'zc_prin_inst_total'}, inplace=True)
    df_zc['day_hk'] = pd.to_datetime(df_zc['day_hk']).apply(lambda x: x.date())
    df_zc['repay_channel'] = df_zc['repay_channel'].astype(int)
    df_yq = df_hk.ix[df_hk['deadline'] < df_hk['repaid_at']]

    df_yq = df_yq.groupby(by=['repaid_at', 'repay_channel'])['real_principle'].sum()
    df_yq = df_yq.to_frame()
    df_yq.reset_index(inplace=True)
    df_yq.rename(columns={'repaid_at': 'day_hk', 'real_principle': 'yq_prin_inst_total'}, inplace=True)
    df_yq['day_hk'] = pd.to_datetime(df_yq['day_hk']).apply(lambda x: x.date())
    df_yq['repay_channel'] = df_yq['repay_channel'].astype(int)
    df_tq = df_hk.ix[df_hk['deadline'] > df_hk['repaid_at']]
    df_tq = df_tq.groupby(by=['deadline', 'repay_channel'])['real_principle'].sum()
    df_tq = df_tq.to_frame()
    df_tq.reset_index(inplace=True)
    df_tq['deadline'] = pd.to_datetime(df_tq['deadline']).apply(lambda x: x.date())
    df_tq.rename(columns={'deadline': 'day_hk', 'real_principle': 'tq_prin_inst_total'}, inplace=True)
    df_tq['repay_channel'] = df_tq['repay_channel'].astype(int)

    df_hk_new = pd.merge(df_zc, df_yq, how='outer', on=['day_hk', 'repay_channel'])
    df_hk_new = pd.merge(df_hk_new, df_tq, how='outer', on=['day_hk', 'repay_channel'])
    df_hk_new = pd.merge(df_plan, df_hk_new, how='outer', on=['day_hk', 'repay_channel'])
    df_hk_new.fillna(0, inplace=True)
    df_hk_new['real_all'] = df_hk_new['zc_prin_inst_total'] + df_hk_new['tq_prin_inst_total'] + df_hk_new[
        'yq_prin_inst_total']
    df_hk_new['fund_name'] = fund_name
    df_hk_new['loan_class'] = loan_class
    df_hk_new = pd.merge(df_hk_new, df_repay_channel, on=['repay_channel'], how='left')
    df_hk_new = df_hk_new[
        ['fund_name', 'day_hk', 'repay_channel', 'channel_name', 'plan_all', 'real_all', 'zc_prin_inst_total',
         'tq_prin_inst_total', 'yq_prin_inst_total', 'loan_class']]
    df_hk_new.rename(
        columns={'day_hk': u'还款日期', 'repay_channel': u'还款账户', 'plan_all': u'应还资金方本息', 'real_all': u'客户实还本息',
                 'zc_prin_inst_total': u'客户正常还款本息', 'yq_prin_inst_total': u'客户逾期还款本息',
                 'tq_prin_inst_total': u'客户提前还款本息', 'loan_class': u'客户等级'},
        inplace=True)
    df_hk_new.to_excel(path + str(fund_code) + '_' + loan_class + '.xlsx', index=None, columns=df_hk_new.columns,
                       encoding='utf8')


def xms_table(fund_code, fund_name, loan_class):
    read_ms_sql = '''
    SELECT '160' as fund_code,`deadline`,`repaid_at`,`funding_principle`+`funding_interest`+`funding_service_fee` as prin_inst_total,`qg_service_fee` real_service_fee ,`qg_overdue_fee` real_punish
    FROM temp.`new_ms_transaction_with_term` t1 JOIN all_back_0531.loan_manifest t2 ON  cast(t1.loan_id as char) = t2.loan_id JOIN temp.loan_class t3 on t2.ref_id=t3.ref_id WHERE t3.class='%s'
    '''
    df_ms = pd.read_sql(read_ms_sql % (loan_class,), con=engine_financial)
    df_ms['repaid_at'].fillna('2016-01-01', inplace=True)
    df_ms['repaid_at'] = pd.to_datetime(df_ms['repaid_at']).apply(lambda x: x.date())
    df_ms['deadline'] = pd.to_datetime(df_ms['deadline']).apply(lambda x: x.date())
    df_zc = df_ms.ix[df_ms['deadline'] == df_ms['repaid_at']]
    df_zc = df_zc.groupby(by=['repaid_at', 'fund_code'])['prin_inst_total'].sum()
    df_zc = df_zc.to_frame()
    df_zc.reset_index(inplace=True)
    df_zc.rename(columns={'repaid_at': 'day_hk', 'prin_inst_total': 'zc_prin_inst_total'}, inplace=True)
    df_zc['day_hk'] = pd.to_datetime(df_zc['day_hk']).apply(lambda x: x.date())
    df_zc['fund_code'] = df_zc['fund_code'].astype(int)
    df_tq = df_ms.ix[df_ms['deadline'] > df_ms['repaid_at']]
    df_tq = df_tq.groupby(by=['deadline', 'fund_code'])['prin_inst_total'].sum()
    df_tq = df_tq.to_frame()
    df_tq.reset_index(inplace=True)
    df_tq['deadline'] = pd.to_datetime(df_tq['deadline']).apply(lambda x: x.date())
    df_tq.rename(columns={'deadline': 'day_hk', 'prin_inst_total': 'tq_prin_inst_total'}, inplace=True)
    df_tq['day_hk'] = pd.to_datetime(df_tq['day_hk']).apply(lambda x: x.date())
    df_tq['fund_code'] = df_tq['fund_code'].astype(int)
    df_yq = df_ms.ix[df_ms['deadline'] < df_ms['repaid_at']]
    df_yq = df_yq.groupby(by=['repaid_at', 'fund_code'])['prin_inst_total'].sum()
    df_yq = df_yq.to_frame()
    df_yq.reset_index(inplace=True)
    df_yq.rename(columns={'repaid_at': 'day_hk', 'prin_inst_total': 'yq_prin_inst_total'}, inplace=True)
    df_yq['day_hk'] = pd.to_datetime(df_yq['day_hk']).apply(lambda x: x.date())
    df_yq['fund_code'] = df_yq['fund_code'].astype(int)
    list_date = pd.date_range('20170101', '20170601')
    df_date = pd.DataFrame(list_date)
    df_date.columns = ['day_hk']
    df_date.reset_index(inplace=True)
    df_date['day_hk'] = pd.to_datetime(df_date['day_hk']).apply(lambda x: x.date())
    fund_id = [160]
    df_date = pd.DataFrame(list(itertools.product(df_date['day_hk'].values.tolist(), fund_id)),
                           columns=['day_hk', 'fund_code'])
    df_date['day_hk'] = pd.to_datetime(df_date['day_hk']).apply(lambda x: x.date())
    df_date['fund_code'] = df_date['fund_code'].astype(int)
    df_out = pd.merge(df_date, df_zc, how='left', on=['day_hk', 'fund_code'])
    df_out = pd.merge(df_out, df_tq, how='left', on=['day_hk', 'fund_code'])
    df_out = pd.merge(df_out, df_yq, how='left', on=['day_hk', 'fund_code'])
    df_out.fillna(0, inplace=True)
    df_out['prin_inst_real'] = df_out['zc_prin_inst_total'] + df_out['yq_prin_inst_total'] + df_out[
        'tq_prin_inst_total']
    df_out.fillna(0, inplace=True)
    df_out['day_hk'] = pd.to_datetime(df_out['day_hk']).apply(lambda x: x.date())
    df_out['fund_code'] = df_out['fund_code'].astype(int)
    read_1_sql = '''
    SELECT DATE(deadline) day_hk,sum(nmrp.`principle`+nmrp.`interest_new`+nmrp.`funding_service_fee`/4) prin_inst_plan FROM `new_ms_repayment_plan` nmrp JOIN all_back_0531.loan_manifest lm ON nmrp.loan_id = lm.loan_id JOIN temp.loan_class lc ON lm.ref_id = lc.ref_id WHERE lc.class='%s' AND lm.loan_paid_at>='%s' AND lm.loan_paid_at<'%s'  group by date(deadline)
    '''
    df_plan = pd.read_sql(read_1_sql % (loan_class, start_time_loanPaidAt, end_time_loanPaidAt),
                          con=engine_financial)
    df_plan.fillna(0, inplace=True)
    df_plan['day_hk'] = pd.to_datetime(df_plan['day_hk']).apply(lambda x: x.date())
    df_plan['fund_code'] = 160
    df_plan['prin_inst_plan'] = df_plan['prin_inst_plan'].astype(float)
    df_out['day_hk'] = pd.to_datetime(df_out['day_hk']).apply(lambda x: x.date())
    df_out['fund_code'] = df_out['fund_code'].astype(int)
    df_out = pd.merge(df_out, df_plan, how='left', on=['day_hk', 'fund_code'])
    df_out.fillna(0, inplace=True)
    df_out['channel_name'] = '新马上'
    df_out['loan_class'] = loan_class
    df_out.rename(columns={'day_hk': u'客户还款日期', 'zc_prin_inst_total': u'客户正常还款本息', 'tq_prin_inst_total': u'客户提前还款本息',
                           'yq_prin_inst_total': u'客户逾期还款本息',
                           'prin_inst_plan': u'客户应还本息',
                           'prin_inst_real': u'客户实还本息',
                           'fund_amount': u'资金方本息', 'loan_class': u'客户等级'}, inplace=True)
    df_out.to_excel(path + str(fund_code) + '_' + loan_class + '.xlsx', index=None, columns=df_out.columns,
                    encoding='utf8')


loan_class_list = ['A', 'B', 'C']
if __name__ == '__main__':
    pool = mtp.Pool(processes=6)
    for index, row in df_fund.iterrows():
        for loan_class_ in loan_class_list:
            fund_code = int(row['fund_code'])
            fund_name = row['fund_name']
            if fund_code in (200, 290, 330):
                pool.apply_async(ky_table, (fund_code, fund_name, loan_class_))
            elif fund_code == 150:
                pool.apply_async(hr_table, (fund_code, fund_name, loan_class_))
            elif fund_code == 160:
                pool.apply_async(xms_table, (fund_code, fund_name, loan_class_))
            else:
                pool.apply_async(other_table, (fund_code, fund_name, loan_class_))
    pool.close()
    pool.join()
    # excel_out = pd.ExcelWriter(path + 'all.xlsx')
    # df_out = pd.DataFrame()
    # for file in os.listdir(path):
    #     tmp_file = file.replace('.xlsx', '')
    #     if tmp_file in ['160_A', '200_A', '290_A', '160_B', '200_B', '290_B', '160_C', '200_C', '290_C'] > 0:
    #         df_tmp = pd.read_excel(path + file)
    #         df_tmp.to_excel(excel_out, sheet_name=tmp_file, columns=df_tmp.columns, index=None, encoding='utf8')
    #     elif tmp_file == 'all':
    #         continue
    #     else:
    #         df_out = df_out.append(pd.read_excel(path + file))
    # df_out.to_excel(excel_out, sheet_name='sheet1', columns=df_out.columns, index=None, encoding='utf8')
    # excel_out.save()
print '=====================main done======================'
