# coding: utf-8
import numpy as np
import pandas as pd
from sqlalchemy import create_engine

"""
关联还款计划，插入到中间表ref表中

"""
pd.options.mode.chained_assignment = None

engine_new_transaction = create_engine(
    'mysql+mysqldb://yulong:ohVU1nPITynSZwR2@172.16.3.201:3306/new_transaction?charset=utf8', echo=True)
engine_xyqb = create_engine('mysql+mysqldb://syzhu_data_r:ulk0989742097y43@192.168.4.6:8066/xyqb?charset=utf8',
                            echo=True)
engine_offline = create_engine('mysql+mysqldb://syzhu_data_r:ulk0989742097y43@192.168.4.6:8066/offline?charset=utf8',
                               echo=True)
engine_pay_center = create_engine(
    'mysql+mysqldb://syzhu_data_r:ulk0989742097y43@192.168.4.6:8066/payment_center?charset=utf8', echo=True)

path = u"E:/baitiao/线上白条还款/2017-09/"

bill_start_time = '2017-09-01'
bill_end_time = '2017-10-01'


def process_plan_id(df):
    print '-------------------------------len(df): %s-------------------------' % len(df)
    df['merchant_repay_order_no'] = df['merchant_repay_order_no'].astype(str)
    df_plan = pd.DataFrame(columns=df.columns)
    for index, row in df.iterrows():
        a = filter(lambda x: x.isdigit(), row['merchant_repay_order_no'].split('_'))
        tmp = pd.DataFrame(index=xrange(len(a)), columns=df.columns)
        for i in xrange(len(a)):
            tmp.ix[i, :] = row[:]
            tmp.ix[i, 'repay_plan_id'] = a[i]
        df_plan = df_plan.append(tmp)
    return df_plan


from threading import Thread


class myThread(Thread):
    def __init__(self, df):
        Thread.__init__(self)
        self.df = df

    def run(self):
        self.result = process_plan_id(self.df)

    def get_result(self):
        return self.result


sql_record = """
  select id record_id, pay_approach repay_channel, order_id 
  from new_transaction.repay_record_online 
  where pay_approach in(3,5,7,22,23,24)
    and enable = 1 
    and is_ref = 0 
    AND  bill_time>='%s' 
    AND bill_time<'%s'
    """

select_zt = """
select repay_plan_id from new_transaction.bt_funding_repayment_record WHERE repay_plan_id in %s
"""
sql_pay_center = """
  select order_id, merchant_repay_order_no, merchant_id 
  from repay_order t 
  where t.repay_status = 3 
  and merchant_id in (2,4,15)
  and t.order_id in %s """
sql_bt_online = """
  select 'XYQB_BT' sys_tag, t1.id repay_plan_id, 
  (ifnull(t1.current_repayment,0)-ifnull(t1.collection_relief,0))current_repayment,
  t2.order_no,t1.term_no ,
  t1.origin_amount principal, t1.interest_amount interest, t1.overstay_amount
  from baitiao_repayment_plan t1 
  join baitiao_order t2 on t2.id = t1.order_id 
  where t1.id in %s
UNION ALL 
 select 'XYQB_SHIGU_BT' sys_tag ,t1.id repay_plan_id, 
  (t1.current_repayment-t1.collection_relief)current_repayment,t2.order_no,t1.term_no ,
  t1.origin_amount principal, t1.interest_amount interest, t1.overstay_amount
 from baitiao_repayment_plan_mall_shigu t1 
 join baitiao_order_mall_shigu t2 on t2.id = t1.order_id 
 where t1.id in %s
"""
sql_bill_online = """
select 'XYQB_BILL_BT' sys_tag, id repay_plan_id, repay_amount current_repayment,order_no,term_no ,
   principal, interest, penalty_interest overstay_amount
from baitiao_bill_repayment_plan 
where id in %s
"""

sql_plan_id = """
select plan.id plan_id, plan.repayment_plan_id repay_plan_id, plan.sys_tag,plan.ref_id,plan.order_no,
  plan.term_no,b1.merchantId,b1.is_active 
from new_transaction.user_bt_repayment_plan plan
JOIN baitiao_audit.baitiao_order b1 ON  b1.ref_id = plan.ref_id
where plan.repayment_plan_id in %s"""

sql_offline = """
  select 'OFFLINE_BT' sys_tag, id repay_plan_id,  
    principal + interest + ifnull(overdue_amount, 0) + collection_relief current_repayment 
  from order_repaid_record 
  where id in %s
"""
update_is_ref = """update repay_record_online set is_ref =1 WHERE  id = %s """


def transfer_df(df):
    df['merchant_repay_order_no'] = df['merchant_repay_order_no'].astype(str)
    df['principal'] = df['principal'].apply(lambda x: np.round(x, 2))
    df['interest'] = df['interest'].apply(lambda x: np.round(x, 2))
    df['overstay_amount'] = df['overstay_amount'].apply(lambda x: np.round(x, 2))
    return df


# 根据文件名和内容，写入文件。成功返回1

def write_file(filename, filecontent):
    try:
        fo = open(filename, 'ab+')
        fo.write(filecontent)
        fo.write('\n')
    finally:
        fo.close()


update_sql = """update baitiao_repay_plan_repay_record_ref set ref_amount=%s WHERE record_id=%s AND plan_id=%s AND repay_channel=%s"""


def online_bt(df_online):
    if len(df_online) > 0:
        df_index = df_online.index
        td_list = []
        for i in xrange(0, len(df_online), frame):
            indexs = df_index[i:i + frame]
            # 多线程处理还款计划
            td = myThread(df_online.ix[indexs])
            td.start()
            td_list.append(td)
        # 等待线程结束
        for td in td_list:
            td.join()
        result_list = []
        # 获取线程返回的结果
        for td in td_list:
            result_list.append(td.get_result())
        df_online_handled = pd.concat(result_list, ignore_index=True)
        with open(path + 'df_online_handled.csv', 'a') as f:
            df_online_handled.to_csv(f, index=None)

        # 区分账单白条和普通白条
        df_bill_bt = df_online_handled.ix[df_online_handled.merchant_repay_order_no.str.contains('BILL')]
        df_online_bt = df_online_handled.ix[~df_online_handled.merchant_repay_order_no.str.contains('BILL')]
        if len(df_bill_bt) > 0:
            df_bill_bt.repay_plan_id = df_bill_bt.repay_plan_id.astype(str)
            bill_plan_id = df_bill_bt.repay_plan_id.tolist()
            repay_bill_online = pd.read_sql(sql=sql_bill_online % str(tuple(bill_plan_id)).replace(',)', ')')
                                            , con=engine_xyqb)
            repay_bill_online.repay_plan_id = repay_bill_online.repay_plan_id.astype(str)
            df_zt = pd.read_sql(sql=select_zt % str(tuple(bill_plan_id)).replace(',)', ')'), con=engine_new_transaction)
            df_zt.repay_plan_id = df_zt.repay_plan_id.astype(str)
            repay_bill_online = repay_bill_online.ix[~repay_bill_online['repay_plan_id'].isin(df_zt['repay_plan_id'])]
            df_bill_bt = df_bill_bt.merge(repay_bill_online, on='repay_plan_id')
        if len(df_online_bt) > 0:
            df_online_bt.repay_plan_id = df_online_bt.repay_plan_id.astype(str)
            bt_plan_id = df_online_bt.repay_plan_id.tolist()
            repay_bt_online = pd.read_sql(sql=sql_bt_online % (
                str(tuple(bt_plan_id)).replace(',)', ')'), str(tuple(bt_plan_id)).replace(',)', ')')), con=engine_xyqb)
            repay_bt_online.repay_plan_id = repay_bt_online.repay_plan_id.astype(str)
            df_online_bt = df_online_bt.merge(repay_bt_online, on='repay_plan_id')
        df_all_bt = pd.concat([df_bill_bt, df_online_bt])
        print '------------------------len(df_all_bt): %s ------------------------------' % len(df_all_bt)
        df_all_bt['ref_amount'] = df_all_bt['current_repayment']
        df_all_online_bt = df_all_bt
        df_all_online_bt['repay_plan_id'] = df_all_online_bt['repay_plan_id'].astype(str)
        repay_plan_ids = df_all_online_bt['repay_plan_id'].tolist()
        df_plan = pd.read_sql(sql=sql_plan_id % str(tuple(repay_plan_ids)).replace(',)', ')'),
                              con=engine_new_transaction)
        # 处理还款计划的 重复的repayment_plan_id 的问题，all 白条
        df_plan = df_plan.merge(
            df_plan.groupby(by=['sys_tag', 'repay_plan_id'])['repay_plan_id'].agg({'count'}).reset_index().rename(
                columns={'count': 'count_by_xyqb_plan_id'}))
        df_plan_1 = df_plan.ix[df_plan['count_by_xyqb_plan_id'] == 1]
        df_plan_n = df_plan.ix[df_plan['count_by_xyqb_plan_id'] > 1]
        # 选择 使用哪个 ref_id--- 提取 is_active = 1 的ref_id
        df_plan_n_1 = df_plan_n.ix[df_plan_n['is_active'] == 1]
        # is_active=0的数据
        df_plan_n_0 = df_plan_n.ix[df_plan_n['is_active'] == 0]
        if len(df_plan_n_0) > 0:
            write_file(path + u'重复还款id的数据.txt', str(df_plan_n_0['repay_plan_id'].tolist()))
        # 合并
        df_plan = pd.concat([df_plan_1, df_plan_n_1, df_plan_n_0], ignore_index=True, axis=0)
        df_all_online_bt = pd.merge(df_all_online_bt, df_plan, on=['repay_plan_id', 'sys_tag'], how='left')
        if len(df_all_online_bt.ix[df_all_online_bt.plan_id.isnull()]) > 0:
            print '----------------------------------online new transaction miss data---------------------'
            with open(path + 'online_new_transaction_miss_data.csv', 'a') as f:
                df_all_online_bt.ix[df_all_online_bt.plan_id.isnull()].to_csv(f, index=None)
        df_all_online_bt = df_all_online_bt.ix[
            df_all_online_bt.plan_id.notnull(), ['record_id', 'ref_amount', 'plan_id', 'repay_channel']]
        df_all_online_bt['repay_type'] = 'ONLINE'
        # df_all_online_bt.to_sql('baitiao_repay_plan_repay_record_ref', con=engine_new_transaction, chunksize=step,
        #                         if_exists='append', index=None)
        # 更新repay_record_onlie的is_ref
        record_ids = df_all_online_bt[['record_id']].values.tolist()
        # engine_new_transaction.execute(update_is_ref, record_ids)
        with open(path + 'online_need_to_insert.csv', 'a') as f:
            df_all_online_bt.to_csv(f, index=None)

        print '--------------------------------------online ok--------------------------------------------'


def offline_bt(df_offline):
    if len(df_offline) > 0:
        df_offline['merchant_repay_order_no'] = df_offline['merchant_repay_order_no'].astype(str)
        columns = list(df_offline.columns)
        # 由于线上数据有部分问题，需要把带有下划线的全部给拆开
        df_offline_handled = pd.DataFrame(columns=columns)
        for index, row in df_offline.iterrows():
            a = row['merchant_repay_order_no'].split(',')
            tmp = pd.DataFrame(index=xrange(len(a)), columns=columns)
            for i in xrange(len(a)):
                tmp.ix[i, :] = row[:]
                tmp.ix[i, 'repay_plan_id'] = a[i]
            df_offline_handled = df_offline_handled.append(tmp)
        with open(path + 'df_offline_handled.csv', 'a') as f:
            df_offline_handled.to_csv(f, index=None)
        df_offline_handled.repay_plan_id = df_offline_handled.repay_plan_id.astype(str)

        plan_id = df_offline_handled.repay_plan_id.tolist()
        repay_offline = pd.read_sql(sql=sql_offline % str(tuple(plan_id)).replace(',)', ')'), con=engine_offline)
        repay_offline.repay_plan_id = repay_offline.repay_plan_id.astype(str)
        df_all_offline = df_offline_handled.merge(repay_offline, on='repay_plan_id')
        print '------------------------------------------------\n', df_all_offline.head(), '-------------------------------------------------------\n'
        if len(df_all_offline) < len(df_offline_handled):
            print '---------------------------offline db miss data-----------------------------'
            with open(path + 'offline_db_miss_data.csv', 'a') as f:
                df_offline_handled.ix[~df_offline_handled.repay_plan_id.isin(df_all_offline.repay_plan_id)].to_csv(f,
                                                                                                                   index=None)
        df_all_offline['ref_amount'] = df_all_offline['current_repayment']
        df_all_offline_bt = df_all_offline
        df_all_offline_bt['repay_plan_id'] = df_all_offline_bt['repay_plan_id'].astype(str)
        repay_plan_ids = df_all_offline_bt['repay_plan_id'].tolist()

        df_plan = pd.read_sql(sql=sql_plan_id % str(tuple(repay_plan_ids)).replace(',)', ')'),
                              con=engine_new_transaction)
        df_all_offline_bt = pd.merge(df_all_offline_bt, df_plan, on=['repay_plan_id', 'sys_tag'], how='left')

        if len(df_all_offline_bt.ix[df_all_offline_bt.plan_id.isnull()]) > 0:
            print '----------------------------------offline new transaction miss data---------------------'
            with open(path + 'offline_new_transaction_miss_data.csv', 'a') as f:
                df_all_offline_bt.ix[df_all_offline_bt.plan_id.isnull()].to_csv(f, index=None)

        df_all_offline_bt = df_all_offline_bt.ix[
            df_all_offline_bt.plan_id.notnull(), ['record_id', 'ref_amount', 'plan_id', 'repay_channel']]

        df_all_offline_bt['repay_type'] = 'ONLINE'
        # df_all_offline_bt.to_sql('baitiao_repay_plan_repay_record_ref', con=engine_new_transaction, chunksize=step,
        #                          if_exists='append', index=None)
        # 更新repay_record_onlie的is_ref
        record_ids = df_all_offline_bt[['record_id']].values.tolist()
        # engine_new_transaction.execute(update_is_ref, record_ids)

        with open(path + 'offline_need_to_insert.csv', 'a') as f:
            df_all_offline_bt.to_csv(f, index=None)

        print '--------------------------------------offline ok--------------------------------------------'


step = 10000
frame = 2000
df_record_gen = pd.read_sql(sql=sql_record % (bill_start_time, bill_end_time), con=engine_new_transaction,
                            chunksize=step)
i = 0
for df_record in df_record_gen:
    print '-----------------------%s---len(df_record): %s ------------------------------' % (i, len(df_record))
    i += 1
    df_record.order_id = df_record.order_id.astype(str)
    order_id_list = df_record.order_id.tolist()
    df_pay_center = pd.read_sql(sql=sql_pay_center % str(tuple(order_id_list)).replace(',)', ')'),
                                con=engine_pay_center)
    df_record = pd.merge(df_record, df_pay_center, on='order_id', how='left')
    df_record_not = df_record.ix[df_record.merchant_repay_order_no.isnull()]
    df_record = df_record.ix[df_record.merchant_repay_order_no.notnull()]
    df_record.merchant_repay_order_no = df_record.merchant_repay_order_no.astype(str)
    if len(df_record_not) > 0:
        with open(path + 'df_record_not_in_pay_center.csv', 'a') as f:
            df_record_not.to_csv(f, index=None)

    # 处理线上数据
    df_online = df_record.ix[df_record['merchant_id'].isin([4, 15])]
    online_bt(df_online)

    # 处理线下数据
    df_offline = df_record.ix[df_record.merchant_id == 2]
    offline_bt(df_offline)

print '--------------------------------all done!----------good luck!----------------------------------------'
