# -*- encoding: utf8 -*-
import sys
import os
import datetime, time
import numpy as np
import pandas as pd
from dateutil.relativedelta import relativedelta
from sqlalchemy import create_engine
import math

reload(sys)
sys.setdefaultencoding("utf-8")
pd.options.mode.chained_assignment = None
# 去哪儿源文件
path = u'E:/核对账单/去哪儿/去哪儿账单文件/201701'
engine_qunaer = create_engine('mysql+mysqldb://yulong:ohVU1nPITynSZwR2@172.16.3.201:3306/qunaer_new?charset=utf8',
                              echo=False).connect()
engine_djy = create_engine('mysql+mysqldb://wanglinfang_data_r:uh90888n7y8ijk8886@192.168.4.6:8066/dujiangyan?charset=utf8',echo=False).connect()
'''
@author linfang
1、qunaer_new 库中补录 16年的 数据
'''

# 取小数点 前两位
def get_2_flo(num):
    begin = str(num).find('.')
    if begin == -1:
        return round(begin, 2)
    return float(str(num)[:begin + 3])
# 小数向上取数
def get_2_ceil(num):
    begin = str(num).find('.')
    if begin == -1:
        return round(begin, 2)
    else:
        if str(num).__len__() - begin > 3:
            return math.ceil(num * 100) / 100
        else:
            return float(str(num)[:begin + 3])
# 补录16年去哪儿的转分期文件---
def loan_trans_16(loan_nos):
    sql_djy = '''
        select loan_no product_no,loan_amount,loan_time trans_time,qunar_user_id user_no,term_no stages,'1' is_ptf
        from qunar_loanpay_history
        where status IN ('CONVERSION','SUCCESS') and
        loan_no in %s
        ''' % str(tuple(loan_nos))
    df_djy = pd.read_sql(sql_djy, engine_djy)
    print len(df_djy)
    sql_lx = '''
    select product_no,loan_time from qunaer.pay_detail where loan_amount >0 and staged_or_not = 1 and product_no in %s
    ''' % str(tuple(loan_nos))
    df_lx = pd.read_sql(sql_lx, engine_qunaer)
    print '----都江堰-李学库====', len(df_djy) - len(df_lx)
    df_new = pd.merge(df_djy, df_lx, on='product_no', how='left')
    # 成功状态
    df_new['status'] = 1
    print df_new.info()
    print df_new.ix[df_new.loan_time.isnull()]
    df_new.to_sql('pay_detail', engine_qunaer, index=None, if_exists='append', chunksize=10000)

# 导入16年的还款计划
def load_16_plan(year, month, insert_db=False):
    start_time = datetime.datetime(year, month, 1, 0, 0, 0)
    end_time = start_time + relativedelta(months=+1)
    # sql_lx = '''
    # SELECT product_no,current_stage_no,fee_rate rate,finish_or_not status,current_real_repay_date repay_time,repay_service_fee repay_fee_amount,
    # real_repay_principle repay_principle,repay_over_due_fine repay_due_amount,
    # supposed_fee_rule fee_amount,supposed_repay_date deadline,supposed_principle_rule principle,supposed_over_due_fine_rule due_amount
    # FROM qunaer.`repayment_plan_2016_deyu`
    # WHERE supposed_repay_date >= '%s' AND supposed_repay_date < '%s'
    # ''' % (start_time, end_time)
    sql_lx = '''
        SELECT product_no,current_stage_no,fee_rate rate,finish_or_not status,
        supposed_fee_rule fee_amount,supposed_repay_date deadline,supposed_principle_rule principle,supposed_over_due_fine_rule due_amount
        FROM qunaer.`repayment_plan_2016_deyu`
        WHERE supposed_repay_date >= '%s' AND supposed_repay_date < '%s'
    ''' % (start_time, end_time)
    df_lx = pd.read_sql(sql_lx, engine_qunaer)
    # status--- 进行调整
    df_lx.status = df_lx.status.astype(int)
    df_lx['require_fee'] = np.round(df_lx['principle'] + df_lx['fee_amount'], 2)
    if insert_db:
        df_lx.to_sql('repayment_plan_2016', engine_qunaer, index=None, if_exists='append', chunksize=10000)


# 导入 16年的放款 对应的费率
def load_16_rate(year, month, is_ptf = False,insert_db=False):
    start_time = datetime.datetime(year, month, 1, 0, 0, 0)
    end_time = start_time + relativedelta(months=+1)
    # 非转分期
    sql_1='''
    select id,product_no from pay_detail where is_ptf = 0 and loan_time >= '%s' and loan_time < '%s'
    ''' % (start_time,end_time)
    # 转分期
    sql_2='''
    select id,product_no from pay_detail where is_ptf = 1 and trans_time >= '%s' and trans_time < '%s'
    ''' % (start_time,end_time)
    sql_3='''
    select product_no,rate from repayment_plan where product_no in %s
    '''
    update_sql='''
    update pay_detail set rate=%s where id = %s
    '''
    if is_ptf:
        df_pay = pd.read_sql(sql_2, engine_qunaer,chunksize=5000)
    else:
        df_pay = pd.read_sql(sql_1, engine_qunaer,chunksize=5000)
    for df in df_pay:
        df.product_no=df.product_no.astype(str)
        df_3=pd.read_sql(sql_3 % str(tuple(df.product_no.tolist())).replace(',)',')'),engine_qunaer)
        df_3.drop_duplicates(subset=['product_no'],inplace=True)
        df=pd.merge(df,df_3,on='product_no',how='left')
        print '----------未找到费率-------------',df.ix[df.rate.isnull()]
        df_good = df.ix[~df.rate.isnull()]
        df_good['id']=df_good['id'].map('{:.0f}'.format)
        if (insert_db) & (len(df_good) > 0):
            print '--------------update 费率-----------',len(df_good)
            up_vals=df_good[['rate', 'id']].values.tolist()
            engine_qunaer.execute(update_sql,up_vals)


# 更新16年的还款流水，没有remain
def load_16_repay(year,month,insert_db=False):
    start_time = datetime.datetime(year, month, 1, 0, 0, 0)
    end_time = start_time + relativedelta(months=+1)
    sql_1='''
     SELECT `product_no`,`current_stage_no`,`repay_no`,`real_repay_principle`,`repay_amount`,`service_fee`,`over_due_fine`
     FROM qunaer.repay_detail WHERE `current_stage_no` > 0 and
     `current_real_repay_date` >= '%s' and  current_real_repay_date < '%s'
    ''' % (start_time,end_time)
    sql_2='''
    select id,repay_no,product_no,current_stage_no from repay_detail where repay_no in %s
    '''
    sql_3='''
    update repay_detail set repay_principle=%s , repay_fee_amount=%s,repay_due_amount=%s where id=%s
    '''
    df_al=pd.read_sql(sql_1,engine_qunaer,chunksize=5000)
    i=0
    for df in df_al:
        print '----------update--progress----',i
        i+=5000
        repay_nos = df.repay_no.astype(str).tolist()
        df_2=pd.read_sql(sql_2 % str(tuple(repay_nos)).replace(',)',')'),engine_qunaer)
        df=pd.merge(df,df_2,on='repay_no',how='inner')
        if insert_db & (len(df) > 0):
            print '---------update------',len(df)
            df['id'] = df['id'].map('{:.0f}'.format)
            up_vals=df[['real_repay_principle','service_fee','over_due_fine','id']].values.tolist()
            engine_qunaer.execute(sql_3,up_vals)


#更新 17年的 还款，其中放款在 16年的数据
def loan_17_repay(year,month,insert_db=False):
    start_time = datetime.datetime(year, month, 1, 0, 0, 0)
    end_time = start_time + relativedelta(months=+1)
    sql_1 = '''
         SELECT `product_no`,`current_stage_no`,`repay_no`,`real_repay_principle`,`repay_amount`,`service_fee`,`over_due_fine`
         FROM qunaer.repay_detail WHERE `current_stage_no` > 0 AND `repay_no` IS NOT NULL and
         `current_real_repay_date` >= '%s' and  current_real_repay_date < '%s'
        ''' % (start_time, end_time)
    sql_2 = '''
        select id,repay_no,product_no from repay_detail where reamin <= 0 and repay_no in %s
        '''
    #保证放款是在 16年
    sql_4='''
    select product_no from pay_detail where  trans_time < '2017-01-01' and product_no in %s
    '''
    sql_3 = '''
        update repay_detail set repay_principle=%s , repay_fee_amount=%s,repay_due_amount=%s where id=%s
        '''
    df_al = pd.read_sql(sql_1, engine_qunaer, chunksize=5000)
    i = 0
    for df in df_al:
        print '----------update--progress----', i
        i += 5000
        repay_nos = df.repay_no.astype(str).tolist()
        df_2 = pd.read_sql(sql_2 % str(tuple(repay_nos)).replace(',)', ')'), engine_qunaer)
        product_nos = df_2.product_no.astype(str).tolist()
        df_4=pd.read_sql(sql_4 % str(tuple(product_nos)).replace(',)', ')'),engine_qunaer)
        #16年的放款
        df_2=df_2.ix[df_2.product_no.isin(df_4.product_no)]
        df = pd.merge(df, df_2, on='repay_no', how='inner')
        if insert_db & (len(df) > 0):
            print '---------update------', len(df)
            df['id'] = df['id'].map('{:.0f}'.format)
            up_vals = df[['real_repay_principle', 'service_fee', 'over_due_fine', 'id']].values.tolist()
            engine_qunaer.execute(sql_3, up_vals)

# 16年的还款流水钩稽
def loan_plan_repay_ref(year,month,insert_db = False,repay_nos=[]):
    start_time = datetime.datetime(year, month, 1, 0, 0, 0)
    end_time = start_time + relativedelta(months=+1)
    if repay_nos.__len__() > 0:  # 指定了 还款流水
        sql_repay = '''
                SELECT product_no,repay_no src_repay_no,repay_time src_repay_time,current_stage_no,repay_amount src_repay_amount,repay_principle src_repay_principle,
                repay_fee_amount src_repay_fee_amount,repay_due_amount src_repay_due_amount,remain src_remain 
                FROM repay_detail
                WHERE repay_status = 1 AND repay_type != 'REFUND_REPAY'
                and repay_no in %s
        ''' % str(tuple(repay_nos)).replace(',)',')')
    else:
        sql_repay = '''
            SELECT product_no,repay_no src_repay_no,repay_time src_repay_time,current_stage_no,repay_amount src_repay_amount,repay_principle src_repay_principle,
            repay_fee_amount src_repay_fee_amount,repay_due_amount src_repay_due_amount,remain src_remain
            FROM repay_detail
            WHERE repay_status = 1 AND repay_type != 'REFUND_REPAY'  AND repay_time >= '%s' AND repay_time < '%s'
            ''' % (start_time, end_time)
        # TODO 注意还款计划表
    sql_plan = '''
        SELECT id plan_id,product_no,current_stage_no,repay_principle,repay_fee_amount,repay_due_amount,repay_no,remark,repay_time,remain
        FROM repayment_plan_2016
        WHERE product_no IN %s
    '''
    update_plan = '''
        update repayment_plan_2016
        set repay_no = %s,repay_time = %s , repay_amount = %s , repay_principle=%s,repay_fee_amount = %s ,repay_due_amount = %s,remain=%s ,remark=%s,status = 2 where id = %s
    '''
    df_repay = pd.read_sql(sql_repay, engine_qunaer, chunksize=5000)
    for df in df_repay:
        product_nos = df.product_no.astype(str).tolist()
        df_plan = pd.read_sql(sql_plan % str(tuple(product_nos)).replace(',)', ')'), engine_qunaer)
        df_plan.current_stage_no = df_plan.current_stage_no.astype(int)
        df_plan.product_no = df_plan.product_no.astype(str)
        df.current_stage_no = df.current_stage_no.astype(int)
        df.product_no = df.product_no.astype(str)
        # 存在还款计划的钩稽关联
        df_new = pd.merge(df, df_plan, on=['product_no', 'current_stage_no'], how='inner')
        if len(df_new) <= 0:
            continue
        df_new.ix[df_new.remark.isnull(), 'remark'] = ''
        df_new.ix[df_new.repay_no.isnull(), 'repay_no'] = ''
        df_new['remark'] = df_new.apply(lambda x: '%s;%s=%s;' % (x.remark, x.src_repay_no, x.src_repay_amount),axis=1)
        # df['remark'] =  df.apply(lambda x:'%s=%s;' % (x.src_repay_no,x.src_repay_amount),axis= 1)
        df_new.fillna(0, inplace=True)
        df_new['al'] = df_new['repay_principle'] + df_new['repay_fee_amount'] + df_new['repay_due_amount']+df_new['remain']
        print '-----------------未找到还款计划--------------', df_new.ix[(df_new.plan_id == 0)]
        # 提取有还款计划的
        df_new.plan_id = df_new.plan_id.astype(int)
        df_new = df_new.ix[(df_new.plan_id != 0)]
        # 更新最新还款时间
        df_new['repay_time'] = pd.to_datetime(df_new['repay_time'])
        df_new['repay_time'] = df_new.apply(lambda x: max(x.src_repay_time, x.repay_time), axis=1)
        df_new.fillna(0,inplace=True)
        # 判断还款计划中是否有已还款的---
        df_bad = df_new.ix[df_new['al'] > 0]
        # 如果是已经还款，那么 分别对应相加
        df_bad['src_repay_amount'] = df_bad['src_repay_amount'] + df_bad['al']
        df_bad['src_repay_principle'] = df_bad['src_repay_principle'] + df_bad['repay_principle']
        df_bad['src_repay_fee_amount'] = df_bad['src_repay_fee_amount'] + df_bad['repay_fee_amount']
        df_bad['src_repay_due_amount'] = df_bad['src_repay_due_amount'] + df_bad['repay_due_amount']
        df_bad['src_remain'] =df_bad['src_remain']+df_bad['remain']
        df_bad.ix[df_bad.repay_no != '', 'repay_no'] = df_bad.ix[df_bad.repay_no != ''].apply(
            lambda x: '%s,%s' % (x.repay_no, x.src_repay_no), axis=1)
        df_bad.ix[df_bad.repay_no == '', 'repay_no'] = df_bad.ix[df_bad.repay_no == '']['src_repay_no']
        if insert_db & (len(df_bad) > 0):
            print '------更新df_bad-----', len(df_bad), df_bad.plan_id.unique()
            up_vals = df_bad[
                ['repay_no', 'repay_time', 'src_repay_amount', 'src_repay_principle', 'src_repay_fee_amount',
                    'src_repay_due_amount','src_remain', 'remark', 'plan_id']].values.tolist()
            engine_qunaer.execute(update_plan, up_vals)
        df_good = df_new.ix[df_new['al'] == 0]
        up_vals = df_good[
            ['src_repay_no', 'repay_time', 'src_repay_amount', 'src_repay_principle', 'src_repay_fee_amount',
                 'src_repay_due_amount', 'src_remain','remark', 'plan_id']].values.tolist()
        if insert_db & (len(df_good) > 0):
            print '-----update df_good len ===', len(df_good)
            engine_qunaer.execute(update_plan, up_vals)





# 还款计划更新
def update_plan(year,month,insert_db = False):
    start_time = datetime.datetime(year, month, 1, 0, 0, 0)
    end_time = start_time + relativedelta(months=+1)
    sql_refund = '''
        select product_no,refund_amount,refund_principle,refund_fee_amount,refund_due_amount,refund_time from refund_detail
        where refund_status = 1 and refund_time >= '%s' and refund_time < '%s'
        ORDER by refund_time asc
        ''' % (start_time, end_time)
    sql_pay = '''
        select product_no,rate,stages from pay_detail where product_no in %s
        '''
    sql_plan = '''
        select id,product_no,current_stage_no,deadline,principle,fee_amount,due_amount
        from repayment_plan_2017
        where product_no = '%s' ORDER by current_stage_no asc
        '''
    sql_repay = '''
        select product_no,current_stage_no,repay_amount,repay_principle,repay_fee_amount,repay_due_amount
        from repay_detail
        where repay_status = 1 and repay_type != 'REFUND_REPAY' and product_no = '%s' and repay_time < '%s'
        '''
    update_sql = '''
        update repayment_plan_2017 set principle=%s ,fee_amount = %s,require_fee=%s ,due_amount=%s,status=%s where id=%s
        '''
    update_sql_2 = '''
        update repayment_plan_2017 set status = 2 where id = %s
        '''
    df_refund = pd.read_sql(sql_refund, engine_qunaer, chunksize=1000)
    i = 0
    for df in df_refund:
        print '-------------update-------------', i
        i += 1000
        df.product_no = df.product_no.astype(str)
        df_pay = pd.read_sql(sql_pay % str(tuple(df.product_no.tolist())).replace(',)', ')'), engine_qunaer)
        df = pd.merge(df, df_pay, on='product_no', how='inner')
        print '-------------无退款本金---------------', df.ix[df.refund_principle == 0]
        df = df.ix[df.refund_principle < 0]
        df.refund_time = pd.to_datetime(df.refund_time).dt.date
        for index, row in df.iterrows():
            list1 = []
            product_no = row['product_no']
            refund_time = row['refund_time']
            df_plan = pd.read_sql(sql_plan % product_no, engine_qunaer)
            # 还款计划一定存在，如果未找到，那么这笔放款在 16年，所以 repayment_plan_2017 表中不存在 2016年的还款计划
            if len(df_plan) == 0:
                continue
            # 还款在退款前
            df_repay = pd.read_sql(sql_repay % (product_no, refund_time), engine_qunaer)
            df_plan.deadline = pd.to_datetime(df_plan.deadline).dt.date
            # 有退款本金的方可更新退款
            refund_principle = row['refund_principle']
            refund_time = row['refund_time']
            rate = float(row['rate'])
            if len(df_repay) == 0:  # 客户无还款情况
                status = 1
                # 如果有逾期的，则进行全额匹配逾期
                df_yq = df_plan.ix[df_plan.deadline < refund_time]
                if len(df_yq) > 0:
                    print '---------逾期退款-----------', df_yq
                    refund_principle = np.round(df_yq['principle'].sum() + refund_principle, 2)
                    up_vals = df_yq[['id']].values.tolist()
                    engine_qunaer.execute(update_sql_2, up_vals)
                # 剔除逾期还款后的
                df_plan = df_plan.ix[df_plan.deadline >= refund_time]
                cur_loan_amount = df_plan['principle'].sum()
                left_principle = np.round(cur_loan_amount + refund_principle, 2)
                if left_principle == 0:
                    status = 2
                # 分期服务费 =  本金 * 利率 * 期数
                # 金额第三位小数向上取值
                stages = len(df_plan)
                all_fee_amount = get_2_ceil(left_principle * rate * stages)
                avg_principle = get_2_flo(left_principle / stages)
                first_principle = left_principle - avg_principle * (stages - 1)
                avg_fee_amount = get_2_flo(left_principle * rate)
                first_fee_amount = all_fee_amount - avg_fee_amount * (stages - 1)
                min_term_no = df_plan['current_stage_no'].min()
                for jindex, jrow in df_plan.iterrows():
                    if int(jrow['current_stage_no']) == min_term_no:
                        list1.append([first_principle, first_fee_amount, first_principle + first_fee_amount, 0, status,
                                      int(jrow['id'])])
                    else:
                        list1.append(
                            [avg_principle, avg_fee_amount, avg_principle + avg_fee_amount, 0, status, int(jrow['id'])])
                if insert_db & (len(list1) > 0):
                    engine_qunaer.execute(update_sql, list1)
            else:  # 客户有还款
                # 如果 剩余应还 >= 退款本金，那么基于剩下的进行本金进行拆分
                sum_repay = df_repay['repay_principle'].sum()
                test = df_plan.ix[df_plan.current_stage_no.isin(df_repay.current_stage_no)]['principle'].sum()
                if np.round(test - sum_repay, 2) != 0:
                    print '-----------客户应还-客户实还---------', product_no
                    continue
                if left_principle >= 0:  # 剩余未还本金 > 退款情况
                    # 剔除已还款的还款计划
                    df_plan = df_plan.ix[~df_plan.current_stage_no.isin(df_repay.current_stage_no)]
                    # 未还是否有逾期
                    df_yq = df_plan.ix[df_plan.deadline < refund_time]
                    if len(df_yq) > 0:
                        print '---------逾期退款-----------', df_yq
                        refund_principle = np.round(df_yq['principle'].sum() + refund_principle, 2)
                        up_vals = df_yq[['id']].values.tolist()
                        engine_qunaer.execute(update_sql_2, up_vals)
                    # 剔除逾期后的
                    df_plan = df_plan.ix[df_plan.deadline >= refund_time]
                    # 分期服务费 =  本金 * 利率 * 期数
                    # 金额第三位小数向上取值
                    left_principle = np.round(df_plan['principle'].sum() + refund_principle, 2)
                    stages = len(df_plan)
                    all_fee_amount = get_2_ceil(left_principle * rate * stages)
                    avg_principle = get_2_flo(left_principle / stages)
                    first_principle = left_principle - avg_principle * (stages - 1)
                    avg_fee_amount = get_2_flo(left_principle * rate)
                    first_fee_amount = all_fee_amount - avg_fee_amount * (stages - 1)
                    status = 1
                    if left_principle == 0:
                        status = 2
                    min_term_no = df_plan['current_stage_no'].min()
                    for jindex, jrow in df_plan.iterrows():
                        if int(jrow['current_stage_no']) == int(min_term_no):
                            list1.append(
                                [first_principle, first_fee_amount, first_principle + first_fee_amount, 0, status,
                                 int(jrow['id'])])
                        else:
                            list1.append([avg_principle, avg_fee_amount, avg_principle + avg_fee_amount, 0, status,
                                          int(jrow['id'])])
                    if insert_db & (len(list1) > 0):
                        engine_qunaer.execute(update_sql, list1)
                else:  # 剩余未还本金 《 退款金额
                    print '----------剩余未还金额<退款金额------', product_no

# 16 年的流水补录
def load_16_refund(year,month):
    start_time = datetime.datetime(year, month, 1, 0, 0, 0)
    end_time = start_time + relativedelta(months=+1)
    sql_1='''
    SELECT product_no,repay_no,repay_amount,real_repay_principle,service_fee,over_due_fine,real_repay_date
    FROM qunaer.`repay_refund_split`
    WHERE `current_stage_no` = 0 AND real_repay_date >= '%s' AND real_repay_date < '%s'
    ''' % (start_time,end_time)
    sql_2='''
    select id,refund_no,refund_amount from refund_detail where refund_time < '2017-01-01' and refund_no in %s
    '''
    sql_3='''
    update refund_detail set refund_principle=%s,refund_fee_amount=%s ,refund_due_amount=%s where id=%s
    '''
    df_al = pd.read_sql(sql_1,engine_qunaer,chunksize=5000)
    for df in df_al:
        repay_nos=df.repay_no.astype(str).tolist()
        df_2=pd.read_sql(sql_2 % str(tuple(repay_nos)).replace(',)',')'),engine_qunaer)
        df=pd.merge(df,df_2,left_on='repay_no',right_on='refund_no',how='inner')
        df['diff']=np.round(df['repay_amount']+df['refund_amount'],2)
        #提取为0 的数据
        df=df.ix[df['diff'] == 0]
        df.real_repay_principle = df.real_repay_principle.apply(lambda x: np.round(0 - x, 2))
        df.service_fee = df.service_fee.apply(lambda x: np.round(0 - x, 2))
        df.over_due_fine = df.over_due_fine.apply(lambda x: np.round(0 - x, 2))
        if len(df) > 0:
            print '------------update----',len(df)
            df['id'] = df['id'].map('{:.0f}'.format)
            up_vals=df[['real_repay_principle','service_fee','over_due_fine','id']].values.tolist()
            engine_qunaer.execute(sql_3,engine_qunaer)


#校验未钩稽上的还款流水
def check_repay_plan(year,month):
    start_time = datetime.datetime(year, month, 1, 0, 0, 0)
    end_time = start_time + relativedelta(months=+1)
    sql_repay='''
    select repay_no,product_no,current_stage_no from repay_detail where repay_status = 1 and repay_type != 'REFUND_REPAY'
    and repay_time >= '%s' and repay_time < '%s'
    ''' % (start_time,end_time)
    df=pd.read_sql(sql_repay,engine_qunaer)
    product_nos = df.product_no.astype(str).tolist()
    sql_plan='''
    select repay_no from repayment_plan_2016 where repay_no is not null and product_no in %s
    ''' % str(tuple(product_nos)).replace(',)',')')
    df_plan=pd.read_sql(sql_plan,engine_qunaer)
    #plan_repay_nos 会有 'A,B'两笔流水在一笔订单的情况,需要转换下
    df_plan_1=df_plan.ix[df_plan.repay_no.str.contains(',')]
    df_plan_2=df_plan.ix[~df_plan.repay_no.str.contains(',')]
    plan_repay_nos=tuple(df_plan_2.repay_no.astype(str).tolist())
    df=df.ix[~df.repay_no.isin(plan_repay_nos)]
    #剔除 plan_1 中的数据
    if len(df_plan_1) > 0:
        list1=[]
        for index,row in df_plan_1.iterrows():
            tmp = row['repay_no']
            tmp = str(tmp).strip().split(',')
            for repay_no in tmp:
                list1.append(str(repay_no))
        df=df.ix[~df.repay_no.isin(list1)]
    return df






# 更新 2017 1- 4月份的还款 放款时间是 2016年的
def repay_sync_remain():
    sql_1='''
    select repay_no,repay_amount,real_repay_principle,service_fee,over_due_fine,remain
    from test_dengcong.repay_detail_source
    where current_real_repay_date >= '2017-01-01' and current_real_repay_date < '2017-05-01' and  remain > 0
    '''
    sql_2='''
    select id,repay_no,product_no from repay_detail
    where repay_time >= '2017-01-01' and repay_time < '2017-05-01' and repay_type != 'REFUND_REPAY'
    and repay_no in %s
    '''
    sql_3='''
   select product_no
        from pay_detail where trans_time < '2017-01-01' and  product_no in %s
    '''
    sql_4='''
    update repay_detail set repay_principle = %s , repay_fee_amount = %s ,repay_due_amount = %s ,reamin=%s where id=%s
    '''
    df_al = pd.read_sql(sql_1,engine_qunaer,chunksize=5000)
    for df in df_al:
        repay_nos=df.repay_no.astype(str).tolist()
        df_repay = pd.read_sql(sql_2 % str(tuple(repay_nos)).replace(',)',')'),engine_qunaer)
        product_nos = df_repay.product_no.astype(str).tolist()
        df_pay=pd.read_sql(sql_3 % str(tuple(product_nos)).replace(',)',')'),engine_qunaer)
        #提取 16年放款的数据
        df_repay=df_repay.ix[df_repay.product_no.isin(df_pay.product_no)]
        #合并 df df_repay
        df=pd.merge(df,df_repay,on='repay_no',how='inner')
        if len(df) > 0:
            print '-------------update remain-----------',len(df)
            df['id'] = df['id'].map('{:.0f}'.format)
            up_vals = df[['real_repay_principle', 'service_fee', 'over_due_fine','remain', 'id']].values.tolist()
            engine_qunaer.execute(sql_4, up_vals)

# 更新 2017 5+6月份的还款 放款时间是 2016年的
def repay_sync_remain_201705():
    sql_1='''
    select repay_detail_id,product_no,current_stage_no,repay_amount,repay_principle,repay_fee_amount,repay_due_amount, remain
    from test_dengcong.repay_detail_new2
    where repay_time >= '2017-06-01' and repay_time < '2017-07-01' and  remain > 0
    '''
    sql_1_1='''
    select id,repay_no from test_dengcong.repay_detail
    where id in %s
    '''
    sql_2='''
    select id,repay_no,product_no,current_stage_no from repay_detail
    where repay_time >= '2017-06-01' and repay_time < '2017-07-01' and repay_type != 'REFUND_REPAY'
    and repay_no in %s
    '''
    sql_3='''
   select product_no from pay_detail where trans_time < '2017-01-01' and  product_no in %s
    '''
    sql_4='''
    update repay_detail set repay_principle = %s , repay_fee_amount = %s ,repay_due_amount = %s ,remain=%s where id=%s
    '''
    df_al = pd.read_sql(sql_1,engine_qunaer,chunksize=5000)
    for df in df_al:
        print '-------------before---------',len(df)
        df_1=pd.read_sql(sql_1_1 % str(tuple(df.repay_detail_id.astype(int).tolist())),engine_qunaer)
        df=pd.merge(df,df_1,left_on='repay_detail_id',right_on='id',how='inner')
        print '-------------after---------', len(df)
        df.drop(['id','repay_detail_id'],inplace=True,axis=1)
        repay_nos = df.repay_no.astype(str).tolist()
        df_repay = pd.read_sql(sql_2 % str(tuple(repay_nos)).replace(',)',')'),engine_qunaer)
        product_nos = df_repay.product_no.astype(str).tolist()
        df_pay=pd.read_sql(sql_3 % str(tuple(product_nos)).replace(',)',')'),engine_qunaer)
        #提取 16年放款的数据
        df_repay=df_repay.ix[df_repay.product_no.isin(df_pay.product_no)]
        #合并 df df_repay
        df=pd.merge(df,df_repay,on='repay_no',how='inner')
        if len(df) > 0:
            print '-------------update remain-----------',len(df)
            df['id'] = df['id'].map('{:.0f}'.format)
            up_vals = df[['repay_principle', 'repay_fee_amount', 'repay_due_amount','remain', 'id']].values.tolist()
            engine_qunaer.execute(sql_4, up_vals)




if __name__ == '__main__':
    year = 2017
    # repay_sync_remain_201705()
    # for month in range(1, 5, 1):
    #     loan_17_repay(year,month,insert_db=True)

    # for month in range(1, 13, 1):
    #     load_16_plan(year,month,insert_db=True)
    ##### 流水补录
    # load_16_repay(year, month, insert_db=True)
    # for month in range(1, 13, 1):
    #     print '-----',year,month
    #     load_16_repay(year, month, insert_db=True)

    # # # 流水钩稽
    for month in range(7,8,1):
        print year,month
        loan_plan_repay_ref(year,month,insert_db=True)
    #     #校验流水是否未钩稽上
        df = check_repay_plan(year,month)
        if len(df) > 0:
    #         这样做是因为，多笔流水还款 1期的时候，会漏掉这些数据。
            repay_nos = df['repay_no'].astype(str).tolist()
            for i in range(0,repay_nos.__len__(),5000):
                print '---------------update--------------',i
                if repay_nos.__len__() < 5000:
                    loan_plan_repay_ref(year,month,insert_db=True,repay_nos=repay_nos)
                    break
                else:
                    loan_plan_repay_ref(year, month, insert_db=True, repay_nos=repay_nos[i:i+5000])


    #还款计划补录
    # for month in range(1, 13, 1):
    #     print '-----',year,month
    #     load_16_plan(year, month, insert_db=True)

    # 还款计划更新








