Skip to content

D
duizhang_tools
Commit fbd55155 authored by zhen.ma's avatar zhen.ma
Browse files
Options

rt

parent ee9dd9fc
Hide whitespace changes
Inline Side-by-side
Showing with 179 additions and 72 deletions
审计第一阶段/现金贷/收入确认表-不同库版本.py
View file @ fbd55155
......@@ -113,7 +113,7 @@ def proc_save(watch_date):
if __name__ == '__main__':
step = 10000
max_limit = 900000
file_path = u'E:/审计出表/现金贷/1021'
file_path = u'E:/审计出表/现金贷/1107'
file_name = u'现金贷收入确认表'
if not os.path.exists(file_path):
os.makedirs(file_path)
......@@ -134,15 +134,15 @@ if __name__ == '__main__':
df_dunjiao[u'repaid_at'] = pd.to_datetime(df_dunjiao[u'repaid_at']).dt.date
df_ms.loan_id = df_ms.loan_id.astype(str)
db_names = ['all_back_0630', 'all_back_0731', 'all_back_0831', 'new_transaction']
date_report_list = [
[datetime.date(2017, 2, 1), datetime.date(2017, 3, 1), datetime.date(2017, 4, 1), datetime.date(2017, 5, 1),
datetime.date(2017, 6, 1), datetime.date(2017, 7, 1)],
[datetime.date(2017, 8, 1)],
[datetime.date(2017, 9, 1)],
[datetime.date(2017, 10, 1)]]
# db_names = ['new_transaction',]
# date_report_list = [[datetime.date(2017, 10, 1)],]
# db_names = ['all_back_0630', 'all_back_0731', 'all_back_0831', 'new_transaction']
# date_report_list = [
# [datetime.date(2017, 2, 1), datetime.date(2017, 3, 1), datetime.date(2017, 4, 1), datetime.date(2017, 5, 1),
# datetime.date(2017, 6, 1), datetime.date(2017, 7, 1)],
# [datetime.date(2017, 8, 1)],
# [datetime.date(2017, 9, 1)],
# [datetime.date(2017, 10, 1)]]
db_names = ['2017-11-01',]
date_report_list = [[datetime.date(2017, 11, 1)],]
for i in xrange(len(db_names)):
db_name = db_names[i]
date_list = date_report_list[i]
......@@ -150,7 +150,7 @@ if __name__ == '__main__':
# #######################
# 查文件
df_loan = pd.read_csv(os.path.join(file_path, 'all_loan_%s.csv' % db_name), encoding='utf8', low_memory=False)
df_ref = pd.read_csv(os.path.join(file_path, 'all_ref_%s.txt' % db_name), sep='\t', low_memory=False)
df_ref = pd.read_csv(os.path.join(file_path, 'all_ref_%s.csv' % db_name), low_memory=False)
df_ref.columns = [u'ref_id', u'实还总额', u'实还本金', u'实还利息', u'实还服务费（减免后）', u'实还罚息', u'其他收益', u'repaid_at']
df_loan[u'放款日期'] = pd.to_datetime(df_loan[u'放款日期']).dt.date
......
审计第一阶段/现金贷/收入确认表-人工控制.py
View file @ fbd55155
......@@ -32,12 +32,13 @@ if not os.path.exists(file_path):
os.makedirs(file_path)
mysql = sql_engine('audit', 'new_transaction', False).get_engine()
sql_loan_plan = """SELECT
sql_loan_plan = """
SELECT
lm.ref_id,
lm.funding_code,
fc.name 资金方,
lm.loan_id,
lc0912.class 风险等级,
lc.class 风险等级,
lm.contract_loan_amount 本金债权,
date(lm.loan_paid_at) 放款日期,
year(lm.loan_paid_at) 年,
......@@ -50,14 +51,14 @@ sql_loan_plan = """SELECT
rc.approach_name 还款账户,
rr.repay_channel_flow_id 流水号
FROM %s.xjd_repay_plan_repay_record_ref xrprrr
JOIN test_mzh.repay_record_amount_1006 rr
JOIN test_mzh.repay_record_amount rr
ON rr.record_id = xrprrr.record_id AND rr.repay_channel = xrprrr.repay_channel
AND rr.repaid_at >= '%s' AND rr.repaid_at < '%s'
JOIN new_transaction.repay_channel rc ON rc.id = xrprrr.repay_channel
JOIN %s.user_repayment_plan urp ON urp.id = xrprrr.plan_id
JOIN %s.loan_manifest lm ON lm.ref_id = urp.ref_id AND lm.funding_code <> 340
JOIN basic.funding_corp fc ON fc.funding_code = lm.funding_code
JOIN test_mzh.loan_class_contract_1002 lc0912 ON lc0912.ref_id = lm.ref_id;
JOIN test_mzh.loan_class_contract lc ON lc.ref_id = lm.ref_id;
"""
sql_yghs = """
......@@ -66,7 +67,7 @@ SELECT
lm.funding_code,
fc.name 资金方,
lm.loan_id,
lc0912.class 风险等级,
lc.class 风险等级,
lm.contract_loan_amount 本金债权,
date(lm.loan_paid_at) 放款日期,
year(lm.loan_paid_at) 年,
......@@ -82,7 +83,7 @@ FROM new_transaction.yg_huishang_repay_term t
JOIN audit.loan_manifest lm ON lm.ref_id = t.ref_id AND lm.funding_code = 340
AND t.repay_date >= '%s' AND t.repay_date < '%s'
JOIN basic.funding_corp fc ON fc.funding_code = lm.funding_code
JOIN test_mzh.loan_class_contract_1002 lc0912 ON lc0912.ref_id = lm.ref_id;
JOIN test_mzh.loan_class_contract lc ON lc.ref_id = lm.ref_id;
"""
sql_dunjiao = """
......@@ -91,7 +92,7 @@ SELECT
lm.funding_code,
fc.name 资金方,
lm.loan_id,
lc0912.class 风险等级,
lc.class 风险等级,
lm.contract_loan_amount 本金债权,
date(lm.loan_paid_at) 放款日期,
year(lm.loan_paid_at) 年,
......@@ -105,13 +106,13 @@ SELECT
rr.repay_channel_flow_id 流水号
FROM new_transaction.xjd_dunjiao_plan_ref xdpr
JOIN new_transaction.xjd_dunjiao_plan xdp ON xdp.id = xdpr.plan_id
JOIN test_mzh.repay_record_amount_1006 rr
JOIN test_mzh.repay_record_amount rr
ON rr.record_id = xdpr.record_id AND rr.repay_channel = xdpr.repay_channel
AND rr.repaid_at >= '%s' AND rr.repaid_at < '%s'
JOIN new_transaction.repay_channel rc ON rc.id = xdpr.repay_channel
JOIN audit.loan_manifest lm ON lm.ref_id = xdp.ref_id
JOIN basic.funding_corp fc ON fc.funding_code = lm.funding_code
JOIN test_mzh.loan_class_contract_1002 lc0912 ON lc0912.ref_id = lm.ref_id;
JOIN test_mzh.loan_class_contract lc ON lc.ref_id = lm.ref_id;
"""
res_cols = [u'资金方', u'loan_id', u'风险等级', u'本金债权', u'放款日期', u'计提风险金比例', u'计提风险金', u'实还总额', u'实还本金', u'实还利息', u'实还服务费',
u'实还日期', u'还款账户', u'流水号']
......
审计第一阶段/现金贷/收入确认表-读数据-最新库.py 0 → 100644
View file @ fbd55155
# -*- encoding: utf8 -*-
"""
目标：
收入确认表-以loan_id为维度比较风险金及客户实还服务费
数据库：
字段名：
loan_id，资金方，风险等级，本金债权，放款日期，计提风险金比例，计提风险金，实还服务费（减免后，含一次性服务费），Asset余额，收入确认金额
注意：
加入一次性服务费，计提风险金比例
每个时间点用对应时间点的备份库，eg：17年6月30号之前的用630数据库，。。。
"""
import os
import datetime
import numpy as np
import pandas as pd
from db_con.db_connect import sql_engine
# ----------------------------------------------
step = 10000
max_limit = 900000
pd.options.mode.chained_assignment = None
pd.set_option('display.float_format', lambda x: '%.5f' % x)
end_date = datetime.date(2017, 11, 1)
file_path = u'E:/审计出表/现金贷/1107'
if not os.path.exists(file_path):
os.makedirs(file_path)
mysql = sql_engine('audit', 'new_transaction').get_engine()
sql_loan_plan = """
SELECT
lm.loan_id,
lm.ref_id,
lc.class 风险等级,
lm.funding_code,
fc.name 资金方,
lm.is_active,
lm.contract_loan_amount 本金债权,
lm.qg_one_time_service_fee,
date(lm.loan_paid_at) 放款日期,
year(lm.loan_paid_at) 年,
month(lm.loan_paid_at) 月
FROM audit.loan_manifest lm
JOIN test_mzh.loan_class_contract lc ON lc.ref_id = lm.ref_id
JOIN basic.funding_corp fc ON fc.funding_code = lm.funding_code
WHERE lm.loan_paid_at < '%s';
"""
sql_ref = """
SELECT
urp.ref_id,
xrprrr.ref_amount 实还总额,
xrprrr.principle - xrprrr.mitigate_principle 实还本金,
xrprrr.interest - xrprrr.mitigate_interest 实还利息,
xrprrr.service_fee - xrprrr.mitigate_service_fee `实还服务费（减免后）`,
xrprrr.punish - xrprrr.mitigate_collection_relief 实还罚息,
xrprrr.remain_income 其他收益,
DATE(rra.repaid_at) `repaid_at`
FROM new_transaction.xjd_repay_plan_repay_record_ref xrprrr
JOIN test_mzh.repay_record_amount rra ON rra.record_id = xrprrr.record_id and rra.repay_channel = xrprrr.repay_channel
JOIN new_transaction.user_repayment_plan urp ON urp.id = xrprrr.plan_id
WHERE rra.repaid_at < '%s';
"""
sql_ms = """
SELECT
t.loan_id,
t.repay_amt 实还总额,
t.repay_principle 实还本金,
t.repay_intrest 实还利息,
t.service_fee `实还服务费（减免后）`,
t.penalty_fee + t.late_fee 滞纳金罚息,
0 其他收益,
date(t.repay_date) `repaid_at`
FROM new_transaction.new_ms_transaction t;
"""
sql_yghs = """
SELECT
t.ref_id,
t.actual_repay_amount 实还总额,
t.actual_principle 实还本金,
t.actual_interest 实还利息,
t.actual_qg_service_fee `实还服务费（减免后）`,
t.actual_qg_overdue_interest + t.actual_fund_overdue_interest 实还罚息,
t.actual_compound_interest 实还复利,
date(t.repay_date) repaid_at
FROM new_transaction.yg_huishang_repay_term t;"""
sql_dunjiao = """
SELECT
xdp.ref_id,
xdpr.ref_amount 趸交服务费,
date(rro.bill_time) repaid_at
FROM
new_transaction.xjd_dunjiao_plan_ref xdpr
JOIN new_transaction.xjd_dunjiao_plan xdp
ON xdp.id = xdpr.plan_id
JOIN new_transaction.repay_record_online rro
ON rro.id = xdpr.record_id
AND rro.pay_approach = xdpr.repay_channel
"""
df_gen = pd.read_sql(sql=sql_loan_plan % (end_date,), con=mysql, chunksize=step)
res = []
for tmp in df_gen:
res.append(tmp)
df_loan = pd.concat(res)
df_loan['qg_one_time_service_fee'].fillna(0, inplace=True)
df_loan.to_csv(os.path.join(file_path, 'all_loan_%s.csv' % end_date), encoding='utf8', index=None)
df_gen = pd.read_sql(sql=sql_ref % (end_date,), con=mysql, chunksize=step)
res = []
for tmp in df_gen:
res.append(tmp)
df_ref = pd.concat(res)
df_ref.to_csv(os.path.join(file_path, 'all_ref_%s.csv' % end_date), encoding='utf8', index=None)
df_ms = pd.read_sql(sql=sql_ms, con=mysql)
df_ms.to_csv(os.path.join(file_path, 'ms.csv'), encoding='utf8', index=None)
df_yghs = pd.read_sql(sql=sql_yghs, con=mysql)
df_yghs.to_csv(os.path.join(file_path, 'yghs.csv'), encoding='utf8', index=None)
df_dunjiao = pd.read_sql(sql=sql_dunjiao, con=mysql)
df_dunjiao.to_csv(os.path.join(file_path, 'dunjiao.csv'), encoding='utf8', index=None)
审计第一阶段/现金贷/收入确认表-读数据.py
View file @ fbd55155
......@@ -26,7 +26,7 @@ max_limit = 900000
pd.options.mode.chained_assignment = None
pd.set_option('display.float_format', lambda x: '%.5f' % x)
db_names = ['audit_temp']
end_dates = [datetime.date(2017, 7, 1)]
end_dates = [datetime.date(2017, 11, 1)]
file_path = u'E:/审计出表/现金贷/1021'
if not os.path.exists(file_path):
......@@ -65,11 +65,11 @@ SELECT
xrprrr.service_fee - xrprrr.mitigate_service_fee `实还服务费（减免后）`,
xrprrr.punish - xrprrr.mitigate_collection_relief 实还罚息,
xrprrr.remain_income 其他收益,
DATE(xrprrrrd.repaid_at) `repaid_at`
DATE(rra.repaid_at) `repaid_at`
FROM %s.xjd_repay_plan_repay_record_ref_0630 xrprrr
JOIN test_mzh.xjd_repay_plan_repay_record_ref_repay_date_flow xrprrrrd ON xrprrr.id = xrprrrrd.xjd_ref_id
JOIN test_mzh.repay_record_amount rra ON rra.record_id = xrprrr.record_id and rra.repay_channel = xrprrr.repay_channel
JOIN %s.user_repayment_plan urp ON urp.id = xrprrr.plan_id
WHERE xrprrrrd.repaid_at < '%s';
WHERE rra.repaid_at < '%s';
"""
sql_ms = """
SELECT
......
审计第一阶段/白条/去哪儿/去哪儿收入确认表.py
View file @ fbd55155
......@@ -15,7 +15,6 @@ from db_con.db_connect import sql_engine
pd.options.mode.chained_assignment = None
pd.set_option('display.float_format', lambda x: '%.5f' % x)
mysql = sql_engine('audit', 'audit').get_engine()
mysql_analysis = sql_engine('analysis', 'analysis_tmp', False).get_engine()
sql_loan = """SELECT
fpd.product_no 产品编号,
......@@ -24,9 +23,9 @@ sql_loan = """SELECT
fpd.loan_time 放款时间,
year(fpd.loan_time) 年,
month(fpd.loan_time) 月
FROM qunaer_new.fund_pay_detail fpd
JOIN qunaer_new.fund_corp fc ON fc.fund_code = fpd.fund_code
AND fpd.status = 1 AND fpd.loan_time >= '2017-08-01';
FROM qunaer_new.pay_detail fpd
JOIN qunaer_new.fund_corp fc ON fc.fund_code = fpd.fund_code and fpd.fund_code <> 1
AND fpd.status = 1 AND fpd.loan_time >= '2017-08-01' and fpd.loan_time < '2017-11-01';
"""
sql_ref = """SELECT
......@@ -35,7 +34,7 @@ sql_ref = """SELECT
rd.repay_fee_amount 实还服务费,
rd.repay_time 还款时间
FROM qunaer_new.repay_detail rd
WHERE rd.fund_code <> 1 AND rd.repay_time >= '2017-08-01'
WHERE rd.fund_code <> 1 AND rd.repay_time >= '2017-08-01' and rd.repay_time < '2017-11-01'
"""
sql_tui = """SELECT
......@@ -44,11 +43,11 @@ sql_tui = """SELECT
-refund_principle 退款服务费,
refund_time 退款时间
FROM qunaer_new.refund_detail
WHERE fund_code != 1 AND refund_time >= '2017-08-01'
WHERE fund_code <> 1 AND refund_time >= '2017-08-01' AND refund_time < '2017-11-01'
"""
sql_class = """
SELECT ref_id 产品编号, class 风险等级 FROM analysis_tmp.nk_bt_order_all_0930 WHERE ref_id IN {}
SELECT ref_id 产品编号, class 风险等级 FROM test_mzh.bt_all_class WHERE ref_id IN {}
"""
sql_ratio = """
......@@ -77,7 +76,7 @@ step = 5000
ref_ids = tuple(df_loan['产品编号'].astype(str).unique())
res = []
for i in xrange(0, len(ref_ids), step):
tmp = pd.read_sql(sql=sql_class.format(str(ref_ids[i:i + step]).replace(',)', ')')), con=mysql_analysis)
tmp = pd.read_sql(sql=sql_class.format(str(ref_ids[i:i + step]).replace(',)', ')')), con=mysql)
res.append(tmp)
df_class = pd.concat(res)
df_class.drop_duplicates(['产品编号'], inplace=True)
......@@ -88,8 +87,8 @@ print(tuple(df_loan.loc[df_loan['风险等级'].isnull(), '产品编号'].astype
df_loan = pd.merge(df_loan, df_ratio, on=['风险等级', '年', '月'])
df_loan['计提风险金'] = 0
view_dates = [datetime.date(2017, 9, 1), datetime.date(2017, 10, 1)]
file_path = 'E:/审计出表/白条/收入确认表'
view_dates = [datetime.date(2017, 9, 1), datetime.date(2017, 10, 1), datetime.date(2017, 11, 1)]
file_path = 'E:/审计出表/白条/收入确认表/1108'
res_sum = []
for v_date in view_dates:
last_day = v_date + datetime.timedelta(days=-1)
......
审计第一阶段/白条/非去哪儿/非去哪儿收入确认表.py
View file @ fbd55155
......@@ -15,7 +15,6 @@ from db_con.db_connect import sql_engine
pd.options.mode.chained_assignment = None
pd.set_option('display.float_format', lambda x: '%.5f' % x)
mysql = sql_engine('audit', 'audit', False).get_engine()
mysql_analysis = sql_engine('analysis', 'analysis_tmp', False).get_engine()
# todo: is_active = -2 的订单，按哪个时间来关联风险金比例
sql_loan = """
......@@ -24,27 +23,15 @@ SELECT
bo.ref_id,
fc.fund_name 资金方,
bo.loan_paid_at 放款时间,
case bo.is_active when 1 then bo.loan_paid_at when -2 then bo.train_cancel_at end train_cancel_at,
year(bo.loan_paid_at) 年,
month(bo.loan_paid_at) 月,
bo.funding_loan_amount 放款金额,
bo.is_active
FROM baitiao_audit.baitiao_order bo
JOIN baitiao_audit.fund_corp fc
ON fc.id = bo.funding_corp_id AND bo.funding_corp_id = 5 AND bo.is_active = 1 AND bo.loan_paid_at >= '2017-08-01'
UNION ALL
SELECT
bo.order_no 产品编号,
bo.ref_id,
fc.fund_name 资金方,
bo.train_cancel_at 放款时间,
year(bo.train_cancel_at) 年,
month(bo.train_cancel_at) 月,
bo.funding_loan_amount 放款金额,
bo.is_active
FROM baitiao_audit.baitiao_order bo
JOIN baitiao_audit.fund_corp fc
ON fc.id = bo.funding_corp_id AND bo.funding_corp_id = 5 AND bo.is_active = -2 AND
bo.train_cancel_at >= '2017-08-01';
ON fc.id = bo.funding_corp_id AND bo.funding_corp_id = 5 AND bo.is_active in (1, -2)
AND bo.loan_paid_at >= '2017-08-01' AND bo.loan_paid_at < '2017-11-01'
"""
sql_ref = """SELECT
......@@ -63,7 +50,7 @@ FROM new_transaction.baitiao_repay_plan_repay_record_ref brprrr
JOIN new_transaction.repay_channel rc ON rc.id = brprrr.repay_channel"""
sql_class = """
SELECT ref_id, class 风险等级 FROM analysis_tmp.nk_bt_order_all_0930 WHERE ref_id IN {}
SELECT order_no 产品编号, class 风险等级 FROM test_mzh.bt_all_class WHERE order_no IN {}
"""
sql_ratio = """
......@@ -83,16 +70,17 @@ df_repay = pd.concat(res)
df_ratio = pd.read_sql(sql=sql_ratio, con=mysql)
step = 5000
ref_ids = tuple(df_loan.ref_id.astype(str).unique())
order_nos = tuple(df_loan['产品编号'].astype(str).unique())
res = []
for i in xrange(0, len(ref_ids), step):
tmp = pd.read_sql(sql=sql_class.format(str(ref_ids[i:i + step]).replace(',)', ')')), con=mysql_analysis)
for i in xrange(0, len(order_nos), step):
tmp = pd.read_sql(sql=sql_class.format(str(order_nos[i:i + step]).replace(',)', ')')), con=mysql)
res.append(tmp)
df_class = pd.concat(res)
df_class.ref_id = df_class.ref_id.astype(int)
df_loan = pd.merge(df_loan, df_class, on='ref_id', how='left')
print('{}, {}'.format(len(ref_ids), df_class.ref_id.nunique()))
print(tuple(df_loan.loc[df_loan['风险等级'].isnull(), 'ref_id'].astype(str)))
df_class.drop_duplicates(['产品编号'], inplace=True)
df_loan = pd.merge(df_loan, df_class, on='产品编号', how='left')
print('{}, {}'.format(len(order_nos), df_class['产品编号'].nunique()))
print(tuple(df_loan.loc[df_loan['风险等级'].isnull(), '产品编号'].astype(str)))
df_loan = pd.merge(df_loan, df_ratio, on=['风险等级', '年', '月'])
df_loan['计提风险金'] = np.round(df_loan['放款金额'] * df_loan['计提风险金比例'], 2)
......@@ -100,11 +88,11 @@ df_loan['计提风险金'] = np.round(df_loan['放款金额'] * df_loan['计提
print(df_loan.groupby([df_loan['资金方'], df_loan['放款时间'].dt.month])['放款金额'].agg('sum').reset_index())
print(df_repay.groupby(df_repay['还款时间'].dt.month)['实还服务费'].agg('sum').reset_index())
view_dates = [datetime.date(2017, 9, 1), datetime.date(2017, 10, 1)]
file_path = 'E:/审计出表/白条/收入确认表'
view_dates = [datetime.date(2017, 9, 1), datetime.date(2017, 10, 1), datetime.date(2017, 11, 1)]
file_path = 'E:/审计出表/白条/收入确认表/1108'
res_sum = []
for v_date in view_dates:
df_loan_v = df_loan.loc[df_loan['放款时间'] < v_date]
df_loan_v = df_loan.loc[(df_loan['放款时间'] < v_date) & (df_loan['train_cancel_at'] < v_date)]
df_repay_v = df_repay.loc[df_repay['还款时间'] < v_date]
no_loan_ref_ids = df_repay_v.loc[~df_repay_v.ref_id.isin(df_loan_v.ref_id), 'ref_id']
......@@ -139,19 +127,9 @@ for v_date in view_dates:
tmp_sum = df_loan_v[[u'放款金额', u'计提风险金', u'实还服务费', u'assets', u'收入']].agg('sum').to_frame().T
tmp_sum[u'截止时间'] = v_date
res_sum.append(tmp_sum)
df_loan_v.drop(['train_cancel_at'], axis=1, inplace=True)
df_loan_v.to_csv(os.path.join(file_path, '白条收入确认表_{}_{:0>2}.csv'.format(v_date.year, v_date.month - 1)), index=None,
encoding='gbk')
df_sum = pd.concat(res_sum)
df_sum.to_excel(os.path.join(file_path, '白条非去哪儿汇总.xlsx'), index=None)
# 98108, 98108
# ()
# 资金方 放款时间 放款金额
# 0 笑脸直投 8 11076756.35000
# 1 笑脸直投 9 40581669.20000
# 还款时间 实还服务费
# 0 8 8561.64000
# 1 9 559489.81000
# 有还款，没有放款的len: 0，ref_id: ()
# 有还款，没有放款的len: 0，ref_id: ()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment