Skip to content

D
duizhang_tools
Commit 8d0f240e authored by Perry_Zhu's avatar Perry_Zhu
Browse files
Options

Merge remote-tracking branch 'origin/master'

parents 9cf7f38e 3f8021c6
Hide whitespace changes
Inline Side-by-side
Showing with 285 additions and 134 deletions
人工控制/现金贷/人工控制-现金贷-客户实还表-加速测试版.py
View file @ 8d0f240e
......@@ -13,35 +13,37 @@ pd.options.mode.chained_assignment = None
max_limit = 800000
import os
path = u'./hanzhenghui/1020_files/'
path = u'd:/Users/Desktop/test/'
engine_xjd = create_engine('mysql+mysqldb://internal_r:ArbNgtvlJzZHXsEu@172.16.3.201:3306/?charset=utf8',
echo=False).connect()
def faster_sql_reader(sql):
cmdstr = 'mysql -h172.16.3.201 -uyulong_rw -pTouBStYwN8wkdxVt -P3306 -e "%s" > %s'
cmdstr = 'mysql -h172.16.3.201 -uinternal_r -pArbNgtvlJzZHXsEu -P3306 -e "%s" > %s'
file_name = 'test.txt'
path_file = os.path.join(os.getcwd(), file_name)
os.system(cmdstr % (sql.replace('`',''), path_file))
exe_sql = cmdstr % (sql.replace('`',''), path_file)
exe_sql = exe_sql.replace('\t','').replace('\n',' ')
os.system(exe_sql)
try:
df = pd.read_csv(path_file, sep='\t',encoding='utf-8')
except:
return pd.DataFrame()
return df
'''
# 注意事项：非常重要 2017年4月份客户实还表中Loan_id = 52532760 的地方要特别地注意一下 ，1月份中loan_id = 39722219 也需要特别注意
跑到2017年4月份不能用了，待调整bug
'''
# 这5个资金方的还款计划来自于用户还款计划
# group by 是由于还资金方计划中有0.01的情况
sql_repay_frr_2 = '''
SELECT
urp.ref_id,
urp.term_no '应还期数',
date(urp.deadline) '应还资金方时间',
sum(urp.principle) '应还资金方本金',
sum(urp.interest) '应还资金方利息'
urp.term_no 'term_no',
date(urp.deadline) 'deadline',
sum(urp.principle) 'principle',
sum(urp.interest) 'interest'
FROM all_back_0930.user_repayment_plan urp
JOIN all_back_0930.loan_manifest lm ON lm.ref_id = urp.ref_id
......@@ -55,10 +57,10 @@ sql_repay_frr_2 = '''
sql_repay_frr_3 = '''
SELECT
frp.ref_id,
frp.term_no '应还期数',
date(frp.deadline) '应还资金方时间',
sum(frp.principle) '应还资金方本金',
sum(frp.interest) '应还资金方利息'
frp.term_no 'term_no',
date(frp.deadline) 'deadline',
sum(frp.principle) 'principle',
sum(frp.interest) 'interest'
FROM audit.funding_repayment_plan frp
JOIN all_back_0930.loan_manifest lm ON lm.ref_id = frp.ref_id
......@@ -69,10 +71,10 @@ sql_repay_frr_3 = '''
# 在线还款部分
read_detail_online_sql = '''
SELECT t1.real_loan_id loan_id,date(lm.loan_paid_at) '放款时间',t1.ref_id,t1.fund_code funding_code,t1.term_no '应还期数',date(t6.bill_time) '实还时间',t2.ref_amount '实还总额',
t2.principle-t2.mitigate_principle '实还本金',t2.interest-t2.mitigate_interest '实还利息',
ifnull(t2.`service_fee`-t2.mitigate_service_fee,0) '实还服务费',IFNULL(t2.`punish`-t2.mitigate_collection_relief, 0) '实还罚息',
t2.remain_income '其他收益',t2.repay_channel,t6.order_id AS '流水号',t6.amount '账单实还金额',t1.service_fee '应收服务费',t1.required_repayment '应还总额'
SELECT t1.real_loan_id loan_id,date(lm.loan_paid_at) 'loan_paid_at',t1.ref_id,t1.fund_code funding_code,t1.term_no 'term_no',date(t6.bill_time) 'bill_time',t2.ref_amount 'ref_amount',
t2.principle-t2.mitigate_principle 'repay_prin',t2.interest-t2.mitigate_interest 'repay_intes',
ifnull(t2.`service_fee`-t2.mitigate_service_fee,0) 'repay_fee',IFNULL(t2.`punish`-t2.mitigate_collection_relief, 0) 'repay_due',
t2.remain_income 'remain',t2.repay_channel,t6.order_id AS 'flow_no',t6.amount 'bank_amount',t1.service_fee 'service_fee',t1.required_repayment 'required_repayment'
FROM audit_temp.xjd_repay_plan_repay_record_ref t2
JOIN all_back_0930.user_repayment_plan t1 ON t1.id=t2.plan_id
JOIN all_back_0930.repay_record_online t6 ON t2.record_id=t6.id AND t2.repay_channel = t6.pay_approach
......@@ -82,10 +84,10 @@ WHERE lm.loan_paid_at >= '%s' AND lm.loan_paid_at < '%s'
# 支付宝线下还款
read_detail_alipay_sql = '''
SELECT t1.real_loan_id loan_id,date(lm.loan_paid_at) '放款时间',t1.ref_id,t1.fund_code funding_code,t1.term_no '应还期数',date(t6.transfer_time) '实还时间',t2.ref_amount '实还总额',
t2.principle-t2.mitigate_principle '实还本金',t2.interest-t2.mitigate_interest '实还利息',
ifnull(t2.`service_fee`-t2.mitigate_service_fee,0) '实还服务费',IFNULL(t2.`punish`-t2.mitigate_collection_relief, 0) '实还罚息',
t2.remain_income '其他收益',t2.repay_channel,t6.business_flow '流水号',t6.income '账单实还金额',t1.service_fee '应收服务费',t1.required_repayment '应还总额'
SELECT t1.real_loan_id loan_id,date(lm.loan_paid_at) 'loan_paid_at',t1.ref_id,t1.fund_code funding_code,t1.term_no 'term_no',date(t6.transfer_time) 'bill_time',t2.ref_amount 'ref_amount',
t2.principle-t2.mitigate_principle 'repay_prin',t2.interest-t2.mitigate_interest 'repay_intes',
ifnull(t2.`service_fee`-t2.mitigate_service_fee,0) 'repay_fee',IFNULL(t2.`punish`-t2.mitigate_collection_relief, 0) 'repay_due',
t2.remain_income 'remain',t2.repay_channel,t6.business_flow 'flow_no',t6.income 'bank_amount',t1.service_fee 'service_fee',t1.required_repayment 'required_repayment'
FROM audit_temp.xjd_repay_plan_repay_record_ref t2
JOIN all_back_0930.`user_repayment_plan` t1 ON t1.id=t2.plan_id
JOIN all_back_0930.`offline_alipay_record` t6 ON t2.record_id=t6.id AND t2.repay_channel = 13
......@@ -95,10 +97,10 @@ WHERE lm.loan_paid_at >= '%s' AND lm.loan_paid_at < '%s'
# 银行线下还款部分，无对应流水号
read_detail_bank_sql = '''
SELECT t1.real_loan_id loan_id,date(lm.loan_paid_at) '放款时间',t1.ref_id,t1.fund_code funding_code,t1.term_no '应还期数',date(t6.transfer_time) '实还时间',t2.ref_amount '实还总额',
t2.principle-t2.mitigate_principle '实还本金',t2.interest-t2.mitigate_interest '实还利息',
ifnull(t2.`service_fee`-t2.mitigate_service_fee,0) '实还服务费',IFNULL(t2.`punish`-t2.mitigate_collection_relief, 0) '实还罚息',
t2.remain_income '其他收益',t2.repay_channel,t6.transfer_amount '账单实还金额',t1.service_fee '应收服务费',t1.required_repayment '应还总额'
SELECT t1.real_loan_id loan_id,date(lm.loan_paid_at) 'loan_paid_at',t1.ref_id,t1.fund_code funding_code,t1.term_no 'term_no',date(t6.transfer_time) 'bill_time',t2.ref_amount 'ref_amount',
t2.principle-t2.mitigate_principle 'repay_prin',t2.interest-t2.mitigate_interest 'repay_intes',
ifnull(t2.`service_fee`-t2.mitigate_service_fee,0) 'repay_fee',IFNULL(t2.`punish`-t2.mitigate_collection_relief, 0) 'repay_due',
t2.remain_income 'remain',t2.repay_channel,t6.transfer_amount 'bank_amount',t1.service_fee 'service_fee',t1.required_repayment 'required_repayment'
FROM audit_temp.xjd_repay_plan_repay_record_ref t2
JOIN all_back_0930.`user_repayment_plan` t1 ON t1.id=t2.plan_id
JOIN all_back_0930.loan_manifest lm ON lm.ref_id = t1.ref_id
......@@ -128,7 +130,10 @@ def handle_hk(year, month):
# df_repay_frr_1 = pd.read_sql(sql_repay_frr_1 % (start_time, end_time), con=engine_xjd)
df_repay_frr_2 = faster_sql_reader(sql_repay_frr_2 % (start_time, end_time))
df_repay_frr_3 = faster_sql_reader(sql_repay_frr_3 % (start_time, end_time))
df_repay_frr = pd.concat([df_repay_frr_2, df_repay_frr_3])
df_repay_frr.rename(columns={'term_no': u'应还期数', 'deadline': u'应还资金方时间', 'principle': u'应还资金方本金', 'interest': u'应还资金方利息'},
inplace=True)
df_repay_frr = df_repay_frr.drop_duplicates([u'ref_id', u'应还期数', u'应还资金方时间', u'应还资金方本金', u'应还资金方利息']).dropna()
......@@ -141,6 +146,10 @@ def handle_hk(year, month):
df_bank_detail = faster_sql_reader(read_detail_bank_sql % (start_time, end_time))
df = pd.concat([df_online_detail, df_alipay_detail, df_bank_detail], axis=0)
df.rename(columns={'loan_paid_at': u'放款时间', 'term_no': u'应还期数', 'bill_time': u'实还时间', 'ref_amount': u'实还总额', \
'repay_prin': u'实还本金', 'repay_intes': u'实还利息', 'repay_fee': u'实还服务费', 'repay_due': u'实还罚息', \
'remain': u'其他收益', 'flow_no': u'流水号', 'bank_amount': u'账单实还金额', 'service_fee': u'应收服务费',
'required_repayment': u'应还总额'}, inplace=True)
df[u'应还期数'] = df[u'应还期数'].astype('int64')
df['ref_id'] = df['ref_id'].astype('int64')
......@@ -185,7 +194,7 @@ def handle_hk(year, month):
temp_list = temp.values.tolist()
temp_list.insert(0, temp.columns.tolist())
all_wb.new_sheet(str(int(i / max_limit + 1)), data=temp_list)
all_wb.save(path + u'人工控制-现金贷-客户实还表-(%d-%02d).xlsx' % (start_time.year, start_time.month))
all_wb.save(path + u'人工控制-现金贷-客户实还表-测试(%d-%02d).xlsx' % (start_time.year, start_time.month))
if __name__ == '__main__':
......
人工控制/现金贷/人工控制-现金贷-放款表-加速测试版.py
View file @ 8d0f240e
......@@ -13,37 +13,12 @@ pd.options.mode.chained_assignment = None
max_limit = 800000
import os
# '''
# 在代码之前，需要进行前期数据的准备，先从数据库中获取所需数据，然后用pandas读取进行操作
# 需要特别注意数据库、表、字段的使用
# '''
# @author Han
# 表名称：
# 现金贷-核算-放款表明细（1-6月份）
# 表字段：
# 唯一编 号 E=A&C
#
#
# 资金方 Loan id A
# 合同号 放款时 间 B
# 放款金 额
# 放款渠 道
# 应收总 服务费
# 应还期 数 C
# 应还资 金方时 间
# 应还资 金方本 息 D
# Merge E&B&D
# 应还期 数
# 应还资 金方时 间
# 应还资 金方本 息
# 应还期 数
# 客户实还服务 费
# 客户实还本 息
# 应还总服务费-实还服务 费
# 应还资金方本息-客 户实还本息
#
# 数据库：
# repay_detail，pay_detail
# 注意事项：
......@@ -51,15 +26,18 @@ import os
# 需要先进行数据的获取，需要时间较长，sql_repay_all中xrd.repaid_at同样需要修改
path = u'./hanzhenghui/1020_files/'
path = u'1020_files/'
path = u'E:/审计出表/非去哪儿白条-客户实还表-1023版/'
engine_xjd = create_engine('mysql+mysqldb://internal_r:ArbNgtvlJzZHXsEu@172.16.3.201:3306/audit?charset=utf8',
echo=False).connect()
def faster_sql_reader(sql):
cmdstr = 'mysql -h172.16.3.201 -uyulong_rw -pTouBStYwN8wkdxVt -P3306 -e "%s" > %s'
cmdstr = 'mysql -h172.16.3.201 -uinternal_r -pArbNgtvlJzZHXsEu -P3306 -e "%s" > %s'
file_name = 'test.txt'
path_file = os.path.join(os.getcwd(), file_name)
os.system(cmdstr % (sql.replace('`',''), path_file))
exe_sql = cmdstr % (sql.replace('`',''), path_file)
exe_sql = exe_sql.replace('\t','').replace('\n',' ')
os.system(exe_sql)
try:
df = pd.read_csv(path_file, sep='\t',encoding='utf-8')
except:
......@@ -68,12 +46,14 @@ def faster_sql_reader(sql):
# #使用时先要修改 deadline，如果出9月份的，deadline > = 2017-09-01即可
sql_fund = '''
select frp.ref_id,frp.term_no '应还期数',date(frp.deadline) '应还资金方时间',frp.principle+frp.interest '应还资金方本息'
from all_back_0831.funding_repayment_plan frp
select frp.ref_id,frp.term_no 'term_no',date(frp.deadline) 'deadline',frp.principle+ifnull(frp.interest,0) 'prin_interes'
from audit.funding_repayment_plan frp
where frp.deadline >= '2017-01-01'
'''
df_fund = faster_sql_reader(sql_fund)
df_fund = pd.read_csv(pah)
df_fund.rename(columns={'term_no':u'应还期数','deadline':u'应还资金方时间','prin_interes':u'应还资金方本息'},inplace=True)
# df_fund = pd.read_csv(path)
#
#此部分不需要进行修改
sql_contract_no = '''
......@@ -98,9 +78,9 @@ def mc_pay(year, month):
sql_repay_all = '''
select
urp.ref_id,
urp.term_no '应还期数',
sum(xrprrr.principle-IFNULL(xrprrr.mitigate_principle,0)+xrprrr.interest-IFNULL(xrprrr.mitigate_interest,0)) '客户实还本息',
sum(xrprrr.service_fee-IFNULL(xrprrr.mitigate_service_fee,0)) '实还服务费'
urp.term_no 'term_no',
sum(xrprrr.principle-IFNULL(xrprrr.mitigate_principle,0)+xrprrr.interest-IFNULL(xrprrr.mitigate_interest,0)) 'principle_interes',
sum(xrprrr.service_fee-IFNULL(xrprrr.mitigate_service_fee,0)) 'service_fee'
from audit_temp.xjd_repay_plan_repay_record_ref_0831 xrprrr
join all_back_0831.user_repayment_plan urp on xrprrr.plan_id = urp.id
join all_back_0831.loan_manifest lm on lm.ref_id = urp.ref_id
......@@ -112,22 +92,33 @@ def mc_pay(year, month):
SELECT
lm.funding_code,
lm.loan_id 'loan_id',
date(lm.loan_paid_at) '放款时间',
lm.contract_loan_amount '放款金额',
la.accountName '放款渠道',
lm.contract_term*lm.per_term_service_fee '应收总服务费',
date(lm.loan_paid_at) 'loan_paid_at',
lm.contract_loan_amount 'contract_loan_amount',
lm.loan_account_id 'account_id',
lm.contract_term*ifnull(lm.per_term_service_fee,0) 'all_service_fee',
lm.ref_id
FROM
audit.loan_manifest lm
join all_back_0831.loan_account la on la.id = lm.loan_account_id
join audit.loan_account la on la.id = lm.loan_account_id
WHERE
lm.loan_paid_at >= '%s'
AND lm.loan_paid_at < '%s'
AND lm.is_active IN (1, -2)
''' % (start_time, end_time)
sql_account_name = '''
select id 'account_id',accountName
from audit.loan_account la
'''
df_account = pd.read_sql(sql_account_name,engine_xjd)
df_pay = faster_sql_reader(sql_pay)
df_pay = pd.merge(df_pay,df_account,on='account_id',how='left')
df_pay.drop(['account_id'],axis=1,inplace=True)
df_pay.rename(columns={'loan_paid_at':u'放款时间','contract_loan_amount':u'放款金额','accountName':u'放款渠道','all_service_fee':u'应收总服务费'},inplace=True)
df_repay_all = faster_sql_reader(sql_repay_all)
df_repay_all.rename(columns={'term_no':u'应还期数','principle_interes':u'客户实还本息','service_fee':u'实还服务费'},inplace=True)
print(sum(df_repay_all[u'客户实还本息']))
df_pay['ref_id'] = df_pay['ref_id'].astype('int64')
......@@ -177,7 +168,7 @@ def mc_pay(year, month):
temp_list = temp.values.tolist()
temp_list.insert(0,temp.columns.tolist())
all_wb.new_sheet(str(int(i/max_limit + 1)),data=temp_list)
all_wb.save(path+u'人工控制-现金贷-放款表_(%d-%02d).xlsx' %(start_time.year,start_time.month))
all_wb.save(path+u'人工控制-现金贷-放款表_测试(%d-%02d).xlsx' %(start_time.year,start_time.month))
# if len(df_pay) > max_limit:
......@@ -189,7 +180,7 @@ def mc_pay(year, month):
if __name__ == '__main__':
year = [2017]
month = range(1, 8, 1)
month = range(1, 2, 1)
for year, month in product(year, month):
mc_pay(year, month)
人工控制/现金贷/人工控制-现金贷-还资金表-加速测试版.py
View file @ 8d0f240e
......@@ -13,7 +13,7 @@ sys.setdefaultencoding("utf-8")
pd.options.mode.chained_assignment = None
#此路径为服务器路径专用
path = u'./hanzhenghui/1020_files/'
path = u'E:/审计出表/人工控制/现金贷-还资金方表-1010版本/'
# path = u'E:/审计出表/人工控制/现金贷-还资金方表-1010版本/'
engine_xjd = create_engine('mysql+mysqldb://internal_r:ArbNgtvlJzZHXsEu@172.16.3.201:3306/?charset=utf8',
echo=False).connect()
......@@ -22,10 +22,12 @@ max_limit = 800000
def faster_sql_reader(sql):
cmdstr = 'mysql -h172.16.3.201 -uyulong_rw -pTouBStYwN8wkdxVt -P3306 -e "%s" > %s'
cmdstr = 'mysql -h172.16.3.201 -uinternal_r -pArbNgtvlJzZHXsEu -P3306 -e "%s" > %s'
file_name = 'test.txt'
path_file = os.path.join(os.getcwd(), file_name)
os.system(cmdstr % (sql.replace('`',''), path_file))
exe_sql = cmdstr % (sql.replace('`',''), path_file)
exe_sql = exe_sql.replace('\t','').replace('\n',' ')
os.system(exe_sql)
try:
df = pd.read_csv(path_file, sep='\t',encoding='utf-8')
except:
......@@ -34,7 +36,7 @@ def faster_sql_reader(sql):
sql_lm = '''
select ref_id,loan_id,funding_code,date(loan_paid_at) '放款时间',contract_term '期数'
select ref_id,loan_id,funding_code,date(loan_paid_at) 'loan_paid_at',contract_term 'contract_term'
from all_back_0930.loan_manifest lm
where loan_paid_at>='%s' and loan_paid_at <'%s'
and is_active in (1, -2)
......@@ -51,9 +53,9 @@ df_fund_name = pd.read_sql(sql_fund_name,engine_xjd)
sql_repay_frr_2 = '''
select
urp.ref_id,
urp.term_no '应还期数',
date(urp.deadline) '应还资金方时间',
urp.principle+urp.interest '应还资金方本息'
urp.term_no 'term_no',
date(urp.deadline) 'deadline',
urp.principle+urp.interest 'prin_intere'
from all_back_0930.user_repayment_plan urp
join all_back_0930.loan_manifest lm on lm.ref_id = urp.ref_id
where lm.loan_paid_at>='%s' and loan_paid_at<'%s'
......@@ -64,9 +66,9 @@ sql_repay_frr_2 = '''
sql_repay_frr_3 = '''
select
frp.ref_id,
frp.term_no '应还期数',
date(frp.deadline) '应还资金方时间',
frp.interest+frp.principle '应还资金方本息'
frp.term_no 'term_no',
date(frp.deadline) 'deadline',
frp.interest+frp.principle 'prin_intere'
from audit.funding_repayment_plan frp
join all_back_0930.loan_manifest lm on lm.ref_id = frp.ref_id
......@@ -78,12 +80,12 @@ sql_repay_frr_3 = '''
sql_refund = '''
SELECT
frr.ref_id,
date(frr.repaid_at) '实还时间',
sum(frr.current_repayment) '实还总额',
sum(frr.principle) '实还本金',
sum(frr.interest) '实还利息',
frr.flow_id '流水号',
frr.term_no '应还期数'
date(frr.repaid_at) 'repaid_at',
sum(frr.current_repayment) 'current_repayment',
sum(frr.principle) 'principle',
sum(frr.interest) 'interest',
frr.flow_id 'flow_id',
frr.term_no 'term_no'
FROM
all_back_0930.funding_repayment_record frr
join all_back_0930.loan_manifest lm on lm.ref_id = frr.ref_id
......@@ -110,6 +112,9 @@ def refund(year,month):
end_time = start_time + relativedelta(months=+1)
df_all = faster_sql_reader(sql_refund %(start_time,end_time))
df_all.rename(
columns={'repaid_at': u'实还时间', 'current_repayment': u'实还总额', 'principle': u'实还本金', 'interest': u'实还利息', \
'flow_id': u'流水号', 'term_no': u'应还期数'}, inplace=True)
#实还部分进行去重
df_all = df_all.drop_duplicates([u'ref_id', u'实还时间', u'实还本金', u'实还总额', u'实还利息', u'流水号',u'应还期数'])
......@@ -119,6 +124,8 @@ def refund(year,month):
#放款部分
df_lm = faster_sql_reader(sql_lm %(start_time,end_time))
df_lm.rename(columns={'loan_paid_at':u'放款时间','contract_term':u'期数'},inplace=True)
df_lm['ref_id'] = df_lm['ref_id'].astype('int64')
df_lm = df_lm.drop_duplicates()
......@@ -127,6 +134,7 @@ def refund(year,month):
df_repay_frr_2 = faster_sql_reader(sql_repay_frr_2 % (start_time, end_time))
df_repay_frr_3 = faster_sql_reader(sql_repay_frr_3 % (start_time, end_time))
df_repay_frr = pd.concat([df_repay_frr_2,df_repay_frr_3])
df_repay_frr.rename(columns={'term_no': u'应还期数', 'deadline': u'应还资金方时间', 'prin_intere': u'应还资金方本息'}, inplace=True)
df_repay_frr['ref_id'] = df_repay_frr['ref_id'].astype('int64')
df_repay_frr[u'应还期数'] = df_repay_frr[u'应还期数'].astype('int64')
......@@ -175,7 +183,7 @@ def refund(year,month):
temp_list = temp.values.tolist()
temp_list.insert(0, temp.columns.tolist())
all_wb.new_sheet(str(int(i / max_limit + 1)), data=temp_list)
all_wb.save(path + u'人工控制-现金贷—还资金方表-(%d-%02d).xlsx' % (start_time.year, start_time.month))
all_wb.save(path + u'人工控制-现金贷—还资金方表-测试(%d-%02d).xlsx' % (start_time.year, start_time.month))
# if len(df) > max_limit:
# for i in xrange(0, len(df), max_limit):
......@@ -186,7 +194,7 @@ def refund(year,month):
if __name__ == '__main__':
year = [2017]
month = range(9, 10, 1)
month = range(1, 2, 1)
for year, month in product(year, month):
print('-----------%s-%s begins') %(year,month)
......
去哪儿/V2/出表/白条-去哪儿-核算-还资金方表.py
View file @ 8d0f240e
......@@ -19,7 +19,7 @@ pd.options.mode.chained_assignment = None
#
#
file_path = u'E:/审计出表'
file_path = u'E:/审计出表/'
engine_qunaer = create_engine('mysql+mysqldb://internal_r:ArbNgtvlJzZHXsEu@172.16.3.201:3306/qunaer_new?charset=utf8',
echo=False).connect()
......@@ -57,12 +57,12 @@ def pay(year, month):
df_sql_pay[u'应还资金方日期'] = df_sql_pay[u'放款日期'] + pd.DateOffset(days=30)
df_sql_pay[u'应还利息'] = df_sql_pay[u'应还本金']*0.01
# 保存导出的结果表
df_sql_pay.to_csv(file_path + u'%s-%s去哪儿-还资金方表.csv', encoding='gb18030', index=False) %(start_time,end_time)
df_sql_pay.to_csv(file_path + u'去哪儿_还资金方表_%s_%s.csv'%(year,month), encoding='gb18030', index=False)
if __name__ == '__main__':
pool = mtp.Pool(processes=2)
# pool = mtp.Pool(processes=2)
# 还款明细
# years = [2017]
# months = range(1,7)
......@@ -72,7 +72,7 @@ if __name__ == '__main__':
# pool.close()
# pool.join()
year,month = 2017,8
year,month = 2017,9
pay(year,month)
print '-----------------------main done---------------------------------------'
\ No newline at end of file
去哪儿/监控不可用/qunaer_not_available.py 0 → 100644
View file @ 8d0f240e
# -*- coding:utf-8 -*-
import sys
import datetime
import pandas as pd
import paramiko
import smtplib
from email.mime.text import MIMEText
from sqlalchemy import create_engine
reload(sys)
sys.setdefaultencoding("utf-8")
engine_qunaer = create_engine('mysql+mysqldb://linfang.wang:&*uijknbghyt63er45@172.16.3.184:7306/xyqb_btzt?charset=utf8', echo=False).connect()
def downloan_file(time_str):
_username = 'fenqixpush'
_password = '2A5ai08uczky'
_host = '59.151.16.190'
path_sftp = '/download/'
t = paramiko.Transport((_host))
t.connect(username=_username, password=_password)
_sftp_fd = paramiko.SFTPClient.from_transport(t)
try:
_sftp_fd.get(path_sftp + '%s_LIANGHP_repaymentInfo.txt' % time_str, '/vpants/wlf/qunar_sftp/' + '%s_LIANGHP_repaymentInfo.txt' % time_str)
_sftp_fd.get(path_sftp + '%s_LIANGHP_refundInfo.txt' % time_str,'/vpants/wlf/qunar_sftp/' + '%s_LIANGHP_refundInfo.txt' % time_str)
except Exception, e:
print 'sftp %s@%s: %s' % (_username, _host, e)
_sftp_fd.close()
def get_now():
n=datetime.datetime.now()
b=datetime.date(n.year,n.month,n.day) + datetime.timedelta(-1)
return b.strftime('%Y-%m-%d')
def read(time_str):
ff = file('/vpants/wlf/qunar_sftp/'+'%s_LIANGHP_repaymentInfo.txt' % time_str).readlines()
while len(ff[-1]) == 0 or ff[-1].startswith('F|'):
del ff[-1]
if ff[0].startswith('H|'):
del ff[0]
df_tmp = pd.DataFrame(ff, columns=['Line'])
df_tmp['product_no'] = df_tmp['Line'].apply(lambda x: x.strip().split('|')[1])
df_tmp['repay_time'] = df_tmp['Line'].apply(lambda x: x.strip().split('|')[3])
df_tmp['repay_amount'] = df_tmp['Line'].apply(lambda x: x.strip().split('|')[6])
ff = file('/vpants/wlf/qunar_sftp/'+'%s_LIANGHP_refundInfo.txt' % time_str).readlines()
while len(ff[-1]) == 0 or ff[-1].startswith('F|'):
del ff[-1]
if ff[0].startswith('H|'):
del ff[0]
df_tmp2 = pd.DataFrame(ff, columns=['Line'])
df_tmp2['product_no'] = df_tmp2['Line'].apply(lambda x: x.strip().split('|')[1])
df_tmp2['repay_time'] = df_tmp2['Line'].apply(lambda x: x.strip().split('|')[3])
df_tmp2['repay_amount'] = df_tmp2['Line'].apply(lambda x: x.strip().split('|')[4])
df_tmp=pd.concat([df_tmp,df_tmp2],ignore_index=True)
d=datetime.datetime.strptime(time_str,'%Y-%m-%d')
df_tmp.repay_time = pd.to_datetime(df_tmp.repay_time)
df_tmp=df_tmp.ix[df_tmp.repay_time >= datetime.datetime(d.year,d.month,d.day,15,0,0)]
df_tmp.repay_amount=df_tmp.repay_amount.astype(float)
#===哈银订单==
sql_hy='''
SELECT upstream_order_id FROM `zt_order` WHERE fund_id = 1 AND channel_id = 66 AND upstream_order_id in %s
''' % str(tuple(df_tmp.product_no.astype(str).tolist()))
df_hy=pd.read_sql(sql_hy,engine_qunaer)
df_tmp=df_tmp.ix[df_tmp.product_no.isin(df_hy.upstream_order_id)]
return df_tmp['repay_amount'].sum()
def send_mail(to_list, sub, content):
mail_host="mail.quantgroup.cn"
mail_user="datamonitor" # user
mail_pass="!Moni123Tor456" # passwd
mail_postfix="quantgroup.cn"
mail_port = "587"
me = mail_user+"@"+mail_postfix
msg = MIMEText(content,_subtype='html',_charset='UTF-8')
msg['Subject'] = sub
msg['From'] = me
msg['To'] = ";".join(to_list)
try:
server = smtplib.SMTP()
server.connect(mail_host,mail_port)
server.starttls()
server.login(me,mail_pass)
server.sendmail(me, to_list, msg.as_string())
server.close()
return True
except Exception, e:
print str(e)
return False
time_str = get_now()
downloan_file(time_str)
repay_amount=read(time_str)
to_list=['linfang.wang@quantgroup.cn','fengli.liu@quantgroup.cn','qian.peng@quantgroup.cn','jianyu.zhu@quantgroup.cn','yao.wang@quantgroup.cn','shanling.yu@quantgroup.cn']
# to_list=['linfang.wang@quantgroup.cn']
sub=u'去哪儿不可用—%s' % time_str
content='''
<table border="1">
<tr>
<td align="center">时间</td>
<td align="center">不可用金额</td>
</tr>
<tr>
<td align="center">%s</td>
<td align="center">%s</td>
</tr>
</table>
''' % (time_str,repay_amount)
send_mail(to_list,sub,content)
\ No newline at end of file
审计第一阶段/现金贷/收入确认表-不同库版本.py
View file @ 8d0f240e
......@@ -25,6 +25,8 @@ pd.options.mode.chained_assignment = None
pd.set_option('display.float_format', lambda x: '%.5f' % x)
res_sum = []
def proc_save(watch_date):
loan = df_loan_ratio.loc[df_loan_ratio[u'放款日期'] < watch_date]
loan.loc[~loan.is_active.isin([1, -2]), u'本金债权'] = 0
......@@ -84,12 +86,16 @@ def proc_save(watch_date):
0)
print(
loan_result[
[u'本金债权', u'计提风险金', u'qg_one_time_service_fee', u'实还服务费（减免后）', u'趸交服务费', u'实还服务费（减免后、含一次性服务费、趸交服务费）', u'asset余额',
[u'本金债权', u'计提风险金', u'qg_one_time_service_fee', u'实还服务费（减免后）', u'趸交服务费', u'实还服务费（减免后、含一次性服务费、趸交服务费）',
u'asset余额',
u'收入确认金额']].sum())
tmp_sum = loan_result[
[u'本金债权', u'计提风险金', u'qg_one_time_service_fee', u'实还服务费（减免后）', u'趸交服务费', u'实还服务费（减免后、含一次性服务费、趸交服务费）',
u'asset余额',
u'收入确认金额']].sum().to_frame().T
# tmp_sum = loan_result.groupby(u'风险等级')[
# [u'本金债权', u'计提风险金', u'qg_one_time_service_fee', u'实还服务费（减免后）', u'趸交服务费', u'实还服务费（减免后、含一次性服务费、趸交服务费）',
# u'asset余额', u'收入确认金额']].agg('sum').reset_index()
tmp_sum[u'截止时间'] = watch_date
res_sum.append(tmp_sum)
......@@ -107,7 +113,7 @@ def proc_save(watch_date):
if __name__ == '__main__':
step = 10000
max_limit = 900000
file_path = u'E:/审计出表/现金贷/0926'
file_path = u'E:/审计出表/现金贷/1021'
file_name = u'现金贷收入确认表'
if not os.path.exists(file_path):
os.makedirs(file_path)
......@@ -129,9 +135,12 @@ if __name__ == '__main__':
df_ms.loan_id = df_ms.loan_id.astype(str)
db_names = ['all_back_0630', 'all_back_0731', 'all_back_0831', 'new_transaction']
date_report_list = [[item.date() for item in pd.date_range(start='2017-02', end='2017-07', freq='MS')],
[datetime.date(2017, 8, 1)],
[datetime.date(2017, 9, 1)], [datetime.date(2017, 10, 1)]]
date_report_list = [
[datetime.date(2017, 2, 1), datetime.date(2017, 3, 1), datetime.date(2017, 4, 1), datetime.date(2017, 5, 1),
datetime.date(2017, 6, 1), datetime.date(2017, 7, 1)],
[datetime.date(2017, 8, 1)],
[datetime.date(2017, 9, 1)],
[datetime.date(2017, 10, 1)]]
# db_names = ['new_transaction',]
# date_report_list = [[datetime.date(2017, 10, 1)],]
for i in xrange(len(db_names)):
......@@ -141,7 +150,8 @@ if __name__ == '__main__':
# #######################
# 查文件
df_loan = pd.read_csv(os.path.join(file_path, 'all_loan_%s.csv' % db_name), encoding='utf8', low_memory=False)
df_ref = pd.read_csv(os.path.join(file_path, 'all_ref_%s.csv' % db_name), encoding='utf8', low_memory=False)
df_ref = pd.read_csv(os.path.join(file_path, 'all_ref_%s.txt' % db_name), sep='\t', low_memory=False)
df_ref.columns = [u'ref_id', u'实还总额', u'实还本金', u'实还利息', u'实还服务费（减免后）', u'实还罚息', u'其他收益', u'repaid_at']
df_loan[u'放款日期'] = pd.to_datetime(df_loan[u'放款日期']).dt.date
df_loan.loc[df_loan[u'风险等级'] == 'D', u'风险等级'] = 'C'
......
审计第一阶段/现金贷/收入确认表-读数据.py
View file @ 8d0f240e
......@@ -25,9 +25,9 @@ step = 10000
max_limit = 900000
pd.options.mode.chained_assignment = None
pd.set_option('display.float_format', lambda x: '%.5f' % x)
db_names = ['all_back_0630', 'all_back_0731', 'all_back_0831']
end_dates = [datetime.date(2017, 7, 1), datetime.date(2017, 8, 1), datetime.date(2017, 9, 1)]
file_path = u'E:/审计出表/现金贷/0926'
db_names = ['audit_temp']
end_dates = [datetime.date(2017, 7, 1)]
file_path = u'E:/审计出表/现金贷/1021'
if not os.path.exists(file_path):
os.makedirs(file_path)
......@@ -66,7 +66,7 @@ SELECT
xrprrr.punish - xrprrr.mitigate_collection_relief 实还罚息,
xrprrr.remain_income 其他收益,
DATE(xrprrrrd.repaid_at) `repaid_at`
FROM %s.xjd_repay_plan_repay_record_ref xrprrr
FROM %s.xjd_repay_plan_repay_record_ref_0630 xrprrr
JOIN test_mzh.xjd_repay_plan_repay_record_ref_repay_date_flow xrprrrrd ON xrprrr.id = xrprrrrd.xjd_ref_id
JOIN %s.user_repayment_plan urp ON urp.id = xrprrr.plan_id
WHERE xrprrrrd.repaid_at < '%s';
......@@ -116,13 +116,13 @@ for i in xrange(0, len(db_names)):
end_date = end_dates[i]
# ##############################################
# 查数据库，并保存文件，时间较长，服务器上在15分钟左右，本机25分钟左右
df_gen = pd.read_sql(sql=sql_loan_plan % (db_name, end_date,), con=mysql, chunksize=step)
res = []
for tmp in df_gen:
res.append(tmp)
df_loan = pd.concat(res)
df_loan['qg_one_time_service_fee'].fillna(0, inplace=True)
df_loan.to_csv(os.path.join(file_path, 'all_loan_%s.csv' % db_name), encoding='utf8', index=None)
# df_gen = pd.read_sql(sql=sql_loan_plan % (db_name, end_date,), con=mysql, chunksize=step)
# res = []
# for tmp in df_gen:
# res.append(tmp)
# df_loan = pd.concat(res)
# df_loan['qg_one_time_service_fee'].fillna(0, inplace=True)
# df_loan.to_csv(os.path.join(file_path, 'all_loan_%s.csv' % db_name), encoding='utf8', index=None)
df_gen = pd.read_sql(sql=sql_ref % (db_name, db_name, end_date,), con=mysql, chunksize=step)
res = []
......@@ -131,9 +131,9 @@ for i in xrange(0, len(db_names)):
df_ref = pd.concat(res)
df_ref.to_csv(os.path.join(file_path, 'all_ref_%s.csv' % db_name), encoding='utf8', index=None)
df_ms = pd.read_sql(sql=sql_ms, con=mysql)
df_ms.to_csv(os.path.join(file_path, 'ms.csv'), encoding='utf8', index=None)
df_yghs = pd.read_sql(sql=sql_yghs, con=mysql)
df_yghs.to_csv(os.path.join(file_path, 'yghs.csv'), encoding='utf8', index=None)
df_dunjiao = pd.read_sql(sql=sql_dunjiao, con=mysql)
df_dunjiao.to_csv(os.path.join(file_path, 'dunjiao.csv'), encoding='utf8', index=None)
# df_ms = pd.read_sql(sql=sql_ms, con=mysql)
# df_ms.to_csv(os.path.join(file_path, 'ms.csv'), encoding='utf8', index=None)
# df_yghs = pd.read_sql(sql=sql_yghs, con=mysql)
# df_yghs.to_csv(os.path.join(file_path, 'yghs.csv'), encoding='utf8', index=None)
# df_dunjiao = pd.read_sql(sql=sql_dunjiao, con=mysql)
# df_dunjiao.to_csv(os.path.join(file_path, 'dunjiao.csv'), encoding='utf8', index=None)
审计第一阶段/白条/去哪儿/去哪儿收入确认表-模型.py
View file @ 8d0f240e
......@@ -100,7 +100,7 @@ def cal_debt_age(x):
df_res['账龄'] = df_res[['放款时间', '应还款日']].apply(lambda x: cal_debt_age(x), axis=1)
asset_remain = df_res.loc[df_res.assets_remain > 0]
asset_remain = df_res.loc[df_res.assets_remain != 0]
asset_remain[['产品编号', '风险等级', '放款时间', '应还服务费', 'assets_remain']].to_excel(
os.path.join(file_path, '去哪儿asset_remain.xlsx'),
index=None)
......@@ -111,12 +111,12 @@ gp1.to_excel(os.path.join(file_path, '去哪儿汇总（期数-应还年月-风
gp2 = df_res.groupby(['账龄', '应还年月', '风险等级'])['assets'].agg('sum').reset_index()
gp2.to_excel(os.path.join(file_path, '去哪儿汇总（账龄-应还年月-风险等级）.xlsx'), index=None)
# df_res.to_csv(os.path.join(file_path, '去哪儿assets减值-details.csv'), index=None, encoding='gbk')
df_res.to_csv(os.path.join(file_path, '去哪儿assets减值-details.csv'), index=None, encoding='gbk')
#
# max_limit = 900000
# if len(df_res) > max_limit:
# for i in xrange(0, len(df_res), max_limit):
# df_res[i:i + max_limit].to_csv(os.path.join(file_path, '去哪儿assets减值-details_%d.csv' % (i / max_limit + 1)), index=None,
# encoding='utf8')
# df_res.loc[df_res['产品编号'].duplicated(), '放款金额'] = 0
# print('{}'.format(df_res['放款金额'].sum()))
df_res.loc[df_res['产品编号'].duplicated(), '放款金额'] = 0
print('{}'.format(df_res['放款金额'].sum()))
审计第一阶段/白条/去哪儿/去哪儿收入确认表.py
View file @ 8d0f240e
......@@ -90,7 +90,7 @@ df_loan['计提风险金'] = 0
view_dates = [datetime.date(2017, 9, 1), datetime.date(2017, 10, 1)]
file_path = 'E:/审计出表/白条/收入确认表'
res_sum = []
for v_date in view_dates:
last_day = v_date + datetime.timedelta(days=-1)
df_loan_v = df_loan.loc[df_loan['放款时间'] < v_date]
......@@ -141,7 +141,13 @@ for v_date in view_dates:
df_loan_v.drop(['减值', '退款服务费'], axis=1, inplace=True)
df_loan_v['放款时间'] = df_loan_v['放款时间'].dt.date
print(df_loan_v[['放款金额', '计提风险金', '实还服务费', 'assets', '收入']].sum())
print(df_loan_v.groupby(['风险等级']).agg('sum'))
# tmp_sum = df_loan_v.groupby(u'风险等级')[
# [u'放款金额', u'计提风险金', u'实还服务费', u'assets', u'收入']].agg('sum').reset_index()
tmp_sum = df_loan_v[[u'放款金额', u'计提风险金', u'实还服务费', u'assets', u'收入']].agg('sum').to_frame().T
tmp_sum[u'截止时间'] = v_date
res_sum.append(tmp_sum)
df_loan_v.to_csv(os.path.join(file_path, '白条去哪儿收入确认表_{}_{:0>2}.csv'.format(v_date.year, v_date.month - 1)),
index=None,
encoding='gbk')
df_sum = pd.concat(res_sum)
df_sum.to_excel(os.path.join(file_path, '白条去哪儿汇总.xlsx'), index=None)
\ No newline at end of file
审计第一阶段/白条/非去哪儿/非去哪儿收入确认表-模型.py
View file @ 8d0f240e
......@@ -149,9 +149,9 @@ def cal_debt_age(x):
df_res['账龄'] = df_res[['放款时间', '应还日']].apply(lambda x: cal_debt_age(x), axis=1)
# asset_remain = df_res.loc[df_res.assets_remain > 0]
# asset_remain[['产品编号', '风险等级', '放款时间', '应还服务费', 'assets_remain']].to_excel(os.path.join(file_path, '非去哪儿asset_remain.xlsx'),
# index=None)
asset_remain = df_res.loc[df_res.assets_remain != 0]
asset_remain[['产品编号', '风险等级', '放款时间', '应还服务费', 'assets_remain']].to_excel(os.path.join(file_path, '非去哪儿asset_remain.xlsx'),
index=None)
df_res['应还年月'] = df_res['应还日'].apply(lambda x: x.strftime('%Y-%m'))
gp1 = df_res.groupby(['当前期数', '应还年月', '风险等级'])['assets'].agg('sum').reset_index()
......@@ -160,7 +160,7 @@ gp1.to_excel(os.path.join(file_path, '非去哪儿汇总（期数-应还年月-
gp2 = df_res.groupby(['账龄', '应还年月', '风险等级'])['assets'].agg('sum').reset_index()
gp2.to_excel(os.path.join(file_path, '非去哪儿汇总（账龄-应还年月-风险等级）.xlsx'), index=None)
#
# df_res.to_csv(os.path.join(file_path, '非去哪儿assets减值-details.csv'), index=None, encoding='gbk')
df_res.to_csv(os.path.join(file_path, '非去哪儿assets减值-details.csv'), index=None, encoding='gbk')
#
# max_limit = 900000
# for i in xrange(0, len(df_res), max_limit):
......
审计第一阶段/白条/非去哪儿/非去哪儿收入确认表.py
View file @ 8d0f240e
......@@ -102,6 +102,7 @@ print(df_repay.groupby(df_repay['还款时间'].dt.month)['实还服务费'].agg
view_dates = [datetime.date(2017, 9, 1), datetime.date(2017, 10, 1)]
file_path = 'E:/审计出表/白条/收入确认表'
res_sum = []
for v_date in view_dates:
df_loan_v = df_loan.loc[df_loan['放款时间'] < v_date]
df_repay_v = df_repay.loc[df_repay['还款时间'] < v_date]
......@@ -132,8 +133,17 @@ for v_date in view_dates:
df_loan_v.drop(['ref_id', '减值'], axis=1, inplace=True)
df_loan_v['放款时间'] = df_loan_v['放款时间'].dt.date
print(df_loan_v[['放款金额', '计提风险金', '实还服务费', 'assets', '收入']].sum())
# df_loan_v.to_csv(os.path.join(file_path, '白条收入确认表_{}_{:0>2}.csv'.format(v_date.year, v_date.month - 1)), index=None, encoding='gbk')
# tmp_sum = df_loan_v.groupby(u'风险等级')[
# [u'放款金额', u'计提风险金', u'实还服务费', u'assets', u'收入']].agg('sum').reset_index()
tmp_sum = df_loan_v[[u'放款金额', u'计提风险金', u'实还服务费', u'assets', u'收入']].agg('sum').to_frame().T
tmp_sum[u'截止时间'] = v_date
res_sum.append(tmp_sum)
df_loan_v.to_csv(os.path.join(file_path, '白条收入确认表_{}_{:0>2}.csv'.format(v_date.year, v_date.month - 1)), index=None,
encoding='gbk')
df_sum = pd.concat(res_sum)
df_sum.to_excel(os.path.join(file_path, '白条非去哪儿汇总.xlsx'), index=None)
# 98108, 98108
# ()
......@@ -144,4 +154,4 @@ for v_date in view_dates:
# 0 8 8561.64000
# 1 9 559489.81000
# 有还款，没有放款的len: 0，ref_id: ()
# 有还款，没有放款的len: 0，ref_id: ()
\ No newline at end of file
# 有还款，没有放款的len: 0，ref_id: ()
审计第二阶段/白条（非去哪儿）/给审计-白条-还款表（7月份开始）.py
View file @ 8d0f240e
......@@ -196,15 +196,15 @@ def hk_hz(year,month):
[u'资金方',u'产品编号',u'放款期数',u'当前期数',u'白条商家',u'客户实还时间', u'总额', u'本金', u'利息', u'服务费', u'逾期罚款收益',
u'其他收益',u'收款账户',u'流水号']]
all_wb = pyexcelerate.Workbook()
for i in xrange(0, len(df), max_limit):
temp = df[i:i + max_limit]
temp_list = temp.values.tolist()
temp_list.insert(0, temp.columns.tolist())
all_wb.new_sheet(str(int(i / max_limit + 1)), data=temp_list)
all_wb.save(file_path + u'白条还款明细表_%s_%s.xlsx' % (year, month))
# df.to_excel(file_path + u'白条还款汇总表_%s_%s.xlsx' %(year,month), index=None)
# all_wb = pyexcelerate.Workbook()
# for i in xrange(0, len(df), max_limit):
# temp = df[i:i + max_limit]
# temp_list = temp.values.tolist()
# temp_list.insert(0, temp.columns.tolist())
# all_wb.new_sheet(str(int(i / max_limit + 1)), data=temp_list)
# all_wb.save(file_path + u'白条还款明细表_%s_%s.xlsx' % (year, month))
df.to_csv(file_path + u'白条还款汇总表_%s_%s.csv' %(year,month), index=None,encoding='gb18030')
if __name__ == '__main__':
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment