Skip to content

C
credit-report-api
Commit 9acec8d0 authored by 张鹏程's avatar 张鹏程
Browse files
Options

新增“”表头信息"

parent b8307955
Hide whitespace changes
Inline Side-by-side
Showing with 320 additions and 1036 deletions
handler/ReportAnalysis_Handler.py
View file @ 9acec8d0
......@@ -5,6 +5,7 @@ from utils import JsonUtil
from handler.Base_Handler import BaseHandler
import json
from config import settings
from service.Head_Service import Heads # 调查信息和被调查者的相关信息
from service.PersonalInformation_Service import PersonalInformation # 个人基本信息
from service.InformationSummary_Service import InformationSummary # 信息概要
from service.TransactionDetails_Service import TransactionDetails # 信贷交易明细
......@@ -36,27 +37,30 @@ class ReportAnalysis(BaseHandler):
if isFile:
try :
import datetime
html = InitHtml(htmlhandle)
asslysis = ['PersonalInformation',
'InformationSummary',
'TransactionDetails',
'QueryInfomation'
]
# maxconnections = 1 #最大并发数
# maxconnections = len(asslysis) #最大并发数
# semlock = threading.BoundedSemaphore(maxconnections)
# t = []
# for i in range(maxconnections):
# semlock.acquire()
# t.append(threading.Thread(target=eval(asslysis[i]),args=(html.menu_dict,)))
# semlock.release()
#
# t.append(threading.Thread(target=eval(asslysis[i]),args=(html.menu_dict,semlock)))
#
# t[i].start()
Heads(html.heads)
PersonalInformation(html.menu_dict)
InformationSummary(html.menu_dict)
TransactionDetails(html.menu_dict)
QueryInfomation(html.menu_dict)
result = Result.get_result()
Result.clear_result()
# '/Users/zhangpengcheng/Documents/量化派代码管理/credit-report-api/html/data.json'
if self._isdownload:
with open(settings.TO_JSON+'/'+self._filepath.split('/')[-1]+'.json', 'w',encoding='utf8') as file_obj:
......
service/Head_Service.py 0 → 100644
View file @ 9acec8d0
# -*- coding:utf-8 -*-
import json
from collections import defaultdict
import bs4
import pandas as pd
from service.Base_Service import Base,Result
class Heads(Base):
"""表头信息 1. 调查信息 2. 被调用户信息"""
def __init__(self,html):
Base.__init__(self,html)
Result.set_result({'表头':self.get_Logic(self.menu_dict)})
pass
def get_Logic(self,tag):
result = {}
result.update(self.get_heads_report(tag[0]))
result.update(self.get_heads_user(tag[1]))
return result
def get_heads_report(self,tag):
"""报告信息"""
result = {}
tds = tag.table.find_all('td')
for ii, i in enumerate(tds):
if isinstance(i,bs4.element.Tag):
if i.get('class') != None and 'high' in i.get('class'):
tm = ''
for msg in i.stripped_strings:
tm+=msg
tm = tm.split(':')
result.update({tm[0]:tm[1]})
return result
def get_heads_user(self,tag):
"""用户信息"""
result = {}
table = self.get_table(tag)
result.update(self.get_json(table))
return result
def get_information_Identity(self,tag,name):
"""身份信息"""
result = {name:{}}
tag_msg = tag[name][0].tbody.children
msg = self.get_json(tag_msg)
for k,v in msg.items():
for i in v:
result[name].update(i)
result.update(result)
return result
\ No newline at end of file
service/InitHtml_Service.py
View file @ 9acec8d0
......@@ -17,6 +17,8 @@ class InitHtml:
# detail = self.children[2].tbody.find_all('tr')
# menu_detail = detail[4:]
menu_detail = children[2].tbody.children
self.heads = []
self.heads.append(children[1]) # 表头 - 调查信息
self.menu_dict = self.get_menu(menu_detail) # 获取报告标题信息
def get_children(self,soup):
......@@ -52,6 +54,9 @@ class InitHtml:
key = None
for ii ,i in enumerate(menu):
if isinstance(i,bs4.element.Tag):
if ii == 1:
# 取得被调查的信息
self.heads.append(i)
con = i.contents
if con :
menu = con[1]
......
service/PersonalInformation_Service.py
View file @ 9acec8d0
......@@ -17,6 +17,7 @@ class PersonalInformation(Base):
Result.set_result(self.get_Logic(self.menu_personalInformation))
else:
Result.set_result({'个人基本信息':None})
def get_Logic(self,tag):
result = {}
if '身份信息' in tag.keys():
......
testfile/征信报告.ipynb
View file @ 9acec8d0
......@@ -277,8 +277,8 @@
]
},
"execution_count": 87,
"output_type": "execute_result",
"metadata": {}
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
......@@ -347,8 +347,8 @@
]
},
"execution_count": 88,
"output_type": "execute_result",
"metadata": {}
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
......@@ -433,8 +433,8 @@
]
},
"execution_count": 103,
"output_type": "execute_result",
"metadata": {}
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
......@@ -554,8 +554,8 @@
]
},
"execution_count": 57,
"output_type": "execute_result",
"metadata": {}
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
......@@ -566,67 +566,7 @@
"cell_type": "code",
"execution_count": 104,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>逾期持续月数</th>\n",
" <th>逾期月份</th>\n",
" <th>逾期金额</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1</td>\n",
" <td>2016.07</td>\n",
" <td>755</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1</td>\n",
" <td>2014.12</td>\n",
" <td>497</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>1</td>\n",
" <td>2016.07</td>\n",
" <td>755</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>1</td>\n",
" <td>2014.12</td>\n",
" <td>497</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
]
},
"output_type": "execute_result",
"metadata": {}
}
],
"outputs": [],
"source": [
"import pandas as pd\n",
"aa = {'逾期持续月数': ['1', '1', '1', '1'],\n",
......@@ -637,959 +577,141 @@
},
{
"cell_type": "code",
"execution_count": 105,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{}"
]
},
"execution_count": 105,
"output_type": "execute_result",
"metadata": {}
}
],
"source": [
"tag = menu_informationSummary['授信及负债信息概要'][1]\n",
"def get_table(tag):\n",
" \"\"\"把table 转list\"\"\"\n",
" tag_list = []\n",
" table = tag.tbody\n",
" for i,t in enumerate(table):\n",
" if isinstance(t,bs4.element.Tag):\n",
" tag_list.append(t)\n",
" return tag_list\n",
"\n",
"table_tag = get_table(tag)\n",
"_json = get_json(table_tag)\n",
"_json"
]
},
{
"cell_type": "code",
"execution_count": 106,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'未结清贷款信息汇总': {}, '未销户贷记卡信息汇总': {}}\n"
]
}
],
"source": [
"result = {}\n",
"for i,tag in enumerate(menu_informationSummary['授信及负债信息概要']):\n",
" if isinstance(tag,bs4.element.Tag):\n",
" if tag.td.get('align'):\n",
" key = tag.td.get_text()\n",
" else:\n",
" if tag.tbody:\n",
" table_tag = get_table(tag)\n",
" _json = get_json(table_tag)\n",
" result[key] = _json\n",
"print(result)"
]
},
{
"cell_type": "code",
"execution_count": 107,
"metadata": {},
"outputs": [],
"source": [
"# _key = menu_personalInformation.keys()\n",
"# for k in _key:\n",
" \n",
"# table_tagmsg_tag = menu_personalInformation[k][0]\n",
"# table_tagmsg = get_table(table_tagmsg_tag)\n",
"# _json = get_json(table_tagmsg)\n",
"# print(_json)\n",
"\n",
"table_tag_live = menu_personalInformation['居住信息'][0]\n",
"# table_tagmsg = get_table(table_tagmsg_tag)\n",
"# _json = get_json(table_tagmsg)\n",
"# _json"
]
},
{
"cell_type": "code",
"execution_count": 108,
"metadata": {},
"outputs": [],
"source": [
"tag = table_tag_live.tbody"
]
},
{
"cell_type": "code",
"execution_count": 109,
"metadata": {},
"outputs": [],
"source": [
"_result = []\n",
"for i in tag:\n",
" td = tag.findAll('td')\n",
" _v = []\n",
" for t in td:\n",
" if t.get('style'):\n",
" for v in t.stripped_strings:\n",
" _v.append(v)\n",
" _result.append(_v)\n",
" \n",
" \n",
"# if isinstance(i,bs4.element.Tag):\n",
"# if i.td.get('colspan') == None:\n",
"# print(i)"
]
},
{
"cell_type": "code",
"execution_count": 110,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[['1', '浙xxxxxxxx司', '浙xxxxxxxxx9号'], ['2', '浙xxxxxxxxxx中心', '--'], ['3', '浙xxx司', '浙xxxxxxxx9号'], ['4', '杭xxxxxx络', '浙xxxxxxxxxxx902'], ['5', '杭xxxxxxxx公司', '浙xxxxxxxxxxxx际'], ['1', '--'], ['--', '--', '--'], ['1900', '2019.02.20', '2'], ['--', '租赁和商务服务业', '--'], ['--', '--', '2018.09.28'], ['3', '--', '--'], ['--', '--', '2017'], ['2018.03.05', '4', '--'], ['信息传输、计算机服务和软件业', '--', '无'], ['--', '2018.02.24', '5'], ['--', '信息传输、计算机服务和软件业', '一般员工'], ['--', '--', '2018.01.16']]\n"
]
}
],
"source": [
"table = menu_personalInformation['职业信息']\n",
"result = []\n",
"bodys= []\n",
"heads = {}\n",
"\n",
"def get_head(tag):\n",
" num = 0\n",
" for i,k in enumerate(table[0].tr.children):\n",
" if isinstance(k,bs4.element.Tag):\n",
" num+=1\n",
" return num\n",
"result_2 = defaultdict(list)\n",
"\n",
"for i,val in enumerate(table):\n",
" tds = val.find_all('td')\n",
" index = 0\n",
" _i = 0\n",
" for ii,td in enumerate(tds):\n",
" if isinstance(td,bs4.element.Tag):\n",
"# print(td)\n",
" td_num = get_head(val)\n",
" \n",
" if td.get('style') == None:\n",
" for h in td.stripped_strings:\n",
" heads[h] = []\n",
"\n",
" else:\n",
" for v in td.stripped_strings:\n",
" bodys.append(v)\n",
" index +=1\n",
" if index%td_num == 0 :\n",
"# print(td)\n",
" if bodys:\n",
" result.append(bodys)\n",
" bodys = []\n",
"# for mm, m in enumerate(td.stripped_strings):\n",
"# if td.get('style') == None:\n",
"# key.append(m)\n",
"# else:\n",
"# result_2[key[index%td_num]].append(m)\n",
"# index +=1\n",
"\n",
"print(result)"
]
},
{
"cell_type": "code",
"execution_count": 111,
"metadata": {},
"outputs": [],
"source": [
"# table = menu_personalInformation['身份信息']\n",
"table = menu_transactionDetailsType['贷款'][1][2]"
]
},
{
"cell_type": "code",
"execution_count": 65,
"metadata": {},
"outputs": [],
"source": [
"table = menu_personalInformation['职业信息']\n",
"tag = table[0].tbody.children\n",
"def get_json(tag):\n",
" result = defaultdict(list)\n",
" key = []\n",
" _num = 0\n",
" for tr in tag:\n",
" _re = []\n",
" index = 0\n",
" if isinstance(tr,bs4.element.Tag):\n",
" tds = tr.find_all('td')\n",
" value = []\n",
" # key = []\n",
" td_num = len(tds)\n",
" for td in tds:\n",
" for m in td.stripped_strings:\n",
" if td.get('style') == None:\n",
" key.append(m)\n",
" else:\n",
" value.append(m)\n",
" index+=1\n",
"\n",
" if len(key)!=td_num:\n",
" key = key[len(key)-td_num:]\n",
" _num += 1\n",
" if value:\n",
" result[_num].append(dict(zip(key,value)))\n",
" return dict(result)\n",
"_identity = get_json(tag)"
]
},
{
"cell_type": "code",
"execution_count": 113,
"metadata": {},
"outputs": [],
"source": [
"# 11.查询记录汇总\n",
"querySummary = menu_queryInformation['查询记录汇总'][0]\n",
"query_Detail = get_table(querySummary)\n",
"query_Detail[1]\n",
"_result = {'最近1个月内的查询机构数':{'贷款审批':None,'信用卡审批':None},\n",
" '最近1个月内的查询次数':{'贷款审批':None,'信用卡审批':None,'本人查询':None},\n",
" '最近2年内的查询次数':{'贷后管理':None,'担保资格审查':None,'特约商户实名审查':None}\n",
" }\n",
"values = []\n",
"\n",
"for i ,td in enumerate(query_Detail[2]):\n",
" if isinstance(td,bs4.element.Tag):\n",
" tm = ''\n",
" for msg in td.stripped_strings:\n",
" tm+=msg\n",
" values.append(tm)\n",
"result = {'查询记录汇总':{}}\n",
"index = 0\n",
"for k ,v in _result.items():\n",
" for kk,vv in v.items():\n",
" result['查询记录汇总'][kk+k] = values[index]\n",
" index+=1"
]
},
{
"cell_type": "code",
"execution_count": 114,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'查询记录汇总': {'信用卡审批最近1个月内的查询机构数': '0',\n",
" '信用卡审批最近1个月内的查询次数': '0',\n",
" '担保资格审查最近2年内的查询次数': '0',\n",
" '本人查询最近1个月内的查询次数': '0',\n",
" '特约商户实名审查最近2年内的查询次数': '0',\n",
" '贷后管理最近2年内的查询次数': '24',\n",
" '贷款审批最近1个月内的查询机构数': '2',\n",
" '贷款审批最近1个月内的查询次数': '2'}}"
]
},
"execution_count": 114,
"output_type": "execute_result",
"metadata": {}
}
],
"source": [
"result"
]
},
{
"cell_type": "code",
"execution_count": 116,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{0: [{'查询原因': '贷款审批', '查询操作员': 'VB', '查询日期': '2019.05.07', '编号': '1'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'XZ', '查询日期': '2019.04.21', '编号': '2'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'ZV', '查询日期': '2019.04.08', '编号': '3'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'OX', '查询日期': '2019.04.01', '编号': '4'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'YM', '查询日期': '2019.03.05', '编号': '5'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'YP', '查询日期': '2019.03.05', '编号': '6'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'SC', '查询日期': '2019.03.04', '编号': '7'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'VB', '查询日期': '2019.03.01', '编号': '8'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'OX', '查询日期': '2019.02.26', '编号': '9'},\n",
" {'查询原因': '信用卡审批', '查询操作员': 'RZ', '查询日期': '2019.02.20', '编号': '10'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'JJ', '查询日期': '2019.02.18', '编号': '11'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'YP', '查询日期': '2019.01.08', '编号': '12'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'OX', '查询日期': '2018.12.05', '编号': '13'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'DG', '查询日期': '2018.11.14', '编号': '14'},\n",
" {'查询原因': '信用卡审批', '查询操作员': 'RZ', '查询日期': '2018.11.14', '编号': '15'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'YM', '查询日期': '2018.11.07', '编号': '16'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'YP', '查询日期': '2018.11.07', '编号': '17'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'TX', '查询日期': '2018.11.07', '编号': '18'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'XZ', '查询日期': '2018.11.07', '编号': '19'},\n",
" {'查询原因': '信用卡审批', '查询操作员': 'OO', '查询日期': '2018.10.29', '编号': '20'},\n",
" {'查询原因': '信用卡审批', '查询操作员': 'HR', '查询日期': '2018.10.12', '编号': '21'},\n",
" {'查询原因': '信用卡审批', '查询操作员': 'NP', '查询日期': '2018.10.08', '编号': '22'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'DG', '查询日期': '2018.10.08', '编号': '23'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'OX', '查询日期': '2018.10.01', '编号': '24'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'XZ', '查询日期': '2018.09.28', '编号': '25'},\n",
" {'查询原因': '信用卡审批', '查询操作员': 'ZH', '查询日期': '2018.09.27', '编号': '26'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'ZV', '查询日期': '2018.09.09', '编号': '27'},\n",
" {'查询原因': '信用卡审批', '查询操作员': 'ZH', '查询日期': '2018.09.05', '编号': '28'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'ZZ', '查询日期': '2018.09.04', '编号': '29'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'XZ', '查询日期': '2018.08.17', '编号': '30'},\n",
" {'查询原因': '信用卡审批', '查询操作员': 'OO', '查询日期': '2018.08.14', '编号': '31'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'PP', '查询日期': '2018.08.08', '编号': '32'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'QC', '查询日期': '2018.08.07', '编号': '33'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'ZV', '查询日期': '2018.07.31', '编号': '34'},\n",
" {'查询原因': '信用卡审批', '查询操作员': 'GH', '查询日期': '2018.07.12', '编号': '35'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'ZV', '查询日期': '2018.06.25', '编号': '36'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'GH', '查询日期': '2018.06.25', '编号': '37'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'GZ', '查询日期': '2018.06.25', '编号': '38'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'XZ', '查询日期': '2018.06.20', '编号': '39'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'PK', '查询日期': '2018.06.07', '编号': '40'},\n",
" {'查询原因': '信用卡审批', '查询操作员': 'ZH', '查询日期': '2018.06.05', '编号': '41'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'BM', '查询日期': '2018.05.26', '编号': '42'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'XZ', '查询日期': '2018.05.14', '编号': '43'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'ZF', '查询日期': '2018.05.12', '编号': '44'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'ZV', '查询日期': '2018.05.10', '编号': '45'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'RZ', '查询日期': '2018.05.09', '编号': '46'},\n",
" {'查询原因': '信用卡审批', '查询操作员': 'RF', '查询日期': '2018.03.09', '编号': '47'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'XZ', '查询日期': '2018.03.07', '编号': '48'},\n",
" {'查询原因': '信用卡审批', '查询操作员': 'BU', '查询日期': '2018.02.28', '编号': '49'},\n",
" {'查询原因': '信用卡审批', '查询操作员': 'XZ', '查询日期': '2018.01.14', '编号': '50'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'YM', '查询日期': '2017.12.14', '编号': '51'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'XZ', '查询日期': '2017.11.05', '编号': '52'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'YM', '查询日期': '2017.11.01', '编号': '53'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'ZN', '查询日期': '2017.11.01', '编号': '54'},\n",
" {'查询原因': '信用卡审批', '查询操作员': 'CB', '查询日期': '2017.10.18', '编号': '55'},\n",
" {'查询原因': '信用卡审批', '查询操作员': 'ZH', '查询日期': '2017.10.17', '编号': '56'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'PZ', '查询日期': '2017.10.14', '编号': '57'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'ST', '查询日期': '2017.09.28', '编号': '58'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'ZN', '查询日期': '2017.09.25', '编号': '59'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'YM', '查询日期': '2017.09.24', '编号': '60'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'JJ', '查询日期': '2017.09.12', '编号': '61'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'RC', '查询日期': '2017.09.10', '编号': '62'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'GH', '查询日期': '2017.08.21', '编号': '63'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'YM', '查询日期': '2017.08.21', '编号': '64'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'RC', '查询日期': '2017.07.13', '编号': '65'},\n",
" {'查询原因': '贷款审批', '查询操作员': 'YM', '查询日期': '2017.05.18', '编号': '66'}]}"
]
},
"execution_count": 116,
"output_type": "execute_result",
"metadata": {}
}
],
"source": [
"# 12.信贷审批查询记录明细\n",
"queryCreditApprovalDetail = menu_queryInformation['信贷审批查询记录明细'][0]\n",
"\n",
"tag = queryCreditApprovalDetail.tbody.children\n",
"def get_json(tag):\n",
" result = defaultdict(list)\n",
" key = []\n",
" _num = 0\n",
" for tr in tag:\n",
" _re = []\n",
" index = 0\n",
" if isinstance(tr,bs4.element.Tag):\n",
" tds = tr.find_all('td')\n",
" value = []\n",
" # key = []\n",
" td_num = len(tds)\n",
" for td in tds:\n",
" for m in td.stripped_strings:\n",
" if td.get('style') == None:\n",
" key.append(m)\n",
" else:\n",
" value.append(m)\n",
" index+=1\n",
"\n",
" if len(key)!=td_num:\n",
" key = key[len(key)-td_num:]\n",
" _num += 1\n",
" if value:\n",
" result[_num].append(dict(zip(key,value)))\n",
" return dict(result)\n",
"get_json(tag)"
]
},
{
"cell_type": "code",
"execution_count": 117,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'逾期（透支）信息汇总': {'准贷记卡60天以上透支': {'单月最高透支余额': '0',\n",
" '最长透支月数': '0',\n",
" '月份数': '0',\n",
" '账户数': '0'},\n",
" '贷款逾期': {'单月最高逾期总额': '3,119', '最长逾期月数': '1', '月份数': '1', '笔数': '1'},\n",
" '贷记卡逾期': {'单月最高逾期总额': '0', '最长逾期月数': '1', '月份数': '2', '账户数': '2'}}}"
]
},
"execution_count": 117,
"output_type": "execute_result",
"metadata": {}
}
],
"source": [
"# 逾期及违约信息概要\n",
"def get_information_overdueAndDefault(tag,name):\n",
" \"\"\"逾期及违约信息概要\"\"\"\n",
" result = {}\n",
" tags = tag[name]\n",
" if tags:\n",
" for i,tag in enumerate(tags):\n",
" if isinstance(tag,bs4.element.Tag):\n",
" if tag.td.get('align'):\n",
" key = tag.td.get_text()\n",
" else:\n",
" if tag.tbody:\n",
" table_tag = get_table(tag)\n",
" _json = get_json_overdueAndDefault(table_tag)\n",
" result[key] = _json\n",
"\n",
" return result\n",
"def get_json_overdueAndDefault(tag):\n",
" column = [] #一共几列\n",
" head = []\n",
" key = []\n",
" body = []\n",
" line_id = 0 #行\n",
" row_id = 0 #列\n",
" values = []\n",
" result = {}\n",
" for i,tds in enumerate(tag):\n",
" tds = tds.children\n",
" for ii ,td in enumerate(tds):\n",
" if isinstance(td,bs4.element.Tag):\n",
" if td.get('colspan'):\n",
" column.append(int(td.get('colspan')))\n",
" tm = ''\n",
" for k in td.stripped_strings:\n",
" tm+=k\n",
" head.append(tm)\n",
" else:\n",
" tm = ''\n",
" for k in td.stripped_strings:\n",
" tm+=k\n",
" values.append(tm)\n",
" row_id += 1\n",
" if line_id<=1:\n",
" if row_id % column[line_id] == 0:\n",
" key.append(values)\n",
" values = []\n",
"# print(row_id,line_id,column[line_id])\n",
" else:\n",
" if row_id % column[line_id] == 0:\n",
" body.append(values)\n",
" values = []\n",
" line_id +=1\n",
" for i,h in enumerate(head):\n",
" result[h] = dict(zip(key[i],body[i]))\n",
" return result\n",
"get_information_overdueAndDefault(menu_informationSummary,'逾期及违约信息概要')"
]
},
{
"cell_type": "code",
"execution_count": 118,
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"def get_DebitCard_Details(tag):\n",
" \"\"\"贷记卡明细拆分\"\"\"\n",
" result = {'贷记卡':{}}\n",
" _result = []\n",
" for v in tag:\n",
" _json = {}\n",
" _json['序号'] =v[0]\n",
" _json['描述'] =v[0]+'.'+v[1]\n",
" r = re.findall(\"(\\d{4}年\\d{1,2}月\\d{1,2}日)(.*?)发放的(.*?)，业务号(.*?)，授信额度[折合人民币]{0,5}(.*?)元，共享授信额度[折合人民币]{0,5}(.*?)元，(.*?)[，|。]{1}截至(\\d{4}年\\d{1,2}月\\d{1,2}日)，(账户状态为“[\\u4E00-\\u9FA5]{1,2}”)*\",v[1])\n",
" if r :\n",
" r = r[0]\n",
" _json['银行名称'] = r[1]\n",
" _json['授信金额'] = r[4]\n",
" _json['共享授信金额'] = r[5]\n",
" _json['授信时间'] = r[0]\n",
" _json['截止时间'] = r[7]\n",
" if r[8]:\n",
" r_temp = re.findall('账户状态为“(.*?)”',r[8])[0]\n",
" _json['账户状态 '] = r_temp\n",
"\n",
" if v[2] != None:\n",
" table_tag = get_table(v[2])\n",
" \n",
" table_json = get_json_DebitCard(table_tag)\n",
" key_name = table_json.keys()\n",
" plan = ''\n",
" overdue = ''\n",
" temp_json = {}\n",
" for name in key_name:\n",
" r = re.findall(\"(\\d{4}年\\d{1,2}月)-(\\d{4}年\\d{1,2}月)的([\\S+]*)\",name)\n",
" if r :\n",
" if r[0][2] == '还款记录':\n",
" temp_json['近24个月的还款记录-开始时间'] =r[0][0]\n",
" temp_json['近24个月的还款记录-截至时间'] =r[0][1]\n",
" plan = name\n",
" if r[0][2] == '逾期记录':\n",
" overdue = name\n",
"\n",
" if plan:\n",
" table_json['近24个月的还款记录-各期还款记录'] = table_json.pop(plan)\n",
" if overdue:\n",
" del table_json[overdue]\n",
"\n",
" if temp_json:\n",
" table_json.update(temp_json)\n",
"\n",
" if '逾期记录' in table_json.keys():\n",
" table_json['近5年逾期记录'] = table_json.pop('逾期记录')\n",
"\n",
" _json.update(table_json)\n",
" _result.append(_json)\n",
" result['贷记卡'] = _result\n",
" return result\n",
"\n",
"table_tag_new = None\n",
"def get_json_DebitCard(tag):\n",
" result = defaultdict(dict)\n",
" key = []\n",
" value = []\n",
" value_list = []\n",
"\n",
" temp_key = None\n",
" for i,val in enumerate(tag):\n",
" tds = val.find_all('td')\n",
" # tds_len = len(tds)\n",
" next_sibling = val.next_sibling\n",
" if (next_sibling == ' ' or next_sibling =='\\n' ) and next_sibling!=None:\n",
" next_sibling = val.next_sibling.next_sibling\n",
" # tds_next = next_sibling.find_all('td')\n",
" # tds_next_len = len(next_sibling.find_all('td'))\n",
"\n",
" for ii ,td in enumerate(tds):\n",
" # print(tds_len,tds_next_len,'---1---')\n",
" if isinstance(td,bs4.element.Tag):\n",
" if val.get('align') or td.get('colspan') == '24':\n",
" tm = ''\n",
" for k in td.stripped_strings:\n",
" tm+= k\n",
" key.append(tm)\n",
" else:\n",
" tm = ''\n",
" for k in td.stripped_strings:\n",
" tm+= k\n",
" value.append(tm)\n",
" if len(key) == 0 and temp_key:\n",
" value_list.append(tm)\n",
" r = re.findall('\\d{4}年\\d{1,2}月-\\d{4}年\\d{1,2}月的([\\S+]*)',temp_key)\n",
" if r:\n",
" r = r[0]\n",
" if result[temp_key][ii] not in result[r].keys():\n",
" result[r][result[temp_key][ii]] = []\n",
" result[r][result[temp_key][ii]].append(value_list[ii])\n",
"\n",
" if key and value:\n",
" if len(key) == len(value):\n",
" result.update(dict(zip(key,value)))\n",
" else:\n",
" result.update(dict({key[0]:value}))\n",
" temp_key = key[0]\n",
" key=[]\n",
" value = []\n",
" return dict(result)"
"result = {\n",
" \"result\": [\n",
" {\n",
" \"表头\": {\n",
" \"被查询者姓名\": \"王xxx\",\n",
" \"查询原因\": \"贷款审批\",\n",
" \"报告时间\": \"2019.05.16 08\",\n",
" \"查询操作员\": \"武汉X\",\n",
" \"查询请求时间\": \"2019.05.16 08\",\n",
" \"被查询者证件号码\": \"1xX\",\n",
" \"报告编号\": \"2019051600007544616066\",\n",
" \"被查询者证件类型\": \"身份证\"\n",
" }\n",
" },\n",
" {\n",
" \"身份信息\": {\n",
" \"性别\": \"x性\",\n",
" \"单位电话\": \"3xxxxxxx0\",\n",
" \"学历\": \"大学本科（简称\\\"大学\\\"）\",\n",
" \"学位\": \"其他\",\n",
" \"户籍地址\": \"河xxxxxxxxxxxxxxxxxxxxx51\",\n",
" \"婚姻状况\": \"未婚\",\n",
" \"通讯地址\": \"河xxxxxxxxxxxxxxxxxxxxxxxx号\",\n",
" \"出生日期\": \"199x23\",\n",
" \"住宅电话\": \"13xxxxxxxxxxxxx56\",\n",
" \"手机号码\": \"13xxxxxxx56\"\n",
" },\n",
" \"居住信息\": [\n",
" {\n",
" \"编号\": \"1\",\n",
" \"居住状况\": \"未知\",\n",
" \"信息更新日期\": \"2019.03.27\",\n",
" \"居住地址\": \"花xxxxxxxxxxxxxxx元501\"\n",
" },\n",
" {\n",
" \"编号\": \"2\",\n",
" \"居住状况\": \"未知\",\n",
" \"信息更新日期\": \"2019.03.25\",\n",
" \"居住地址\": \"河xxxxxxxxxxxxxxxx14号\"\n",
" },\n",
" {\n",
" \"编号\": \"3\",\n",
" \"居住状况\": \"未知\",\n",
" \"信息更新日期\": \"2019.03.23\",\n",
" \"居住地址\": \"河xxxxxxxxxxxxxxxxxxx51\"\n",
" },\n",
" {\n",
" \"编号\": \"4\",\n",
" \"居住状况\": \"其他\",\n",
" \"信息更新日期\": \"2019.03.03\",\n",
" \"居住地址\": \"河xxxxxxxxxxxxxxxxxx14号\"\n",
" },\n",
" {\n",
" \"编号\": \"5\",\n",
" \"居住状况\": \"其他\",\n",
" \"信息更新日期\": \"2019.02.27\",\n",
" \"居住地址\": \"河xxxxxxxxxxxxxxxx9号楼\"\n",
" }\n",
" ],\n",
" \"职业信息\": [\n",
" {\n",
" \"进入本单位年份\": \"2018\",\n",
" \"编号\": 1,\n",
" \"职称\": \"--\",\n",
" \"信息更新日期\": \"2019.03.28\",\n",
" \"行业\": \"--\",\n",
" \"单位地址\": \"--\",\n",
" \"工作单位\": \"一般企业\",\n",
" \"职务\": \"--\",\n",
" \"职业\": \"不便分类的其它从业人员\"\n",
" },\n",
" {\n",
" \"进入本单位年份\": \"2019\",\n",
" \"编号\": 2,\n",
" \"职称\": \"--\",\n",
" \"信息更新日期\": \"2019.03.27\",\n",
" \"行业\": \"--\",\n",
" \"单位地址\": \"--\",\n",
" \"工作单位\": \"承德xxxxxx店\",\n",
" \"职务\": \"--\",\n",
" \"职业\": \"不便分类的其它从业人员\"\n",
" },\n",
" {\n",
" \"进入本单位年份\": \"--\",\n",
" \"编号\": 3,\n",
" \"职称\": \"--\",\n",
" \"信息更新日期\": \"2019.02.19\",\n",
" \"行业\": \"--\",\n",
" \"单位地址\": \"--\",\n",
" \"工作单位\": \"承德xxxxxxxx公司\",\n",
" \"职务\": \"--\",\n",
" \"职业\": \"专业技术人员\"\n",
" },\n",
" {\n",
" \"进入本单位年份\": \"--\",\n",
" \"编号\": 4,\n",
" \"职称\": \"--\",\n",
" \"信息更新日期\": \"2019.01.13\",\n",
" \"行业\": \"--\",\n",
" \"单位地址\": \"--\",\n",
" \"工作单位\": \"鸭xxxxxxx团\",\n",
" \"职务\": \"--\",\n",
" \"职业\": \"不便分类的其它从业人员\"\n",
" },\n",
" {\n",
" \"进入本单位年份\": \"--\",\n",
" \"编号\": 5,\n",
" \"职称\": \"--\",\n",
" \"信息更新日期\": \"2018.04.12\",\n",
" \"行业\": \"住宿和餐饮业\",\n",
" \"单位地址\": \"--\",\n",
" \"工作单位\": \"承德xxxxxxxxx体户\",\n",
" \"职务\": \"--\",\n",
" \"职业\": \"--\"\n",
" }\n",
" ]\n",
" }]}"
]
},
{
"cell_type": "code",
"execution_count": 122,
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"menu_transactionDetailsType = {}\n",
"for k,value in menu_transactionDetails.items():\n",
" menu_transactionDetailsType[k] = get_detailsList(value)\n",
"# get_DebitCard_Details(menu_transactionDetailsType['贷款'])\n",
"tag = None\n",
"for t in menu_transactionDetailsType['贷款']:\n",
" if t[0] == '1':\n",
" tag = t\n",
"table_tag = get_table(tag[2])"
]
},
{
"cell_type": "code",
"execution_count": 123,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[<tr>\n",
" <td align=\"center\" class=\"tdStyle\" colspan=\"3\">\n",
" <div align=\"center\" class=\"high\"><b><font color=\"#0066cc\"><span class=\"high\">账户状态</span></font></b></div></td>\n",
" <td align=\"center\" class=\"tdStyle\" colspan=\"3\">\n",
" <div align=\"center\" class=\"high\"><b><font color=\"#0066cc\"><span class=\"high\">五级分类</span></font></b></div></td>\n",
" <td align=\"center\" class=\"tdStyle\" colspan=\"3\">\n",
" <div align=\"center\" class=\"high\"><b><font color=\"#0066cc\"><span class=\"high\">本金余额</span></font></b></div></td>\n",
" <td align=\"center\" class=\"tdStyle\" colspan=\"3\">\n",
" <div align=\"center\" class=\"high\"><b><font color=\"#0066cc\"><span class=\"high\">剩余还<br/>款期数</span></font></b></div></td>\n",
" <td align=\"center\" class=\"tdStyle\" colspan=\"3\"><b><font color=\"#0066cc\"><span class=\"high\">本月应还款</span></font></b> </td>\n",
" <td align=\"center\" class=\"tdStyle\" colspan=\"3\"><b><font color=\"#0066cc\"><span class=\"high\">应还款日</span></font></b> </td>\n",
" <td align=\"center\" class=\"tdStyle\" colspan=\"3\"><b><font color=\"#0066cc\"><span class=\"high\">本月实还款</span></font></b> </td>\n",
" <td align=\"center\" class=\"tdStyle\" colspan=\"3\" style=\"BORDER-RIGHT-WIDTH: 1px\"><b><font color=\"#0066cc\"><span class=\"high\">最近一次<br/>还款日期</span></font></b> </td></tr>,\n",
" <tr>\n",
" <td class=\"tdStyle\" colspan=\"3\" style=\"WORD-BREAK: break-all\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">正常</span></font></div></td>\n",
" <td class=\"tdStyle\" colspan=\"3\" style=\"WORD-BREAK: break-all\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">正常</span></font></div></td>\n",
" <td class=\"tdStyle\" colspan=\"3\" style=\"WORD-BREAK: break-all\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">327,097</span></font></div></td>\n",
" <td class=\"tdStyle\" colspan=\"3\" style=\"WORD-BREAK: break-all\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">136</span></font></div></td>\n",
" <td class=\"tdStyle\" colspan=\"3\" style=\"WORD-BREAK: break-all\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">3,119</span></font></div></td>\n",
" <td class=\"tdStyle\" colspan=\"3\" style=\"WORD-BREAK: break-all\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">2019.04.14</span></font></div></td>\n",
" <td class=\"tdStyle\" colspan=\"3\" style=\"WORD-BREAK: break-all\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">3,119</span></font></div></td>\n",
" <td class=\"tdStyle\" colspan=\"3\" style=\"BORDER-RIGHT-WIDTH: 1px; WORD-BREAK: break-all\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">2019.04.14</span></font></div></td></tr>,\n",
" <tr>\n",
" <td align=\"center\" class=\"tdStyle\" colspan=\"4\">\n",
" <div align=\"center\" class=\"high\"><b><font color=\"#0066cc\"><span class=\"high\">当前逾期期数</span></font></b></div></td>\n",
" <td align=\"center\" class=\"tdStyle\" colspan=\"4\">\n",
" <div align=\"center\" class=\"high\"><b><font color=\"#0066cc\"><span class=\"high\">当前逾期金额</span></font></b></div></td>\n",
" <td align=\"center\" class=\"tdStyle\" colspan=\"4\">\n",
" <div align=\"center\" class=\"high\"><b><font color=\"#0066cc\"><span class=\"high\">逾期31-60天<br/>未还本金</span></font></b></div></td>\n",
" <td align=\"center\" class=\"tdStyle\" colspan=\"4\">\n",
" <div align=\"center\" class=\"high\"><b><font color=\"#0066cc\"><span class=\"high\">逾期61－90天<bt>未还本金</bt></span></font></b></div></td>\n",
" <td align=\"center\" class=\"tdStyle\" colspan=\"4\">\n",
" <div align=\"center\" class=\"high\"><b><font color=\"#0066cc\"><span class=\"high\">逾期91－180天<br/>未还本金</span></font></b></div></td>\n",
" <td align=\"center\" class=\"tdStyle\" colspan=\"4\" style=\"BORDER-RIGHT-WIDTH: 1px\">\n",
" <div align=\"center\" class=\"high\"><b><font color=\"#0066cc\"><span class=\"high\">逾期180天以<br/>上未还本金</span></font></b></div></td></tr>,\n",
" <tr>\n",
" <td class=\"tdStyle\" colspan=\"4\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">0</span></font></div></td>\n",
" <td class=\"tdStyle\" colspan=\"4\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">0</span></font></div></td>\n",
" <td class=\"tdStyle\" colspan=\"4\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">0</span></font></div></td>\n",
" <td class=\"tdStyle\" colspan=\"4\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">0</span></font></div></td>\n",
" <td class=\"tdStyle\" colspan=\"4\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">0</span></font></div></td>\n",
" <td class=\"tdStyle\" colspan=\"4\" style=\"BORDER-RIGHT-WIDTH: 1px\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">0</span></font></div></td></tr>,\n",
" <tr>\n",
" <td class=\"tdStyle\" colspan=\"24\" style=\"BORDER-RIGHT-WIDTH: 1px\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><strong><b>2017年05月-2019年04月的还款记录</b></strong></font></div></td></tr>,\n",
" <tr>\n",
" <td class=\"tdStyle\" width=\"37\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">N</span></font></div></td>\n",
" <td class=\"tdStyle\" width=\"37\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">N</span></font></div></td>\n",
" <td class=\"tdStyle\" width=\"38\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">N</span></font></div></td>\n",
" <td class=\"tdStyle\" width=\"38\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">N</span></font></div></td>\n",
" <td class=\"tdStyle\" width=\"38\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">N</span></font></div></td>\n",
" <td class=\"tdStyle\" width=\"38\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">N</span></font></div></td>\n",
" <td class=\"tdStyle\" width=\"38\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">N</span></font></div></td>\n",
" <td class=\"tdStyle\" width=\"38\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">N</span></font></div></td>\n",
" <td class=\"tdStyle\" width=\"38\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">N</span></font></div></td>\n",
" <td class=\"tdStyle\" width=\"38\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">N</span></font></div></td>\n",
" <td class=\"tdStyle\" width=\"38\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">N</span></font></div></td>\n",
" <td class=\"tdStyle\" width=\"38\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">N</span></font></div></td>\n",
" <td class=\"tdStyle\" width=\"38\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">N</span></font></div></td>\n",
" <td class=\"tdStyle\" width=\"38\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">N</span></font></div></td>\n",
" <td class=\"tdStyle\" width=\"38\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">N</span></font></div></td>\n",
" <td class=\"tdStyle\" width=\"38\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">N</span></font></div></td>\n",
" <td class=\"tdStyle\" width=\"38\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">N</span></font></div></td>\n",
" <td class=\"tdStyle\" width=\"38\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">N</span></font></div></td>\n",
" <td class=\"tdStyle\" width=\"38\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">N</span></font></div></td>\n",
" <td class=\"tdStyle\" width=\"38\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">N</span></font></div></td>\n",
" <td class=\"tdStyle\" width=\"38\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">N</span></font></div></td>\n",
" <td class=\"tdStyle\" width=\"38\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">N</span></font></div></td>\n",
" <td class=\"tdStyle\" width=\"38\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">N</span></font></div></td>\n",
" <td class=\"tdStyle\" style=\"BORDER-RIGHT-WIDTH: 1px\" width=\"28\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">N</span></font></div></td></tr>,\n",
" <tr>\n",
" <td class=\"tdStyle\" colspan=\"24\" style=\"BORDER-RIGHT-WIDTH: 1px\">\n",
" <div align=\"center\" class=\"high\"><b><font color=\"#0066cc\"><span class=\"high\">2015年08月-2017年04月的逾期记录 </span></font></b></div></td></tr>,\n",
" <tr>\n",
" <td class=\"tdStyle\" colspan=\"4\">\n",
" <div align=\"center\" class=\"high\"><b><font color=\"#0066cc\"><span class=\"high\">逾期月份</span></font></b></div></td>\n",
" <td class=\"tdStyle\" colspan=\"4\">\n",
" <div align=\"center\" class=\"high\"><b><font color=\"#0066cc\"><span class=\"high\">逾期持续月数</span></font></b></div></td>\n",
" <td class=\"tdStyle\" colspan=\"4\">\n",
" <div align=\"center\" class=\"high\"><b><font color=\"#0066cc\"><span class=\"high\">逾期金额</span></font></b></div></td>\n",
" <td class=\"tdStyle\" colspan=\"4\">\n",
" <div align=\"center\" class=\"high\"><b><font color=\"#0066cc\"><span class=\"high\">逾期月份</span></font></b></div></td>\n",
" <td class=\"tdStyle\" colspan=\"4\">\n",
" <div align=\"center\" class=\"high\"><b><font color=\"#0066cc\"><span class=\"high\">逾期持续月数</span></font></b></div></td>\n",
" <td class=\"tdStyle\" colspan=\"4\" style=\"BORDER-RIGHT-WIDTH: 1px\">\n",
" <div align=\"center\" class=\"high\"><b><font color=\"#0066cc\"><span class=\"high\">逾期金额</span></font></b></div></td></tr>,\n",
" <tr>\n",
" <td class=\"tdStyle\" colspan=\"4\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">2016.12</span></font></div></td>\n",
" <td class=\"tdStyle\" colspan=\"4\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">1</span></font></div></td>\n",
" <td class=\"tdStyle\" colspan=\"4\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">3,119</span></font></div></td>\n",
" <td class=\"tdStyle\" colspan=\"4\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">--</span></font></div></td>\n",
" <td class=\"tdStyle\" colspan=\"4\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">--</span></font></div></td>\n",
" <td class=\"tdStyle\" colspan=\"4\" style=\"BORDER-RIGHT-WIDTH: 1px\">\n",
" <div align=\"center\" class=\"high\"><font color=\"#0066cc\"><span class=\"high\">--</span></font></div></td></tr>]"
]
},
"execution_count": 123,
"output_type": "execute_result",
"metadata": {}
}
],
"source": [
"table_tag"
"import json"
]
},
{
"cell_type": "code",
"execution_count": 310,
"metadata": {},
"outputs": [
{
"ename": "IndexError",
"evalue": "list index out of range",
"output_type": "error",
"traceback": [
"---------------------------------------------------------------------------",
"IndexError Traceback (most recent call last)",
"<ipython-input-310-6f3d66682be4> in <module>()\n 65 # _k = []\n 66 return dict(result)\n---> 67 get_json_DebitCard(table_tag)\n",
"<ipython-input-310-6f3d66682be4> in get_json_DebitCard(tag)\n 35 if r:\n 36 r = r[0]\n---> 37 if result[temp_key][ii] not in result[r].keys():\n 38 result[r][result[temp_key][ii]] = []\n 39 result[r][result[temp_key][ii]].append(value_list[ii])\n",
"IndexError: list index out of range"
]
}
],
"source": [
"def get_json_DebitCard(tag):\n",
" result = defaultdict(dict)\n",
" key = []\n",
" value = []\n",
" value_list = []\n",
" _k = []\n",
" _v = []\n",
"\n",
" temp_key = None\n",
" for i,val in enumerate(tag):\n",
" tds = val.find_all('td')\n",
" # tds_len = len(tds)\n",
"# next_sibling = val.next_sibling\n",
"# if (next_sibling == ' ' or next_sibling =='\\n' ) and next_sibling!=None:\n",
"# next_sibling = val.next_sibling.next_sibling\n",
"# # tds_next = next_sibling.find_all('td')\n",
"# # tds_next_len = len(next_sibling.find_all('td'))\n",
"\n",
" for ii ,td in enumerate(tds):\n",
" # print(tds_len,tds_next_len,'---1---')\n",
" if isinstance(td,bs4.element.Tag):\n",
" if td.get('align') or td.get('colspan') == '24':\n",
" tm = ''\n",
" for k in td.stripped_strings:\n",
" tm+= k\n",
" key.append(tm)\n",
" else:\n",
" tm = ''\n",
" for k in td.stripped_strings:\n",
" tm+= k\n",
" value.append(tm)\n",
" if len(key) == 0 and temp_key:\n",
" value_list.append(tm)\n",
" r = re.findall('\\d{4}年\\d{1,2}月-\\d{4}年\\d{1,2}月的([\\S+]*)',temp_key)\n",
" if r:\n",
" r = r[0]\n",
" if result[temp_key][ii] not in result[r].keys():\n",
" result[r][result[temp_key][ii]] = []\n",
" result[r][result[temp_key][ii]].append(value_list[ii])\n",
"\n",
" if key and value:\n",
" if len(key) == len(value):\n",
" result.update(dict(zip(key,value)))\n",
" else:\n",
" result.update(dict({key[0]:value}))\n",
" temp_key = key[0]\n",
" key=[]\n",
" value = []\n",
" \n",
" \n",
" \n",
"# if key and value_list:\n",
"# print(len(next_sibling.find_all('td')))\n",
"# print(next_sibling.find_all('td'))\n",
"# key = []\n",
"# value_list = []\n",
"# if len(key)==0 and len(value_list)==0:\n",
"# if value and len(value)<= len(tds):\n",
"# _k=value\n",
"# else :\n",
"# _v = value[len(tds):]\n",
"# result.update(dict(zip(_k,_v)))\n",
" \n",
"# value = []\n",
"# _k = []\n",
"# return dict(result)\n",
"get_json_DebitCard(table_tag)"
]
},
{
"cell_type": "code",
"execution_count": 312,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'2015年08月-2017年04月的逾期记录': ['逾期月份',\n",
" '逾期持续月数',\n",
" '逾期金额',\n",
" '逾期月份',\n",
" '逾期持续月数',\n",
" '逾期金额'],\n",
" '2017年05月-2019年04月的还款记录': ['N',\n",
" 'N',\n",
" 'N',\n",
" 'N',\n",
" 'N',\n",
" 'N',\n",
" 'N',\n",
" 'N',\n",
" 'N',\n",
" 'N',\n",
" 'N',\n",
" 'N',\n",
" 'N',\n",
" 'N',\n",
" 'N',\n",
" 'N',\n",
" 'N',\n",
" 'N',\n",
" 'N',\n",
" 'N',\n",
" 'N',\n",
" 'N',\n",
" 'N',\n",
" 'N'],\n",
" '五级分类': '正常',\n",
" '剩余还款期数': '136',\n",
" '应还款日': '2019.04.14',\n",
" '当前逾期期数': '0',\n",
" '当前逾期金额': '0',\n",
" '最近一次还款日期': '2019.04.14',\n",
" '本月实还款': '3,119',\n",
" '本月应还款': '3,119',\n",
" '本金余额': '327,097',\n",
" '账户状态': '正常',\n",
" '逾期180天以上未还本金': '0',\n",
" '逾期31-60天未还本金': '0',\n",
" '逾期61－90天未还本金': '0',\n",
" '逾期91－180天未还本金': '0',\n",
" '逾期记录': {'逾期持续月数': ['1', '--'],\n",
" '逾期月份': ['2016.12', '--'],\n",
" '逾期金额': ['3,119', '--']}}"
]
},
"execution_count": 312,
"output_type": "execute_result",
"metadata": {}
}
],
"source": [
"def get_json_DebitCard(tag):\n",
" result = defaultdict(dict)\n",
" key = []\n",
" value = []\n",
" value_list = []\n",
"\n",
" temp_key = None\n",
" for i,val in enumerate(tag):\n",
" tds = val.find_all('td')\n",
" # tds_len = len(tds)\n",
" # next_sibling = val.next_sibling\n",
" # if (next_sibling == ' ' or next_sibling =='\\n' ) and next_sibling!=None:\n",
" # next_sibling = val.next_sibling.next_sibling\n",
" # tds_next = next_sibling.find_all('td')\n",
" # tds_next_len = len(next_sibling.find_all('td'))\n",
"\n",
" for ii ,td in enumerate(tds):\n",
" # print(tds_len,tds_next_len,'---1---')\n",
" if isinstance(td,bs4.element.Tag):\n",
" if td.get('align') or td.get('colspan') == '24':\n",
" tm = ''\n",
" for k in td.stripped_strings:\n",
" tm+= k\n",
" key.append(tm)\n",
" else:\n",
" tm = ''\n",
" for k in td.stripped_strings:\n",
" tm+= k\n",
" value.append(tm)\n",
" if len(key) == 0 and temp_key:\n",
" value_list.append(tm)\n",
" r = re.findall('\\d{4}年\\d{1,2}月-\\d{4}年\\d{1,2}月的([\\S+]*)',temp_key)\n",
" if r:\n",
" r = r[0]\n",
" if result[temp_key][ii] not in result[r].keys():\n",
" result[r][result[temp_key][ii]] = []\n",
" result[r][result[temp_key][ii]].append(value_list[ii])\n",
"\n",
" if key and value:\n",
" if len(key) == len(value):\n",
" result.update(dict(zip(key,value)))\n",
" else:\n",
" result.update(dict({key[0]:value}))\n",
" temp_key = key[0]\n",
" key=[]\n",
" value = []\n",
" return dict(result)\n",
"get_json_DebitCard(table_tag)"
]
},
{
"cell_type": "code",
"execution_count": 315,
"execution_count": 11,
"metadata": {},
"outputs": [
{
......@@ -1613,59 +735,159 @@
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>逾期持续月数</th>\n",
" <th>逾期月份</th>\n",
" <th>逾期金额</th>\n",
" <th>居住信息</th>\n",
" <th>职业信息</th>\n",
" <th>表头</th>\n",
" <th>身份信息</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1</td>\n",
" <td>2016.12</td>\n",
" <td>3,119</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>{'被查询者证件号码': '1xX', '查询操作员': '武汉X', '报告编号': '2...</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2</td>\n",
" <td>NaN</td>\n",
" <td>[{'信息更新日期': '2019.03.27', '居住状况': '未知', '编号': ...</td>\n",
" <td>[{'信息更新日期': '2019.03.28', '单位地址': '--', '职务': ...</td>\n",
" <td>NaN</td>\n",
" <td>{'单位电话': '3xxxxxxx0', '通讯地址': '河xxxxxxxxxxxxxx...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" 居住信息 \\\n",
"0 NaN \n",
"1 [{'信息更新日期': '2019.03.27', '居住状况': '未知', '编号': ... \n",
"\n",
" 职业信息 \\\n",
"0 NaN \n",
"1 [{'信息更新日期': '2019.03.28', '单位地址': '--', '职务': ... \n",
"\n",
" 表头 \\\n",
"0 {'被查询者证件号码': '1xX', '查询操作员': '武汉X', '报告编号': '2... \n",
"1 NaN \n",
"\n",
" 身份信息 \n",
"0 NaN \n",
"1 {'单位电话': '3xxxxxxx0', '通讯地址': '河xxxxxxxxxxxxxx... "
]
},
"output_type": "execute_result",
"metadata": {}
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"aa = [{'逾期持续月数': '1', '逾期月份': '2016.12', '逾期金额': '3,119'},{'逾期持续月数':2}]\n",
"\n",
"pd.DataFrame("
"import pandas as pd\n",
"pd.read_json(json.dumps(result['result']))"
]
},
{
"cell_type": "code",
"execution_count": 317,
"execution_count": 15,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<filter at 0x10b612048>"
"{'居住信息': [{'信息更新日期': '2019.03.27',\n",
" '居住地址': '花xxxxxxxxxxxxxxx元501',\n",
" '居住状况': '未知',\n",
" '编号': '1'},\n",
" {'信息更新日期': '2019.03.25',\n",
" '居住地址': '河xxxxxxxxxxxxxxxx14号',\n",
" '居住状况': '未知',\n",
" '编号': '2'},\n",
" {'信息更新日期': '2019.03.23',\n",
" '居住地址': '河xxxxxxxxxxxxxxxxxxx51',\n",
" '居住状况': '未知',\n",
" '编号': '3'},\n",
" {'信息更新日期': '2019.03.03',\n",
" '居住地址': '河xxxxxxxxxxxxxxxxxx14号',\n",
" '居住状况': '其他',\n",
" '编号': '4'},\n",
" {'信息更新日期': '2019.02.27',\n",
" '居住地址': '河xxxxxxxxxxxxxxxx9号楼',\n",
" '居住状况': '其他',\n",
" '编号': '5'}],\n",
" '职业信息': [{'信息更新日期': '2019.03.28',\n",
" '单位地址': '--',\n",
" '工作单位': '一般企业',\n",
" '编号': 1,\n",
" '职业': '不便分类的其它从业人员',\n",
" '职务': '--',\n",
" '职称': '--',\n",
" '行业': '--',\n",
" '进入本单位年份': '2018'},\n",
" {'信息更新日期': '2019.03.27',\n",
" '单位地址': '--',\n",
" '工作单位': '承德xxxxxx店',\n",
" '编号': 2,\n",
" '职业': '不便分类的其它从业人员',\n",
" '职务': '--',\n",
" '职称': '--',\n",
" '行业': '--',\n",
" '进入本单位年份': '2019'},\n",
" {'信息更新日期': '2019.02.19',\n",
" '单位地址': '--',\n",
" '工作单位': '承德xxxxxxxx公司',\n",
" '编号': 3,\n",
" '职业': '专业技术人员',\n",
" '职务': '--',\n",
" '职称': '--',\n",
" '行业': '--',\n",
" '进入本单位年份': '--'},\n",
" {'信息更新日期': '2019.01.13',\n",
" '单位地址': '--',\n",
" '工作单位': '鸭xxxxxxx团',\n",
" '编号': 4,\n",
" '职业': '不便分类的其它从业人员',\n",
" '职务': '--',\n",
" '职称': '--',\n",
" '行业': '--',\n",
" '进入本单位年份': '--'},\n",
" {'信息更新日期': '2018.04.12',\n",
" '单位地址': '--',\n",
" '工作单位': '承德xxxxxxxxx体户',\n",
" '编号': 5,\n",
" '职业': '--',\n",
" '职务': '--',\n",
" '职称': '--',\n",
" '行业': '住宿和餐饮业',\n",
" '进入本单位年份': '--'}],\n",
" '身份信息': {'住宅电话': '13xxxxxxxxxxxxx56',\n",
" '出生日期': '199x23',\n",
" '单位电话': '3xxxxxxx0',\n",
" '婚姻状况': '未婚',\n",
" '学位': '其他',\n",
" '学历': '大学本科（简称\"大学\"）',\n",
" '性别': 'x性',\n",
" '户籍地址': '河xxxxxxxxxxxxxxxxxxxxx51',\n",
" '手机号码': '13xxxxxxx56',\n",
" '通讯地址': '河xxxxxxxxxxxxxxxxxxxxxxxx号'}}"
]
},
"execution_count": 317,
"output_type": "execute_result",
"metadata": {}
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"aa = '3,119'\n",
"filter (lambda x:x,aa)"
"result['result'][1]\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
......@@ -1677,16 +899,16 @@
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3.0
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.0"
"version": "3.4.4"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
\ No newline at end of file
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment