1. 提交征信解析第一版版

config/settings.py

+import os
+
+FILE_PATH = os.path.realpath(os.curdir)
+
+FEATURE_HOST = 'http://172.20.1.131:23010'
+FEATURE_HOST_MY = 'http://localhost:23010'
+
+
+url_reportanalysis = '/report'
+

handler/Base_Handler.py

+# -*- coding:utf-8 -*-
+
+import tornado.web
+
+class BaseHandler(tornado.web.RequestHandler):
+    # 初始化映射关系
+    def prepare(self):
+        self.set_header('Content-Type', "application/json; charset=UTF-8")
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

handler/ReportAnalysis_Handler.py

+# -*- coding:utf-8 -*-
+
+import threading
+from utils import JsonUtil
+from handler.Base_Handler import BaseHandler
+from service.PersonalInformation_Service import PersonalInformation # 个人基本信息
+from service.InformationSummary_Service import InformationSummary # 信息概要
+from service.TransactionDetails_Service import TransactionDetails # 信贷交易明细
+from service.QueryInfomation_Service import QueryInfomation # 查询记录
+
+from service.Base_Service import Result # 返回结果存储
+from service.InitHtml_Service import InitHtml # 初始化HTML
+
+class ReportAnalysis(BaseHandler):
+    def post(self):
+        self._filepath = self.get_argument('filepath', default=None)
+
+        if self._filepath == None or self._filepath == '':
+            self.write(JsonUtil.build_json(code = JsonUtil.Constants.Code_Params_Error,
+                                           mssage=JsonUtil.Constants.Msg_Params_Error.format('filepath',self._filepath)))
+            self.flush()
+
+        else:
+            filepath = self._filepath
+            isFile = False
+            try :
+                with open(filepath, 'rb') as f:
+                    htmlhandle = f.read()
+                    f.close()
+                    isFile = True
+            except IOError as e:
+                self.write(JsonUtil.build_json(code = JsonUtil.Constants.Code_File_Error,
+                                               mssage = JsonUtil.Constants.Msg_File_Error))
+                self.flush()
+
+            if isFile:
+                try :
+                    html = InitHtml(htmlhandle)
+                    asslysis = ['PersonalInformation',
+                                'InformationSummary',
+                                'TransactionDetails',
+                                'QueryInfomation'
+                                ]
+                    maxconnections = len(asslysis) #最大并发数
+                    semlock = threading.BoundedSemaphore(maxconnections)
+                    for i in range(maxconnections):
+                        semlock.acquire()
+                        t = threading.Thread(target=eval(asslysis[i]),args=(html.menu_dict,))
+                        semlock.release()
+                        t.start()
+                    # PersonalInformation(html.menu_dict)
+                    # InformationSummary(html.menu_dict)
+                    # TransactionDetails(html.menu_dict)
+                    # QueryInfomation(html.menu_dict)
+                    result = Result.get_result()
+                    Result.clear_result()
+                    self.write(JsonUtil.build_json(report = result,
+                                                   code=JsonUtil.Constants.Code_Success,
+                                                   mssage=JsonUtil.Constants.Msg_Success))
+                    self.flush()
+                    self.finish()
+                except :
+                    self.write(JsonUtil.build_json(code=JsonUtil.Constants.Code_Analysis_Error,
+                                                   mssage=JsonUtil.Constants.Msg_Analysis_Error
+                                                   ))
+                    self.flush()
+
+
+        # import json
+        # outpath = r'/Users/zhangpengcheng/Documents/量化派代码管理/credit_report/html/provinces.json'
+        # with open(outpath,"w+",encoding="utf-8") as  f:
+        #     _json = json.dumps(Result.get_result())
+        #     f.write(_json)
+        #     f.close()
+
+
+
+        # filepath = r'/Users/zhangpengcheng/Documents/量化派代码管理/credit_report/html/一代征信报告/432325197803211379.html'
+        # filepath = r'/Users/zhangpengcheng/Documents/量化派代码管理/credit_report/html/一代征信报告/13082119950823527X.htm'
+        # filepath = r'/Users/zhangpengcheng/Documents/量化派代码管理/credit_report/html/一代征信报告/32052219780226051X.htm'
+        # filepath = r'/Users/zhangpengcheng/Documents/量化派代码管理/credit_report/html/一代征信报告/140427198607030038.htm'
+        # filepath = r'/Users/zhangpengcheng/Documents/量化派代码管理/credit_report/html/一代征信报告/330682198805122815.htm'
+        # filepath = r'/Users/zhangpengcheng/Documents/量化派代码管理/credit_report/html/一代征信报告/440421198210078001.html'
+
+
+
+    # def run(self):
+    #     if self._target is not None:
+    #         self._return = self._target(*self._args, **self._kwargs)
+    #
+    # def get_return(self):
+    #     try:
+    #         return self._return
+    #     except Exception:
+    #         return None
+    #
+    # def join(self):
+    #     threading.Thread.join(self)
+    #     return self._return
\ No newline at end of file

service/Base_Service.py

+# -*- coding:utf-8 -*-
+
+import bs4
+import re
+from collections import defaultdict
+
+class Result:
+    """html to json"""
+    result = {}
+    html = None
+    @classmethod
+    def set_result(cls,result):
+        cls.result.update(result)
+
+    @classmethod
+    def get_result(cls):
+        return cls.result
+
+    @classmethod
+    def clear_result(cls):
+        cls.result = {}
+        return cls.result
+
+    @classmethod
+    def get_htmlMsg(cls,html):
+        if cls.html == None:
+            cls.html = 1
+        return cls.html
+
+
+class Base(object):
+    def __init__(self,html):
+        self.menu_dict = html
+        pass
+
+    def get_menuDetail(self,tag):
+        """按报告内部小标题拆分"""
+        _dict = defaultdict(list)
+        key = None
+        for i in range(len(tag)):
+            index = []
+            for m in tag[i].stripped_strings:
+                #             ret=re.findall(r'(\（一\）|\（二\）)[\ ]?(\w+)',m)
+                # ret=re.findall(r'(\（\S\）)[\ ]?(\w+)',m)
+                ret=re.findall(r'(\（[\u4E00-\u9FA5]\）)[\ ]?(\w+)',m)
+                if ret:
+                    key = ret[0][1]
+                    index.append(i)
+            if key:
+                if i not in index:
+                    _dict[key].append(tag[i])
+        return dict(_dict)
+
+    def get_menuDetail_personalInformation(self,tag):
+        """按报告内部小标题拆分 - 公共信息明细"""
+        _dict = defaultdict(list)
+        key = None
+        for i in tag:
+            if i.table==None:
+                if i.b:
+                    key = i.b.get_text()
+            else:
+                _dict[key].append( i.table)
+        return _dict
+
+    def get_menuDetail_queryInformation(self,tag):
+        """按报告内部小标题拆分 - """
+        _dict = defaultdict(list)
+        key = None
+        for i in tag:
+            if i.table==None:
+                if i.b:
+                    key = i.b.get_text()
+            else:
+                _dict[key].append( i.table)
+        return _dict
+
+    def get_table(self,tag):
+        """把table 转list"""
+        tag_list = []
+        table = tag.tbody
+        for i,t in enumerate(table):
+            if isinstance(t,bs4.element.Tag):
+                tag_list.append(t)
+        return tag_list
+
+    def get_json(self,tag):
+        """获取table解析后的json , 支持表头与值循环"""
+        result = defaultdict(list)
+        _key = []
+        # _colspan=0
+        index = 0
+        for i,val in enumerate(tag):
+            tds = val.find_all('td')
+            if i %2 == 0:
+                key = []
+                for ii ,td in enumerate(tds):
+                    if isinstance(td,bs4.element.Tag):
+                        # _colspan = td.get('colspan')
+                        tm = ''
+                        for k in td.stripped_strings:
+                            tm+= k
+                    key.append(tm)
+                _key.append(key)
+                index+=1
+            else:
+                if len(_key[index-1]) == len(tds):
+                    for ii ,td in enumerate(tds):
+                        if isinstance(td,bs4.element.Tag):
+                            value = ''
+                            for k in td.stripped_strings:
+                                value+=k
+                        result[_key[index-1][ii]] = value
+                else:
+                    value = []
+                    for ii ,td in enumerate(tds):
+                        value.append(td.get_text().strip())
+                    result[_key[index-1][0]] = value
+        return dict(result)
+
+    # def get_json_tableList(self,tag):
+    #     """支持 table 列表形式 存储
+    #     ::return list
+    #     """
+    #     _result = []
+    #     tag = tag.tbody
+    #     for i in tag:
+    #         td = tag.findAll('td')
+    #         _v = []
+    #         for t in td:
+    #             if t.get('style'):
+    #                 for v in t.stripped_strings:
+    #                     _v.append(v)
+    #         _result.append(_v)
+    #     return _result
\ No newline at end of file

service/InformationSummary_Service.py

+import bs4
+
+from service.Base_Service import Base,Result
+
+
+class InformationSummary(Base):
+    """信息概要"""
+    def __init__(self,filepath):
+        Base.__init__(self,filepath)
+
+        if '信息概要' in self.menu_dict.keys():
+            self.menu_informationSummary = self.get_menuDetail(self.menu_dict['信息概要'])  # 信息概要
+
+            Result.set_result(self.get_Logic(self.menu_informationSummary))
+        else:
+            Result.set_result({'信息概要':None})
+        pass
+
+    def get_Logic(self,tag):
+        result = {}
+        if '信用提示' in tag.keys(): # 信用汇总
+            result.update(self.get_information_creditSummary(tag,'信用提示'))
+            result.update(self.get_numberUnscramble(tag,'信用提示'))
+        if '逾期及违约信息概要' in tag.keys():
+            result.update(self.get_information_overdueAndDefault(tag,'逾期及违约信息概要'))
+
+        if '授信及负债信息概要' in tag.keys(): # 未结清贷款信息汇总 / 未销户贷记卡信息汇总 / 未销户准贷记卡信息汇总
+            result.update(self.get_information_uncLoanSummary(tag,'授信及负债信息概要'))
+
+        return result
+
+    def get_information_creditSummary(self,tag,name):
+        """信用提示 - 信用汇总or信用提示"""
+        result = {'信用汇总':{}}
+        tag = self.menu_informationSummary['信用提示'][0]
+        table_tag = self.get_table(tag)
+        result['信用汇总'].update(self.get_json(table_tag))
+        return result
+
+    def get_numberUnscramble(self,tag,name):
+        """信用提示 - 个人信用报告 “数字解读”"""
+        result = {}
+        tag = tag[name]
+        istrue = False
+        key = ''
+        if tag:
+            for i,t in enumerate(tag):
+                if isinstance(t,bs4.element.Tag):
+                    if '个人信用报告 “数字解读”' in t.td.get_text():
+                        istrue = True
+                        key = t.td.get_text()
+                        table = t.next_sibling.next_sibling
+        if istrue:
+            tag = table
+            if tag:
+                if tag.find_all('tbody'):
+                    for i,t in enumerate(tag):
+                        if isinstance(t,bs4.element.Tag):
+                            table_tag = self.get_table(t)
+                            _json = self.get_json(table_tag)
+                            result[key] = _json
+        return result
+
+    def get_information_uncLoanSummary(self,tag,name):
+        """未结清贷款信息汇总 / 未销户贷记卡信息汇总 / 未销户准贷记卡信息汇总"""
+        result = {}
+        tag = tag[name]
+        if tag:
+            for i,t in enumerate(tag):
+                if isinstance(t,bs4.element.Tag):
+                    if t.td.get('align'):
+                        key = t.td.get_text()
+                    else:
+                        if t.tbody:
+                            table_tag = self.get_table(t)
+                            _json = self.get_json(table_tag)
+                            result[key] = _json
+        return result
+
+    def get_information_overdueAndDefault(self,tag,name):
+        """逾期及违约信息概要"""
+        result = {}
+        tags = tag[name]
+        if tags:
+            for i,tag in enumerate(tags):
+                if isinstance(tag,bs4.element.Tag):
+                    if tag.td.get('align'):
+                        key = tag.td.get_text() # 获取第一行,设置为head
+                    else:
+                        if tag.tbody:
+                            table_tag = self.get_table(tag)
+                            _json = self.get_json_overdueAndDefault(table_tag)
+                            result[key] = _json
+        result['逾期信息汇总'] = result.pop('逾期（透支）信息汇总')
+        return result
+    def get_json_overdueAndDefault(self,tag):
+        """解析table"""
+        column = [] #一共几列
+        head = []
+        key = []
+        body = []
+        line_id = 0 #行
+        row_id = 0 #列
+        values = []
+        result = {}
+        for i,tds in enumerate(tag):
+            tds = tds.children
+            for ii ,td in enumerate(tds):
+                if isinstance(td,bs4.element.Tag):
+                    if td.get('colspan'):
+                        column.append(int(td.get('colspan')))
+                        tm = ''
+                        for k in td.stripped_strings:
+                            tm+=k
+                        head.append(tm)
+                    else:
+                        tm = ''
+                        for k in td.stripped_strings:
+                            tm+=k
+                        values.append(tm)
+                        row_id += 1
+                        if line_id<=1:
+                            if row_id % column[line_id] == 0:
+                                key.append(values)
+                                values = []
+                            #                         print(row_id,line_id,column[line_id])
+                        else:
+                            if row_id % column[line_id] == 0:
+                                body.append(values)
+                                values = []
+            line_id +=1
+        for i,h in enumerate(head):
+            result[h] = dict(zip(key[i],body[i]))
+        return result
+
+
+
+if __name__ == '__main__':
+    InformationSummary()
\ No newline at end of file

service/InitHtml_Service.py

+# -*- coding:utf-8 -*-
+
+from bs4 import BeautifulSoup,Comment
+import bs4,re
+from collections import defaultdict
+
+class InitHtml:
+    """初始化html 文件,拆分标题"""
+
+    def __init__(self,html):
+        htmlhandle = html
+        soup = BeautifulSoup(htmlhandle,'lxml')
+        comments = soup.findAll(text=lambda text: isinstance(text, Comment))
+        [comment.extract() for comment in comments] # 去除注释
+        [s.extract() for s in soup("style")] # 去除指定标签
+        children = self.get_children(soup)
+        # detail = self.children[2].tbody.find_all('tr')
+        # menu_detail = detail[4:]
+        menu_detail = children[2].tbody.children
+        self.menu_dict = self.get_menu(menu_detail) # 获取报告标题信息
+
+    def get_children(self,soup):
+        """获取子节点"""
+        children = []
+        for child in soup.body.children:
+            if child != '\n':
+                children.append(child)
+        return children
+
+
+
+    # def get_menu(self,menu):
+    #     """按报告标题大标题拆分"""
+    #     menu_dict = defaultdict(list)
+    #     index = []
+    #     for i in range(len(menu)):
+    #         for m in menu[i].stripped_strings:
+    #             # #         ret = re.findall(r"[一|二|三|四|五] .*", m)
+    #             # ret=re.findall('(一|二|三|四|五|六|十一)[\ ]{1,1}(\w+)',m)
+    #             ret=re.findall('([\u4E00-\u9FA5])[\ ]{1,1}(\w+)',m)
+    #             if ret:
+    #                 key = ret[0][1]
+    #                 index.append(i)
+    #         if i not in index:
+    #             menu_dict[key].append(menu[i])
+    #     return dict(menu_dict)
+
+    def get_menu(self,menu):
+        """按报告标题大标题拆分"""
+        menu_dict = defaultdict(list)
+        index = []
+        key = None
+        for ii ,i in enumerate(menu):
+            if isinstance(i,bs4.element.Tag):
+                con = i.contents
+                if con :
+                    menu = con[1]
+                    # menu = i.contents[1]
+                    for mm,m in enumerate(menu.stripped_strings):
+                        #                 ret = re.findall(r"[一|二|三|四|五] .*", m)
+                        ret=re.findall('([\u4E00-\u9FA5])[\ ]{1,1}(\w+)',m)
+                        if ret:
+                            key = ret[0][1]
+                            index.append(ii)
+
+                    if key:
+                        if ii not in index:
+                            menu_dict[key].append(i)
+        return dict(menu_dict)
+

service/PersonalInformation_Service.py

+# -*- coding:utf-8 -*-
+
+import json
+from collections import defaultdict
+import bs4
+import pandas as pd
+from service.Base_Service import Base,Result
+
+
+class PersonalInformation(Base):
+    """个人基本信息"""
+    def __init__(self,html):
+        Base.__init__(self,html)
+
+        if '个人基本信息' in self.menu_dict.keys():
+            self.menu_personalInformation = self.get_menuDetail_personalInformation(self.menu_dict['个人基本信息'])  # 个人基本信息
+            Result.set_result(self.get_Logic(self.menu_personalInformation))
+        else:
+            Result.set_result({'个人基本信息':None})
+    def get_Logic(self,tag):
+        result = {}
+        if '身份信息' in tag.keys():
+            result.update(self.get_information_Identity(tag,'身份信息'))
+        if '居住信息' in tag.keys():
+            result.update(self.get_information_Residential(tag,'居住信息'))
+        if '职业信息' in tag.keys():
+            result.update(self.get_information_Occupation(tag,'职业信息'))
+        return result
+
+    def get_information_Identity(self,tag,name):
+        """身份信息"""
+        result = {name:{}}
+        tag_msg = tag[name][0].tbody.children
+        msg = self.get_json(tag_msg)
+        for k,v in msg.items():
+            for i in v:
+                result[name].update(i)
+        result.update(result)
+        return result
+
+    def get_information_Residential(self,tag,name):
+        """居住信息"""
+        result = {name:{}}
+        tag_msg = tag[name][0].tbody.children
+        msg = self.get_json(tag_msg)
+        result[name] = msg[0]
+        return result
+
+    def get_information_Occupation(self,tag,name):
+        """职业信息"""
+        result = {name:{}}
+        tag_msg = tag[name][0].tbody.children
+        msg = self.get_json(tag_msg)
+        if msg :
+            basic = pd.read_json(json.dumps(msg[0]))
+            details =  pd.read_json(json.dumps(msg[1]))
+            df = pd.merge(basic,details,on='编号',how = 'left')
+            js = df.to_dict(orient='records')
+            result[name] = js
+        return result
+
+    def get_json(self,tag):
+        result = defaultdict(list)
+        key = []
+        _num = 0
+        for tr in tag:
+            index = 0
+            if isinstance(tr,bs4.element.Tag):
+                tds = tr.find_all('td')
+                value = []
+                #         key = []
+                td_num = len(tds)
+                for td in tds:
+                    for m in td.stripped_strings:
+                        if td.get('style') == None:
+                            key.append(m)
+                        else:
+                            value.append(m)
+                        index+=1
+
+                if len(key)!=td_num:
+                    key = key[len(key)-td_num:]
+                    _num += 1
+                if value:
+                    result[_num].append(dict(zip(key,value)))
+        return result
+
+if __name__ == '__main__':
+    PersonalInformation()
\ No newline at end of file

service/QueryInfomation_Service.py

+from collections import defaultdict
+
+import bs4
+
+from service.Base_Service import Base,Result
+
+
+class QueryInfomation(Base):
+    """查询记录"""
+    def __init__(self,filepath):
+        Base.__init__(self,filepath)
+        if '查询记录' in self.menu_dict.keys():
+            self.menu_queryInformation = self.get_menuDetail_personalInformation(self.menu_dict['查询记录'])
+            Result.set_result(self.get_Logic(self.menu_queryInformation))
+        else:
+            Result.set_result({'查询记录':None})
+        pass
+
+    def get_Logic(self,tag):
+        result = {}
+        if '查询记录汇总' in tag.keys():
+            result.update(self.get_querySummary(tag,'查询记录汇总'))
+        if '信贷审批查询记录明细' in tag.keys():
+            result.update(self.get_queryCreditApprovalDetail(tag,'信贷审批查询记录明细'))
+        return result
+
+    def get_querySummary(self,tag,name):
+        """查询记录汇总"""
+        _result = {'最近1个月内的查询机构数':{'贷款审批':None,'信用卡审批':None},
+                  '最近1个月内的查询次数':{'贷款审批':None,'信用卡审批':None,'本人查询':None},
+                  '最近2年内的查询次数':{'贷后管理':None,'担保资格审查':None,'特约商户实名审查':None}
+                  }
+        result = {name:{}}
+        values = []
+        tds = self.get_table(tag[name][0])
+        for i ,td in enumerate(tds[2]):
+            if isinstance(td,bs4.element.Tag):
+                tm = ''
+                for msg in td.stripped_strings:
+                    tm+=msg
+                values.append(tm)
+        index = 0
+        for k ,v in _result.items():
+            for kk,vv in v.items():
+                result[name][kk+k] = values[index]
+                index+=1
+        return result
+
+    def get_queryCreditApprovalDetail(self,tag,name):
+        """信贷审批查询记录明细"""
+        result = {}
+        table = tag[name][0].tbody.children
+        result.update(self.get_json(table))
+        result[name] = result.pop(0)
+        return result
+
+    def get_json(self,tag):
+        result = defaultdict(list)
+        key = []
+        _num = 0
+        for tr in tag:
+            index = 0
+            if isinstance(tr,bs4.element.Tag):
+                tds = tr.find_all('td')
+                value = []
+                #         key = []
+                td_num = len(tds)
+                for td in tds:
+                    for m in td.stripped_strings:
+                        if td.get('style') == None:
+                            key.append(m)
+                        else:
+                            value.append(m)
+                        index+=1
+
+                if len(key)!=td_num:
+                    key = key[len(key)-td_num:]
+                    _num += 1
+                if value:
+                    result[_num].append(dict(zip(key,value)))
+        return result
+
+
+if __name__ == '__main__':
+    QueryInfomation()
\ No newline at end of file

startup.py

+# -*- coding:utf-8 -*-
+
+'''
+服务启动接口
+'''
+from tornado.web import Application
+from tornado.httpserver import HTTPServer
+from tornado.ioloop import IOLoop
+import tornado.log
+import logging
+import tornado.options
+from config import settings as URL
+
+# from handler import LoanDueRateHandler
+from tornado.options import define, options
+
+from handler import ReportAnalysis_Handler
+
+from tornado.options import define, options
+define("port", default=23010, help="run on the given port ", type=int)
+define("log_path", default='/tmp', help="log path ", type=str)
+
+class LogFormatter(tornado.log.LogFormatter):
+    def __init__(self):
+        print('%(color)s[%(asctime)s %(filename)s:%(funcName)s:%(lineno)d %(levelname)s]%(end_color)s %(message)s')
+        super(LogFormatter, self).__init__(
+                fmt='%(color)s[%(asctime)s %(filename)s:%(funcName)s:%(lineno)d %(levelname)s]%(end_color)s %(message)s',
+                datefmt='%Y-%m-%d %H:%M:%S'
+        )
+
+def apps():
+    return Application([
+        (URL.url_reportanalysis, ReportAnalysis_Handler.ReportAnalysis),
+
+    ])
+
+if __name__ == "__main__":
+
+    # app = apps()
+    # server = HTTPServer(app)
+    # [i.setFormatter(LogFormatter()) for i in logging.getLogger().handlers]
+    # tornado.options.parse_command_line()
+    #
+    # # #== 本地调试
+    # app.listen(23011)
+    # IOLoop.instance().start()
+
+
+    tornado.options.parse_command_line()
+    app = apps()
+    http_server = tornado.httpserver.HTTPServer(app)
+    http_server.bind(options.port)
+    http_server.start()
+    tornado.ioloop.IOLoop.instance().start()

testfile/analysis.py

+# -*- coding:utf-8 -*-
+
+from bs4 import BeautifulSoup,Comment
+from abc import ABCMeta,abstractmethod
+import re
+from lxml import etree
+# path = "/Users/zhangpengcheng/Documents/量化派代码管理/credit_report/html/一代征信报告/432325197803211379.html"
+# path = "/Users/zhangpengcheng/Documents/量化派代码管理/credit_report/html/一代征信报告/32052219780226051X.htm"
+path = "/Users/zhangpengcheng/Documents/量化派代码管理/credit_report/html/一代征信报告/test.html"
+
+htmlfile = open(path, 'rb')
+# print(htmlfile)
+htmlhandle = htmlfile.read()
+soup = BeautifulSoup(htmlhandle,'lxml')
+
+class Base(object):
+    def __init__(self):
+        self.path = "/Users/zhangpengcheng/Documents/量化派代码管理/credit_report/html/一代征信报告/test.html"
+        htmlfile = open(path, 'rb')
+        htmlhandle = htmlfile.read()
+        soup = BeautifulSoup(htmlhandle,'lxml')
+        comments = soup.findAll(text=lambda text: isinstance(text, Comment))
+        [comment.extract() for comment in comments] # 去除注释
+        # [s.extract() for s in soup("style")] # 去除指定标签
+        self.children = self.get_children(soup)
+
+        pass
+
+    def get_children(self,soup):
+        children = []
+        for child in soup.body.children:
+            if child != '\n':
+                children.append(child)
+        return children
+    @abstractmethod
+    def get_json(self,detail,msgJson):
+        return msgJson
+
+class ReportDetail(Base):
+    """报告详情"""
+    def __init__(self):
+        Base.__init__(self)
+        detail = self.children[1].find_all('b')
+        msgJson = {'报告编号':None,'查询请求时间':None,'报告时间':None}
+        self.result = {'报告详情':self.get_json(detail,msgJson)}
+        pass
+
+    def get_json(self,detail,msgJson):
+        for hl in detail:
+            text = hl.get_text()
+            for k,v in msgJson.items():
+                if k in text:
+                    msgJson[k] = text.split(k+':')[1]
+        return msgJson
+
+class SelectDetail(Base):
+    """查询信息"""
+    def __init__(self):
+        Base.__init__(self)
+        detail = self.children[2].table.find_all('tr')
+        msgJson = {}
+        self.result = {'查询信息':self.get_json(detail,msgJson)}
+        pass
+
+
+
+# r = ReportDetail()
+# s = SelectDetail()
+a = AllDatail()
+
+
+
+children = []
+
+for child in soup.body.children:
+    if child != '\n':
+        children.append(child)
+
+d_identity = {}
+table = soup.table
+
+tbody = table.find_all('tbody')
+
+
+tr_arr = table.find_all("tr")
+
+tds = []
+for tr in tr_arr:
+    tds.append(tr.find_all('td'))
+
+# print(tds[0][0].get_text())
+#
+# print(tds[1][0].table['class'])
+#
+# print(tds[1][0].tr.contents[3])
+name = []
+value = []
+tds[0][0].div.get_text()
+for i in range(len(tds[1][0].find_all('tr'))):
+    if tds[1][0].find_all('tr')[i].find(style="WORD-BREAK: break-all") == None:
+        span = tds[1][0].find_all('tr')[i].find_all('div')
+        for s in span:
+            name.append(s.get_text())
+    else:
+        span = tds[1][0].find_all('tr')[i].find_all('div')
+        for s in span:
+            value.append(s.get_text())
+d_identity = dict(zip(name,value))
+
+
+# for child in children:
+#
+#     # print(type(child))
+#     tr_arr = child.find_all("tr")
+#     for tr in tr_arr:
+#         print(tr)
+#
+#     # print(child.find_all(text="性别"))
+#     print('----------')
+# #     print('--------')
+
+
+# from lxml import etree
+# html=etree.HTML(htmlhandle,etree.HTMLParser())
+# print(html.text())
+# print(html.xpath("//b[contains(text(),'一 个人基本信息')]"))
+
+

testfile/test.py

+import urllib3
+import sys
+import bs4
+import chardet
+# reload(sys)
+# sys.setdefaultencoding('utf-8')
+def download(url):
+    htmlfile = open('/Users/zhangpengcheng/Documents/量化派代码管理/credit_report/html/一代征信报告/432325197803211379_1.html','w')
+    try:
+        # result = urllib3.urlopen(url)
+        result = open('/Users/zhangpengcheng/Documents/量化派代码管理/credit_report/html/一代征信报告/432325197803211379.html', 'rb')
+        content = result.read()
+        info = result.info()
+        result.close()
+    except Exception as e:
+        print ('download error!!!')
+        print (e)
+    else:
+        if content != None:
+            charset1 = (chardet.detect(content))['encoding'] #real encoding type
+            charset2 = info.getparam('charset') #declared encoding type
+            print (charset1,' ', charset2)
+            # case1: charset is not None.
+            if charset1 != None and charset2 != None and charset1.lower() != charset2.lower():
+                newcont = bs4.BeautifulSoup(content, from_encoding='GB18030')  #coding: GB18030
+                for cont in newcont:
+                    htmlfile.write('%s\n'%cont)
+            # case2: either charset is None, or charset is the same.
+            else:
+                #print sys.getdefaultencoding()
+                htmlfile.write(content) #default coding: utf-8
+    htmlfile.close()
+if __name__ == "__main__":
+    url = '//www.jb51.net'
+    download(url)
\ No newline at end of file

utils/JsonUtil.py

+# -*- coding:utf-8 -*-
+
+'''
+json 组装
+'''
+import json
+
+
+## 常数变量定义
+## json 返回中的 错误码定义
+class Constants:
+    Code_Success = 200
+    Msg_Success = 'SUCCESS'
+
+    Code_Fail = 1
+    Msg_Fail = '失败'
+
+    Code_Param_Error = -1
+    Msg_Param_Error = '参数错误'
+
+    Code_LoginToken_Auth_Error = -2
+    Msg_LoginToken_Auth_Error = 'token登陆认证错误'
+
+
+    Code_Error = -3
+    Msg_Error = '数据异常'
+
+    Code_Params_Error = -4
+    Msg_Params_Error = '{0} is error,{0} is {1}'
+
+    Code_File_Error = -5
+    Msg_File_Error = '文件读取错误'
+
+    Code_Analysis_Error = -6
+    Msg_Analysis_Error = 'html 解析错误'
+
+    Code_Error_Value = 506
+    Msg_Error_Value = '没找到指定参数的返回值'
+
+
+
+def build_json(code=None,report=None,mssage=None,**kwargs):
+
+    if report == None:
+        report = []
+
+    body = {}
+    for key,value in kwargs.items():
+        body[str(key)] = value
+    try :
+        if body:
+            ddata = {'code':code,'mssage':mssage,'body':body}
+        else:
+            ddata = {'code':code,'mssage':mssage,'result':report}
+    except Exception as e:
+        print(e.args)
+
+    return json.dumps(ddata,ensure_ascii=False)
+
+# businessType=int(_business_type), orderId=_orderId, loanId=_loanId,uuid = _uuid,
+# data=_result, flag=True)
+#
+# def build_json_feature (features=None, code=Constants.Code_Success, msg=Constants.Msg_Success):
+#
+#     if data == None:
+#         data = []
+#     ddata = dict(code=code, businessCode=code, msg=msg, features=features)
+#
+# def build_json_feature_two(data):
+#     return json.dumps(data,ensure_ascii=False)
+#
+# """
+# 只返回 错误码 + 错误提示
+# """
+#
+#
+# def build_json(code, msg):
+#     ddata = dict(code=code, msg=msg, businessCode=code)
+#     # return json.dumps(ddata,encoding='UTF-8',ensure_ascii=False) #python2
+#     return json.dumps(ddata, ensure_ascii=False)  # python3
+#
+#
+# """
+# 错误码 默认为 成功；
+# data may be dict array
+# """
+#
+#
+# def build_json_with_data(data=None, code=Constants.Code_Success, msg=Constants.Msg_Success):
+#     if data == None:
+#         data = []
+#     ddata = dict(code=code, businessCode=code, msg=msg, data=data)
+#     # return json.dumps(ddata,encoding='UTF-8',ensure_ascii=False) # python2
+#     return json.dumps(ddata, ensure_ascii=False)  # python3
+#
+#
+# def build_json_with_data_page(data=None, total=0, page_no=1, page_size=20, code=Constants.Code_Success,
+#                               msg=Constants.Msg_Success):
+#     page_no = int(page_no)
+#     page_size = int(page_size)
+#     total = int(total)
+#     if total % page_size == 0:
+#         _page_total = total / page_size
+#     else:
+#         _page_total = total / page_size + 1
+#     # ==没有查询到数据，则data 为空
+#     if total == 0:
+#         data = []
+#     _data = dict(data=data, page_no=int(page_no), page_size=int(page_size), page_total=int(_page_total),
+#                  total=int(total))
+#     ddata = dict(code=code, businessCode=code, msg=msg, data=_data)
+#     # return json.dumps(ddata,encoding='UTF-8',ensure_ascii=False) # python2
+#     return json.dumps(ddata, ensure_ascii=False)  # python3
+#
+#
+# def build_json_with_block(data=None, code=Constants.Code_Success, msg=Constants.Msg_Success):
+#     ddata = dict(code=code, businessCode=code, msg=msg, data=data)
+#     # return json.dumps(ddata,encoding='UTF-8',ensure_ascii=False) # python2
+#     return json.dumps(ddata, ensure_ascii=False)  # python3
+#
+#
+# """
+# 转换为 dict 对象; 直接 json.key 获取数据
+# 如果data 为空，返回一个空的data数据
+# """
+#
+#
+# def str_parse_json(data):
+#     if any(data):
+#         return json.loads(data, encoding='UTF-8')
+#     return dict()