black_list_new表数据往tmp_black_grey_list清洗

src/main/java/cn/quantgroup/qgblservice/controller/BlackListToolsManagerController.java

@@ -2,6 +2,7 @@ package cn.quantgroup.qgblservice.controller;
 
 import cn.quantgroup.qgblservice.response.GlobalResponse;
 import cn.quantgroup.qgblservice.service.IBlackListToolsManagerService;
+import org.apache.commons.lang.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
@@ -49,4 +50,18 @@ public class BlackListToolsManagerController {
         return iBlackListToolsManagerService.buildMd5Tools(privateKey,operatType,idArray,gtCreatUtc);
     }
 
+    @RequestMapping("/cleanTableData")
+    public GlobalResponse cleanTableData(String privateKey,String operatType){
+        if(StringUtils.isNotEmpty(privateKey) && "aHVhdDI0V0JvT3hwTkJwQmk1d2Q4dz09".equals(privateKey)){
+            if(StringUtils.isEmpty(operatType)){
+                return GlobalResponse.error("cleanTableData参数为空！");
+            }
+
+            iBlackListToolsManagerService.cleanTableData(privateKey,operatType);
+            return GlobalResponse.success("copyBlackList任务调度成功");
+        }else{
+            return GlobalResponse.error("cleanTableData私钥为空或不正确，请联系管理员！");
+        }
+    }
+
 }

src/main/java/cn/quantgroup/qgblservice/repository/mybatis/entity/tidb/TmpBlackGreyList.java

+package cn.quantgroup.qgblservice.repository.mybatis.entity.tidb;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.Data;
+
+import java.io.Serializable;
+import java.sql.Timestamp;
+
+/**
+ * -----------------------------------------------------------------------------<br>
+ * 描述: tmp_black_grey_list表对应实体<br>
+ * 作者：yanhui.Hao <br>
+ * 时间：2020.04.02 <br>
+ * 授权: (C) Copyright (c) 2017 <br>
+ * 公司: 北京众信利民信息技术有限公司 <br>
+ * -----------------------------------------------------------------------------
+ */
+@Data
+public class TmpBlackGreyList implements Serializable {
+
+    private static final long serialVersionUID = 3504867579864272343L;
+
+    private Long id;
+    private String uuid;
+    private String name;
+    private String phoneNo;
+    private String idNo;
+
+    /**phoneNo md5加密*/
+    private String phoneNoMd5;
+    /**idNo md5加密*/
+    private String idNoMd5;
+
+    private String blackType;
+    private String type;
+    private String joinBlackReason;
+
+    private String maxOverdueDays;
+    private String totalOverdueDays;
+    private String remark;
+
+    @JsonIgnore
+    private Timestamp createdAt;
+    @JsonIgnore
+    private Timestamp updatedAt;
+
+
+}

src/main/java/cn/quantgroup/qgblservice/repository/mybatis/entity/tidb/TmpBlackGreyListRowMapper.java

+package cn.quantgroup.qgblservice.repository.mybatis.entity.tidb;
+
+import org.springframework.jdbc.core.RowMapper;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+/**
+ * -----------------------------------------------------------------------------<br>
+ * 描述: tmp_black_grey_list表对应实体<br>
+ * 作者：yanhui.Hao <br>
+ * 时间：2020.04.02 <br>
+ * 授权: (C) Copyright (c) 2017 <br>
+ * 公司: 北京众信利民信息技术有限公司 <br>
+ * -----------------------------------------------------------------------------
+ */
+
+public class TmpBlackGreyListRowMapper implements RowMapper<TmpBlackGreyList> {
+
+    /**
+     * Implementations must implement this method to map each row of data
+     * in the ResultSet. This method should not call {@code next()} on
+     * the ResultSet; it is only supposed to map values of the current row.
+     *
+     * @param rs     the ResultSet to map (pre-initialized for the current row)
+     * @param rowNum the number of the current row
+     * @return the result object for the current row
+     * @throws SQLException if a SQLException is encountered getting
+     *                      column values (that is, there's no need to catch SQLException)
+     */
+    @Override
+    public TmpBlackGreyList mapRow(ResultSet rs, int rowNum) throws SQLException {
+        TmpBlackGreyList bean = new TmpBlackGreyList();
+        bean.setId(rs.getLong("id"));
+        bean.setUuid(rs.getString("uuid"));
+        bean.setName(rs.getString("name"));
+        bean.setPhoneNo(rs.getString("phone_no"));
+        bean.setIdNo(rs.getString("id_no"));
+        bean.setBlackType(rs.getString("black_type"));
+        bean.setType(rs.getString("type"));
+        bean.setJoinBlackReason(rs.getString("join_black_reason"));
+        bean.setMaxOverdueDays(rs.getString("max_overdue_days"));
+        bean.setTotalOverdueDays(rs.getString("total_overdue_days"));
+        bean.setCreatedAt(rs.getTimestamp("created_at"));
+        bean.setUpdatedAt(rs.getTimestamp("updated_at"));
+        return bean;
+    }
+}

src/main/java/cn/quantgroup/qgblservice/service/IBlackListToolsManagerService.java

@@ -35,4 +35,15 @@ public interface IBlackListToolsManagerService {
 
 
 
+    /**
+     * -----------------------------------------------------------------------------<br>
+     * 描述: black_list_new 数据，往tmp_black_grey_list清洗<br>
+     * 作者：yanhui.Hao <br>
+     * 时间：2020.04.02 <br>
+     * 授权: (C) Copyright (c) 2017 <br>
+     * 公司: 北京众信利民信息技术有限公司 <br>
+     * -----------------------------------------------------------------------------
+     */
+    public void cleanTableData(String privateKey, String operatType);
+
 }

src/main/java/cn/quantgroup/qgblservice/service/impl/BlackListToolsManagerServiceImpl.java

@@ -3,6 +3,8 @@ package cn.quantgroup.qgblservice.service.impl;
 import cn.quantgroup.qgblservice.constant.Constant;
 import cn.quantgroup.qgblservice.repository.mybatis.entity.blacklist.BlackListChannelExpireConfigVo0;
 import cn.quantgroup.qgblservice.repository.mybatis.entity.tidb.BlackListQueryTidbVo0;
+import cn.quantgroup.qgblservice.repository.mybatis.entity.tidb.TmpBlackGreyList;
+import cn.quantgroup.qgblservice.repository.mybatis.entity.tidb.TmpBlackGreyListRowMapper;
 import cn.quantgroup.qgblservice.response.GlobalResponse;
 import cn.quantgroup.qgblservice.service.IBlackListToolsManagerService;
 import cn.quantgroup.qgblservice.utils.MD5Util;
@@ -17,6 +19,7 @@ import org.apache.commons.lang.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.jdbc.core.BeanPropertyRowMapper;
 import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Service;
 
 import javax.annotation.PostConstruct;
@@ -30,6 +33,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 
@@ -49,6 +54,9 @@ public class BlackListToolsManagerServiceImpl implements IBlackListToolsManagerS
     private final int LIMIT = 10000;
     private final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
 
+    private final int LIMIT_1000 = 2000;
+    private static Pattern p_chinese = Pattern.compile("[\u4e00-\u9fa5]");//中文
+
     @PostConstruct
     public void initChannelBlackListExpireConfig() {
         List<BlackListChannelExpireConfigVo0> queryBlackListChannelExpireConfigVo0List = blackListJdbcTemplate.query(Constant.SQL.BLACK_LIST_NEW_QUERY_CHANNEL_BLACK_LIST_EXPIRE_CONFIG_SQL, new BeanPropertyRowMapper<>(BlackListChannelExpireConfigVo0.class));
@@ -426,4 +434,226 @@ public class BlackListToolsManagerServiceImpl implements IBlackListToolsManagerS
     }
 
 
+
+
+    /**
+     * -----------------------------------------------------------------------------<br>
+     * 描述: black_list_new 数据，往tmp_black_grey_list清洗<br>
+     * 作者：yanhui.Hao <br>
+     * 时间：2020.04.02 <br>
+     * 授权: (C) Copyright (c) 2017 <br>
+     * 公司: 北京众信利民信息技术有限公司 <br>
+     * -----------------------------------------------------------------------------
+     */
+    @Async
+    @Override
+    public void cleanTableData(String privateKey, String operatType) {
+        if(StringUtils.isEmpty(operatType)){
+            log.error("cleanTableData参数为空！");
+            return;
+        }
+
+        if("copyBlackList".equals(operatType)){
+            copyBlackListNew();
+        }
+        else{
+            log.error("cleanTableData未知的操作类型！");
+            return;
+        }
+    }
+
+    private static boolean isContainChinese(String str) {
+        Matcher m = p_chinese.matcher(str);
+        if (m.find()) {
+            return true;
+        }
+        return false;
+    }
+
+    private static boolean isIDNumber(String IDNumber) {
+        if (IDNumber == null || "".equals(IDNumber)) {
+            return false;
+        }
+       /* String s = "61011319780908211x";
+        String regex = "\\d{15}|\\d{17}[\\dxX]";
+        if(s.matches(regex)) {
+            System.out.println("格式正确");
+        }else {
+            System.out.println("格式错误");
+        }*/
+
+        // 定义判别用户身份证号的正则表达式（15位或者18位，最后一位可以为字母）
+        String regularExpression = "(^[1-9]\\d{5}(18|19|20)\\d{2}((0[1-9])|(10|11|12))(([0-2][1-9])|10|20|30|31)\\d{3}[0-9Xx]$)|" +
+                "(^[1-9]\\d{5}\\d{2}((0[1-9])|(10|11|12))(([0-2][1-9])|10|20|30|31)\\d{3}$)";
+        //假设18位身份证号码:41000119910101123X  410001 19910101 123X
+        //^开头
+        //[1-9] 第一位1-9中的一个      4
+        //\\d{5} 五位数字           10001（前六位省市县地区）
+        //(18|19|20)                19（现阶段可能取值范围18xx-20xx年）
+        //\\d{2}                    91（年份）
+        //((0[1-9])|(10|11|12))     01（月份）
+        //(([0-2][1-9])|10|20|30|31)01（日期）
+        //\\d{3} 三位数字            123（第十七位奇数代表男，偶数代表女）
+        //[0-9Xx] 0123456789Xx其中的一个 X（第十八位为校验值）
+        //$结尾
+
+        //假设15位身份证号码:410001910101123  410001 910101 123
+        //^开头
+        //[1-9] 第一位1-9中的一个      4
+        //\\d{5} 五位数字           10001（前六位省市县地区）
+        //\\d{2}                    91（年份）
+        //((0[1-9])|(10|11|12))     01（月份）
+        //(([0-2][1-9])|10|20|30|31)01（日期）
+        //\\d{3} 三位数字            123（第十五位奇数代表男，偶数代表女），15位身份证不含X
+        //$结尾
+
+        boolean matches = IDNumber.matches(regularExpression);
+        //判断第18位校验值
+        if (matches) {
+
+            if (IDNumber.length() == 18) {
+                try {
+                    char[] charArray = IDNumber.toCharArray();
+                    //前十七位加权因子
+                    int[] idCardWi = {7, 9, 10, 5, 8, 4, 2, 1, 6, 3, 7, 9, 10, 5, 8, 4, 2};
+                    //这是除以11后，可能产生的11位余数对应的验证码
+                    String[] idCardY = {"1", "0", "X", "9", "8", "7", "6", "5", "4", "3", "2"};
+                    int sum = 0;
+                    for (int i = 0; i < idCardWi.length; i++) {
+                        int current = Integer.parseInt(String.valueOf(charArray[i]));
+                        int count = current * idCardWi[i];
+                        sum += count;
+                    }
+                    char idCardLast = charArray[17];
+                    int idCardMod = sum % 11;
+                    if (idCardY[idCardMod].toUpperCase().equals(String.valueOf(idCardLast).toUpperCase())) {
+                        return true;
+                    } else {
+                        return false;
+                    }
+
+                } catch (Exception e) {
+                    e.printStackTrace();
+                    return false;
+                }
+            }
+
+        }
+        return matches;
+    }
+
+    private void copyBlackListNew(){
+        long star = System.currentTimeMillis();
+        final String SQL_MAXID = "SELECT max(id) as 'maxId' FROM black_list_new;";
+        String SQL_QUERY = "SELECT b.id, b.uuid, b.name, b.phone_no, b.id_no, " +
+                "if(type=11,'1','2') black_type, b.type, if(b.type=11,'4','5') join_black_reason, " +
+                " b.max_overdue_days, b.total_overdue_days, b.created_at, b.updated_at" +
+                " FROM black_list_new b WHERE id > ? and id<=?;";
+
+        int maxId = 0;
+        int totalCount = 0, runOkCount = 0;
+        Map<String, Object> maxIdMap = blackListJdbcTemplate.queryForMap(SQL_MAXID);
+        if(maxIdMap!=null && maxIdMap.get("maxId")!=null){
+            maxId = Integer.parseInt(String.valueOf(maxIdMap.get("maxId")));
+            log.info("dealWith all query  maxId: {} ",maxId);
+
+            int totalPage = maxId/LIMIT_1000;
+            if((maxId%LIMIT_1000)!=0){
+                totalPage+=1;
+            }
+
+            int startId = 0,endId = 0;
+            for (int page = 1; page <= totalPage; page++) {
+                startId = (page-1) * LIMIT_1000;
+                endId = page * LIMIT_1000;
+
+                int batchExecuteResult = 0;
+                long startUtc = System.currentTimeMillis();
+                List<TmpBlackGreyList> updateBeanList = null;
+
+                try{
+                    //String query_sql = String.format(SQL_QUERLISTBYID,startId,endId);
+
+                    log.info("dealWith all Start page: {} , startId: {} , endId: {} ------>>>>>>", page, startId, endId);
+
+                    List<TmpBlackGreyList> tmpQueryList = blackListJdbcTemplate.query(SQL_QUERY, new Object[]{startId, endId}, new TmpBlackGreyListRowMapper());
+
+                    if(tmpQueryList!=null && tmpQueryList.size()>0){
+                        totalCount = totalCount + tmpQueryList.size();
+                        updateBeanList = new ArrayList<>();
+                        for(TmpBlackGreyList queryBean : tmpQueryList){
+
+                            if(queryBean.getUuid()!=null){
+                                if(queryBean.getUuid().equals("null")){
+                                    queryBean.setUuid(null);
+                                }
+                            }
+                            if(queryBean.getName()!=null){
+                                if(queryBean.getName().equals("null")){
+                                    queryBean.setName(null);
+                                }
+
+                            }
+                            if(queryBean.getPhoneNo()!=null){
+                                if(queryBean.getPhoneNo().equals("null")){
+                                    queryBean.setPhoneNo(null);
+                                }
+                                //手机号等于姓名的
+                                else if(queryBean.getName()!=null && queryBean.getPhoneNo().equals(queryBean.getName()) && isContainChinese(queryBean.getPhoneNo())){
+                                    queryBean.setPhoneNo(null);
+                                }
+                            }
+                            if(queryBean.getIdNo()!=null){
+                                if(queryBean.getIdNo().equals("null") || queryBean.getIdNo().equals("1")){
+                                    queryBean.setIdNo(null);
+                                }
+                                //`440582199711272032
+                                else if(queryBean.getIdNo().length()==19 && queryBean.getIdNo().startsWith("`")){
+                                    queryBean.setIdNo(queryBean.getIdNo().substring(1, queryBean.getIdNo().length()));
+                                }
+                                //3.5058219811212e+17
+                                else if (queryBean.getIdNo().length()!=15 && queryBean.getIdNo().length()!=18
+                                        && queryBean.getIdNo().contains(".") && queryBean.getIdNo().contains("+")){
+                                    queryBean.setIdNo(null);
+                                }
+                            }
+
+                            //姓名和身份证反了
+                            if(queryBean.getName()!=null && queryBean.getIdNo()!=null && (queryBean.getName().length()==15 || queryBean.getName().length()==18)){
+                                String idNo = queryBean.getIdNo();
+                                String  name = queryBean.getName();
+                                if(isContainChinese(idNo) && isIDNumber(name)){
+                                    queryBean.setName(idNo);
+                                    queryBean.setIdNo(name);
+                                }
+                            }
+
+                            if(queryBean.getIdNo()!=null){
+                                queryBean.setIdNoMd5(MD5Util.getMD5Digest(queryBean.getIdNo().toUpperCase()));
+                            }
+                            if(queryBean.getPhoneNo()!=null){
+                                queryBean.setPhoneNoMd5(MD5Util.getMD5Digest(queryBean.getPhoneNo()));
+                            }
+
+                            updateBeanList.add(queryBean);
+                        }
+
+                        if(updateBeanList.size()>0){
+                            batchExecuteResult = JdbcExecuters.insertBatchExecute(updateBeanList, blackListJdbcTemplate);
+                            runOkCount = runOkCount+batchExecuteResult;
+                        }
+                    }
+                }catch (Exception e){
+                    e.printStackTrace();
+                    log.error("方法copyBlackListNew()处理异常, page: {} , startId: {} , endId: {} ", page, startId, endId);
+                }finally {
+                    log.info("方法copyBlackListNew()处理End page: {} , startId: {} , endId: {} , list.size: {} , batchExecuteResult: {} <<<<<<------\n",page,startId,endId,(updateBeanList!=null ? updateBeanList.size(): 0),batchExecuteResult,((System.currentTimeMillis()-startUtc)+".ms"));
+                }
+            }
+        }
+        log.info("\n>>>>>>方法copyBlackListNew()处理 Method End, totalCount: {} , runOkCount: {} , total cost: {} <<<<<<",totalCount,runOkCount,(System.currentTimeMillis()-star)+".ms");
+        //return "dealWith all OK, totalCount="+totalCount+",runOkCount="+runOkCount;
+    }
+
+
 }

src/main/java/cn/quantgroup/qgblservice/utils/jdbc/JdbcExecuters.java

 package cn.quantgroup.qgblservice.utils.jdbc;
 
 import cn.quantgroup.qgblservice.repository.mybatis.entity.tidb.BlackListQueryTidbVo0;
+import cn.quantgroup.qgblservice.repository.mybatis.entity.tidb.TmpBlackGreyList;
 import cn.quantgroup.qgblservice.utils.MD5Util;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
@@ -161,6 +162,54 @@ public class JdbcExecuters {
         return atomicInteger.get();
     }
 
+    public static int insertBatchExecute(List<TmpBlackGreyList> tmpBlackGreyList, JdbcTemplate jdbcTemplate) {
+        String sql = "insert ignore into tmp_black_grey_list (`id`, `uuid`, `name`, `phone_no`, `id_no`, `phone_no_md5`, `id_no_md5`, `black_type`, `type`, `join_black_reason`, `max_overdue_days`, `total_overdue_days`, `created_at`, `updated_at`) " +
+                " values(?,?,?,?,?,?,?,?,?,?,?,?,?,?) ";
+
+        Connection conn = null;
+        PreparedStatement ps = null;
+        AtomicInteger atomicInteger = new AtomicInteger();
+        try {
+            conn = jdbcTemplate.getDataSource().getConnection();
+            ps = conn.prepareStatement(sql);
+            conn.setAutoCommit(false);
+            for (int i = 0; i < tmpBlackGreyList.size(); i++) {
+
+                TmpBlackGreyList vo = tmpBlackGreyList.get(i);
+                ps.setLong(1, vo.getId());
+                ps.setString(2, vo.getUuid());
+                ps.setString(3, vo.getName());
+                ps.setString(4, vo.getPhoneNo());
+                ps.setString(5, vo.getIdNo());
+                ps.setString(6, vo.getPhoneNoMd5());
+                ps.setString(7, vo.getIdNoMd5());
+                ps.setString(8, vo.getBlackType());
+                ps.setString(9, vo.getType());
+                ps.setString(10, vo.getJoinBlackReason());
+                ps.setString(11, vo.getMaxOverdueDays());
+                ps.setString(12, vo.getTotalOverdueDays());
+                ps.setTimestamp(13, vo.getCreatedAt());
+                ps.setTimestamp(14, vo.getUpdatedAt());
+
+                ps.addBatch();
+                atomicInteger.getAndIncrement();
+
+                if (i > 0 && i % 1000 == 0) {
+                    ps.executeBatch();
+                    conn.commit();
+                    ps.clearBatch();
+                }
+            }
+            ps.executeBatch();
+            conn.commit();
+        } catch (Exception e) {
+            log.error("tmp_black_grey_list表批量插入数据异常", e);
+        } finally {
+            close(conn, ps, null);
+        }
+        return atomicInteger.get();
+    }
+
 
     /**
      * 关闭资源