Skip to content

Q
qg-bl-service
Commit d6a05dc7 authored by 郝彦辉's avatar 郝彦辉
Browse files
Options

黑灰名单2020.04.22

parent e239fccf
Show whitespace changes
Inline Side-by-side
Showing with 402 additions and 133 deletions
src/main/java/cn/quantgroup/qgblservice/constant/ConstantBlackGrey.java
View file @ d6a05dc7
package cn.quantgroup.qgblservice.constant;
import java.io.File;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
/**
* -----------------------------------------------------------------------------<br>
* 类描述: <br>
......@@ -47,6 +51,10 @@ public class ConstantBlackGrey {
" where br.black_type='1' " +
" ORDER BY br.r_id LIMIT ?, ? " +
") AS page USING(r_id); ";
public static String BL_QUERY_BLACKLIST_RESULT_MAX_ID = " SELECT max(id) as 'maxId' FROM black_grey_list_result r WHERE r.black_type='1';";
public static String BL_QUERY_BLACKLIST_RESULT_BY_LIMIT_ID = " SELECT ##r.*## " +
" FROM black_grey_list_result r " +
" WHERE r.black_type='1' AND id >? AND id<=?; ";
//现金分期在逾转已还清
......@@ -110,56 +118,20 @@ public class ConstantBlackGrey {
" and (plan_status <>'Finish' or (plan_status ='Finish' and paid_off_date>repay_date)) " +
" and a.user_id=?; ";
public static String BLACK_LIST_NEW_QUERY_THIRD_PART_BLACK_LIST_CONFIG_SQL = "select * from third_part_black_list_config where status = true;";
public static String BLACK_LIST_NEW_QUERY_CHANNEL_BLACK_LIST_EXPIRE_CONFIG_SQL = "select * from black_list_channel_expire_config where status = true";
public static String BLACK_LIST_NEW_BATCH_INSERT_BLACK_THREE_ELE_LIST_SQL = "insert ignore into black_list_new (`uuid`, `name`, `phone_no`, `id_no`, `major_type`, `type`, `total_overdue_days`, `max_overdue_days`, `black_level`, `join_black_reason`, `created_at`, `updated_at`, `phone_no_md5`, `id_no_md5`) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
public static String BLACK_LIST_NEW_QUERY_BY_UNIQUE_KEY_SQL = "select * from black_list_new where phone_no = '%s' and name = '%s' and id_no = '%s' and type = '%s';";
public static String BLACK_LIST_NEW_QUERY_BY_UNIQUE_KEY_SQL2_MD5 = "select * from black_list_new where phone_no = '%s' and name = '%s' and id_no = '%s' and type = '%s' and phone_no_md5 = '%s' and id_no_md5 = '%s';";
public static String BLACK_LIST_NEW_UPDATE_DISABLE_BY_RECORD_ID_SQL = "update black_list_new set status = false, updated_at = '%s' where id = %s;";
public static String BLACK_LIST_NEW_UPDATE_THREE_ELE_AND_BLACK_LEVEL_SQL = "update black_list_new set uuid = '%s', name = '%s', phone_no = '%s' , id_no = '%s', black_level = '%s', phone_no_md5 = '%s', id_no_md5 = '%s' where id = %s;";
public static String BLACK_LIST_NEW_UPDATE_BLACK_LIST_LEVEL_SQL = "update black_list_new set black_level = '%s' where id = %s;";
public static String BLACK_LIST_NEW_INSERT_THIRD_PART_BLACK_LIST_SQL = "insert into `black_list_new` (`uuid`, `name`, `phone_no`, `id_no`, `major_type`, `type`, `black_level`, `join_black_reason`, `created_at`, `updated_at`,`phone_no_md5`,`id_no_md5`) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ? ,? ,?);";
public static String BLACK_LIST_NEW_UPDATE_FIRST_OR_MAX_OVERDUE_DAYS_SQL = "update black_list_new set total_overdue_days = ? , max_overdue_days = ?, updated_at = ? where id = ?";
public static String BLACK_LIST_NEW_QUERY_BY_PHONENO_SQL = "select * from black_list_new where phone_no = '%s' and status = true;";
public static String BLACK_LIST_NEW_QUERY_BY_IDNO_SQL = "select * from black_list_new where id_no = '%s' and status = true;";
public static String BLACK_LIST_NEW_QUERY_BY_PHONENO_IDNO_SQL = "select * from black_list_new where phone_no = '%s' and id_no = '%s' and status = true;";
public static String BLACK_LIST_NEW_QUERY_BY_THREE_ELE_SQL = "select * from black_list_new where phone_no = '%s' and id_no = '%s' and name = '%s' and status = true;";
public static String BLACK_LIST_NEW_QUERY_BY_IDNO_OR_PHONENO_SQL = "select * from black_list_new where phone_no = '%s' or id_no = '%s'";
public static String BLACK_LIST_NEW_QUERY_BY_UUID_SQL = "select * from black_list_new where uuid = '%s' and status = true;";
public static String BLACK_LIST_NEW_QUERY_BY_PHONENOMD5_SQL = "select * from black_list_new where phone_no_md5 = '%s' and status = true;";
public static String BLACK_LIST_NEW_QUERY_BY_IDNOMD5_SQL = "select * from black_list_new where id_no_md5 = '%s' and status = true;";
public static String BLACK_LIST_NEW_QUERY_BY_PHONENOMD5_AND_IDNOMD5_SQL = "select * from black_list_new where phone_no_md5 = '%s' and id_no_md5 = '%s' and status = true;";
public static String BLACK_LIST_NEW_QUERY_BY_PHONENOMD5_OR_IDNOMD5_SQL = "select * from black_list_new where phone_no_md5 = '%s' or id_no_md5 = '%s';";
// public static String XYQB_QUERY_USER_FIRST_TERM_OVERDUE_SQL = "select if(repayment_status=3,DATEDIFF(repaid_at,deadline)+1,DATEDIFF(CURDATE(),deadline)+1) total_overdue_days from repayment_plan where user_id in (%s) and deadline<=CURDATE() and (repayment_status not in (3,4) or (repayment_status=3 and repaid_at>=deadline)) order by deadline desc limit 1;";
public static String XYQB_QUERY_USER_TOTAL_OVERDUE_SQL = "select sum(if(repayment_status=3,DATEDIFF(repaid_at,deadline)+1,DATEDIFF(CURDATE(),deadline)+1)) total_overdue_days from repayment_plan where user_id = '%s' and deadline<=CURDATE() and (repayment_status not in (3,4) or (repayment_status=3 and repaid_at>=deadline));";
public static String XYQB_QUERY_USER_MAX_OVERDUE_SQL = "select max(if(repayment_status=3,DATEDIFF(repaid_at,deadline)+1,DATEDIFF(CURDATE(),deadline)+1)) max_overdue_days from repayment_plan where user_id in (%s) and deadline<=CURDATE() and (repayment_status not in (3,4) or (repayment_status=3 and repaid_at>=deadline));";
public static String XYQB_USER_QUERY_USER_DETAIL_INFO_SQL = "select c.id, c.uuid uuid, b.name, c.phone_no phoneNo, b.id_no idNo from user_detail b left join user c on b.user_id=c.id where b.user_id='%s';";
public static String XYQB_USER_QUERY_USER_DETAIL_BY_PHONE_NO_SQL = "select user_id from user a left join user_detail b on a.id=b.user_id where b.phone_no='%s';";
public static String XYQB_USER_QUERY_USER_DETAIL_BY_ID_NO_SQL = "select user_id from user a left join user_detail b on a.id=b.user_id where b.id_no='%s';";
public static String XYQB_USER_QUERY_USER_DETAIL_BY_UUID_SQL = "select user_id from user a left join user_detail b on a.id=b.user_id where a.uuid = '%s'";
public static String BLACK_LIST_NEW_QUERY_BY_IDNO_SQL2 = "select * from black_list_new where id_no = '%s' and created_at < '%s' and status = true;";
public static String BLACK_LIST_NEW_QUERY_BY_PHONENO_SQL2 = "select * from black_list_new where phone_no = '%s' and created_at < '%s' and status = true;";
public static String BLACK_LIST_NEW_UPDATE_PHONENOMD5_AND_IDNOMD5_SQL = "update black_list_new set `phone_no_md5`=? ,`id_no_md5`=? where `id`=? ";
}
public static class PARAM {
public static final String[] BLACK_LIST_TYPES = {"1","2","10"};
public static final String SEPA = File.separator; //System.getProperty("file.separator");
public static final DateFormat DF_YMD = new SimpleDateFormat("yyyyMMdd");
public static final String SAVELOGPATH = "/home/quant_group/qg-bl-service/logs/";
/**当前服务器CPU个数*/
public static final int SYS_CPU_COUNT = Runtime.getRuntime().availableProcessors();
}
public static class DAYE_FORMAT{
......
src/main/java/cn/quantgroup/qgblservice/repository/mybatis/xml/blacklist/BlackGreyListMapper.xml
View file @ d6a05dc7
......@@ -122,7 +122,8 @@
</select>
<select id="findBlackGreyListDetails" parameterType="cn.quantgroup.qgblservice.repository.mybatis.entity.blacklist.BlackGreyListQueryVo" resultMap="BlackGreyListDetailsMap">
select * from black_grey_list_details
select id, r_id, name, phone_no, id_no, type, reason_code, max_overdue_days, total_overdue_days, status, remark, created_at, updated_at
from black_grey_list_details
<trim prefix="where" prefixOverrides="and">
<if test="rId != null">
and r_id = #{rId,jdbcType=VARCHAR}
......@@ -189,7 +190,6 @@
<select id="findBlackGreyListRjoinD" parameterType="cn.quantgroup.qgblservice.repository.mybatis.entity.blacklist.BlackGreyListRjoinDVo" resultMap="BlackGreyListRjoinDVoMap">
select r.r_id , r.uuid, r.`name`, r.phone_no, r.id_no, r.black_type, r.types,
d.id, d.type, d.reason_code, d.max_overdue_days, d.total_overdue_days, d.`status`, d.remark, d.created_at, d.updated_at
from black_grey_list_result r
......@@ -260,9 +260,8 @@
<if test="reasons!=null"> reasons = #{reasons,jdbcType=VARCHAR}, </if>
<if test="updatedAt!=null"> updated_at = #{updatedAt,jdbcType=VARCHAR} </if>
where
r_id = #{rId,jdbcType=VARCHAR};
</set>
where r_id = #{rId,jdbcType=VARCHAR};
</update>
<update id="updateDetailsStatusByparams" parameterType="cn.quantgroup.qgblservice.repository.mybatis.entity.blacklist.DetailsUpdate">
......
src/main/java/cn/quantgroup/qgblservice/service/impl/BlackGreyListServiceImpl.java
View file @ d6a05dc7
......@@ -3,24 +3,25 @@ package cn.quantgroup.qgblservice.service.impl;
import cn.quantgroup.qgblservice.constant.ConstantBlackGrey;
import cn.quantgroup.qgblservice.repository.mybatis.entity.blacklist.*;
import cn.quantgroup.qgblservice.repository.mybatis.entity.tidb.TmpBlackGreyList;
import cn.quantgroup.qgblservice.repository.mybatis.entity.tidb.TmpBlackGreyListRowMapper;
import cn.quantgroup.qgblservice.repository.mybatis.mapper.blacklist.BlackGreyListMapper;
import cn.quantgroup.qgblservice.service.IBlackGreyListService;
import cn.quantgroup.qgblservice.utils.MD5Util;
import cn.quantgroup.qgblservice.utils.blacklist.BlackListUtils;
import cn.quantgroup.qgblservice.utils.jdbc.JdbcExecuters;
import com.alibaba.fastjson.JSON;
import com.google.common.base.Stopwatch;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.dao.EmptyResultDataAccessException;
import org.springframework.jdbc.core.BeanPropertyRowMapper;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Service;
import java.sql.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
......@@ -50,6 +51,8 @@ public class BlackGreyListServiceImpl implements IBlackGreyListService {
@Autowired
private UpdateBlackListOverdueDayParallel updateBlackListOverdueDayParallel;
private static int pageSize = 10000;
//private static Map<String, Integer> channelBlackListExpireConfigMap = new ConcurrentHashMap<>();
/*@PostConstruct
public void initChannelBlackListExpireConfig() {
......@@ -676,18 +679,7 @@ public class BlackGreyListServiceImpl implements IBlackGreyListService {
return String.format(resMsgTemp, vccBlackGreyList.size(), details_haveCount.get(), newIncreasedList.size(), saveOkCount);
}
/**
* -----------------------------------------------------------------------------<br>
* 描 述: 逾期已还清黑名单>>转灰名单 <br>
* 创建人: yanhui.Hao <br>
* 创建时间: 2020.04.16 21:12 <br>
* 最后修改人: <br>
* 最后修改时间: 2020.04.16 21:12 <br>
* 入参说明: <br>
* 出参说明: <br>
* -----------------------------------------------------------------------------
*/
@Override
/*@Override
public String removeBlackToGreyList() {
long startTime = System.currentTimeMillis();
List<BlackGreyListResult> blackListResults = new ArrayList<>();
......@@ -734,20 +726,96 @@ public class BlackGreyListServiceImpl implements IBlackGreyListService {
return blackToGreyListParallel.runMain(blackListResults);
}
return null;
}
}*/
/**
* -----------------------------------------------------------------------------<br>
* 描 述: 更新黑名单>>逾期天数 <br>
* 描 述: 逾期已还清黑名单>>转灰名单 <br>
* 创建人: yanhui.Hao <br>
* 创建时间: 2020.04.16 21:13 <br>
* 创建时间: 2020.04.16 21:12 <br>
* 最后修改人: <br>
* 最后修改时间: 2020.04.16 21:13 <br>
* 最后修改时间: 2020.04.16 21:12 <br>
* 入参说明: <br>
* 出参说明: <br>
* -----------------------------------------------------------------------------
*/
@Override
public String removeBlackToGreyList() {
long startTime = System.currentTimeMillis();
Integer maxId = 0;
try {
maxId = xyqbJdbcTemplate.queryForObject(ConstantBlackGrey.SQL.BL_QUERY_BLACKLIST_RESULT_MAX_ID, Integer.class);
}catch (EmptyResultDataAccessException e){
log.error("查询结果表-黑名单最大Id异常, sql: {} ", ConstantBlackGrey.SQL.BL_QUERY_BLACKLIST_RESULT_MAX_ID, e);
}
if(maxId==null || maxId.intValue() <=0){
return "查询结果表-黑名单最大Id异常";
}
int totalPage = maxId / pageSize;
if((maxId % pageSize)!=0){
totalPage+=1;
}
String sql_query_tmp = ConstantBlackGrey.SQL.BL_QUERY_BLACKLIST_RESULT_BY_LIMIT_ID;
sql_query_tmp = sql_query_tmp.replace("##r.*##", " r.r_id, r.uuid, r.black_type, r.types, r.reasons ");
List<BlackGreyListResult> blackListResults = new ArrayList<>();
int count_black = 0, count_type_11=0;
int startId = 0,endId = 0;
for (int page = 1; page <= totalPage; page++) {
long thisPageStart = System.currentTimeMillis();
startId = (page-1) * pageSize;
endId = startId + pageSize;
List<BlackGreyListResult> tmpQueryList = null;
try{
tmpQueryList = blackListJdbcTemplate.query(sql_query_tmp, new Object[]{startId, endId}, new RowMapper<BlackGreyListResult>() {
@Override
public BlackGreyListResult mapRow(ResultSet rs, int rowNum) throws SQLException {
BlackGreyListResult bean = new BlackGreyListResult();
bean.setRId(rs.getString("r_id"));
bean.setUuid(rs.getString("uuid"));
bean.setBlackType(rs.getString("black_type"));
bean.setTypes(rs.getString("types"));
bean.setReasons(rs.getString("reasons"));
return bean;
}
});
if(tmpQueryList!=null && tmpQueryList.size() >0){
count_black+=tmpQueryList.size();
//blackListResults.addAll(tmpQueryList);
for (BlackGreyListResult result: tmpQueryList){
String types = ","+result.getTypes().trim()+",";
if(!types.contains(",11,")){
blackListResults.add(result);
}else {
count_type_11++;
}
}
blackListResults.addAll(tmpQueryList);
}
}catch (Exception e){
e.printStackTrace();
log.error("查询结果表-黑名单列表异常, totalPage: {} , page: {} , startId: {} , endId: {} , 耗时: {} ", totalPage, page, startId, endId, (System.currentTimeMillis()-thisPageStart), e);
}finally {
log.error("查询结果表-黑名单列表结束, totalPage: {} , page: {} , startId: {} , endId: {} , 大小: {} , 耗时: {} ", totalPage, page, startId, endId, tmpQueryList!=null?tmpQueryList.size():"null", (System.currentTimeMillis()-thisPageStart));
}
}
log.info("黑转灰-查询黑灰名单结果表结束, 查询大小: {} , type11大小: {} , list大小: {} , 总查询耗时: {} ", count_black, count_type_11, blackListResults!=null?blackListResults.size():"null", (System.currentTimeMillis()-startTime));
//批量 逾期已还清黑名单>>转灰名单
if(blackListResults!=null && blackListResults.size() >0 ){
return blackToGreyListParallel.runMain(blackListResults);
}
return null;
}
/*@Override
public String updateBlackListOverdueDay() {
long startTime = System.currentTimeMillis();
List<BlackGreyListResult> blackListResults = new ArrayList<>();
......@@ -781,6 +849,78 @@ public class BlackGreyListServiceImpl implements IBlackGreyListService {
}
log.info("更新黑名单逾期天数-查询黑灰名单结果表结束, list大小: {} , 耗时: {} ", blackListResults!=null?blackListResults.size():"null", (System.currentTimeMillis()-startTime));
//批量更新黑名单>>逾期天数
if(blackListResults!=null && blackListResults.size() >0 ){
return updateBlackListOverdueDayParallel.runMain(blackListResults);
}
return null;
}*/
/**
* -----------------------------------------------------------------------------<br>
* 描 述: 更新黑名单>>逾期天数 <br>
* 创建人: yanhui.Hao <br>
* 创建时间: 2020.04.16 21:13 <br>
* 最后修改人: <br>
* 最后修改时间: 2020.04.16 21:13 <br>
* 入参说明: <br>
* 出参说明: <br>
* -----------------------------------------------------------------------------
*/
@Override
public String updateBlackListOverdueDay() {
long startTime = System.currentTimeMillis();
Integer maxId = 0;
try {
maxId = xyqbJdbcTemplate.queryForObject(ConstantBlackGrey.SQL.BL_QUERY_BLACKLIST_RESULT_MAX_ID, Integer.class);
}catch (EmptyResultDataAccessException e){
log.error("查询结果表-黑名单最大Id异常, sql: {} ", ConstantBlackGrey.SQL.BL_QUERY_BLACKLIST_RESULT_MAX_ID, e);
}
if(maxId==null || maxId.intValue() <=0){
return "查询结果表-黑名单最大Id异常";
}
int totalPage = maxId / pageSize;
if((maxId % pageSize)!=0){
totalPage+=1;
}
String sql_query_tmp = ConstantBlackGrey.SQL.BL_QUERY_BLACKLIST_RESULT_BY_LIMIT_ID;
sql_query_tmp = sql_query_tmp.replace("##r.*##","r.r_id, r.uuid, r.black_type");
List<BlackGreyListResult> blackListResults = new ArrayList<>();
int startId = 0,endId = 0;
for (int page = 1; page <= totalPage; page++) {
long thisPageStart = System.currentTimeMillis();
startId = (page-1) * pageSize;
endId = startId + pageSize;
List<BlackGreyListResult> tmpQueryList = null;
try{
tmpQueryList = blackListJdbcTemplate.query(sql_query_tmp, new Object[]{startId, endId}, new RowMapper<BlackGreyListResult>() {
@Override
public BlackGreyListResult mapRow(ResultSet rs, int rowNum) throws SQLException {
BlackGreyListResult bean = new BlackGreyListResult();
bean.setRId(rs.getString("r_id"));
bean.setUuid(rs.getString("uuid"));
bean.setBlackType(rs.getString("black_type"));
return bean;
}
});
if(tmpQueryList!=null && tmpQueryList.size() >0){
blackListResults.addAll(tmpQueryList);
}
}catch (Exception e){
e.printStackTrace();
log.error("查询结果表-黑名单列表异常, totalPage: {} , page: {} , startId: {} , endId: {} , 耗时: {} ", totalPage, page, startId, endId, (System.currentTimeMillis()-thisPageStart), e);
}finally {
log.error("查询结果表-黑名单列表结束, totalPage: {} , page: {} , startId: {} , endId: {} , 大小: {} , 耗时: {} ", totalPage, page, startId, endId, tmpQueryList!=null?tmpQueryList.size():"null", (System.currentTimeMillis()-thisPageStart));
}
}
log.info("更新黑名单逾期天数-查询黑灰名单结果表结束, list大小: {} , 总查询耗时: {} ", blackListResults!=null?blackListResults.size():"null", (System.currentTimeMillis()-startTime));
//批量更新黑名单>>逾期天数
if(blackListResults!=null && blackListResults.size() >0 ){
return updateBlackListOverdueDayParallel.runMain(blackListResults);
......
src/main/java/cn/quantgroup/qgblservice/service/impl/BlackListToolsManagerServiceImpl.java
View file @ d6a05dc7
......@@ -458,6 +458,10 @@ public class BlackListToolsManagerServiceImpl implements IBlackListToolsManagerS
}
//-----------------------------------------------------------------------------------------------------------------------
/**
* -----------------------------------------------------------------------------<br>
* 描述: black_list_new 数据，往tmp_black_grey_list清洗<br>
......@@ -504,9 +508,9 @@ public class BlackListToolsManagerServiceImpl implements IBlackListToolsManagerS
//tmpBlackGreyToProduct();
}
if("copyBlackGreyListResult_old2".equals(operatType)){
/* if("copyBlackGreyListResult_old2".equals(operatType)){
copyBlackGreyListResult_old2();
}
}*/
//测试job中的方法
......
src/main/java/cn/quantgroup/qgblservice/service/impl/BlackToGreyListParallel.java
View file @ d6a05dc7
......@@ -22,8 +22,6 @@ import java.io.IOException;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
......@@ -42,18 +40,10 @@ public class BlackToGreyListParallel implements ParallelComputingProcess<BlackGr
private final String log_inf = "BlackToGreyListParallel";
private final String sepa = java.io.File.separator; //System.getProperty("file.separator");
private final DateFormat df = new SimpleDateFormat(ConstantBlackGrey.DAYE_FORMAT.YYYYMMDD);
private final String saveLogPath = "/home/quant_group/qg-bl-service/logs/";
//并行处理框架
private ParallelComputing<BlackGreyListResult, Set<String>> p = new ParallelComputing<BlackGreyListResult, Set<String>>("BlackToGreyListParallel");
/**
* 当前服务器CPU个数
*/
private static int SYS_CPU_COUNT = Runtime.getRuntime().availableProcessors();//4
//private static int SYS_CPU_COUNT = 1;
private static int THREAD_COUNT = ConstantBlackGrey.PARAM.SYS_CPU_COUNT * 2;
@Autowired
private BlackGreyListMapper blackGreyListMapper;
......@@ -71,12 +61,12 @@ public class BlackToGreyListParallel implements ParallelComputingProcess<BlackGr
public String runMain(List<BlackGreyListResult> tmpQueryList) {
long startProcess = System.currentTimeMillis();
int listSize = tmpQueryList.size();
log.info(log_inf+"-runMain黑名单转灰名单开始执行, listSize: {} , SYS_CPU_COUNT: {} ", listSize, SYS_CPU_COUNT);
log.info(log_inf+"-runMain黑名单转灰名单开始执行, listSize: {} , SYS_CPU_COUNT: {} ", listSize, THREAD_COUNT);
try {
//并行框架---执行
Set<String> optSet = Collections.synchronizedSet(new HashSet<>());
p.processForThread(tmpQueryList, this, SYS_CPU_COUNT, optSet);
p.processForThread(tmpQueryList, this, THREAD_COUNT, optSet);
//并行框架执行结果
if (optSet.size() > 0) {//并行框架执行异常
......@@ -98,19 +88,188 @@ public class BlackToGreyListParallel implements ParallelComputingProcess<BlackGr
@Override
public void process(int threadId, List<BlackGreyListResult> blackResultList, Set<String> optSet) {
//yyyyMMdd
String dateName = ConstantBlackGrey.PARAM.DF_YMD.format(Calendar.getInstance().getTime());
int list_size = blackResultList.size();
AtomicInteger remove_count = new AtomicInteger();
AtomicInteger err_count = new AtomicInteger();
AtomicInteger type_11 = new AtomicInteger();
int list_size = blackResultList.size();
//yyyyMMdd
Calendar calendar = Calendar.getInstance();
String dateName = df.format(calendar.getTime());
int index = 0;
int index = 0;
for (BlackGreyListResult blackResult : blackResultList) {
//blackResult r_id,uuid,black_type,types,reasons
//`status` tinyint(2) DEFAULT '1' COMMENT '状态 0:正常有效的； -1:由灰名单进入黑名单(无效，逻辑删除的)； -2:由黑名单进入灰名单(无效，逻辑删除的)； -3:在插入灰名单时，已经是黑名单； 10：当前在黑名单，并且该条数据逾期已还清的',
log.info("threadId_{}: 开始执行list_size: {} , this: {} ", threadId , list_size, (++index));
String rId = blackResult.getRId();
String uuid = blackResult.getUuid();
if(StringUtils.isNotEmpty(uuid)){
try {
BlackGreyListQueryVo queryParam = BlackGreyListQueryVo.builder().rId(blackResult.getRId()).build();
//关联查询太慢
//List<BlackGreyListRjoinDVo> rJoinDVoList = blackGreyListMapper.findBlackGreyListRjoinD(queryParam);
List<BlackGreyListDetails> detailsList = blackGreyListMapper.findBlackGreyListDetails(queryParam);
if (detailsList!=null && detailsList.size()>0) {
boolean is_type_11 = false;
for(BlackGreyListDetails detailsVo : detailsList){
//type=11, 恶意投诉客服，一直在黑名单
if("11".equals(detailsVo.getType()) && detailsVo.getStatus()==0){
type_11.getAndIncrement();
is_type_11=true;
break;
}
//现金贷
else if("1".equals(detailsVo.getType()) && detailsVo.getStatus()==0){
//#现金分期在逾转已还 (条件为step1的表内join_black_reason=1的用户，判断其是否还处于在逾状态，如果in_overdue=0，则更新step1的表的black_type为2并更新join_black_reason为6,否则不做修改)
//查询现金贷逾期，是否逾期已还清
Integer in_overdue_Obj = null;
try {
in_overdue_Obj = xyqbJdbcTemplate.queryForObject(ConstantBlackGrey.SQL.XYQB_QUERY_XIANJINDAI_IS_PAYOFF, new Object[]{uuid}, Integer.class);
}catch (EmptyResultDataAccessException e){
log.error("查询[现金贷]逾期逾期黑名单是否已还清-查询xyqb异常, r_Id: {} , id: {} , uuid: {} , e: {} ", rId, detailsVo.getId(), uuid, e.toString());
}finally {
log.info("查询[现金贷]逾期逾期黑名单是否已还清-结束, r_Id: {} , id: {} , uuid: {} , in_overdue_Obj : {} ", rId, detailsVo.getId(), uuid, in_overdue_Obj!=null? in_overdue_Obj.toString(): "null");
}
//修改结果表状态为10
updateDetailStatus10(in_overdue_Obj, detailsVo, "现金贷");
}
//白条
else if("2".equals(detailsVo.getType()) && detailsVo.getStatus()==0){
//#白条在逾转已还 (条件为step1的表内join_black_reason=3的用户，判断其是否还处于在逾状态，如果in_overdue=0，则更新step1的表的black_type为2并更新join_black_reason为6,否则不做修改)
//查询白条逾期逾期黑名单是否已还清
Integer in_overdue_Obj = null;
try {
in_overdue_Obj = xyqbJdbcTemplate.queryForObject(ConstantBlackGrey.SQL.XYQB_QUERY_BAITIAO_IS_PAYOFF, new Object[]{uuid}, Integer.class);
}catch (EmptyResultDataAccessException e){
log.error("查询[白条]逾期逾期黑名单是否已还清-查询xyqb异常, r_Id: {} , id: {} , uuid: {} , e: {} ", rId, detailsVo.getId(), uuid, e.toString());
}finally {
log.info("查询[白条]逾期逾期黑名单是否已还清-结束, r_Id: {} , id: {} , uuid: {} , in_overdue_Obj : {} ", rId, detailsVo.getId(), uuid, in_overdue_Obj!=null? in_overdue_Obj.toString(): "null");
}
//修改结果表状态为10
updateDetailStatus10(in_overdue_Obj, detailsVo, "白条");
}
//Vcc
else if("10".equals(detailsVo.getType()) && detailsVo.getStatus()==0){
// #vcc在逾转已还 (条件为step1的表内join_black_reason=2的用户，判断其是否还处于在逾状态，如果in_overdue=0，则更新step1的表的black_type为2并更新join_black_reason为6,否则不做修改)
Integer in_overdue_Obj = null;
XyqbUser xyqbUser = queryXyqbUserByUuid(uuid);
if(xyqbUser!=null && xyqbUser.getId()!=0L){
try {
in_overdue_Obj = acsdbJdbcTemplate.queryForObject(ConstantBlackGrey.SQL.XYQB_QUERY_VCC_IS_PAYOFF, new Object[]{xyqbUser.getId()}, Integer.class);
}catch (EmptyResultDataAccessException e){
log.error("查询[VCC]逾期逾期黑名单是否已还清-查询xyqb异常, r_Id: {} , id: {} , uuid: {} , user_id: {} , e: {} ", rId, detailsVo.getId(), uuid, xyqbUser.getId(), e.toString());
}finally {
log.info("查询[VCC]逾期逾期黑名单是否已还清-结束, r_Id: {} , id: {} , uuid: {} , user_id: {} , in_overdue_Obj : {} ", rId, detailsVo.getId(), uuid, xyqbUser.getId(), in_overdue_Obj!=null? in_overdue_Obj.toString(): "null");
}
//修改结果表状态为10
updateDetailStatus10(in_overdue_Obj, detailsVo, "VCC");
}
}else{
err_count.getAndIncrement();
optSet.add("otherType:"+blackResult.getRId());
writeLogByName(ConstantBlackGrey.PARAM.SAVELOGPATH + dateName + ConstantBlackGrey.PARAM.SEPA + "error.log", "black_list_result_not_find_details>>"+blackResult.getRId());
}
}
//type=11, 恶意投诉客服，一直待在黑名单
if(is_type_11){
continue;
}
boolean is_grey_result = true;
List<ReasonsVo> reasonsList = null;
for(BlackGreyListDetails detailsVo : detailsList){
if(detailsVo.getStatus()==0){
//可以有多个type，说明至少有一个逾期还没还清，就不能转出黑名单
is_grey_result = false;
}else if(detailsVo.getStatus()==10){
//Set<String> typeSet = BlackGreyListResult.typesToSet(blackResult.getTypes(), rJoinDVo.getType());
reasonsList = BlackGreyListResult.reasonsToList(blackResult.getReasons(), "6", detailsVo.getType());
}
}
if(is_grey_result){
Timestamp timestamp = new Timestamp(System.currentTimeMillis());
//update black_grey_list_details set status = ?, updated_at = ? where r_id = ? AND status =10 AND type IN ('1','2','10') ;
//int update = blackListJdbcTemplate.update(ConstantBlackGrey.SQL.BL_DELETE_DATAILS_BLACK, -2, timestamp, blackResult.getRId());
//更新明细表
DetailsUpdate detailUpdateParam = new DetailsUpdate();
detailUpdateParam.setStatus(-2);//由黑名单进入灰名单
detailUpdateParam.setUpdatedAt(new Timestamp(System.currentTimeMillis()));
//条件
detailUpdateParam.setRId(blackResult.getRId());
detailUpdateParam.setWEqStatus(10);
detailUpdateParam.setWInType(Arrays.asList(ConstantBlackGrey.PARAM.BLACK_LIST_TYPES)); //('1','2','10')
//int update_d = blackGreyListMapper.updateDetailsStatusByparams(detailUpdateParam);
//测试不操作数据
int update_d = 1;
log.warn("测试不操作数据,updateDetailsStatusByparams:{}", JSONObject.toJSONString(detailUpdateParam));
//更新结果表
BlackGreyListResult updateResult = new BlackGreyListResult();
updateResult.setRId(blackResult.getRId());//条件
updateResult.setUpdatedAt(timestamp);
updateResult.setBlackType("2");
//updateResult.setTypes(blackResult.getTypes());
if(reasonsList!=null){
updateResult.setReasons(JSON.toJSONString(reasonsList));
}
//int update_r = blackGreyListMapper.updateBlackGreyListResult(updateResult);
//测试不操作数据
int update_r = 0;
log.warn("测试不操作数据,updateBlackGreyListResult:{}", JSONObject.toJSONString(updateResult));
if(update_d>0 && update_r>0){
remove_count.getAndIncrement();
}
}
}else {
err_count.getAndIncrement();
optSet.add("notFind:"+blackResult.getRId());
writeLogByName(ConstantBlackGrey.PARAM.SAVELOGPATH + dateName + ConstantBlackGrey.PARAM.SEPA + "error.log", "black_list_result_not_find_details>>"+blackResult.getRId());
}
} catch (Exception e) {
optSet.add("ERR:" + blackResult.getRId());
log.error(log_inf + "process() threadId:" + threadId + ",ERROR!", e);
}
}else {
log.error("查询逾期黑名单是否已还清-发现uuid为空, threadId_{} , r_Id: {} , uuid: {} , blackType: {} ", threadId, rId, uuid, blackResult.getBlackType());
}
}
log.info(log_inf + "处理结束 CPU_{} , remove_count:{} , 失败:{} , this_list_size: {} ", threadId, remove_count.get(), err_count.get(), list_size);
}
/* @Override
public void process(int threadId, List<BlackGreyListResult> blackResultList, Set<String> optSet) {
//yyyyMMdd
String dateName = df.format(Calendar.getInstance().getTime());
int list_size = blackResultList.size();
AtomicInteger remove_count = new AtomicInteger();
AtomicInteger err_count = new AtomicInteger();
AtomicInteger type_11 = new AtomicInteger();
int index = 0;
for (BlackGreyListResult blackResult : blackResultList) {
//blackResult r_id,uuid,black_type,types,reasons
//`status` tinyint(2) DEFAULT '1' COMMENT '状态 0:正常有效的； -1:由灰名单进入黑名单(无效，逻辑删除的)； -2:由黑名单进入灰名单(无效，逻辑删除的)； -3:在插入灰名单时，已经是黑名单； 10：当前在黑名单，并且该条数据逾期已还清的',
log.info("threadId_{}: 开始执行list_size: {} , this: {} ", threadId , list_size, (++index));
try {
BlackGreyListQueryVo queryParam = BlackGreyListQueryVo.builder().rId(blackResult.getRId()).build();
List<BlackGreyListRjoinDVo> rJoinDVoList = blackGreyListMapper.findBlackGreyListRjoinD(queryParam);
......@@ -269,11 +428,7 @@ public class BlackToGreyListParallel implements ParallelComputingProcess<BlackGr
}
log.info(log_inf + "处理结束 CPU_{} , remove_count:{} , 失败:{} , this_list_size: {} ", threadId, remove_count.get(), err_count.get(), list_size);
}
*/
//-------------------------------------------------------------------------
private static void writeLogByName(String fileName, String msg) {
......@@ -313,7 +468,7 @@ public class BlackToGreyListParallel implements ParallelComputingProcess<BlackGr
}
//将黑名单明细表的状态改为10
private int updateDetailStatus10(Integer in_overdue_Obj, BlackGreyListRjoinDVo rJoinDVo, String log_inf){
private int updateDetailStatus10(Integer in_overdue_Obj, BlackGreyListDetails rJoinDVo, String log_inf){ //BlackGreyListRjoinDVo >> BlackGreyListDetails
//修改结果表状态为10
try {
if(in_overdue_Obj!=null && StringUtils.isNotEmpty(in_overdue_Obj.toString())){
......@@ -337,7 +492,7 @@ public class BlackToGreyListParallel implements ParallelComputingProcess<BlackGr
}
}
}catch (Exception e){
log.error("查询["+log_inf+"]逾期逾期黑名单是否已还清-修改结果表状态为10出现异常, r_Id: {} , id: {} , uuid: {} ", rJoinDVo.getRId(), rJoinDVo.getId(), rJoinDVo.getUuid(), e);
log.error("查询["+log_inf+"]逾期逾期黑名单是否已还清-修改结果表状态为10出现异常, r_Id: {} , id: {} ", rJoinDVo.getRId(), rJoinDVo.getId(), e);
}
return 0;
}
......
src/main/java/cn/quantgroup/qgblservice/service/impl/UpdateBlackListOverdueDayParallel.java
View file @ d6a05dc7
......@@ -12,11 +12,10 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.dao.DataAccessException;
import org.springframework.dao.EmptyResultDataAccessException;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.ResultSetExtractor;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import java.io.File;
......@@ -24,8 +23,6 @@ import java.io.IOException;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
......@@ -42,20 +39,12 @@ import java.util.concurrent.atomic.AtomicInteger;
@Service
public class UpdateBlackListOverdueDayParallel implements ParallelComputingProcess<BlackGreyListResult, Set<String>> {
private final String log_inf = "BlackToGreyListParallel";
private final String sepa = File.separator; //System.getProperty("file.separator");
private final DateFormat df = new SimpleDateFormat(ConstantBlackGrey.DAYE_FORMAT.YYYYMMDD);
private final String saveLogPath = "/home/quant_group/qg-bl-service/logs/";
private final String log_inf = "UpdateBlackListOverdueDay";
//并行处理框架
private ParallelComputing<BlackGreyListResult, Set<String>> p = new ParallelComputing<BlackGreyListResult, Set<String>>("BlackToGreyListParallel");
/**
* 当前服务器CPU个数
*/
private static int SYS_CPU_COUNT = Runtime.getRuntime().availableProcessors();//4
//private static int SYS_CPU_COUNT = 1;
private ParallelComputing<BlackGreyListResult, Set<String>> p = new ParallelComputing<BlackGreyListResult, Set<String>>("UpdateBlackListOverdueDay");
private static int THREAD_COUNT = ConstantBlackGrey.PARAM.SYS_CPU_COUNT * 2;
@Autowired
private BlackGreyListMapper blackGreyListMapper;
......@@ -73,12 +62,12 @@ public class UpdateBlackListOverdueDayParallel implements ParallelComputingProce
public String runMain(List<BlackGreyListResult> tmpQueryList) {
long startProcess = System.currentTimeMillis();
int listSize = tmpQueryList.size();
log.info(log_inf+"-runMain更新逾期天数开始执行, listSize: {} , SYS_CPU_COUNT: {} ", listSize, SYS_CPU_COUNT);
log.info(log_inf+"-runMain更新逾期天数开始执行, listSize: {} , SYS_CPU_COUNT: {} ", listSize, THREAD_COUNT);
try {
//并行框架---执行
Set<String> optSet = Collections.synchronizedSet(new HashSet<>());
p.processForThread(tmpQueryList, this, SYS_CPU_COUNT, optSet);
p.processForThread(tmpQueryList, this, THREAD_COUNT, optSet);
//并行框架执行结果
if (optSet.size() > 0) {//并行框架执行异常
......@@ -100,19 +89,18 @@ public class UpdateBlackListOverdueDayParallel implements ParallelComputingProce
@Override
public void process(int threadId, List<BlackGreyListResult> blackResultList, Set<String> optSet) {
//yyyyMMdd
String dateName = ConstantBlackGrey.PARAM.DF_YMD.format(Calendar.getInstance().getTime());
int list_size = blackResultList.size();
AtomicInteger update_count = new AtomicInteger();
AtomicInteger err_count = new AtomicInteger();
AtomicInteger type_11 = new AtomicInteger();
int list_size = blackResultList.size();
//yyyyMMdd
Calendar calendar = Calendar.getInstance();
String dateName = df.format(calendar.getTime());
int index = 0;
int index = 0;
for (BlackGreyListResult blackResult : blackResultList) {
//blackResult>>r_id,uuid,black_type
log.info("threadId_{}: 开始执行修改逾期天数list_size: {} , this: {} ", threadId , list_size, (++index));
String rId = blackResult.getRId();
String uuid = blackResult.getUuid();
......@@ -214,27 +202,26 @@ public class UpdateBlackListOverdueDayParallel implements ParallelComputingProce
}else{
err_count.getAndIncrement();
optSet.add("otherType:"+blackResult.getRId());
writeLogByName(saveLogPath + dateName + sepa + "error.log", "black_list_result_not_find_details>>"+blackResult.getRId());
writeLogByName(ConstantBlackGrey.PARAM.SAVELOGPATH + dateName + ConstantBlackGrey.PARAM.SEPA + "error.log", "black_list_result_not_find_details>>"+blackResult.getRId());
}
}
//如果明细表有修改，更新一下结果表修改时间
if(details_is_update){
//更新结果表
/* //更新结果表
BlackGreyListResult updateResult = new BlackGreyListResult();
updateResult.setRId(blackResult.getRId());//条件
updateResult.setUpdatedAt(new Timestamp(System.currentTimeMillis()));//结果
//int update_r = blackGreyListMapper.updateBlackGreyListResult(updateResult);
//测试不操作数据
int update_r = 1;
log.warn("测试不操作数据, 更新时间, updateBlackGreyListResult:{}", JSONObject.toJSONString(updateResult));
int update_r = blackGreyListMapper.updateBlackGreyListResult(updateResult);
*/
update_count.getAndIncrement();
updateResultUpdatedAt(blackResult.getRId());
}
}else {
err_count.getAndIncrement();
optSet.add("notFind:"+blackResult.getRId());
writeLogByName(saveLogPath + dateName + sepa + "error.log", "black_list_result_not_find_details>>"+blackResult.getRId());
writeLogByName(ConstantBlackGrey.PARAM.SAVELOGPATH + dateName + ConstantBlackGrey.PARAM.SEPA + "error.log", "black_list_result_not_find_details>>"+blackResult.getRId());
}
} catch (Exception e) {
......@@ -266,7 +253,7 @@ public class UpdateBlackListOverdueDayParallel implements ParallelComputingProce
private XyqbUser queryXyqbUserByUuid(String uuid) {
String sql_byUUid = "select u.id, u.uuid, u.phone_no from `user` u where u.uuid=? AND u.`enable`=1 ";
String sql_byUUid = "select u.id, u.uuid from `user` u where u.uuid=? AND u.`enable`=1 ";
List<XyqbUser> xyqbUserList = null;
try {
xyqbUserList = xyqbUserJdbcTemplate.query(sql_byUUid, new Object[]{uuid}, new RowMapper<XyqbUser>() {
......@@ -275,12 +262,9 @@ public class UpdateBlackListOverdueDayParallel implements ParallelComputingProce
XyqbUser bean = new XyqbUser();
bean.setId(rs.getLong("id"));
bean.setUuid(rs.getString("uuid"));
bean.setPhoneNo(rs.getString("phone_no"));
return bean;
}
});
} catch (Exception e) {
log.error("根据uuid查询xyqb用户异常, uuid: {} ", uuid, e);
}
......@@ -319,5 +303,20 @@ public class UpdateBlackListOverdueDayParallel implements ParallelComputingProce
}
//更新结果表,updatedAt时间
@Async
private int updateResultUpdatedAt(String rId){
int update_r = 0;
try{
BlackGreyListResult updateResult = new BlackGreyListResult();
updateResult.setRId(rId);//条件
updateResult.setUpdatedAt(new Timestamp(System.currentTimeMillis()));//结果
//测试不操作数据
//update_r = blackGreyListMapper.updateBlackGreyListResult(updateResult);
}catch (Exception e){
log.error("异步更新黑灰名单结果表-updatedAt时间异常, rId : {} ", rId, e);
}
return update_r;
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment