Skip to content

Q
qg-bl-service
Commit 7fe229af authored by 郝彦辉's avatar 郝彦辉
Browse files
Options

黑灰名单2020.04.22_07

parent 97447aa4
Hide whitespace changes
Inline Side-by-side
Showing with 30 additions and 82 deletions
src/main/java/cn/quantgroup/qgblservice/service/impl/BlackGreyListServiceImpl.java
View file @ 7fe229af
......@@ -288,10 +288,7 @@ public class BlackGreyListServiceImpl implements IBlackGreyListService {
insertdetails.setCreatedAt(blackGreyObj.getCreatedAt());
insertdetails.setUpdatedAt(blackGreyObj.getUpdatedAt());
/* int insert_result = blackGreyListMapper.insertBlackGreyListResult(insertResult);
int insert_details =blackGreyListMapper.insertBlackGreyListDetails(insertdetails);*/
log.info("黑灰名单增加结束-结果表插入: {} , 明细表插入: {} ", insertResult.toString(), insertdetails.toString());
}else{
if(blackGreyResultList.size()!=1){
......@@ -301,11 +298,7 @@ public class BlackGreyListServiceImpl implements IBlackGreyListService {
BlackGreyListResult oldResult = blackGreyResultList.get(0);
updateResult = new BlackGreyListResult();
//updateParams.setUpdatedAt("")
//updateParams.setBlackType("");
//updateParams.setTypes("");
//updateParams.setReasons("");
//UpdatedAt BlackType Types Reasons
updateResult.setRId(oldResult.getRId());
if(oldResult.getUpdatedAt().getTime() >= blackGreyObj.getUpdatedAt().getTime()){
updateResult.setUpdatedAt(oldResult.getUpdatedAt());
......@@ -358,7 +351,6 @@ public class BlackGreyListServiceImpl implements IBlackGreyListService {
throw new SQLException("黑灰名单新增发现其他类别BlackType, oldResult:"+oldResult.getBlackType()+", blackGrey:"+blackGreyObj.getBlackType());
}
insertdetails = new BlackGreyListDetails();
insertdetails.setRId(oldResult.getRId());
insertdetails.setName(oldResult.getName());
......@@ -373,25 +365,15 @@ public class BlackGreyListServiceImpl implements IBlackGreyListService {
insertdetails.setCreatedAt(blackGreyObj.getCreatedAt());
insertdetails.setUpdatedAt(blackGreyObj.getUpdatedAt());
/* int delete_details = 0;
if(iupdatedetails!=null){
delete_details = blackGreyListMapper.updateDetailsStatus(iupdatedetails);
}
int insert_details = blackGreyListMapper.insertBlackGreyListDetails(insertdetails);
int update_result = blackGreyListMapper.updateBlackGreyListResult(updateResult);*/
log.info("黑灰名单增加结束-结果表修改: {} , 明细表插入: {} , 明细表修改: {} , blackGreyObj : {} ", updateResult.toString(), insertdetails.toString(), deleteDetails, blackGreyObj.toString());
}
}catch (Exception e){
log.error("黑灰名单增加筛选结果异常, blackGreyObj : {}", blackGreyObj.toString(), e);
insertResult = null; updateResult = null; insertdetails = null; deleteDetails = null;
continue;
}
/**
//测试不操作数据
if(deleteDetails!=null){
StringBuffer sql_buff = new StringBuffer("UPDATE black_grey_list_details set ");
sql_buff.append(" status=").append(deleteDetails.getStatus()).append(", ");
......@@ -419,7 +401,6 @@ public class BlackGreyListServiceImpl implements IBlackGreyListService {
// 批处理多条sql操作
statement.addBatch(sql_buff.toString());
}
if(insertResult!=null){
StringBuffer sql_buff = new StringBuffer("insert ignore into black_grey_list_result (`r_id`, `uuid`, `name`, `phone_no`, `id_no`, `phone_no_md5`, `id_no_md5`, `black_type`, `types`, `reasons`, `created_at`, `updated_at`) ");
sql_buff.append(" values (");
......@@ -458,7 +439,6 @@ public class BlackGreyListServiceImpl implements IBlackGreyListService {
statement.addBatch(sql_buff.toString());
}
// 关闭自动提交
conn.setAutoCommit(false);
try {
......@@ -468,7 +448,6 @@ public class BlackGreyListServiceImpl implements IBlackGreyListService {
conn.commit();
atomicInteger.getAndIncrement();
statement.clearBatch();
} catch (Exception e) {
log.error("黑灰名单增加数据提交异常, blackGreyObj: {} , insertResult: {} , insertdetails: {} , updateResult: {} , updatedetails: {} ",
blackGreyObj.toString(), insertResult!=null? insertResult.toString(): null, insertdetails!=null? insertdetails.toString(): null, updateResult!=null? updateResult.toString(): null, deleteDetails!=null? deleteDetails.toString(): null, e);
......@@ -480,11 +459,8 @@ public class BlackGreyListServiceImpl implements IBlackGreyListService {
log.error("黑灰名单增加数据回滚异常, blackGreyObj: {} ", blackGreyObj.toString(),e);
}
}
*/
}
}catch (Exception e){
log.error("黑灰名单增加数据异常", e);
}finally {
......@@ -532,13 +508,12 @@ public class BlackGreyListServiceImpl implements IBlackGreyListService {
*/
@Override
public String importXianJinDaiBlackGreyList() {
List<TmpBlackGreyList> newIncreasedList = new ArrayList<>();
Stopwatch queryStopwatch = Stopwatch.createStarted();
List<TmpBlackGreyList> xjdBlackGreyList = xyqbJdbcTemplate.query(ConstantBlackGrey.SQL.XYQB_QUERY_XianJinDai_15DAY, new BeanPropertyRowMapper<>(TmpBlackGreyList.class));
log.info("每天执行-现金分期15+逾期用户黑名单-查询结束, listSize: {} , 耗时: {} ", xjdBlackGreyList.size(), queryStopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
Stopwatch checkStopwatch = Stopwatch.createStarted();
List<TmpBlackGreyList> newIncreasedList = new ArrayList<>();
AtomicInteger details_haveCount = new AtomicInteger();
int saveOkCount = 0;
TmpBlackGreyList blackGreyVo = null;
......@@ -574,7 +549,6 @@ public class BlackGreyListServiceImpl implements IBlackGreyListService {
}
log.info("每天执行-现金分期15+逾期用户黑名单-插入结束, listSize: {} , details_haveCount: {} , newIncreasedList: {} , saveOkCount: {} , 耗时: {} ", xjdBlackGreyList.size(), details_haveCount.get(), newIncreasedList.size(), saveOkCount, saveStopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
String resMsgTemp = "新增黑名单(现金分期15+逾期) 查询 %d条, 明细表已存在 %d条, 需要插入 %d条, 插入成功 %d条;";
return String.format(resMsgTemp, xjdBlackGreyList.size(), details_haveCount.get(), newIncreasedList.size(), saveOkCount);
}
......@@ -590,7 +564,6 @@ public class BlackGreyListServiceImpl implements IBlackGreyListService {
}
log.info("每天执行-vcc逾期15+用户黑名单-查询acsdb用户userId结束, vccUserIdList: {} , 耗时: {} ", (vccUserIdList!=null ? vccUserIdList.size(): "null"), queryStopwatch1.stop().elapsed(TimeUnit.MILLISECONDS));
Stopwatch queryStopwatch2 = Stopwatch.createStarted();
List<TmpBlackGreyList> vccBlackGreyList = new ArrayList<TmpBlackGreyList>();
......@@ -664,7 +637,6 @@ public class BlackGreyListServiceImpl implements IBlackGreyListService {
}
log.info("每天执行-vcc逾期15+用户黑名单-检测结束, vccBlackGreyList: {} , details_haveCount: {} , newIncreasedList: {} , 耗时: {} ", vccBlackGreyList.size(), details_haveCount.get(), newIncreasedList.size(), checkStopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
Stopwatch saveStopwatch = Stopwatch.createStarted();
if(newIncreasedList!=null && newIncreasedList.size()> 0){
try {
......@@ -675,7 +647,6 @@ public class BlackGreyListServiceImpl implements IBlackGreyListService {
}
log.info("每天执行-vcc逾期15+用户黑名单-插入结束, vccBlackGreyList: {} , details_haveCount: {} , newIncreasedList: {} , saveOkCount: {} , 耗时: {} ", vccBlackGreyList.size(), details_haveCount.get(), newIncreasedList.size(), saveOkCount, saveStopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
String resMsgTemp = "新增黑名单(vcc逾期15+逾期) 查询 %d条, 明细表已存在 %d条, 需要插入 %d条, 插入成功 %d条;";
return String.format(resMsgTemp, vccBlackGreyList.size(), details_haveCount.get(), newIncreasedList.size(), saveOkCount);
}
......@@ -786,8 +757,8 @@ public class BlackGreyListServiceImpl implements IBlackGreyListService {
if(tmpQueryList!=null && tmpQueryList.size() >0){
count_black+=tmpQueryList.size();
//blackListResults.addAll(tmpQueryList);
for (BlackGreyListResult result: tmpQueryList){
String types = ","+result.getTypes().trim()+",";
if(!types.contains(",11,")){
......
src/main/java/cn/quantgroup/qgblservice/service/impl/BlackToGreyListParallel.java
View file @ 7fe229af
......@@ -43,7 +43,8 @@ public class BlackToGreyListParallel implements ParallelComputingProcess<BlackGr
//并行处理框架
private ParallelComputing<BlackGreyListResult, Set<String>> p = new ParallelComputing<BlackGreyListResult, Set<String>>("BlackToGreyListParallel");
private static int THREAD_COUNT = ConstantBlackGrey.PARAM.SYS_CPU_COUNT * 2;
//private static int THREAD_COUNT = ConstantBlackGrey.PARAM.SYS_CPU_COUNT * 2;
private static int THREAD_COUNT = 5;
@Autowired
private BlackGreyListMapper blackGreyListMapper;
......@@ -61,7 +62,7 @@ public class BlackToGreyListParallel implements ParallelComputingProcess<BlackGr
public String runMain(List<BlackGreyListResult> tmpQueryList) {
long startProcess = System.currentTimeMillis();
int listSize = tmpQueryList.size();
log.info(log_inf+"-runMain黑名单转灰名单开始执行, listSize: {} , SYS_CPU_COUNT: {} ", listSize, THREAD_COUNT);
log.info(log_inf+"-runMain黑名单转灰名单开始执行>>>>>, listSize: {} , SYS_CPU_COUNT: {} ", listSize, THREAD_COUNT);
try {
//并行框架---执行
......@@ -70,12 +71,11 @@ public class BlackToGreyListParallel implements ParallelComputingProcess<BlackGr
//并行框架执行结果
if (optSet.size() > 0) {//并行框架执行异常
log.error("黑名单转灰名单runMain() ERROR! >> optSet size:{} , Json:{} ", optSet.size(), JSONObject.toJSONString(optSet));
log.info(log_inf + "-runMain黑名单转灰名单执行结束-部分失败, cost: {} <<<<<< process optSet size: {} , json: {} ", (System.currentTimeMillis()-startProcess), optSet.size(), JSONObject.toJSONString(optSet));
} else {
//执行成功
log.info(log_inf + " >> to process() is OK!");
log.info(log_inf + "-runMain黑名单转灰名单执行结束-成功, cost: {} <<<<<< process is OK!", (System.currentTimeMillis() - startProcess));
}
log.info(log_inf +"-runMain黑名单转灰名单执行结束, cost:" + (System.currentTimeMillis() - startProcess) + "ms.");
String resMsgTemp = "总共 %d条, 执行成功 %d条, 失败 %d条;";
return String.format(resMsgTemp, listSize, listSize-optSet.size(), optSet.size());
......@@ -91,18 +91,18 @@ public class BlackToGreyListParallel implements ParallelComputingProcess<BlackGr
//yyyyMMdd
String dateName = ConstantBlackGrey.PARAM.DF_YMD.format(Calendar.getInstance().getTime());
int list_size = blackResultList.size();
long start = System.currentTimeMillis();
AtomicInteger remove_count = new AtomicInteger();
AtomicInteger err_count = new AtomicInteger();
AtomicInteger type_11 = new AtomicInteger();
int index = 0;
//blackResult r_id,uuid,black_type,types,reasons
for (BlackGreyListResult blackResult : blackResultList) {
//blackResult r_id,uuid,black_type,types,reasons
//`status` tinyint(2) DEFAULT '1' COMMENT '状态 0:正常有效的； -1:由灰名单进入黑名单(无效，逻辑删除的)； -2:由黑名单进入灰名单(无效，逻辑删除的)； -3:在插入灰名单时，已经是黑名单； 10：当前在黑名单，并且该条数据逾期已还清的',
log.info("threadId_{}: 开始执行list_size: {} , this: {} ", threadId , list_size, (++index));
//status 状态 0:正常有效的； -1:由灰名单进入黑名单(无效，逻辑删除的)； -2:由黑名单进入灰名单(无效，逻辑删除的)； -3:在插入灰名单时，已经是黑名单； 10：当前在黑名单，并且该条数据逾期已还清的',
String rId = blackResult.getRId();
String uuid = blackResult.getUuid();
log.info("threadId_{}: 开始执行list_size: {} , this: {} , rId: {} , uuid: {} ", threadId , list_size, (++index), rId, uuid);
if(StringUtils.isNotEmpty(uuid)){
try {
......@@ -201,9 +201,6 @@ public class BlackToGreyListParallel implements ParallelComputingProcess<BlackGr
if(is_grey_result){
Timestamp timestamp = new Timestamp(System.currentTimeMillis());
//update black_grey_list_details set status = ?, updated_at = ? where r_id = ? AND status =10 AND type IN ('1','2','10') ;
//int update = blackListJdbcTemplate.update(ConstantBlackGrey.SQL.BL_DELETE_DATAILS_BLACK, -2, timestamp, blackResult.getRId());
//更新明细表
DetailsUpdate detailUpdateParam = new DetailsUpdate();
detailUpdateParam.setStatus(-2);//由黑名单进入灰名单
......@@ -212,25 +209,19 @@ public class BlackToGreyListParallel implements ParallelComputingProcess<BlackGr
detailUpdateParam.setRId(blackResult.getRId());
detailUpdateParam.setWEqStatus(10);
detailUpdateParam.setWInType(Arrays.asList(ConstantBlackGrey.PARAM.BLACK_LIST_TYPES)); //('1','2','10')
//int update_d = blackGreyListMapper.updateDetailsStatusByparams(detailUpdateParam);
//测试不操作数据
int update_d = 1;
log.warn("测试不操作数据,updateDetailsStatusByparams:{}", JSONObject.toJSONString(detailUpdateParam));
int update_d = blackGreyListMapper.updateDetailsStatusByparams(detailUpdateParam);
//更新结果表
BlackGreyListResult updateResult = new BlackGreyListResult();
updateResult.setRId(blackResult.getRId());//条件
updateResult.setUpdatedAt(timestamp);
updateResult.setBlackType("2");
//updateResult.setTypes(blackResult.getTypes());
////updateResult.setTypes(blackResult.getTypes());
if(reasonsList!=null){
updateResult.setReasons(JSON.toJSONString(reasonsList));
}
//int update_r = blackGreyListMapper.updateBlackGreyListResult(updateResult);
//测试不操作数据
int update_r = 0;
log.warn("测试不操作数据,updateBlackGreyListResult:{}", JSONObject.toJSONString(updateResult));
int update_r = blackGreyListMapper.updateBlackGreyListResult(updateResult);
if(update_d>0 && update_r>0){
remove_count.getAndIncrement();
}
......@@ -249,10 +240,9 @@ public class BlackToGreyListParallel implements ParallelComputingProcess<BlackGr
}else {
log.error("查询逾期黑名单是否已还清-发现uuid为空, threadId_{} , r_Id: {} , uuid: {} , blackType: {} ", threadId, rId, uuid, blackResult.getBlackType());
}
}
log.info(log_inf + "处理结束 CPU_{} , remove_count:{} , type_11: {} , 失败:{} , this_list_size: {} ", threadId, remove_count.get(), type_11.get(), err_count.get(), list_size);
log.info(log_inf + "黑名单转灰名单process处理结束CPU_{} , remove_count: {} , type_11: {} , 失败: {} , this_list_size: {} , 耗时: {} ", threadId, remove_count.get(), type_11.get(), err_count.get(), list_size, (System.currentTimeMillis()-start));
}
......@@ -394,10 +384,7 @@ public class BlackToGreyListParallel implements ParallelComputingProcess<BlackGr
detailUpdateParam.setRId(blackResult.getRId());
detailUpdateParam.setWEqStatus(10);
detailUpdateParam.setWInType(Arrays.asList(ConstantBlackGrey.PARAM.BLACK_LIST_TYPES)); //('1','2','10')
//int update_d = blackGreyListMapper.updateDetailsStatusByparams(detailUpdateParam);
//测试不操作数据
int update_d = 1;
log.warn("测试不操作数据,updateDetailsStatusByparams:{}", JSONObject.toJSONString(detailUpdateParam));
int update_d = blackGreyListMapper.updateDetailsStatusByparams(detailUpdateParam);
//更新结果表
BlackGreyListResult updateResult = new BlackGreyListResult();
......@@ -408,11 +395,8 @@ public class BlackToGreyListParallel implements ParallelComputingProcess<BlackGr
if(reasonsList!=null){
updateResult.setReasons(JSON.toJSONString(reasonsList));
}
//int update_r = blackGreyListMapper.updateBlackGreyListResult(updateResult);
//测试不操作数据
int update_r = 0;
log.warn("测试不操作数据,updateBlackGreyListResult:{}", JSONObject.toJSONString(updateResult));
int update_r = blackGreyListMapper.updateBlackGreyListResult(updateResult);
if(update_d>0 && update_r>0){
remove_count.getAndIncrement();
}
......@@ -484,9 +468,7 @@ public class BlackToGreyListParallel implements ParallelComputingProcess<BlackGr
detailUpdateParam.setRId(rJoinDVo.getRId());
detailUpdateParam.setWEqType(rJoinDVo.getType());
detailUpdateParam.setWEqStatus(0);
//int update = blackGreyListMapper.updateDetailsStatusByparams(detailUpdateParam);
int update = 1;
log.warn("测试不操作数据, log_inf: {} , updateDetailStatus10:{}", log_inf, JSONObject.toJSONString(detailUpdateParam));
int update = blackGreyListMapper.updateDetailsStatusByparams(detailUpdateParam);
if(update>0){
//用于修改结果表标记
rJoinDVo.setStatus(10);
......
src/main/java/cn/quantgroup/qgblservice/service/impl/UpdateBlackListOverdueDayParallel.java
View file @ 7fe229af
......@@ -44,7 +44,8 @@ public class UpdateBlackListOverdueDayParallel implements ParallelComputingProce
//并行处理框架
private ParallelComputing<BlackGreyListResult, Set<String>> p = new ParallelComputing<BlackGreyListResult, Set<String>>("UpdateBlackListOverdueDay");
private static int THREAD_COUNT = ConstantBlackGrey.PARAM.SYS_CPU_COUNT * 2;
//private static int THREAD_COUNT = ConstantBlackGrey.PARAM.SYS_CPU_COUNT * 2;
private static int THREAD_COUNT = 5;
@Autowired
private BlackGreyListMapper blackGreyListMapper;
......@@ -62,7 +63,7 @@ public class UpdateBlackListOverdueDayParallel implements ParallelComputingProce
public String runMain(List<BlackGreyListResult> tmpQueryList) {
long startProcess = System.currentTimeMillis();
int listSize = tmpQueryList.size();
log.info(log_inf+"-runMain更新逾期天数开始执行, listSize: {} , SYS_CPU_COUNT: {} ", listSize, THREAD_COUNT);
log.info(log_inf+"-runMain更新逾期天数开始执行>>>>>>, listSize: {} , SYS_CPU_COUNT: {} ", listSize, THREAD_COUNT);
try {
//并行框架---执行
......@@ -71,12 +72,11 @@ public class UpdateBlackListOverdueDayParallel implements ParallelComputingProce
//并行框架执行结果
if (optSet.size() > 0) {//并行框架执行异常
log.error("黑名单转灰名单runMain() ERROR! >> optSet size:{} , Json:{} ", optSet.size(), JSONObject.toJSONString(optSet));
log.info(log_inf + "-runMain更新逾期天数执行结束-部分失败, cost: {} <<<<<< process optSet size: {} , json: {} ", (System.currentTimeMillis()-startProcess), optSet.size(), JSONObject.toJSONString(optSet));
} else {
//执行成功
log.info(log_inf + " >> to process() is OK!");
log.info(log_inf + "-runMain更新逾期天数执行结束-成功, cost: {} <<<<<< process is OK!", (System.currentTimeMillis() - startProcess));
}
log.info(log_inf +"-runMain更新逾期天数执行结束, cost:" + (System.currentTimeMillis() - startProcess) + "ms.");
String resMsgTemp = "更新逾期天数总共 %d条, 执行成功 %d条, 失败 %d条;";
return String.format(resMsgTemp, listSize, listSize-optSet.size(), optSet.size());
......@@ -89,6 +89,7 @@ public class UpdateBlackListOverdueDayParallel implements ParallelComputingProce
@Override
public void process(int threadId, List<BlackGreyListResult> blackResultList, Set<String> optSet) {
long start = System.currentTimeMillis();
//yyyyMMdd
String dateName = ConstantBlackGrey.PARAM.DF_YMD.format(Calendar.getInstance().getTime());
int list_size = blackResultList.size();
......@@ -228,11 +229,10 @@ public class UpdateBlackListOverdueDayParallel implements ParallelComputingProce
err_count.getAndIncrement();
log.error("查询黑名单逾期天-发现uuid为空, threadId_{} , r_Id: {} , uuid: {} , blackType: {} ", threadId, rId, uuid, blackResult.getBlackType());
}
}
log.info(log_inf + "处理结束 CPU_{} , update_count:{} , type_11: {} , err_count:{} , this_list_size: {} ", threadId, update_count.get(), type_11.get(), err_count.get(), list_size);
}
log.info(log_inf + "更新黑名单逾期天数process处理结束CPU_{} , update_count: {} , type_11: {} , err_count: {} , this_list_size: {} , 耗时: {} ", threadId, update_count.get(), type_11.get(), err_count.get(), list_size, (System.currentTimeMillis()-start));
}
......@@ -286,12 +286,8 @@ public class UpdateBlackListOverdueDayParallel implements ParallelComputingProce
//detailUpdateParam.setWEqType(detailsVo.getType());
//detailUpdateParam.setWEqStatus(0);
//int update = blackGreyListMapper.updateDetailOverdueDayById(detailUpdateParam);
log.warn("测试不操作数据, log_inf: {} , updateOverdueDay:{}", log_inf, JSONObject.toJSONString(detailUpdateParam));
//测试不操作数据
int update = 1;
int update = blackGreyListMapper.updateDetailOverdueDayById(detailUpdateParam);
return update;
}
} catch (Exception e) {
log.error("修改[" + log_inf + "]黑名单逾期天数-异常, r_Id: {} , id: {} , overdueDays: {} ", detailsVo.getRId(), detailsVo.getId(), overdueDays!=null?JSONObject.toJSONString(overdueDays):"null", e);
......@@ -307,8 +303,7 @@ public class UpdateBlackListOverdueDayParallel implements ParallelComputingProce
BlackGreyListResult updateResult = new BlackGreyListResult();
updateResult.setRId(rId);//条件
updateResult.setUpdatedAt(new Timestamp(System.currentTimeMillis()));//结果
//测试不操作数据
//update_r = blackGreyListMapper.updateBlackGreyListResult(updateResult);
update_r = blackGreyListMapper.updateBlackGreyListResult(updateResult);
}catch (Exception e){
log.error("异步更新黑灰名单结果表-updatedAt时间异常, rId : {} ", rId, e);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment