检测transaction_log重复工具

src/main/java/cn/quantgroup/report/controller/ManualToolController.java

@@ -164,6 +164,11 @@ public class ManualToolController {
     cleanningTransactionLogService.checkCallRecordCF(newYnrTime,isExecuteOnce);
     return "checkCallRecordCF调度完成";
   }
+  @RequestMapping("/checkTransactionLogCF")
+  public String checkTransactionLogCF(String newYnrTime,String isExecuteOnce){
+    cleanningTransactionLogService.checkTransactionLogCF(newYnrTime,isExecuteOnce);
+    return "checkTransactionLogCF调度完成";
+  }
 
   @RequestMapping("/checkCallRecordCFByTransactionLog")
   public String checkCallRecordCFByTransactionLog(String newYnrTime,String isExecuteOnce){

src/main/java/cn/quantgroup/report/service/manualTool/CleanningTransactionLogService.java

@@ -483,7 +483,7 @@ public class CleanningTransactionLogService {
      * 创建时间: 2020.01.02  <br/>
      */
     @Async
-    public void checkCallRecordCF(String newYnrTime, String isExecuteOnce) {//yyyy-MM-dd
+    public void checkCallRecordCF(String newYnrTime, String isExecuteOnce) { //yyyy-MM-dd
         try {
             boolean executeOnce = false;
             if(StringUtils.isNotEmpty(isExecuteOnce) && "true".equals(isExecuteOnce)){
@@ -554,6 +554,77 @@ public class CleanningTransactionLogService {
     }
 
 
+    @Async
+    public void checkTransactionLogCF(String newYnrTime, String isExecuteOnce) { //yyyy-MM-dd
+        try {
+            boolean executeOnce = false;
+            if(StringUtils.isNotEmpty(isExecuteOnce) && "true".equals(isExecuteOnce)){
+                executeOnce = true;
+            }
+
+            SimpleDateFormat simpleDateFormat1 = new SimpleDateFormat("yyyy-MM-dd");
+            LocalDateTime now = new Timestamp(simpleDateFormat1.parse(newYnrTime).getTime()).toLocalDateTime();
+
+            //开始时间 2018-10-17 11:30:39
+            //结束时间 2019-12-31 10:36:34
+            //相差440天
+            for (int i = 0; i < 500; i++) {
+                String startTime = now.minusDays(i+1).format(DateTimeFormatter.ISO_DATE);
+                String endTime = now.minusDays(i).format(DateTimeFormatter.ISO_DATE);
+
+                Stopwatch callRStopwatch = Stopwatch.createStarted();
+                String CALL_SQL = "select transaction_id, uuid, url_type, code, created_at " +
+                        " from transaction_log t where t.time_created >= '" + startTime + "' and t.time_created < '" + endTime + "'";
+                List<TransactionLogPO> queryResult = riskDatasourceJdbcTemplate.query(CALL_SQL, new BeanPropertyRowMapper<>(TransactionLogPO.class));
+                log.info("checkTransactionLogCF检测重复transaction_log查询数据结束, startTime: {} , endTime: {} , size: {} , sql: {} , 耗时: {} ", startTime, endTime, CollectionUtils.isEmpty(queryResult) ? 0 : queryResult.size(), CALL_SQL, callRStopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
+
+                Map<String, Integer> callRecord1Map = new HashMap<>();
+
+                if (CollectionUtils.isNotEmpty(queryResult)) {
+                    Stopwatch callStopwatch = Stopwatch.createStarted();
+                    for (int call = 0; call < queryResult.size(); call++) {
+                        TransactionLogPO callRecord1 = queryResult.get(call);
+                        String format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(callRecord1.getTimeCreated());
+                        String key = new StringBuffer(callRecord1.getTransactionId())
+                                .append(StringUtils.isNotBlank(callRecord1.getUuid()) ? callRecord1.getUuid() : "")
+                                .append(callRecord1.getUrlType())
+                                .append(callRecord1.getCode())
+                                .append(format).toString();
+
+                        if(callRecord1Map.containsKey(key)){
+                            callRecord1Map.put(key, callRecord1Map.get(key).intValue()+1);
+                        }else{
+                            callRecord1Map.put(key, 1);
+                        }
+                    }
+                    log.info("checkTransactionLogCF检测重复transaction组装数据完成, startTime: {} , 查询List大小: {} , 组装后Map大小: {} , 耗时: {} ", startTime, queryResult.size(), callRecord1Map.size(), callStopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
+
+                    Stopwatch delStopwatch = Stopwatch.createStarted();
+                    Iterator<Map.Entry<String, Integer>> iterator = callRecord1Map.entrySet().iterator();
+                    while (iterator.hasNext()) {
+                        Map.Entry<String, Integer> next = iterator.next();
+                        String key = next.getKey();
+                        Integer value = next.getValue();
+                        if (value!=null && value.intValue() >1 ) {
+                            log.info("checkTransactionLogCF检测重复发现有重复数据, startTime: {} , 重复次数value: {} , key: {} ", startTime, value, key);
+                        }else{
+                            iterator.remove();
+                        }
+                    }
+                    log.info("checkTransactionLogCF检测重复数据结束, startTime: {} , 查询List大小: {} , 过滤后transaction Map大小: {} , 耗时: {} ", startTime, queryResult.size(), callRecord1Map.size(), delStopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
+                }
+
+                if(executeOnce){
+                    log.info("checkTransactionLogCF检测重复一天数据完成, newYnrTime: {} , isExecuteOnce: {} , startTime: {} , endTime: {} ,",newYnrTime, isExecuteOnce, startTime, endTime);
+                    break;
+                }
+            }
+            log.info("----checkTransactionLogCF---All检测重复数据结束----");
+        } catch (Exception e) {
+            log.error("checkTransactionLogCF检测重复数据异常", e);
+        }
+    }
+
     @Async
     public void checkCallRecordCFByTransactionLog(String newYnrTime, String isExecuteOnce) {//yyyy-MM-dd
         try {