检测tidb的call_record表数据是否有重复数据工具

src/main/java/cn/quantgroup/report/controller/ManualToolController.java

@@ -159,4 +159,10 @@ public class ManualToolController {
     return "synCallRecordNew调度完成";
   }
 
+  @RequestMapping("/checkCallRecordCF")
+  public String checkCallRecordCF(String newYnrTime,String isExecuteOnce){
+    cleanningTransactionLogService.checkCallRecordCF(newYnrTime,isExecuteOnce);
+    return "checkCallRecordCF调度完成";
+  }
+
 }

src/main/java/cn/quantgroup/report/service/manualTool/CleanningTransactionLogService.java

@@ -467,4 +467,83 @@ public class CleanningTransactionLogService {
             log.error("清洗数据异常", e);
         }
     }
+
+
+    /**
+     * 描述: 检测tidb的call_record表数据是否有重复数据 <br/>
+     * 参数: [newYnrTime, isExecuteOnce]  <br/>
+     * 返回值: void  <br/>
+     * 创建人: yanhui.Hao  <br/>
+     * 创建时间: 2020.01.02  <br/>
+     */
+    @Async
+    public void checkCallRecordCF(String newYnrTime, String isExecuteOnce) {//yyyy-MM-dd
+        try {
+            boolean executeOnce = false;
+            if(StringUtils.isNotEmpty(isExecuteOnce) && "true".equals(isExecuteOnce)){
+                executeOnce = true;
+            }
+
+            SimpleDateFormat simpleDateFormat1 = new SimpleDateFormat("yyyy-MM-dd");
+            LocalDateTime now = new Timestamp(simpleDateFormat1.parse(newYnrTime).getTime()).toLocalDateTime();
+
+            //开始时间 2018-10-17 11:30:39
+            //结束时间 2019-12-31 10:36:34
+            //相差440天
+            for (int i = 0; i < 500; i++) {
+                String startTime = now.minusDays(i+1).format(DateTimeFormatter.ISO_DATE);
+                String endTime = now.minusDays(i).format(DateTimeFormatter.ISO_DATE);
+
+                Stopwatch callRStopwatch = Stopwatch.createStarted();
+                String CALL_SQL = "select transaction_id, uuid, url_type, code, created_at " +
+                        " from call_record where created_at >= '" + startTime + "' and created_at < '" + endTime + "'";
+                List<CallRecord1> queryResult = tidbRiskJdbcTemplate.query(CALL_SQL, new BeanPropertyRowMapper<>(CallRecord1.class));
+                log.info("callRecord查询数据结束, startTime: {} , endTime: {} , size: {} , sql: {} , 耗时: {} ", startTime, endTime, CollectionUtils.isEmpty(queryResult) ? 0 : queryResult.size(), CALL_SQL, callRStopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
+
+                Map<String, Integer> callRecord1Map = new HashMap<>();
+
+                if (CollectionUtils.isNotEmpty(queryResult)) {
+                    Stopwatch callStopwatch = Stopwatch.createStarted();
+                    for (int call = 0; call < queryResult.size(); call++) {
+                        CallRecord1 callRecord1 = queryResult.get(call);
+                        String format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(callRecord1.getCreated_at().getTime()));
+                        String key = new StringBuffer(callRecord1.getTransactionId())
+                                .append(StringUtils.isNotBlank(callRecord1.getUuid()) ? callRecord1.getUuid() : "")
+                                .append(callRecord1.getUrlType())
+                                .append(callRecord1.getCode())
+                                .append(format).toString();
+
+                        if(callRecord1Map.containsKey(key)){
+                            callRecord1Map.put(key, callRecord1Map.get(key).intValue()+1);
+                        }else{
+                            callRecord1Map.put(key, 1);
+                        }
+                    }
+                    log.info("callRecord组装数据完成, startTime: {} , 查询List大小: {} , 组装后Map大小: {} , 耗时: {} ", startTime, queryResult.size(), callRecord1Map.size(), callStopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
+
+                    Stopwatch delStopwatch = Stopwatch.createStarted();
+                    Iterator<Map.Entry<String, Integer>> iterator = callRecord1Map.entrySet().iterator();
+                    while (iterator.hasNext()) {
+                        Map.Entry<String, Integer> next = iterator.next();
+                        String key = next.getKey();
+                        Integer value = next.getValue();
+                        if (value!=null && value.intValue() >1 ) {
+                            log.info("检测发现有重复数据, startTime: {} , 重复次数value: {} , key: {} ", startTime, value, key);
+                        }else{
+                            iterator.remove();
+                        }
+                    }
+                    log.info("检测重复数据结束, startTime: {} , 查询List大小: {} , 过滤后callRecord1Map大小: {} , 耗时: {} ", startTime, queryResult.size(), callRecord1Map.size(), delStopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
+                }
+
+                if(executeOnce){
+                    log.info("一天数据插入完成, newYnrTime: {} , isExecuteOnce: {} , startTime: {} , endTime: {} ,",newYnrTime, isExecuteOnce, startTime, endTime);
+                    break;
+                }
+            }
+            log.info("----All检测重复数据结束----");
+        } catch (Exception e) {
+            log.error("检测重复数据异常", e);
+        }
+    }
 }