检测tidb的call_record表数据是否有重复数据工具

src/main/java/cn/quantgroup/report/service/manualTool/CleanningTransactionLogService.java

@@ -504,7 +504,7 @@ public class CleanningTransactionLogService {
                 String CALL_SQL = "select transaction_id, uuid, url_type, code, created_at " +
                         " from call_record where created_at >= '" + startTime + "' and created_at < '" + endTime + "'";
                 List<CallRecord1> queryResult = tidbRiskJdbcTemplate.query(CALL_SQL, new BeanPropertyRowMapper<>(CallRecord1.class));
-                log.info("callRecord查询数据结束, startTime: {} , endTime: {} , size: {} , sql: {} , 耗时: {} ", startTime, endTime, CollectionUtils.isEmpty(queryResult) ? 0 : queryResult.size(), CALL_SQL, callRStopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
+                log.info("checkCallRecordCF检测重复callRecord查询数据结束, startTime: {} , endTime: {} , size: {} , sql: {} , 耗时: {} ", startTime, endTime, CollectionUtils.isEmpty(queryResult) ? 0 : queryResult.size(), CALL_SQL, callRStopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
 
                 Map<String, Integer> callRecord1Map = new HashMap<>();
 
@@ -525,7 +525,7 @@ public class CleanningTransactionLogService {
                             callRecord1Map.put(key, 1);
                         }
                     }
-                    log.info("callRecord组装数据完成, startTime: {} , 查询List大小: {} , 组装后Map大小: {} , 耗时: {} ", startTime, queryResult.size(), callRecord1Map.size(), callStopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
+                    log.info("checkCallRecordCF检测重复callRecord组装数据完成, startTime: {} , 查询List大小: {} , 组装后Map大小: {} , 耗时: {} ", startTime, queryResult.size(), callRecord1Map.size(), callStopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
 
                     Stopwatch delStopwatch = Stopwatch.createStarted();
                     Iterator<Map.Entry<String, Integer>> iterator = callRecord1Map.entrySet().iterator();
@@ -534,22 +534,22 @@ public class CleanningTransactionLogService {
                         String key = next.getKey();
                         Integer value = next.getValue();
                         if (value!=null && value.intValue() >1 ) {
-                            log.info("检测发现有重复数据, startTime: {} , 重复次数value: {} , key: {} ", startTime, value, key);
+                            log.info("checkCallRecordCF检测重复发现有重复数据, startTime: {} , 重复次数value: {} , key: {} ", startTime, value, key);
                         }else{
                             iterator.remove();
                         }
                     }
-                    log.info("检测重复数据结束, startTime: {} , 查询List大小: {} , 过滤后callRecord1Map大小: {} , 耗时: {} ", startTime, queryResult.size(), callRecord1Map.size(), delStopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
+                    log.info("checkCallRecordCF检测重复数据结束, startTime: {} , 查询List大小: {} , 过滤后callRecord1Map大小: {} , 耗时: {} ", startTime, queryResult.size(), callRecord1Map.size(), delStopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
                 }
 
                 if(executeOnce){
-                    log.info("一天数据插入完成, newYnrTime: {} , isExecuteOnce: {} , startTime: {} , endTime: {} ,",newYnrTime, isExecuteOnce, startTime, endTime);
+                    log.info("checkCallRecordCF检测重复一天数据完成, newYnrTime: {} , isExecuteOnce: {} , startTime: {} , endTime: {} ,",newYnrTime, isExecuteOnce, startTime, endTime);
                     break;
                 }
             }
-            log.info("----All检测重复数据结束----");
+            log.info("----checkCallRecordCF---All检测重复数据结束----");
         } catch (Exception e) {
-            log.error("检测重复数据异常", e);
+            log.error("checkCallRecordCF检测重复数据异常", e);
         }
     }
 

src/main/java/cn/quantgroup/report/service/manualTool/ManualToolService.java

@@ -2615,48 +2615,74 @@ public class ManualToolService implements CommonSuperService {
 
     private static void transaction_call2() {
         try{
-            String path = "D:\\JavaTeam\\test\\transaction_call\\";
-            List<String> jsonList_1 = ReadOrWriteTxt.readTxtList(path + "not_transaction_log.txt");
+            String path = "D:\\JavaTeam\\test\\transaction_call\\01_03\\";
+            List<String> jsonList_1 = ReadOrWriteTxt.readTxtList(path + "not_transaction_log.tmp");
             HashMap<String,String> map_transaction = new HashMap<>();
             int i=0;
             for(String str : jsonList_1){
-                map_transaction.put(str.substring(0,str.length()-6), ""+(++i));
+                str = str.trim();
+                //map_transaction.put(str.substring(0,str.length()-6), ""+(++i));
+                map_transaction.put(str , str.substring(str.lastIndexOf(",")+1, str.length()) );
             }
             log.info("read files jsonList_1 size:{} , map_transaction siez:{} ", jsonList_1.size(), map_transaction.size());
             jsonList_1 = null;
 
             manualGC();
 
-            List<String> jsonList_2 = ReadOrWriteTxt.readTxtList(path + "not_call_record.txt");
+            List<String> jsonList_2 = ReadOrWriteTxt.readTxtList(path + "not_call_record.tmp");
             HashMap<String,String> map_call = new HashMap<>();
             int j=0;
             for(String str : jsonList_2){
-                map_call.put(str.substring(0,str.length()-6), ""+(++j));
+                str = str.trim();
+                //map_call.put(str.substring(0,str.length()-6), ""+(++j));
+                map_call.put(str , str.substring(str.lastIndexOf(",")+1, str.length()) );
             }
             log.info("read files jsonList_2 size:{} , map_call siez:{} ", jsonList_2.size(), map_call.size());
             jsonList_2 = null;
 
             manualGC();
 
+            DateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
             
             Iterator<Map.Entry<String, String>> iterator_call = map_call.entrySet().iterator();
             while (iterator_call.hasNext()) {
                 Map.Entry<String, String> next = iterator_call.next();
                 String key = next.getKey();
+                String call_time = next.getValue();
+
+                Date date1 = format.parse(call_time);
+                date1.setTime(date1.getTime() + 1000);
+                String time_a1 = format.format(date1);
+
+                Date date2 = format.parse(call_time);
+                date2.setTime(date2.getTime() - 1000);
+                String time_j1 = format.format(date2);
+
+                String key_a1 = key.substring(0,key.lastIndexOf(","))+","+time_a1;
+                String key_j1 = key.substring(0,key.lastIndexOf(","))+","+time_j1;
 
                 if (map_transaction.containsKey(key)) {
                     iterator_call.remove();
                     map_transaction.remove(key);
+                }else if(map_transaction.containsKey(key_a1)){
+                    iterator_call.remove();
+                    map_transaction.remove(key_a1);
+                }else if(map_transaction.containsKey(key_j1)){
+                    iterator_call.remove();
+                    map_transaction.remove(time_j1);
+                }else{
+                    log.info("其他情况");
                 }
             }
             log.info("去重过滤后 map_transaction size:{} , map_call siez:{} ", map_transaction.size(), map_call.size());
 
             for(String str1 : map_transaction.keySet()){
-                FileUtils.write(new File(path + "2not_transaction_log.txt"), str1+"\n", "UTF-8", true);
+                FileUtils.write(new File(path + "2not_transaction_log.tmp"), str1+"\n", "UTF-8", true);
             }
+            log.info("写入map_transaction结束");
 
             for(String str2 : map_call.keySet()){
-                FileUtils.write(new File(path + "2not_call_record.txt"), str2+"\n", "UTF-8", true);
+                FileUtils.write(new File(path + "2not_call_record.tmp"), str2+"\n", "UTF-8", true);
             }
 
             log.info("All去重过滤后保存放款记录结束");
@@ -2780,25 +2806,12 @@ public class ManualToolService implements CommonSuperService {
         //build_d2_1226();
 
         //transaction_call();
-        //transaction_call2();
+        transaction_call2();
         //transaction_call3();
 
 
 
 
-        try {
-            SimpleDateFormat simpleDateFormat1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-            //Timestamp time1 = new Timestamp(System.currentTimeMillis());
-            Timestamp time1 = new Timestamp(simpleDateFormat1.parse("2019-12-24 15:18:07").getTime());
-            Timestamp time2 = new Timestamp(simpleDateFormat1.parse("2019-12-24 15:18:09").getTime());
-
-            System.out.println(Math.abs(time1.compareTo(time2)));
-            System.out.println(time2.compareTo(time1));
-
-        } catch (ParseException e) {
-            e.printStackTrace();
-        }
-
 
 
     }

src/main/java/cn/quantgroup/report/utils/JdbcUtils.java

@@ -96,5 +96,30 @@ public class JdbcUtils {
         }
     }
 
+    public static void batchInsertExecuteCallRecord(String sql, List<CallRecord3> callRecord3List){
+        Connection conn = null;
+        PreparedStatement ps = null;
+        try {
+            conn = HIK_DATA_SOURCE_TIDB_RISK.dataSourceJdbc.getConnection();
+            ps = conn.prepareStatement(sql);
+            conn.setAutoCommit(false);
+            for (int i = 0; i < callRecord3List.size(); i++) {
+                CallRecord3 callRecord3 = callRecord3List.get(i);
+                //delete from call_record where id=? AND transaction_id =?
+                ps.setLong(1, callRecord3.getId());
+                ps.setString(2, callRecord3.getTransactionId());
+                ps.addBatch();
+            }
+            ps.executeBatch();
+            conn.commit();
+        }catch(Exception e){
+            log.error("======执行batchInsertExecuteCallRecord异常======"+sql+"\r\n",e);
+            log.error("方法batchInsertExecuteCallRecord异常, DATA: {} \n", JSON.toJSON(callRecord3List));
+            e.printStackTrace();
+        }finally {
+            close(conn, ps,null);
+        }
+    }
+
 
 }