检测tidb的call_record表数据是否有重复数据工具

src/main/java/cn/quantgroup/report/controller/ManualToolController.java

@@ -177,4 +177,10 @@ public class ManualToolController {
     return "checkCallRecordCFByTransactionLog2调度完成";
   }
 
+  @RequestMapping("/deleteTidbCallRecordCF")
+  public String deleteTidbCallRecordCF(String cfFileName, String bakFileName){
+    cleanningTransactionLogService.deleteTidbCallRecordCF(cfFileName,bakFileName);
+    return "deleteTidbCallRecordCF调度完成";
+  }
+
 }

src/main/java/cn/quantgroup/report/domain/tidbrisk/CallRecord3.java

@@ -33,4 +33,7 @@ public class CallRecord3 implements Serializable {
     private Timestamp createdAt;
     private Timestamp updatedAt;
 
+
+    private Long id;
+
 }

src/main/java/cn/quantgroup/report/service/manualTool/CleanningTransactionLogService.java

@@ -4,6 +4,7 @@ import cn.quantgroup.report.domain.master.CallRecord1;
 import cn.quantgroup.report.domain.master.TransactionLogPO;
 import cn.quantgroup.report.domain.tidbrisk.CallRecord3;
 import cn.quantgroup.report.utils.JdbcUtils;
+import cn.quantgroup.report.utils.ReadOrWriteTxt;
 import com.alibaba.fastjson.JSON;
 import com.google.common.base.Stopwatch;
 import lombok.extern.slf4j.Slf4j;
@@ -783,4 +784,71 @@ public class CleanningTransactionLogService {
             log.error("比较数据异常", e);
         }
     }
+
+
+    @Async
+    public void deleteTidbCallRecordCF(String cfFileName, String bakFileName) {
+        try {
+            List<String> tidbCallRecordCFList = ReadOrWriteTxt.readTxtList(cfFileName);
+            log.info("读取文件cfFileName: {} , tidbCallRecordCFList大小:{} ",cfFileName,tidbCallRecordCFList.size());
+
+            for (int i = 0; i < tidbCallRecordCFList.size(); i++) {
+                //ee86810a076d45d58a5d87d7cfc60e8f,032f14da-b905-4a1c-9c38-9db3dd0b1304,ZhiChengAFuAntiFraud,1004,2019-12-24 21:48:53
+                String[] arry = tidbCallRecordCFList.get(i).trim().split(",");
+                Stopwatch callRStopwatch = Stopwatch.createStarted();
+
+                String DELETE_CALL_SQL = "delete from call_record where id=##ID## AND transaction_id ='##TRANSACTION_ID##';";
+
+                String CALL_SQL = "select * from call_record where transaction_id ='" + arry[0] + "' and uuid ='" + arry[1] +"'"
+                        +" and url_type ='" + arry[2] + "' and code ='" +arry[3]+ "'; ";
+                List<CallRecord3> queryResult = tidbRiskJdbcTemplate.query(CALL_SQL, new BeanPropertyRowMapper<>(CallRecord3.class));
+                if(queryResult!=null && queryResult.size()==2){
+                    CallRecord3 bean1 = queryResult.get(0);
+                    CallRecord3 bean2 = queryResult.get(1);
+                    int second = Math.abs(bean1.getCreatedAt().compareTo(bean2.getCreatedAt()));
+                    if(second <=3){
+                        if(StringUtils.isEmpty(bean1.getRequestUrl()) && StringUtils.isEmpty(bean1.getChannelId())
+                                && StringUtils.isNotEmpty(bean2.getRequestUrl()) && StringUtils.isNotEmpty(bean2.getChannelId())){
+
+                            String sql = DELETE_CALL_SQL.replace("##ID##", ""+bean1.getId());
+                            sql = sql.replace("##TRANSACTION_ID##", bean1.getTransactionId());
+                            try {
+                                int update =tidbRiskJdbcTemplate.update(sql);
+                                log.info("删除bean1成功, sql: {} ,update: {} ",sql,update);
+                                FileUtils.write(new File(bakFileName), JSON.toJSONString(bean1)+"\r\n", "UTF-8", true);
+                            } catch (IOException e) {
+                                log.error("删除bean1数据是不, sql: {} ",sql, e);
+                            }
+
+                        }else if(StringUtils.isEmpty(bean2.getRequestUrl()) && StringUtils.isEmpty(bean2.getChannelId())
+                                && StringUtils.isNotEmpty(bean1.getRequestUrl()) && StringUtils.isNotEmpty(bean1.getChannelId())){
+
+                            String sql = DELETE_CALL_SQL.replace("##ID##", ""+bean2.getId());
+                            sql = sql.replace("##TRANSACTION_ID##", bean2.getTransactionId());
+                            try {
+                                int update = tidbRiskJdbcTemplate.update(sql);
+                                log.info("删除bean2成功, sql: {} , update: {} ",sql,update);
+                                FileUtils.write(new File(bakFileName), JSON.toJSONString(bean2)+"\r\n", "UTF-8", true);
+                            } catch (IOException e) {
+                                log.error("删除bean2数据是不, sql: {} ",sql, e);
+                            }
+
+                        }else {
+                            log.info("查询call_record表,发现其他情况, param: {} , List: {} ", tidbCallRecordCFList.get(i), (queryResult!=null ? JSON.toJSONString(queryResult): "null") );
+                        }
+
+                    }else{
+                        log.info("查询call_record表,相差时间大于3秒, param: {} , List: {} ", tidbCallRecordCFList.get(i), (queryResult!=null ? JSON.toJSONString(queryResult): "null") );
+                    }
+                }else{
+                    log.info("查询call_record表,list大小不为2, param: {} , List: {} ", tidbCallRecordCFList.get(i), (queryResult!=null ? JSON.toJSONString(queryResult): "null") );
+                }
+            }
+            log.info("----删除bean数据完成----");
+
+        } catch (Exception e) {
+            log.error("删除bean数据异常", e);
+        }
+    }
+
 }

src/main/java/cn/quantgroup/report/service/manualTool/ManualToolService.java

@@ -34,8 +34,12 @@ import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Service;
 
 import java.io.*;
+import java.sql.Timestamp;
 import java.text.DateFormat;
+import java.text.ParseException;
 import java.text.SimpleDateFormat;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
 import java.util.*;
 
 /**
@@ -2609,6 +2613,100 @@ public class ManualToolService implements CommonSuperService {
         }
     }
 
+    private static void transaction_call2() {
+        try{
+            String path = "D:\\JavaTeam\\test\\transaction_call\\";
+            List<String> jsonList_1 = ReadOrWriteTxt.readTxtList(path + "not_transaction_log.txt");
+            HashMap<String,String> map_transaction = new HashMap<>();
+            int i=0;
+            for(String str : jsonList_1){
+                map_transaction.put(str.substring(0,str.length()-6), ""+(++i));
+            }
+            log.info("read files jsonList_1 size:{} , map_transaction siez:{} ", jsonList_1.size(), map_transaction.size());
+            jsonList_1 = null;
+
+            manualGC();
+
+            List<String> jsonList_2 = ReadOrWriteTxt.readTxtList(path + "not_call_record.txt");
+            HashMap<String,String> map_call = new HashMap<>();
+            int j=0;
+            for(String str : jsonList_2){
+                map_call.put(str.substring(0,str.length()-6), ""+(++j));
+            }
+            log.info("read files jsonList_2 size:{} , map_call siez:{} ", jsonList_2.size(), map_call.size());
+            jsonList_2 = null;
+
+            manualGC();
+
+
+            Iterator<Map.Entry<String, String>> iterator_call = map_call.entrySet().iterator();
+            while (iterator_call.hasNext()) {
+                Map.Entry<String, String> next = iterator_call.next();
+                String key = next.getKey();
+
+                if (map_transaction.containsKey(key)) {
+                    iterator_call.remove();
+                    map_transaction.remove(key);
+                }
+            }
+            log.info("去重过滤后 map_transaction size:{} , map_call siez:{} ", map_transaction.size(), map_call.size());
+
+            for(String str1 : map_transaction.keySet()){
+                FileUtils.write(new File(path + "2not_transaction_log.txt"), str1+"\n", "UTF-8", true);
+            }
+
+            for(String str2 : map_call.keySet()){
+                FileUtils.write(new File(path + "2not_call_record.txt"), str2+"\n", "UTF-8", true);
+            }
+
+            log.info("All去重过滤后保存放款记录结束");
+
+        }catch (Exception e){
+            log.error("处理放款记录异常！", e);
+        }
+    }
+
+
+    private static void transaction_call3() {
+        try{
+            List<String> d2JsonList = ReadOrWriteTxt.readTxtList("D:\\JavaTeam\\test\\transaction_call\\比对ok的.txt");
+            log.info("files d2JsonList size="+d2JsonList.size());
+
+            HashMap<String,String> map_transaction = new HashMap<>();
+            for(int i=0; i<d2JsonList.size(); i++){
+                String str = d2JsonList.get(i);
+                if(str.length()<2){
+                    continue;
+                }
+                map_transaction.put(str.split(" , ")[0], ""+i);
+            }
+
+            SimpleDateFormat simpleDateFormat1 = new SimpleDateFormat("yyyy-MM-dd");
+            LocalDateTime now = new Timestamp(simpleDateFormat1.parse("2020-01-01").getTime()).toLocalDateTime();
+
+            //开始时间 2018-10-17 11:30:39
+            //结束时间 2019-12-31 10:36:34
+            //相差440天
+            for (int i = 0; i < 500; i++) {
+                String startTime = now.minusDays(i + 1).format(DateTimeFormatter.ISO_DATE);
+
+                if(!map_transaction.containsKey(startTime)){
+                    log.info("err Data:{}",startTime);
+                }
+
+                if(startTime.equals("2018-10-17")){
+                    break;
+                }
+            }
+
+            log.info("All D2 LOG:保存放款记录结束");
+
+        }catch (Exception e){
+            log.error("处理放款记录异常！", e);
+        }
+    }
+
+
 
 
 
@@ -2681,7 +2779,27 @@ public class ManualToolService implements CommonSuperService {
 
         //build_d2_1226();
 
-        transaction_call();
+        //transaction_call();
+        //transaction_call2();
+        //transaction_call3();
+
+
+
+
+        try {
+            SimpleDateFormat simpleDateFormat1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+            //Timestamp time1 = new Timestamp(System.currentTimeMillis());
+            Timestamp time1 = new Timestamp(simpleDateFormat1.parse("2019-12-24 15:18:07").getTime());
+            Timestamp time2 = new Timestamp(simpleDateFormat1.parse("2019-12-24 15:18:09").getTime());
+
+            System.out.println(Math.abs(time1.compareTo(time2)));
+            System.out.println(time2.compareTo(time1));
+
+        } catch (ParseException e) {
+            e.printStackTrace();
+        }
+
+
 
     }