数据处理-批量插入异常

src/main/java/cn/quantgroup/report/controller/BaiHangDataDisposeController.java

+package cn.quantgroup.report.controller;
+
+import cn.quantgroup.report.service.baihang.BaiHangDataDisposeService;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+@Slf4j
+@RestController
+@RequestMapping(value = "/baiHangDataDispose")
+public class BaiHangDataDisposeController {
+
+  @Autowired
+  private BaiHangDataDisposeService baiHangDataDisposeService;
+
+  @PostMapping("/fixA1RefuseDataCauseByBatchSave")
+  public String fixA1RefuseDataCauseByBatchSave(String plainTextFileName,Boolean saveDb){
+    try {
+      baiHangDataDisposeService.fixA1RefuseDataCauseByBatchSave(plainTextFileName,saveDb);
+      return "SUCCESS";
+    } catch (Exception e){
+      return e.getMessage();
+    }
+  }
+
+}

src/main/java/cn/quantgroup/report/service/baihang/BaiHangDataDisposeService.java

+package cn.quantgroup.report.service.baihang;
+
+import cn.hutool.core.io.IoUtil;
+import cn.hutool.json.JSONUtil;
+import cn.quantgroup.report.domain.baihang.ApplyLoanInfoZhuDai;
+import cn.quantgroup.report.mapper.master.ApplyLoanInfoZhuDaiMapper;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.BeanUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.annotation.Async;
+import org.springframework.stereotype.Service;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+
+@Slf4j
+@Service
+public class BaiHangDataDisposeService {
+
+    private static final String HOST_PATH = "/home/quant_group/reportFile/";
+
+    @Autowired
+    private BaiHangFileReportService baiHangFileReportService;
+    @Autowired
+    public ApplyLoanInfoZhuDaiMapper applyLoanInfoZhuDaiMapper;
+
+    @Async
+    public void fixA1RefuseDataCauseByBatchSave(String plainTextFileName,Boolean saveDb){
+        BufferedReader bis = null;
+        String dataStr = null;
+        List<ApplyLoanInfoZhuDai> zhuDaiLogList = null;
+        ApplyLoanInfoZhuDai record = null;
+        AtomicInteger atomicInteger = new AtomicInteger();
+        try {
+            bis = new BufferedReader(new FileReader(HOST_PATH + plainTextFileName));
+            while(null != (dataStr = bis.readLine())) {
+                if (StringUtils.isNotBlank(dataStr) && !baiHangFileReportService.hasBaiHangFileHeadMark(dataStr)) {
+                    ApplyLoanInfoZhuDai applyLoanInfoZhuDai = JSONUtil.toBean(dataStr, ApplyLoanInfoZhuDai.class);
+                    zhuDaiLogList = applyLoanInfoZhuDaiMapper.findByApplyId(applyLoanInfoZhuDai.getApplyId());
+                    if (CollectionUtils.isEmpty(zhuDaiLogList)) {
+                        atomicInteger.getAndIncrement();
+                        if (saveDb) {
+                            record = new ApplyLoanInfoZhuDai();
+                            BeanUtils.copyProperties(applyLoanInfoZhuDai, record);
+                            record.setRecordId(UUID.randomUUID().toString().replaceAll("-", ""));
+                            applyLoanInfoZhuDaiMapper.saveApplyLoanInfoZhuDaiLog(record);
+                        }
+                    }
+                }
+            }
+            log.error("fixA1RefuseDataCauseByBatchSave处理数据结束,处理总量={}",atomicInteger.get());
+        } catch (Exception e) {
+            log.error("fixA1RefuseDataCauseByBatchSave处理数据异常",e.getMessage());
+        } finally {
+            IoUtil.close(bis);
+            dataStr = null;
+        }
+    }
+
+}

src/main/java/cn/quantgroup/report/service/baihang/BaiHangFileReportService.java

@@ -19,6 +19,7 @@ import cn.quantgroup.report.service.baihang.response.FileUploadResponse;
 import cn.quantgroup.report.service.baihang.util.TuoMinUtils;
 import cn.quantgroup.report.service.http.IHttpService;
 import cn.quantgroup.report.utils.DateUtils;
+import cn.quantgroup.report.utils.IdUtils;
 import cn.quantgroup.report.utils.SftpUtil;
 import cn.quantgroup.report.utils.dingtalk.DingTalk;
 import com.alibaba.fastjson.JSON;
@@ -249,7 +250,7 @@ public class BaiHangFileReportService {
                     log.error("量化派助贷TO百行报送(A1)-贷款申请信息异常, reqId_log: {} ", reqId_log, e);
                 }
             }
-            File file = new File(HOST_PATH+"量子数科科技有限公司_A1_"+fileNameReplaceAll(starTime.split("T")[0])+"_"+fileNameReplaceAll(endTime.split("T")[0])+"_"+String.format("%06d",new Random().nextInt(999999))+".txt");
+            File file = new File(HOST_PATH+"量子数科科技有限公司_A1_"+fileNameReplaceAll(starTime.split("T")[0])+"_"+fileNameReplaceAll(endTime.split("T")[0])+"_"+IdUtils.genRandom(7)+".txt");
             FileUtils.writeLines(file,reportList);
             if (reportList.size()<=1) {
                 log.info("量化派助贷TO百行报送(A1)-贷款申请信息为空,加密文件不生成!starTime={},endTime={}",starTime,endTime);
@@ -326,7 +327,7 @@ public class BaiHangFileReportService {
             } else {
                 encryptFileName = createFile(file.getAbsolutePath());
             }
-            log.info("量化派助贷TO百行报送(A1Refuse)-贷款申请信息完成, 实际大小: {} , 报送成功大小: {} , 耗时: {} ", applyLoanInfos.size(), atomicInteger.get(), sendWatch.stop().elapsed(TimeUnit.MILLISECONDS));
+            log.info("量化派助贷TO百行报送(A1Refuse)处理完成, 实际大小: {} , 报送成功大小: {} , 耗时: {} ", applyLoanInfos.size(), atomicInteger.get(), sendWatch.stop().elapsed(TimeUnit.MILLISECONDS));
         } catch (Exception e) {
             log.error("量化派助贷TO百行报送(A1Refuse)-异常", e);
         } finally {
@@ -406,7 +407,7 @@ public class BaiHangFileReportService {
                 }
             }
             saveLoanInfoRecordLog(recordList,starTime,endTime);
-            File file = new File(HOST_PATH+"量子数科科技有限公司_D2_"+fileNameReplaceAll(starTime.split("T")[0])+"_"+fileNameReplaceAll(endTime.split("T")[0])+"_"+String.format("%06d",new Random().nextInt(999999))+".txt");
+            File file = new File(HOST_PATH+"量子数科科技有限公司_D2_"+fileNameReplaceAll(starTime.split("T")[0])+"_"+fileNameReplaceAll(endTime.split("T")[0])+"_"+IdUtils.genRandom(7)+".txt");
             FileUtils.writeLines(file,reportList);
             if (reportList.size()<=1) {
                 log.info("量化派助贷TO百行报送-放款信息为空,加密文件不生成!starTime={},endTime={}",starTime,endTime);
@@ -586,7 +587,7 @@ public class BaiHangFileReportService {
                 }
             }
             saveRepaymentLoanInfoLog(recordList,startTime,endTime);
-            File file = new File(HOST_PATH+"量子数科科技有限公司_D3O_"+fileNameReplaceAll(startTime.split("T")[0])+"_"+fileNameReplaceAll(endTime.split("T")[0])+"_"+String.format("%06d",new Random().nextInt(999999))+".txt");
+            File file = new File(HOST_PATH+"量子数科科技有限公司_D3O_"+fileNameReplaceAll(startTime.split("T")[0])+"_"+fileNameReplaceAll(endTime.split("T")[0])+"_"+IdUtils.genRandom(7)+".txt");
             FileUtils.writeLines(file,reportList);
             if (reportList.size()<=1) {
                 log.info("量化派助贷TO百行报送(D3)-逾期记录为空,加密文件不生成!starTime={},endTime={}",startTime,endTime);
@@ -660,7 +661,7 @@ public class BaiHangFileReportService {
                 }
             }
             saveRepaymentLoanInfoLog(recordList,startTime,endTime);
-            File file = new File(HOST_PATH+"量子数科科技有限公司_D3O30_"+fileNameReplaceAll(startTime.split("T")[0])+"_"+fileNameReplaceAll(endTime.split("T")[0])+"_"+String.format("%06d",new Random().nextInt(999999))+".txt");
+            File file = new File(HOST_PATH+"量子数科科技有限公司_D3O30_"+fileNameReplaceAll(startTime.split("T")[0])+"_"+fileNameReplaceAll(endTime.split("T")[0])+"_"+IdUtils.genRandom(7)+".txt");
             FileUtils.writeLines(file,reportList);
             if (reportList.size()<=1) {
                 log.info("量化派助贷TO百行报送(D3)-每30天逾期记录为空,加密文件不生成!starTime={},endTime={}",startTime,endTime);
@@ -725,7 +726,7 @@ public class BaiHangFileReportService {
                 }
             }
             saveRepaymentLoanInfoLog(recordList,startTime,endTime);
-            File file = new File(HOST_PATH+"量子数科科技有限公司_D3R_"+fileNameReplaceAll(startTime.split("T")[0])+"_"+fileNameReplaceAll(endTime.split("T")[0])+"_"+String.format("%06d",new Random().nextInt(999999))+".txt");
+            File file = new File(HOST_PATH+"量子数科科技有限公司_D3R_"+fileNameReplaceAll(startTime.split("T")[0])+"_"+fileNameReplaceAll(endTime.split("T")[0])+"_"+IdUtils.genRandom(7)+".txt");
             FileUtils.writeLines(file,reportList);
             if (reportList.size()<=1) {
                 log.info("量化派助贷TO百行报送(D3)-还款记录为空,加密文件不生成!starTime={},endTime={}",startTime,endTime);
@@ -970,4 +971,12 @@ public class BaiHangFileReportService {
         log.info("sourceFileName:{} , targetFileName:{} 文件数据校验拷贝结束",sourceFileName,targetFileName);
     }
 
+    public Boolean hasBaiHangFileHeadMark (String dataStr) {
+        Boolean headMark = Boolean.FALSE;
+        if (StringUtils.equalsAny(dataStr,"#applyInfo","#singleLoanAccountInfo","#singleLoanRepayInfo")) {
+            headMark = Boolean.TRUE;
+        }
+        return headMark;
+    }
+
 }

src/main/java/cn/quantgroup/report/utils/IdUtils.java

@@ -93,4 +93,12 @@ public class IdUtils {
         }
     }
 
+    public static String genRandom(Integer maxLength) {
+        String radom = "";
+        for(int i = 0; i < maxLength; ++i) {
+            radom = radom + (int)(Math.random() * 10.0D);
+        }
+        return radom;
+    }
+
 }