写文件用批量写入

src/main/java/cn/quantgroup/report/service/baihang/BaiHangZhuDaiService.java

@@ -909,6 +909,8 @@ public class BaiHangZhuDaiService {
 
         int counter = 0;
         log.info("开始>>整理百行-助贷还款&逾期存量数据 startDate : {} , noEndDate : {} ......",startDate,noEndDate);
+
+        StringBuffer  tmpWriteBuffer = null;
         while (true) {
             if(D3_Stop.get()){
                 log.error("All还款&逾期存量数据查询STOP, D3_Stop: {} , endTime: {} ",D3_Stop.get(), erlyDate.plusDays(counter).format(DateTimeFormatter.ISO_DATE));
@@ -919,6 +921,7 @@ public class BaiHangZhuDaiService {
             if (erlyDate.plusDays(counter).compareTo(endDate) > 0) {
                 break;
             }
+
             List<RepaymentInfoZhuDai> repaymentInfoList = new ArrayList<>();
             List<RepaymentInfoZhuDai> repaymentLoanInfosLh = new ArrayList<>();
 
@@ -931,11 +934,9 @@ public class BaiHangZhuDaiService {
                 long log_stUtc = System.currentTimeMillis();
                 try {
                     if (type == 0) {
-
                         if(StringUtils.isNotEmpty(d3Type) && d3Type.equals("1")){//只要逾期
                             continue;
                         }
-
                         //Stopwatch stopwatch = Stopwatch.createStarted();
                        /* repaymentInfoList = repaymentLoanInfoMapper.queryStockRepayMentInfoZhuDai(timeRecord);*/
 
@@ -951,7 +952,6 @@ public class BaiHangZhuDaiService {
                         }
 
                     } else {
-
                         if(StringUtils.isNotEmpty(d3Type) && d3Type.equals("0")){//只要还款
                             continue;
                         }
@@ -975,7 +975,8 @@ public class BaiHangZhuDaiService {
                         repaymentInfoList.addAll(repaymentLoanInfosLh);
                     }
 
-
+                    tmpWriteBuffer = new StringBuffer();
+                    int buffCount = 0;
                     for (int j = 0; j < repaymentInfoList.size(); j++) {
                         RepaymentInfoZhuDai repaymentInfo = repaymentInfoList.get(j);
                         String id = UUID.randomUUID().toString().replaceAll("-", "");
@@ -994,12 +995,20 @@ public class BaiHangZhuDaiService {
                             repaymentInfo.setUploadTs(repaymentInfo.getStatusConfirmAt());
                         }
 
-                        FileUtils.write(new File(fileName+"_01.txt"), JSON.toJSONString(repaymentInfo)+"\r\n", "UTF-8", true);
-
+                        //这么写太慢了
+                        ///////FileUtils.write(new File(fileName+"_01.txt"), JSON.toJSONString(repaymentInfo)+"\r\n", "UTF-8", true);
                         //脱敏数据写一份
                         //testTuoMinFileWrite(fileName+"_TEST_01.txt",starTime,null,null,repaymentInfo);
+
+                        tmpWriteBuffer.append(JSON.toJSONString(repaymentInfo)+"\r\n");
+                        buffCount++;
                     }
 
+                    FileUtils.write(new File(fileName+"_01.txt"), tmpWriteBuffer.toString(), "UTF-8", true);
+                    log.info("批量写入文件结束,repaymentInfoList="+repaymentInfoList.size()+",buffCount="+buffCount+",Check Size="+(repaymentInfoList.size()==buffCount));
+                    tmpWriteBuffer=null;
+                    buffCount=0;
+
                 } catch (Exception e) {
                     log.error("还款&逾期存量数据处理异常, startTime: {} , endTime: {} , type: {} ", starTime, endTime, type, e);
                     try{

src/main/java/cn/quantgroup/report/service/manualTool/ManualToolService.java

@@ -1198,7 +1198,29 @@ public class ManualToolService implements CommonSuperService {
         System.err.println("build data buildD3_uploadTs end.");
     }
 
-
+    public static void manualGC() {
+        try {
+            long totalMemory = Runtime.getRuntime().totalMemory();
+            long freeMemory = Runtime.getRuntime().freeMemory();
+            long usedMemory = totalMemory - freeMemory;
+            java.text.NumberFormat format = new java.text.DecimalFormat("###,###");
+
+            String memoryInfo = "垃圾回收前，内存使用情况为：已用内存/总内存：" + format.format(usedMemory) + "/" + format.format(totalMemory);
+            System.out.println("gc:" + memoryInfo);
+            System.out.println("gc:垃圾回收开始，时间为:" + System.currentTimeMillis());
+
+            System.gc();
+
+            totalMemory = Runtime.getRuntime().totalMemory();
+            freeMemory = Runtime.getRuntime().freeMemory();
+            usedMemory = totalMemory - freeMemory;
+            memoryInfo = "垃圾回收后，内存使用情况为：已用内存/总内存：" + format.format(usedMemory) + "/" + format.format(totalMemory);
+            System.out.println(memoryInfo);
+            System.out.println("垃圾回收结束，时间为:" + System.currentTimeMillis());
+        } catch (Exception e) {
+            System.err.println("gc: error!" + e.toString());
+        }
+    }
 
     public static void testMd5() {
         String newJsonFile = "E:\\桌面_工作\\新阳Md5\\aa.txt";
@@ -1261,6 +1283,224 @@ public class ManualToolService implements CommonSuperService {
     }
 
 
+    public static void d3_logicerror_shaixuan_Id(String fileName,String logName) {
+        //"D3_041", 贷款账户编号loanId+还款期数termNo+还款状态确认时间statusConfirmAt+业务发生机构代码在还款记录中唯一。
+       /* List<String> logicerrorList = ReadOrWriteTxt.readTxtList("D:\\用户目录\\Downloads\\生产p2p\\err041_7.log");
+        Map<String,String> err_log_loanIdMap = new HashMap<>(logicerrorList.size());
+        for (String strs : logicerrorList){
+            String[] tmpArr = strs.split("[|]");
+            err_log_loanIdMap.put(tmpArr[0]+","+tmpArr[1]+","+tmpArr[2], strs);
+        }
+        System.out.println("err_log_loanIdMap 大小:"+err_log_loanIdMap.size());*/
+
+
+        /*List<String> lineList = ReadOrWriteTxt.readTxtList("D:\\用户目录\\Downloads\\生产p2p\\sc_D3_20190201_20190410_4.txt");
+        System.out.println("lineList:"+lineList.size());
+        String ok_fileName = "D:\\用户目录\\Downloads\\生产p2p\\build_2\\" + "id_4.txt";*/
+
+        List<String> lineList = ReadOrWriteTxt.readTxtList(fileName);
+        System.out.println("lineList:"+lineList.size());
+        String ok_fileName = logName;
+
+        StringBuffer logIdBuff = new StringBuffer(lineList.size());
+        int totalCount = 0;
+        for(int i=0;i<lineList.size();i++){
+            System.out.println("START:"+i);
+            String str = lineList.get(i);
+            if(str.length() < 30 ){//#applyInfo   #singleLoanAccountInfo
+                continue;
+            }
+            RepaymentInfoZhuDai repaymentLoanInfo = new Gson().fromJson(str, new TypeToken<RepaymentInfoZhuDai>(){}.getType());
+            logIdBuff.append(repaymentLoanInfo.getReqID()+"="+repaymentLoanInfo.getLoanId());
+            logIdBuff.append("\n");
+            totalCount++;
+        }
+
+        System.out.println("build D3 end, lineList="+lineList.size()+",totalCount="+totalCount);
+        try {
+            FileUtils.write(new File(ok_fileName), logIdBuff.toString()+"\n", "UTF-8", true);
+        } catch (IOException e) {
+            System.err.println("build D3 data,"+e);
+        }
+
+        System.out.println("build D3 write file end.");
+    }
+
+
+    public static void d3_logicerror_fengxi() {
+
+       List<String> logicerrorList = ReadOrWriteTxt.readTxtList("D:\\用户目录\\Downloads\\生产p2p\\build_2_err\\err_d3_035_reqId.tmp");
+        Map<String,String> err_log_reqIdMap = new HashMap<>(logicerrorList.size());
+        int count = 0;
+        for (String reqId : logicerrorList){
+            count++;
+            err_log_reqIdMap.put(reqId, ""+count);
+        }
+        System.out.println("err_log_loanIdMap 大小:"+err_log_reqIdMap.size());
+
+
+        String[] all_ids_file = {"id_1.txt","id_2.txt","id_3.txt","id_4.txt","id_5.txt","id_6.txt","id_9.txt"};
+        for(String fileName : all_ids_file){
+
+            List<String> lineList = ReadOrWriteTxt.readTxtList("D:\\用户目录\\Downloads\\生产p2p\\build_2_err\\"+fileName);
+            System.out.println("lineList:"+ fileName + ", size:"+lineList.size());
+
+            StringBuffer in_buff = new StringBuffer(lineList.size());
+            StringBuffer not_in_buff = new StringBuffer(lineList.size());
+
+            for(int i=0;i<lineList.size();i++){
+                System.out.println("START:"+i);
+
+                String[] msgs = lineList.get(i).split("=");
+                // reqId=loddId
+                if(err_log_reqIdMap.containsKey(msgs[0])){
+                    in_buff.append(lineList.get(i)+"\n");
+                }else{
+                    not_in_buff.append(lineList.get(i)+"\n");
+                }
+            }
+            System.out.println(fileName+" build D3 筛选 end.");
+
+            try {
+                FileUtils.write(new File("D:\\用户目录\\Downloads\\生产p2p\\build_2_err\\in_"+fileName), in_buff.toString()+"\n", "UTF-8", true);
+                FileUtils.write(new File("D:\\用户目录\\Downloads\\生产p2p\\build_2_err\\notIn_"+fileName), not_in_buff.toString()+"\n", "UTF-8", true);
+            } catch (IOException e) {
+                System.err.println("build D3 data,"+e);
+            }
+            System.out.println(fileName+" build D3 写入 end.");
+
+        }
+        System.out.println("build D3 write file end.");
+    }
+
+
+    public static void d3_logicerror_fengxi2() {
+        manualGC();
+
+        Map<String,String> all_reqIdMap = new HashMap<>(4729825);
+
+        String[] all_ids_file = {"id_1.txt","id_2.txt","id_3.txt","id_4.txt","id_5.txt","id_6.txt","id_9.txt"};
+        for(String fileName : all_ids_file){
+            List<String> lineList = ReadOrWriteTxt.readTxtList("D:\\用户目录\\Downloads\\生产p2p\\build_2_err\\"+fileName);
+            System.out.println("lineList:"+ fileName + ", size:"+lineList.size());
+            for(int i=0;i<lineList.size();i++){
+                String[] msgs = lineList.get(i).split("=");
+                // reqId=loddId
+                if(msgs.length>=2){
+                    all_reqIdMap.put(msgs[0],msgs[1]);
+                }else{
+                    System.err.println(fileName+">>"+lineList.get(i));
+                }
+            }
+            System.out.println(fileName+" build D3 筛选 end.");
+        }
+
+        manualGC();
+
+        List<String> logicerrorList = ReadOrWriteTxt.readTxtList("D:\\用户目录\\Downloads\\生产p2p\\build_2_err\\err_d3_035_reqId.tmp");
+        //StringBuffer haveBuff = new StringBuffer(logicerrorList.size());
+        int count = 0;
+        for (String reqId : logicerrorList){
+            if(all_reqIdMap.containsKey(reqId)){
+                count++;
+                //haveBuff.append(reqId+"="+all_reqIdMap.get(reqId)+"\n");
+
+                System.out.println("st:"+count);
+                try {
+                    FileUtils.write(new File("D:\\用户目录\\Downloads\\生产p2p\\build_2_err\\have_fill_err_d3_035_reqId.tmp"), reqId+"="+all_reqIdMap.get(reqId)+"\n", "UTF-8", true);
+                } catch (IOException e) {
+                    System.err.println("build D3 data,"+e);
+                }
+            }
+        }
+        System.out.println("build D3 file logicerrorList="+logicerrorList.size()+", count="+count);
+
+        logicerrorList = null;
+        all_reqIdMap = null;
+
+        manualGC();
+
+        /*try {
+            FileUtils.write(new File("D:\\用户目录\\Downloads\\生产p2p\\build_2_err\\have_fill_err_d3_035_reqId.tmp"), haveBuff.toString()+"\n", "UTF-8", true);
+        } catch (IOException e) {
+            System.err.println("build D3 data,"+e);
+        }*/
+
+        System.out.println("build D3 write file end.");
+    }
+
+    public static void d3_logicerror_fengxi3() {
+        manualGC();
+
+        //Set<String> loandSet = new HashSet<>();
+
+        HashMap<String,Integer> loandMap = new HashMap<>();
+
+        List<String> lineList = ReadOrWriteTxt.readTxtList("D:\\用户目录\\Downloads\\生产p2p\\build_2_err\\have_fill_err_d3_035_reqId.tmp");
+        System.out.println("lineList size:"+lineList.size());
+
+        int count = 0;
+        for(int i=0;i<lineList.size();i++){
+            if(StringUtils.isNotBlank(lineList.get(i))){
+                String[] msgs = lineList.get(i).split("=");
+                // reqId=loddId
+                if(msgs.length>=2){
+                    if(loandMap.containsKey(msgs[1])){
+                        loandMap.put(msgs[1], loandMap.get(msgs[1])+1);
+                    }else{
+                        loandMap.put(msgs[1], 1);
+                    }
+                    count++;
+                }else{
+                    System.err.println("reqId=loddId is null >>"+lineList.get(i));
+                }
+            }
+        }
+        System.out.println("have_fill_err_d3_035_reqId lineList="+lineList.size()+",count="+count+",loandMap="+loandMap.size());
+        lineList = null;
+        manualGC();
+
+        List<String> d2List = ReadOrWriteTxt.readTxtList("D:\\用户目录\\Downloads\\生产p2p\\bulid_ok\\lhp_D2_20161101_20191025_01.txt");
+        int st=0,have = 0;
+        int allcount = loandMap.size();
+        for (String d2JsonStr : d2List){
+            System.out.println("st:"+(st++));
+            if(d2JsonStr.length()<30){
+                continue;
+            }
+
+            JSONObject d2Json = JSONObject.parseObject(d2JsonStr);
+            String loanId = d2Json.getString("loanId");
+            if(loandMap.containsKey(loanId)){
+                have++;
+                System.out.println("have:"+have);
+                try {
+                    FileUtils.write(new File("D:\\用户目录\\Downloads\\生产p2p\\build_2_err\\ind2_d3_035_loanId.txt"),
+                            loanId+",d3_count="+loandMap.get(loanId)+"\n", "UTF-8", true);
+
+                    loandMap.remove(loanId);
+                } catch (IOException e) {
+                    System.err.println("build D3 data,"+e);
+                }
+            }
+        }
+        System.out.println("allcount size="+allcount+", in have="+have+", not in="+loandMap.size());
+
+        if(loandMap.size()>0){
+            int wr = 0;
+            for(String key : loandMap.keySet()){
+                System.out.println("wr:"+(wr++));
+                try {
+                    FileUtils.write(new File("D:\\用户目录\\Downloads\\生产p2p\\build_2_err\\notind2_d3_035_loanId.txt"),
+                            key+",d3_count=SSS"+loandMap.get(key)+"\n", "UTF-8", true);
+                } catch (IOException e) {
+                    System.err.println("build D3 data,"+e);
+                }
+            }
+        }
+    }
+
+
         public static void main(String[] args) {
 //        List<String> lineList = ReadOrWriteTxt.readTxtList("D:\\用户目录\\Downloads\\D2_M_01.txt");
 //        System.out.println("lineList:"+lineList.size());
@@ -1290,6 +1530,22 @@ public class ManualToolService implements CommonSuperService {
             //testMd5_2();
 
 
-            buildD3_logicerror();
+            //buildD3_logicerror();
+
+
+
+           /* String dataFile = "D:\\用户目录\\Downloads\\生产p2p\\bulid_ok\\ok9_D3_20191025_0027.txt";
+            String logFile = "D:\\用户目录\\Downloads\\生产p2p\\build_2_err\\id_9.txt";
+            d3_logicerror_shaixuan_Id(dataFile,logFile);*/
+
+
+            //d3_logicerror_fengxi();
+
+            //d3_logicerror_fengxi2();
+
+            d3_logicerror_fengxi3();
         }
+
+
+
 }