工具修改

src/main/java/cn/quantgroup/report/constant/BaiHangHistoryConstant.java

+package cn.quantgroup.report.constant;/**
+ * -----------------------------------------------------------------------------<br>
+ * 描述: <br>
+ * 作者: Administrator <br>
+ * 时间: 2019.11.14 10:56 <br>
+ * 授权: (C) Copyright (c) 2017 <br>
+ * 公司: 北京众信利民信息技术有限公司 <br>
+ * -----------------------------------------------------------------------------
+ */
+
+/**
+ * -----------------------------------------------------------------------------<br>
+ * 描述: <br>
+ * 作者：yanhui.Hao <br>
+ * 时间：2019.11.14 <br>
+ * 授权: (C) Copyright (c) 2017 <br>
+ * 公司: 北京众信利民信息技术有限公司 <br>
+ * -----------------------------------------------------------------------------
+ */
+public class BaiHangHistoryConstant {
+
+    public static final int MAX_COMMIT_COUT = 20000;//大于2万条会出现事务提交过长报错
+
+    public static final String SQL_COMMONE_TEMPLATE_HISTORY = "INSERT xyqb_i_loan_a_m_history_bak " +
+            " select * from xyqb_i_loan_application_manifest_history b " +
+            " where b.funding_corp_id in (420,520,580,650,670,810,240,640,700) " +
+            " and b.created_at >= '##STARTTIME##'  and b.created_at < '##ENDTIME##'; ";
+    //报送查询 b.loan_paid_at<@t2  公司打款时间
+
+    public static final String SQL_COMMONE_TEMPLATE_PLAN = "INSERT xyqb_i_repayment_plan_bak " +
+            "select a.* from xyqb_i_repayment_plan a " +
+            " join xyqb_i_loan_a_m_history_bak b " +
+            " on a.loan_application_history_id = b.loan_application_history_id " +
+            "where b.created_at >= '##STARTTIME##'  and b.created_at < '##ENDTIME##'; ";
+    //报送查询 a.repaid_at>=@t2 还款时间
+
+
+    //---------------------根据id去同步---------------------------
+
+
+    public static final String SQL_COMMONE_TEMPLATE_HISTORY_BAK_MAX_ID = "select MAX(b.id) as bakMaxId from xyqb_i_loan_a_m_history_bak b; ";
+
+    public static final String SQL_COMMONE_TEMPLATE_HISTORY_SYN_COUNT = "select count(b.id) from xyqb_i_loan_application_manifest_history b " +
+            " where b.funding_corp_id in (420,520,580,650,670,810,240,640,700) " +
+            " and b.id > ##BAKMAXID## ;";
+    public static final String SQL_COMMONE_TEMPLATE_HISTORY_SYN_DATA = " INSERT xyqb_i_loan_a_m_history_bak " +
+            " select b.* from xyqb_i_loan_application_manifest_history b " +
+            " where b.funding_corp_id in (420,520,580,650,670,810,240,640,700) " +
+            " and b.id > ##STARTID## ";//AND b.id <= ##ENDID##
+
+
+    public static final String SQL_COMMONE_TEMPLATE_PLAN_SYN_COUNT = "select count(a.id) from xyqb_i_repayment_plan a " +
+            " join xyqb_i_loan_a_m_history_bak b on a.loan_application_history_id = b.loan_application_history_id " +
+            " where b.id  > ##BAKMAXID##;";
+    public static final String SQL_COMMONE_TEMPLATE_PLAN_SYN_DATA = "INSERT xyqb_i_repayment_plan_bak " +
+            "select a.* from xyqb_i_repayment_plan a " +
+            " join xyqb_i_loan_a_m_history_bak b on a.loan_application_history_id = b.loan_application_history_id " +
+            " where b.id > ##STARTID## ";//AND b.id <= ##ENDID##
+
+}

src/main/java/cn/quantgroup/report/job/LoanApplicationManifestHistoryTask.java

 package cn.quantgroup.report.job;
 
+import cn.quantgroup.report.constant.BaiHangHistoryConstant;
 import cn.quantgroup.report.service.baihang.constant.Constant;
 import com.google.common.base.Stopwatch;
 import org.slf4j.Logger;
@@ -42,20 +43,8 @@ public class LoanApplicationManifestHistoryTask {
 
     private static AtomicBoolean SYN_Stop = new AtomicBoolean(false);
 
-    private static String SQL_COMMONE_TEMPLATE = "INSERT xyqb_i_loan_a_m_history_bak " +
-            " select * from xyqb_i_loan_application_manifest_history b " +
-            " where b.funding_corp_id in (420,520,580,650,670,810,240,640,700) " +
-            " and b.created_at >= '##STARTTIME##'  and b.created_at < '##ENDTIME##'; ";
-
-    private static String SQL_COMMONE_TEMPLATE_PLAN = " INSERT xyqb_i_repayment_plan_bak " +
-            "select a.* from xyqb_i_repayment_plan a " +
-            " join xyqb_i_loan_a_m_history_bak b " +
-            " on a.loan_application_history_id = b.loan_application_history_id " +
-            "where b.created_at >= '##STARTTIME##'  and b.created_at < '##ENDTIME##'; ";
-
-
     @Async
-    /*@Scheduled(cron = "0 06 16 * * ?")*/
+    @Scheduled(cron = "0 17 16 * * ?")//2019.11.14 15:25
     public void startHistoryDateCopy(){
         if(increment()){
             redisTemplate.expire(Constant.XYQB_HISTORY_DAI_REPORT_LOCK_KEY, 10, TimeUnit.SECONDS);
@@ -67,8 +56,13 @@ public class LoanApplicationManifestHistoryTask {
             //String endnyr = LocalDateTime.now().format(DateTimeFormatter.ISO_DATE);
 
             log.info("量化派-同步表xyqb_i_loan_application_manifest_history数据开始, newTime: {} ", LocalDateTime.now());
-            //table_xyqb_i_loan_a_m_history_bak();
-            table_xyqb_i_repayment_plan_bak();
+
+            syn_history_by_CreatedAt();
+
+            syn_plan_by_CreatedAt();
+
+            //syn_historyAndplan_by_id();
+
             log.info("量化派-同步表xyqb_i_loan_application_manifest_history数据结束, endTime: {}, 耗时: {} ", LocalDateTime.now(), stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
         }
     }
@@ -79,7 +73,8 @@ public class LoanApplicationManifestHistoryTask {
         return increment <= 1;
     }
 
-    public void table_xyqb_i_loan_a_m_history_bak(){
+
+    public void syn_history_by_CreatedAt(){
         LocalDateTime erlyDate = LocalDateTime.parse("2016-08-23T00:00:00");//2016-08-23T16:29:10
         LocalDateTime endDate = LocalDateTime.parse("2019-11-01T00:00:00");//2019-11-11 12:32:08
         int counter = 0;
@@ -98,7 +93,8 @@ public class LoanApplicationManifestHistoryTask {
             String starTime = erlyDate.plusDays(counter - 1).format(DateTimeFormatter.ISO_DATE);
             String endTime = erlyDate.plusDays(counter).format(DateTimeFormatter.ISO_DATE);
             try {
-                String tmp_sql = SQL_COMMONE_TEMPLATE.replace("##STARTTIME##",starTime);
+                String tmp_sql = BaiHangHistoryConstant.SQL_COMMONE_TEMPLATE_HISTORY;
+                tmp_sql = tmp_sql.replace("##STARTTIME##",starTime);
                 tmp_sql = tmp_sql.replace("##ENDTIME##",endTime);
 
                 int upCount = xyqbHistoryJdbcTemplate.update(tmp_sql);
@@ -110,7 +106,7 @@ public class LoanApplicationManifestHistoryTask {
         }
     }
 
-    public void table_xyqb_i_repayment_plan_bak(){
+    public void syn_plan_by_CreatedAt(){
         LocalDateTime erlyDate = LocalDateTime.parse("2016-08-23T00:00:00");//2016-08-23T16:29:10
         LocalDateTime endDate = LocalDateTime.parse("2019-11-01T00:00:00");//2019-11-11 12:32:08
         int counter = 0;
@@ -129,7 +125,8 @@ public class LoanApplicationManifestHistoryTask {
             String starTime = erlyDate.plusDays(counter - 1).format(DateTimeFormatter.ISO_DATE);
             String endTime = erlyDate.plusDays(counter).format(DateTimeFormatter.ISO_DATE);
             try {
-                String tmp_sql = SQL_COMMONE_TEMPLATE_PLAN.replace("##STARTTIME##",starTime);
+                String tmp_sql = BaiHangHistoryConstant.SQL_COMMONE_TEMPLATE_PLAN;
+                tmp_sql = tmp_sql.replace("##STARTTIME##",starTime);
                 tmp_sql = tmp_sql.replace("##ENDTIME##",endTime);
 
                 int upCount = xyqbHistoryJdbcTemplate.update(tmp_sql);
@@ -142,5 +139,143 @@ public class LoanApplicationManifestHistoryTask {
     }
 
 
+    private boolean syn_history_by_id(long bakMaxId){
+        String count_sql = BaiHangHistoryConstant.SQL_COMMONE_TEMPLATE_HISTORY_SYN_COUNT;
+        count_sql = count_sql.replace("##BAKMAXID##",""+bakMaxId);
+
+        Integer count = xyqbHistoryJdbcTemplate.queryForObject(count_sql,Integer.class);
+        if(count==null){
+            log.error("表xyqb_i_loan_application_manifest_history需要同步的count为null.");
+            return false;
+        }
+
+        String inset_sql = BaiHangHistoryConstant.SQL_COMMONE_TEMPLATE_HISTORY_SYN_DATA;
+        inset_sql = inset_sql.replace("##STARTID##",""+bakMaxId);
+
+        if(count.intValue() <= BaiHangHistoryConstant.MAX_COMMIT_COUT){
+            Stopwatch startwatch = Stopwatch.createStarted();
+            int update = xyqbHistoryJdbcTemplate.update(inset_sql);
+            log.info("增量同步xyqb_i_loan_application_manifest_history表结束, bakMaxId: {} , count: {} , 插入条数: {} , 耗时: {} ", bakMaxId, count, update, startwatch.stop().elapsed(TimeUnit.MILLISECONDS)+".ms");
+            return true;
+        }else{
+
+            int page = count.intValue() / BaiHangHistoryConstant.MAX_COMMIT_COUT;
+            int mode = count.intValue() % BaiHangHistoryConstant.MAX_COMMIT_COUT;
+            if(mode!=0){
+                page = page+1;
+            }
+
+            int totalUpdate = 0;
+            for (int p=0 ; p < page; p++){
+                Stopwatch startwatch = Stopwatch.createStarted();
+                int limit1 = p * BaiHangHistoryConstant.MAX_COMMIT_COUT;
+                int limit2 = (p+1) * BaiHangHistoryConstant.MAX_COMMIT_COUT;
+                if( limit2 > count.intValue()){
+                    limit2 = count.intValue();
+                }
+                String tmpInsertSql = inset_sql + (" ORDER BY b.id ASC LIMIT "+limit1+", "+limit2+" ;");
+
+                int update = xyqbHistoryJdbcTemplate.update(tmpInsertSql);
+
+                totalUpdate+=update;
+                log.info("增量同步xyqb_i_loan_application_manifest_history表页数page["+p+"], limit1: {} , limit2: {} , 插入条数: {} ", limit1, limit2, update, startwatch.stop().elapsed(TimeUnit.MILLISECONDS)+".ms");
+            }
+            log.info("增量同步xyqb_i_loan_application_manifest_history表结束, bakMaxId: {} , count: {} , 总插入条数: {} , page: {} , mode: {} ", bakMaxId, count, totalUpdate, page, mode);
+
+
+            if(totalUpdate==count){
+                return true;
+            }else{
+                return false;
+            }
+        }
+    }
+
+
+    private boolean syn_plan_by_id(long bakMaxId){
+        String count_sql = BaiHangHistoryConstant.SQL_COMMONE_TEMPLATE_PLAN_SYN_COUNT;
+        count_sql = count_sql.replace("##BAKMAXID##",""+bakMaxId);
+
+        Integer count = xyqbHistoryJdbcTemplate.queryForObject(count_sql,Integer.class);
+        if(count==null){
+            log.error("表xyqb_i_repayment_plan需要同步的count为null.");
+            return false;
+        }
+
+        String inset_sql = BaiHangHistoryConstant.SQL_COMMONE_TEMPLATE_PLAN_SYN_DATA;
+        inset_sql = inset_sql.replace("##STARTID##",""+bakMaxId);
+
+        if(count.intValue() <= BaiHangHistoryConstant.MAX_COMMIT_COUT){
+            Stopwatch startwatch = Stopwatch.createStarted();
+            int update = xyqbHistoryJdbcTemplate.update(inset_sql);
+            log.info("增量同步xyqb_i_repayment_plan表结束, bakMaxId: {} , count: {} , 插入条数: {} , 耗时: {} ", bakMaxId, count, update, startwatch.stop().elapsed(TimeUnit.MILLISECONDS)+".ms");
+            return true;
+        }else{
+
+            int page = count.intValue() / BaiHangHistoryConstant.MAX_COMMIT_COUT;
+            int mode = count.intValue() % BaiHangHistoryConstant.MAX_COMMIT_COUT;
+            if(mode!=0){
+                page = page+1;
+            }
+
+            int totalUpdate = 0;
+            for (int p=0 ; p < page; p++){
+                Stopwatch startwatch = Stopwatch.createStarted();
+                int limit1 = p * BaiHangHistoryConstant.MAX_COMMIT_COUT;
+                int limit2 = (p+1) * BaiHangHistoryConstant.MAX_COMMIT_COUT;
+                if( limit2 > count.intValue()){
+                    limit2 = count.intValue();
+                }
+                String tmpInsertSql = inset_sql + (" ORDER BY b.id ASC LIMIT "+limit1+", "+limit2+" ;");
+
+                int update = xyqbHistoryJdbcTemplate.update(tmpInsertSql);
+
+                totalUpdate+=update;
+                log.info("增量同步xyqb_i_repayment_plan表页数page["+p+"], limit1: {} , limit2: {} , 插入条数: {} ", limit1, limit2, update, startwatch.stop().elapsed(TimeUnit.MILLISECONDS)+".ms");
+            }
+            log.info("增量同步xyqb_i_repayment_plan表结束, bakMaxId: {} , count: {} , 总插入条数: {} , page: {} , mode: {} ", bakMaxId, count, totalUpdate, page, mode);
+
+
+            if(totalUpdate==count){
+                return true;
+            }else{
+                return false;
+            }
+        }
+    }
+
+    public boolean syn_historyAndplan_by_id(){
+        Stopwatch startwatch = Stopwatch.createStarted();
+        try {
+            String tmp_sql = BaiHangHistoryConstant.SQL_COMMONE_TEMPLATE_HISTORY_BAK_MAX_ID;
+            Long bakMaxId = xyqbHistoryJdbcTemplate.queryForObject(tmp_sql,Long.class);
+            if(bakMaxId!=null && bakMaxId.longValue() >= 0L){
+
+                //-------------------------------
+               boolean table_history = syn_history_by_id(bakMaxId.longValue());
+               if(table_history){
+
+                   boolean table_plan = syn_plan_by_id(bakMaxId.longValue());
+                   if(table_plan){
+                       log.info("增量同步history&plan表数据结束, nowTime: {} ", LocalDateTime.now(), startwatch.stop().elapsed(TimeUnit.MILLISECONDS)+".ms");
+                       return true;
+                   }else{
+                       log.error("增量同步[xyqb_i_repayment_plan_bak]表数据失败,请尽快处理! nowTime: {} , bakMaxId: {} ",LocalDateTime.now(), bakMaxId);
+                   }
+
+               }else{
+                   log.error("增量同步[xyqb_i_loan_a_m_history_bak]表数据失败,请尽快处理! nowTime: {} , bakMaxId: {} ",LocalDateTime.now(), bakMaxId);
+               }
+               //-------------------------------
+
+            }else {
+                log.error("增量同步xyqb_i_repayment_plan数据-历史id为空,停止同步! nowTime: {} , bakMaxId: {} ",LocalDateTime.now(), bakMaxId);
+            }
+        }catch (Exception e){
+            log.error("增量同步history&plan表数据异常, nowTime: {} ", LocalDateTime.now(), e);
+        }
+
+        return false;
+    }
 
 }