Skip to content

B
baihang-report
Commit 3824b779 authored by 郝彦辉's avatar 郝彦辉
Browse files
Options

数据同步修改，增加钉钉报警

parent 6d785121
Hide whitespace changes
Inline Side-by-side
Showing with 315 additions and 58 deletions
src/main/java/cn/quantgroup/report/constant/BaiHangHistoryConstant.java
View file @ 3824b779
...@@ -19,7 +19,7 @@ package cn.quantgroup.report.constant;/** ...@@ -19,7 +19,7 @@ package cn.quantgroup.report.constant;/**
*/ */
public class BaiHangHistoryConstant { public class BaiHangHistoryConstant {
public static final int MAX_COMMIT_COUT = 20000;//大于2万条会出现事务提交过长报错 public static final int MAX_COMMIT_COUT = 24000;//大于2万条会出现事务提交过长报错
public static final String SQL_COMMONE_TEMPLATE_HISTORY = "INSERT xyqb_i_loan_a_m_history_bak " + public static final String SQL_COMMONE_TEMPLATE_HISTORY = "INSERT xyqb_i_loan_a_m_history_bak " +
" select * from xyqb_i_loan_application_manifest_history b " + " select * from xyqb_i_loan_application_manifest_history b " +
...@@ -39,20 +39,23 @@ public class BaiHangHistoryConstant { ...@@ -39,20 +39,23 @@ public class BaiHangHistoryConstant {
public static final String SQL_COMMONE_TEMPLATE_HISTORY_BAK_MAX_ID = "select MAX(b.id) as bakMaxId from xyqb_i_loan_a_m_history_bak b; "; public static final String SQL_COMMONE_TEMPLATE_HISTORY_BAK_MAX_ID = "select MAX(b.id) as bakMaxId from xyqb_i_loan_a_m_history_bak b; ";
public static final String SQL_COMMONE_TEMPLATE_HISTORY_MAX_ID = " select MAX(b2.id) from xyqb_i_loan_application_manifest_history b2 " +
" where b2.funding_corp_id in (420,520,580,650,670,810,240,640,700) ";
//借款清单表xyqb_i_loan_application_manifest_history
public static final String SQL_COMMONE_TEMPLATE_HISTORY_SYN_COUNT = "select count(b.id) from xyqb_i_loan_application_manifest_history b " + public static final String SQL_COMMONE_TEMPLATE_HISTORY_SYN_COUNT = "select count(b.id) from xyqb_i_loan_application_manifest_history b " +
" where b.funding_corp_id in (420,520,580,650,670,810,240,640,700) " + " where b.funding_corp_id in (420,520,580,650,670,810,240,640,700) " +
" and b.id > ##BAKMAXID## ;"; " and b.id > ##BAKMAXID## AND b.id <= ##NEWMAXID## ;";
public static final String SQL_COMMONE_TEMPLATE_HISTORY_SYN_DATA = " INSERT xyqb_i_loan_a_m_history_bak " + public static final String SQL_COMMONE_TEMPLATE_HISTORY_SYN_DATA = "REPLACE INTO xyqb_i_loan_a_m_history_bak " + //"INSERT xyqb_i_loan_a_m_history_bak "
" select b.* from xyqb_i_loan_application_manifest_history b " + " select b.* from xyqb_i_loan_application_manifest_history b " +
" where b.funding_corp_id in (420,520,580,650,670,810,240,640,700) " + " where b.funding_corp_id in (420,520,580,650,670,810,240,640,700) " +
" and b.id > ##STARTID## ";//AND b.id <= ##ENDID## " and b.id > ##STARTID## ";//AND b.id <= ##ENDID##
//还款计划表还款计划表xyqb_i_repayment_plan
public static final String SQL_COMMONE_TEMPLATE_PLAN_SYN_COUNT = "select count(a.id) from xyqb_i_repayment_plan a " + public static final String SQL_COMMONE_TEMPLATE_PLAN_SYN_COUNT = "select count(a.id) from xyqb_i_repayment_plan a " +
" join xyqb_i_loan_a_m_history_bak b on a.loan_application_history_id = b.loan_application_history_id " + " join xyqb_i_loan_a_m_history_bak b on a.loan_application_history_id = b.loan_application_history_id " +
" where b.id > ##BAKMAXID##;"; " where b.id > ##BAKMAXID## AND b.id <= ##NEWMAXID## ;";
public static final String SQL_COMMONE_TEMPLATE_PLAN_SYN_DATA = "INSERT xyqb_i_repayment_plan_bak " + public static final String SQL_COMMONE_TEMPLATE_PLAN_SYN_DATA = "REPLACE INTO xyqb_i_repayment_plan_bak " + //"INSERT xyqb_i_repayment_plan_bak "
"select a.* from xyqb_i_repayment_plan a " + "select a.* from xyqb_i_repayment_plan a " +
" join xyqb_i_loan_a_m_history_bak b on a.loan_application_history_id = b.loan_application_history_id " + " join xyqb_i_loan_a_m_history_bak b on a.loan_application_history_id = b.loan_application_history_id " +
" where b.id > ##STARTID## ";//AND b.id <= ##ENDID## " where b.id > ##STARTID## ";//AND b.id <= ##ENDID##
......
src/main/java/cn/quantgroup/report/job/LoanApplicationManifestHistoryTask.java
View file @ 3824b779
...@@ -2,6 +2,7 @@ package cn.quantgroup.report.job; ...@@ -2,6 +2,7 @@ package cn.quantgroup.report.job;
import cn.quantgroup.report.constant.BaiHangHistoryConstant; import cn.quantgroup.report.constant.BaiHangHistoryConstant;
import cn.quantgroup.report.service.baihang.constant.Constant; import cn.quantgroup.report.service.baihang.constant.Constant;
import cn.quantgroup.report.utils.dingtalk.DingTalk;
import com.google.common.base.Stopwatch; import com.google.common.base.Stopwatch;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
...@@ -41,15 +42,17 @@ public class LoanApplicationManifestHistoryTask { ...@@ -41,15 +42,17 @@ public class LoanApplicationManifestHistoryTask {
@Autowired @Autowired
private JdbcTemplate xyqbHistoryJdbcTemplate; private JdbcTemplate xyqbHistoryJdbcTemplate;
@Autowired
private DingTalk dingTalk;
private static AtomicBoolean SYN_Stop = new AtomicBoolean(false); private static AtomicBoolean SYN_Stop = new AtomicBoolean(false);
@Async @Async
@Scheduled(cron = "0 17 16 * * ?")//2019.11.14 15:25 @Scheduled(cron = "0 05 17 * * ?")//2019.11.15 15:25
public void startHistoryDateCopy(){ public void startHistoryDateCopy(){
if(increment()){ if(increment()){
redisTemplate.expire(Constant.XYQB_HISTORY_DAI_REPORT_LOCK_KEY, 10, TimeUnit.SECONDS); redisTemplate.expire(Constant.XYQB_HISTORY_DAI_REPORT_LOCK_KEY, 10, TimeUnit.SECONDS);
//将xyqb_i_loan_application_manifest_history表的数据，同步到xyqb_i_loan_a_m_history_bak
Stopwatch stopwatch = Stopwatch.createStarted(); Stopwatch stopwatch = Stopwatch.createStarted();
//yyyy-MM-dd //yyyy-MM-dd
//String startnyr = LocalDateTime.now().plusDays(-1).format(DateTimeFormatter.ISO_DATE); //String startnyr = LocalDateTime.now().plusDays(-1).format(DateTimeFormatter.ISO_DATE);
...@@ -57,11 +60,13 @@ public class LoanApplicationManifestHistoryTask { ...@@ -57,11 +60,13 @@ public class LoanApplicationManifestHistoryTask {
log.info("量化派-同步表xyqb_i_loan_application_manifest_history数据开始, newTime: {} ", LocalDateTime.now()); log.info("量化派-同步表xyqb_i_loan_application_manifest_history数据开始, newTime: {} ", LocalDateTime.now());
syn_history_by_CreatedAt(); //将xyqb_i_loan_application_manifest_history表的数据，同步到xyqb_i_repayment_plan_bak
//syn_history_by_CreatedAt();
syn_plan_by_CreatedAt(); //将xyqb_i_repayment_plan表的数据，同步到xyqb_i_loan_a_m_history_bak
//syn_plan_by_CreatedAt();
//syn_historyAndplan_by_id(); syn_historyAndplan_by_id();
log.info("量化派-同步表xyqb_i_loan_application_manifest_history数据结束, endTime: {}, 耗时: {} ", LocalDateTime.now(), stopwatch.stop().elapsed(TimeUnit.MILLISECONDS)); log.info("量化派-同步表xyqb_i_loan_application_manifest_history数据结束, endTime: {}, 耗时: {} ", LocalDateTime.now(), stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
} }
...@@ -139,28 +144,33 @@ public class LoanApplicationManifestHistoryTask { ...@@ -139,28 +144,33 @@ public class LoanApplicationManifestHistoryTask {
} }
private boolean syn_history_by_id(long bakMaxId){ private boolean syn_history_by_id(long bakMaxId,long newMaxId){
String count_sql = BaiHangHistoryConstant.SQL_COMMONE_TEMPLATE_HISTORY_SYN_COUNT; String count_sql = BaiHangHistoryConstant.SQL_COMMONE_TEMPLATE_HISTORY_SYN_COUNT;
count_sql = count_sql.replace("##BAKMAXID##",""+bakMaxId); count_sql = count_sql.replace("##BAKMAXID##",""+bakMaxId);
count_sql = count_sql.replace("##NEWMAXID##",""+newMaxId);//防止查询的过程中再增长
Integer count = xyqbHistoryJdbcTemplate.queryForObject(count_sql,Integer.class); Integer count = xyqbHistoryJdbcTemplate.queryForObject(count_sql,Integer.class);
if(count==null){ if(count==null){
log.error("表xyqb_i_loan_application_manifest_history需要同步的count为null."); log.error("表xyqb_i_loan_application_manifest_history查询增量count为Null.");
dingTalk.talk("Warn","同步[借款清单]数据失败","查询增量count为Null,请尽快处理!"+" bakMaxId:"+bakMaxId+", newMaxId:"+newMaxId);
dingTalk.talk_ToUser("同步[借款清单]表查询增量count为Null,停止T+1报送！发现问题，请及时处理！");
return false; return false;
} }
String inset_sql = BaiHangHistoryConstant.SQL_COMMONE_TEMPLATE_HISTORY_SYN_DATA; //小于2万条，可以commit
inset_sql = inset_sql.replace("##STARTID##",""+bakMaxId);
if(count.intValue() <= BaiHangHistoryConstant.MAX_COMMIT_COUT){ if(count.intValue() <= BaiHangHistoryConstant.MAX_COMMIT_COUT){
Stopwatch startwatch = Stopwatch.createStarted(); Stopwatch startwatch = Stopwatch.createStarted();
String inset_sql = BaiHangHistoryConstant.SQL_COMMONE_TEMPLATE_HISTORY_SYN_DATA;
inset_sql = inset_sql.replace("##STARTID##",""+bakMaxId);
int update = xyqbHistoryJdbcTemplate.update(inset_sql); int update = xyqbHistoryJdbcTemplate.update(inset_sql);
log.info("增量同步xyqb_i_loan_application_manifest_history表结束, bakMaxId: {} , count: {} , 插入条数: {} , 耗时: {} ", bakMaxId, count, update, startwatch.stop().elapsed(TimeUnit.MILLISECONDS)+".ms"); log.info("增量同步xyqb_i_loan_application_manifest_history表结束, bakMaxId: {} , count: {} , 插入条数: {} , 耗时: {} ", bakMaxId, count, update, startwatch.stop().elapsed(TimeUnit.MILLISECONDS)+".ms");
dingTalk.talk("Info","同步[借款清单]数据成功","实际条数:"+update +", bakMaxId:"+bakMaxId+", newMaxId:"+newMaxId);
return true; return true;
}else{ }else{
//大于2万条会报事务太长，需要分页插入
int page = count.intValue() / BaiHangHistoryConstant.MAX_COMMIT_COUT; int id_limit_count = (int) (newMaxId - bakMaxId);
int mode = count.intValue() % BaiHangHistoryConstant.MAX_COMMIT_COUT; int page = id_limit_count / BaiHangHistoryConstant.MAX_COMMIT_COUT;
int mode = id_limit_count % BaiHangHistoryConstant.MAX_COMMIT_COUT;
if(mode!=0){ if(mode!=0){
page = page+1; page = page+1;
} }
...@@ -168,52 +178,64 @@ public class LoanApplicationManifestHistoryTask { ...@@ -168,52 +178,64 @@ public class LoanApplicationManifestHistoryTask {
int totalUpdate = 0; int totalUpdate = 0;
for (int p=0 ; p < page; p++){ for (int p=0 ; p < page; p++){
Stopwatch startwatch = Stopwatch.createStarted(); Stopwatch startwatch = Stopwatch.createStarted();
int limit1 = p * BaiHangHistoryConstant.MAX_COMMIT_COUT; long startId = bakMaxId + p * BaiHangHistoryConstant.MAX_COMMIT_COUT;
int limit2 = (p+1) * BaiHangHistoryConstant.MAX_COMMIT_COUT; long endId = bakMaxId + (p+1) * BaiHangHistoryConstant.MAX_COMMIT_COUT;
if( limit2 > count.intValue()){ if( endId > newMaxId){
limit2 = count.intValue(); endId = newMaxId;
} }
String tmpInsertSql = inset_sql + (" ORDER BY b.id ASC LIMIT "+limit1+", "+limit2+" ;"); String tmpInsertSql = BaiHangHistoryConstant.SQL_COMMONE_TEMPLATE_HISTORY_SYN_DATA;
tmpInsertSql = tmpInsertSql.replace("##STARTID##",""+startId);
tmpInsertSql = tmpInsertSql + (" AND b.id <="+endId+" ;");
int update = xyqbHistoryJdbcTemplate.update(tmpInsertSql); int update = xyqbHistoryJdbcTemplate.update(tmpInsertSql);
totalUpdate+=update; totalUpdate+=update;
log.info("增量同步xyqb_i_loan_application_manifest_history表页数page["+p+"], limit1: {} , limit2: {} , 插入条数: {} ", limit1, limit2, update, startwatch.stop().elapsed(TimeUnit.MILLISECONDS)+".ms"); log.info("增量同步xyqb_i_loan_application_manifest_history表页数page["+p+"], startId: {} , endId: {} , 插入条数: {} , 耗时: {} ", startId, endId, update, startwatch.stop().elapsed(TimeUnit.MILLISECONDS)+".ms");
} }
log.info("增量同步xyqb_i_loan_application_manifest_history表结束, bakMaxId: {} , count: {} , 总插入条数: {} , page: {} , mode: {} ", bakMaxId, count, totalUpdate, page, mode); log.info("增量同步xyqb_i_loan_application_manifest_history表结束, bakMaxId: {} , newMaxId: {} , ID差: {} , count: {} , totalUpdate: {} , page: {} , mode: {} ", bakMaxId, newMaxId, (newMaxId-bakMaxId), count, totalUpdate, page, mode);
if(totalUpdate==count){ if(totalUpdate==count){
dingTalk.talk("Info","同步[借款清单]数据成功","实际条数(分页):"+totalUpdate+", bakMaxId:"+bakMaxId+", newMaxId:"+newMaxId);
return true; return true;
}else{ }else{
dingTalk.talk("Warn","同步[借款清单]数据失败","查询条数:"+count+",实际条数="+totalUpdate+", bakMaxId:"+bakMaxId+", newMaxId:"+newMaxId);
dingTalk.talk_ToUser("同步[借款清单]数据失败,停止T+1报送！发现问题，请及时处理！");
return false; return false;
} }
} }
} }
private boolean syn_plan_by_id(long bakMaxId){ private boolean syn_plan_by_id(long bakMaxId,long newMaxId){
String count_sql = BaiHangHistoryConstant.SQL_COMMONE_TEMPLATE_PLAN_SYN_COUNT; String count_sql = BaiHangHistoryConstant.SQL_COMMONE_TEMPLATE_PLAN_SYN_COUNT;
count_sql = count_sql.replace("##BAKMAXID##",""+bakMaxId); count_sql = count_sql.replace("##BAKMAXID##",""+bakMaxId);
count_sql = count_sql.replace("##NEWMAXID##",""+newMaxId);//防止查询的过程中再增长
Integer count = xyqbHistoryJdbcTemplate.queryForObject(count_sql,Integer.class); Integer count = xyqbHistoryJdbcTemplate.queryForObject(count_sql,Integer.class);
if(count==null){ if(count==null){
log.error("表xyqb_i_repayment_plan需要同步的count为null."); log.error("表xyqb_i_repayment_plan查询增量count为Null.");
dingTalk.talk("Warn","同步[还款计划]数据失败","查询增量count为Null,请尽快处理!" + ", bakMaxId:"+bakMaxId+", newMaxId:"+newMaxId);
dingTalk.talk_ToUser("同步[还款计划]表查询增量count为Null,停止T+1报送！发现问题，请及时处理！");
return false; return false;
} }
String inset_sql = BaiHangHistoryConstant.SQL_COMMONE_TEMPLATE_PLAN_SYN_DATA; //小于2万条，可以commit
inset_sql = inset_sql.replace("##STARTID##",""+bakMaxId);
if(count.intValue() <= BaiHangHistoryConstant.MAX_COMMIT_COUT){ if(count.intValue() <= BaiHangHistoryConstant.MAX_COMMIT_COUT){
Stopwatch startwatch = Stopwatch.createStarted(); Stopwatch startwatch = Stopwatch.createStarted();
String inset_sql = BaiHangHistoryConstant.SQL_COMMONE_TEMPLATE_PLAN_SYN_DATA;
inset_sql = inset_sql.replace("##STARTID##",""+bakMaxId);
int update = xyqbHistoryJdbcTemplate.update(inset_sql); int update = xyqbHistoryJdbcTemplate.update(inset_sql);
log.info("增量同步xyqb_i_repayment_plan表结束, bakMaxId: {} , count: {} , 插入条数: {} , 耗时: {} ", bakMaxId, count, update, startwatch.stop().elapsed(TimeUnit.MILLISECONDS)+".ms"); log.info("增量同步xyqb_i_repayment_plan表结束, bakMaxId: {} , count: {} , 插入条数: {} , 耗时: {} ", bakMaxId, count, update, startwatch.stop().elapsed(TimeUnit.MILLISECONDS)+".ms");
dingTalk.talk("Info","同步[还款计划]数据成功","实际条数:"+update +", bakMaxId:"+bakMaxId+", newMaxId:"+newMaxId);
return true; return true;
}else{ }else{
//大于2万条会报事务太长，需要分页插入
//xyqb_i_repayment_plan表目前发现有24期的，也就是说xyqb_i_loan_a_m_history_bak关联最大会出现24条
int PLAN_MAX_COMMIT_COUT = BaiHangHistoryConstant.MAX_COMMIT_COUT / 24;
int page = count.intValue() / BaiHangHistoryConstant.MAX_COMMIT_COUT; int id_limit_count = (int) (newMaxId - bakMaxId);
int mode = count.intValue() % BaiHangHistoryConstant.MAX_COMMIT_COUT; int page = id_limit_count / PLAN_MAX_COMMIT_COUT;
int mode = id_limit_count % PLAN_MAX_COMMIT_COUT;
if(mode!=0){ if(mode!=0){
page = page+1; page = page+1;
} }
...@@ -221,58 +243,78 @@ public class LoanApplicationManifestHistoryTask { ...@@ -221,58 +243,78 @@ public class LoanApplicationManifestHistoryTask {
int totalUpdate = 0; int totalUpdate = 0;
for (int p=0 ; p < page; p++){ for (int p=0 ; p < page; p++){
Stopwatch startwatch = Stopwatch.createStarted(); Stopwatch startwatch = Stopwatch.createStarted();
int limit1 = p * BaiHangHistoryConstant.MAX_COMMIT_COUT; long startId = bakMaxId + p * PLAN_MAX_COMMIT_COUT;
int limit2 = (p+1) * BaiHangHistoryConstant.MAX_COMMIT_COUT; long endId = bakMaxId + (p+1) * PLAN_MAX_COMMIT_COUT;
if( limit2 > count.intValue()){ if( endId > newMaxId){
limit2 = count.intValue(); endId = newMaxId;
} }
String tmpInsertSql = inset_sql + (" ORDER BY b.id ASC LIMIT "+limit1+", "+limit2+" ;"); String tmpInsertSql = BaiHangHistoryConstant.SQL_COMMONE_TEMPLATE_PLAN_SYN_DATA;
tmpInsertSql = tmpInsertSql.replace("##STARTID##",""+startId);
tmpInsertSql = tmpInsertSql + (" AND b.id <="+endId+" ;");
int update = xyqbHistoryJdbcTemplate.update(tmpInsertSql); int update = xyqbHistoryJdbcTemplate.update(tmpInsertSql);
totalUpdate+=update; totalUpdate+=update;
log.info("增量同步xyqb_i_repayment_plan表页数page["+p+"], limit1: {} , limit2: {} , 插入条数: {} ", limit1, limit2, update, startwatch.stop().elapsed(TimeUnit.MILLISECONDS)+".ms"); log.info("增量同步xyqb_i_repayment_plan表页数page["+p+"], startId: {} , endId: {} , 插入条数: {} , 耗时: {} ", startId, endId, update, startwatch.stop().elapsed(TimeUnit.MILLISECONDS)+".ms");
} }
log.info("增量同步xyqb_i_repayment_plan表结束, bakMaxId: {} , count: {} , 总插入条数: {} , page: {} , mode: {} ", bakMaxId, count, totalUpdate, page, mode); log.info("增量同步xyqb_i_repayment_plan表结束, bakMaxId: {} , newMaxId: {} , ID差: {} , count: {} , totalUpdate: {} , page: {} , mode: {} ", bakMaxId, newMaxId, (newMaxId-bakMaxId), count, totalUpdate, page, mode);
if(totalUpdate==count){ if(totalUpdate==count){
dingTalk.talk("Info","同步[还款计划]数据成功","实际条数(分页):"+totalUpdate + ", bakMaxId:"+bakMaxId+", newMaxId:"+newMaxId);
return true; return true;
}else{ }else{
dingTalk.talk("Warn","同步[还款计划]数据失败","查询条数:"+count+",实际条数="+totalUpdate + ", bakMaxId:"+bakMaxId+", newMaxId:"+newMaxId);
dingTalk.talk_ToUser("同步[还款计划]数据失败,停止T+1报送！发现问题，请及时处理！");
return false; return false;
} }
} }
} }
public boolean syn_historyAndplan_by_id(){ public boolean syn_historyAndplan_by_id(){
Stopwatch startwatch = Stopwatch.createStarted(); Stopwatch startwatch = Stopwatch.createStarted();
try { try {
String tmp_sql = BaiHangHistoryConstant.SQL_COMMONE_TEMPLATE_HISTORY_BAK_MAX_ID; String tmp_sql1 = BaiHangHistoryConstant.SQL_COMMONE_TEMPLATE_HISTORY_BAK_MAX_ID;
Long bakMaxId = xyqbHistoryJdbcTemplate.queryForObject(tmp_sql,Long.class); Long bakMaxId = xyqbHistoryJdbcTemplate.queryForObject(tmp_sql1,Long.class);
if(bakMaxId!=null && bakMaxId.longValue() >= 0L){
//------------------------------- String tmp_sql2 = BaiHangHistoryConstant.SQL_COMMONE_TEMPLATE_HISTORY_MAX_ID;
boolean table_history = syn_history_by_id(bakMaxId.longValue()); Long newMaxId = xyqbHistoryJdbcTemplate.queryForObject(tmp_sql2,Long.class);
if(table_history){
//2019.11.15 13:37 bakMaxId: 1857710 , newMaxId: 13732716 , count: 1062175 , 差:11875006 , 总插入条数: 1062174 , page: 594
//bakMaxId = 1857710L;
//newMaxId = 13732716L;
boolean table_plan = syn_plan_by_id(bakMaxId.longValue()); if(bakMaxId!=null && bakMaxId.longValue() >= 0L && newMaxId!=null && newMaxId.longValue() >= 0L){
//同步到xyqb_i_loan_a_m_history_bak表-------------------------------
boolean table_history = syn_history_by_id(bakMaxId.longValue(), newMaxId.longValue());
if(table_history){
//同步到xyqb_i_repayment_plan_bak表
boolean table_plan = syn_plan_by_id(bakMaxId.longValue(), newMaxId.longValue());
if(table_plan){ if(table_plan){
log.info("增量同步history&plan表数据结束, nowTime: {} ", LocalDateTime.now(), startwatch.stop().elapsed(TimeUnit.MILLISECONDS)+".ms"); log.info("ALL增量同步history&plan表数据结束, nowTime: {} ", LocalDateTime.now(), startwatch.stop().elapsed(TimeUnit.MILLISECONDS)+".ms");
return true; return true;
}else{ }else{
log.error("增量同步[xyqb_i_repayment_plan_bak]表数据失败,请尽快处理! nowTime: {} , bakMaxId: {} ",LocalDateTime.now(), bakMaxId); log.error("All增量同步[xyqb_i_repayment_plan_bak]表数据失败,请尽快处理! nowTime: {} , bakMaxId: {} , newMaxId: {} ",LocalDateTime.now(), bakMaxId, newMaxId);
////dingTalk.talk("Warn","同步[还款计划]数据失败","同步xyqb_i_repayment_plan_bak表数据失败,请尽快处理! newMaxId:"+newMaxId+",newMaxId="+newMaxId);
} }
}else{ }else{
log.error("增量同步[xyqb_i_loan_a_m_history_bak]表数据失败,请尽快处理! nowTime: {} , bakMaxId: {} ",LocalDateTime.now(), bakMaxId); log.error("增量同步[xyqb_i_loan_a_m_history_bak]表数据失败,请尽快处理! nowTime: {} , bakMaxId: {} , newMaxId: {} ",LocalDateTime.now(), bakMaxId, newMaxId);
////dingTalk.talk("Warn","同步[借款清单]数据失败","同步xyqb_i_loan_a_m_history_bak表数据失败,请尽快处理! newMaxId:"+newMaxId+",newMaxId="+newMaxId);
} }
//------------------------------- //-------------------------------
}else { }else {
log.error("增量同步xyqb_i_repayment_plan数据-历史id为空,停止同步! nowTime: {} , bakMaxId: {} ",LocalDateTime.now(), bakMaxId); log.error("增量同步xyqb_i_repayment_plan数据-历史id为空,停止同步! nowTime: {} , bakMaxId: {} , newMaxId: {} ",LocalDateTime.now(), bakMaxId, newMaxId);
dingTalk.talk("Warn","同步临时表数据失败","查询数据历史bakMaxId或newMaxId为空.");
dingTalk.talk_ToUser("查询数据历史bakMaxId或newMaxId为空,停止T+1报送！发现问题，请及时处理！");
} }
}catch (Exception e){ }catch (Exception e){
log.error("增量同步history&plan表数据异常, nowTime: {} ", LocalDateTime.now(), e); log.error("增量同步history和plan表数据异常, nowTime: {} ", LocalDateTime.now(), e);
dingTalk.talk("Error","同步history和plan表数据异常",e);
dingTalk.talk_ToUser("同步history和plan表数据异常,停止T+1报送！发现问题，请及时处理！");
} }
return false; return false;
......
src/main/java/cn/quantgroup/report/service/manualTool/ManualToolService.java
View file @ 3824b779
...@@ -767,7 +767,7 @@ public class ManualToolService implements CommonSuperService { ...@@ -767,7 +767,7 @@ public class ManualToolService implements CommonSuperService {
public static void writeLog(String msg) { public static void writeLog(String msg) {
System.out.println(msg); System.out.println(msg);
try { try {
String fileName = "D:\\用户目录\\Downloads\\out_log.txt"; String fileName = "D:\\JavaTeam\\D3_build\\out_log.txt";
FileUtils.write(new File(fileName), msg+"\r\n", "UTF-8", true); FileUtils.write(new File(fileName), msg+"\r\n", "UTF-8", true);
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
...@@ -933,6 +933,66 @@ public class ManualToolService implements CommonSuperService { ...@@ -933,6 +933,66 @@ public class ManualToolService implements CommonSuperService {
System.err.println("build data finTechAgencyBusiness end."); System.err.println("build data finTechAgencyBusiness end.");
} }
public static void buildD3_logicerror() {
List<String> logicerrorList = ReadOrWriteTxt.readTxtList("D:\\JavaTeam\\D3_build\\d3_041_errData.txt");
Map<String,String> err_log_loanIdMap = new HashMap<>(logicerrorList.size());
for (String strs : logicerrorList){
String[] tmpArr = strs.split("[|]");
err_log_loanIdMap.put(tmpArr[1],strs);
}
System.out.println("err_log_loanIdMap 大小:"+err_log_loanIdMap.size());
List<String> lineList = ReadOrWriteTxt.readTxtList("D:\\JavaTeam\\D3_build\\D3_04.txt");
System.out.println("lineList:"+lineList.size());
String fileName = "D:\\JavaTeam\\D3_build\\new_D3_04.txt";
int totalCount = 0,removeCount=0;
for(int i=0;i<lineList.size();i++){
System.out.println("START:"+i);
String str = lineList.get(i);
if(str.length() < 30 ){//#applyInfo #singleLoanAccountInfo
continue;
}
RepaymentInfoZhuDai repaymentLoanInfo = new Gson().fromJson(str, new TypeToken<RepaymentInfoZhuDai>(){}.getType());
String loanId = repaymentLoanInfo.getLoanId();
if(err_log_loanIdMap.containsKey(loanId)){
//2|100011189|9|2019-02-22T00:00:00|
String[] errMsgArr = err_log_loanIdMap.get(loanId).split("[|]");
if(repaymentLoanInfo.getTermNo()==Integer.parseInt(errMsgArr[2]) && repaymentLoanInfo.getStatusConfirmAt().equals(errMsgArr[3])){
if(repaymentLoanInfo.getRealRepayment()==null || repaymentLoanInfo.getRealRepayment().intValue()==0){
if(repaymentLoanInfo.getRealRepaymentDate()==null || repaymentLoanInfo.getRealRepaymentDate().equals("")){
writeLog(JSON.toJSONString(repaymentLoanInfo));
removeCount++;
continue;
}
}
}
}
try {
FileUtils.write(new File(fileName), JSON.toJSONString(repaymentLoanInfo)+"\r\n", "UTF-8", true);
} catch (IOException e) {
System.err.println("build D3 data,"+e);
}
totalCount++;
}
System.out.println("build D3 end, lineList="+lineList.size()+",totalCount="+totalCount+",removeCount="+removeCount+", 合="+(totalCount+removeCount));
}
public static void main(String[] args) { public static void main(String[] args) {
// List<String> lineList = ReadOrWriteTxt.readTxtList("D:\\用户目录\\Downloads\\D2_M_01.txt"); // List<String> lineList = ReadOrWriteTxt.readTxtList("D:\\用户目录\\Downloads\\D2_M_01.txt");
// System.out.println("lineList:"+lineList.size()); // System.out.println("lineList:"+lineList.size());
...@@ -950,7 +1010,9 @@ public class ManualToolService implements CommonSuperService { ...@@ -950,7 +1010,9 @@ public class ManualToolService implements CommonSuperService {
//buildD2_inputerror(); //buildD2_inputerror();
buildD2_logicerror(); // buildD2_logicerror();
buildD3_logicerror();
} }
} }
src/main/java/cn/quantgroup/report/utils/dingtalk/DingTalk.java 0 → 100644
View file @ 3824b779
package cn.quantgroup.report.utils.dingtalk;
import cn.quantgroup.report.service.http.IHttpService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.HashMap;
import java.util.Map;
/**
* @author ：dongjianhua
* @date ：Created in 2019/11/1 10:34
* @description：钉钉提醒
* @modified By：
* @version: 1.0
*/
@Component
@Slf4j
public class DingTalk {
@Autowired
private IHttpService iHttpService;
public static final String talkUri = "https://alertserv-dataservice.quantgroup.cn/common/alert/dingtalk";
public static final String webHook = "https://oapi.dingtalk.com/robot/send?access_token=ef3cafb299ec6cb7442b2795d4d0c48e940756f7f267cfa895dbb419adb00128";
@Async
public void talk(String alarmLevel, String msgTitle, String msgContent) {
try{
Map<String, String> param = new HashMap<>();
param.put("webhook", webHook);
param.put("alarmLevel", alarmLevel);//"Warn"
param.put("msgTitle", "量化派2百行：" + msgTitle);
param.put("msgContent", msgContent);
// param.put("toUsers", "15542661376,18612632691");
String resp = iHttpService.post(talkUri, param);
log.info("钉钉报警接口调用结束, resp: {} ", resp);
}catch (Exception e){
log.warn("钉钉报警接口调用异常! ",e);
}
}
@Async
public void talk(String alarmLevel, String msgTitle, Exception e) {
try{
if (e == null) {
return;
}
StringWriter sw = new StringWriter();
e.printStackTrace(new PrintWriter(sw));
Map<String, String> param = new HashMap<>();
param.put("webhook", webHook);
param.put("alarmLevel", alarmLevel);//"Warn"
param.put("msgTitle", "量化派2百行：" + msgTitle);
param.put("msgContent", sw.toString().length() > 500 ? sw.toString().substring(0, 499) : sw.toString());
// param.put("toUsers", "15542661376,18612632691");
String resp = iHttpService.post(talkUri, param);
log.info("测试钉钉报警接口结果!resp:{}", resp);
log.info("钉钉报警接口调用结束, resp: {} ", resp);
}catch (Exception e2){
log.warn("钉钉报警接口调用异常! ",e2);
}
}
@Async
public void talk_ToUser(String msgContent) {
try{
Map<String, String> param = new HashMap<>();
param.put("webhook", webHook);
param.put("alarmLevel", "ERROE");//
param.put("msgTitle", "量化派2百行：数据同步失败");
param.put("msgContent", msgContent);
param.put("toUsers", "18010105506");
String resp = iHttpService.post(talkUri, param);
log.info("钉钉报警接口调用结束, resp: {} ", resp);
}catch (Exception e){
log.warn("钉钉报警接口调用异常! ",e);
}
}
@Async
public void talk(String alarmLevel, String msgTitle, QgException e) {
Map<String, String> param = new HashMap<>();
param.put("webhook", webHook);
param.put("alarmLevel", alarmLevel);//"Warn"
param.put("msgTitle", "量化派2百行：" + msgTitle);
param.put("msgContent", exceptionToString(e));
// param.put("toUsers", "15542661376,18612632691");
String resp = iHttpService.post(talkUri, param);
log.info("测试钉钉报警接口结果!resp:{}", resp);
}
public String exceptionToString(QgException e) {
return new StringBuffer()
.append(e.alarm)
.append(":")
.append(e.message == null ? "" : (e.message.length() > 500 ? e.message.substring(0, 499) : e.message)).toString();
}
}
src/main/java/cn/quantgroup/report/utils/dingtalk/QgException.java 0 → 100644
View file @ 3824b779
package cn.quantgroup.report.utils.dingtalk;
import java.io.PrintWriter;
import java.io.StringWriter;
/**
* @author ：dongjianhua
* @date ：Created in 2019/11/4 10:38
* @description：异常类
* @modified By：
* @version: 1.0
*/
public class QgException extends RuntimeException {
//提醒标题
public String alarm;
//堆栈信息 或者报错信息
public String message;
public QgException(String exMsg) {
alarm = exMsg;
}
public QgException(String exMsg, String message) {
super(message,null);
alarm = exMsg;
this.message = message;
}
public QgException(String exMsg, Exception e) {
super(e);
alarm = exMsg;
message = getStackTrace(e);
}
public static String getStackTrace(Throwable t) {
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
try {
t.printStackTrace(pw);
return sw.toString();
} finally {
pw.close();
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment