Skip to content

B
baihang-report
Commit c9e4fb02 authored by 郝彦辉's avatar 郝彦辉
Browse files
Options

D3筛选工具开发

parent 81ca39a3
Hide whitespace changes
Inline Side-by-side
Showing with 153 additions and 136 deletions
src/main/java/cn/quantgroup/report/service/manualTool/ManualToolService.java
View file @ c9e4fb02
...@@ -1092,67 +1092,72 @@ public class ManualToolService implements CommonSuperService { ...@@ -1092,67 +1092,72 @@ public class ManualToolService implements CommonSuperService {
* 创建时间: 2019.12.22 <br/> * 创建时间: 2019.12.22 <br/>
*/ */
public static String remove_D3_loanId_chunfu(String oldDataFile, String cfLoanIdLogFile, String newFileName, String rmIdFileName,String rmPhoneFileName) { public static String remove_D3_loanId_chunfu(String oldDataFile, String cfLoanIdLogFile, String newFileName, String rmIdFileName,String rmPhoneFileName) {
List<String> logicerrorList = ReadOrWriteTxt.readTxtList(cfLoanIdLogFile); try{
Map<String,String> err_log_loanIdMap = new HashMap<>(logicerrorList.size()); List<String> logicerrorList = ReadOrWriteTxt.readTxtList(cfLoanIdLogFile);
for (String strs : logicerrorList){ Map<String,String> err_log_loanIdMap = new HashMap<>(logicerrorList.size());
//2 SP290206392723262586192081:11_2019-11-03T00:00:00 for (String strs : logicerrorList){
if(strs.trim().length()<10){ //2 SP290206392723262586192081:11_2019-11-03T00:00:00
log.info("remove_D3_loanId_chunfu warn strs="+strs); if(strs.trim().length()<10){
continue; log.info("remove_D3_loanId_chunfu warn strs="+strs);
continue;
}
String[] tmpArr = strs.trim().split("[ ]");
tmpArr = tmpArr[1].split("[_]");
//SP290206392723262586192081,11,2019-11-03T00:00:00
err_log_loanIdMap.put(tmpArr[0].replace(":",",")+","+tmpArr[1], strs.trim());
} }
String[] tmpArr = strs.trim().split("[ ]"); logicerrorList = null;
tmpArr = tmpArr[1].split("[_]"); log.info("remove_D3_loanId_chunfu err_log_loanIdMap 大小:"+err_log_loanIdMap.size());
//SP290206392723262586192081,11,2019-11-03T00:00:00
err_log_loanIdMap.put(tmpArr[0].replace(":",",")+","+tmpArr[1], strs.trim());
}
logicerrorList = null;
log.info("remove_D3_loanId_chunfu err_log_loanIdMap 大小:"+err_log_loanIdMap.size());
List<String> lineList = ReadOrWriteTxt.readTxtList(oldDataFile); List<String> lineList = ReadOrWriteTxt.readTxtList(oldDataFile);
log.info("remove_D3_loanId_chunfu lineList:"+lineList.size()); log.info("remove_D3_loanId_chunfu lineList:"+lineList.size());
int totalCount = 0,removeCount=0,removeCount2=0; int totalCount = 0,removeCount=0,removeCount2=0;
for(int i=0;i<lineList.size();i++){ for(int i=0;i<lineList.size();i++){
log.info("START:"+i); log.info("START:"+i);
String JsonStr = lineList.get(i); String JsonStr = lineList.get(i);
if(JsonStr.length() < 30 ){//#singleLoanRepayInfo if(JsonStr.length() < 30 ){//#singleLoanRepayInfo
continue; continue;
} }
RepaymentInfoZhuDai repaymentLoanInfo = new Gson().fromJson(JsonStr, new TypeToken<RepaymentInfoZhuDai>(){}.getType()); RepaymentInfoZhuDai repaymentLoanInfo = new Gson().fromJson(JsonStr, new TypeToken<RepaymentInfoZhuDai>(){}.getType());
String loanId = repaymentLoanInfo.getLoanId(); String loanId = repaymentLoanInfo.getLoanId();
String key = loanId+","+repaymentLoanInfo.getTermNo()+","+repaymentLoanInfo.getStatusConfirmAt(); String key = loanId+","+repaymentLoanInfo.getTermNo()+","+repaymentLoanInfo.getStatusConfirmAt();
//"errorCode":"D3_041" //"errorCode":"D3_041"
if (err_log_loanIdMap.containsKey(key)) { if (err_log_loanIdMap.containsKey(key)) {
//100011189,9,2019-02-22T00:00:00 //100011189,9,2019-02-22T00:00:00
//删掉 本次还款金额 实际还款时间 为空的一条 "realRepayment":0,"realRepaymentDate":"" //删掉 本次还款金额 实际还款时间 为空的一条 "realRepayment":0,"realRepaymentDate":""
if (repaymentLoanInfo.getRealRepayment() == null || repaymentLoanInfo.getRealRepayment().intValue() == 0) { if (repaymentLoanInfo.getRealRepayment() == null || repaymentLoanInfo.getRealRepayment().intValue() == 0) {
if (repaymentLoanInfo.getRealRepaymentDate() == null || repaymentLoanInfo.getRealRepaymentDate().equals("")) { if (repaymentLoanInfo.getRealRepaymentDate() == null || repaymentLoanInfo.getRealRepaymentDate().equals("")) {
writeLogByName(rmIdFileName, JsonStr); writeLogByName(rmIdFileName, JsonStr);
removeCount++; removeCount++;
continue; continue;
}
} }
} }
}
//{"errorCode":"ERR_30003" //{"errorCode":"ERR_30003"
if(repaymentLoanInfo.getMobile()==null || repaymentLoanInfo.getMobile().length()!=11){ if(repaymentLoanInfo.getMobile()==null || repaymentLoanInfo.getMobile().length()!=11){
writeLogByName(rmPhoneFileName, JsonStr); writeLogByName(rmPhoneFileName, JsonStr);
removeCount2++; removeCount2++;
continue; continue;
} }
try { try {
FileUtils.write(new File(newFileName), JsonStr+"\r\n", "UTF-8", true); FileUtils.write(new File(newFileName), JsonStr+"\r\n", "UTF-8", true);
} catch (IOException e) { } catch (IOException e) {
log.error("remove_D3_loanId_chunfu write newD3 data error,"+e); log.error("remove_D3_loanId_chunfu write newD3 data error,"+e);
}
totalCount++;
} }
totalCount++; log.info("remove_D3_loanId_chunfu end, lineList="+lineList.size()+",totalCount="+totalCount+",removeCount="+removeCount+",removeCount2="+removeCount2+
} ", 合="+(totalCount+removeCount+removeCount2));
log.info("remove_D3_loanId_chunfu end, lineList="+lineList.size()+",totalCount="+totalCount+",removeCount="+removeCount+",removeCount2="+removeCount2+ return "lineList="+lineList.size()+",totalCount="+totalCount+",removeCount="+removeCount+",removeCount2="+removeCount2+ ", 合="+(totalCount+removeCount+removeCount2);
", 合="+(totalCount+removeCount+removeCount2));
return "lineList="+lineList.size()+",totalCount="+totalCount+",removeCount="+removeCount+",removeCount2="+removeCount2+ ", 合="+(totalCount+removeCount+removeCount2); }catch (Exception e){
log.error("check_D3loanId_InD2 Error,",e);
}
return null;
} }
/** /**
...@@ -1163,10 +1168,11 @@ public class ManualToolService implements CommonSuperService { ...@@ -1163,10 +1168,11 @@ public class ManualToolService implements CommonSuperService {
* 创建时间: 2019.12.22 <br/> * 创建时间: 2019.12.22 <br/>
*/ */
public static String check_D3loanId_InD2(String d3LoanIdFile, String d2File, String inFileName, String notInFileName) { public static String check_D3loanId_InD2(String d3LoanIdFile, String d2File, String inFileName, String notInFileName) {
List<String> d2DataList = ReadOrWriteTxt.readTxtList(d2File); try{
Map<String,String> d2_loanIdMap = new HashMap<>(d2DataList.size()); List<String> d2DataList = ReadOrWriteTxt.readTxtList(d2File);
int d2size = 0; Map<String,String> d2_loanIdMap = new HashMap<>(d2DataList.size());
for (String strs : d2DataList){ int d2size = 0;
for (String strs : d2DataList){
/* if(strs.trim().length()<30){ /* if(strs.trim().length()<30){
log.info("check_D3loanId_InD2 warn strs="+strs); log.info("check_D3loanId_InD2 warn strs="+strs);
continue; continue;
...@@ -1174,47 +1180,51 @@ public class ManualToolService implements CommonSuperService { ...@@ -1174,47 +1180,51 @@ public class ManualToolService implements CommonSuperService {
JSONObject d2Json = JSONObject.parseObject(strs); JSONObject d2Json = JSONObject.parseObject(strs);
d2_loanIdMap.put(d2Json.getString("loanId"), d2Json.getString("reqID"));*/ d2_loanIdMap.put(d2Json.getString("loanId"), d2Json.getString("reqID"));*/
if(strs.trim().length()<1){ if(strs.trim().length()<1){
log.info("check_D3loanId_InD2 warn strs="+strs); log.info("check_D3loanId_InD2 warn strs="+strs);
continue; continue;
}
//key=loanId
d2_loanIdMap.put(strs.trim(), ""+(d2size++));
} }
//key=loanId d2DataList = null;
d2_loanIdMap.put(strs.trim(), ""+(d2size++)); log.info("check_D3loanId_InD2 d2DataList大小: {} , d2_loanIdMap大小: {} ", d2DataList.size(), d2_loanIdMap.size());
}
d2DataList = null;
log.info("check_D3loanId_InD2 d2DataList大小: {} , d2_loanIdMap大小: {} ", d2DataList.size(), d2_loanIdMap.size());
//cat lhp_D3.txt |grep -v '#singleLoanRepayInfo' |awk -F '"loanId":"' '{print $2}' |awk -F '","' '{print $1}' |sort |uniq -c |more //cat lhp_D3.txt |grep -v '#singleLoanRepayInfo' |awk -F '"loanId":"' '{print $2}' |awk -F '","' '{print $1}' |sort |uniq -c |more
List<String> d3LoanIdList = ReadOrWriteTxt.readTxtList(d3LoanIdFile); List<String> d3LoanIdList = ReadOrWriteTxt.readTxtList(d3LoanIdFile);
log.info("check_D3loanId_InD2 d3LoanIdList:"+d3LoanIdList.size()); log.info("check_D3loanId_InD2 d3LoanIdList:"+d3LoanIdList.size());
int in=0, notin=0; int in=0, notin=0;
for(int i=0; i<d3LoanIdList.size(); i++){ for(int i=0; i<d3LoanIdList.size(); i++){
if(d3LoanIdList.get(i).trim().length()<1){ if(d3LoanIdList.get(i).trim().length()<1){
log.info("check_D3loanId_InD2 warn strs="+d3LoanIdList.get(i).trim()); log.info("check_D3loanId_InD2 warn strs="+d3LoanIdList.get(i).trim());
continue; continue;
}
// 2 SP12323232
String[] tmpArr = d3LoanIdList.get(i).trim().split("[ ]");
if(d2_loanIdMap.containsKey(tmpArr[1])){
try {
FileUtils.write(new File(inFileName), d3LoanIdList.get(i).trim()+"\r\n", "UTF-8", true);
} catch (IOException e) {
log.error("check_D3loanId_InD2 write in data error,"+e);
} }
in++;
}else{ // 2 SP12323232
try { String[] tmpArr = d3LoanIdList.get(i).trim().split("[ ]");
FileUtils.write(new File(notInFileName), d3LoanIdList.get(i).trim()+"\r\n", "UTF-8", true); if(d2_loanIdMap.containsKey(tmpArr[1])){
} catch (IOException e) { try {
log.error("check_D3loanId_InD2 write notin data error,"+e); FileUtils.write(new File(inFileName), d3LoanIdList.get(i).trim()+"\r\n", "UTF-8", true);
} catch (IOException e) {
log.error("check_D3loanId_InD2 write in data error,"+e);
}
in++;
}else{
try {
FileUtils.write(new File(notInFileName), d3LoanIdList.get(i).trim()+"\r\n", "UTF-8", true);
} catch (IOException e) {
log.error("check_D3loanId_InD2 write notin data error,"+e);
}
notin++;
} }
notin++;
} }
log.info("d3LoanIdList="+d3LoanIdList.size()+",d2_loanIdMap="+d2_loanIdMap.size()+",in="+in+",notin="+notin);
return "d3LoanIdList="+d3LoanIdList.size()+",d2_loanIdMap="+d2_loanIdMap.size()+",in="+in+",notin="+notin;
}catch (Exception e){
log.error("check_D3loanId_InD2 Error,",e);
} }
log.info("d3LoanIdList="+d3LoanIdList.size()+",d2_loanIdMap="+d2_loanIdMap.size()+",in="+in+",notin="+notin); return null;
return "d3LoanIdList="+d3LoanIdList.size()+",d2_loanIdMap="+d2_loanIdMap.size()+",in="+in+",notin="+notin;
} }
/** /**
...@@ -1225,10 +1235,11 @@ public class ManualToolService implements CommonSuperService { ...@@ -1225,10 +1235,11 @@ public class ManualToolService implements CommonSuperService {
* 创建时间: 2019.12.22 <br/> * 创建时间: 2019.12.22 <br/>
*/ */
public static String check_D2loanId_InD3WanJie(String d3LoanIdFile, String d2File, String notWJFileName) { public static String check_D2loanId_InD3WanJie(String d3LoanIdFile, String d2File, String notWJFileName) {
List<String> d2DataList = ReadOrWriteTxt.readTxtList(d2File); try{
Map<String,String> d2_loanIdMap = new HashMap<>(d2DataList.size()); List<String> d2DataList = ReadOrWriteTxt.readTxtList(d2File);
int d2size=0; Map<String,String> d2_loanIdMap = new HashMap<>(d2DataList.size());
for (String strs : d2DataList){ int d2size=0;
for (String strs : d2DataList){
/*if(strs.trim().length()<30){ /*if(strs.trim().length()<30){
log.info("check_D2loanId_InD3WanJie warn strs="+strs); log.info("check_D2loanId_InD3WanJie warn strs="+strs);
continue; continue;
...@@ -1237,23 +1248,23 @@ public class ManualToolService implements CommonSuperService { ...@@ -1237,23 +1248,23 @@ public class ManualToolService implements CommonSuperService {
JSONObject d2Json = JSONObject.parseObject(strs); JSONObject d2Json = JSONObject.parseObject(strs);
d2_loanIdMap.put(d2Json.getString("loanId"), ""+d2Json.getIntValue("totalTerm"));*/ d2_loanIdMap.put(d2Json.getString("loanId"), ""+d2Json.getIntValue("totalTerm"));*/
if(strs.trim().length()<1){ if(strs.trim().length()<1){
log.info("check_D2loanId_InD3WanJie warn strs="+strs); log.info("check_D2loanId_InD3WanJie warn strs="+strs);
continue; continue;
} }
//key=loanId //key=loanId
d2_loanIdMap.put(strs.trim(), ""+(d2size++)); d2_loanIdMap.put(strs.trim(), ""+(d2size++));
} }
log.info("check_D2loanId_InD3WanJie d2DataList大小: {}, d2_loanIdMap大小: {} ",d2DataList.size(), d2_loanIdMap.size()); log.info("check_D2loanId_InD3WanJie d2DataList大小: {}, d2_loanIdMap大小: {} ",d2DataList.size(), d2_loanIdMap.size());
d2DataList = null; d2DataList = null;
//cat lhp_D3.txt |grep '"loanStatus":3,' |awk -F '"loanId":"' '{print $2}' |awk -F '","' '{print $1}' |sort |uniq -c |more //cat lhp_D3.txt |grep '"loanStatus":3,' |awk -F '"loanId":"' '{print $2}' |awk -F '","' '{print $1}' |sort |uniq -c |more
List<String> d3LoanIdList = ReadOrWriteTxt.readTxtList(d3LoanIdFile); List<String> d3LoanIdList = ReadOrWriteTxt.readTxtList(d3LoanIdFile);
log.info("check_D2loanId_InD3WanJie d3LoanIdList:"+d3LoanIdList.size()); log.info("check_D2loanId_InD3WanJie d3LoanIdList:"+d3LoanIdList.size());
int d2rmCount=0, d2_loanIdMapSize=d2_loanIdMap.size(); int d2rmCount=0, d2_loanIdMapSize=d2_loanIdMap.size();
for(int i=0; i<d3LoanIdList.size(); i++){ for(int i=0; i<d3LoanIdList.size(); i++){
/*if(d3LoanIdList.get(i).length()<30){ /*if(d3LoanIdList.get(i).length()<30){
log.info("check_D2loanId_InD3WanJie warn strs="+d3LoanIdList.get(i)); log.info("check_D2loanId_InD3WanJie warn strs="+d3LoanIdList.get(i));
continue; continue;
...@@ -1269,40 +1280,44 @@ public class ManualToolService implements CommonSuperService { ...@@ -1269,40 +1280,44 @@ public class ManualToolService implements CommonSuperService {
} }
}*/ }*/
if(d3LoanIdList.get(i).length()<1){ if(d3LoanIdList.get(i).length()<1){
log.info("check_D2loanId_InD3WanJie warn d3Sort="+d3LoanIdList.get(i)); log.info("check_D2loanId_InD3WanJie warn d3Sort="+d3LoanIdList.get(i));
continue; continue;
} }
// 2 SP12323232 // 2 SP12323232
String[] tmpArr = d3LoanIdList.get(i).trim().split("[ ]"); String[] tmpArr = d3LoanIdList.get(i).trim().split("[ ]");
if(tmpArr.length>=2){ if(tmpArr.length>=2){
if(d2_loanIdMap.containsKey(tmpArr[1])){ if(d2_loanIdMap.containsKey(tmpArr[1])){
log.info("CHECK_D2LOANID_IND3WANJIE IN d2Map rm>>"+tmpArr[1]+","+d2_loanIdMap.get(tmpArr[1])); log.info("CHECK_D2LOANID_IND3WANJIE IN d2Map rm>>"+tmpArr[1]+","+d2_loanIdMap.get(tmpArr[1]));
d2_loanIdMap.remove(tmpArr[1]); d2_loanIdMap.remove(tmpArr[1]);
d2rmCount++; d2rmCount++;
}else{
log.error("CHECK_D2LOANID_IND3WANJIE not IN d2Map d3Msg>>>"+d3LoanIdList.get(i));
}
}else{ }else{
log.error("CHECK_D2LOANID_IND3WANJIE not IN d2Map d3Msg>>>"+d3LoanIdList.get(i)); log.error("CHECK_D2LOANID_IND3WANJIE d3Sort !length>=2 :"+d3LoanIdList.get(i));
} }
}else{
log.error("CHECK_D2LOANID_IND3WANJIE d3Sort !length>=2 :"+d3LoanIdList.get(i));
} }
} d3LoanIdList = null;
d3LoanIdList = null; log.info("check_D2loanId_InD3WanJie d2_loanIdMapSize:"+d2_loanIdMapSize+",d2rmCount="+d2rmCount+",d2have="+d2_loanIdMap.size());
log.info("check_D2loanId_InD3WanJie d2_loanIdMapSize:"+d2_loanIdMapSize+",d2rmCount="+d2rmCount+",d2have="+d2_loanIdMap.size());
if(d2_loanIdMap.size()>0){ if(d2_loanIdMap.size()>0){
for(Map.Entry<String,String> d2Id_totalTerm : d2_loanIdMap.entrySet()){ for(Map.Entry<String,String> d2Id_totalTerm : d2_loanIdMap.entrySet()){
String msg = d2Id_totalTerm.getKey()+","+d2Id_totalTerm.getValue(); String msg = d2Id_totalTerm.getKey()+","+d2Id_totalTerm.getValue();
try { try {
FileUtils.write(new File(notWJFileName), msg+"\r\n", "UTF-8", true); FileUtils.write(new File(notWJFileName), msg+"\r\n", "UTF-8", true);
} catch (IOException e) { } catch (IOException e) {
log.error("check_D2loanId_InD3WanJie notWJFile error,"+e); log.error("check_D2loanId_InD3WanJie notWJFile error,"+e);
}
} }
} }
}
return "check_D2loanId_InD3WanJie d2_loanIdMapSize:"+d2_loanIdMapSize+",d2rmCount="+d2rmCount+",d2have="+d2_loanIdMap.size(); return "check_D2loanId_InD3WanJie d2_loanIdMapSize:"+d2_loanIdMapSize+",d2rmCount="+d2rmCount+",d2have="+d2_loanIdMap.size();
}catch (Exception e){
log.error("check_D3loanId_InD2 Error,",e);
}
return null;
} }
...@@ -1748,9 +1763,8 @@ public class ManualToolService implements CommonSuperService { ...@@ -1748,9 +1763,8 @@ public class ManualToolService implements CommonSuperService {
logicerror_035_List = null; logicerror_035_List = null;
manualGC(); manualGC();
//"ok1_D3_20161101_20180801.txt","ok2_D3_20180801_20181117.txt","ok3_D3_20181117_20190201.txt", "ok4_D3_20190201_20190410.txt","ok5_D3_20190410_20190701.txt","ok6_D3_20190701_20190910.txt","ok9_D3_20191025_0027.txt"
String[] all_id3_fileNames = {"ok1_D3_20161101_20180801.txt","ok2_D3_20180801_20181117.txt","ok3_D3_20181117_20190201.txt", String[] all_id3_fileNames = {"ok6_D3_20190701_20190910.txt","ok9_D3_20191025_0027.txt"};
"ok4_D3_20190201_20190410.txt","ok5_D3_20190410_20190701.txt","ok6_D3_20190701_20190910.txt","ok9_D3_20191025_0027.txt"};
List<String> d3lineList = null; List<String> d3lineList = null;
RepaymentInfoZhuDai repaymentLoanInfo = null; RepaymentInfoZhuDai repaymentLoanInfo = null;
...@@ -1789,6 +1803,9 @@ public class ManualToolService implements CommonSuperService { ...@@ -1789,6 +1803,9 @@ public class ManualToolService implements CommonSuperService {
} }
} }
System.out.println(fileName+":build D3 end."); System.out.println(fileName+":build D3 end.");
d3lineList = null;
repaymentLoanInfo = null;
manualGC();
} }
System.out.println("All d3_logicerror_20191223() end."); System.out.println("All d3_logicerror_20191223() end.");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment