Skip to content

B
baihang-report
Commit c9e4fb02 authored by 郝彦辉's avatar 郝彦辉
Browse files
Options

D3筛选工具开发

parent 81ca39a3
Hide whitespace changes
Inline Side-by-side
Showing with 153 additions and 136 deletions
src/main/java/cn/quantgroup/report/service/manualTool/ManualToolService.java
View file @ c9e4fb02
......@@ -1092,67 +1092,72 @@ public class ManualToolService implements CommonSuperService {
* 创建时间: 2019.12.22 <br/>
*/
public static String remove_D3_loanId_chunfu(String oldDataFile, String cfLoanIdLogFile, String newFileName, String rmIdFileName,String rmPhoneFileName) {
List<String> logicerrorList = ReadOrWriteTxt.readTxtList(cfLoanIdLogFile);
Map<String,String> err_log_loanIdMap = new HashMap<>(logicerrorList.size());
for (String strs : logicerrorList){
//2 SP290206392723262586192081:11_2019-11-03T00:00:00
if(strs.trim().length()<10){
log.info("remove_D3_loanId_chunfu warn strs="+strs);
continue;
try{
List<String> logicerrorList = ReadOrWriteTxt.readTxtList(cfLoanIdLogFile);
Map<String,String> err_log_loanIdMap = new HashMap<>(logicerrorList.size());
for (String strs : logicerrorList){
//2 SP290206392723262586192081:11_2019-11-03T00:00:00
if(strs.trim().length()<10){
log.info("remove_D3_loanId_chunfu warn strs="+strs);
continue;
}
String[] tmpArr = strs.trim().split("[ ]");
tmpArr = tmpArr[1].split("[_]");
//SP290206392723262586192081,11,2019-11-03T00:00:00
err_log_loanIdMap.put(tmpArr[0].replace(":",",")+","+tmpArr[1], strs.trim());
}
String[] tmpArr = strs.trim().split("[ ]");
tmpArr = tmpArr[1].split("[_]");
//SP290206392723262586192081,11,2019-11-03T00:00:00
err_log_loanIdMap.put(tmpArr[0].replace(":",",")+","+tmpArr[1], strs.trim());
}
logicerrorList = null;
log.info("remove_D3_loanId_chunfu err_log_loanIdMap 大小:"+err_log_loanIdMap.size());
logicerrorList = null;
log.info("remove_D3_loanId_chunfu err_log_loanIdMap 大小:"+err_log_loanIdMap.size());
List<String> lineList = ReadOrWriteTxt.readTxtList(oldDataFile);
log.info("remove_D3_loanId_chunfu lineList:"+lineList.size());
List<String> lineList = ReadOrWriteTxt.readTxtList(oldDataFile);
log.info("remove_D3_loanId_chunfu lineList:"+lineList.size());
int totalCount = 0,removeCount=0,removeCount2=0;
for(int i=0;i<lineList.size();i++){
log.info("START:"+i);
int totalCount = 0,removeCount=0,removeCount2=0;
for(int i=0;i<lineList.size();i++){
log.info("START:"+i);
String JsonStr = lineList.get(i);
if(JsonStr.length() < 30 ){//#singleLoanRepayInfo
continue;
}
RepaymentInfoZhuDai repaymentLoanInfo = new Gson().fromJson(JsonStr, new TypeToken<RepaymentInfoZhuDai>(){}.getType());
String loanId = repaymentLoanInfo.getLoanId();
String key = loanId+","+repaymentLoanInfo.getTermNo()+","+repaymentLoanInfo.getStatusConfirmAt();
//"errorCode":"D3_041"
if (err_log_loanIdMap.containsKey(key)) {
//100011189,9,2019-02-22T00:00:00
//删掉 本次还款金额 实际还款时间 为空的一条 "realRepayment":0,"realRepaymentDate":""
if (repaymentLoanInfo.getRealRepayment() == null || repaymentLoanInfo.getRealRepayment().intValue() == 0) {
if (repaymentLoanInfo.getRealRepaymentDate() == null || repaymentLoanInfo.getRealRepaymentDate().equals("")) {
writeLogByName(rmIdFileName, JsonStr);
removeCount++;
continue;
String JsonStr = lineList.get(i);
if(JsonStr.length() < 30 ){//#singleLoanRepayInfo
continue;
}
RepaymentInfoZhuDai repaymentLoanInfo = new Gson().fromJson(JsonStr, new TypeToken<RepaymentInfoZhuDai>(){}.getType());
String loanId = repaymentLoanInfo.getLoanId();
String key = loanId+","+repaymentLoanInfo.getTermNo()+","+repaymentLoanInfo.getStatusConfirmAt();
//"errorCode":"D3_041"
if (err_log_loanIdMap.containsKey(key)) {
//100011189,9,2019-02-22T00:00:00
//删掉 本次还款金额 实际还款时间 为空的一条 "realRepayment":0,"realRepaymentDate":""
if (repaymentLoanInfo.getRealRepayment() == null || repaymentLoanInfo.getRealRepayment().intValue() == 0) {
if (repaymentLoanInfo.getRealRepaymentDate() == null || repaymentLoanInfo.getRealRepaymentDate().equals("")) {
writeLogByName(rmIdFileName, JsonStr);
removeCount++;
continue;
}
}
}
}
//{"errorCode":"ERR_30003"
if(repaymentLoanInfo.getMobile()==null || repaymentLoanInfo.getMobile().length()!=11){
writeLogByName(rmPhoneFileName, JsonStr);
removeCount2++;
continue;
}
//{"errorCode":"ERR_30003"
if(repaymentLoanInfo.getMobile()==null || repaymentLoanInfo.getMobile().length()!=11){
writeLogByName(rmPhoneFileName, JsonStr);
removeCount2++;
continue;
}
try {
FileUtils.write(new File(newFileName), JsonStr+"\r\n", "UTF-8", true);
} catch (IOException e) {
log.error("remove_D3_loanId_chunfu write newD3 data error,"+e);
try {
FileUtils.write(new File(newFileName), JsonStr+"\r\n", "UTF-8", true);
} catch (IOException e) {
log.error("remove_D3_loanId_chunfu write newD3 data error,"+e);
}
totalCount++;
}
totalCount++;
}
log.info("remove_D3_loanId_chunfu end, lineList="+lineList.size()+",totalCount="+totalCount+",removeCount="+removeCount+",removeCount2="+removeCount2+
", 合="+(totalCount+removeCount+removeCount2));
log.info("remove_D3_loanId_chunfu end, lineList="+lineList.size()+",totalCount="+totalCount+",removeCount="+removeCount+",removeCount2="+removeCount2+
", 合="+(totalCount+removeCount+removeCount2));
return "lineList="+lineList.size()+",totalCount="+totalCount+",removeCount="+removeCount+",removeCount2="+removeCount2+ ", 合="+(totalCount+removeCount+removeCount2);
return "lineList="+lineList.size()+",totalCount="+totalCount+",removeCount="+removeCount+",removeCount2="+removeCount2+ ", 合="+(totalCount+removeCount+removeCount2);
}catch (Exception e){
log.error("check_D3loanId_InD2 Error,",e);
}
return null;
}
/**
......@@ -1163,10 +1168,11 @@ public class ManualToolService implements CommonSuperService {
* 创建时间: 2019.12.22 <br/>
*/
public static String check_D3loanId_InD2(String d3LoanIdFile, String d2File, String inFileName, String notInFileName) {
List<String> d2DataList = ReadOrWriteTxt.readTxtList(d2File);
Map<String,String> d2_loanIdMap = new HashMap<>(d2DataList.size());
int d2size = 0;
for (String strs : d2DataList){
try{
List<String> d2DataList = ReadOrWriteTxt.readTxtList(d2File);
Map<String,String> d2_loanIdMap = new HashMap<>(d2DataList.size());
int d2size = 0;
for (String strs : d2DataList){
/* if(strs.trim().length()<30){
log.info("check_D3loanId_InD2 warn strs="+strs);
continue;
......@@ -1174,47 +1180,51 @@ public class ManualToolService implements CommonSuperService {
JSONObject d2Json = JSONObject.parseObject(strs);
d2_loanIdMap.put(d2Json.getString("loanId"), d2Json.getString("reqID"));*/
if(strs.trim().length()<1){
log.info("check_D3loanId_InD2 warn strs="+strs);
continue;
if(strs.trim().length()<1){
log.info("check_D3loanId_InD2 warn strs="+strs);
continue;
}
//key=loanId
d2_loanIdMap.put(strs.trim(), ""+(d2size++));
}
//key=loanId
d2_loanIdMap.put(strs.trim(), ""+(d2size++));
}
d2DataList = null;
log.info("check_D3loanId_InD2 d2DataList大小: {} , d2_loanIdMap大小: {} ", d2DataList.size(), d2_loanIdMap.size());
d2DataList = null;
log.info("check_D3loanId_InD2 d2DataList大小: {} , d2_loanIdMap大小: {} ", d2DataList.size(), d2_loanIdMap.size());
//cat lhp_D3.txt |grep -v '#singleLoanRepayInfo' |awk -F '"loanId":"' '{print $2}' |awk -F '","' '{print $1}' |sort |uniq -c |more
List<String> d3LoanIdList = ReadOrWriteTxt.readTxtList(d3LoanIdFile);
log.info("check_D3loanId_InD2 d3LoanIdList:"+d3LoanIdList.size());
int in=0, notin=0;
for(int i=0; i<d3LoanIdList.size(); i++){
//cat lhp_D3.txt |grep -v '#singleLoanRepayInfo' |awk -F '"loanId":"' '{print $2}' |awk -F '","' '{print $1}' |sort |uniq -c |more
List<String> d3LoanIdList = ReadOrWriteTxt.readTxtList(d3LoanIdFile);
log.info("check_D3loanId_InD2 d3LoanIdList:"+d3LoanIdList.size());
int in=0, notin=0;
for(int i=0; i<d3LoanIdList.size(); i++){
if(d3LoanIdList.get(i).trim().length()<1){
log.info("check_D3loanId_InD2 warn strs="+d3LoanIdList.get(i).trim());
continue;
}
// 2 SP12323232
String[] tmpArr = d3LoanIdList.get(i).trim().split("[ ]");
if(d2_loanIdMap.containsKey(tmpArr[1])){
try {
FileUtils.write(new File(inFileName), d3LoanIdList.get(i).trim()+"\r\n", "UTF-8", true);
} catch (IOException e) {
log.error("check_D3loanId_InD2 write in data error,"+e);
if(d3LoanIdList.get(i).trim().length()<1){
log.info("check_D3loanId_InD2 warn strs="+d3LoanIdList.get(i).trim());
continue;
}
in++;
}else{
try {
FileUtils.write(new File(notInFileName), d3LoanIdList.get(i).trim()+"\r\n", "UTF-8", true);
} catch (IOException e) {
log.error("check_D3loanId_InD2 write notin data error,"+e);
// 2 SP12323232
String[] tmpArr = d3LoanIdList.get(i).trim().split("[ ]");
if(d2_loanIdMap.containsKey(tmpArr[1])){
try {
FileUtils.write(new File(inFileName), d3LoanIdList.get(i).trim()+"\r\n", "UTF-8", true);
} catch (IOException e) {
log.error("check_D3loanId_InD2 write in data error,"+e);
}
in++;
}else{
try {
FileUtils.write(new File(notInFileName), d3LoanIdList.get(i).trim()+"\r\n", "UTF-8", true);
} catch (IOException e) {
log.error("check_D3loanId_InD2 write notin data error,"+e);
}
notin++;
}
notin++;
}
log.info("d3LoanIdList="+d3LoanIdList.size()+",d2_loanIdMap="+d2_loanIdMap.size()+",in="+in+",notin="+notin);
return "d3LoanIdList="+d3LoanIdList.size()+",d2_loanIdMap="+d2_loanIdMap.size()+",in="+in+",notin="+notin;
}catch (Exception e){
log.error("check_D3loanId_InD2 Error,",e);
}
log.info("d3LoanIdList="+d3LoanIdList.size()+",d2_loanIdMap="+d2_loanIdMap.size()+",in="+in+",notin="+notin);
return "d3LoanIdList="+d3LoanIdList.size()+",d2_loanIdMap="+d2_loanIdMap.size()+",in="+in+",notin="+notin;
return null;
}
/**
......@@ -1225,10 +1235,11 @@ public class ManualToolService implements CommonSuperService {
* 创建时间: 2019.12.22 <br/>
*/
public static String check_D2loanId_InD3WanJie(String d3LoanIdFile, String d2File, String notWJFileName) {
List<String> d2DataList = ReadOrWriteTxt.readTxtList(d2File);
Map<String,String> d2_loanIdMap = new HashMap<>(d2DataList.size());
int d2size=0;
for (String strs : d2DataList){
try{
List<String> d2DataList = ReadOrWriteTxt.readTxtList(d2File);
Map<String,String> d2_loanIdMap = new HashMap<>(d2DataList.size());
int d2size=0;
for (String strs : d2DataList){
/*if(strs.trim().length()<30){
log.info("check_D2loanId_InD3WanJie warn strs="+strs);
continue;
......@@ -1237,23 +1248,23 @@ public class ManualToolService implements CommonSuperService {
JSONObject d2Json = JSONObject.parseObject(strs);
d2_loanIdMap.put(d2Json.getString("loanId"), ""+d2Json.getIntValue("totalTerm"));*/
if(strs.trim().length()<1){
log.info("check_D2loanId_InD3WanJie warn strs="+strs);
continue;
}
//key=loanId
d2_loanIdMap.put(strs.trim(), ""+(d2size++));
if(strs.trim().length()<1){
log.info("check_D2loanId_InD3WanJie warn strs="+strs);
continue;
}
//key=loanId
d2_loanIdMap.put(strs.trim(), ""+(d2size++));
}
log.info("check_D2loanId_InD3WanJie d2DataList大小: {}, d2_loanIdMap大小: {} ",d2DataList.size(), d2_loanIdMap.size());
d2DataList = null;
}
log.info("check_D2loanId_InD3WanJie d2DataList大小: {}, d2_loanIdMap大小: {} ",d2DataList.size(), d2_loanIdMap.size());
d2DataList = null;
//cat lhp_D3.txt |grep '"loanStatus":3,' |awk -F '"loanId":"' '{print $2}' |awk -F '","' '{print $1}' |sort |uniq -c |more
List<String> d3LoanIdList = ReadOrWriteTxt.readTxtList(d3LoanIdFile);
log.info("check_D2loanId_InD3WanJie d3LoanIdList:"+d3LoanIdList.size());
//cat lhp_D3.txt |grep '"loanStatus":3,' |awk -F '"loanId":"' '{print $2}' |awk -F '","' '{print $1}' |sort |uniq -c |more
List<String> d3LoanIdList = ReadOrWriteTxt.readTxtList(d3LoanIdFile);
log.info("check_D2loanId_InD3WanJie d3LoanIdList:"+d3LoanIdList.size());
int d2rmCount=0, d2_loanIdMapSize=d2_loanIdMap.size();
for(int i=0; i<d3LoanIdList.size(); i++){
int d2rmCount=0, d2_loanIdMapSize=d2_loanIdMap.size();
for(int i=0; i<d3LoanIdList.size(); i++){
/*if(d3LoanIdList.get(i).length()<30){
log.info("check_D2loanId_InD3WanJie warn strs="+d3LoanIdList.get(i));
continue;
......@@ -1269,40 +1280,44 @@ public class ManualToolService implements CommonSuperService {
}
}*/
if(d3LoanIdList.get(i).length()<1){
log.info("check_D2loanId_InD3WanJie warn d3Sort="+d3LoanIdList.get(i));
continue;
}
// 2 SP12323232
String[] tmpArr = d3LoanIdList.get(i).trim().split("[ ]");
if(tmpArr.length>=2){
if(d2_loanIdMap.containsKey(tmpArr[1])){
log.info("CHECK_D2LOANID_IND3WANJIE IN d2Map rm>>"+tmpArr[1]+","+d2_loanIdMap.get(tmpArr[1]));
d2_loanIdMap.remove(tmpArr[1]);
d2rmCount++;
if(d3LoanIdList.get(i).length()<1){
log.info("check_D2loanId_InD3WanJie warn d3Sort="+d3LoanIdList.get(i));
continue;
}
// 2 SP12323232
String[] tmpArr = d3LoanIdList.get(i).trim().split("[ ]");
if(tmpArr.length>=2){
if(d2_loanIdMap.containsKey(tmpArr[1])){
log.info("CHECK_D2LOANID_IND3WANJIE IN d2Map rm>>"+tmpArr[1]+","+d2_loanIdMap.get(tmpArr[1]));
d2_loanIdMap.remove(tmpArr[1]);
d2rmCount++;
}else{
log.error("CHECK_D2LOANID_IND3WANJIE not IN d2Map d3Msg>>>"+d3LoanIdList.get(i));
}
}else{
log.error("CHECK_D2LOANID_IND3WANJIE not IN d2Map d3Msg>>>"+d3LoanIdList.get(i));
log.error("CHECK_D2LOANID_IND3WANJIE d3Sort !length>=2 :"+d3LoanIdList.get(i));
}
}else{
log.error("CHECK_D2LOANID_IND3WANJIE d3Sort !length>=2 :"+d3LoanIdList.get(i));
}
}
d3LoanIdList = null;
log.info("check_D2loanId_InD3WanJie d2_loanIdMapSize:"+d2_loanIdMapSize+",d2rmCount="+d2rmCount+",d2have="+d2_loanIdMap.size());
d3LoanIdList = null;
log.info("check_D2loanId_InD3WanJie d2_loanIdMapSize:"+d2_loanIdMapSize+",d2rmCount="+d2rmCount+",d2have="+d2_loanIdMap.size());
if(d2_loanIdMap.size()>0){
for(Map.Entry<String,String> d2Id_totalTerm : d2_loanIdMap.entrySet()){
String msg = d2Id_totalTerm.getKey()+","+d2Id_totalTerm.getValue();
try {
FileUtils.write(new File(notWJFileName), msg+"\r\n", "UTF-8", true);
} catch (IOException e) {
log.error("check_D2loanId_InD3WanJie notWJFile error,"+e);
if(d2_loanIdMap.size()>0){
for(Map.Entry<String,String> d2Id_totalTerm : d2_loanIdMap.entrySet()){
String msg = d2Id_totalTerm.getKey()+","+d2Id_totalTerm.getValue();
try {
FileUtils.write(new File(notWJFileName), msg+"\r\n", "UTF-8", true);
} catch (IOException e) {
log.error("check_D2loanId_InD3WanJie notWJFile error,"+e);
}
}
}
}
return "check_D2loanId_InD3WanJie d2_loanIdMapSize:"+d2_loanIdMapSize+",d2rmCount="+d2rmCount+",d2have="+d2_loanIdMap.size();
return "check_D2loanId_InD3WanJie d2_loanIdMapSize:"+d2_loanIdMapSize+",d2rmCount="+d2rmCount+",d2have="+d2_loanIdMap.size();
}catch (Exception e){
log.error("check_D3loanId_InD2 Error,",e);
}
return null;
}
......@@ -1748,9 +1763,8 @@ public class ManualToolService implements CommonSuperService {
logicerror_035_List = null;
manualGC();
String[] all_id3_fileNames = {"ok1_D3_20161101_20180801.txt","ok2_D3_20180801_20181117.txt","ok3_D3_20181117_20190201.txt",
"ok4_D3_20190201_20190410.txt","ok5_D3_20190410_20190701.txt","ok6_D3_20190701_20190910.txt","ok9_D3_20191025_0027.txt"};
//"ok1_D3_20161101_20180801.txt","ok2_D3_20180801_20181117.txt","ok3_D3_20181117_20190201.txt", "ok4_D3_20190201_20190410.txt","ok5_D3_20190410_20190701.txt","ok6_D3_20190701_20190910.txt","ok9_D3_20191025_0027.txt"
String[] all_id3_fileNames = {"ok6_D3_20190701_20190910.txt","ok9_D3_20191025_0027.txt"};
List<String> d3lineList = null;
RepaymentInfoZhuDai repaymentLoanInfo = null;
......@@ -1789,6 +1803,9 @@ public class ManualToolService implements CommonSuperService {
}
}
System.out.println(fileName+":build D3 end.");
d3lineList = null;
repaymentLoanInfo = null;
manualGC();
}
System.out.println("All d3_logicerror_20191223() end.");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment