Skip to content

Z
zhj-report
Commit 7270228a authored by 董建华's avatar 董建华
Browse files
Options

新增文件存到数据库功能

parent 872bf201
Hide whitespace changes
Inline Side-by-side
Showing with 122 additions and 12 deletions
src/main/java/com/controller/ReportController.java
View file @ 7270228a
......@@ -40,6 +40,15 @@ public class ReportController {
private ReportTimer reportTimer;
public static boolean isrun = false;
@RequestMapping("insertLog")
public String insertLog(String token) {
if (!"4745589d-f394-9028-26e5-2913aa252917".equals(token)) {
return "token有误";
}
reportService.insertLog();
return "任务开始执行";
}
@RequestMapping("report")
public String report(String token,String type) {
if (!"4745589d-f394-9028-26e5-2913aa252917".equals(token)) {
......
src/main/java/com/mapper/report/ReportRecordMapper.java
View file @ 7270228a
......@@ -14,6 +14,6 @@ public interface ReportRecordMapper {
int updateByPrimaryKeySelectiveByFileName(ReportRecord record);
int banchInsert(@Param("records") List<Map<String,Object>> params);
List<Map<String,Object>> getDataByTYpe(String type);
List<ReportRecord> getAllRecord();
}
\ No newline at end of file
src/main/java/com/service/report/ReportService.java
View file @ 7270228a
......@@ -43,4 +43,10 @@ public interface ReportService {
void runMonthData(String date);
void autoReReport(String path);
void insertReportLog(String filePath);
void insertReportLogAsync(List<Map<String, Object>> list,FileEntry fileEntry,String type);
void insertLog();
}
src/main/java/com/service/report/impl/IReportServiceImpl.java
View file @ 7270228a
......@@ -18,21 +18,21 @@ import com.sun.jersey.api.client.ClientResponse;
import com.timer.ReportTimer;
import com.util.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.io.FileUtils;
import org.apache.ibatis.annotations.Case;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import java.io.*;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
......@@ -518,6 +518,7 @@ public class IReportServiceImpl implements ReportService {
reportData(fileEntry);
log.info("放款数据上报结束");
ReportTimer.result.put("T+1放款数据条数：",loanDayData.size());
insertReportLogAsync(loanDayData, fileEntry,"1");
} catch (QgException e) {
log.error("上报T+1放款数据异常，e : {} ", e);
dingTalk.talk("上报T+1放款数据告警", e);
......@@ -544,6 +545,7 @@ public class IReportServiceImpl implements ReportService {
reportData(fileEntry);
log.info("还款数据上报结束");
ReportTimer.result.put("T+1还款数据条数：",repayDayData.size());
insertReportLogAsync(repayDayData,fileEntry,"2");
} catch (QgException e) {
log.error("上报T+1还款数据异常，e : {} ", e);
dingTalk.talk("上报T+1还款数据告警", e);
......@@ -572,6 +574,7 @@ public class IReportServiceImpl implements ReportService {
reportData(fileEntry);
log.info("逾期数据上报结束");
ReportTimer.result.put("T+1逾期数据条数：",overDayData.size());
insertReportLogAsync(overDayData, fileEntry,"3");
} catch (QgException e) {
log.error("上报T+1逾期数据异常，e : {} ", e);
dingTalk.talk("上报T+1逾期数据告警", e);
......@@ -599,6 +602,7 @@ public class IReportServiceImpl implements ReportService {
reportData(fileEntry);
log.info("月逾期数据上报结束 result: {}");
ReportTimer.result.put("月逾期数据条数：",overMonthData.size());
insertReportLogAsync(overMonthData, fileEntry,"4");
} catch (QgException e) {
log.error("上报月逾期数据异常，e : {} ", e);
dingTalk.talk("上报月逾期数据告警", e);
......@@ -623,7 +627,7 @@ public class IReportServiceImpl implements ReportService {
Map<String, List<Map<String, Object>>> dataSync = getDataSync(param);
Set<Map.Entry<String, List<Map<String, Object>>>> entries = dataSync.entrySet();
for (Map.Entry<String, List<Map<String, Object>>> entry : entries) {
List<List<Map<String, Object>>> lists = utils.subList(entry.getValue(), 1000, entry.getKey());
List<List<Map<String, Object>>> lists = utils.subList(entry.getValue(), 1000, entry.getKey(),null);
for (List<Map<String, Object>> list : lists) {
reportRecordMapper.banchInsert(list);
}
......@@ -646,7 +650,7 @@ public class IReportServiceImpl implements ReportService {
param.put("startDate", DateUtil.getCurrentDay(date, -1, DateUtil.YYYY_MM_DD, DateUtil.YYYY_MM_DD));
param.put("endDate", date);
List<Map<String, Object>> monthData = getMonthData(param);
List<List<Map<String, Object>>> lists = utils.subList(monthData, 1000, "4");
List<List<Map<String, Object>>> lists = utils.subList(monthData, 1000, "4",null);
for (List<Map<String, Object>> list : lists) {
reportRecordMapper.banchInsert(list);
}
......@@ -667,6 +671,41 @@ public class IReportServiceImpl implements ReportService {
reReportClassify(classify);
}
@Override
public void insertReportLog(String filePath) {
log.info("开始处理filename:{}",filePath);
List<Map<String, Object>> maps = Utils.unZip(filePath);
List<List<Map<String, Object>>> lists = utils.subList(maps, 1000, null,null);
for (List<Map<String, Object>> logs : lists) {
reportRecordMapper.banchInsert(logs);
}
log.info("filename:{},处理完成",filePath);
}
@Override
public void insertReportLogAsync(final List<Map<String, Object>> list,final FileEntry fileEntry,final String type) {
ThreadPoolUtil.getInstance().execute(()->{
List<List<Map<String, Object>>> lists = utils.subList(list, 1000, type,fileEntry.getFileName());
for (List<Map<String, Object>> logs : lists) {
reportRecordMapper.banchInsert(logs);
}
});
}
@Override
public void insertLog() {
List<ReportRecord> reportedRecord = reportRecordMapper.getAllRecord();
int count = 0;
for (ReportRecord record : reportedRecord) {
count++;
String filepath = record.getFilepath();
String filename = record.getFilename();
String path = filepath +File.separator+filename+ZIP_SUFFIX;
insertReportLog(path);
log.info("处理进度 {}/{}",count,reportedRecord.size());
}
}
public Map<ErrorReson, List<String>> classify(List<String> data) {
Map<ErrorReson, List<String>> rsult = new HashMap<>();
for (String datum : data) {
......
src/main/java/com/util/Utils.java
View file @ 7270228a
......@@ -4,8 +4,10 @@ import com.QgException;
import com.cfcc.jaf.crypto.CryptoUtil;
import com.cfcc.jaf.crypto.sm.SMUtil;
import com.config.ReportConfig;
import com.service.report.impl.IReportServiceImpl;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.bouncycastle.math.ec.ECPoint;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
......@@ -22,6 +24,8 @@ import java.security.NoSuchAlgorithmException;
import java.util.*;
import java.util.Map.Entry;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
public class Utils {
......@@ -316,7 +320,7 @@ public class Utils {
* @param <T>
* @return
*/
public <T> List<List<T>> subList(List<T> data, int size, String type) {
public <T> List<List<T>> subList(List<T> data, int size, String type,String filename) {
List<List<T>> result = new ArrayList<>();
int allSize = data.size();
......@@ -326,7 +330,12 @@ public class Utils {
for (int j = i * size; (j < (i + 1) * size && j < allSize); j++) {
T t = data.get(j);
if (t instanceof Map) {
((Map<String, Object>) t).put("type", type);
if(!StringUtils.isEmpty(type)){
((Map<String, Object>) t).put("type", type);
}
if(!StringUtils.isEmpty(filename)){
((Map<String, Object>) t).put("filename", filename);
}
}
item.add(t);
}
......@@ -427,6 +436,42 @@ public class Utils {
}
return charset;
}
public static List<Map<String, Object>> unZip(String filePath){
List<Map<String, Object>> result = new ArrayList<>();
try {
ZipFile zipFile = new ZipFile(filePath);
Enumeration<? extends ZipEntry> entries = zipFile.entries();
while (entries.hasMoreElements()){
ZipEntry zipEntry = entries.nextElement();
InputStream inputStream = zipFile.getInputStream(zipEntry);
InputStreamReader isr = new InputStreamReader(inputStream, "GBK");
BufferedReader br = new BufferedReader(isr);
try {
String line = "";
while ((line = br.readLine()) != null) {
result.add(getLine(line,zipFile.getName().substring(26,53)));
}
}finally {
br.close();
isr.close();
zipEntry.clone();
}
}
zipFile.close();
} catch (IOException e) {
e.printStackTrace();
}
return result;
}
private static Map getLine(String line,String filename){
Map result = new HashMap();
String [] colums = line.split(",");
String[] keys = IReportServiceImpl.KEYS;
for (int i = 0; i<keys.length;i++){
result.put(keys[i],colums[i]);
}
result.put("filename",filename);
return result;
}
}
src/main/resources/mybatis/report/ReportRecordMapper.xml
View file @ 7270228a
......@@ -79,10 +79,12 @@
<insert id="banchInsert" parameterType="ArrayList">
INSERT INTO
report_data_tmp(name,pidType,pid,businessHappenOrgan,loanId,businessType,businessKind,openDate,dueDate,creditMoney,businessHappenDate,balance,currentOverdueTotal,repaymentStatus,type,createTime)
report_data_tmp(filename,name,pidType,pid,businessHappenOrgan,loanId,businessType,businessKind,openDate,dueDate,creditMoney,businessHappenDate,balance,currentOverdueTotal,repaymentStatus,type,createTime)
VALUES
<foreach collection="records" item="emp" separator=",">
(#{emp.name}
(
#{emp.filename}
,#{emp.name}
,#{emp.pidType}
,#{emp.pid}
,#{emp.businessHappenOrgan}
......@@ -106,4 +108,13 @@
name,pidType,pid,businessHappenOrgan,loanId,businessType,businessKind,openDate,dueDate,creditMoney,businessHappenDate,balance,currentOverdueTotal,repaymentStatus
from report_data_tmp WHERE type =#{type} ORDER BY businessHappenDate asc
</select>
<select id="getAllRecord" resultMap="BaseResultMap">
SELECT
<include refid="Base_Column_List"/>
from reportrecord
order by id
</select>
</mapper>
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment