Commit 8a338355 authored by data爬虫-冯 军凯's avatar data爬虫-冯 军凯

transactionLog 数据清洗 main类修改1234

parent d7877d2f
package cn.quantgroup.dbc.spark.transactionlog; package cn.quantgroup.dbc.spark.transactionlog;
import cn.quantgroup.dbc.utils.JdbcExecuters; import cn.quantgroup.dbc.utils.JdbcExecuters;
import cn.quantgroup.dbc.utils.PagingUtil;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.SparkSession;
...@@ -9,9 +8,6 @@ import org.apache.spark.sql.SparkSession; ...@@ -9,9 +8,6 @@ import org.apache.spark.sql.SparkSession;
import java.sql.Timestamp; import java.sql.Timestamp;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
/** /**
* @Author fengjunkai * @Author fengjunkai
...@@ -49,7 +45,7 @@ public class CleanningTransactionLogMain { ...@@ -49,7 +45,7 @@ public class CleanningTransactionLogMain {
transactionLog.setUpdated_at(timestamp); transactionLog.setUpdated_at(timestamp);
if (transactionLogs.size() != 0 && transactionLogs.size() % 200 == 0) { if (transactionLogs.size() != 0 && transactionLogs.size() % 200 == 0) {
// JdbcExecuters.prepareBatchUpdateExecuteTransactionid(sql, transactionLogs); JdbcExecuters.prepareBatchUpdateExecuteTransactionid(sql, transactionLogs);
System.out.println("执行sql集合: "+JSON.toJSONString(transactionLogs)); System.out.println("执行sql集合: "+JSON.toJSONString(transactionLogs));
transactionLogs.clear(); transactionLogs.clear();
} }
...@@ -58,7 +54,7 @@ public class CleanningTransactionLogMain { ...@@ -58,7 +54,7 @@ public class CleanningTransactionLogMain {
e.printStackTrace(); e.printStackTrace();
} }
}); });
// JdbcExecuters.prepareBatchUpdateExecuteTransactionid(sql, transactionLogs); JdbcExecuters.prepareBatchUpdateExecuteTransactionid(sql, transactionLogs);
}); });
ss.stop(); ss.stop();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment