Skip to content

Q
qg-dbc-spark
Commit 78a25d8e authored by data爬虫-冯 军凯's avatar data爬虫-冯 军凯
Browse files
Options

申请单数据清洗 fix 正式写

parent c7f3575d
Hide whitespace changes
Inline Side-by-side
Showing with 6 additions and 3 deletions
src/main/java/cn/quantgroup/dbc/spark/applyList/ApplyListExportToBizFlowMain.java
View file @ 78a25d8e
......@@ -23,7 +23,8 @@ public class ApplyListExportToBizFlowMain {
SparkSession ss = SparkSession.builder().appName("DBC").getOrCreate();
System.out.println("开始加载数据");
Dataset<String> userInfoListDataSet = ss.read().textFile("hdfs:///app/user/data/user_data/feng.ren/applyList/userInfoNew11/part-m-00000");
// Dataset<String> userInfoListDataSet = ss.read().textFile("hdfs:///app/user/data/user_data/feng.ren/applyList/userInfoNew11/part-m-00000");
Dataset<String> userInfoListDataSet = ss.read().textFile("hdfs:///app/user/data/user_data/feng.ren/applyList/userInfoNew22/part-m-00000");
Dataset<UserInfoDataRow> userInfoListDataSetMap = userInfoListDataSet.map((MapFunction<String, UserInfoDataRow>) line -> {
UserInfoDataRow person = new UserInfoDataRow();
......@@ -35,7 +36,8 @@ public class ApplyListExportToBizFlowMain {
System.out.println("转换用户信息完成" + userInfoListDataSetMap.count());
Dataset<String> quotaCreditDataSet = ss.read().textFile("hdfs:///app/user/data/user_data/feng.ren/applyList/quota_credit11/part-m-00000");
// Dataset<String> quotaCreditDataSet = ss.read().textFile("hdfs:///app/user/data/user_data/feng.ren/applyList/quota_credit11/part-m-00000");
Dataset<String> quotaCreditDataSet = ss.read().textFile("hdfs:///app/user/data/user_data/feng.ren/applyList/quota_credit22/part-m-00000");
Dataset<QuotaCredit> quotaCreditDataSetMap = quotaCreditDataSet.map((MapFunction<String, QuotaCredit>) line -> {
String quotaCreditTmp[] = line.split("\t");
......@@ -59,7 +61,8 @@ public class ApplyListExportToBizFlowMain {
System.out.println("转换申请单完成" + quotaCreditDataSetMap.count());
Dataset<String> applyQuotaRecordDataSet = ss.read().textFile("hdfs:///app/user/data/user_data/feng.ren/applyList/apply_quota_record11/part-m-00000");
// Dataset<String> applyQuotaRecordDataSet = ss.read().textFile("hdfs:///app/user/data/user_data/feng.ren/applyList/apply_quota_record11/part-m-00000");
Dataset<String> applyQuotaRecordDataSet = ss.read().textFile("hdfs:///app/user/data/user_data/feng.ren/applyList/apply_quota_record22/part-m-00000");
Dataset<ApplyQuotaRecord> applyQuotaRecordDataSetMap = applyQuotaRecordDataSet.map((MapFunction<String, ApplyQuotaRecord>) line -> {
String applyQuotaRecordTmp[] = line.split("\t");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment