Skip to content

Q
qg-dbc-spark
Commit 4236b8e1 authored by data爬虫-冯 军凯's avatar data爬虫-冯 军凯
Browse files
Options

申请单数据清洗test

parent abc4d176
Hide whitespace changes
Inline Side-by-side
Showing with 3 additions and 3 deletions
src/main/java/cn/quantgroup/dbc/spark/applyList/ApplyListExportToBizFlowMain.java
View file @ 4236b8e1
...@@ -23,7 +23,7 @@ public class ApplyListExportToBizFlowMain { ...@@ -23,7 +23,7 @@ public class ApplyListExportToBizFlowMain {
SparkSession ss = SparkSession.builder().appName("DBC").getOrCreate(); SparkSession ss = SparkSession.builder().appName("DBC").getOrCreate();
System.out.println("开始加载数据"); System.out.println("开始加载数据");
Dataset<String> userInfoListDataSet = ss.read().textFile("hdfs:///app/user/data/user_data/feng.ren/applyList/userInfoNew1/part-m-00000"); Dataset<String> userInfoListDataSet = ss.read().textFile("hdfs:///app/user/data/user_data/feng.ren/applyList/userInfoNew11/part-m-00000");
Dataset<UserInfoDataRow> userInfoListDataSetMap = userInfoListDataSet.map((MapFunction<String, UserInfoDataRow>) line -> { Dataset<UserInfoDataRow> userInfoListDataSetMap = userInfoListDataSet.map((MapFunction<String, UserInfoDataRow>) line -> {
UserInfoDataRow person = new UserInfoDataRow(); UserInfoDataRow person = new UserInfoDataRow();
...@@ -35,7 +35,7 @@ public class ApplyListExportToBizFlowMain { ...@@ -35,7 +35,7 @@ public class ApplyListExportToBizFlowMain {
System.out.println("转换用户信息完成" + userInfoListDataSetMap.count()); System.out.println("转换用户信息完成" + userInfoListDataSetMap.count());
Dataset<String> quotaCreditDataSet = ss.read().textFile("hdfs:///app/user/data/user_data/feng.ren/applyList/quota_credit1/part-m-00000"); Dataset<String> quotaCreditDataSet = ss.read().textFile("hdfs:///app/user/data/user_data/feng.ren/applyList/quota_credit11/part-m-00000");
Dataset<QuotaCredit> quotaCreditDataSetMap = quotaCreditDataSet.map((MapFunction<String, QuotaCredit>) line -> { Dataset<QuotaCredit> quotaCreditDataSetMap = quotaCreditDataSet.map((MapFunction<String, QuotaCredit>) line -> {
String quotaCreditTmp[] = line.split("\t"); String quotaCreditTmp[] = line.split("\t");
...@@ -59,7 +59,7 @@ public class ApplyListExportToBizFlowMain { ...@@ -59,7 +59,7 @@ public class ApplyListExportToBizFlowMain {
System.out.println("转换申请单完成" + quotaCreditDataSetMap.count()); System.out.println("转换申请单完成" + quotaCreditDataSetMap.count());
Dataset<String> applyQuotaRecordDataSet = ss.read().textFile("hdfs:///app/user/data/user_data/feng.ren/applyList/apply_quota_record1/part-m-00000"); Dataset<String> applyQuotaRecordDataSet = ss.read().textFile("hdfs:///app/user/data/user_data/feng.ren/applyList/apply_quota_record11/part-m-00000");
Dataset<ApplyQuotaRecord> applyQuotaRecordDataSetMap = applyQuotaRecordDataSet.map((MapFunction<String, ApplyQuotaRecord>) line -> { Dataset<ApplyQuotaRecord> applyQuotaRecordDataSetMap = applyQuotaRecordDataSet.map((MapFunction<String, ApplyQuotaRecord>) line -> {
String applyQuotaRecordTmp[] = line.split("\t"); String applyQuotaRecordTmp[] = line.split("\t");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment