开始数据清洗(腾讯云tidb)

qg-data-service/pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>cn.quantgroup</groupId>
     <artifactId>qg-data-service</artifactId>
-    <version>1.2.0-SNAPSHOT</version>
+    <version>1.2.1-SNAPSHOT</version>
     <packaging>jar</packaging>
     <name>qg-data-service</name>
 
@@ -41,6 +41,12 @@
             <version>3.4</version>
         </dependency>
 
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+            <version>5.1.44</version>
+        </dependency>
+
     </dependencies>
 
     <build>

qg-data-service/src/main/java/cn/quantgroup/qgdataservice/config/datasource/TidbDataSource.java

@@ -17,9 +17,12 @@ public enum TidbDataSource {
         if (dataSource == null) {
             System.out.println("======创建TIDB数据库连接======");
             HikariConfig config = new HikariConfig();
-            config.setJdbcUrl("jdbc:mysql://10.17.115.6:4010/rc_comservice_data_pool_v2?useUnicode=true&characterEncoding=UTF8&rewriteBatchedStatements=true");
-            config.setUsername("rc_comservice_data_pool_v2_w");
-            config.setPassword("w9pr8IPJkLmUSBe4");
+//            config.setJdbcUrl("jdbc:mysql://10.17.115.6:4010/rc_comservice_data_pool_v2?useUnicode=true&characterEncoding=UTF8&rewriteBatchedStatements=true");
+//            config.setUsername("rc_comservice_data_pool_v2_w");
+//            config.setPassword("w9pr8IPJkLmUSBe4");
+            config.setJdbcUrl("jdbc:mysql://fengkong-tidb.quantgroups.com:4010/rc_comservice_data_pool_v3?useUnicode=true&characterEncoding=UTF8&rewriteBatchedStatements=true");
+            config.setUsername("rc_comservice_data_pool_v3_w");
+            config.setPassword("L4letZzT1LmPCxAt");
             config.setDriverClassName("com.mysql.jdbc.Driver");
             config.setMaximumPoolSize(20);
             config.setMinimumIdle(10);

qg-data-service/src/main/java/cn/quantgroup/qgdataservice/service/hbase/OperatorInfoService.java

@@ -49,7 +49,7 @@ public class OperatorInfoService {
         else
             SpiderUserItemDataService.put(spiderUserItemDataRow);
 
-//        OperatorInfoCleaningService.cleaningAndSaveUserInfo(spiderUserItemDataRow, uuid, ka);
+        OperatorInfoCleaningService.cleaningAndSaveUserInfo(spiderUserItemDataRow, uuid, ka);
 
     }
 
@@ -66,7 +66,7 @@ public class OperatorInfoService {
         else
             MobileCallInfoDataService.put(mobileCallInfoRows);
 
-//        OperatorInfoCleaningService.cleaningAndSaveMobileCallDetailInfo(mobileCallInfoRows, uuid, ka);
+        OperatorInfoCleaningService.cleaningAndSaveMobileCallDetailInfo(mobileCallInfoRows, uuid, ka);
 
     }
 
@@ -83,7 +83,7 @@ public class OperatorInfoService {
         else
             MobileSmsInfoDataService.put(mobileSmsInfoRows);
 
-//        OperatorInfoCleaningService.cleaningAndSaveMobileSmsDetainInfo(mobileSmsInfoRows, uuid, ka);
+        OperatorInfoCleaningService.cleaningAndSaveMobileSmsDetainInfo(mobileSmsInfoRows, uuid, ka);
 
     }
 
@@ -100,7 +100,7 @@ public class OperatorInfoService {
         else
             MobileFlowInfoDataService.put(mobileFlowInfoRows);
 
-//        OperatorInfoCleaningService.cleaningAndSaveMobileFlowDetailInfo(mobileFlowInfoRows, uuid, ka);
+        OperatorInfoCleaningService.cleaningAndSaveMobileFlowDetailInfo(mobileFlowInfoRows, uuid, ka);
 
     }
 
@@ -117,7 +117,7 @@ public class OperatorInfoService {
         else
             TelecomCallInfoDataService.put(telecomCallInfoRows);
 
-//        OperatorInfoCleaningService.cleaningAndSaveTelecomCallDetailInfo(telecomCallInfoRows, uuid, ka);
+        OperatorInfoCleaningService.cleaningAndSaveTelecomCallDetailInfo(telecomCallInfoRows, uuid, ka);
 
     }
 
@@ -134,7 +134,7 @@ public class OperatorInfoService {
         else
             TelecomSmsInfoDataService.put(telecomSmsInfoRows);
 
-//        OperatorInfoCleaningService.cleaningAndSaveTelecomSmsDetailInfo(telecomSmsInfoRows, uuid, ka);
+        OperatorInfoCleaningService.cleaningAndSaveTelecomSmsDetailInfo(telecomSmsInfoRows, uuid, ka);
 
     }
 
@@ -151,7 +151,7 @@ public class OperatorInfoService {
         else
             TelecomFlowInfoDataService.put(telecomFlowInfoRows);
 
-//        OperatorInfoCleaningService.cleaningAndSaveTelecomFlowDetailInfo(telecomFlowInfoRows, uuid, ka);
+        OperatorInfoCleaningService.cleaningAndSaveTelecomFlowDetailInfo(telecomFlowInfoRows, uuid, ka);
 
     }
 
@@ -168,7 +168,7 @@ public class OperatorInfoService {
         else
             UnicomCallInfoDataService.put(unicomCallInfoRows);
 
-//        OperatorInfoCleaningService.cleaningAndSaveUnicomCallDetailInfo(unicomCallInfoRows, uuid, ka);
+        OperatorInfoCleaningService.cleaningAndSaveUnicomCallDetailInfo(unicomCallInfoRows, uuid, ka);
 
     }
 
@@ -185,7 +185,7 @@ public class OperatorInfoService {
         else
             UnicomSmsInfoDataService.put(unicomSmsInfoRows);
 
-//        OperatorInfoCleaningService.cleaningAndSaveUnicomSmsDetailInfo(unicomSmsInfoRows, uuid, ka);
+        OperatorInfoCleaningService.cleaningAndSaveUnicomSmsDetailInfo(unicomSmsInfoRows, uuid, ka);
 
     }
 
@@ -202,7 +202,7 @@ public class OperatorInfoService {
         else
             UnicomFlowInfoDataService.put(unicomFlowInfoRows);
 
-//        OperatorInfoCleaningService.cleaningAndSaveUnicomFlowDetailInfo(unicomFlowInfoRows, uuid, ka);
+        OperatorInfoCleaningService.cleaningAndSaveUnicomFlowDetailInfo(unicomFlowInfoRows, uuid, ka);
 
     }
 
@@ -219,7 +219,7 @@ public class OperatorInfoService {
         else
             PhoneBillDataService.put(phoneBillDataRow);
 
-//        OperatorInfoCleaningService.cleaningAndSavePhoneBillInfo(phoneBillDataRow, uuid, ka);
+        OperatorInfoCleaningService.cleaningAndSavePhoneBillInfo(phoneBillDataRow, uuid, ka);
 
     }
 
@@ -233,7 +233,7 @@ public class OperatorInfoService {
 
         PhonePaymentDataService.put(phonePaymentDataRow);
 
-//        OperatorInfoCleaningService.cleaningAndSaveRechargeInfo(phonePaymentDataRow, uuid, ka);
+        OperatorInfoCleaningService.cleaningAndSaveRechargeInfo(phonePaymentDataRow, uuid, ka);
 
     }
 

qg-data-service/src/main/java/cn/quantgroup/qgdataservice/utils/JdbcExecuters.java

@@ -162,5 +162,16 @@ public class JdbcExecuters {
 
     }
 
+//    public static void main(String[] args) {
+//        Connection conn = null;
+//        PreparedStatement ps = null;
+//        AtomicInteger atomicInteger = new AtomicInteger();
+//        try {
+//
+//            conn = HIK_DATA_SOURCE.dataSource.getConnection();
+//            System.out.println(conn);
+//        }catch(Exception e){}
+//    }
+
 
 }