设备指纹实时数据清洗

qg-data-service/pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>cn.quantgroup</groupId>
     <artifactId>qg-data-service</artifactId>
-    <version>1.2.5-SNAPSHOT</version>
+    <version>1.2.6-SNAPSHOT</version>
     <packaging>jar</packaging>
     <name>qg-data-service</name>
 

qg-data-service/src/main/java/cn/quantgroup/qgdataservice/constant/Constant.java

@@ -6,7 +6,7 @@ package cn.quantgroup.qgdataservice.constant;
  */
 public class Constant {
 
-    public static class SQL{
+    public static class SQL {
 
         public static String USER_INFO_ITEM = "INSERT IGNORE INTO comservice_i_spider_user_info (uuid,realName,registerDate,idCard,phoneRemain,phone,addr,merry,cardType,cardNo,sex,loginName,userSource,timestamp) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?)";  //14
 
@@ -32,10 +32,25 @@ public class Constant {
 
         public static String PHONE_RECHARGE_INFOS = "INSERT IGNORE INTO comservice_i_phone_payment_info (uuid,phone,payamount,paytime,paymethod,paychannel,payspare,timestamp) values (?,?,?,?,?,?,?,?)";//8
 
+        public static String DEVICE_FINGERS_INFOS = "INSERT IGNORE INTO comservice_i_finger_print_info(userid,user_type,createdate,memo,params,timestamp,htime," +
+                "cpuArchitecture,cpuSpeed,cpuHardware,cpuMaxFreq,cpuMinFreq,cpuCurFreq,cpuCount,cpuSerial,cpuABI,cpuABI2,user,totalStorage," +
+                "totalSDStorage,freeSDStorage,totalMemory,totalsys,avaisys,diskUsedStorage,diskFreeStorage,usedMemory,simState," +
+                "imsi,simSerial,phoneType,country,simName,simMcc,simMnc,simLocation,simCid,baseStationLongitude,baseStationLatitude," +
+                "simRssi,nbasestation,wifiDNS,wifiGateway,wifiIP,wifiNetmask,wifiRssi,wifiMac,wifiList,longitude,latitude,osVersion," +
+                "osName,osVersionInt,codeName,signatures,networkType,activetime,systemUpTime,userAgent,tags,language,type,deviceId," +
+                "androidId,bluetoothMac,bluetoothName,baseBandVersion,board,bootLoader,brand,device,display,host,id,manufacturer," +
+                "model,product,resolution,screenDensity,brightness,touchScreen,simulator,rooted,hasCellular,hasCellular1,hasWiFi," +
+                "hasGps,hasTelephony,hasNFC,hasNFCHost,hasBluetooth,hasWiFiDirect,hasOTG,hasAOA,timeZone,sensors,deviceType,deviceName," +
+                "deviceModel,screenW,screenH,breakFlag,appName,appVersion,idfv,idfa,carrier,isConnWifi,isConnCell,isHasCellular,isHasWiFi," +
+                "isHasTelephony,isHasBluetooth,isHeadphonesAttached,batteryStat,isFullCharged,ip,allApps)" +
+                " values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?," +
+                "?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?," +
+                "?,?,?,?,?,?,?)";
+
     }
 
 
-    public static class OPERATOR{
+    public static class OPERATOR {
 
         public static final String MOBILE_USERSOURCE_CALL = "MOBILE_USERSOURCE_CALL";
 
@@ -64,7 +79,13 @@ public class Constant {
 
     }
 
-    public static class BATCH{
+    public static class DEVICE {
+
+        public static final String DEVICE_FINGERS_INFOS = "DEVICE_FINGERS_INFOS";
+
+    }
+
+    public static class BATCH {
         public static final int BATCH_SIZE = 1000;
     }
 

qg-data-service/src/main/java/cn/quantgroup/qgdataservice/service/hbase/DeviceFingerInfoService.java

+package cn.quantgroup.qgdataservice.service.hbase;
+
+import cn.quantgroup.qgdataservice.service.tidb.DeviceFingerInfoCleaningService;
+import com.lkb.data.hbase.dataservice.MobileFingerPrintDataService;
+import com.lkb.data.hbase.row.MobileFingerPrintDataRow;
+
+/**
+ * 设备指纹
+ *
+ * @Author fengjunkai
+ * @Date 2019-06-27 18:38
+ */
+public class DeviceFingerInfoService {
+
+    public static void storageDeviceFingerInfo(MobileFingerPrintDataRow mobileFingerPrintDataRow, String uuid, boolean ka) {
+
+        MobileFingerPrintDataService.put(mobileFingerPrintDataRow);
+
+        DeviceFingerInfoCleaningService.cleanningAndSaveDeviceFingersInfo(mobileFingerPrintDataRow, uuid, ka);
+
+    }
+
+}

qg-data-service/src/main/java/cn/quantgroup/qgdataservice/utils/JdbcExecuters.java

 package cn.quantgroup.qgdataservice.utils;
 
 import cn.quantgroup.qgdataservice.constant.Constant;
+import com.alibaba.fastjson.JSON;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -72,6 +73,43 @@ public class JdbcExecuters {
         }
     }
 
+    public static void prepareBatchUpdate(String sql, List<List<Object>> paramList) {
+        Connection conn = null;
+        PreparedStatement statement = null;
+        try {
+            conn = HIK_DATA_SOURCE.dataSource.getConnection();
+            statement = conn.prepareStatement(sql);
+            conn.setAutoCommit(false);
+            for (List<Object> param : paramList) {
+                try {
+                    for (int i = 0; i < param.size(); i++) {
+                        Object object = param.get(i);
+                        if (object == null || "".equals(object)) {
+                            statement.setObject(i + 1, null);
+                        } else if (object instanceof String) {
+                            statement.setString(i + 1, (String) object);
+                        } else if (object instanceof Long) {
+                            statement.setLong(i + 1, (Long) object);
+                        } else {
+                            statement.setObject(i + 1, object);
+                        }
+                    }
+                    statement.addBatch();
+                } catch (SQLException e) {
+                    log.error("设备指纹清洗数据批量插入数据SQL异常 , param: {} ", JSON.toJSONString(param), e);
+                } catch (Exception e) {
+                    log.error("设备指纹清洗数据批量插入数据未知异常, param: {} ", JSON.toJSONString(param), e);
+                }
+            }
+            statement.executeBatch();
+            conn.commit();
+        } catch (Exception e) {
+            log.error("设备指纹清洗数据批量插入数据未知异常, sql: {} ", sql, e);
+        } finally {
+            close(conn, statement, null);
+        }
+    }
+
     public static int batchExecute(List<List<String>> list, String sql) {
         Connection conn = null;
         PreparedStatement ps = null;
@@ -157,7 +195,7 @@ public class JdbcExecuters {
             if (conn != null)
                 conn.close();
         } catch (SQLException e) {
-            e.printStackTrace();
+            log.error("清洗数据关闭jdbc资源异常", e);
         }
 
     }