Skip to content

Q
qg-rt-dc
Commit 9ac7a242 authored by data爬虫-冯 军凯's avatar data爬虫-冯 军凯
Browse files
Options

设备指纹实时数据清洗

parent 85b675e6
Show whitespace changes
Inline Side-by-side
Showing with 751 additions and 5 deletions
qg-data-service/pom.xml
View file @ 9ac7a242
......@@ -5,7 +5,7 @@
<groupId>cn.quantgroup</groupId>
<artifactId>qg-data-service</artifactId>
<version>1.2.5-SNAPSHOT</version>
<version>1.2.6-SNAPSHOT</version>
<packaging>jar</packaging>
<name>qg-data-service</name>
......
qg-data-service/src/main/java/cn/quantgroup/qgdataservice/constant/Constant.java
View file @ 9ac7a242
......@@ -6,7 +6,7 @@ package cn.quantgroup.qgdataservice.constant;
*/
public class Constant {
public static class SQL{
public static class SQL {
public static String USER_INFO_ITEM = "INSERT IGNORE INTO comservice_i_spider_user_info (uuid,realName,registerDate,idCard,phoneRemain,phone,addr,merry,cardType,cardNo,sex,loginName,userSource,timestamp) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; //14
......@@ -32,10 +32,25 @@ public class Constant {
public static String PHONE_RECHARGE_INFOS = "INSERT IGNORE INTO comservice_i_phone_payment_info (uuid,phone,payamount,paytime,paymethod,paychannel,payspare,timestamp) values (?,?,?,?,?,?,?,?)";//8
public static String DEVICE_FINGERS_INFOS = "INSERT IGNORE INTO comservice_i_finger_print_info(userid,user_type,createdate,memo,params,timestamp,htime," +
"cpuArchitecture,cpuSpeed,cpuHardware,cpuMaxFreq,cpuMinFreq,cpuCurFreq,cpuCount,cpuSerial,cpuABI,cpuABI2,user,totalStorage," +
"totalSDStorage,freeSDStorage,totalMemory,totalsys,avaisys,diskUsedStorage,diskFreeStorage,usedMemory,simState," +
"imsi,simSerial,phoneType,country,simName,simMcc,simMnc,simLocation,simCid,baseStationLongitude,baseStationLatitude," +
"simRssi,nbasestation,wifiDNS,wifiGateway,wifiIP,wifiNetmask,wifiRssi,wifiMac,wifiList,longitude,latitude,osVersion," +
"osName,osVersionInt,codeName,signatures,networkType,activetime,systemUpTime,userAgent,tags,language,type,deviceId," +
"androidId,bluetoothMac,bluetoothName,baseBandVersion,board,bootLoader,brand,device,display,host,id,manufacturer," +
"model,product,resolution,screenDensity,brightness,touchScreen,simulator,rooted,hasCellular,hasCellular1,hasWiFi," +
"hasGps,hasTelephony,hasNFC,hasNFCHost,hasBluetooth,hasWiFiDirect,hasOTG,hasAOA,timeZone,sensors,deviceType,deviceName," +
"deviceModel,screenW,screenH,breakFlag,appName,appVersion,idfv,idfa,carrier,isConnWifi,isConnCell,isHasCellular,isHasWiFi," +
"isHasTelephony,isHasBluetooth,isHeadphonesAttached,batteryStat,isFullCharged,ip,allApps)" +
" values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?," +
"?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?," +
"?,?,?,?,?,?,?)";
}
public static class OPERATOR{
public static class OPERATOR {
public static final String MOBILE_USERSOURCE_CALL = "MOBILE_USERSOURCE_CALL";
......@@ -64,7 +79,13 @@ public class Constant {
}
public static class BATCH{
public static class DEVICE {
public static final String DEVICE_FINGERS_INFOS = "DEVICE_FINGERS_INFOS";
}
public static class BATCH {
public static final int BATCH_SIZE = 1000;
}
......
qg-data-service/src/main/java/cn/quantgroup/qgdataservice/service/hbase/DeviceFingerInfoService.java 0 → 100644
View file @ 9ac7a242
package cn.quantgroup.qgdataservice.service.hbase;
import cn.quantgroup.qgdataservice.service.tidb.DeviceFingerInfoCleaningService;
import com.lkb.data.hbase.dataservice.MobileFingerPrintDataService;
import com.lkb.data.hbase.row.MobileFingerPrintDataRow;
/**
* 设备指纹
*
* @Author fengjunkai
* @Date 2019-06-27 18:38
*/
public class DeviceFingerInfoService {
public static void storageDeviceFingerInfo(MobileFingerPrintDataRow mobileFingerPrintDataRow, String uuid, boolean ka) {
MobileFingerPrintDataService.put(mobileFingerPrintDataRow);
DeviceFingerInfoCleaningService.cleanningAndSaveDeviceFingersInfo(mobileFingerPrintDataRow, uuid, ka);
}
}
qg-data-service/src/main/java/cn/quantgroup/qgdataservice/service/tidb/DeviceFingerInfoCleaningService.java 0 → 100644
View file @ 9ac7a242
package cn.quantgroup.qgdataservice.service.tidb;
import cn.quantgroup.qgdataservice.constant.Constant;
import cn.quantgroup.qgdataservice.utils.JdbcExecuters;
import cn.quantgroup.qgdataservice.utils.ThreadPoolExecutorUtils;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.common.base.Stopwatch;
import com.lkb.data.hbase.row.MobileFingerPrintDataRow;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
/**
* @Author fengjunkai
* @Date 2019-06-27 18:41
*/
public class DeviceFingerInfoCleaningService {
private static final Logger log = LoggerFactory.getLogger(DeviceFingerInfoCleaningService.class);
public static final DecimalFormat format = new DecimalFormat("#0.000000");
public static final DecimalFormat storageFormat = new DecimalFormat("#0.00");
private static final String UNEXPECTED = "-1";
static Set<String> dataColumns = new LinkedHashSet<>();
static Set<String> booleanColumns = new HashSet<>();
static Set<String> longColumns = new HashSet<>();
static {
dataColumns.add("cpuArchitecture");
dataColumns.add("cpuSpeed");
dataColumns.add("cpuHardware");
dataColumns.add("cpuMaxFreq");
dataColumns.add("cpuMinFreq");
dataColumns.add("cpuCurFreq");
dataColumns.add("cpuCount");
dataColumns.add("cpuSerial");
dataColumns.add("cpuABI");
dataColumns.add("cpuABI2");
dataColumns.add("user");
dataColumns.add("totalStorage");
dataColumns.add("totalSDStorage");
dataColumns.add("freeSDStorage");
dataColumns.add("totalMemory");
dataColumns.add("totalsys");
dataColumns.add("avaisys");
dataColumns.add("diskUsedStorage");
dataColumns.add("diskFreeStorage");
dataColumns.add("usedMemory");
dataColumns.add("simState");
dataColumns.add("imsi");
dataColumns.add("simSerial");
dataColumns.add("phoneType");
dataColumns.add("country");
dataColumns.add("simName");
dataColumns.add("simMcc");
dataColumns.add("simMnc");
dataColumns.add("simLocation");
dataColumns.add("simCid");
dataColumns.add("baseStationLongitude");
dataColumns.add("baseStationLatitude");
dataColumns.add("simRssi");
dataColumns.add("nbasestation");
dataColumns.add("wifiDNS");
dataColumns.add("wifiGateway");
dataColumns.add("wifiIP");
dataColumns.add("wifiNetmask");
dataColumns.add("wifiRssi");
dataColumns.add("wifiMac");
dataColumns.add("wifiList");
dataColumns.add("longitude");
dataColumns.add("latitude");
dataColumns.add("osVersion");
dataColumns.add("osName");
dataColumns.add("osVersionInt");
dataColumns.add("codeName");
dataColumns.add("signatures");
dataColumns.add("networkType");
dataColumns.add("activetime");
dataColumns.add("systemUpTime");
dataColumns.add("userAgent");
dataColumns.add("tags");
dataColumns.add("language");
dataColumns.add("type");
dataColumns.add("deviceId");
dataColumns.add("androidId");
dataColumns.add("bluetoothMac");
dataColumns.add("bluetoothName");
dataColumns.add("baseBandVersion");
dataColumns.add("board");
dataColumns.add("bootLoader");
dataColumns.add("brand");
dataColumns.add("device");
dataColumns.add("display");
dataColumns.add("host");
dataColumns.add("id");
dataColumns.add("manufacturer");
dataColumns.add("model");
dataColumns.add("product");
dataColumns.add("resolution");
dataColumns.add("screenDensity");
dataColumns.add("brightness");
dataColumns.add("touchScreen");
dataColumns.add("simulator");
dataColumns.add("rooted");
dataColumns.add("hasCellular");
dataColumns.add("hasCellular1");
dataColumns.add("hasWiFi");
dataColumns.add("hasGps");
dataColumns.add("hasTelephony");
dataColumns.add("hasNFC");
dataColumns.add("hasNFCHost");
dataColumns.add("hasBluetooth");
dataColumns.add("hasWiFiDirect");
dataColumns.add("hasOTG");
dataColumns.add("hasAOA");
dataColumns.add("timeZone");
dataColumns.add("sensors");
dataColumns.add("deviceType");
dataColumns.add("deviceName");
dataColumns.add("deviceModel");
dataColumns.add("screenW");
dataColumns.add("screenH");
dataColumns.add("breakFlag");
dataColumns.add("appName");
dataColumns.add("appVersion");
dataColumns.add("idfv");
dataColumns.add("idfa");
dataColumns.add("carrier");
dataColumns.add("isConnWifi");
dataColumns.add("isConnCell");
dataColumns.add("isHasCellular");
dataColumns.add("isHasWiFi");
dataColumns.add("isHasTelephony");
dataColumns.add("isHasBluetooth");
dataColumns.add("isHeadphonesAttached");
dataColumns.add("batteryStat");
dataColumns.add("isFullCharged");
dataColumns.add("ip");
dataColumns.add("allApps");
booleanColumns.add("simulator");
booleanColumns.add("rooted");
booleanColumns.add("hasCellular");
booleanColumns.add("hasCellular1");
booleanColumns.add("hasWiFi");
booleanColumns.add("hasGps");
booleanColumns.add("hasTelephony");
booleanColumns.add("hasNFC");
booleanColumns.add("hasNFCHost");
booleanColumns.add("hasBluetooth");
booleanColumns.add("hasWiFiDirect");
booleanColumns.add("hasOTG");
booleanColumns.add("hasAOA");
booleanColumns.add("breakFlag");
booleanColumns.add("isConnWifi");
booleanColumns.add("isConnCell");
booleanColumns.add("isHasCellular");
booleanColumns.add("isHasWiFi");
booleanColumns.add("isHasTelephony");
booleanColumns.add("isHasBluetooth");
booleanColumns.add("isHeadphonesAttached");
booleanColumns.add("isFullCharged");
longColumns.add("activetime");
longColumns.add("systemUpTime");
}
public static void cleanningAndSaveDeviceFingersInfo(MobileFingerPrintDataRow mobileFingerPrintDataRow, String uuid, boolean ka) {
ThreadPoolExecutorUtils.getThreadPoolUtil().execute(new Runnable() {
@Override
public void run() {
List<List<Object>> paramList = new ArrayList<>();
Stopwatch stopwatch = Stopwatch.createStarted();
try {
String data = mobileFingerPrintDataRow.getData();
JSONObject jsonObject = JSONObject.parseObject(data);
//把所有字段拆出来，到一级json
jsonObject = flatJson(jsonObject);
//合并字段
combineColumn(jsonObject);
//清洗逻辑
cleanColumn(jsonObject);
List<Object> list = new ArrayList<>();
list.add(uuid);
list.add(mobileFingerPrintDataRow.getType());
list.add(mobileFingerPrintDataRow.getCreateDate());
list.add(mobileFingerPrintDataRow.getMemo());
list.add(mobileFingerPrintDataRow.getParams());
list.add(mobileFingerPrintDataRow.getTimestamp());
SimpleDateFormat sdf = new SimpleDateFormat("yyy-MM-dd HH:mm:ss");
list.add(sdf.format(new Date(mobileFingerPrintDataRow.getTimestamp())));
for (String column : dataColumns) {
try {
if (booleanColumns.contains(column)) {
list.add(jsonObject.getBoolean(column));
} else if (longColumns.contains(column)) {
list.add(jsonObject.getLong(column));
} else {
list.add(jsonObject.getString(column));
}
} catch (Exception e) {
//如果上述jsonObject获取值出现问题，则使用null
list.add(null);
}
}
paramList.add(list);
} catch (Exception e) {
log.error("设备指纹清洗异常, uuid: {} , ka: {} , param: {} ", uuid, ka, JSON.toJSONString(mobileFingerPrintDataRow));
}
JdbcExecuters.prepareBatchUpdate(Constant.SQL.DEVICE_FINGERS_INFOS, paramList);
log.info("充值记录清洗完成, uuid: {} , 入库大小: {} , ka: {} , 耗时: {} ", uuid, 1, ka, stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
}
});
}
//把所有json都放在一个大的json里，去掉子json
public static JSONObject flatJson(JSONObject jsonObject) {
JSONObject result = new JSONObject();
Set<String> keySet = jsonObject.keySet();
for (String key : keySet) {
String value = jsonObject.getString(key);
if (StringUtils.isNotBlank(value) && (value = value.trim()).startsWith("{") && value.endsWith("}")) {
JSONObject sonJson = JSON.parseObject(value);
result.putAll(sonJson);
} else if ("sensorList".equalsIgnoreCase(key)) {
result.put("sensors", value);
} else {
result.put(key, value);
}
}
jsonObject = null;
return result;
}
public static void cleanColumn(JSONObject jsonObject) {
if (jsonObject != null) {
String networkType = jsonObject.getString("networkType");
networkType = StringUtils.isBlank(networkType) || "NaN".equalsIgnoreCase(networkType) ? UNEXPECTED
: networkType.replace("WWAN", "Wwan").replace("WiFi", "Wifi").replace("NotReachable", UNEXPECTED);
jsonObject.put("networkType", networkType);
String latitude = jsonObject.getString("latitude") != null ? jsonObject.getString("latitude") : jsonObject.getString("Latitude");
latitude = StringUtils.isBlank(latitude) || "NaN".equalsIgnoreCase(latitude) ? UNEXPECTED : formatString(latitude);
jsonObject.put("latitude", latitude);
String osVersion = jsonObject.getString("osVersion");
osVersion = StringUtils.isBlank(osVersion) || "NaN".equalsIgnoreCase(osVersion) ? UNEXPECTED
: osVersion.replace(" ", "").replaceAll("[a-zA-Z]+", "").split("\\.")[0];
jsonObject.put("osVersion", osVersion);
String deviceId = jsonObject.getString("deviceId");
deviceId = StringUtils.isBlank(deviceId) || "NaN".equalsIgnoreCase(deviceId) ? UNEXPECTED
: deviceId.replace("00000000-0000-0000-0000-000000000000", UNEXPECTED);
jsonObject.put("deviceId", deviceId);
String idfv = jsonObject.getString("idfv");
idfv = StringUtils.isBlank(idfv) || "NaN".equalsIgnoreCase(idfv) || deviceId.equalsIgnoreCase(idfv) ? UNEXPECTED
: idfv.replace("00000000-0000-0000-0000-000000000000", UNEXPECTED);
jsonObject.put("idfv", idfv);
String androidId = jsonObject.getString("androidId");
androidId = StringUtils.isBlank(androidId) || "NaN".equalsIgnoreCase(androidId) ? UNEXPECTED : androidId;
jsonObject.put("androidId", androidId);
String resolution = jsonObject.getString("resolution");
resolution = StringUtils.isBlank(resolution) || "NaN".equalsIgnoreCase(resolution) ? UNEXPECTED
: dealResolution(resolution);
jsonObject.put("resolution", resolution);
String wifiIp = jsonObject.getString("wifiIP");
wifiIp = StringUtils.isBlank(wifiIp) || "NaN".equalsIgnoreCase(wifiIp) ? UNEXPECTED : wifiIp;
jsonObject.put("wifiIP", wifiIp);
String wifiDns = jsonObject.getString("wifiDNS");
wifiDns = StringUtils.isBlank(wifiDns) || "NaN".equalsIgnoreCase(wifiDns) ? UNEXPECTED : wifiDns;
jsonObject.put("wifiDNS", wifiDns);
String wifiGateway = jsonObject.getString("wifiGateway");
wifiGateway = StringUtils.isBlank(wifiGateway) || "NaN".equalsIgnoreCase(wifiGateway) ? UNEXPECTED : wifiGateway;
jsonObject.put("wifiGateway", wifiGateway);
String wifiNetmask = jsonObject.getString("wifiNetmask");
wifiNetmask = StringUtils.isBlank(wifiNetmask) || "NaN".equalsIgnoreCase(wifiNetmask) ? UNEXPECTED : wifiNetmask;
jsonObject.put("wifiNetmask", wifiNetmask);
String cpuArchitecture = jsonObject.getString("cpuArchitecture");
cpuArchitecture = StringUtils.isBlank(cpuArchitecture) || "NaN".equalsIgnoreCase(cpuArchitecture) ? UNEXPECTED
: dealCpuArchitecture(cpuArchitecture);
jsonObject.put("cpuArchitecture", cpuArchitecture);
String baseStationLongitude = jsonObject.getString("baseStationLongitude");
baseStationLongitude = StringUtils.isBlank(baseStationLongitude) || "NaN".equalsIgnoreCase(baseStationLongitude) ? UNEXPECTED
: formatString(baseStationLongitude);
jsonObject.put("baseStationLongitude", baseStationLongitude);
String baseStationLatitude = jsonObject.getString("baseStationLatitude");
baseStationLatitude = StringUtils.isBlank(baseStationLatitude) || "NaN".equalsIgnoreCase(baseStationLatitude) ? UNEXPECTED
: formatString(baseStationLatitude);
jsonObject.put("baseStationLatitude", baseStationLatitude);
String simCid = jsonObject.getString("simCid");
simCid = StringUtils.isBlank(simCid) || "NaN".equalsIgnoreCase(simCid) ? UNEXPECTED : simCid;
jsonObject.put("simCid", simCid);
String imsi = jsonObject.getString("imsi");
imsi = StringUtils.isBlank(imsi) || "NaN".equalsIgnoreCase(imsi) || imsi.contains("unknown") || imsi.contains("Unknown") ?
UNEXPECTED : imsi;
jsonObject.put("imsi", imsi);
String timeZone = jsonObject.getString("timeZone");
timeZone = StringUtils.isBlank(timeZone) || "NaN".equalsIgnoreCase(timeZone) ? UNEXPECTED : dealTimeZone(timeZone);
jsonObject.put("timeZone", timeZone);
String idfa = jsonObject.getString("idfa");
idfa = StringUtils.isBlank(idfa) || "NaN".equalsIgnoreCase(idfa) || deviceId.equalsIgnoreCase(idfa) ? UNEXPECTED
: idfa.replace("00000000-0000-0000-0000-000000000000", UNEXPECTED);
jsonObject.put("idfa", idfa);
String breakFlag = jsonObject.getString("breakFlag");
breakFlag = "FALSE".equalsIgnoreCase(breakFlag) || "TRUE".equalsIgnoreCase(breakFlag) ? breakFlag : UNEXPECTED;
jsonObject.put("breakFlag", breakFlag);
String totalStorage = jsonObject.getString("totalStorage");
totalStorage = StringUtils.isBlank(totalStorage) || "NaN".equalsIgnoreCase(totalStorage) ? UNEXPECTED : dealTotalStorage(totalStorage);
jsonObject.put("totalStorage", totalStorage);
String brand = jsonObject.getString("brand");
brand = StringUtils.isBlank(brand) || "NaN".equalsIgnoreCase(brand) ? UNEXPECTED : dealBrand(brand);
jsonObject.put("brand", brand);
String longitude = jsonObject.getString("longitude") != null ? jsonObject.getString("longitude")
: jsonObject.getString("Longitude");
longitude = StringUtils.isBlank(longitude) || "NaN".equalsIgnoreCase(longitude) ? UNEXPECTED : formatString(longitude);
jsonObject.put("longitude", longitude);
String osName = jsonObject.getString("osName");
osName = StringUtils.isBlank(osName) || "NaN".equalsIgnoreCase(osName) ? UNEXPECTED
: osName;
if (osName.contains("iOS")) {
osName = "Ios";
} else {
osName = "Android";
}
jsonObject.put("osName", osName);
String cpuCount = jsonObject.getString("cpuCount");
cpuCount = StringUtils.isBlank(cpuCount) || "NaN".equalsIgnoreCase(cpuCount) ? UNEXPECTED : cpuCount;
jsonObject.put("cpuCount", cpuCount);
String totalsys = jsonObject.getString("totalsys");
totalsys = StringUtils.isBlank(totalsys) || "NaN".equalsIgnoreCase(totalsys) ? UNEXPECTED : dealTotalSys(totalsys);
jsonObject.put("totalsys", totalsys);
JSONArray allApps = jsonObject.getJSONArray("allApps");
String allAppsStr = allApps == null || allApps.size() == 0 ? UNEXPECTED
: String.join("#", allApps.stream().map(Object::toString).collect(Collectors.toList()).toArray(new String[]{}));
jsonObject.put("allApps", allAppsStr);
String userAgent = jsonObject.getString("userAgent");
userAgent = StringUtils.isBlank(userAgent) || "NaN".equalsIgnoreCase(userAgent) ? UNEXPECTED : userAgent;
jsonObject.put("userAgent", userAgent);
}
}
public static String formatString(String str) {
try {
if (str.contains(".") && str.split("\\.", -1)[1].length() > 6) {
str = format.format(Double.valueOf(str));
}
} catch (Exception e) {
e.printStackTrace();
}
return str;
}
public static void combineColumn(JSONObject jsonObject) {
// String cpuABI = jsonObject.getString("cpuABI");
// String cpuABI2 = jsonObject.getString("cpuABI2");
// if (StringUtils.isBlank(cpuABI) && StringUtils.isNotBlank(cpuABI2)) {
// jsonObject.put("cpuABI", cpuABI2);
// jsonObject.remove("cpuABI2");
// }
String simState = jsonObject.getString("simState");
String simState1 = jsonObject.getString("simState1");
String simState2 = jsonObject.getString("simState2");
if (StringUtils.isBlank(simState) && (StringUtils.isNotBlank(simState1) || StringUtils.isNotBlank(simState2))) {
jsonObject.put("simState", StringUtils.isNotBlank(simState1) ? simState1 : simState2);
jsonObject.remove("simState1");
jsonObject.remove("simState2");
}
String imsi = jsonObject.getString("imsi");
String imsi1 = jsonObject.getString("imsi1");
String imsi2 = jsonObject.getString("imsi2");
if (StringUtils.isBlank(imsi) && (StringUtils.isNotBlank(imsi1) || StringUtils.isNotBlank(imsi2))) {
jsonObject.put("imsi", StringUtils.isNotBlank(imsi1) ? imsi1 : imsi2);
jsonObject.remove("imsi1");
jsonObject.remove("imsi2");
}
String simSerial = jsonObject.getString("simSerial");
String simSerial1 = jsonObject.getString("simSerial1");
String simSerial2 = jsonObject.getString("simSerial2");
if (StringUtils.isBlank(simSerial) && (StringUtils.isNotBlank(simSerial1) || StringUtils.isNotBlank(simSerial2))) {
jsonObject.put("simSerial", StringUtils.isNotBlank(simSerial1) ? simSerial1 : simSerial2);
jsonObject.remove("simSerial1");
jsonObject.remove("simSerial2");
}
String phoneType = jsonObject.getString("phoneType");
String phoneType1 = jsonObject.getString("phoneType1");
String phoneType2 = jsonObject.getString("phoneType2");
if (StringUtils.isBlank(phoneType) && (StringUtils.isNotBlank(phoneType1) || StringUtils.isNotBlank(phoneType2))) {
jsonObject.put("phoneType", StringUtils.isNotBlank(phoneType1) ? phoneType1 : phoneType2);
jsonObject.remove("phoneType1");
jsonObject.remove("phoneType2");
}
String country = jsonObject.getString("country");
String country1 = jsonObject.getString("country1");
String country2 = jsonObject.getString("country2");
if (StringUtils.isBlank(country) && (StringUtils.isNotBlank(country1) || StringUtils.isNotBlank(country2))) {
jsonObject.put("country", StringUtils.isNotBlank(country1) ? country1 : country2);
jsonObject.remove("country1");
jsonObject.remove("country2");
}
String simName = jsonObject.getString("simName");
String simName1 = jsonObject.getString("simName1");
String simName2 = jsonObject.getString("simName2");
if (StringUtils.isBlank(simName) && (StringUtils.isNotBlank(simName1) || StringUtils.isNotBlank(simName2))) {
jsonObject.put("simName", StringUtils.isNotBlank(simName1) ? simName1 : simName2);
jsonObject.remove("simName1");
jsonObject.remove("simName2");
}
String simMcc = jsonObject.getString("simMcc");
String simMcc1 = jsonObject.getString("simMcc1");
String simMcc2 = jsonObject.getString("simMcc2");
if (StringUtils.isBlank(simMcc) && (StringUtils.isNotBlank(simMcc1) || StringUtils.isNotBlank(simMcc2))) {
jsonObject.put("simMcc", StringUtils.isNotBlank(simMcc1) ? simMcc1 : simMcc2);
jsonObject.remove("simMcc1");
jsonObject.remove("simMcc2");
}
String simMnc = jsonObject.getString("simMnc");
String simMnc1 = jsonObject.getString("simMnc1");
String simMnc2 = jsonObject.getString("simMnc2");
if (StringUtils.isBlank(simMnc) && (StringUtils.isNotBlank(simMnc1) || StringUtils.isNotBlank(simMnc2))) {
jsonObject.put("simMnc", StringUtils.isNotBlank(simMnc1) ? simMnc1 : simMnc2);
jsonObject.remove("simMnc1");
jsonObject.remove("simMnc2");
}
String simLocation = jsonObject.getString("simLocation");
String simLocation1 = jsonObject.getString("simLocation1");
String simLocation2 = jsonObject.getString("simLocation2");
if (StringUtils.isBlank(simLocation) && (StringUtils.isNotBlank(simLocation1) || StringUtils.isNotBlank(simLocation2))) {
jsonObject.put("simLocation", StringUtils.isNotBlank(simLocation1) ? simLocation1 : simLocation2);
jsonObject.remove("simLocation1");
jsonObject.remove("simLocation2");
}
String simCid = jsonObject.getString("simCid");
String simCid1 = jsonObject.getString("simCid1");
String simCid2 = jsonObject.getString("simCid2");
if (StringUtils.isBlank(simCid) && (StringUtils.isNotBlank(simCid1) || StringUtils.isNotBlank(simCid2))) {
jsonObject.put("simCid", StringUtils.isNotBlank(simCid1) ? simCid1 : simCid2);
jsonObject.remove("simCid1");
jsonObject.remove("simCid2");
}
String baseStationLongitude = jsonObject.getString("baseStationLongitude");
String baseStationLongitude1 = jsonObject.getString("baseStationLongitude1");
String baseStationLongitude2 = jsonObject.getString("baseStationLongitude2");
if (StringUtils.isBlank(baseStationLongitude) && (StringUtils.isNotBlank(baseStationLongitude1) || StringUtils.isNotBlank(baseStationLongitude2))) {
jsonObject.put("baseStationLongitude", StringUtils.isNotBlank(baseStationLongitude1) ? baseStationLongitude1 : baseStationLongitude2);
jsonObject.remove("baseStationLongitude1");
jsonObject.remove("baseStationLongitude2");
}
String baseStationLatitude = jsonObject.getString("baseStationLatitude");
String baseStationLatitude1 = jsonObject.getString("baseStationLatitude1");
String baseStationLatitude2 = jsonObject.getString("baseStationLatitude2");
if (StringUtils.isBlank(baseStationLatitude) && (StringUtils.isNotBlank(baseStationLatitude1) || StringUtils.isNotBlank(baseStationLatitude2))) {
jsonObject.put("baseStationLatitude", StringUtils.isNotBlank(baseStationLatitude1) ? baseStationLatitude1 : baseStationLatitude2);
jsonObject.remove("baseStationLatitude1");
jsonObject.remove("baseStationLatitude2");
}
String simRssi = jsonObject.getString("simRssi");
String simRssi1 = jsonObject.getString("simRssi1");
String simRssi2 = jsonObject.getString("simRssi2");
if (StringUtils.isBlank(simRssi) && (StringUtils.isNotBlank(simRssi1) || StringUtils.isNotBlank(simRssi2))) {
jsonObject.put("simRssi", StringUtils.isNotBlank(simRssi1) ? simRssi1 : simRssi2);
jsonObject.remove("simRssi1");
jsonObject.remove("simRssi2");
}
String nbasestation = jsonObject.getString("nbasestation");
String nbasestation1 = jsonObject.getString("nbasestation1");
String nbasestation2 = jsonObject.getString("nbasestation2");
if (StringUtils.isBlank(nbasestation) && (StringUtils.isNotBlank(nbasestation1) || StringUtils.isNotBlank(nbasestation2))) {
jsonObject.put("nbasestation", StringUtils.isNotBlank(nbasestation1) ? nbasestation1 : nbasestation2);
jsonObject.remove("nbasestation1");
jsonObject.remove("nbasestation2");
}
}
public static String dealResolution(String resolution) {
String[] values = resolution.split("\\*");
if (values.length == 2) {
try {
return String.valueOf(Long.valueOf(values[0]) * Long.valueOf(values[1]));
} catch (Exception e) {
System.out.println("计算分辨率出错");
e.printStackTrace();
}
}
return UNEXPECTED;
}
public static String dealCpuArchitecture(String cpuArchitecture) {
cpuArchitecture = cpuArchitecture.trim();
if (cpuArchitecture.startsWith("ARMv")) {
return "Armv";
} else if (cpuArchitecture.startsWith("AArch")) {
return "Aarch";
} else {
return "Other";
}
}
public static String dealTimeZone(String timeZone) {
if (timeZone.trim().startsWith("GMT+08:00")) {
return "beijingStandardTime";
} else {
return "otherTimeZone";
}
}
public static String dealTotalStorage(String totalStorage) {
String total = "";
try {
if (totalStorage.contains("k") || totalStorage.contains("K")) {
totalStorage = totalStorage.replaceAll("[kK][Bb]*", "");
totalStorage = totalStorage.replace(",", ".").trim();
float gb = Float.valueOf(totalStorage) / (1024 * 1024);
total = storageFormat.format(gb);
} else if (totalStorage.contains("m") || totalStorage.contains("M")) {
totalStorage = totalStorage.replaceAll("[mM][Bb]*", "");
totalStorage = totalStorage.replace(",", ".").trim();
float gb = Float.valueOf(totalStorage) / 1024;
total = storageFormat.format(gb);
} else {
totalStorage = totalStorage.replace(",", ".").replace("吉字节", "").trim();
total = totalStorage.replaceAll("[gG][Bb]*", "");
}
} catch (Exception e) {
e.printStackTrace();
}
return total;
}
public static String dealBrand(String brand) {
brand = brand.trim();
if (brand.startsWith("Apple") || brand.startsWith("apple") || brand.startsWith("iPhone") || brand.startsWith("iphone")) {
return "Apple";
} else if (brand.startsWith("HUAWEI")) {
return "Huawei";
} else if (brand.startsWith("HONOR")) {
return "Honor";
} else if (brand.startsWith("Xiaomi")) {
return "Xiaomi";
} else if (brand.startsWith("OPPO")) {
return "Oppo";
} else if (brand.startsWith("samsung")) {
return "Samsung";
} else if (brand.startsWith("vivo")) {
return "Vivo";
}
return "Other";
}
public static String dealTotalSys(String totalSys) {
String total = "";
try {
if (totalSys.contains("k") || totalSys.contains("K")) {
totalSys = totalSys.replaceAll("[kK][Bb]*", "");
totalSys = totalSys.replace(",", ".").trim();
float gb = Float.valueOf(totalSys) / 1024;
total = storageFormat.format(gb);
} else if (totalSys.contains("g") || totalSys.contains("G") || totalSys.contains("吉字节")) {
totalSys = totalSys.replaceAll("[gG][Bb]*", "");
totalSys = totalSys.replace(",", ".").replace("吉字节", "").trim();
float gb = Float.valueOf(totalSys) * 1024;
total = storageFormat.format(gb);
} else {
totalSys = totalSys.replace(",", ".").trim();
total = totalSys.replaceAll("[mM][Bb]*", "");
}
} catch (Exception e) {
e.printStackTrace();
}
return total;
}
// public static void main(String[] args) {
//// String uuid = "9391cb3c-edad-4e04-bf4a-955c4d0ea987";
// String uuid = "d0324881-e32f-4832-93d7-7d7f0feefdfc";
// List<MobileFingerPrintDataRow> mobileFingerPrintDataRows = MobileFingerPrintDataService.get(uuid);
// System.out.println(JSON.toJSONString(mobileFingerPrintDataRows.get(0)));
// System.out.println(JSON.parseObject(mobileFingerPrintDataRows.get(0).getData()));
// cleanningAndSaveDeviceFingersInfo(mobileFingerPrintDataRows.get(0), uuid+"1a1", false);
// }
}
qg-data-service/src/main/java/cn/quantgroup/qgdataservice/utils/JdbcExecuters.java
View file @ 9ac7a242
package cn.quantgroup.qgdataservice.utils;
import cn.quantgroup.qgdataservice.constant.Constant;
import com.alibaba.fastjson.JSON;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -72,6 +73,43 @@ public class JdbcExecuters {
}
}
public static void prepareBatchUpdate(String sql, List<List<Object>> paramList) {
Connection conn = null;
PreparedStatement statement = null;
try {
conn = HIK_DATA_SOURCE.dataSource.getConnection();
statement = conn.prepareStatement(sql);
conn.setAutoCommit(false);
for (List<Object> param : paramList) {
try {
for (int i = 0; i < param.size(); i++) {
Object object = param.get(i);
if (object == null || "".equals(object)) {
statement.setObject(i + 1, null);
} else if (object instanceof String) {
statement.setString(i + 1, (String) object);
} else if (object instanceof Long) {
statement.setLong(i + 1, (Long) object);
} else {
statement.setObject(i + 1, object);
}
}
statement.addBatch();
} catch (SQLException e) {
log.error("设备指纹清洗数据批量插入数据SQL异常 , param: {} ", JSON.toJSONString(param), e);
} catch (Exception e) {
log.error("设备指纹清洗数据批量插入数据未知异常, param: {} ", JSON.toJSONString(param), e);
}
}
statement.executeBatch();
conn.commit();
} catch (Exception e) {
log.error("设备指纹清洗数据批量插入数据未知异常, sql: {} ", sql, e);
} finally {
close(conn, statement, null);
}
}
public static int batchExecute(List<List<String>> list, String sql) {
Connection conn = null;
PreparedStatement ps = null;
......@@ -157,7 +195,7 @@ public class JdbcExecuters {
if (conn != null)
conn.close();
} catch (SQLException e) {
e.printStackTrace();
log.error("清洗数据关闭jdbc资源异常", e);
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment