Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Q
qg-rt-dc
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
data-spider
qg-rt-dc
Commits
8d65ad12
Commit
8d65ad12
authored
Aug 28, 2019
by
data爬虫-冯 军凯
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
支持发送三大运营商数据至kafka(清洗后数据)
parent
4005affd
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
8 additions
and
5 deletions
+8
-5
pom.xml
qg-data-service/pom.xml
+1
-1
Constant.java
...n/java/cn/quantgroup/qgdataservice/constant/Constant.java
+2
-1
OperatorInfoCleaningService.java
...dataservice/service/tidb/OperatorInfoCleaningService.java
+5
-3
No files found.
qg-data-service/pom.xml
View file @
8d65ad12
...
@@ -6,7 +6,7 @@
...
@@ -6,7 +6,7 @@
<groupId>
cn.quantgroup
</groupId>
<groupId>
cn.quantgroup
</groupId>
<artifactId>
qg-data-service
</artifactId>
<artifactId>
qg-data-service
</artifactId>
<!--<version>1.0.0-qa-SNAPSHOT</version>-->
<!--<version>1.0.0-qa-SNAPSHOT</version>-->
<version>
1.4.
5
-SNAPSHOT
</version>
<version>
1.4.
6
-SNAPSHOT
</version>
<packaging>
jar
</packaging>
<packaging>
jar
</packaging>
<name>
qg-data-service
</name>
<name>
qg-data-service
</name>
...
...
qg-data-service/src/main/java/cn/quantgroup/qgdataservice/constant/Constant.java
View file @
8d65ad12
...
@@ -108,7 +108,8 @@ public class Constant {
...
@@ -108,7 +108,8 @@ public class Constant {
}
}
public
static
class
KAFKA
{
public
static
class
KAFKA
{
public
static
final
String
OPERATOR_CALL_TOPIC
=
"rc.comservice.call"
;
// public static final String OPERATOR_CALL_TOPIC = "rc.comservice.call";
public
static
final
String
OPERATOR_CALL_TOPIC
=
"rc.comservice.calls"
;
public
static
final
String
BOOT_STRAPSERVER
=
"kafka.bootStrapServer"
;
public
static
final
String
BOOT_STRAPSERVER
=
"kafka.bootStrapServer"
;
public
static
final
String
APPLICATION
=
"kafka_application"
;
public
static
final
String
APPLICATION
=
"kafka_application"
;
}
}
...
...
qg-data-service/src/main/java/cn/quantgroup/qgdataservice/service/tidb/OperatorInfoCleaningService.java
View file @
8d65ad12
...
@@ -8,6 +8,9 @@ import cn.quantgroup.qgdataservice.service.kafka.OperatorCallDetailListSendToKaf
...
@@ -8,6 +8,9 @@ import cn.quantgroup.qgdataservice.service.kafka.OperatorCallDetailListSendToKaf
import
cn.quantgroup.qgdataservice.utils.*
;
import
cn.quantgroup.qgdataservice.utils.*
;
import
com.alibaba.fastjson.JSON
;
import
com.alibaba.fastjson.JSON
;
import
com.google.common.base.Stopwatch
;
import
com.google.common.base.Stopwatch
;
import
com.lkb.data.hbase.dataservice.operators.mobile.MobileCallInfoDataService
;
import
com.lkb.data.hbase.dataservice.operators.telecom.TelecomCallInfoDataService
;
import
com.lkb.data.hbase.dataservice.operators.unicom.UnicomCallInfoDataService
;
import
com.lkb.data.hbase.row.SpiderUserItemDataRow
;
import
com.lkb.data.hbase.row.SpiderUserItemDataRow
;
import
com.lkb.data.hbase.row.operators.PhoneBillDataRow
;
import
com.lkb.data.hbase.row.operators.PhoneBillDataRow
;
import
com.lkb.data.hbase.row.operators.PhonePaymentDataRow
;
import
com.lkb.data.hbase.row.operators.PhonePaymentDataRow
;
...
@@ -369,7 +372,7 @@ public class OperatorInfoCleaningService {
...
@@ -369,7 +372,7 @@ public class OperatorInfoCleaningService {
}
}
int
size
=
JdbcExecuters
.
batchExecute
(
sqls
,
Constant
.
SQL
.
TELECOM_DETAIL_INFOS
);
int
size
=
JdbcExecuters
.
batchExecute
(
sqls
,
Constant
.
SQL
.
TELECOM_DETAIL_INFOS
);
OperatorCallDetailListSendToKafkaService
.
sendTelecomCallDetailListMessage
(
telecomCallDetailInfos
);
log
.
info
(
"电信通话详单清洗完成, uuid: {} , 原始大小: {} , 清洗后大小: {} , 入库大小: {} , ka: {} , 耗时: {} "
,
uuid
,
telecomCallInfoRows
.
size
(),
sqls
.
size
(),
size
,
ka
,
stopwatch
.
stop
().
elapsed
(
TimeUnit
.
MILLISECONDS
));
log
.
info
(
"电信通话详单清洗完成, uuid: {} , 原始大小: {} , 清洗后大小: {} , 入库大小: {} , ka: {} , 耗时: {} "
,
uuid
,
telecomCallInfoRows
.
size
(),
sqls
.
size
(),
size
,
ka
,
stopwatch
.
stop
().
elapsed
(
TimeUnit
.
MILLISECONDS
));
}
}
});
});
...
@@ -565,7 +568,7 @@ public class OperatorInfoCleaningService {
...
@@ -565,7 +568,7 @@ public class OperatorInfoCleaningService {
}
}
int
size
=
JdbcExecuters
.
batchExecute
(
sqls
,
Constant
.
SQL
.
UNICOM_DETAIL_INFOS
);
int
size
=
JdbcExecuters
.
batchExecute
(
sqls
,
Constant
.
SQL
.
UNICOM_DETAIL_INFOS
);
OperatorCallDetailListSendToKafkaService
.
sendUnicomCallDetailListMessage
(
unicomCallDetailInfos
);
log
.
info
(
"联通通话详单清洗完成, uuid: {} , 原始大小: {} , 清洗后大小: {} , 入库大小: {} , ka: {} , 耗时: {} "
,
uuid
,
unicomCallInfoRows
.
size
(),
sqls
.
size
(),
size
,
ka
,
stopwatch
.
stop
().
elapsed
(
TimeUnit
.
MILLISECONDS
));
log
.
info
(
"联通通话详单清洗完成, uuid: {} , 原始大小: {} , 清洗后大小: {} , 入库大小: {} , ka: {} , 耗时: {} "
,
uuid
,
unicomCallInfoRows
.
size
(),
sqls
.
size
(),
size
,
ka
,
stopwatch
.
stop
().
elapsed
(
TimeUnit
.
MILLISECONDS
));
}
}
});
});
...
@@ -863,5 +866,4 @@ public class OperatorInfoCleaningService {
...
@@ -863,5 +866,4 @@ public class OperatorInfoCleaningService {
return
unicomCallDetailInfo
;
return
unicomCallDetailInfo
;
}
}
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment