Skip to content

H
holmes
Commit 5e3d377a authored by 黎博's avatar 黎博
Browse files
Options

Merge branch 'master' into ldap

parents 5bc05069 881a7266
Show whitespace changes
Inline Side-by-side
Showing with 98 additions and 75 deletions
src/main/java/cn/qg/holmes/config/RedisConfig.java
View file @ 5e3d377a
......@@ -45,10 +45,11 @@ public class RedisConfig extends CachingConfigurerSupport {
public JedisPoolConfig jedisPoolConfig() {
JedisPoolConfig jedisPoolConfig = new JedisPoolConfig();
jedisPoolConfig.setMaxTotal(32);
jedisPoolConfig.setMaxIdle(16);
jedisPoolConfig.setMinIdle(8);
jedisPoolConfig.setMaxWaitMillis(5000);
jedisPoolConfig.setMaxIdle(32);
jedisPoolConfig.setMinIdle(0);
jedisPoolConfig.setTestOnBorrow(true);
jedisPoolConfig.setTestWhileIdle(true);
jedisPoolConfig.setTestOnReturn(true);
return jedisPoolConfig;
}
......
src/main/java/cn/qg/holmes/service/effect/impl/DatabaseSyncServiceImpl.java
View file @ 5e3d377a
......@@ -12,10 +12,7 @@ import org.springframework.stereotype.Service;
import java.math.BigDecimal;
import java.sql.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
@Slf4j
@Service
......@@ -51,15 +48,19 @@ public class DatabaseSyncServiceImpl implements DatabaseSyncService {
public boolean getDbInfoFromSource(String ip, String port, String username, String password, String dbName) {
String driver = "com.mysql.jdbc.Driver";
String url = "jdbc:mysql://" + ip + ":" + port + "/" + dbName;
String createTableKey = dbSyncPrefix + dbName + ":create" ;
String insertTableKey = dbSyncPrefix + dbName + ":insert";
String dbRedisKey = dbSyncPrefix + dbName;
String tableListKey = dbSyncPrefix + dbName + ":tables";
Connection connection = null;
PreparedStatement preparedStatement = null;
// 创建数据库的redis值
String dbRedisValue = "";
// 表数据redis值
String insertRedisValue = "";
List<Object> createTableRedisValue = new ArrayList<>();
// 建表语句redis值
String createTableRedisValue = "";
// 表名列表redis值
String tableListStr = "";
List<String> columnNameList = new ArrayList<>();
List<String> columnTypeList = new ArrayList<>();
......@@ -79,19 +80,30 @@ public class DatabaseSyncServiceImpl implements DatabaseSyncService {
redisUtils.set(dbRedisKey, dbRedisValue, 86400);
}
if (!redisUtils.hasKey(createTableKey)) {
DatabaseMetaData databaseMetaData = connection.getMetaData();
// 获取所有表名
ResultSet tableResultSet = databaseMetaData.getTables(null, null, null, new String[]{"TABLE"});
while (tableResultSet.next()) {
String tableName = tableResultSet.getString("TABLE_NAME");
tableListStr += tableName;
tableListStr += "\n";
long startTime = System.currentTimeMillis();
log.info("开始获取表{}的数据", tableName);
String createTableKey = dbSyncPrefix + dbName + ":" + tableName + ":create" ;
String insertTableKey = dbSyncPrefix + dbName + ":" + tableName + ":insert" ;
if (!redisUtils.hasKey(createTableKey)) {
// 获取所有建表语句
String sql = String.format("SHOW CREATE TABLE %s", tableName);
preparedStatement = connection.prepareStatement(sql);
ResultSet pResultSet = preparedStatement.executeQuery();
while (pResultSet.next()) {
createTableRedisValue.add(pResultSet.getString(2));
createTableRedisValue = pResultSet.getString(2);
}
redisUtils.set(createTableKey, createTableRedisValue, 86400);
createTableRedisValue = "";
}
if (!redisUtils.hasKey(insertTableKey)) {
......@@ -126,12 +138,13 @@ public class DatabaseSyncServiceImpl implements DatabaseSyncService {
}
columnNameList.clear();
columnTypeList.clear();
}
}
redisUtils.lSet(createTableKey, createTableRedisValue, 86400);
redisUtils.set(insertTableKey, insertRedisValue, 86400);
long endTime = System.currentTimeMillis();
log.info("{}表数据获取完成，共花费{}秒", tableName, (endTime - startTime) / 1000);
insertRedisValue = "";
}
}
redisUtils.set(tableListKey, tableListStr, 86400);
} catch (Exception e) {
e.printStackTrace();
} finally {
......@@ -163,9 +176,8 @@ public class DatabaseSyncServiceImpl implements DatabaseSyncService {
String driver = "com.mysql.jdbc.Driver";
String url = "jdbc:mysql://" + ip + ":" + port;
String createTableKey = dbSyncPrefix + dbName + ":create" ;
String insertTableKey = dbSyncPrefix + dbName + ":insert";
String dbRedisKey = dbSyncPrefix + dbName;
String tableListKey = dbSyncPrefix + dbName + ":tables";
Connection connection = null;
Connection newConnection = null;
......@@ -192,31 +204,33 @@ public class DatabaseSyncServiceImpl implements DatabaseSyncService {
newConnection = DriverManager.getConnection(url + "/" + dbName, username, password);
newConnection.setAutoCommit(false);
// 从redis中获取要同步的表结构
List<Object> createTableRedisValue = redisUtils.lGet(createTableKey, 0, redisUtils.lGetListSize(createTableKey));
List<String> tableList = Arrays.asList(redisUtils.get(tableListKey).toString().split("\n"));
Statement statement = newConnection.createStatement();
log.info("开始同步表结构！");
long structureStartTime = System.currentTimeMillis();
for (Object sql: createTableRedisValue) {
statement.addBatch(sql.toString());
// 循环处理每个表
for (String tableName: tableList) {
log.info("开始同步表：{}", tableName);
long dataStartTime = System.currentTimeMillis();
String createTableKey = dbSyncPrefix + dbName + ":" + tableName + ":create" ;
String insertTableKey = dbSyncPrefix + dbName + ":" + tableName + ":insert" ;
String createTableValue = redisUtils.get(createTableKey).toString();
String insertTableValue = redisUtils.get(insertTableKey).toString();
// 不为空时才执行建表语句
if (!createTableValue.isEmpty()) {
statement.execute(createTableValue);
}
statement.executeBatch();
statement.clearBatch();
long structureEndTime = System.currentTimeMillis();
log.info("表结构同步完成，共花费{}秒", (structureEndTime - structureStartTime) / 1000);
// 从redis中同步表数据
String insertTableRedisValue = redisUtils.get(insertTableKey).toString();
log.info("开始同步表数据！");
long dataStartTime = System.currentTimeMillis();
for (String insertSql: insertTableRedisValue.split("\n")) {
// statement.addBatch(replaceDomain(insertSql, namespace));
// 不为空时才插入数据
if (!insertTableValue.isEmpty()) {
for (String insertSql: insertTableValue.split("\n")) {
statement.addBatch(insertSql);
}
statement.executeBatch();
statement.clearBatch();
}
long dataEndTime = System.currentTimeMillis();
log.info("表数据同步完成，共花费{}秒", (dataEndTime - dataStartTime) / 1000);
log.info("{}表同步完成，共花费{}秒", tableName, (dataEndTime - dataStartTime) / 1000);
}
// 判断是否需要update
if (dbName.equals("cash_loan_flow")) {
......@@ -574,7 +588,6 @@ public class DatabaseSyncServiceImpl implements DatabaseSyncService {
if ("int".equals(columnType) || "INT".equals(columnType)) {
// 整数
Object intValue = resultSet.getObject(index);
if (null == intValue) {
return null;
}
......@@ -686,6 +699,15 @@ public class DatabaseSyncServiceImpl implements DatabaseSyncService {
} else if ("timestamp".equals(columnType) || "TIMESTAMP".equals(columnType)) {
// 时间类型：范围 1970-01-01 00:00:00/2037 年某时 格式 YYYYMMDD HHMMSS 混合日期和时间值，时间戳
return resultSet.getString(index);
} else if ("bit".equals(columnType) || "BIT".equals(columnType)) {
String value = resultSet.getString(index);
if (value.equals("false")) {
return "0";
} else if (value.equals("true")) {
return "1";
} else {
return value;
}
} else {
Object value = resultSet.getObject(index);
if (null == value) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment