Skip to content

H
holmes
Commit 5e3d377a authored by 黎博's avatar 黎博
Browse files
Options

Merge branch 'master' into ldap

parents 5bc05069 881a7266
Show whitespace changes
Inline Side-by-side
Showing with 98 additions and 75 deletions
src/main/java/cn/qg/holmes/config/RedisConfig.java
View file @ 5e3d377a
...@@ -45,10 +45,11 @@ public class RedisConfig extends CachingConfigurerSupport { ...@@ -45,10 +45,11 @@ public class RedisConfig extends CachingConfigurerSupport {
public JedisPoolConfig jedisPoolConfig() { public JedisPoolConfig jedisPoolConfig() {
JedisPoolConfig jedisPoolConfig = new JedisPoolConfig(); JedisPoolConfig jedisPoolConfig = new JedisPoolConfig();
jedisPoolConfig.setMaxTotal(32); jedisPoolConfig.setMaxTotal(32);
jedisPoolConfig.setMaxIdle(16); jedisPoolConfig.setMaxIdle(32);
jedisPoolConfig.setMinIdle(8); jedisPoolConfig.setMinIdle(0);
jedisPoolConfig.setMaxWaitMillis(5000);
jedisPoolConfig.setTestOnBorrow(true); jedisPoolConfig.setTestOnBorrow(true);
jedisPoolConfig.setTestWhileIdle(true);
jedisPoolConfig.setTestOnReturn(true);
return jedisPoolConfig; return jedisPoolConfig;
} }
......
src/main/java/cn/qg/holmes/service/effect/impl/DatabaseSyncServiceImpl.java
View file @ 5e3d377a
...@@ -12,10 +12,7 @@ import org.springframework.stereotype.Service; ...@@ -12,10 +12,7 @@ import org.springframework.stereotype.Service;
import java.math.BigDecimal; import java.math.BigDecimal;
import java.sql.*; import java.sql.*;
import java.util.ArrayList; import java.util.*;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Slf4j @Slf4j
@Service @Service
...@@ -51,15 +48,19 @@ public class DatabaseSyncServiceImpl implements DatabaseSyncService { ...@@ -51,15 +48,19 @@ public class DatabaseSyncServiceImpl implements DatabaseSyncService {
public boolean getDbInfoFromSource(String ip, String port, String username, String password, String dbName) { public boolean getDbInfoFromSource(String ip, String port, String username, String password, String dbName) {
String driver = "com.mysql.jdbc.Driver"; String driver = "com.mysql.jdbc.Driver";
String url = "jdbc:mysql://" + ip + ":" + port + "/" + dbName; String url = "jdbc:mysql://" + ip + ":" + port + "/" + dbName;
String createTableKey = dbSyncPrefix + dbName + ":create" ;
String insertTableKey = dbSyncPrefix + dbName + ":insert";
String dbRedisKey = dbSyncPrefix + dbName; String dbRedisKey = dbSyncPrefix + dbName;
String tableListKey = dbSyncPrefix + dbName + ":tables";
Connection connection = null; Connection connection = null;
PreparedStatement preparedStatement = null; PreparedStatement preparedStatement = null;
// 创建数据库的redis值
String dbRedisValue = ""; String dbRedisValue = "";
// 表数据redis值
String insertRedisValue = ""; String insertRedisValue = "";
List<Object> createTableRedisValue = new ArrayList<>(); // 建表语句redis值
String createTableRedisValue = "";
// 表名列表redis值
String tableListStr = "";
List<String> columnNameList = new ArrayList<>(); List<String> columnNameList = new ArrayList<>();
List<String> columnTypeList = new ArrayList<>(); List<String> columnTypeList = new ArrayList<>();
...@@ -79,19 +80,30 @@ public class DatabaseSyncServiceImpl implements DatabaseSyncService { ...@@ -79,19 +80,30 @@ public class DatabaseSyncServiceImpl implements DatabaseSyncService {
redisUtils.set(dbRedisKey, dbRedisValue, 86400); redisUtils.set(dbRedisKey, dbRedisValue, 86400);
} }
if (!redisUtils.hasKey(createTableKey)) {
DatabaseMetaData databaseMetaData = connection.getMetaData(); DatabaseMetaData databaseMetaData = connection.getMetaData();
// 获取所有表名 // 获取所有表名
ResultSet tableResultSet = databaseMetaData.getTables(null, null, null, new String[]{"TABLE"}); ResultSet tableResultSet = databaseMetaData.getTables(null, null, null, new String[]{"TABLE"});
while (tableResultSet.next()) { while (tableResultSet.next()) {
String tableName = tableResultSet.getString("TABLE_NAME"); String tableName = tableResultSet.getString("TABLE_NAME");
tableListStr += tableName;
tableListStr += "\n";
long startTime = System.currentTimeMillis();
log.info("开始获取表{}的数据", tableName);
String createTableKey = dbSyncPrefix + dbName + ":" + tableName + ":create" ;
String insertTableKey = dbSyncPrefix + dbName + ":" + tableName + ":insert" ;
if (!redisUtils.hasKey(createTableKey)) {
// 获取所有建表语句 // 获取所有建表语句
String sql = String.format("SHOW CREATE TABLE %s", tableName); String sql = String.format("SHOW CREATE TABLE %s", tableName);
preparedStatement = connection.prepareStatement(sql); preparedStatement = connection.prepareStatement(sql);
ResultSet pResultSet = preparedStatement.executeQuery(); ResultSet pResultSet = preparedStatement.executeQuery();
while (pResultSet.next()) { while (pResultSet.next()) {
createTableRedisValue.add(pResultSet.getString(2)); createTableRedisValue = pResultSet.getString(2);
}
redisUtils.set(createTableKey, createTableRedisValue, 86400);
createTableRedisValue = "";
} }
if (!redisUtils.hasKey(insertTableKey)) { if (!redisUtils.hasKey(insertTableKey)) {
...@@ -126,12 +138,13 @@ public class DatabaseSyncServiceImpl implements DatabaseSyncService { ...@@ -126,12 +138,13 @@ public class DatabaseSyncServiceImpl implements DatabaseSyncService {
} }
columnNameList.clear(); columnNameList.clear();
columnTypeList.clear(); columnTypeList.clear();
}
}
redisUtils.lSet(createTableKey, createTableRedisValue, 86400);
redisUtils.set(insertTableKey, insertRedisValue, 86400); redisUtils.set(insertTableKey, insertRedisValue, 86400);
long endTime = System.currentTimeMillis();
log.info("{}表数据获取完成，共花费{}秒", tableName, (endTime - startTime) / 1000);
insertRedisValue = "";
} }
}
redisUtils.set(tableListKey, tableListStr, 86400);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} finally { } finally {
...@@ -163,9 +176,8 @@ public class DatabaseSyncServiceImpl implements DatabaseSyncService { ...@@ -163,9 +176,8 @@ public class DatabaseSyncServiceImpl implements DatabaseSyncService {
String driver = "com.mysql.jdbc.Driver"; String driver = "com.mysql.jdbc.Driver";
String url = "jdbc:mysql://" + ip + ":" + port; String url = "jdbc:mysql://" + ip + ":" + port;
String createTableKey = dbSyncPrefix + dbName + ":create" ;
String insertTableKey = dbSyncPrefix + dbName + ":insert";
String dbRedisKey = dbSyncPrefix + dbName; String dbRedisKey = dbSyncPrefix + dbName;
String tableListKey = dbSyncPrefix + dbName + ":tables";
Connection connection = null; Connection connection = null;
Connection newConnection = null; Connection newConnection = null;
...@@ -192,31 +204,33 @@ public class DatabaseSyncServiceImpl implements DatabaseSyncService { ...@@ -192,31 +204,33 @@ public class DatabaseSyncServiceImpl implements DatabaseSyncService {
newConnection = DriverManager.getConnection(url + "/" + dbName, username, password); newConnection = DriverManager.getConnection(url + "/" + dbName, username, password);
newConnection.setAutoCommit(false); newConnection.setAutoCommit(false);
// 从redis中获取要同步的表结构 List<String> tableList = Arrays.asList(redisUtils.get(tableListKey).toString().split("\n"));
List<Object> createTableRedisValue = redisUtils.lGet(createTableKey, 0, redisUtils.lGetListSize(createTableKey));
Statement statement = newConnection.createStatement(); Statement statement = newConnection.createStatement();
log.info("开始同步表结构！"); // 循环处理每个表
long structureStartTime = System.currentTimeMillis(); for (String tableName: tableList) {
for (Object sql: createTableRedisValue) { log.info("开始同步表：{}", tableName);
statement.addBatch(sql.toString()); long dataStartTime = System.currentTimeMillis();
String createTableKey = dbSyncPrefix + dbName + ":" + tableName + ":create" ;
String insertTableKey = dbSyncPrefix + dbName + ":" + tableName + ":insert" ;
String createTableValue = redisUtils.get(createTableKey).toString();
String insertTableValue = redisUtils.get(insertTableKey).toString();
// 不为空时才执行建表语句
if (!createTableValue.isEmpty()) {
statement.execute(createTableValue);
} }
statement.executeBatch();
statement.clearBatch();
long structureEndTime = System.currentTimeMillis();
log.info("表结构同步完成，共花费{}秒", (structureEndTime - structureStartTime) / 1000);
// 从redis中同步表数据 // 不为空时才插入数据
String insertTableRedisValue = redisUtils.get(insertTableKey).toString(); if (!insertTableValue.isEmpty()) {
log.info("开始同步表数据！"); for (String insertSql: insertTableValue.split("\n")) {
long dataStartTime = System.currentTimeMillis();
for (String insertSql: insertTableRedisValue.split("\n")) {
// statement.addBatch(replaceDomain(insertSql, namespace));
statement.addBatch(insertSql); statement.addBatch(insertSql);
} }
statement.executeBatch(); statement.executeBatch();
statement.clearBatch(); statement.clearBatch();
}
long dataEndTime = System.currentTimeMillis(); long dataEndTime = System.currentTimeMillis();
log.info("表数据同步完成，共花费{}秒", (dataEndTime - dataStartTime) / 1000); log.info("{}表同步完成，共花费{}秒", tableName, (dataEndTime - dataStartTime) / 1000);
}
// 判断是否需要update // 判断是否需要update
if (dbName.equals("cash_loan_flow")) { if (dbName.equals("cash_loan_flow")) {
...@@ -574,7 +588,6 @@ public class DatabaseSyncServiceImpl implements DatabaseSyncService { ...@@ -574,7 +588,6 @@ public class DatabaseSyncServiceImpl implements DatabaseSyncService {
if ("int".equals(columnType) || "INT".equals(columnType)) { if ("int".equals(columnType) || "INT".equals(columnType)) {
// 整数 // 整数
Object intValue = resultSet.getObject(index); Object intValue = resultSet.getObject(index);
if (null == intValue) { if (null == intValue) {
return null; return null;
} }
...@@ -686,6 +699,15 @@ public class DatabaseSyncServiceImpl implements DatabaseSyncService { ...@@ -686,6 +699,15 @@ public class DatabaseSyncServiceImpl implements DatabaseSyncService {
} else if ("timestamp".equals(columnType) || "TIMESTAMP".equals(columnType)) { } else if ("timestamp".equals(columnType) || "TIMESTAMP".equals(columnType)) {
// 时间类型：范围 1970-01-01 00:00:00/2037 年某时 格式 YYYYMMDD HHMMSS 混合日期和时间值，时间戳 // 时间类型：范围 1970-01-01 00:00:00/2037 年某时 格式 YYYYMMDD HHMMSS 混合日期和时间值，时间戳
return resultSet.getString(index); return resultSet.getString(index);
} else if ("bit".equals(columnType) || "BIT".equals(columnType)) {
String value = resultSet.getString(index);
if (value.equals("false")) {
return "0";
} else if (value.equals("true")) {
return "1";
} else {
return value;
}
} else { } else {
Object value = resultSet.getObject(index); Object value = resultSet.getObject(index);
if (null == value) { if (null == value) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment