Skip to content

C
commons
Commit 70a7c44a authored by xiaoguang.xu's avatar xiaoguang.xu
Browse files
Options

Merge remote-tracking branch 'remotes/origin/new_id_generator' into feature/0.2.0

parents 3b3deea6 7e7aa7cc
Show whitespace changes
Inline Side-by-side
Showing with 239 additions and 127 deletions
commons-spring/src/main/java/cn/quantgroup/tech/util/IDGenerator.java → commons-spring/src/main/java/cn/quantgroup/tech/util/OIDGenerator.java
View file @ 70a7c44a
......@@ -35,7 +35,7 @@ import java.util.concurrent.locks.ReentrantLock;
@Component
@ConditionalOnClass(RedisTemplate.class)
@ConditionalOnProperty(name = "data.center.id")
public class IDGenerator {
public class OIDGenerator {
private static final String REDIS_WORK_ID_KEY = "GLOBAL:WORK:ID:";
private static final String ID_FORMAT = "yyyyMMddHHmmss";
......
commons-spring/src/main/java/cn/quantgroup/tech/util/id/BitsAllocator.java
View file @ 70a7c44a
/*
* Copyright (c) 2017 Baidu, Inc. All Rights Reserve.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cn.quantgroup.tech.util.id;
import org.apache.commons.lang3.builder.ToStringBuilder;
......@@ -24,8 +9,6 @@ import java.math.BigInteger;
/**
* Allocate 64 bits for the UID(long)<br>
* sign (fixed 1bit) -> deltaSecond -> workerId -> sequence(within the same second)
*
* @author yutianbao
*/
public class BitsAllocator {
/**
......@@ -81,7 +64,6 @@ public class BitsAllocator {
/**
* Allocate bits for UID according to delta seconds & workerId & sequence<br>
* <b>Note that: </b>The highest bit will always be 0 for sign
*
* @param deltaSeconds
* @param workerId
......
commons-spring/src/main/java/cn/quantgroup/tech/util/id/DisposableWorkerIdAssigner.java deleted 100644 → 0
View file @ 3b3deea6
/*
* Copyright (c) 2017 Baidu, Inc. All Rights Reserve.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cn.quantgroup.tech.util.id;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
@Component
public class DisposableWorkerIdAssigner {
private static final Logger LOGGER = LoggerFactory.getLogger(DisposableWorkerIdAssigner.class);
private static final String REDIS_WORK_ID_KEY = "GLOBAL:WORK:ID:";
@Autowired
private StringRedisTemplate redisTemplate;
/**
* Assign worker id base on database.<p>
* If there is host name & port in the environment, we considered that the node runs in Docker container<br>
* Otherwise, the node runs on an actual machine.
*
* @param dataCenterId
* @param bitsAllocator
* @return assigned worker id
*/
public long assignWorkerId(long dataCenterId, BitsAllocator bitsAllocator) {
return redisTemplate.opsForValue().increment(REDIS_WORK_ID_KEY + dataCenterId, 1) % bitsAllocator.getMaxWorkerId();
}
}
commons-spring/src/main/java/cn/quantgroup/tech/util/id/UidGenerateException.java → commons-spring/src/main/java/cn/quantgroup/tech/util/id/IDGenerateException.java
View file @ 70a7c44a
......@@ -16,11 +16,9 @@
package cn.quantgroup.tech.util.id;
/**
* UidGenerateException
*
* @author yutianbao
* IDGenerateException
*/
public class UidGenerateException extends RuntimeException {
public class IDGenerateException extends RuntimeException {
/**
* Serial Version UID
......@@ -30,45 +28,16 @@ public class UidGenerateException extends RuntimeException {
/**
* Default constructor
*/
public UidGenerateException() {
public IDGenerateException() {
super();
}
/**
* Constructor with message & cause
*
* @param message
* @param cause
*/
public UidGenerateException(String message, Throwable cause) {
super(message, cause);
}
/**
* Constructor with message
*
* @param message
*/
public UidGenerateException(String message) {
super(message);
}
/**
* Constructor with message format
*
* @param msgFormat
* @param args
*/
public UidGenerateException(String msgFormat, Object... args) {
super(String.format(msgFormat, args));
}
/**
* Constructor with cause
*
* @param cause
*/
public UidGenerateException(Throwable cause) {
public IDGenerateException(Throwable cause) {
super(cause);
}
......
commons-spring/src/main/java/cn/quantgroup/tech/util/id/DefaultUidGenerator.java → commons-spring/src/main/java/cn/quantgroup/tech/util/id/IDGenerator.java
View file @ 70a7c44a
......@@ -8,6 +8,10 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
import org.springframework.util.Assert;
......@@ -17,128 +21,137 @@ import java.util.Date;
import java.util.concurrent.TimeUnit;
/**
* Represents an implementation of {@link UidGenerator}
* Represents an implementation of {@link IDGenerator}
* 基于百度开源项目 uid-generator 的增强版，Snowflake Java 实现版本。项目 Github：https://github.com/baidu/uid-generator
* <p>
* The unique id has 64bits (long), default allocated as blow:<br>
* <li>sign: The highest bit is 0
* <li>delta seconds: The next 28 bits, represents delta seconds since a customer epoch(2016-05-20 00:00:00.000).
* Supports about 8.7 years until to 2024-11-20 21:24:16
* <li>worker id: The next 22 bits, represents the worker's id which assigns based on database, max id is about 420W
* <li>sequence: The next 13 bits, represents a sequence within the same second, max for 8192/s<br><br>
* uid-generator 通过对 64 位数字分区来生成唯一 ID，由以下组成：
* <p>
* -----------------------------------------------------------------------------------
* | sign | delta seconds | worker id | SEQUENCE
* -----------------------------------------------------------------------------------
* 1bits 28bits 22bits 13bits
* -----------------------------------------------------------------------------------
* 其中 delta seconds 为 当前时间 - 指定起始时间。
* 该版本有三个问题
* 1. delta seconds 位数有限，28bits 也只能允许运行 8.7 年左右。
* 2. worker id 生成号码为用后即弃，可容纳重启次数有限。
* 3. 微服务分布式的情况下，无法使用统一数据源，则不同服务生成 worker id 时会重复
* <p>
* <pre>{@code
* +------+----------------------+----------------+-----------+
* | sign | delta seconds | worker node id | sequence |
* +------+----------------------+----------------+-----------+
* 1bit 28bits 22bits 13bits
* }</pre>
* 于是做出以下改进
* 1. worker id 拆分成 data center id，每个服务通过约定指定自己的 data center id 。
* 2. worker id 通过 redis 自增指定，设计为首尾相连的环形，自增数字达到设定的最大值时，会从0开始。
* 2. 不限制使用 delta seconds 的位数，则实现了无限时间的使用。当位数增长到 64 为后，改用 BigInteger 的位运算实现。
* <p>
* You can also specified the bits by Spring property setting.
* <li>timeBits: default as 28
* <li>workerBits: default as 22
* <li>seqBits: default as 13
* <li>epochStr: Epoch date string format 'yyyy-MM-dd'. Default as '2016-05-20'<p>
* 经测试，BigInteger 实现时，性能降低 60% 左右，每秒发号约为 100w~150w。
* 现在 uid 由以下组成
* ---------------------------------------------------------------------------------------------------------
* | sign(length < 64) | delta seconds (unlimited) | data center id | worker id | SEQUENCE
* ---------------------------------------------------------------------------------------------------------
* 1bits 28bits 22bits 22bits 13bits
* ---------------------------------------------------------------------------------------------------------
* 其中 data center id + worker id + SEQUENCE 设定的位数不大于 63。
* <p>
* <b>Note that:</b> The total bits must be 64 -1
* 使用注意：
* 1. 号码的位数不固定，会随着时间增长。data center id + worker id + SEQUENCE 总数设定越大，号码位数越长
* 2. 各个分区的位数、起始时间一旦设定完成投入使用，则后续不能更改。否则会导致发号重复。
*
* @author yutianbao
* @author zhiguo.liu
*/
@Component
public class DefaultUidGenerator implements UidGenerator, InitializingBean {
public static final String DATETIME_PATTERN = "yyyy-MM-dd HH:mm:ss";
public static final String DAY_PATTERN = "yyyy-MM-dd";
private static final Logger LOGGER = LoggerFactory.getLogger(DefaultUidGenerator.class);
@ConditionalOnClass(RedisTemplate.class)
@ConditionalOnProperty(name = "id.epochStr")
public class IDGenerator implements InitializingBean {
private static final String DATETIME_PATTERN = "yyyy-MM-dd HH:mm:ss";
private static final String DAY_PATTERN = "yyyy-MM-dd";
private static final Logger LOGGER = LoggerFactory.getLogger(IDGenerator.class);
private static final String REDIS_WORK_ID_KEY = "GLOBAL:WORK:ID:";
/**
* Bits allocate
*/
@Value("${uid.timeBits:28}")
protected int timeBits;
@Value("${uid.dataCenterIdBits:9}")
@Value("${id.dataCenterIdBits:0}")
protected int dataCenterIdBits;
@Value("${uid.workerBits:13}")
protected int workerBits;
@Value("${uid.seqBits:13}")
@Value("${id.seqBits:13}")
protected int seqBits;
@Value("${id.workerBits:8}")
protected int workerBits;
protected static long DATA_CENTER_ID;
/**
* Customer epoch, unit as second. For example 2018-03-01 (ms: 1463673600000)
*/
protected String epochStr;
protected long epochSeconds;
private static long EPOCH_SECONDS;
/**
* Stable fields after spring bean initializing
*/
protected BitsAllocator bitsAllocator;
protected long workerId;
protected long dataCenterId;
private static BitsAllocator ALLOCATOR;
private static long WORKER_ID;
/**
* Volatile fields caused by nextId()
*/
protected long sequence = 0L;
protected long lastSecond = -1L;
private static long SEQUENCE = 0L;
private static long LAST_SECOND = -1L;
/**
* Spring property
*/
@Autowired
protected DisposableWorkerIdAssigner workerIdAssigner;
private StringRedisTemplate redisTemplate;
@Override
public void afterPropertiesSet() throws Exception {
// initialize bits allocator
bitsAllocator = new BitsAllocator(timeBits, dataCenterIdBits, workerBits, seqBits);
workerIdAssigner = new DisposableWorkerIdAssigner();
int timeBits = 64 - 1 - dataCenterIdBits - workerBits - seqBits;
ALLOCATOR = new BitsAllocator(timeBits, dataCenterIdBits, workerBits, seqBits);
// initialize worker id
workerId = workerIdAssigner.assignWorkerId(dataCenterId, bitsAllocator);
Assert.isTrue(workerId < bitsAllocator.getMaxWorkerId(), "workerId is too big");
Assert.isTrue(dataCenterId < bitsAllocator.getMaxDataCenterId(), "dataCenterId is too big");
WORKER_ID = redisTemplate.opsForValue().increment(REDIS_WORK_ID_KEY + DATA_CENTER_ID, 1) % ALLOCATOR.getMaxWorkerId();
Assert.isTrue(WORKER_ID < ALLOCATOR.getMaxWorkerId(), "WORKER_ID is too big");
Assert.isTrue(ALLOCATOR.getMaxDataCenterId() != 0 && DATA_CENTER_ID < ALLOCATOR.getMaxDataCenterId(), "DATA_CENTER_ID is too big");
LOGGER.info("Initialized bits dataCenterBits:{}, workerBits:{}, seqBits:{}", dataCenterIdBits, workerBits, seqBits);
LOGGER.info("Initialized nodes, workerId:{}, dataCenterId:{}", workerId, dataCenterId);
LOGGER.info("Initialized nodes, WORKER_ID:{}, DATA_CENTER_ID:{}", WORKER_ID, DATA_CENTER_ID);
}
@Override
public String getUID(String preFix) throws UidGenerateException {
public String getID(String prefix) throws IDGenerateException {
try {
return nextId(preFix);
return nextId(prefix);
} catch (Exception e) {
LOGGER.error("Generate unique id exception. ", e);
throw new UidGenerateException(e);
throw new IDGenerateException(e);
}
}
// TODO: 2018/3/5 反序列化 uid
@Override
public String parseUID(String uidStr) {
BigInteger bigInteger = new BigInteger(uidStr);
public String parseID(String idStr) {
BigInteger bigInteger = new BigInteger(idStr);
int totalBits = bigInteger.bitLength();
long dataCenterIdBits = bitsAllocator.getDataCenterIdBits();
long workerIdBits = bitsAllocator.getWorkerIdBits();
long sequenceBits = bitsAllocator.getSequenceBits();
long dataCenterIdBits = ALLOCATOR.getDataCenterIdBits();
long workerIdBits = ALLOCATOR.getWorkerIdBits();
long sequenceBits = ALLOCATOR.getSequenceBits();
if (totalBits < 64) {
totalBits = 64;
long uid = bigInteger.longValue();
long sequence = (uid << (totalBits - sequenceBits)) >>> (totalBits - sequenceBits);
long workerId = (uid << (totalBits - workerIdBits - sequenceBits)) >>> (totalBits - workerIdBits);
long dataCenterId = (uid << (totalBits - dataCenterIdBits - workerIdBits - sequenceBits)) >>> (totalBits - dataCenterIdBits);
long deltaSeconds = uid >>> (dataCenterIdBits + workerIdBits + sequenceBits);
Date thatTime = new Date(TimeUnit.SECONDS.toMillis(epochSeconds + deltaSeconds));
long id = bigInteger.longValue();
long sequence = (id << (totalBits - sequenceBits)) >>> (totalBits - sequenceBits);
long workerId = (id << (totalBits - workerIdBits - sequenceBits)) >>> (totalBits - workerIdBits);
long dataCenterId = (id << (totalBits - dataCenterIdBits - workerIdBits - sequenceBits)) >>> (totalBits - dataCenterIdBits);
if (dataCenterIdBits == 0) {
dataCenterId = 0;
}
long deltaSeconds = id >>> (dataCenterIdBits + workerIdBits + sequenceBits);
Date thatTime = new Date(TimeUnit.SECONDS.toMillis(EPOCH_SECONDS + deltaSeconds));
String thatTimeStr = DateFormatUtils.format(thatTime, DATETIME_PATTERN);
return String.format("{\"UID\":\"%d\",\"timestamp\":\"%s\",\"dataCenterId\":\"%d\",\"workerId\":\"%d\",\"sequence\":\"%d\"}",
uid, thatTimeStr, dataCenterId, workerId, sequence);
return String.format("{\"ID\":\"%d\",\"timestamp\":\"%s\",\"DATA_CENTER_ID\":\"%d\",\"WORKER_ID\":\"%d\",\"SEQUENCE\":\"%d\"}",
id, thatTimeStr, dataCenterId, workerId, sequence);
} else {
BigInteger workerBig = getBigIntegerFromLength(workerIdBits).shiftLeft((int) sequenceBits).and(bigInteger);
System.out.println(workerBig);
long sequence = getBigIntegerFromLength(sequenceBits).and(bigInteger).longValue();
long workerId = getBigIntegerFromLength(workerIdBits).and(bigInteger.shiftRight((int)sequenceBits)).longValue();
long dataCenterId = getBigIntegerFromLength(dataCenterIdBits).and(bigInteger.shiftRight((int)sequenceBits+(int)workerIdBits)).longValue();
long workerId = getBigIntegerFromLength(workerIdBits).and(bigInteger.shiftRight((int) sequenceBits)).longValue();
long dataCenterId = getBigIntegerFromLength(dataCenterIdBits).and(bigInteger.shiftRight((int) sequenceBits + (int) workerIdBits)).longValue();
if (dataCenterIdBits == 0) {
dataCenterId = 0;
}
long deltaSeconds = bigInteger.shiftRight((int) dataCenterIdBits + (int) workerIdBits + (int) sequenceBits).longValue();
Date thatTime = new Date(TimeUnit.SECONDS.toMillis(epochSeconds + deltaSeconds));
Date thatTime = new Date(TimeUnit.SECONDS.toMillis(EPOCH_SECONDS + deltaSeconds));
String thatTimeStr = DateFormatUtils.format(thatTime, DATETIME_PATTERN);
return String.format("{\"UID\":\"%d\",\"timestamp\":\"%s\",\"dataCenterId\":\"%d\",\"workerId\":\"%d\",\"sequence\":\"%d\"}",
return String.format("{\"ID\":\"%d\",\"timestamp\":\"%s\",\"DATA_CENTER_ID\":\"%d\",\"WORKER_ID\":\"%d\",\"SEQUENCE\":\"%d\"}",
bigInteger, thatTimeStr, dataCenterId, workerId, sequence);
}
}
......@@ -148,38 +161,38 @@ public class DefaultUidGenerator implements UidGenerator, InitializingBean {
}
/**
* Get UID
* Get ID
*
* @return UID
* @throws UidGenerateException in the case: Clock moved backwards; Exceeds the max timestamp
* @return ID
* @throws IDGenerateException in the case: Clock moved backwards; Exceeds the max timestamp
*/
protected synchronized String nextId(String preFix) {
long currentSecond = getCurrentSecond();
// Clock moved backwards, wait for newest time
if (currentSecond < lastSecond) {
getNextSecond(lastSecond);
if (currentSecond < LAST_SECOND) {
getNextSecond(LAST_SECOND);
}
// At the same second, increase sequence
if (currentSecond == lastSecond) {
sequence = (sequence + 1) & bitsAllocator.getMaxSequence();
// Exceed the max sequence, we wait the next second to generate uid
if (sequence == 0) {
currentSecond = getNextSecond(lastSecond);
// At the same second, increase SEQUENCE
if (currentSecond == LAST_SECOND) {
SEQUENCE = (SEQUENCE + 1) & ALLOCATOR.getMaxSequence();
// Exceed the max SEQUENCE, we wait the next second to generate ID
if (SEQUENCE == 0) {
currentSecond = getNextSecond(LAST_SECOND);
}
// At the different second, sequence restart from zero
// At the different second, SEQUENCE restart from zero
} else {
sequence = 0L;
SEQUENCE = 0L;
}
lastSecond = currentSecond;
LAST_SECOND = currentSecond;
// 当前时间小于设定的最大时间，即总位数在 64 位以下，用 long 生成数字
if (currentSecond - epochSeconds <= bitsAllocator.getMaxDeltaSeconds()) {
return preFix + bitsAllocator.allocate(currentSecond - epochSeconds, dataCenterId, workerId, sequence);
if (currentSecond - EPOCH_SECONDS <= ALLOCATOR.getMaxDeltaSeconds()) {
return preFix + ALLOCATOR.allocate(currentSecond - EPOCH_SECONDS, DATA_CENTER_ID, WORKER_ID, SEQUENCE);
}
return preFix + bitsAllocator.allocateBigInteger(currentSecond - epochSeconds, dataCenterId, workerId, sequence);
return preFix + ALLOCATOR.allocateBigInteger(currentSecond - EPOCH_SECONDS, DATA_CENTER_ID, WORKER_ID, SEQUENCE);
}
/**
......@@ -201,33 +214,21 @@ public class DefaultUidGenerator implements UidGenerator, InitializingBean {
return TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
}
public void setTimeBits(int timeBits) {
if (timeBits > 0) {
this.timeBits = timeBits;
}
}
public void setWorkerBits(int workerBits) {
if (workerBits > 0) {
this.workerBits = workerBits;
}
@Value("${data.center.id}")
public void setDataCenterId(Integer dataCenterId) {
DATA_CENTER_ID = dataCenterId;
}
public void setSeqBits(int seqBits) {
if (seqBits > 0) {
this.seqBits = seqBits;
}
}
@Value("${uid.epochStr:2018-03-01}")
@Value("${id.epochStr:2018-04-01}")
public void setEpochStr(String epochStr) {
if (StringUtils.isNotBlank(epochStr)) {
this.epochStr = epochStr;
try {
this.epochSeconds = TimeUnit.MILLISECONDS.toSeconds(DateUtils.parseDate(epochStr, new String[]{DAY_PATTERN}).getTime());
EPOCH_SECONDS = TimeUnit.MILLISECONDS.toSeconds(DateUtils.parseDate(epochStr, new String[]{DAY_PATTERN}).getTime());
} catch (ParseException e) {
e.printStackTrace();
}
}
}
}
commons-spring/src/main/java/cn/quantgroup/tech/util/id/UidGenerator.java deleted 100644 → 0
View file @ 3b3deea6
package cn.quantgroup.tech.util.id;
/**
* Represents a unique id generator.
*
* @author yutianbao
*/
public interface UidGenerator {
/**
* Get a unique ID
*
* @return UID
* @throws UidGenerateException
*/
String getUID(String preFix) throws UidGenerateException;
/**
* Parse the UID into elements which are used to generate the UID. <br>
* Such as timestamp & workerId & sequence...
*
* @param uid
* @return Parsed info
*/
String parseUID(String uid);
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment