一、rocketmq順序消費的原理
1、消息的有序性是指消息的消費順序能夠與消息的發送順序一致。但是有時候我們從業務需要上面並不需要保證所有消息嚴格按照消費順序完全一致。例如，一個訂單的下單、付款、出庫等操作是不同替換順序。但是有A訂單和B訂單，並不需要保證A訂單與B訂單的順序。
2、RocketMQ採用了局部順序一致性的機制，一組消息發送到同一個隊列中來保證發送順序的有序性，然後再由消費者進行。消費的時候通過一個隊列只會被一個線程取到 ，第二個線程無法訪問這個隊列 來保證隊列有序性。rocketmq可以同時多個隊列並列消費提高，提高rocketmq的消費速度。其實這個方案很像jdk7 裏面ConcurrentHashMap 實現分段鎖的實現，通過保證每段的線程安全，多段並行消費提高消費能力。
二、代碼的具體實現
1、引入依賴
<!-- https://mvnrepository.com/artifact/org.apache.rocketmq/rocketmq-client -->
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-client</artifactId>
<version>4.2.0</version>
</dependency>
生產方：
2、rocketmq 生產者的配置文件
###producer
#該應用是否啓用生產者
rocketmq.producer.isOnOff=on
#發送同一類消息的設置爲同一個group，保證唯一,默認不需要設置，
# rocketmq會使用ip@pid(pid代表jvm名字)作爲唯一標示
rocketmq.producer.groupName=rocketProducerGroup
#mq的nameserver地址
rocketmq.producer.addr=192.168.25.128:9876
#消息最大長度 默認1024*4(4M)
rocketmq.producer.maxMessage=4096
#發送消息超時時間,默認3000
rocketmq.send.outTime=3000
#發送消息失敗重試次數，默認2
rocketmq.producer.retryNum=2
rocketmq.producer.topic=orderTopi
rocketmq.producer.tag=demoTag
2、生產者的配置類
@Configuration
@Component
public class RocketProduceConfig {
private static final Logger LOGGER = LoggerFactory.getLogger(RocketProduceConfig.class);
/**
* 發送同一類消息的設置爲同一個group，
* 保證唯一,默認不需要設置，rocketmq會使用ip@pid(pid代表jvm名字)作爲唯一標示
*/
@Value("${rocketmq.producer.groupName}")
private String groupName;
@Value("${rocketmq.producer.addr}")
private String producerAddr;
/**
* 消息最大大小，默認4M
*/
@Value("${rocketmq.producer.maxMessage}")
private Integer maxMessageSize;
/**
* 消息發送超時時間，默認3秒
*/
@Value("${rocketmq.send.outTime}")
private Integer sendMsgTimeOut;
/**
* 消息發送失敗重試次數，默認2次
*/
@Value("${rocketmq.producer.retryNum}")
private Integer retryNum;
@Bean
public DefaultMQProducer getRocketMQProducer() throws RocketMQException {
DefaultMQProducer producer = new DefaultMQProducer(this.groupName);
producer.setNamesrvAddr(this.producerAddr);
//如果需要同一個jvm中不同的producer往不同的mq集羣發送消息，需要設置不同的instanceName
//producer.setInstanceName(instanceName);
if(this.maxMessageSize!=null){
producer.setMaxMessageSize(this.maxMessageSize);
}
if(this.sendMsgTimeOut!=null){
producer.setSendMsgTimeout(this.sendMsgTimeOut);
}
//如果發送消息失敗，設置重試次數，默認爲2次
if(this.retryNum!=null){
producer.setRetryTimesWhenSendFailed(this.retryNum);
}
try {
producer.start();
LOGGER.info("producer is start, groupName:{},namesrvAddr:{}"
, this.groupName, this.producerAddr);
} catch (MQClientException e) {
LOGGER.error(String.format("producer is error {}"
, e.getMessage(),e));
throw new RocketMQException(e);
}
return producer;
}
}
3、生產者需要保證同一個消息發送到對應隊列需要實現MessageQueueSelector ，
redis cluster鍵的crc16算法均勻分配策略 這個依賴和需要CRC16 在附錄給出，如果本身arg是數值且自增長可以直接求餘
public class SeqMessageQueueSelector implements MessageQueueSelector {
/**
*這裏需要注意的是，我這邊採用了利用redis cluster鍵的crc16算法均勻分配策略，如果本身arg是數值且自增長可以直接求餘
* @param mqs 隊列
* @param message 消息
* @param arg 這個arg是來自DefaultMQProducer.send(Message msg, MessageQueueSelector selector, Object arg) 中的msg
* @return
*/
@Override
public MessageQueue select(List<MessageQueue> mqs, Message message, Object arg) {
String s = String.valueOf(arg);
int crc16 = CRC16.getCRC16(s);
return mqs.get(crc16 % mqs.size());
}
}
4、發送消息測試：
@Value("${rocketmq.producer.topic}")
private String topic;
@Value("${rocketmq.producer.tag}")
private String tag;
@Autowired
private DefaultMQProducer defaultMQProducer;
@Test
public void sendRocketMessage() throws InterruptedException {
ExecutorService service = Executors.newFixedThreadPool(5);
service.execute(new SeqRunnable("abc"));
service.execute(new SeqRunnable("efg"));
service.execute(new SeqRunnable("hig"));
Thread.currentThread().join();
}
class SeqRunnable implements Runnable {
Logger logger = LoggerFactory.getLogger(SeqRunnable.class);
private String orderId;
public SeqRunnable(String orderId) {
this.orderId = orderId;
}
@Override
public void run() {
for (int i = 0; i < 3; i++) {
try {
System.out.println("開始發送消息：" + orderId + ",第 " + i + "次");
String msg = orderId + ",第 " + i + "次消息";
String keys = msg;
SeqMessageQueueSelector selector = new SeqMessageQueueSelector();
Message sendMsg = new Message(topic, tag, keys, msg.getBytes(RemotingHelper.DEFAULT_CHARSET));
//默認3秒超時
SendResult sendResult = defaultMQProducer.send(sendMsg, selector, orderId);
System.out.println("消息發送響應信息：" + sendResult.toString());
} catch (Exception e) {
logger.error("發送消息發生異常:{}", orderId, e);
}
}
}
}
消費方：
1、消費方配置文件
###consumer
##該應用是否啓用消費者
rocketmq.consumer.isOnOff=on
rocketmq.consumer.groupName=rocketCustomerGroup
#mq的nameserver地址
rocketmq.consumer.namesrvAddr=192.168.25.128:9876
#該消費者訂閱的主題和tags("*"號表示訂閱該主題下所有的tags),格式：topic~tag1||tag2||tag3;topic2~*;
rocketmq.consumer.topics=orderTopi~*;;
rocketmq.consumer.consumeThreadMin=20
rocketmq.consumer.consumeThreadMax=64
#設置一次消費消息的條數，默認爲1條
rocketmq.consumer.consumeMessageBatchMaxSize=10
2、消費方的配置類
@Configuration
public class MQConsumerConfiguration {
public static final Logger LOGGER = LoggerFactory.getLogger(MQConsumerConfiguration.class);
@Value("${rocketmq.consumer.namesrvAddr}")
private String namesrvAddr;
@Value("${rocketmq.consumer.groupName}")
private String groupName;
@Value("${rocketmq.consumer.consumeThreadMin}")
private int consumeThreadMin;
@Value("${rocketmq.consumer.consumeThreadMax}")
private int consumeThreadMax;
@Value("${rocketmq.consumer.topics}")
private String topics;
@Value("${rocketmq.consumer.consumeMessageBatchMaxSize}")
private int consumeMessageBatchMaxSize;
@Autowired
private SeqConsumeListenerProcessor seqConsumeListenerProcessor;
@Bean
public DefaultMQPushConsumer getRocketMQConsumer() throws RocketMQException {
if (StringUtils.isEmpty(groupName)){
throw new RocketMQException("groupName is null");
}
if (StringUtils.isEmpty(namesrvAddr)){
throw new RocketMQException("namesrvAddr is null");
}
if(StringUtils.isEmpty(topics)){
throw new RocketMQException("topics is null");
}
DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(groupName);
consumer.setNamesrvAddr(namesrvAddr);
consumer.setConsumeThreadMin(consumeThreadMin);
consumer.setConsumeThreadMax(consumeThreadMax);
consumer.registerMessageListener(seqConsumeListenerProcessor);
/**
* 設置Consumer第一次啓動是從隊列頭部開始消費還是隊列尾部開始消費
* 如果非第一次啓動，那麼按照上次消費的位置繼續消費
*/
consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);
/**
* 設置消費模型，集羣還是廣播，默認爲集羣
*/
consumer.setMessageModel(MessageModel.CLUSTERING);
/**
* 設置一次消費消息的條數，默認爲1條
*/
consumer.setConsumeMessageBatchMaxSize(consumeMessageBatchMaxSize);
try {
/**
* 設置該消費者訂閱的主題和tag，如果是訂閱該主題下的所有tag，則tag使用*；
* 如果需要指定訂閱該主題下的某些tag，則使用||分割，例如tag1||tag2||tag3
*/
String[] topicTagsArr = topics.split(";");
for (String topicTags : topicTagsArr) {
String[] topicTag = topicTags.split("~");
consumer.subscribe(topicTag[0],topicTag[1]);
}
consumer.start();
LOGGER.info("consumer is start, groupName:{},topics:{},namesrvAddr:{}",groupName,topics,namesrvAddr);
}catch (MQClientException e){
LOGGER.error("consumer is start ,groupName:{},topics:{},namesrvAddr:{}",groupName,topics,namesrvAddr,e);
throw new RocketMQException(e);
}
return consumer;
}
}
2、消費方需要實現的類，這裏需要順序消費的監聽器：必須實現MessageListenerOrderly ，來保證一個隊列只有一個線程消費。保證消費有序性
@Slf4j
@Component
public class SeqConsumeListenerProcessor implements MessageListenerOrderly {
@Override
public ConsumeOrderlyStatus consumeMessage(List<MessageExt> list, ConsumeOrderlyContext consumeOrderlyContext) {
if (CollectionUtils.isEmpty(list)){
return ConsumeOrderlyStatus.SUCCESS;
}
//設置自動提交
consumeOrderlyContext.setAutoCommit(true);
for (MessageExt msg : list) {
String messageBody="";
try {
messageBody = new String(msg.getBody(), RemotingHelper.DEFAULT_CHARSET);
System.out.println("messageId: " + msg.getMsgId() + ",topic: " + msg.getTopic() + ",tags: "
+ msg.getTags() + ",keys: " + msg.getKeys() + ",messageBody: " + messageBody);
} catch (Exception e) {
log.error("{} consume message has a error",messageBody);
throw new RuntimeException(e);
}
}
return ConsumeOrderlyStatus.SUCCESS;
}
}
測試結果（生產方）
測試結果（消費方）
可以看到不同隊列的消費並沒有順序，消費方保證同一個隊列的前後有序性。
在消費方使用到CRC16的分配策略，這個以附錄形式，展示，供大家測試：
附錄：
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>2.9.0</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId>
<version>2.0.5.RELEASE</version>
</dependency>
public class CRC16 {
static final int[] CRC16_TAB256 = {0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, 0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce,
0xf1ef, 0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6, 0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de,
0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485, 0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d, 0x3653,
0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4, 0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc, 0x48c4, 0x58e5,
0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823, 0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b, 0x5af5, 0x4ad4, 0x7ab7,
0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12, 0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a, 0x6ca6, 0x7c87, 0x4ce4, 0x5cc5,
0x2c22, 0x3c03, 0x0c60, 0x1c41, 0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49, 0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13,
0x2e32, 0x1e51, 0x0e70, 0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78, 0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d,
0xf14e, 0xe16f, 0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067, 0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f,
0xf35e, 0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256, 0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d,
0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405, 0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c, 0x26d3,
0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634, 0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab, 0x5844, 0x4865,
0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3, 0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a, 0x4a75, 0x5a54, 0x6a37,
0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92, 0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9, 0x7c26, 0x6c07, 0x5c64, 0x4c45,
0x3ca2, 0x2c83, 0x1ce0, 0x0cc1, 0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8, 0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93,
0x3eb2, 0x0ed1, 0x1ef0};
private CRC16() throws InstantiationException {
throw new InstantiationException("Must not instantiate this class");
}
public static int getSlot(String key) {
key = JedisClusterHashTagUtil.getHashTag(key);
return getCRC16(key) & 0x3FFF;
}
public static int getSlot(byte[] key) {
int s = -1;
int e = -1;
boolean sFound = false;
for (int i = 0, length = key.length; i < length; i++) {
if (key[i] == '{' && !sFound) {
s = i;
sFound = true;
}
if (key[i] == '}' && sFound) {
e = i;
break;
}
}
if (s > -1 && e > -1 && e != s + 1) {
return getCRC16(key, s + 1, e) & 0x3FFF;
}
return getCRC16(key) & 0x3FFF;
}
public static int getCRC16(byte[] bytes, int s, int e) {
int crc = 0x0000;
for (int i = s; i < e; i++) {
crc = ((crc << 8) ^ CRC16_TAB256[((crc >>> 8) ^ (bytes[i] & 0xff)) & 0xff]);
}
return crc & 0xffff;
}
public static int getCRC16(byte[] bytes){
return getCRC16(bytes, 0, bytes.length);
}
public static int getCRC16(String key) {
byte[] bytesKey = SafeEncoder.encode(key);
return getCRC16(bytesKey, 0, bytesKey.length);
}
}