RocketMq實用

文章目錄

1. rocketmq 生產者

package com.becom.qoe.qoeservice.service.rocketmq.impl;

import java.io.UnsupportedEncodingException;

import javax.annotation.PostConstruct;

import org.apache.commons.lang3.time.StopWatch;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.client.producer.SendStatus;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.remoting.common.RemotingHelper;
import org.apache.rocketmq.remoting.exception.RemotingException;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import com.becom.qoe.qoeservice.entity.stuscorestatistics.StuNormScore;
import com.becom.qoe.qoeservice.service.rocketmq.IRocketMqProducer;

import lombok.extern.slf4j.Slf4j;
import net.sf.json.JSONObject;
@Slf4j
@Service
public class RocketMqProducer implements IRocketMqProducer<StuNormScore>{
	@Value("${apache.rocketmq.producer.producerGroup}")
	private String producerGroup;
	
	@Value("${apache.rocketmq.producer.topic}")
	private String topic;
	
	@Value("${apache.rocketmq.producer.tag}")
	private String tag;
	
	@Value("${apache.rocketmq.namesrvAddr}")
	private String namesrvAddr;
	
	private DefaultMQProducer producer;
	
	@PostConstruct
	public void defaultMQProducer() {
		// 生產者的組名
		producer = new DefaultMQProducer(producerGroup);
		// 指定NameServer地址，多個地址以 ; 隔開
		producer.setNamesrvAddr(namesrvAddr);
		producer.setVipChannelEnabled(false);
		try {
			producer.start();
			log.info("MQ：啓動啓動生產者");
		} catch (MQClientException e) {
			e.printStackTrace();
		}
	}
	
	@Override
	public boolean send(StuNormScore om) {
		JSONObject json = JSONObject.fromObject(om);
		Message message;
		StopWatch stop = new StopWatch();
		stop.start();
		boolean stat = false;
		try {
			message = new Message(topic, tag, json.toString().getBytes(RemotingHelper.DEFAULT_CHARSET));
			SendResult result = producer.send(message);
			log.info("MQ 消息生產者發送消息狀態: "+ result.getSendStatus());
			if (SendStatus.SEND_OK.equals(result.getSendStatus())) {
				stat = true;
			}
		} catch (UnsupportedEncodingException | MQClientException | RemotingException | MQBrokerException | InterruptedException e) {
			log.error(e.getMessage(), e);
		}
		stop.stop();
		return stat;
	}

}

2. rocketmq 消費者

消費者類型1：PushConsumer

package com.becom.qoe.asyncproccess.rocketmq;

import java.util.List;

import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component;

import com.becom.qoe.asyncproccess.entity.StuNormScore;
import com.becom.qoe.asyncproccess.service.MainExecuteStatistic;

import lombok.extern.slf4j.Slf4j;
import net.sf.json.JSONObject;
@Slf4j
@Component
public class RocketMqConsumer implements CommandLineRunner{

	@Value("${apache.rocketmq.consumer.ConsumerGroup}")
	private String consumerGroupName;
	@Value("${apache.rocketmq.namesrvAddr}")
	private String namesrvAddr;
	@Value("${apache.rocketmq.producer.topic}")
	private String topic;
	@Value("${apache.rocketmq.producer.tag}")
	private String tag;
	@Autowired
	private MainExecuteStatistic mainExecuteStatistic;
	/**
	 * MQ 消息消費端
	 * 消費策略：
	 * CONSUME_FROM_LAST_OFFSET 默認策略，從該隊列最尾開始消費，即跳過歷史消息
	 * CONSUME_FROM_FIRST_OFFSET 從隊列最開始開始消費，即歷史消息（還儲存在broker的）全部消費一遍
	 * CONSUME_FROM_TIMESTAMP 從某個時間點開始消費，和setConsumeTimestamp()配合使用，默認是半個小時以前
	 * 消費模式：
	 * CLUSTERING：集羣，默認
	 * 同一個Group裏每個consumer只消費訂閱消息的一部分內容，也就是同一groupName,所有消費的內容加起來纔是訂閱topic內容的整體，達到負載均衡的目的
	 * BROADCASTING:廣播模式
	 * 同一個Group裏每個consumer都能消費到所訂閱topic的全部消息，也就是一個消息會被分發多次，被多個consumer消費
	 * 廣播消息只發送一次，沒有重試
	 * 返回消費狀態：
	 * CONSUME_SUCCESS 消費成功  
	 * RECONSUME_LATER 消費失敗，需要稍後重新消費
	 * 重試機制(consumer),僅限於CLUSTERING模式
	 * 1.exception的情況，一般重複16次 10s、30s、1分鐘、2分鐘、3分鐘等等  獲取重試次數：msgs.get(0).getReconsumeTimes()
	 * 2.超時的情況，這種情況MQ會無限制的發送給消費端 就是由於網絡的情況，MQ發送數據之後，Consumer端並沒有收到導致超時。也就是消費端沒有給我返回
	 * return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;這樣的就認爲沒有到達Consumer端
	 */
	public void messageListener() {
		log.info("MQ：啓動消費者");
		DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(consumerGroupName);

		consumer.setNamesrvAddr(namesrvAddr);
		try {
			// 訂閱PushTopic下Tag爲push的消息,都訂閱消息
			consumer.subscribe(topic, tag);
			// 程序第一次啓動從消息隊列頭獲取數據
			consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
			consumer.setMessageModel(MessageModel.CLUSTERING);
			// 可以修改每次消費消息的數量，默認設置是每次消費一條
			consumer.setConsumeMessageBatchMaxSize(1);
			// 在此監聽中消費信息，並返回消費的狀態信息
			/*consumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {
				// 會把不同的消息分別放置到不同的隊列中
				for (Message msg : msgs) {
					StuNormScore stuNormScore = (StuNormScore) JSONObject.toBean(JSONObject.fromObject(new String(msg.getBody())), StuNormScore.class);
					log.info("rocketMQ client 成功獲取消息" + msg);
					mainExecuteStatistic.executeStatistic(stuNormScore);
				}
				return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
			});*/
			consumer.registerMessageListener(new MessageListenerConcurrently() {
				@Override
				public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
						ConsumeConcurrentlyContext context) {
					
					int index = 0;
					try {
						for (; index < msgs.size(); index++) {
							log.info("MQ 消費者 獲取消息重試次數" + msgs.get(index).getReconsumeTimes());
							MessageExt msg = msgs.get(index);
							String messageBody = new String(msg.getBody());
							log.info("MQ 消費者 成功獲取消息" + messageBody);
							StuNormScore stuNormScore = (StuNormScore) JSONObject.toBean(JSONObject.fromObject(messageBody), StuNormScore.class);
							Integer num = mainExecuteStatistic.executeStatistic(stuNormScore);
							if (num == 0) {
								log.info("MQ 消費者處理消息失敗");
								return ConsumeConcurrentlyStatus.RECONSUME_LATER;
							}
						}
					} catch (Exception e) {
						log.info("MQ 消費者 處理消息失敗");
						log.error(e.getMessage(), e);	
						return ConsumeConcurrentlyStatus.RECONSUME_LATER;
					}finally {
						if (index < msgs.size()) {
                            context.setAckIndex(index + 1);
                        }
					}
					log.info("MQ 消費者 處理消息成功");
					return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
				}
			});
			
			consumer.start();
		} catch (Exception e) {
			log.error("MQ：啓動消費者失敗：{}-{}");	
			throw new RuntimeException(e.getMessage(), e);
		}
	}
	
	@Override
	public void run(String... arg0) throws Exception {
		this.messageListener();
	}

}

消費者類型2：PullConsumer

package com.becom.qoe.asyncproccess.rocketmq;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;

import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;

import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;
import org.apache.rocketmq.client.consumer.PullResult;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.message.MessageQueue;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import com.becom.qoe.asyncproccess.entity.StuNormScore;
import com.becom.qoe.asyncproccess.service.MainExecuteStatistic;

import lombok.extern.slf4j.Slf4j;
import net.sf.json.JSONObject;
@Slf4j
@Component
public class RocketMqConsumer{

	@Value("${apache.rocketmq.consumer.ConsumerGroup}")
	private String consumerGroupName;
	@Value("${apache.rocketmq.namesrvAddr}")
	private String namesrvAddr;
	@Value("${apache.rocketmq.producer.topic}")
	private String topic;
	@Value("${apache.rocketmq.producer.tag}")
	private String tag;
	@Autowired
	private MainExecuteStatistic mainExecuteStatistic;
	
	private static final Map<MessageQueue, Long> OFFSE_TABLE = new HashMap<MessageQueue, Long>();
	private DefaultMQPullConsumer pullConsumer = null;
	private static AtomicBoolean RUNNING = new AtomicBoolean(true);
	
	@PostConstruct
	public void messageListener() {
		log.info("MQ：啓動消費者");
		pullConsumer = new DefaultMQPullConsumer(consumerGroupName);
		pullConsumer.setNamesrvAddr(namesrvAddr);
		pullConsumer.setVipChannelEnabled(false);
		pullConsumer.setConsumerPullTimeoutMillis(1000 * 10);  //超時時間
		pullConsumer.setBrokerSuspendMaxTimeMillis(1000 * 10);
		pullConsumer.setConsumerTimeoutMillisWhenSuspend(1000 * 15);
		try {
			pullConsumer.start();
			//拉取topic下的所有消息隊列
			Set<MessageQueue> mqs = pullConsumer.fetchSubscribeMessageQueues(topic);
			for (MessageQueue mq : mqs) {
				Long endTime = null;
				Long startTime = null;
				while (RUNNING.get()) {
					if (null != endTime && null != startTime) {
						log.info("MQ 消費者 開始拉取消息,與上次拉取時間間隔爲：" + (endTime - startTime));
					}
					startTime = System.currentTimeMillis();
					try {
						//設置上次消費消息下標
						PullResult pullResult = pullConsumer.pullBlockIfNotFound(mq, null, getMessageQueueOffset(mq), 10);
						putMessageQueueOffset(mq, pullResult.getNextBeginOffset());
						switch (pullResult.getPullStatus()) {
							//根據結果狀態，如果找到消息，批量消費消息
							case FOUND:
								List<MessageExt> messageExtList = pullResult.getMsgFoundList();
								log.info("MQ 消費者 拉取" + messageExtList.size() + "條消息");
								for (MessageExt m : messageExtList) {
									String messageBody = new String(m.getBody());
									log.info("MQ 消費者 成功獲取消息" + messageBody);
									StuNormScore stuNormScore = (StuNormScore) JSONObject.toBean(JSONObject.fromObject(messageBody), StuNormScore.class);
									Integer num = mainExecuteStatistic.executeStatistic(stuNormScore);
									if (num > 0) {
										log.info("MQ 消費者 處理消息成功");
									}
	                            }
								break;
							case NO_MATCHED_MSG:
								log.info("MQ 消費者 沒有拉取到匹配的消息");
								break;
							case NO_NEW_MSG:
								endTime = System.currentTimeMillis();
								log.info("MQ 消費者 沒有拉取到消息");
								break;
							case OFFSET_ILLEGAL:
								log.info("MQ Consumer offset,may be too big or too small");
								break;
							default:
								break;
						}
					} catch (Exception e) {
						e.printStackTrace();
					} 
				}
			}
		} catch (MQClientException e) {
			e.printStackTrace();
		}
	}
	//獲取上次消費的消息的下表
    private static long getMessageQueueOffset(MessageQueue mq) {
        Long offset = (Long) OFFSE_TABLE.get(mq);
        if (offset != null) {
            return offset;
        }
        return 0;
    }
	//保存上次消費的消息下標，這裏使用了一個全局HashMap來保存
    private static void putMessageQueueOffset(MessageQueue mq, long offset) {
    	OFFSE_TABLE.put(mq, offset);
    }
	
    @PreDestroy
    public void shutDownConsumer() {
    	RUNNING.set(false);
    	if (pullConsumer != null) {
            pullConsumer.shutdown();
            log.info("MQ Consumer shutDown");
        }
    }
}

文章目錄

1. rocketmq 生產者

2. rocketmq 消費者

RocketMq實用

MySql中的存儲過程

EXT結合POI導出數據到Excel表格

highchars項目中的實戰，與ajax結合接收後臺傳過來的數據

關於SpringMVC中找不到標籤的解決辦法

Mac下配置sublime實現LaTeX

https://yachay.unat.edu.pe/blog/index.php?comment_area=format_blog&comment_component=blog&comment_co

linux以太網驅動總結