記錄一下怎麼保證MQ消費消息去重，消息重試

先說 背景，有消息生產，有很多SQL表名稱，對應去統計不同表的數據，更新數量，但是這些消息會重複，可能有很多邏輯都要重複執行，可能會速度慢

生產：

這是SQL解析，重要的是這段 ，

tableName是枚舉裏面固定的，圖片中有顯示

  RabbitMQSender.sendMessage(MQConfig.FIRST_PAGE_SQL_ROUTINGKEY, tableName, MessageType.COMMON, uuid);

///僞代碼 　　　　
　　StatementHandler statementHandler = PluginUtils.realTarget(invocation.getTarget());
            MetaObject metaObject = SystemMetaObject.forObject(statementHandler);
            MappedStatement mappedStatement = (MappedStatement)metaObject.getValue("delegate.mappedStatement");
            if(!SqlCommandType.SELECT.equals(mappedStatement.getSqlCommandType())){
                Object proceed = invocation.proceed();
                firstPageSqlParse(metaObject);
                return proceed;
　　　　　　}

private void firstPageSqlParse(MetaObject metaObject) {
    BoundSql boundSql = (BoundSql) metaObject.getValue("delegate.boundSql");
    String sql = boundSql.getSql();
    CompletableFuture.runAsync(() -> {
        // 語句提取表名：
        String lowSql = sql.toLowerCase();
        String tableName = extractTableName(lowSql);
        FirstPageCountEnum[] values = FirstPageCountEnum.values();
        Set<String> collect = Arrays.stream(values).map(FirstPageCountEnum::getTableName).collect(Collectors.toSet());
        if (collect.contains(tableName)) {
            String uuid = IdUtil.randomUUID();
            RabbitMQSender.sendMessage(MQConfig.FIRST_PAGE_SQL_ROUTINGKEY, tableName, MessageType.COMMON, uuid);
        }
    });
}

private static String extractTableName(String sql) {
    String regex = "(insert\\s+into\\s+|update\\s+|delete\\s+from\\s+)(\\w+)";
    Pattern pattern = Pattern.compile(regex);
    Matcher matcher = pattern.matcher(sql);
    if (matcher.find()) {
        String tableName = matcher.group(2);
        System.out.println(tableName);
        return tableName;
    }
    return "";
}

 

 

消費端：

這就是普通接受部分

 /**
     * 監聽單個隊列
     * concurrency:併發處理消息數
     */
    @RabbitListener(queues = MQConfig.FIRST_PAGE_SQL)
    @RabbitHandler
    public void notificationQueueReceiver(Message message, Channel channel) throws IOException {
        messageHandler(message, channel);
    }

    private void messageHandler(Message message, Channel channel) throws IOException {
        long deliveryTag = message.getMessageProperties().getDeliveryTag();
        Action action = Action.ACCEPT;
        try {
            MessageBody messageBody = MessageBody.getMessageBody(message);
          
            String sql = MessageBody.getMessageBody(message).getData().toString();
           　// 鎖，　
            Object o = CACHE_MQ_MAP_LOCK.get(sql);
            if (o != null) {
                if (CACHE_MQ_MAP.getOrDefault(sql, 0) == 0) {
                    synchronized (o) {
                        if (CACHE_MQ_MAP.getOrDefault(sql, 0) == 0) {
                            CACHE_MQ_MAP.put(sql, 1);
                        }
                        o.notifyAll();
                    }
                }
            }
        } catch (Exception e) {
            log.error("MQ處理消息出錯", e);
        } finally {
            // 通過 finally 塊來保證 Ack/Nack 會且只會執行一次
            if (action == Action.ACCEPT) {
                // false 只確認當前 consumer 一個消息收到，true 確認所有 consumer 獲得的消息。
                channel.basicAck(deliveryTag, false);
            } else {
                // 第二個 boolean 爲 false 表示不會重試，爲 true 會重新放回隊列
                channel.basicReject(deliveryTag, false);
            }
        }
    }

初始化兩個MAP 鎖，以及去除重複的數據Map

 @PostConstruct
    public void init() {
        CompletableFuture.runAsync(() -> {
            for (FirstPageCountEnum value : FirstPageCountEnum.values()) {
                CACHE_MQ_MAP_LOCK.put(value.getTableName(), new Object());
                try {
//　　爲了初始化可以數據更新
                    String uuid = IdUtil.randomUUID();
                    RabbitMQSender.sendMessage(MQConfig.FIRST_PAGE_SQL_ROUTINGKEY, value.getTableName(), MessageType.COMMON, uuid);
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
            // 創建消費者線程，處理邏輯在下面，這裏有點問題，創建的線程多了，因爲表名稱重複了。
            for (FirstPageCountEnum value : FirstPageCountEnum.values()) {
                Thread consumerThread = new Thread(new Consumer(CACHE_MQ_MAP, value.getTableName(), userFirstPageCountService));
                consumerThread.start();
            }
        });
    }
　　// 保存消息的去重複的MAP
    private final ConcurrentHashMap<String, Integer> CACHE_MQ_MAP = new ConcurrentHashMap<>(32);
　　// 保存鎖的
    private static final ConcurrentHashMap<String, Object> CACHE_MQ_MAP_LOCK = new ConcurrentHashMap<>(32);

 

創建的消費線程處理的邏輯

 static class Consumer implements Runnable {
        private final ConcurrentHashMap<String, Integer> dataMap;
        private final String tableName;
        private final UserFirstPageCountService userFirstPageCountService;

        public Consumer(ConcurrentHashMap<String, Integer> dataMap, String tableName, UserFirstPageCountService userFirstPageCountService) {
            this.userFirstPageCountService = userFirstPageCountService;
            this.tableName = tableName;
            this.dataMap = dataMap;
        }

        @Override
        public void run() {
            // 消費數據
            while (true) {
                for (String key : dataMap.keySet()) {
                    if (!key.equals(tableName)) {
                        continue;
                    }

                    Object lock = CACHE_MQ_MAP_LOCK.get(key);
                    // 獲取數據
                    try {
                        Thread.sleep(2000);
                        int value = dataMap.getOrDefault(key, 0);
                        log.info("Consumed: {},  value:{}", key, value);
                        if (value == 1) {
                            dataMap.put(key, 0);
　　　　　　　　　　　　　　　　//這裏業務處理邏輯，裏面加了重試。
                            userFirstPageCountService.sqlParse(tableName);
                        }
                        synchronized (lock) {
                            if (dataMap.getOrDefault(key, 0) == 0) {
                                // 設置對應的key爲0
                                log.info("wait {},{}", key, 0);
                                lock.wait();
                                log.info("notify {},{}", key, 0);
                            }
                        }
                    } catch (Exception e) {
                        log.error("Consumed error ", e);
                    }

                }
            }
        }
    }

 

這是重試，初始化邏輯可以保證不丟失。 以及這裏出錯後調用不成功可以不斷重試