序
最近項目改造,原先是使用的rabbitMq。此次改造,我們重新調研整體流程,爲結合原已有服務,所以使用的Kafka。
我的集成思路簡單清晰,不過還有提升的地方,直接上菜。
一、消費者配置
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
import org.springframework.kafka.config.KafkaListenerContainerFactory;
import org.springframework.kafka.core.ConsumerFactory;
import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
import org.springframework.kafka.listener.ConcurrentMessageListenerContainer;
import org.springframework.kafka.listener.ContainerProperties;
import java.util.HashMap;
import java.util.Map;
/**
* kafka消費者配置
* @author zhengwen
*/
@EnableKafka
@Configuration
public class KafkaConsumerConfig {
/**
* 服務地址
*/
@Value("${spring.kafka.bootstrap-servers}")
private String servers;
/**
* 分組id
*/
@Value("${spring.kafka.consumer.group-id}")
private String groupId;
/**
* 自動提交
*/
@Value("${spring.kafka.consumer.enable-auto-commit}")
private boolean enableAutoCommit;
/**
* 自動提交的間隔時間,默認值是 5000,單位是毫秒
*/
@Value("${spring.kafka.consumer.auto-commit-interval}")
private String autoCommitInterval;
/**
* 批量消費一次最大拉取的數據量
*/
@Value("${spring.kafka.consumer.max-poll-records}")
private int maxPollRecordsConfig;
/**
* 消費者配置
* @return 配置map
*/
public Map<String, Object> consumerConfigs() {
Map<String, Object> propsMap = new HashMap<>(10);
propsMap.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, servers);
propsMap.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, enableAutoCommit);
propsMap.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, autoCommitInterval);
propsMap.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
propsMap.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
// 組名
propsMap.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
propsMap.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecordsConfig);
return propsMap;
}
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
factory.setConsumerFactory(consumerFactory());
factory.setConcurrency(10);
factory.getContainerProperties().setPollTimeout(1500);
factory.setBatchListener(true);
factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE);
return factory;
}
public ConsumerFactory<String, String> consumerFactory() {
return new DefaultKafkaConsumerFactory<>(consumerConfigs());
}
}
二、生產者配置
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.StringSerializer;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.core.DefaultKafkaProducerFactory;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.core.ProducerFactory;
import java.util.HashMap;
import java.util.Map;
/**
* kafka生產者配置
* @author zhengwen
*/
@EnableKafka
@Configuration
public class KafkaProducerConfig {
/**
* 服務地址
*/
@Value("${spring.kafka.bootstrap-servers}")
private String servers;
/**
* 重試失敗的發送次數
*/
@Value("${spring.kafka.producer.retries}")
private int retries;
/**
* 批量數據大小
*/
@Value("${spring.kafka.producer.batch-size}")
private int batchSize;
/**
* 緩衝內存大小
*/
@Value("${spring.kafka.producer.buffer-memory}")
private int bufferMemory;
/**
* 生產者配置
* @return 配置map
*/
public Map<String, Object> producerConfigs() {
Map<String, Object> props = new HashMap<>();
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, servers);
props.put(ProducerConfig.RETRIES_CONFIG, retries);
props.put(ProducerConfig.BATCH_SIZE_CONFIG, batchSize);
props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, bufferMemory);
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
return props;
}
@Bean
public KafkaTemplate<String, String> kafkaTemplate() {
return new KafkaTemplate(producerFactory());
}
public ProducerFactory<String, String> producerFactory() {
return new DefaultKafkaProducerFactory<>(producerConfigs());
}
}
三、多線程配置
import lombok.extern.slf4j.Slf4j;
import org.springframework.aop.interceptor.AsyncUncaughtExceptionHandler;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.AsyncConfigurer;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import java.lang.reflect.Method;
import java.util.concurrent.Executor;
/**
* @author zhengwen
*/
@Configuration
@EnableAsync
@Slf4j
public class ThreadConfig implements AsyncConfigurer {
/**
* 核心線程樹
*/
@Value("${thread.config.corePoolSize:5}")
private Integer corePoolSize;
/**
* 最大線程池數量
*/
@Value("${thread.config.maxPoolSize:10}")
private Integer maxPoolSize;
/**
* 隊列長度
*/
@Value("${thread.config.queueCapacity:10}")
private Integer queueCapacity;
@Override
public Executor getAsyncExecutor() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
log.info("---------[多線程配置初始化],最大核心線程數:{},最大線程池數量:{},線程處理隊列長度:{}", corePoolSize, maxPoolSize, queueCapacity);
//核心線程數
executor.setCorePoolSize(corePoolSize);
//最大線程池數量
executor.setMaxPoolSize(maxPoolSize);
//線程處理隊列長度
executor.setQueueCapacity(queueCapacity);
executor.initialize();
return executor;
}
@Override
public AsyncUncaughtExceptionHandler getAsyncUncaughtExceptionHandler() {
log.info("-----------多線程異常handler------------");
return new SpringAsyncExceptionHandler();
}
class SpringAsyncExceptionHandler implements AsyncUncaughtExceptionHandler {
@Override
public void handleUncaughtException(Throwable ex, Method method, Object... params) {
log.error("Asyn返回異常:" + ex.getCause().getMessage() + method.getName());
}
}
}
四、監聽方式消費
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.kafka.support.Acknowledgment;
import org.springframework.stereotype.Component;
import java.util.List;
/**
* 消費者listener
*
* @author zhengwen
**/
@Slf4j
@Component
public class AnalyzeConsumer {
@Autowired
private AnalyzeService analyzeService;
/**
* 車輛抓拍
* @param records 消費信息
* @param ack Ack機制
*/
@KafkaListener(topics = "${fillersmart.analyze.car.topic.consumer}", containerFactory = "kafkaListenerContainerFactory")
public void carListen(List<ConsumerRecord> records, Acknowledgment ack) {
log.info("=====車輛carListen消費者接收信息====");
try {
for (ConsumerRecord record : records) {
log.info("---開啓線程解析車輛數據:{}",record.toString());
analyzeService.carAnalyze(record);
}
} catch (Exception e) {
e.printStackTrace();
log.error("----車輛消費者解析數據異常:{}",e.getMessage(),e);
} finally {
//手動提交偏移量
ack.acknowledge();
}
}
}
四、生產者
我這裏是消費信息做了處理以後再放回去到另一個topic。如果不需要生產的,可以不要了。
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.stereotype.Component;
import org.springframework.util.concurrent.FailureCallback;
import org.springframework.util.concurrent.ListenableFuture;
import org.springframework.util.concurrent.SuccessCallback;
/**
* 解析生產者
* @author zhengwen
**/
@Slf4j
@Component
public class AnalyzeProducer {
@Autowired
private KafkaTemplate kafkaTemplate;
/**
* 發送數據到kafka
* @param topic topic名稱
* @param msg 發送信息字符串
* @param sendCallBack 發送回調
*/
public void send(String topic, String msg, SendCallBack sendCallBack) {
ListenableFuture listenableFuture = kafkaTemplate.send(topic,msg);
//發送成功後回調
SuccessCallback<String> successCallback = new SuccessCallback() {
@Override
public void onSuccess(Object result) {
sendCallBack.sendSuccessCallBack(topic,msg);
}
};
//發送失敗回調
FailureCallback failureCallback = new FailureCallback() {
@Override
public void onFailure(Throwable ex) {
sendCallBack.sendFailCallBack(topic,msg,ex);
}
};
listenableFuture.addCallback(successCallback,failureCallback);
}
}
五、回調方法函數
定義了回調接口,發送的時候傳入。
/**
* @author zhengwen
**/
public interface SendCallBack {
/**
* 生產成功回調
* @param topic topic
* @param msg 信息字符串
*/
void sendSuccessCallBack(String topic,String msg);
/**
* 生產失敗回調
* @param topic topic
* @param msg 信息字符串
* @param ex 異常
*/
void sendFailCallBack(String topic,String msg,Throwable ex);
}
六、使用方法
就這上面幾步步就ok了,AnalyzeService就是普通的service,注意它的實現類的方法要體現多線程,需要再實現方法上加 @Async標籤,否則無效。多線程配置之前其實我也分享過,我是個怕麻煩的人,都是喜歡簡潔的集成方法。實現方法裏就可以放飛自我了,這裏就看看實現方法怎麼調用的。
//解析完成的數據發送到kafka
String msg = JSON.toJSONString(collectDataDto);
analyzeProducer.send(carProducerTopic,msg, new SendCallBack() {
@Override
public void sendSuccessCallBack(String topic, String msg) {
log.info("----車輛抓拍kafka記錄解析完成放入topic:{},發送成功:{}",topic, msg);
}
@Override
public void sendFailCallBack(String topic, String msg, Throwable ex) {
log.error("----車輛抓拍kafka記錄解析完成放入topic:{},發送失敗{}",topic,msg,ex);
}
});
實現方法裏處理完成後生產到kafka就是這樣優雅的調用,回調函數裏我這裏只打印了topic,沒有做其他處理,有需要的就再這裏放飛自我。
七、關鍵配置文件與引包
我是springBoot2.3.0 + springCloud:Hoxton.SR4
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
</dependency>
//配置文件
spring:
kafka:
bootstrap-servers: 127.0.0.1:9092
consumer:
auto-commit-interval: 1000
enable-auto-commit: false
group-id: test-consumer-group
max-poll-records: 10
producer:
batch-size: 4096
buffer-memory: 40960
retries: 1
其他配置文件都就隨意了。
使用就是這麼優雅,我這個場景同事說更適合使用kafkaStream,下次分享。