Kafka API調用示例

1、調用代碼示例

• NewTopic

• alterReplicaLogDirs

• KafkaProducer

package com.lin.demo.producer;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.kafka.clients.CommonClientConfigs;
import org.apache.kafka.clients.admin.*;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.TopicPartitionReplica;
import org.apache.kafka.common.errors.ProducerFencedException;
import org.apache.kafka.clients.producer.KafkaProducer;

import java.util.*;
import java.util.concurrent.ExecutionException;

public class KafkaProducerTest {
    private static Log logger = LogFactory.getLog(KafkaProducerTest.class);
    private static final String broker = "yourIP:9092";   //配置的是kafka broker的ip
    public static Producer<String, String> producer;
    public static AdminClient client;

    //用於開始的屬性初始化
    public static void beforeClass() {
        Properties properties = new Properties();
        properties.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, broker);
        client = AdminClient.create(properties);
    }
    //用於關閉client
    public  static  void afterClass(){
        client.close();
    }

    //調用NewTopic  API的demo
    public static void createTopics(String topic) {
        try {
            NewTopic newTopic = new NewTopic(topic, 3, (short) 1);
            Collection<NewTopic> newTopics = new ArrayList<>();
            newTopics.add(newTopic);
            System.out.println(newTopics);
            // 創建主題
            CreateTopicsResult result=client.createTopics(newTopics);
            result.all().get();
            System.out.println("Topic創建成功！");
        } catch (InterruptedException e) {
            System.out.println("創建失敗：" + e.getMessage());
            e.printStackTrace();
        } catch (ExecutionException e) {
            e.printStackTrace();
        }
    }

    //使用kafka的alterReplicaLogDirs修改replica的dir
    public  static  void  alterReplicaLogDir(String topic){
        try {
            TopicPartitionReplica topicPartitionReplica=new TopicPartitionReplica(topic,3,0);  //第二個參數是你想要移動的partition編號（編號從0開始），第三個參數是kafka集羣上對應的broker 編號，此處是broker 0
            Map<TopicPartitionReplica, String> replicaAssignment=new HashMap<TopicPartitionReplica, String>();
            replicaAssignment.put(topicPartitionReplica,"/home/yhl/test/kafkalogs/ssd");  //此處的file path必須是kafka集羣 broker 0 上server.properties 中 log.dirs配置中的一個
            System.out.println("開始移動log文件");
            AlterReplicaLogDirsResult  result=client.alterReplicaLogDirs(replicaAssignment);
            result.all().get();
            System.out.println("Topic移動文件夾成功！");
        }catch (InterruptedException e) {
            System.out.println("移動文件失敗：" + e.getMessage());
            e.printStackTrace();
        } catch (ExecutionException e) {
            e.printStackTrace();
        }
    }


    //producer初始化
    static synchronized void initProducer() {
        if (producer != null) {
            producer.close();
            producer = null;
        }
        Properties props = new Properties();
        props.put("bootstrap.servers", broker);
        props.put("acks", "all");
        props.put("batch.size", 16384);
        props.put("linger.ms", 1);
        props.put("buffer.memory", 33554432);
        props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        //定義生產者對象，該類指定了兩個參數的泛型，第一個參數表示分區鍵值的類型，第二參數表示消息類型
        producer = new KafkaProducer<String, String>(props);
    }

    //批量發送數據
    public void testBachProfucuer(String topic) {
        try {
            for (int i = 0; i < 10000; i++) {
                producer.send(new ProducerRecord<String, String>(topic, Integer.toString(i), Integer.toString(i)));
            }
        } catch (ProducerFencedException e) {
            producer.close();
        }
        producer.close();
    }

    public static void main(String[] args) {
        beforeClass();   //必須初始化properties，否則不能正常創建topic，會提示空指針異常
        createTopics("test");
        initProducer();
        System.out.println("開始發送"+System.currentTimeMillis());
        new KafkaProducerTest().testBachProfucuer("test");
        System.out.println("發送完成"+System.currentTimeMillis());
        alterReplicaLogDir("test");
        afterClass();
    }
}

2、錯誤解決

1、 提示出現如下錯誤

java.util.concurrent.ExecutionException: org.apache.kafka.common.errors.UnknownTopicOrPartitionException: This server does not host this topic-partition.
	at org.apache.kafka.common.internals.KafkaFutureImpl.wrapAndThrow(KafkaFutureImpl.java:45)
	at org.apache.kafka.common.internals.KafkaFutureImpl.access$000(KafkaFutureImpl.java:32)
	at org.apache.kafka.common.internals.KafkaFutureImpl$SingleWaiter.await(KafkaFutureImpl.java:89)
	at org.apache.kafka.common.internals.KafkaFutureImpl.get(KafkaFutureImpl.java:260)
	at com.lin.demo.producer.KafkaProducerTest.alterReplicaLogDir(KafkaProducerTest.java:84)
	at com.lin.demo.producer.KafkaProducerTest.main(KafkaProducerTest.java:137)
Caused by: org.apache.kafka.common.errors.UnknownTopicOrPartitionException: This server does not host this topic-partition.

原因分析：調用以下代碼時，第二個參數指定錯誤，超出了partition的編號範圍
TopicPartitionReplica topicPartitionReplica=new TopicPartitionReplica(topic,3,0);
因爲第二個參數指定了partition的編號是3，但是我們創建topic test 的時候指定了partition參數是3，但是編號是從0開始的，即磁盤上存在test-0，test-1，test-2 ，但是不存在test-3，所以是參數指定錯誤導致了上面報錯。

2、提示出現以下錯誤

Exception in thread "main" java.lang.NullPointerException
	at com.lin.demo.producer.KafkaProducerTest.createTopics(KafkaProducerTest.java:62)
	at com.lin.demo.producer.KafkaProducerTest.main(KafkaProducerTest.java:131)

原因分析：沒有進行AdminClient的初始化，即沒有在創建topic的之前調用以下方法

 beforeClass();