Kafka監控——獲取Partition的LogSize、Lag、BrokerId

一、引入kafka Pom依賴

		<dependency>
			  <groupId>org.apache.kafka</groupId>
			  <artifactId>kafka_2.10</artifactId>
			  <version>0.8.1.1</version>
		</dependency>

二、LogSize、BrokerId獲取實現類

package com.xgd.log.common;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import kafka.api.OffsetRequest;
import kafka.api.PartitionOffsetRequestInfo;
import kafka.cluster.Broker;
import kafka.common.TopicAndPartition;
import kafka.javaapi.OffsetResponse;
import kafka.javaapi.PartitionMetadata;
import kafka.javaapi.TopicMetadata;
import kafka.javaapi.TopicMetadataRequest;
import kafka.javaapi.TopicMetadataResponse;
import kafka.javaapi.consumer.SimpleConsumer;

public class KafkaHelper {

	/**
	 * 獲取kafka logSize
	 * @param host
	 * @param port
	 * @param topic
	 * @param partition
	 * @return
	 */
	public static long getLogSize(String host,int port,String topic,int partition){
		String clientName = "Client_" + topic + "_" + partition;
		Broker leaderBroker = getLeaderBroker(host, port, topic, partition);
		String reaHost = null;
		if (leaderBroker != null) {
			reaHost = leaderBroker.host();
		}else {
			System.out.println("Partition of Host is not find");
		}
		SimpleConsumer simpleConsumer = new SimpleConsumer(reaHost, port, 10000, 64*1024, clientName);
		TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
		Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
		requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(OffsetRequest.LatestTime(), 1));
		kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo, OffsetRequest.CurrentVersion(), clientName);
		OffsetResponse response = simpleConsumer.getOffsetsBefore(request);
		if (response.hasError()) {
			System.out.println("Error fetching data Offset , Reason: " + response.errorCode(topic, partition) );
			return 0;
		}
		long[] offsets = response.offsets(topic, partition);
		return offsets[0];
	}
	
	/**
	 * 獲取broker ID
	 * @param host
	 * @param port
	 * @param topic
	 * @param partition
	 * @return
	 */
	public static Integer getBrokerId(String host,int port,String topic,int partition){
		Broker leaderBroker = getLeaderBroker(host, port, topic, partition);
		if (leaderBroker != null) {
			return leaderBroker.id();
		}
		return null;
	}
	/**
	 * 獲取leaderBroker
	 * @param host
	 * @param port
	 * @param topic
	 * @param partition
	 * @return
	 */
	private static Broker getLeaderBroker(String host,int port,String topic,int partition){
		String clientName = "Client_Leader_LookUp";
		SimpleConsumer consumer = null;
		PartitionMetadata partitionMetaData = null;
		try {
			consumer = new SimpleConsumer(host, port, 10000, 64*1024, clientName);
			List<String> topics = new ArrayList<String>();
			topics.add(topic);
			TopicMetadataRequest request = new TopicMetadataRequest(topics);
			TopicMetadataResponse reponse = consumer.send(request);
			List<TopicMetadata> topicMetadataList = reponse.topicsMetadata();
			for(TopicMetadata topicMetadata : topicMetadataList){
				for(PartitionMetadata metadata : topicMetadata.partitionsMetadata()){
					if (metadata.partitionId() == partition) {
						partitionMetaData = metadata;
						break;
					}
				}
			}
			if (partitionMetaData != null) {
				return partitionMetaData.leader();
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
		return null;
	}

}

得到LogSize後，Lag=LogSize - Offset，其中Offset值可以從Zookeeper中獲取。