啓動：
./bin/kafka-server-start.sh -daemon config/server.properties
創建1replica 1 partition 的topic
bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test
查看詳情
bin/kafka-topics.sh --describe --zookeeper localhost:2181 --topic test
添加分區
添加新的kafka實例，broker.id=1，然後執行如下命令，修改test的partition數爲2
bin/kafka-topics.sh --alter --zookeeper localhost:2181 --partitions 2 --topic test
添加新的kafka實例後，新建的topic的partition會自動分配到新的broker
[root@kudu001 kafka]# bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 2 --partitions 2 --topic test2
[root@kudu001 kafka]# bin/kafka-topics.sh --describe --zookeeper localhost:2181 --topic test2
Topic:test2 PartitionCount:2 ReplicationFactor:2 Configs:
Topic: test2 Partition: 0 Leader: 1 Replicas: 1,0 Isr: 1,0
Topic: test2 Partition: 1 Leader: 0 Replicas: 0,1 Isr: 0,1
分區遷移,分區重新劃分
使用命令kafka-reassign-partitions.sh，命令接受broker-list參數，表示數據要遷移到哪些broker；
將topic test的第1個分區移動到broker.id=1的那個節點；
第一步：創建test配置文件test.json
{"topics":[{"topic":"test"}],"version": 1}
第二步：生成test的新的邏輯圖
[root@kudu001 kafka]# bin/kafka-reassign-partitions.sh --zookeeper localhost:2181 --topics-to-move-json-file test.json --broker-list "0,1" --generate
Current partition replica assignment
#當前partition 副本配置
{"version":1,"partitions":[{"topic":"test","partition":1,"replicas":[0]},{"topic":"test","partition":0,"replicas":[0]}]}
Proposed partition reassignment configuration
#建議的partition 配置
{"version":1,"partitions":[{"topic":"test","partition":1,"replicas":[1]},{"topic":"test","partition":0,"replicas":[0]}]}
第三步：生成test 新的邏輯結構描述文件，test-reassignment.json
{
"version":1,
"partitions":[
{
"topic":"test",
"partition":1,
"replicas":[
1
]
},
{
"topic":"test",
"partition":0,
"replicas":[
0
]
}
]
}
第四步：重新分區
bin/kafka-reassign-partitions.sh --zookeeper localhost:2181 --reassignment-json-file test-reassignment.json --execute
查看重新分區進度
[root@kudu001 kafka]# bin/kafka-reassign-partitions.sh --zookeeper localhost:2181 --reassignment-json-file test-reassignment.json --verify
Status of partition reassignment:
Reassignment of partition [test,1] completed successfully
Reassignment of partition [test,0] completed successfully
test 詳情
[root@kudu001 kafka]# bin/kafka-topics.sh --describe --zookeeper localhost:2181 --topic test
Topic:test PartitionCount:2 ReplicationFactor:1 Configs:
Topic: test Partition: 0 Leader: 0 Replicas: 0 Isr: 0
Topic: test Partition: 1 Leader: 1 Replicas: 1 Isr: 1
添加副本
給test添加1個副本,新建test-addreplica.json
{
"version":1,
"partitions":[
{
"topic":"test",
"partition":0,
"replicas":[
0,
1
]
},
{
"topic":"test",
"partition":1,
"replicas":[
1,
0
]
}
]
}
執行添加副本命令
bin/kafka-reassign-partitions.sh --zookeeper localhost:2181 --reassignment-json-file test-addreplica.json --execute
查看test詳情
[root@kudu001 kafka]# bin/kafka-topics.sh --describe --zookeeper localhost:2181 --topic test
Topic:test PartitionCount:2 ReplicationFactor:2 Configs:
Topic: test Partition: 0 Leader: 0 Replicas: 0,1 Isr: 0,1
Topic: test Partition: 1 Leader: 1 Replicas: 1,0 Isr: 1,0
topic partition leader 負載均衡
kafka集羣運行一段時間後，由於一些節點的啓停，topic的parition leader不在均勻分佈在各個broker上，這將影響整個集羣的讀寫性能;解決辦法有兩種
1.配置文件，默認是開啓的
auto.leader.rebalance.enable=true
leader.imbalance.per.broker.percentage =10
#leader的不平衡比例，若是超過這個數值，會對分區進行重新的平衡
leader.imbalance.check.interval.seconds =300
#檢查leader是否不平衡的時間間隔
2.使用命令行的方式手動均衡
執行leader均衡使用的命令是
kafka-preferred-replica-election.sh --zookeeper localhost:2181 --path-to-json-file test_maxwell_1-rebalance.json
test_maxwell_1詳情
[root@hzv_stat_es1 ~]# kafka-topics.sh --describe --zookeeper localhost:2181 --topic test_maxwell_1
Topic:test_maxwell_1 PartitionCount:6 ReplicationFactor:3 Configs:retention.ms=172800000,cleanup.policy=delete
Topic: test_maxwell_1 Partition: 0 Leader: 1 Replicas: 1,2,3 Isr: 1,2,3
Topic: test_maxwell_1 Partition: 1 Leader: 1 Replicas: 1,2,3 Isr: 1,2,3
Topic: test_maxwell_1 Partition: 2 Leader: 1 Replicas: 1,2,3 Isr: 1,2,3
Topic: test_maxwell_1 Partition: 3 Leader: 1 Replicas: 1,2,3 Isr: 1,2,3
Topic: test_maxwell_1 Partition: 4 Leader: 1 Replicas: 1,2,3 Isr: 1,2,3
Topic: test_maxwell_1 Partition: 5 Leader: 1 Replicas: 1,2,3 Isr: 1,2,3
但在執行rebalance時，每個分區默認會使用Replicas中的第一個broker作爲leader，例如上面的topic：test_maxwell_1，每個分區都是Replicas: 1,2,3，每個partitiion的所有replicas叫做"assigned replicas"，“assigned replicas"中的第一個replicas叫"preferred replica”，剛創建的topic一般"preferred replica"是leader；所以即便執行kafka-preferred-replica-election.sh命令，leader也沒辦法重新分配，那就要重新分配每個partition的replica，其實就是重新分配一次，調整順序（注意：如果preferred replica不相同，直接跳到步驟5）
第一步：
編寫topic文件：test_maxwell_1.json
{"topics":[{"topic":"test_maxwell_1"}],"version": 1}
第二步，生成topic 邏輯圖
[root@hzv_stat_es1 ~]# kafka-reassign-partitions.sh --zookeeper localhost:2181 --topics-to-move-json-file test_maxwell_1.json --broker-list "1,2,3" --generate
Current partition replica assignment
{"version":1,"partitions":[{"topic":"test_maxwell_1","partition":3,"replicas":[1,2,3]},{"topic":"test_maxwell_1","partition":5,"replicas":[1,2,3]},{"topic":"test_maxwell_1","partition":2,"replicas":[1,2,3]},{"topic":"test_maxwell_1","partition":0,"replicas":[1,2,3]},{"topic":"test_maxwell_1","partition":4,"replicas":[1,2,3]},{"topic":"test_maxwell_1","partition":1,"replicas":[1,2,3]}]}
Proposed partition reassignment configuration
{"version":1,"partitions":[{"topic":"test_maxwell_1","partition":5,"replicas":[1,3,2]},{"topic":"test_maxwell_1","partition":3,"replicas":[2,1,3]},{"topic":"test_maxwell_1","partition":2,"replicas":[1,2,3]},{"topic":"test_maxwell_1","partition":0,"replicas":[2,3,1]},{"topic":"test_maxwell_1","partition":4,"replicas":[3,2,1]},{"topic":"test_maxwell_1","partition":1,"replicas":[3,1,2]}]}
第三步：創建文件test_maxwell_1-reassignment.json
{
"version":1,
"partitions":[
{
"topic":"test_maxwell_1",
"partition":5,
"replicas":[
1,
3,
2
]
},
{
"topic":"test_maxwell_1",
"partition":3,
"replicas":[
2,
1,
3
]
},
{
"topic":"test_maxwell_1",
"partition":2,
"replicas":[
1,
2,
3
]
},
{
"topic":"test_maxwell_1",
"partition":0,
"replicas":[
2,
3,
1
]
},
{
"topic":"test_maxwell_1",
"partition":4,
"replicas":[
3,
2,
1
]
},
{
"topic":"test_maxwell_1",
"partition":1,
"replicas":[
3,
1,
2
]
}
]
}
第四步：執行
kafka-reassign-partitions.sh --zookeeper localhost:2181 \
--reassignment-json-file test_maxwell_1-reassignment.json --execute
查看topic 詳情
[root@hzv_stat_es1 ~]# kafka-topics.sh --describe --zookeeper localhost:2181 --topic test_maxwell_1
Topic:test_maxwell_1 PartitionCount:6 ReplicationFactor:3 Configs:retention.ms=172800000,cleanup.policy=delete
Topic: test_maxwell_1 Partition: 0 Leader: 1 Replicas: 2,3,1 Isr: 1,2,3
Topic: test_maxwell_1 Partition: 1 Leader: 1 Replicas: 3,1,2 Isr: 1,2,3
Topic: test_maxwell_1 Partition: 2 Leader: 1 Replicas: 1,2,3 Isr: 1,2,3
Topic: test_maxwell_1 Partition: 3 Leader: 1 Replicas: 2,1,3 Isr: 1,2,3
Topic: test_maxwell_1 Partition: 4 Leader: 1 Replicas: 3,2,1 Isr: 1,2,3
Topic: test_maxwell_1 Partition: 5 Leader: 1 Replicas: 1,3,2 Isr: 1,2,3
第五步：partition leader 均衡 test_maxwell_1-rebalance.json
{
"partitions":[
{
"topic":"test_maxwell_1",
"partition":0
},
{
"topic":"test_maxwell_1",
"partition":1
},
{
"topic":"test_maxwell_1",
"partition":2
},
{
"topic":"test_maxwell_1",
"partition":3
},
{
"topic":"test_maxwell_1",
"partition":4
},
{
"topic":"test_maxwell_1",
"partition":5
}
]
}
執行均衡
kafka-preferred-replica-election.sh --zookeeper localhost:2181 --path-to-json-file test_maxwell_1-rebalance.json
查看topic詳情
[root@hzv_stat_es1 ~]# kafka-topics.sh --describe --zookeeper localhost:2181 --topic test_maxwell_1
Topic:test_maxwell_1 PartitionCount:6 ReplicationFactor:3 Configs:retention.ms=172800000,cleanup.policy=delete
Topic: test_maxwell_1 Partition: 0 Leader: 1 Replicas: 1,2,3 Isr: 1,2,3
Topic: test_maxwell_1 Partition: 1 Leader: 2 Replicas: 2,3,1 Isr: 1,2,3
Topic: test_maxwell_1 Partition: 2 Leader: 3 Replicas: 3,1,2 Isr: 1,2,3
Topic: test_maxwell_1 Partition: 3 Leader: 1 Replicas: 1,3,2 Isr: 1,2,3
Topic: test_maxwell_1 Partition: 4 Leader: 2 Replicas: 2,1,3 Isr: 1,2,3
Topic: test_maxwell_1 Partition: 5 Leader: 3 Replicas: 3,2,1 Isr: 1,2,3
