kafka .net Core 分區(Partition)數據分發，實現 負載均衡功能

nuget  安裝  Confluent.Kafka 最新版

    /// <summary>
    /// 消費者
    /// </summary>
    public class KafkaConsumer
    {
        /// <summary>
        /// 消費者
        /// </summary>
        public static void Consumer(string Topic, string id, string GroupId, string BootstrapServers)
        {
            var conf = new ConsumerConfig
            {
                GroupId = GroupId,
                BootstrapServers = BootstrapServers,
                AutoOffsetReset = AutoOffsetReset.Earliest,
                EnableAutoCommit = false  // 設置非自動偏移，業務邏輯完成後手動處理偏移，防止數據丟失
            };

            using (var consumer = new ConsumerBuilder<string, string>(conf).Build())
            {
                // 訂閱topic
                consumer.Assign(new TopicPartition(Topic, new Partition()));
                consumer.Subscribe(Topic);
                while (true)
                {
                    try
                    {
                        var consume = consumer.Consume();
                        string receiveMsg = consume.Value;
                        string key = consume.Key;
                        Console.WriteLine($"消息隊列:{id} Consumed key '{key}' value {receiveMsg} at: '{consume.TopicPartitionOffset}'.");
                        consume = consumer.Consume();
                        receiveMsg = consume.Value;
                        key = consume.Key;
                        consumer.Commit(consume); //手動提交偏移
                    }
                    catch (Exception e)
                    {
                        Console.WriteLine(e.Message);
                    }
                }
            }
        }
    }

    /// <summary>
    /// 生產者
    /// </summary>
    public class KafkaProduce
    {
        /// <summary>
        /// 生產者
        /// </summary>
        /// <param name="Topic">主題</param>
        /// <param name="partitionNumber">分區索引  需要跟kafka配置的數量一致</param>
        /// <param name="key"></param>
        /// <param name="value"></param>
        /// <param name="BootstrapServers">kafka服務地址</param>
        public static void Produce(string Topic, int partitionNumber,string key, string value, string BootstrapServers)
        {
            var config = new ProducerConfig { BootstrapServers = BootstrapServers };

            using (var producer = new ProducerBuilder<string, string>(config).Build())
            {
                // 異步發送消息到主題
                var dr = producer.ProduceAsync(new TopicPartition(Topic, new Partition(partitionNumber)), new Message<string, string> { Value = value, Key = value }).GetAwaiter().GetResult();
                // 3秒後 Flush到磁盤
                producer.Flush(TimeSpan.FromSeconds(3));
            }
        }
    }

        static void Main(string[] args)
        {
            string topic = "22222";
            string ip = "192.168.1.14:9092";
            string groupid = "groupID1";
            int Split = 4;


            #region 單生產，單消費
            //生產10個
            for (int b = 0; b < 10; b++)
            {
                KafkaProduce.Produce(topic, 0, b.ToString(), b.ToString(), ip);
            }
            KafkaConsumer.Consumer(topic, 1.ToString(), groupid, ip);
            #endregion
            Console.ReadLine();
            #region 消費者 間隔5秒開始啓動消費
            //生產者生產
            Task.Run(() =>
            {
                for (int c = 0; c < 1000; c++)
                {
                    for (int i = 0; i < Split; i++)
                    {
                        for (int b = 0; b < 10; b++)
                        {
                            KafkaProduce.Produce(topic, i, b.ToString(), (b + i * 10).ToString(), ip);
                        }
                    }
                    Console.WriteLine("生產一組成功!");
                    Thread.Sleep(1 * 1000);
                }
            });
            //消費者開始消費
            for (int i = 0; i < Split; i++)
            {
                Console.WriteLine("啓動消費者:" + i.ToString());
                Task.Run(() =>
                {
                    KafkaConsumer.Consumer(topic, i.ToString(), groupid, ip);
                });
                Thread.Sleep(5 * 1000);
            }
            #endregion
            Console.ReadLine();
            #region 消費者 同時消費
            //生產者生產
            Task.Run(() =>
            {
                for (int c = 0; c < 1000; c++)
                {
                    for (int i = 0; i < Split; i++)
                    {
                        for (int b = 0; b < 10; b++)
                        {
                            KafkaProduce.Produce(topic, i, i.ToString(), (b + i * 10).ToString(), ip);
                        }
                    }
                    Console.WriteLine("生產一組成功!");
                    Thread.Sleep(1 * 1000);
                }
            });
            ////消費者開始消費
            //for (int i = 0; i < Split; i++)
            //{
            //    Console.WriteLine("啓動消費者:" + i.ToString());
            //    Task.Run(() =>
            //    {
            //        KafkaConsumer.Consumer(topic, i.ToString(), groupid, ip);
            //    });
            //    Thread.Sleep(5 * 1000);
            //}

            //消費者開始消費
            Task.Run(() =>
            {
                KafkaConsumer.Consumer(topic, 0.ToString(), groupid, ip);
            });
            //消費者開始消費
            Task.Run(() =>
            {
                KafkaConsumer.Consumer(topic, 1.ToString(), groupid, ip);
            });
            //消費者開始消費
            Task.Run(() =>
            {
                KafkaConsumer.Consumer(topic, 2.ToString(), groupid, ip);
            });
            //消費者開始消費
            Task.Run(() =>
            {
                KafkaConsumer.Consumer(topic, 3.ToString(), groupid, ip);
            });
            #endregion
            Console.ReadLine();
        }