Go語言（二十）日誌採集項目（二）Etcd的使用

日誌採集項目（二）Etcd的使用

ETCD 介紹

• 概念： 高可用的分佈式key-value存儲，實現配置共享和服務發現
• 類似項目： zookeeper和consul
• 開發語言： Go
• 接口： 提供restful的http接口，使用簡單
• 實現算法： 基於raft算法的強一致性，高可用的服務存儲目錄

ETCD的應用場景

• 服務發現和註冊
• 配置中心
• 分佈式鎖
• master選舉

ETCD環境搭建

• 下載地址：https://github.com/etcd-io/etcd/releases/download/v3.4.7/etcd-v3.4.7-linux-amd64.tar.gz
• 啓動方式： bin/etcd 可以直接啓動

[root@centos7-node1 etcd]# nohup ./etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://0.0.0.0:2379 --listen-peer-urls http://0.0.0.0:2380 --initial-advertise-peer-urls http://0.0.0.0:2380 &        #啓動etcd

• etcdctl使用

[root@centos7-node1 ~]# cd /opt/application/etcd/
[root@centos7-node1 etcd]# ./etcdctl --endpoints "http://localhost:2379" put /logagent/conf 333333
[root@centos7-node1 etcd]# ./etcdctl --endpoints "http://localhost:2379" watch  /logagent/conf   
[root@centos7-node1 etcd]# ./etcdctl --endpoints "http://localhost:2379" del /logagent/conf

• go實現watch功能

安裝v3插件
go get go.etcd.io/etcd/clientv3

代碼

package main

import (
   "context"
   "fmt"
   "go.etcd.io/etcd/clientv3"
   "time"
)

func main() {
   client,err := clientv3.New(clientv3.Config{
      Endpoints: []string{"192.168.56.11:2379"},
      DialTimeout: time.Second*3,
   })

   defer client.Close()
   fmt.Printf("conn succ\n")
   for {
      resultChan := client.Watch(context.Background(),"/logagent/conf")
      for v := range resultChan{
         if v.Err() != nil {
            fmt.Printf("watch faild,err:%v\n",err)
            continue
         }
         for _,e := range v.Events {
            fmt.Printf("event_type:%v,key:%v,val:%v\n",e.Type,e.Kv.Key,string(e.Kv.Value))
         }
      }
   }
}

• go 實現put功能

package main

import (
   "context"
   "fmt"
   "go.etcd.io/etcd/clientv3"
   "time"
)

func main() {
   client,err := clientv3.New(clientv3.Config{
      Endpoints: []string{"192.168.56.11:2379"},
      DialTimeout: time.Second*3,
   })

   defer client.Close()
   fmt.Printf("conn succ\n")

   _,err = client.Put(context.Background(),"/logagent/conf","sddadas")

   if err != nil {
      fmt.Printf("Put faild,err:%v\n",err)
   }
}

• kafka消費代碼

package main

import (
   "fmt"
   "github.com/Shopify/sarama"
   "sync"
)

var wg sync.WaitGroup

func main() {
   //連接配置
   consumer,err := sarama.NewConsumer([]string{"192.168.56.11:9092"},nil)
   if err != nil {
      fmt.Printf("consumer message faild,error:%v\n",err)
      return
   }
   fmt.Printf("conn succ\n")
   pt,err := consumer.Partitions("nginx_log")
   if err != nil {
      fmt.Printf("get partions aild,err:%v\n",err)
      return
   }

   for _,p := range pt {
      pc, err := consumer.ConsumePartition("nginx_log",p,sarama.OffsetNewest)

      if err !=  nil {
         fmt.Printf("consumer faild,error:%v\n",err)
         continue
      }
      wg.Add(1)
      go func() {
         for m := range pc.Messages() {
            fmt.Printf("topic:%v,value:%v\n",m.Topic,string(m.Value))
         }
         wg.Done()
      }()
   }
   wg.Wait()
}