flume自定義攔截器進行日誌採集,並寫入kafka

• 項目架構
  
• 創建maven項目，引入依賴

  <dependency>
      <groupId>org.apache.flume</groupId>
        <artifactId>flume-ng-core</artifactId>
        <version>1.8.0</version>
        <scope>provided</scope>
     </dependency>
  

  • 創建LogETLInterceptor類實現Interceptor接口，進行初步清洗

  public class LogETLInterceptor implements Interceptor {
      
      @Override
      public void initialize() {
      }
  
      @Override
      public Event intercept(Event event) {
          // 主頁日誌{json}、 時間日誌 時間戳|{json}
          byte[] body = event.getBody();
          String log = new String(body, Charset.forName("UTF-8"));
          if (log.contains("start")) {
              // 啓動日誌
              if (LogUtil.validateStartLog(log)) {
                  return event;
              }
          } else {
              // 時間日誌
              if (LogUtil.validateEventLog(log)) {
                  return event;
              }
          }
          return null;
      }
  
      @Override
      public List<Event> intercept(List<Event> events) {
          ArrayList<Event> eventList = new ArrayList<>();
          for (Event event : events) {
              Event validEvent = intercept(event);
              if (validEvent != null) {
                  eventList.add(validEvent);
              }
          }
          return eventList;
      }
  
      @Override
      public void close() {
      }
  
      public static class Builder implements Interceptor.Builder {
          
          @Override
          public Interceptor build() {
              return new LogETLInterceptor();
          }
  
          @Override
          public void configure(Context context) {
          }
      }
  
  }
  

  • 創建LogTypeInterceptor實現Interceptor,用於將不同類型的日誌寫到不同kafka topic中

  public class LogTypeInterceptor implements Interceptor {
  
      @Override
      public void initialize() {
      }
  
      @Override
      public Event intercept(Event event) {
          byte[] body = event.getBody();
          Map<String, String> headers = event.getHeaders();
          String log = new String(body, Charset.forName("UTF-8"));
          if (log.contains("start")) {
              // 啓動日誌
              headers.put("topic", "topic_start");
          } else {
              // 事件日誌
              headers.put("topic", "topic_event");
          }
          return event;
      }
  
      @Override
      public List<Event> intercept(List<Event> events) {
          ArrayList<Event> eventList = new ArrayList<>();
          for (Event event : events) {
              Event interceptedEvent = intercept(event);
              eventList.add(interceptedEvent);
          }
          return eventList;
      }
  
      @Override
      public void close() {
      }
  
      public static class Builder implements Interceptor.Builder {
  
          @Override
          public Interceptor build() {
              return new LogTypeInterceptor();
          }
  
          @Override
          public void configure(Context context) {
  
          }
      }
  
  }
  

• 打包後放到flume/lib/目錄下
• 編寫conf文件

  # Name the components on this agent
  a1.sources = r1
  a1.channels = c1 c2
  
  # Describe/configure the source
  a1.sources.r1.type = TAILDIR
  a1.sources.r1.positionFile = /soft/flume/data/taildir_position.json
  a1.sources.r1.filegroups = f1
  a1.sources.r1.filegroups.f1 = /tmp/flume/.*log
  a1.sources.ri.maxBatchCount = 1000
  
  # Describe c1
  a1.channels.c1.type = org.apache.flume.channel.kafka.KafkaChannel
  a1.channels.c1.kafka.bootstrap.servers = cdh01:9092,cdh02:9092,cdh03:9092
  a1.channels.c1.kafka.topic = topic_start
  
  # Describe c2
  a1.channels.c2.type = org.apache.flume.channel.kafka.KafkaChannel
  a1.channels.c2.kafka.bootstrap.servers = cdh01:9092,cdh02:9092,cdh03:9092
  a1.channels.c2.kafka.topic = topic_event
  
  
  # Describe interceptor
  a1.sources.r1.interceptors = i1 i2
  a1.sources.r1.interceptors.i1.type = com.spark.interceptors.LogETLInterceptor$Builder
  a1.sources.r1.interceptors.i2.type = com.spark.interceptors.LogTypeInterceptor$Builder
  
  # Describe selector
  a1.sources.r1.selector.type = multiplexing
  a1.sources.r1.selector.header = topic
  a1.sources.r1.selector.mapping.topic_start = c1
  a1.sources.r1.selector.mapping.topic_event = c2
  
  # Bind the source and sink to the channel
  a1.sources.r1.channels = c1 c2
  

• 啓動flume

  /soft/flume/bin/flume-ng agent -n a1 -f /soft/flume/conf/file-flume-kafka.conf -c /soft/flume/conf/  -Dflume.root.logger=INFO,console  > /soft/flume/logs/test.log 2>&1  &