elk筆記2--使用docker啓一套elk實例

1 需求簡介

1）使用 docker 啓動一個elasticsearch 實例
2）使用 docker 啓動一個kibana 實例
3）使用 docker 啓動一個logstash 實例
4）使用 logstash 收集dmsg和syslog日誌

2 啓動步驟

2.1 下載docker鏡像

筆者此處直接從dockerhub拉取， 也可以按照官網的制定路徑拉鏡像（使用官方路徑可能下載速度較慢）

1. docker pull elasticsearch:7.6.1
2. docker pull kibana:7.6.1
3. docker pull logstash:7.6.1

2.2 啓動docker實例

1. 啓動elasticsearch

    docker run -d --name=elasticsearch_7.6.1 -p 9203:9200 -p 9303:9300 -e "discovery.type=single-node" -e ES_JAVA_OPTS="-Xms512m -Xmx512m" \
    -v /home/xg/soft/bigdata/elk7.6.1/docker/es_config:/usr/share/elasticsearch/config \
    elasticsearch:7.6.1
   

   此處將/usr/share/elasticsearch/config 拷貝到本地目錄，以便於更改配置，此處可以去掉-v參數
2. 啓動kibana

    docker run -d --name=kibana_7.6.1 --link elasticsearch_7.6.1:elasticsearch -p 5603:5601 \
    -v /home/xg/soft/bigdata/elk7.6.1/docker/kibana_config:/usr/share/kibana/config \
    kibana:7.6.1
   

   此處將/usr/share/kibana/config 拷貝到本地目錄，以便於更改配置，此處可以去掉-v參數
3. 啓動logstash

    docker run -d --name=logstash_7.6.1 --link elasticsearch_7.6.1:elasticsearch \
    		-v /home/xg/soft/bigdata/elk7.6.1/docker/logstash_config:/usr/share/logstash/config \
    		-v /home/xg/soft/bigdata/elk7.6.1/docker/pipline:/usr/share/logstash/pipeline \
    		-v /home/xg/soft/bigdata/log/testlog:/var/log/testlog \
    		-v /var/log/syslog:/var/log/syslog_host \
    		-v /var/log/dmesg:/var/log/dmesg_host \
    		logstash:7.6.1   
   

   此處有多個目錄映射， 其中syslog，dmesg主要爲了logstash能正常讀取宿主上的日誌，pipline目錄文件主要存放input、output 和 filter規則， logstash_config和testlog可以根據需要去掉。
   注意： logstash對應的pipeline不能爲空，若爲空則logstsh會自動退出。
   logstash的pipeline configuration 在pipeline目錄下，以下爲筆者寫的一個pipeline配置， default.conf 中包含syslog，dmesg，es_error(此處沒有映射過來，可以刪掉):

   input{
   file{
   	path => "/var/log/syslog_host"
   	type => "syslog" 
   	start_position => "beginning"
   }
   file{
   	path => "/var/log/dmesg_host"
   	type => "dmesg"
   	start_position => "beginning"
   }
   file{
   	path => "/home/xg/soft/bigdata/log/es6.8.8/es6.8.log"
   	type => "es_error"
   	start_position => "beginning"
   	codec => multiline {
         # Grok pattern names are valid! :)
         pattern => "^\["
         negate => true
         what => "previous"
       }
   }
   }
   
   filter{
   }
   
   output{
   if [type] == "syslog" {
   	elasticsearch {
   		hosts => ["elasticsearch:9200"]
   		index => "syslog-%{+YYYY.MM.dd}"
   	}
   }
   if [type] == "dmesg" {
   	elasticsearch {
   		hosts => ["elasticsearch:9200"]
   		index => "dmesg-%{+YYYY.MM.dd}"
   	}
   }
   if [type] == "es_error" {
   	elasticsearch {
   		hosts => ["elasticsearch:9200"]
   		index => "es_error-%{+YYYY.MM.dd}"
   	}
   }
   }
   

2.3 多功能elk實例管理腳本

筆者根據個人使用需要，寫了一個小腳本，以便於統一管理elk實例，可以根據需要修改。

#!/bin/bash 

help()
{
	cat <<_EOF
Help function:
	bash updown_docker_elk.sh help|start|stop|restart|new|es|kibana|logstash|rm
		=> start|stop|restart es|kibana|logstash|all
_EOF
}

elk_rm()
{
	docker stop elasticsearch_7.6.1
	docker rm elasticsearch_7.6.1
	docker stop kibana_7.6.1
	docker rm kibana_7.6.1
	docker stop logstash_7.6.1
	docker rm logstash_7.6.1
}

elk_start()
{
case "$1" in
    es)
		docker start elasticsearch_7.6.1
		;;
	kibana)
		docker start kibana_7.6.1
		;;
	logstash)
		docker start logstash_7.6.1
		;;
    all)
    	docker start elasticsearch_7.6.1
		sleep 15
		docker start kibana_7.6.1
		sleep 10
		docker start logstash_7.6.1
        ;;
    *)
    	help
    	exit 1
    	;;
esac
}

elk_restart()
{
case "$1" in
    es)
		docker restart elasticsearch_7.6.1
		;;
	kibana)
		docker restart kibana_7.6.1
		;;
	logstash)
		docker restart logstash_7.6.1
		;;
    all)
    	docker restart elasticsearch_7.6.1
		sleep 15
		docker restart kibana_7.6.1
		sleep 10
		docker restart logstash_7.6.1
        ;;
    *)
    	help
    	exit 1
    	;;
esac
}

elk_stop()
{
case "$1" in
    es)
		docker stop elasticsearch_7.6.1
		;;
	kibana)
		docker stop kibana_7.6.1
		;;
	logstash)
		docker stop logstash_7.6.1
		;;
    all)
        docker stop logstash_7.6.1
		docker stop kibana_7.6.1
		docker stop elasticsearch_7.6.1
        ;;
    *)
    	help
    	exit 1
    	;;
esac
}

elk_new(){
	docker stop elasticsearch_7.6.1
	docker rm elasticsearch_7.6.1
	docker stop kibana_7.6.1
	docker rm kibana_7.6.1
	docker stop logstash_7.6.1
	docker rm logstash_7.6.1
	# es
	elk_es
	# kibana
	sleep 15
	elk_kibana
	# logstash
	elk_logstash
}

elk_es()
{
	docker run -d --name=elasticsearch_7.6.1 -p 9203:9200 -p 9303:9300 -e "discovery.type=single-node" -e ES_JAVA_OPTS="-Xms512m -Xmx512m" \
	-v /home/xg/soft/bigdata/elk7.6.1/docker/es_config:/usr/share/elasticsearch/config \
	elasticsearch:7.6.1
}

elk_kibana()
{
	docker run -d --name=kibana_7.6.1 --link elasticsearch_7.6.1:elasticsearch -p 5603:5601 \
	-v /home/xg/soft/bigdata/elk7.6.1/docker/kibana_config:/usr/share/kibana/config \
	kibana:7.6.1
}

elk_logstash()
{
	docker run -d --name=logstash_7.6.1 --link elasticsearch_7.6.1:elasticsearch \
			-v /home/xg/soft/bigdata/elk7.6.1/docker/logstash_config:/usr/share/logstash/config \
			-v /home/xg/soft/bigdata/elk7.6.1/docker/pipline:/usr/share/logstash/pipeline \
			-v /home/xg/soft/bigdata/log/testlog:/var/log/testlog \
			-v /var/log/syslog:/var/log/syslog_host \
			-v /var/log/dmesg:/var/log/dmesg_host \
			logstash:7.6.1
}

case "$1" in
    help)
        help
        ;;
    start)
        elk_start $2
        ;;
    restart)
        elk_restart $2
        ;;
    stop)
        elk_stop $2
        ;;
    new)
        elk_new
        ;;
    es)
		elk_es
		;;
	kibana)
		elk_kibana
		;;
	logstash)
		elk_logstash
		;;
    rm)
	elk_rm
	;;
    *)
        echo "Unknown command: $1"
        help
        exit 1
        ;;
esac

2.4 測試結果

1. es 查看所有index
   
2. kibana 查看所有logstash上傳的syslog日誌
   

3 說明

1. 軟件環境
   筆者測試系統爲Ubuntu 2004 Desktop
   elk 版本爲7.6.1
2. 參考文檔
   installing-elastic-stack