使用docker安裝elk數據分析

docker:

    sudo yum install -y yum-utils device-mapper-persistent-data lvm2
    sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
    sudo yum install docker-ce

    sudo systemctl start docker
    sudo docker run hello-world

    # docker中-d參數是在後臺運行, 若調試的話, 可以改爲--rm

Install Elasticsearch with Docker

    sudo docker pull docker.elastic.co/elasticsearch/elasticsearch:6.4.2

    sys config
        /etc/sysctl.conf中添加: vm.max_map_count=262144
        sudo grep vm.max_map_count /etc/sysctl.conf
    
    run:
        docker run -p 9200:9200 -p 9300:9300 -d --name elasticsearch -d -e "discovery.type=single-node" -e network.publish_host=0.0.0.0 docker.elastic.co/elasticsearch/elasticsearch:6.4.2
    
    inspect status of cluster:
        curl http://127.0.0.1:9200/_cat/health
    
    查看索引: curl -X GET 'http://localhost:9200/_cat/indices?v'
    查看type: curl 'localhost:9200/_mapping?pretty=true'
    新建 Index: curl -X PUT 'localhost:9200/weather'
    查詢記錄: curl 'localhost:9200/accounts/person/_search'  # /Index/Type/_search

Kibana

    docker pull docker.elastic.co/kibana/kibana:6.4.2

    docker run -p 5601:5601 --name kibana -d --link elasticsearch -e ELASTICSEARCH_URL=http://10.97.88.71:9200 -e elasticsearch.ssl.verify=false -e server.host=0.0.0.0 docker.elastic.co/kibana/kibana:6.4.2
    # elasticsearch 爲 Elasticsearch 的docker name
    # 10.97.88.71, 使用docker的時候, 不能用localhost

    http://10.97.88.71:5601

Logstash

    docker pull docker.elastic.co/logstash/logstash:6.4.2

    docker run --name logstash --rm -p 5144:5144 --link elasticsearch -e xpack.monitoring.enabled=true -e ELASTICSEARCH_URL=http://10.97.88.71:9200 -v ~/pipeline/:/usr/share/logstash/pipeline/ docker.elastic.co/logstash/logstash:6.4.2

    配置文件 logstash.yml
        input{
            syslog {
                type => "icc_rpc_log"
                port => "5144"
            }
        }

        filter {
            if [type] == "icc_rpc_log"{
                grok {
                patterns_dir => "/usr/local/logstash/patterns"        //設置自定義正則路徑
                # match => { "message" => "%{IP:client_id_address} %{LOGLEVEL:loglevel}" }
                match => { "message" => "%{TIMESTAMP_ISO8601:log_date} %{LOGLEVEL:loglevel}\:index code %{NUMBER:index_code} is invoked by %{IP:client_id_address}\(%{DATA:user_cd}\-%{DATA:user_name}\) with parameter_list %{DATA:parameter_list} \+ default_parameter %{GREEDYDATA:parameter_default}" }
                }
            }
        }

        output {
            if [type] == "icc_rpc_log" and [loglevel] == "INFO"{
                stdout { codec => rubydebug }
            }
            if [type] == "icc_rpc_log" and [loglevel] == "ERROR"{
                elasticsearch {
                    hosts => ["10.97.88.71:9200"]
                    index => "icc_calc_log"
                    # index => "system-syslog-log-%{+YYYY.MM.dd}"
                }
            }

        }
    測試數據: 
        2018-10-19 08:50:47 INFO:index code 000000001 is invoked by 1.2.2.2(-Anonymous) with parameter_list [{'asset_code': '000001', 'benm_code': '000002', 'yield_date_type': None, 'yield_type': None}] + default_parameter {'end_date': '2017-05-01', 'start_date': '2017-05-01', 'freq_code': 'D', 'riskfree_benm_code': '000003', 'annual_flag': False}

其他相關站點:
Logstash 最佳實踐: https://doc.yonyoucloud.com/doc/logstash-best-practice-cn/output/elasticsearch.html
Kibana（一張圖片勝過千萬行日誌）: https://www.cnblogs.com/cjsblog/p/9476813.html
kibana visualize添加自定義查詢: https://blog.csdn.net/xr568897472/article/details/71540937
全文搜索引擎 Elasticsearch 入門教程: http://www.ruanyifeng.com/blog/2017/08/elasticsearch.html
使用Docker搭建ELK日誌系統: http://chenzhijun.me/2017/12/27/elk-docker/
logstash-patterns-core: https://github.com/logstash-plugins/logstash-patterns-core/blob/master/patterns/grok-patterns
1分鐘搭建Elasticsearch可視化(沒試過): https://blog.csdn.net/dounine/article/details/78887792

Elasticsearch 查詢

curl -X GET 'localhost:9200/icc_calc_log/_search' -H 'Content-Type: application/json' -d '{"query":{"match":{"message":"invoked"}}}'

import calendar
import datetime

import requests
import json

"""
curl -X GET 'localhost:9200/icc_calc_log/_search' -H 'Content-Type: application/json' 
-d '{"query":{"match":{"message":"invoked"}}}'

?size=3
"""
# json_param = {"query": {"match": {"message": "invoked"}}}
# json_param = {"query": {"range": {"log_date": {"gte": "2018-01-01", "lte": "2018-12-31"}}}}
# json_param = {"query": {"range": {"client_id_address": {"gte": "1.2.2.1", "lte": "1.2.2.2"}}}}
# json_param = {"query": {"regexp": {"log_date": "2018.*?"}}}


# def get_log_date_reg():
#     start_month_end_date = calendar.monthrange(start_date.year, start_date.month)[1]
#     end_month_end_date = calendar.monthrange(end_date.year, end_date.month)[1]
#     year_list, month_list, day_list = [], [], []
#
#     year_list = [str(i) for i in range(start_date.year + 1, end_date.year)]
#
#     if start_date.year != end_date.year:
#         for i in range(start_date.month + 1, 13):
#             month_list.append(str(start_date.year) + "-" + str(i).rjust(2, "0"))
#
#         for i in range(1, end_date.month):
#             month_list.append(str(end_date.year) + "-" + str(i).rjust(2, "0"))
#     else:
#         for i in range(start_date.month + 1, end_date.month):
#             month_list.append(str(start_date.year) + "-" + str(i).rjust(2, "0"))
#
#     if start_date.day == 1:
#         month_list.append(str(start_date.year) + "-" + str(start_date.month).rjust(2, "0"))
#         start_date_is_month_begin_date_flag = True
#     else:
#         start_date_is_month_begin_date_flag = False
#     if end_date.day == end_month_end_date:
#         month_list.append(str(end_date.year) + "-" + str(end_date.month).rjust(2, "0"))
#         end_date_is_month_end_date_flag = True
#     else:
#         end_date_is_month_end_date_flag = False
#
#     if start_date_is_month_begin_date_flag is False:
#         for i in range(start_date.day, start_month_end_date + 1):
#             day_list.append(str(start_date.year) + "-" + str(start_date.month).rjust(2, "0") + "-" + str(i).rjust(2, "0"))
#     if end_date_is_month_end_date_flag is False:
#         for i in range(1, end_date.day + 1):
#             day_list.append(str(end_date.year) + "-" + str(end_date.month).rjust(2, "0") + "-" + str(i).rjust(2, "0"))
#
#     print(year_list)
#     print(month_list)
#     print(day_list)
#     result = year_list + month_list + day_list
#     result_str = "|".join(result)
#     print(result_str)
#     return result_str
# # "log_date": "2018-10-19 08:50:47",


ip = "10.97.88.71"
index = "icc_calc_log"
file_name = r"./qqqqqqqqqqqqqqqqqqqqq_view_log"


def save_result(result, f_handler):
    # print(json.dumps(result, ensure_ascii=False, indent=4))
    result_list = result.get("hits", {}).get("hits", [])
    for result_now in result_list:
        f_handler.write(json.dumps(result_now, ensure_ascii=False, indent=4))
        f_handler.write("\n")
    f_handler.flush()
    return result["_scroll_id"]


def start():
    # query_param = {"query": {"regexp": {"log_date": "{0}".format(get_log_date_reg())}}, "post_filter": {"regexp": {"message": "invoked"}}}
    start_date, end_date = "2018-01-01", "2018-12-31"
    query_param = {"query": {"range": {"@timestamp": {"gte": start_date, "lte": end_date}}}, "post_filter": {"regexp": {"message": "invoked"}}}

    with open(file_name, 'w') as f:
        result = requests.post(r"http://{0}:9200/{1}/_search?size=1&scroll=1m".format(ip, index), json=query_param).json()
        _scroll_id = save_result(result, f)

        while True:
            result = requests.post(r"http://{0}:9200/_search/scroll?scroll=2m&scroll_id={1}".format(ip, _scroll_id), json={}).json()
            _scroll_id = save_result(result, f)
            if not result.get("hits", {}).get("hits"):
                break


if __name__ == "__main__":
    start()