緣起
最近在使用騰訊雲，想對訪問日誌進行收集與分析，發現CLB（負責均衡）日誌只能保存到COS上面，而且是每個CLB沒小時壓發送個gz壓縮包到COS。
實現方式
CLB配置日誌存儲到COS，Filebeat客戶端CVM安裝cosfs掛載COS，並配置Filebeat輸出到Elasticsearch集羣，最後通過Kibana和Grafana分析。
參考文檔
https://www.elastic.co/guide/en/logstash/current/lookup-enrichment.html
https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-reference-yml.html
https://www.elastic.co/guide/en/beats/filebeat/current/configuration-template.html
https://www.elastic.co/guide/en/logstash-versioned-plugins/versioned_plugin_docs/v4.1.5-plugins-inputs-file.html
https://www.elastic.co/guide/en/logstash-versioned-plugins/versioned_plugin_docs/v3.1.0-plugins-filters-translate.html
https://www.elastic.co/guide/en/logstash-versioned-plugins/versioned_plugin_docs/v1.0.4-plugins-filters-jdbc_streaming.html
https://www.elastic.co/guide/en/logstash-versioned-plugins/versioned_plugin_docs/v9.2.0-plugins-outputs-elasticsearch.html
/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output-elasticsearch-9.2.0-java/lib/logstash/outputs/elasticsearch/elasticsearch-template-es6x.json實施步驟
1. COS創建bucket用於存儲CLB日誌
2. CLB配置存儲日誌到COS
日誌訪問：當前僅支持HTTP/HTTPS訪問日誌的收集，騰訊雲默認在CLB底層爲客戶保留3天的日誌；開啓日誌訪問後，日誌將存入COS，支持更長期地存儲，詳情請見https://cloud.tencent.com/document/product/214/10329。
3. 客戶端配置COSFS工具
安裝Filebeat或Logstash客戶端上安裝COSFS工具，掛載COS，安裝參考https://cloud.tencent.com/document/product/436/6883。
4. 客戶端配置傳輸日誌到Elasticsearch集羣
a. Logstash主要配置信息如下：
input {
file{
path => "/nginxlog/clb_log_bj/*/*/*" #監控日誌路徑，可通配
exclude => "*.gz" #排除掉gz壓縮文件，因爲騰訊CLB日誌沒小時生成一個壓縮包，這邊需要腳本自動解壓縮
codec => "json" #日誌格式爲JSON
}
}
filter {
jdbc_streaming{ #使用jdbc_streaming filter是給域名加上部門，默認CLB日誌沒有部門字段，也可以用translate filter但是不方便動態更新
jdbc_driver_library => "/etc/logstash/mysql-connector.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
jdbc_connection_string => "jdbc:mysql://localhost:3306/domain"
jdbc_user => "domain_rw"
jdbc_password => "domain_password"
statement => "select ifnull((select department from domain.domain WHERE url = :server_name),CONVERT('其他' USING utf8)) as department"
parameters => { "server_name" => "server_name"}
target => "department"
}
}
output {
elasticsearch {
hosts => ["elasticsearch01:9200","elasticsearch02:9200","elasticsearch03:9200"]
index => "logstash-nginxlog-%{+YYYY.MM.dd}"
manage_template => true
template_overwrite => true
template_name => "nginx_template" #自定義模板名字
template => "/etc/logstash/templates/nginx_template" #自定義模板路徑
}
}b. Nginx模板配置信息如下：
{
"template" : "logstash-nginxlog-*", #在默認logstash模板上面修改的
"version" : 999,
"settings" : { #主要是改動index分片，副本，其他也可以更加需要修改
"index.refresh_interval" : "60s",
"index.number_of_shards": "3",
"index.number_of_replicas": "1"
},
"mappings" : {
"_default_" : {
"dynamic_templates" : [ {
"message_field" : {
"path_match" : "message",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text",
"norms" : false
}
}
}, {
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text", "norms" : false,
"fields" : {
"keyword" : { "type": "keyword", "ignore_above": 256 }
}
}
}
} ],
"properties" : {
"@timestamp": { "type": "date"},
"@version": { "type": "keyword"},
"geoip" : {
"dynamic": true,
"properties" : {
"ip": { "type": "ip" },
"location" : { "type" : "geo_point" },
"latitude" : { "type" : "half_float" },
"longitude" : { "type" : "half_float" }
}
}
}
}
}
}c. Filebeat主配置文件如下：
filebeat.inputs: #默認簡單需求filebeat就能滿足，記錄下
- type: log
enabled: true
paths:
- /nginxlog/clb_log_bj/*/*/* #監控日誌路徑
exclude_files: ['.gz/pre>] #排除壓縮文件
json.message_key: log #如下三行設置json格式
json.keys_under_root: true
json.overwrite_keys: true
filebeat.config.modules: #設置日誌字段，可以刪除不需要的字段信息
path: ${path.config}/modules.d/*.yml
reload.enabled: true
setup.template.enabled: true #如下設置索引名稱、分片、副本信息
setup.template.name: "filebeat-nginxlog"
setup.template.pattern: "filebeat-nginxlog-*"
setup.template.overwrite: true
setup.template.settings:
index.number_of_shards: 3
index.number_of_replicas: 1
output.elasticsearch: #如下輸出elasticsearch集羣配置
hosts: ["elasticsearch01:9200","elasticsearch02:9200","elasticsearch03:9200"]
index: "filebeat-nginxlog-%{+yyyy.MM.dd}"5. Grafana或者Kibana製圖
a. Grafana製圖模板如下：
{
"__inputs": [
{
"name": "DS_LOGSTASH-NGINXLOG",
"label": "logstash-nginxlog",
"description": "",
"type": "datasource",
"pluginId": "elasticsearch",
"pluginName": "Elasticsearch"
}
],
"__requires": [
{
"type": "datasource",
"id": "elasticsearch",
"name": "Elasticsearch",
"version": "5.0.0"
},
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "5.0.1"
},
{
"type": "panel",
"id": "grafana-piechart-panel",
"name": "Pie Chart",
"version": "1.3.3"
},
{
"type": "panel",
"id": "graph",
"name": "Graph",
"version": "5.0.0"
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"id": null,
"iteration": 1534240475758,
"links": [],
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 16,
"panels": [],
"repeat": "department",
"title": "$department",
"type": "row"
},
{
"aliasColors": {},
"breakPoint": "50%",
"cacheTimeout": null,
"combine": {
"label": "Others",
"threshold": 0
},
"datasource": "${DS_LOGSTASH-NGINXLOG}",
"description": "",
"fontSize": "80%",
"format": "none",
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 1
},
"id": 6,
"interval": null,
"legend": {
"percentage": true,
"percentageDecimals": 2,
"show": true,
"values": true
},
"legendType": "Right side",
"links": [],
"maxDataPoints": 3,
"nullPointMode": "connected",
"pieType": "pie",
"strokeWidth": "0",
"targets": [
{
"bucketAggs": [
{
"fake": true,
"field": "http_host.keyword",
"id": "3",
"settings": {
"min_doc_count": 1,
"order": "desc",
"orderBy": "_count",
"size": "10"
},
"type": "terms"
},
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto",
"min_doc_count": 0,
"trimEdges": 0
},
"type": "date_histogram"
}
],
"expr": "",
"format": "time_series",
"intervalFactor": 1,
"metrics": [
{
"field": "select field",
"id": "1",
"type": "count"
}
],
"query": "http_host.keyword:$hostname AND status.keyword:$status AND upstream_addr.keyword:$upstream_addr AND department.department.keyword:$department",
"refId": "A",
"timeField": "@timestamp"
}
],
"title": "域名訪問比例圖",
"type": "grafana-piechart-panel",
"valueName": "total"
},
{
"aliasColors": {},
"breakPoint": "50%",
"cacheTimeout": null,
"combine": {
"label": "Others",
"threshold": 0
},
"datasource": "${DS_LOGSTASH-NGINXLOG}",
"fontSize": "80%",
"format": "none",
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 1
},
"id": 4,
"interval": null,
"legend": {
"percentage": true,
"percentageDecimals": 2,
"show": true,
"sortDesc": true,
"values": false
},
"legendType": "Right side",
"links": [],
"maxDataPoints": 3,
"nullPointMode": "connected",
"pieType": "pie",
"strokeWidth": "0",
"targets": [
{
"bucketAggs": [
{
"fake": true,
"field": "status.keyword",
"id": "3",
"settings": {
"min_doc_count": 1,
"order": "desc",
"orderBy": "_count",
"size": "10"
},
"type": "terms"
},
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto",
"min_doc_count": 0,
"trimEdges": 0
},
"type": "date_histogram"
}
],
"expr": "",
"format": "time_series",
"intervalFactor": 1,
"metrics": [
{
"field": "select field",
"id": "1",
"type": "count"
}
],
"query": "http_host.keyword:$hostname AND status.keyword:$status AND upstream_addr.keyword:$upstream_addr AND department.department.keyword:$department",
"refId": "A",
"timeField": "@timestamp"
}
],
"title": "狀態碼比例圖",
"type": "grafana-piechart-panel",
"valueName": "total"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_LOGSTASH-NGINXLOG}",
"fill": 1,
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 10
},
"id": 2,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"bucketAggs": [
{
"fake": true,
"field": "http_host.keyword",
"id": "3",
"settings": {
"min_doc_count": 1,
"order": "desc",
"orderBy": "_count",
"size": "10"
},
"type": "terms"
},
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto",
"min_doc_count": 0,
"trimEdges": 0
},
"type": "date_histogram"
}
],
"expr": "",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "",
"metrics": [
{
"field": "select field",
"id": "1",
"type": "count"
}
],
"query": "http_host.keyword:$hostname AND status.keyword:$status AND upstream_addr.keyword:$upstream_addr AND department.department.keyword:$department",
"refId": "A",
"timeField": "@timestamp"
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "域名訪問趨勢TOP10",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
},
{
"aliasColors": {},
"bars": true,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_LOGSTASH-NGINXLOG}",
"fill": 1,
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 10
},
"hideTimeOverride": false,
"id": 18,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": true,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": false,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "Count"
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"bucketAggs": [
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto",
"min_doc_count": 0,
"trimEdges": 0
},
"type": "date_histogram"
}
],
"expr": "",
"format": "time_series",
"intervalFactor": 1,
"metrics": [
{
"field": "request_time",
"id": "1",
"meta": {},
"settings": {
"percents": [
"90",
"95",
"99",
"99.9",
"99.99",
"100"
]
},
"type": "percentiles"
}
],
"query": "http_host.keyword:$hostname AND status.keyword:$status AND upstream_addr.keyword:$upstream_addr AND department.department.keyword:$department",
"refId": "A",
"timeField": "@timestamp"
}
],
"thresholds": [
{
"colorMode": "critical",
"fill": true,
"line": true,
"op": "gt"
},
{
"colorMode": "warning",
"fill": false,
"line": false,
"op": "gt",
"value": 5000
}
],
"timeFrom": null,
"timeShift": null,
"title": "延遲",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"decimals": null,
"format": "s",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
]
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 17
},
"id": 14,
"panels": [
{
"columns": [],
"datasource": "logstash-nginxlog",
"fontSize": "100%",
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 70
},
"id": 12,
"links": [],
"pageSize": null,
"scroll": true,
"showHeader": true,
"sort": {
"col": 1,
"desc": true
},
"styles": [
{
"alias": "Time",
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"link": false,
"pattern": "Time",
"type": "date"
},
{
"alias": "",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"decimals": 2,
"pattern": "/.*/",
"thresholds": [],
"type": "number",
"unit": "short"
}
],
"targets": [
{
"bucketAggs": [
{
"field": "http_user_agent.keyword",
"id": "2",
"settings": {
"min_doc_count": 1,
"order": "desc",
"orderBy": "_count",
"size": "10"
},
"type": "terms"
}
],
"expr": "",
"format": "table",
"intervalFactor": 1,
"metrics": [
{
"field": "select field",
"id": "1",
"type": "count"
}
],
"query": "http_host.keyword:$hostname AND status.keyword:$status AND upstream_addr.keyword:$upstream_addr AND department.department.keyword:$department",
"refId": "A",
"timeField": "@timestamp"
}
],
"title": "域名訪問Agent TOP 10",
"transform": "table",
"transparent": false,
"type": "table"
},
{
"columns": [],
"datasource": "logstash-nginxlog",
"fontSize": "100%",
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 70
},
"id": 8,
"links": [],
"pageSize": null,
"scroll": true,
"showHeader": true,
"sort": {
"col": 1,
"desc": true
},
"styles": [
{
"alias": "Time",
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"pattern": "Time",
"type": "number",
"unit": "none"
},
{
"alias": "",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"decimals": 2,
"pattern": "/.*/",
"thresholds": [],
"type": "number",
"unit": "short"
}
],
"targets": [
{
"bucketAggs": [
{
"field": "remote_addr.keyword",
"id": "2",
"settings": {
"min_doc_count": 1,
"order": "desc",
"orderBy": "_count",
"size": "10"
},
"type": "terms"
}
],
"expr": "",
"format": "table",
"intervalFactor": 1,
"metrics": [
{
"field": "select field",
"id": "1",
"type": "count"
}
],
"query": "http_host.keyword:$hostname AND status.keyword:$status AND upstream_addr.keyword:$upstream_addr AND department.department.keyword:$department",
"refId": "A",
"timeField": "@timestamp"
}
],
"title": "域名客戶端IP TOP 10",
"transform": "table",
"type": "table"
},
{
"columns": [],
"datasource": "logstash-nginxlog",
"fontSize": "100%",
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 79
},
"id": 10,
"links": [],
"pageSize": null,
"scroll": true,
"showHeader": true,
"sort": {
"col": 1,
"desc": true
},
"styles": [
{
"alias": "Time",
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"pattern": "Time",
"type": "date"
},
{
"alias": "",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"decimals": 2,
"pattern": "/.*/",
"thresholds": [],
"type": "number",
"unit": "short"
}
],
"targets": [
{
"bucketAggs": [
{
"fake": true,
"field": "request.keyword",
"id": "3",
"settings": {
"min_doc_count": 1,
"order": "desc",
"orderBy": "_count",
"size": "10"
},
"type": "terms"
}
],
"expr": "",
"format": "table",
"intervalFactor": 1,
"metrics": [
{
"field": "select field",
"id": "1",
"type": "count"
}
],
"query": "http_host.keyword:$hostname AND status.keyword:$status AND upstream_addr.keyword:$upstream_addr AND department.department.keyword:$department",
"refId": "A",
"timeField": "@timestamp"
}
],
"title": "域名Request TOP 10",
"transform": "table",
"type": "table"
}
],
"title": "Row title",
"type": "row"
}
],
"refresh": false,
"schemaVersion": 16,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"allValue": null,
"current": {},
"datasource": "${DS_LOGSTASH-NGINXLOG}",
"hide": 0,
"includeAll": true,
"label": "業務",
"multi": true,
"name": "department",
"options": [],
"query": "{\"find\": \"terms\", \"field\": \"department.department.keyword\"}",
"refresh": 1,
"regex": "",
"sort": 2,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": null,
"current": {},
"datasource": "${DS_LOGSTASH-NGINXLOG}",
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "hostname",
"options": [],
"query": "{\"find\": \"terms\", \"field\": \"http_host.keyword\"}",
"refresh": 1,
"regex": "",
"sort": 3,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": null,
"current": {},
"datasource": "${DS_LOGSTASH-NGINXLOG}",
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "status",
"options": [],
"query": "{\"find\": \"terms\", \"field\": \"status.keyword\" ,\"query\": \"http_host:$hostname\"}",
"refresh": 1,
"regex": "",
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": null,
"current": {},
"datasource": "${DS_LOGSTASH-NGINXLOG}",
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "upstream_addr",
"options": [],
"query": "{\"find\": \"terms\", \"field\": \"upstream_addr.keyword\",\"query\": \"http_host:$hostname\"}",
"refresh": 1,
"regex": "",
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
}
]
},
"time": {
"from": "now-24h",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"timezone": "",
"title": "騰訊雲CLB日誌分析",
"uid": "7PJIMq5ik",
"version": 8
}