prometheus數據採集exporter全家桶

1. exporter介紹

exporter是prometheus監控中重要的組成部分，負責數據指標的採集。上篇文章介紹了prometheus server的相關內容，本文將介紹數據採集插件。官方給出的插件有node_exporter、blackbox_exporter、mysqld_exporter、snmp_exporter等，第三方的插件有redis_exporter，cadvisor等。
下面我將結合實際工作中的使用到的插件來分別介紹。

2. node_exporter

node_exporter主要用來採集機器的性能指標數據，包括cpu，內存，磁盤，io等基本信息。上邊文章介紹promehteus server時已詳細介紹了node_exporter，這裏就不在贅述。

3. mysqld_exporter

mysqld_exporter主要用於監控採集mysql數據庫服務器相關指標。
#下載二進制文件

root@db-100-51:~# cd /data/
root@db-100-51:/data# wget https://github.com/prometheus/mysqld_exporter/releases/download/v0.10.0/mysqld_exporter-0.10.0.linux-amd64.tar.gz
root@db-100-51:/data#  tar -xf mysqld_exporter-0.10.0.linux-amd64.tar.gz
root@db-100-51:/data# cd mysqld_exporter-0.10.0/
root@db-100-51:/data/mysqld_exporter-0.10.0# ls
LICENSE  mysqld_exporter  NOTICE
root@db-100-51:/data#  mkdir log

#mysqld_exporter需要連接到數據庫，創建一個登錄數據庫的用戶

mysql> GRANT PROCESS, REPLICATION CLIENT, SELECT ON *.* TO 'exporter'@'localhost' identified by Cds20180808!';
mysql> flush privileges;

#創建用於連接數據庫的配置文件

root@db-100-51:/data/mysqld_exporter-0.10.0# vim .my.cnf 
[client]
user=exporter
password=Cds20180808!

#創建supervisor啓動mysqld_exporter

root@db-100-51:/data/mysqld_exporter-0.10.0# vim /etc/supervisor/conf.d/mysqld_exporter.conf
[program:mysqld_exporter]
command = /data/mysqld_exporter-0.10.0/mysqld_exporter -config.my-cnf="/data/mysqld_exporter-0.10.0/.my.cnf"
autostart = true
autorestart = true
startsecs = 5
startretries = 3
redirect_stderr = true
stdout_logfile=/data/mysqld_exporter-0.10.0/log/out-mysqld_exporter.log
stderr_logfile=/data/mysqld_exporter-0.10.0/log/err-mysqld_exporter.log
stdout_logfile_maxbytes = 20MB
stdout_logfile_backups = 20

#啓動mysqld_exporter

root@db-100-51:/data/mysqld_exporter-0.10.0# supervisorctl update mysqld_exporter
root@db-100-51:/data/mysqld_exporter-0.10.0# supervisorctl status mysqld_exporter

4. redis_exporter

redis_exporter主要用於監控採集redis數據庫服務器相關指標。
#下載二進制文件

root@redis-node1-slave:~# cd /data/ wget https://github.com/oliver006/redis_exporter/releases/download/v0.24.0/redis_exporter-v0.24.0.linux-amd64.tar.gz
root@redis-node1-slave:/data# tar -xf redis_exporter-v0.24.0.linux-amd64.tar.gz
root@redis-node1-slave:/data# mkdir /data/redis_exporter-v0.24.0
root@redis-node1-slave:/data# mv redis_exporter redis_exporter-v0.24.0
root@redis-node1-slave:/data# cd redis_exporter-v0.24.0/
root@redis-node1-slave:/data/redis_exporter-v0.24.0# mkdir log

#配置supervisor啓動redis_exporter

root@redis-node1-slave:/data/redis_exporter-v0.24.0# vim /etc/supervisor/conf.d/redis_exporter.conf
[program:redis_exporter]
command = /data/redis_exporter-v0.24.0/redis_exporter -redis.addr 10.13.225.112:6379 -redis.password cds-china
autostart = true
startsecs = 5
startretries = 3
redirect_stderr = true
stout_logfile = /data/redis_exporter-v0.24.0/log/out-redis_exporter.log
stderr_logfile = /data/redis_exporter-v0.24.0/log/err-redis_exporter.log
stdout_logfile_maxbytes = 20MB
stdout_logfile_backups = 20

#啓動redis_exporter

root@redis-node1-slave:/data/redis_exporter-v0.24.0# supervisorctl update redis_exporter
root@redis-node1-slave:/data/redis_exporter-v0.24.0# supervisorctl status redis_exporter

5. black_exporter

black_exporter是prometheus社區提供的官方黑盒監控解決方案，其允許用戶通過：http、https、dns、tcp以及icmp的方式對網絡進行探測。我們利用icmp探針可以堅持網絡是否通暢，利用http，https可以堅持網頁是否可以正常訪問，利用tcp檢測服務端口判斷服務是否正常。
#下載二進制文件

root@prometheus:~# cd /data/
root@prometheus:/data# wget https://github.com/prometheus/blackbox_exporter/releases/download/v0.13.0/blackbox_exporter-0.13.0.linux-amd64.tar.gz
root@prometheus:/data# tar -xf blackbox_exporter-0.13.0.linux-amd64.tar.gz

#修改blackbox_exporter配置文件

root@prometheus:/data# cd blackbox_exporter-0.13.0/
root@prometheus:/data/blackbox_exporter-0.13.0# vim blackbox.yml 
modules:
  http_2xx:
    prober: http
    timeout: 5s
    http:
      preferred_ip_protocol: "ip4"
      no_follow_redirects: true
      valid_http_versions: ["HTTP/1.1", "HTTP/2"]
      valid_status_codes: [200,302]  # Defaults to 2xx
      method: GET
  http_post_2xx:
    prober: http
    http:
      method: POST
  tcp_connect:
    prober: tcp
    timeout: 5s
  pop3s_banner:
    prober: tcp
    tcp:
      query_response:
      - expect: "^+OK"
      tls: true
      tls_config:
        insecure_skip_verify: false
  ssh_banner:
    prober: tcp
    timeout: 5s
    tcp:
      query_response:
      - expect: "^SSH-2.0-"
  irc_banner:
    prober: tcp
    tcp:
      query_response:
      - send: "NICK prober"
      - send: "USER prober prober prober :prober"
      - expect: "PING :([^ ]+)"
        send: "PONG ${1}"
      - expect: "^:[^ ]+ 001"
  icmp:
    prober: icmp
    timeout: 5s

#配置systemd啓動blackbox_exporter

root@prometheus:/data/blackbox_exporter-0.13.0# vim /usr/lib/systemd/system/blackbox_exporter.service 

[Unit]
Description=Prometheus blackbox exporter
After=local-fs.target network-online.target network.target
Wants=local-fs.target network-online.target network.target

[Service]
User=root
Type=simple
WorkingDirectory=/data/blackbox_exporter-0.13.0/
ExecStart=/data/blackbox_exporter-0.13.0/blackbox_exporter --config.file=/data/blackbox_exporter-0.13.0/blackbox.yml
Restart=on-failure

[Install]
WantedBy=multi-user.target
root@prometheus:/data/blackbox_exporter-0.13.0# supervisorctl start blackbox_exporter 
root@prometheus:/data/blackbox_exporter-0.13.0# supervisorctl enable blackbox_exporter 
root@prometheus:/data/blackbox_exporter-0.13.0# supervisorctl status blackbox_exporter

6. cadvisor

cadvisor是google開源的用於監控容器運行的工具。
#下載二進制文件

root@GZH-vSPC02:~# cd /data/
root@GZH-vSPC02:/data# wget https://github.com/google/cadvisor/releases/download/v0.33.0/cadvisor
root@GZH-vSPC02:/data# mkdir cadvisor-v0.33.0
root@GZH-vSPC02:/data# mv cadvisor cadvisor-v0.33.0/
root@GZH-vSPC02:/data# cd cadvisor-v0.33.0/
root@GZH-vSPC02:/data/cadvisor-v0.33.0# mkdir log

#配置supervisor啓動cadvisor

root@GZH-vSPC02:/data/cadvisor-v0.33.0# vim /etc/supervisor/conf.d/cadvisor-server.conf 
[program:cadvisor-server]
command = /data/cadvisor-v0.33.0/cadvisor
autostart = true
autorestart = true
startsecs = 5
startretries = 3
redirect_stderr = true
stdout_logfile=/data/cadvisor-v0.33.0/log/out-cadvisor.log
stderr_logfile=/data/cadvisor-v0.33.0/log/err-cadvisor.log
stdout_logfile_maxbytes = 20MB
stdout_logfile_backups = 20
root@GZH-vSPC02:/data/cadvisor-v0.33.0# supervisorctl update cadvisor 
root@GZH-vSPC02:/data/cadvisor-v0.33.0# supervisorctl status cadvisor

7. 配置prometheus服務端

上面佈置了exporter，我們需要配置prometheus server來拿到exporter採集到數據。
#修改pormetheus配置文件，配置文件上篇文章已經介紹了，這裏就不在解釋

root@prometheus:~# cd /data/prometheus-2.4.3/
root@prometheus:/data/prometheus-2.4.3# vim prometheus.yml 
#my global config
global:
  scrape_interval:     30s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 25s # Evaluate rules every 15 seconds. The default is every 1 minute.
  scrape_timeout: 25s #is set to the global default (10s).

#Alertmanager configuration
alerting:
  alertmanagers:
  - static_configs:
    - targets:
      - 10.13.103.152:9093
      - 10.128.120.218:9093

#Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
  - "/data/prometheus-2.4.3/rules/node_down.yml"
  - "/data/prometheus-2.4.3/rules/memory_over.yml"
  - "/data/prometheus-2.4.3/rules/disk_over.yml"
  - "/data/prometheus-2.4.3/rules/cpu_over.yml"
  - "/data/prometheus-2.4.3/rules/http_check.yml"
  - "/data/prometheus-2.4.3/rules/tcp_check.yml"
  - "/data/prometheus-2.4.3/rules/mysql_check.yml"
  - "/data/prometheus-2.4.3/rules/redis_down_check.yml"
  - "/data/prometheus-2.4.3/rules/redis_rule_check.yml"
  - "/data/prometheus-2.4.3/rules/container_down.yml"
  - "/data/prometheus-2.4.3/rules/ping_check.yml"

#A scrape configuration containing exactly one endpoint to scrape:
#Here it's Prometheus itself.
scrape_configs:
  #The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: 'prometheus'

    #metrics_path defaults to '/metrics'
    #scheme defaults to 'http'.

    static_configs:
    - targets: ['localhost:9090']

  - job_name: 'GICHOST'
    file_sd_configs:
    - files: ['./node_exporter/host.json']

  - job_name: 'federate'
    scrape_interval: 30s
    scrape_timeout: 25s
    honor_labels: true
    metrics_path: '/federate'
    params:
      'match[]':
        - '{job=~"kubernetes-.*"}'
    static_configs:
      - targets:
        - '10.13.103.12:9090'

  - job_name: 'blackbox-http'
    scrape_interval: 5s
    metrics_path: /probe
    params:
      module: [http_2xx]  # Look for a HTTP 200 response.
    file_sd_configs:
    - files: ['./blackbox_exporter/http.json']
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: 10.13.103.151:9115

  - job_name: 'blackbox-tcp'
    scrape_interval: 5s
    metrics_path: /probe
    params:
      module: [tcp_connect]
    file_sd_configs:
    - files: ['./blackbox_exporter/tcp.json']
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: 10.13.103.151:9115

  - job_name: 'mysqld-exporter'
    file_sd_configs:
    - files: ['./mysqld_exporter/mysqld.json']

  - job_name: 'blackbox-ping'
    scrape_interval: 5s
    metrics_path: /probe
    params:
      module: [icmp]
    file_sd_configs:
    - files: ['./blackbox_exporter/ping.json']
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: 10.13.103.151:9115

  - job_name: 'redis-exporter'
    file_sd_configs:
    - files: ['./redis_exporter/redis.json']

  - job_name: 'container-exporter'
    file_sd_configs:
    - files: ['./container_exporter/container.json']

8. 配置prometheus 主機監控文件和告警規則

8.1 主機監控文件

#mysqld_exporter主機監控文件

root@prometheus:/data/prometheus-2.4.3# mkdir mysqld_exporter
root@prometheus:/data/prometheus-2.4.3# cd mysqld_exporter/
root@prometheus:/data/prometheus-2.4.3/mysqld_exporter# vim mysqld.json
[
{
"targets":[
"10.13.100.51:9104"
],
"labels":{
"dbinstance":"db100.51"
}
}
]

#redis_exporter主機監控文件

root@prometheus:/data/prometheus-2.4.3# mkdir redis_exporter
root@prometheus:/data/prometheus-2.4.3# cd redis_exporter/
root@prometheus:/data/prometheus-2.4.3/redis_exporter# vim redis.json 
[
{
"targets":[
"10.13.0.235:9121",
"10.13.0.236:9121"
],
"labels":{
"service":"redis"
}
}
]

#blackbox_exporter主機監控文件

root@prometheus:/data/prometheus-2.4.3# mkdir blackbox_exporter
root@prometheus:/data/prometheus-2.4.3# cd blackbox_exporter
root@prometheus:/data/prometheus-2.4.3/blackbox_exporter# vim http.json
[
{
"targets":[
"http://10.13.227.134:6011/health",
"http://10.13.102.134:6011/health"
],
"labels":{
"service":"WSREP"
}
}
]

root@prometheus:/data/prometheus-2.4.3/blackbox_exporter# vim tcp.json
[
{
"targets":[
"10.128.107.53:13371",
"10.128.107.57:13371"
],
"labels":{
"service":"vspc"
}
}
]

root@prometheus:/data/prometheus-2.4.3/blackbox_exporter# vim ping.json
[
{
"targets":[
"10.13.101.131",
"10.13.101.132",
"10.13.101.141"
],
"labels":{
"service":"mysql"
}
}
]

#配置cadvisor主機監控文件

root@prometheus:/data/prometheus-2.4.3# mkdir container_exporter
root@prometheus:/data/prometheus-2.4.3# cd container_exporter/
root@prometheus:/data/prometheus-2.4.3/container_exporter# vim container.json
[
{
"targets":[
"10.13.103.153:8080",
"10.128.87.5:8080"
],
"labels":{
"service":"docker-monitor"
}
}
]

8.2 告警規則

node_exporter監控獲取的cpu，磁盤，實例存活規則上篇文章已經介紹，這裏不再介紹

root@prometheus:/data/prometheus-2.4.3# cd rules
root@prometheus:/data/prometheus-2.4.3/rules# ls
container_down.yml  disk_over.yml  mysql_check.yml  node_down.yml   redis_down_check.yml  tcp_check.yml
cpu_over.yml  http_check.yml  memory_over.yml  ping_check.yml  redis_rule_check.yml

#mysqld_exporter監控mysql數據告警規則

root@prometheus:/data/prometheus-2.4.3/rules# vim mysql_check.yml
groups:
- name: MySQLStatsAlert
  rules:
  - alert: MySQL is down
    expr: mysql_up == 0
    for: 1m
    labels:
      severity: critical
    annotations:
      summary: "Instance {{$labels.instance}} MySQL is down"
      description: "MySQL database is down. This requires immediate action!(current value is: {{$value}})"
  - alert: Mysql_High_QPS
    expr: rate(mysql_global_status_questions[5m]) > 8000 
    for: 1m
    labels:
      severity: warning
    annotations:
      summary: "{{$labels.instance}}: Mysql_High_QPS detected"
      description: "{{$labels.instance}}: Mysql opreation is more than 5000 per second ,(current value is: {{$value}})" 
  - alert: Mysql_Too_Many_Slow_Query
    expr: rate(mysql_global_status_slow_queries[30m]) > 3
    for: 1m
    labels:
      severity: warning
    annotations:
      summary: "{{$labels.instance}}: Mysql_Too_Many_Slow_Query detected"
      description: "{{$labels.instance}}: Mysql current Slow_Query Sql is more than 3 ,(current value is: {{$value}})"
  - alert: Mysql_Deadlock
    expr: mysql_global_status_innodb_deadlocks > 300
    for: 1m
    labels:
      severity: warning
    annotations:
      summary: "{{$labels.instance}}: Mysql_Deadlock detected"
      description: "{{$labels.instance}}: Mysql Deadlock was found ,(current value is: {{$value}})"
  - alert: open files high
    expr: mysql_global_status_innodb_num_open_files > (mysql_global_variables_open_files_limit) * 0.75
    for: 1m
    labels:
      severity: warning
    annotations:
      summary: "Instance {{$labels.instance}} open files high"
      description: "Open files is high. Please consider increasing open_files_limit.(current value is: {{$value}})"
  - alert: Used more than 80% of max connections limited 
    expr: mysql_global_status_max_used_connections > mysql_global_variables_max_connections * 0.8
    for: 1m
    labels:
      severity: warning
    annotations:
      summary: "Instance {{$labels.instance}} Used more than 80% of max connections limited"
      description: "Used more than 80% of max connections limited.(current value is: {{$value}})"
  - alert: InnoDB Log File size is too small
    expr: mysql_global_variables_innodb_log_file_size < 16777216 
    for: 1m
    labels:
      severity: warning
    annotations:
      summary: "Instance {{$labels.instance}} InnoDB Log File size is too small"
      description: "The InnoDB Log File size is possibly too small. Choosing a small InnoDB Log File size can have significant performance impacts.(current value is: {{$value}})"
  - alert: Binary Log is disabled
    expr: mysql_global_variables_log_bin != 1
    for: 1m
    labels:
      severity: warning
    annotations:
      summary: "Instance {{$labels.instance}} Binary Log is disabled"
      description: "Binary Log is disabled. This prohibits you to do Point in Time Recovery (PiTR).(current value is: {{$value}})"
  - alert: IO thread stopped
    expr: mysql_slave_status_slave_io_running != 1
    for: 1m
    labels:
      severity: critical
    annotations:
      summary: "Instance {{$labels.instance}} IO thread stopped"
      description: "IO thread has stopped. This is usually because it cannot connect to the Master any more.(current value is: {{$value}})"
  - alert: SQL thread stopped
    expr: mysql_slave_status_slave_sql_running != 1
    for: 1m
    labels:
      severity: critical
    annotations:
      summary: "Instance {{$labels.instance}} Sync Binlog is enabled"
      description: "SQL thread has stopped. This is usually because it cannot apply a SQL statement received from the master.(current value is: {{$value}})"
  - alert: Slave lagging behind Master
    expr: rate(mysql_slave_status_seconds_behind_master[1m]) >30 
    for: 1m
    labels:
      severity: warning 
    annotations:
      summary: "Instance {{$labels.instance}} Slave lagging behind Master"
      description: "Slave is lagging behind Master. Please check if Slave threads are running and if there are some performance issues!(current value is: {{$value}})"

#redis_exporter監控redis指標告警規則

root@prometheus:/data/prometheus-2.4.3/rules# vim redis_down_check.yml
groups:
- name: redis檢測規則
  rules:
  - alert: redis存活檢測
    expr: redis_up{job="redis-exporter"} == 0
    for: 1m
    annotations:
      description: "機器:{{ $labels.instance }} 所屬 job:{{ $labels.job }} redis宕機，請檢查！"
      summary: "redis服務"

root@prometheus:/data/prometheus-2.4.3/rules# vim redis_rule_check.yml
    groups:
- name: RedisStatsAlert
  rules:
  - alert: last create rdb failed                       
    expr: redis_rdb_last_bgsave_status != 1
    for: 1m
    labels:
      severity: warning
    annotations:
      summary: " Instance {{ $labels.instance }} rdb_last_bgsave_status  "
      description: "last create rdb failed"
  - alert: Redis linked too many clients
    expr: redis_connected_clients / redis_config_maxclients * 100 > 80
    for: 1m
    labels:
      severity: warning
    annotations:
      summary: "Instance {{ $labels.instance }} Redis linked clients too many"
      description: "Redis linked clients too many. This requires immediate action!"
  - alert: master link status failed                                     
    expr: redis_master_link_up == 0
    for: 1m
    labels:
      severity: warning
    annotations:
      summary: "Instance {{ $labels.instance }} link failed"
      description: "redis_master_link=0 link failed"
  - alert: last AOF failed
    expr: redis_aof_last_bgrewrite_status != 1
    for: 1m
    labels:
      severity: warning
    annotations:
      summary: "Instance {{ $labels.instance }} redis aof last rewrite duration sec"
      description: "last AOF failed"
  - alert: Redis Cluster State Wrong
    expr: redis_cluster_state != 1
    for: 1m
    labels:
      severity: warning
    annotations:
      summary: "Instance {{ $labels.instance }} redis cluster status wrong"
      description: "Redis Cluster State Wrong"

#blackbox_exporter監控指標告警規則

root@prometheus:/data/prometheus-2.4.3/rules# vim ping_check.yml
groups:
- name: 機器網絡存活檢測
  rules:
  - alert: 網絡檢測
    expr: probe_success{job="blackbox-ping"} == 0
    for: 1m
    annotations:
      description: "機器:{{ $labels.instance }} 所屬 job:{{ $labels.job }} 網絡不通或者宕機超過1分鐘，請檢查！"
      summary: "網絡檢測"
root@prometheus:/data/prometheus-2.4.3/rules# vim http_check.yml
groups:
- name: 服務檢測規則
  rules:
  - alert: http服務檢測
    expr: probe_success{job="blackbox-http"} == 0
    for: 1m
    annotations:
      description: "機器:{{ $labels.instance }} 所屬 job:{{ $labels.job }} http狀態碼: {{ printf `probe_http_status_code{instance='%s'}` $labels.instance | query | first | value }} http檢測失敗，請檢查！"
      summary: "http檢測"
root@prometheus:/data/prometheus-2.4.3/rules# vim tcp_check.yml
groups:
- name: 服務檢測規則
  rules:
  - alert: tcp服務檢測
    expr: probe_success{job="blackbox-tcp"} == 0
    for: 1m
    annotations:
      description: "機器:{{ $labels.instance }} 所屬 job:{{ $labels.job }} tcp檢測失敗，請檢查！"
      summary: "tcp檢測"

#cadvisor監控指標告警規則

root@prometheus:/data/prometheus-2.4.3/rules# vim container_down.yml 

groups:
- name: 容器存活報警規則
  rules:
  - alert: DockerInstanceDown
    expr: absent(container_last_seen{name="core_vspc"}) == 1
    for: 1m
    annotations:
      description: "vspc client容器:{{ $labels.name }} (所屬主機{{ $labels.instance }}) 已經異常退出超過1分鐘，請檢查！"
      summary: "容器:Instance {{ $labels.name }} 存活檢測"

參考文檔：

https://github.com/prometheus
https://github.com/oliver006/redis_exporter
https://github.com/google/cadvisor

prometheus數據採集exporter全家桶

1. exporter介紹

2. node_exporter

3. mysqld_exporter

4. redis_exporter

5. black_exporter

6. cadvisor

7. 配置prometheus服務端

8. 配置prometheus 主機監控文件和告警規則

8.1 主機監控文件

8.2 告警規則

參考文檔：

Kafka存儲機制

aws語音呼叫調用，告警電話

【轉】[C#] WebAPI 防止併發調用二（冥等性）

HTTP URL 詳解

創新工具：2024年開發者必備的一款表格控件（二）

車牌識別控制檯可快速整合二次開發

金山雲api簽名（go語言）

linux工作利器之二，網絡分析工具tcpdump

linux網絡分析、性能分析、文本格式化、文件讀寫操作之利器(mtr、top、jq、sponge)

kubernetes高可用集羣（多master，v1.15官方最新版）

利用python爬取貝殼網租房信息

https://yachay.unat.edu.pe/blog/index.php?comment_area=format_blog&comment_component=blog&comment_co

linux以太網驅動總結