Kubernetes 監控體系基本結構

實驗環境

三節點 Kubernetes 1.17.2

• 192.168.220.110 kmaster
• 192.168.220.111 knode1
• 192.168.220.112 knode2

組件

• Prometheus：收集、查詢、分析、存儲監控數據，觸發告警，部署方式爲 deployment
• Grafana：圖形化展示監控數據，部署方式爲 deployment
• Alertmanager：發送告警通知，部署方式爲 deployment
• node-exporter：收集節點數據，部署方式爲 daemonset
• blackbox-exporter：黑盒監控非 kubernetes 服務，部署方式爲 deployment

我在自己的實驗環境部署的所有組件，每個組件的 configmap、deployment、service 都寫在一個 yaml 文件中一起啓停了，爲了避免多次下載鏡像我綁定了 node，存儲都配置的是宿主機本地存儲。Prometheus、Alertmanager 和 blackbox-exporter 的 configmap 最好單獨部署，這樣修改配置後向 /-/reload 端點 post 一下就可以重新加載配置而不用重啓 deployment。

部署兩個 service 的目的：-inner 用於組件間通信，-server用於外部訪問組件的web頁面。

全部組件運行在 monitoring 命名空間內。

apiVersion: v1
kind: Namespace
metadata:
   name: monitoring
   labels:
     name: monitoring

Prometheus

配置告警規則，數據採集參數，Alertmanager 地址。除了 Kubernetes 一些標準的監控對象，我額外通過 blackbox-exporter 監控了非Kubernetes 部署的一個 web 服務和一個 logstash 服務，告警的對象包括節點的基礎資源和 blackbox-exporter 的兩個監控對象。web 界面的 Targets 裏除了 kube-state-metrics 都 up 了，這個後面再研究。

apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-server-conf
  labels:
    name: prometheus-server-conf
  namespace: monitoring
data:
  prometheus.rules: |-
    groups:
    - name: web_probe
      rules:
       - alert: ProbeFailing
         expr: probe_success == 0
         # expr: up{job="blackbox"} == 0 or probe_success{job="blackbox"} == 0
         for: 10s
         labels:
          severity: critical
          team: blackbox
         annotations:
          summary: "網站/端口 {{ $labels.instance }} 失聯 {{ $value }}。"
          description: "Job {{$labels.job}} 中的 網站/端口 {{ $labels.instance }} 已失聯超過1分鐘。"
    - name: node_alert
      rules:
      - alert: HighNodeMemory
        expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 5
        # expr: (node_memory_Buffers_bytes+node_memory_Cached_bytes+node_memory_MemFree_bytes)/node_memory_MemTotal_bytes*100 > 90 
        for: 10s
        labels:
          severity: critical
          team: node
        annotations:
          summary: "Instance {{ $labels.instance }} 節點內存使用率超過95%"
          description: "{{ $labels.instance }} of job {{$labels.job}}內存使用率超過95%,當前使用率[{{ $value }}]."
      - alert: HighNodeCpuLoad
        expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
        for: 5m
        labels:
          severity: warning
          team: node
        annotations:
          summary: "(instance {{ $labels.instance }}) 節點CPU 5分鐘負載大於80%"
          description: "CPU load is > 80%\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
      - alert: OutOfRootDiskSpace
        expr: (node_filesystem_avail_bytes{mountpoint="/"}  * 100) / node_filesystem_size_bytes{mountpoint="/"} < 10
        for: 5m
        labels:
          severity: warning
          team: node
        annotations:
          summary: "(instance {{ $labels.instance }}) 節點根目錄剩餘空間小於10%"
          description: "Disk is almost full (< 10% left)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
      - alert: OutOfHomeDiskSpace
        expr: (node_filesystem_avail_bytes{mountpoint="/home"}  * 100) / node_filesystem_size_bytes{mountpoint="/home"} < 10
        for: 5m
        labels:
          severity: warning
          team: node
        annotations:
          summary: "(instance {{ $labels.instance }}) 節點家目錄剩餘空間小於10%"
          description: "Disk is almost full (< 10% left)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
      - alert: OutOfRootInodes
        expr: node_filesystem_files_free{mountpoint ="/"} / node_filesystem_files{mountpoint ="/"} * 100 < 10
        for: 5m
        labels:
          severity: warning
          team: node
        annotations:
          summary: "(instance {{ $labels.instance }}) 節點根目錄剩餘inode小於10%"
          description: "Disk is almost running out of available inodes (< 10% left)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
      - alert: OutOfHomeInodes
        expr: node_filesystem_files_free{mountpoint ="/home"} / node_filesystem_files{mountpoint ="/home"} * 100 < 10
        for: 5m
        labels:
          severity: warning
          team: node
        annotations:
          summary: "(instance {{ $labels.instance }}) 節點家目錄剩餘inode小於10%"
          description: "Disk is almost running out of available inodes (< 10% left)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

  prometheus.yml: |-
    global:
      scrape_interval: 5s
      evaluation_interval: 5s
    rule_files:
      - /etc/prometheus/prometheus.rules
    alerting:
      alertmanagers:
      - scheme: http
        static_configs:
        - targets: 
          - alertmanager-inner.monitoring.svc:9093
          # - 192.168.220.112:30002
    scrape_configs:
      - job_name: 'kubernetes-apiservers'
        kubernetes_sd_configs:
        - role: endpoints
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        relabel_configs:
        - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
          action: keep
          regex: default;kubernetes;https
      - job_name: 'kubernetes-nodes'
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        kubernetes_sd_configs:
        - role: node
        relabel_configs:
        - action: labelmap
          regex: __meta_kubernetes_node_label_(.+)
        - target_label: __address__
          replacement: kubernetes.default.svc:443
        - source_labels: [__meta_kubernetes_node_name]
          regex: (.+)
          target_label: __metrics_path__
          replacement: /api/v1/nodes/${1}/proxy/metrics
      
      - job_name: 'kubernetes-pods'
        kubernetes_sd_configs:
        - role: pod
        relabel_configs:
        - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
          action: keep
          regex: true
        - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
          action: replace
          target_label: __metrics_path__
          regex: (.+)
        - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
          action: replace
          regex: ([^:]+)(?::\d+)?;(\d+)
          replacement: $1:$2
          target_label: __address__
        - action: labelmap
          regex: __meta_kubernetes_pod_label_(.+)
        - source_labels: [__meta_kubernetes_namespace]
          action: replace
          target_label: kubernetes_namespace
        - source_labels: [__meta_kubernetes_pod_name]
          action: replace
          target_label: kubernetes_pod_name
      
      - job_name: 'kube-state-metrics'
        static_configs:
          - targets: ['kube-state-metrics.kube-system.svc.cluster.local:9090']

      - job_name: 'kubernetes-cadvisor'
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        kubernetes_sd_configs:
        - role: node
        relabel_configs:
        - action: labelmap
          regex: __meta_kubernetes_node_label_(.+)
        - target_label: __address__
          replacement: kubernetes.default.svc:443
        - source_labels: [__meta_kubernetes_node_name]
          regex: (.+)
          target_label: __metrics_path__
          replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
      
      - job_name: 'kubernetes-service-endpoints'
        kubernetes_sd_configs:
        - role: endpoints
        relabel_configs:
        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
          action: keep
          regex: true
        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
          action: replace
          target_label: __scheme__
          regex: (https?)
        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
          action: replace
          target_label: __metrics_path__
          regex: (.+)
        - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
          action: replace
          target_label: __address__
          regex: ([^:]+)(?::\d+)?;(\d+)
          replacement: $1:$2
        - action: labelmap
          regex: __meta_kubernetes_service_label_(.+)
        - source_labels: [__meta_kubernetes_namespace]
          action: replace
          target_label: kubernetes_namespace
        - source_labels: [__meta_kubernetes_service_name]
          action: replace
          target_label: kubernetes_name

      - job_name: 'web_status'
        scrape_interval: 5s
        metrics_path: /probe
        params:
          module: [http_2xx]
        static_configs:
            - targets:
              - https://www.baidu.com/
              - http://192.168.220.112:5000/health
        relabel_configs:
            - source_labels: [__address__]
              target_label: __param_target   
            - source_labels: [__param_target]
              target_label: instance         
            - target_label: __address__
              replacement: blackbox-exporter-inner.monitoring.svc:9115

      - job_name: "port_status"
        scrape_interval: 5s
        metrics_path: /probe
        params:
          module: [tcp_connect]
        static_configs:
            - targets: [ '192.168.220.111:30102' ]
        relabel_configs:
            - source_labels: [__address__]
              target_label: __param_target
            - source_labels: [__param_target]
              target_label: instance
            - target_label: __address__
              replacement: blackbox-exporter-inner.monitoring.svc:9115
---        
apiVersion: apps/v1
kind: Deployment
metadata:
  name: prometheus
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: prometheus-server
  template:
    metadata:
      labels:
        app: prometheus-server
    spec:
      nodeSelector:
        kubernetes.io/hostname: knode2
      containers:
        - name: prometheus
          image: prom/prometheus:v2.16.0
          imagePullPolicy: IfNotPresent
          args:
            - "--config.file=/etc/prometheus/prometheus.yml"
            - "--storage.tsdb.path=/prometheus/"
          ports:
            - containerPort: 9090
          volumeMounts:
            - name: prometheus-config-volume
              mountPath: /etc/prometheus/
            - name: prometheus-storage-volume
              mountPath: /prometheus/              
      volumes:
        - name: prometheus-config-volume
          configMap:
            defaultMode: 420
            name: prometheus-server-conf
        - name: prometheus-storage-volume
          emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
  name: prometheus-service
  namespace: monitoring
  annotations:
      prometheus.io/scrape: 'true'
      prometheus.io/port:   '9090'
spec:
  selector: 
    app: prometheus-server
  type: NodePort  
  ports:
    - port: 8080
      targetPort: 9090 
      nodePort: 30000

Grafana

apiVersion: apps/v1
kind: Deployment
metadata:
  name: grafana
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: grafana
  template:
    metadata:
      name: grafana
      annotations:
        prometheus.io/scrape: "true"
      labels:
        app: grafana
    spec:
      containers:
        - name: grafana
          image: grafana/grafana
          imagePullPolicy: IfNotPresent
          ports:
          - name: grafana
            containerPort: 3000
          env:
            # The following env variables set up basic auth twith the default admin user and admin password.
            - name: GF_AUTH_BASIC_ENABLED
              value: "true"
            - name: GF_AUTH_ANONYMOUS_ENABLED
              value: "false"
            # - name: GF_AUTH_ANONYMOUS_ORG_ROLE
            #   value: Admin
            # does not really work, because of template variables in exported dashboards:
            # - name: GF_DASHBOARDS_JSON_ENABLED
            #   value: "true"
          readinessProbe:
            httpGet:
              path: /login
              port: 3000
            # initialDelaySeconds: 30
            # timeoutSeconds: 1
          volumeMounts:
            - mountPath: /var/lib/grafana
              name: grafana-storage
      volumes:
        - name: grafana-storage
          emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
  name: grafana-service
  namespace: monitoring
spec:
  selector: 
    app: grafana
  type: NodePort
  selector:
    app: grafana
  ports:
    - port: 3000
      targetPort: 3000
      nodePort: 30001

啓動以後在grafana裏面配置prometheus源。

dashboard:

• 315 kubernetes cluster
• 7587 blackbox exporter

Node-exporter

apiVersion: v1
kind: Service
metadata:
  annotations:
    prometheus.io/scrape: 'true'
  labels:
    app: node-exporter
  name: node-exporter-inner
  namespace: monitoring
spec:
  clusterIP: None
  ports:
  - name: scrape
    port: 9100
    protocol: TCP
  selector:
    app: node-exporter
  type: ClusterIP
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: node-exporter
  namespace: monitoring
spec:
  selector:
    matchLabels:
      app: node-exporter
  template:
    metadata:
      labels:
        app: node-exporter
      name: node-exporter
    spec:
      hostNetwork: true
      hostPID: true
      hostIPC: true
      securityContext:
        runAsUser: 0
      containers:
      - image: prom/node-exporter
        imagePullPolicy: IfNotPresent 
        name: node-exporter
        volumeMounts:
          - mountPath: /run/systemd/private
            name: systemd-socket
            readOnly: true
        args:
          - "--collector.systemd"
          - "--collector.systemd.unit-whitelist=(docker|ssh|rsyslog|kubelet).service"
        ports:
          - containerPort: 9100
            hostPort: 9100
            name: scrape
        livenessProbe:
          httpGet:
            path: /metrics
            port: 9100
          initialDelaySeconds: 30
          timeoutSeconds: 10
          periodSeconds: 1
        readinessProbe:
          failureThreshold: 5
          httpGet:
            path: /metrics
            port: 9100
          initialDelaySeconds: 10
          timeoutSeconds: 10
          periodSeconds: 2
      volumes:
        - hostPath:  
            path: /run/systemd/private
          name: systemd-socket

Alertmanager

apiVersion: v1
kind: ConfigMap
metadata:
  name: alertmanager-conf
  namespace: monitoring
data:
  config.yml: |-
    global:
      # 在沒有告警的情況下聲明爲已解決的時間
      resolve_timeout: 1m
      # 配置郵件發送信息
      smtp_smarthost: 'smtp.163.com:25'
      smtp_from: '[email protected]'
      smtp_auth_username: '[email protected]'
      smtp_auth_password: 'sadfds'
      # smtp_hello: '163.com'
      smtp_require_tls: false
    # 所有報警信息進入後的根路由，用來設置告警的分發策略
    route:
      # 這裏的標籤列表是接收到報警信息後的重新分組標籤，
      # 例如，接收到的報警信息裏面有許多具有 cluster=A 和 alertname=LatncyHigh 
      # 這樣的標籤的報警信息將會批量被聚合到一個分組裏面。
      group_by: ['alertname']

      # 當觸發該組的一個告警後，需要等待至少group_wait時間來發送通知，
      # 這段時間內觸發的同組告警將一起發送通知，減少告警條數。
      group_wait: 2s

      # 當第一個報警發送後，等待'group_interval'時間後發送該組新告警通知。
      group_interval: 1s

      # 如果一個告警通知已經發送，等待'repeat_interval'時間後再次發送，
      # 減少重複發送條數。
      repeat_interval: 10s

      # 默認的receiver：如果一個告警沒有被一個route匹配，則發送給默認的接收器
      receiver: 'sms'

      # 以上屬性都由所有子路由繼承。
      routes:
      - receiver: 'sms'
        group_wait: 1s
        match:
          team: node
    receivers:
    - name: 'sms'
      webhook_configs:
      - url: 'http://192.168.220.112:5000/'
        send_resolved: false
    # - name: 'email'
    #   email_configs:
    #   - to: '[email protected]'
    #     send_resolved: true
---        
apiVersion: apps/v1
kind: Deployment
metadata:
  name: alertmanager
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: alertmanager-server
  template:
    metadata:
      labels:
        app: alertmanager-server
    spec:
      nodeSelector:
        kubernetes.io/hostname: knode2
      containers:        
        - name: alermanager
          image: prom/alertmanager:v0.20.0
          imagePullPolicy: IfNotPresent
          args:
          - "--config.file=/etc/alertmanager/config.yml"
          - "--storage.path=/alertmanager/data"
          ports:
          - containerPort: 9093
            name: http
          volumeMounts:
          - mountPath: "/etc/alertmanager"
            name: alertmanager-config-volume
          resources:
            requests:
              cpu: 100m
              memory: 256Mi
            limits:
              cpu: 100m
              memory: 256Mi
      volumes:
        - name: alertmanager-config-volume
          configMap:
            name: alertmanager-conf
---
apiVersion: v1
kind: Service
metadata:
  name: alertmanager-service
  namespace: monitoring
  annotations:
      prometheus.io/scrape: 'true'
      prometheus.io/port:   '9093'
spec:
  selector: 
    app: alertmanager-server
  type: NodePort  
  ports:
    - port: 9093
      targetPort: 9093 
      nodePort: 30002
---          
apiVersion: v1
kind: Service
metadata:
  name: alertmanager-inner
  namespace: monitoring
spec:
  clusterIP: None
  ports:
  - name: http
    port: 9093
  selector:
    app: alertmanager-server

說明：

• prometheus.rules 配置中設定的 labels 作爲 Alertmanager 的通知路由匹配規則，如team:node，annotations 是告警內容的說明，通常是發送的通知內容。
• Prometheus 的 alert 頁面中告警有三種狀態:
  • inactive：未觸發
  • pending：達到閾值，觸發告警，等待 group_wait 和 group_interval超時
  • firing：已將告警發送至 Alertmanager，只要告警沒有恢復，就持續在這個狀態
• 配置文件裏的 send_resolved 我本來理解應該是指告警恢復是否發送通知 ，但實際是指告警恢復後是否仍然發送，所以要配置成 false，否則告警恢復了還會繼續發送通知。
• 通過 webhook 接收器自定義通知方式，Alertmanager 會把通知信息 post 到 webhook_configs 配置的url，我們自己啓個服務接收通知然後處理，發微信、郵件、釘釘還是短信都可以自己寫。
• Alertmanager web界面自帶的 silence 功能比較簡單，只能根據 label 匹配設置固定的靜默時間，通過自定義接收器可以靈活設置靜默時間，比如每天的某一時段爲部署窗口。

我用 Go 寫了一個簡單的通知接收程序，將收到的通知打印出來

package main

import (
	"encoding/json"
	"fmt"
	"github.com/julienschmidt/httprouter"
	"io/ioutil"
	"log"
	"net/http"
)

type Msg struct {
	Tel []string `json:"tel"`
	Msg string   `json:"msg"`
}

type Alert struct {
	Labels      map[string]string `json:"labels"`
	Annotations map[string]string `json:"annotations"`
	StartsAt    string            `json:"startsAt"`
	EndsAt      string            `json:"endsAt"`
}

type Alerts struct {
	Version string  ` json:"version"`
	Status  string  `json:"status"`
	Alerts  []Alert `json:"alerts"`
}

func homePage(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	reqBody, _ := ioutil.ReadAll(r.Body)
	var alerts Alerts
	err := json.Unmarshal(reqBody, &alerts)
	if err != nil {
		log.Fatal("unmarshal request error: ", err)
	}

	log.Println(alerts)
	_, _ = fmt.Fprintln(w, alerts)
}

func healthCheck(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) {

	health := map[string]string{
		"status": "up",
	}
	_ = json.NewEncoder(w).Encode(health)
}

func handleRequests() {
	router := httprouter.New()

	router.POST("/", homePage)
	router.GET("/health", healthCheck)
	log.Fatal(http.ListenAndServe(":5000", router))
}

func main() {
	handleRequests()
}

blackbox-exporter

apiVersion: apps/v1
kind: Deployment
metadata:
  name: blackbox-exporter
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: blackbox-exporter
  template:
    metadata:
      labels:
        app: blackbox-exporter
    spec:
      nodeSelector:
        kubernetes.io/hostname: knode2
      containers:
        - name: blackbox-exporter
          image: prom/blackbox-exporter:v0.16.0
          imagePullPolicy: IfNotPresent
          ports:
          - containerPort: 9115
            name: http
---
apiVersion: v1
kind: Service
metadata:
  name: blackbox-exporter-inner
  namespace: monitoring
  annotations:
      prometheus.io/scrape: 'true'
      prometheus.io/port:   '9115'
spec:
  selector:
    app: blackbox-exporter
  clusterIP: None
  ports:
  - name: http
    port: 9115