k8s 集羣下微服務 pod 的各種指標信息監控

{"type":"doc","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"作者：Damon","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"博客：","attrs":{}},{"type":"link","attrs":{"href":"http://www.damon8.cn","title":"","type":null},"content":[{"type":"text","text":"http://www.damon8.cn","attrs":{}}]}]},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"程序猿Damon | 微服務 | 容器化 | 自動化","attrs":{}}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"今天主要分享下，在 k8s 集羣下，微服務的各種狀態指標情況的監控，我們都知道Prometheus是做數據監控的，但說白點，其獨特格式的數據，其實都是靠一些源來的，那麼這些源有哪些呢？已經有了cadvisor、heapster、metric-server，幾乎容器運行的所有指標都能拿到，但是下面這種情況卻無能爲力：","attrs":{}}]},{"type":"codeblock","attrs":{"lang":null},"content":[{"type":"text","text":"調度了N個replicas？現在可用的有 N 個？\nN 個 Pod 是 running/stopped/terminated 狀態？\nPod 重啓了N次？\n我有 N 個job在運行中\n","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"而這些則是 kube-state-metrics 提供的內容，它基於client-go開發，輪詢Kubernetes API，並將Kubernetes的結構化信息轉換爲metrics。kube-state-metrics是kubernetes開源的一個插件。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"廢話不多說，直接上教程。。。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"部署教程","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":5},"content":[{"type":"text","text":"下載","attrs":{}}]},{"type":"numberedlist","attrs":{"start":null,"normalizeStart":1},"content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":1,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"在官網 ","attrs":{}},{"type":"link","attrs":{"href":"https://github.com/kubernetes/kube-state-metrics","title":"","type":null},"content":[{"type":"text","text":"kube-state-metrics","attrs":{}}],"marks":[{"type":"color","attrs":{"color":"#1e6bb8","name":"user"}},{"type":"strong"}]},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":" 下載相應的源碼以及部署腳本，本次使用release1.9.7，即v1.9.7版本的 kube-state-metrics","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":2,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"執行 cd /kube-state-metrics/examples/standard，可以看到幾個文件：","attrs":{}}]}]}]},{"type":"codeblock","attrs":{"lang":null},"content":[{"type":"text","text":"cluster-role-binding.yaml\ncluster-role.yaml\ndeployment.yaml\nprometheus-configmap.yaml\nservice-account.yaml\nservice.yaml\n","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如果Prometheus已經部署，且部署在kube-system空間下，則源碼中的namespace不需更改，否則可自定義爲monitoring。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":5},"content":[{"type":"text","text":"更新","attrs":{}}]},{"type":"numberedlist","attrs":{"start":null,"normalizeStart":1},"content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":1,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"首先修改 service.yaml","attrs":{}}]}]}]},{"type":"codeblock","attrs":{"lang":null},"content":[{"type":"text","text":"apiVersion: v1\nkind: Service\nmetadata:\n  annotations:\n    prometheus.io/scrape: \"true\"\n  labels:\n    app.kubernetes.io/name: kube-state-metrics\n    app.kubernetes.io/version: v1.9.7\n  name: kube-state-metrics\n  namespace: kube-system\nspec:\n  clusterIP: None\n  ports:\n  - name: http-metrics\n    port: 8080\n    targetPort: http-metrics\n  - name: telemetry\n    port: 8081\n    targetPort: telemetry\n  selector:\n    app.kubernetes.io/name: kube-state-metrics\n","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"很簡單，增加了註解方便後面使用","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"坑：源碼中的角色授權綁定的是其寫的kind爲ClusterRole的資源，但後來發現部署kube-state-metrics服務時，其無法成功訪問k8s的api-server，故需要修改，棄用其ClusterRole，使用k8s系統最高權限cluster-admin。","attrs":{}}]},{"type":"numberedlist","attrs":{"start":"2","normalizeStart":"2"},"content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":2,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"更改訪問權限","attrs":{}}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"vi cluster-role-binding.yaml","attrs":{}}]},{"type":"codeblock","attrs":{"lang":null},"content":[{"type":"text","text":"apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n  labels:\n    app.kubernetes.io/name: kube-state-metrics\n    app.kubernetes.io/version: v1.9.7\n  name: kube-state-metrics\nroleRef:\n  apiGroup: rbac.authorization.k8s.io\n  kind: ClusterRole\n  name: cluster-admin #kube-state-metrics\nsubjects:\n- kind: ServiceAccount\n  name: kube-state-metrics\n  namespace: kube-system\n","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"部署","attrs":{}}]},{"type":"codeblock","attrs":{"lang":null},"content":[{"type":"text","text":"cd /kube-state-metrics/examples/standard\nkubectl create -f .\n","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"此時還需要更新Prometheus的掛載的configMap，因爲前面說了只抓取帶有prometheus.io/scrape: \"true\"註解的endpoint","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"vi prometheus-configmap.yaml","attrs":{}}]},{"type":"codeblock","attrs":{"lang":null},"content":[{"type":"text","text":"apiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: prometheus-config\n  namespace: kube-system\ndata:\n  prometheus.yaml: |\n    global:\n      scrape_interval:     15s\n      evaluation_interval: 15s\n    scrape_configs:\n    - job_name: 'kubernetes-apiservers'\n      kubernetes_sd_configs:\n      - role: endpoints\n      scheme: https\n      tls_config:\n        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\n      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\n      relabel_configs:\n      - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]\n        action: keep\n        regex: default;kubernetes;https\n\n    - job_name: 'kubernetes-nodes'\n      kubernetes_sd_configs:\n      - role: node\n      scheme: https\n      tls_config:\n        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\n      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\n      relabel_configs:\n      - action: labelmap\n        regex: __meta_kubernetes_node_label_(.+)\n      - target_label: __address__\n        replacement: kubernetes.default.svc:443\n      - source_labels: [__meta_kubernetes_node_name]\n        regex: (.+)\n        target_label: __metrics_path__\n        replacement: /api/v1/nodes/${1}/proxy/metrics\n\n    - job_name: 'kubernetes-cadvisor'\n      kubernetes_sd_configs:\n      - role: node\n      scheme: https\n      tls_config:\n        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\n      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\n      relabel_configs:\n      - action: labelmap\n        regex: __meta_kubernetes_node_label_(.+)\n      - target_label: __address__\n        replacement: kubernetes.default.svc:443\n      - source_labels: [__meta_kubernetes_node_name]\n        regex: (.+)\n        target_label: __metrics_path__\n        replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor\n\n    - job_name: 'kubernetes-service-endpoints'\n      kubernetes_sd_configs:\n      - role: endpoints\n      relabel_configs:\n      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]\n        action: keep\n        regex: true\n      - action: labelmap\n        regex: __meta_kubernetes_service_label_(.+)\n      - source_labels: [__meta_kubernetes_namespace]\n        action: replace\n        target_label: kubernetes_namespace\n      - source_labels: [__meta_kubernetes_service_name]\n        action: replace\n        target_label: service_name\n\n    - job_name: 'kubernetes-services'\n      kubernetes_sd_configs:\n      - role: service\n      metrics_path: /probe\n      params:\n        module: [http_2xx]\n      relabel_configs:\n      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]\n        action: keep\n        regex: true\n      - source_labels: [__address__]\n        target_label: __param_target\n      - target_label: __address__\n        replacement: blackbox-exporter.example.com:9115\n      - source_labels: [__param_target]\n        target_label: instance\n      - action: labelmap\n        regex: __meta_kubernetes_service_label_(.+)\n      - source_labels: [__meta_kubernetes_namespace]\n        target_label: kubernetes_namespace\n      - source_labels: [__meta_kubernetes_service_name]\n        target_label: kubernetes_name\n\n    - job_name: 'kubernetes-ingresses'\n      kubernetes_sd_configs:\n      - role: ingress\n      relabel_configs:\n      - source_labels: [__meta_kubernetes_ingress_annotation_prometheus_io_probe]\n        action: keep\n        regex: true\n      - source_labels: [__meta_kubernetes_ingress_scheme,__address__,__meta_kubernetes_ingress_path]\n        regex: (.+);(.+);(.+)\n        replacement: ${1}://${2}${3}\n        target_label: __param_target\n      - target_label: __address__\n        replacement: blackbox-exporter.example.com:9115\n      - source_labels: [__param_target]\n        target_label: instance\n      - action: labelmap\n        regex: __meta_kubernetes_ingress_label_(.+)\n      - source_labels: [__meta_kubernetes_namespace]\n        target_label: kubernetes_namespace\n      - source_labels: [__meta_kubernetes_ingress_name]\n        target_label: kubernetes_name\n\n    - job_name: 'kubernetes-pods'\n      kubernetes_sd_configs:\n      - role: pod\n      relabel_configs:\n      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]\n        action: keep\n        regex: true\n      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]\n        action: replace\n        target_label: __metrics_path__\n        regex: (.+)\n      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]\n        action: replace\n        regex: ([^:]+)(?::\\d+)?;(\\d+)\n        replacement: $1:$2\n        target_label: __address__\n      - action: labelmap\n        regex: __meta_kubernetes_pod_label_(.+)\n      - source_labels: [__meta_kubernetes_namespace]\n        action: replace\n        target_label: kubernetes_namespace\n      - source_labels: [__meta_kubernetes_pod_name]\n        action: replace\n        target_label: kubernetes_pod_name\n","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"更新 configmap 後，需要重啓 Prometheus 使其生效，如果沒部署，則創建 configmap 後執行腳本部署即可。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"導入模板","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"最後從 grafana.com 下載 state-metrics 監控模版","attrs":{}},{"type":"link","attrs":{"href":"https://grafana.com/grafana/dashboards/8588","title":"","type":null},"content":[{"type":"text","text":"導入模板","attrs":{}}],"marks":[{"type":"color","attrs":{"color":"#1e6bb8","name":"user"}},{"type":"strong"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"導入到 grafana 後，即可看到效果咯：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/6c/6c0bfdf90d8f630821cc3fc4e8061823.png","alt":null,"title":"","style":[{"key":"width","value":"100%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/88/88966e1cde86e1916ac6f91ff97e57a4.png","alt":null,"title":"","style":[{"key":"width","value":"100%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"結束福利","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"開源實戰利用 k8s 作微服務的架構設計代碼：","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://gitee.com/damon_one/spring-cloud-k8s","title":"","type":null},"content":[{"type":"text","text":"https://gitee.com/damon_one/spring-cloud-k8s","attrs":{}}]}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://github.com/damon008/spring-cloud-k8s","title":"","type":null},"content":[{"type":"text","text":"https://github.com/damon008/spring-cloud-k8s","attrs":{}}]}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://gitee.com/damon_one/spring-cloud-oauth2","title":"","type":null},"content":[{"type":"text","text":"https://gitee.com/damon_one/spring-cloud-oauth2","attrs":{}}]}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://github.com/damon008/spring-cloud-oauth2","title":"","type":null},"content":[{"type":"text","text":"https://github.com/damon008/spring-cloud-oauth2","attrs":{}}]}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://gitee.com/damon_one/Springcloud-Learning-Dalston","title":"","type":null},"content":[{"type":"text","text":"https://gitee.com/damon_one/Springcloud-Learning-Dalston","attrs":{}}]}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"歡迎大家 star","attrs":{}},{"type":"text","text":"，多多指教。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"關於作者","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"  ","attrs":{}},{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"筆名：Damon，技術愛好者，長期從事 Java 開發、Spring Cloud 的微服務架構設計，以及結合 Docker、K8s 做微服務容器化，自動化部署等一站式項目部署、落地。目前主要從事基於 K8s 雲原生架構研發的工作。Golang 語言開發，長期研究邊緣計算框架 KubeEdge、調度框架 Volcano 等。公衆號 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"程序猿Damon","attrs":{}}],"marks":[{"type":"italic"}],"attrs":{}},{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":" 發起人。個人微信 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"MrNull008","attrs":{}}],"marks":[{"type":"italic"}],"attrs":{}},{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"，個人網站：","attrs":{}},{"type":"link","attrs":{"href":"http://www.damon8.cn","title":"","type":null},"content":[{"type":"text","text":"Damon | Micro-Service | Containerization | DevOps","attrs":{}}],"marks":[{"type":"italic"},{"type":"color","attrs":{"color":"#1e6bb8","name":"user"}},{"type":"strong"}]},{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"，歡迎來撩。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"歡迎關注：","attrs":{}},{"type":"link","attrs":{"href":"https://www.infoq.cn/profile/1905020/following/user","title":"InfoQ","type":null},"content":[{"type":"text","text":"InfoQ","attrs":{}}],"marks":[{"type":"color","attrs":{"color":"#1e6bb8","name":"user"}},{"type":"strong"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"歡迎關注：","attrs":{}},{"type":"link","attrs":{"href":"https://cloud.tencent.com/developer/column/invite/533b1afb","title":"","type":null},"content":[{"type":"text","text":"騰訊自媒體專欄","attrs":{}}],"marks":[{"type":"color","attrs":{"color":"#1e6bb8","name":"user"}},{"type":"strong"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"精彩推薦","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://www.damon8.cn/2021/05/06/micro-cloud-native/","title":"","type":null},"content":[{"type":"text","text":"實戰 | 雲原生時代的微服務架構","attrs":{}}]}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://www.damon8.cn/2020/07/24/micro-service01/","title":"","type":null},"content":[{"type":"text","text":"淺談微服務安全架構設計","attrs":{}}]}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://www.damon8.cn/2020/07/27/mq-01/","title":"","type":null},"content":[{"type":"text","text":"消息中間件那點事兒","attrs":{}}]}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://xie.infoq.cn/article/fe449751ec689f4d4f97923f1","title":"","type":null},"content":[{"type":"text","text":"Spring Cloud 與 K8s 在微服務層面的不同","attrs":{}}]}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://xie.infoq.cn/article/637299b17ab089a5ad2a2ac8a","title":"","type":null},"content":[{"type":"text","text":"如何利用k8s拉取私有倉庫鏡像","attrs":{}}]}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://xie.infoq.cn/article/a69ef3df1b8de21c906a3a369","title":"","type":null},"content":[{"type":"text","text":"個站建立基礎教程","attrs":{}}]}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://www.damon8.cn/2020/07/23/core-java02/","title":"","type":null},"content":[{"type":"text","text":"淺談 Java 集合 | 底層源碼解析","attrs":{}}]}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://www.damon8.cn/2020/07/23/mysql-norm/","title":"","type":null},"content":[{"type":"text","text":"大佬整理的mysql規範，分享給大家","attrs":{}}]}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://www.damon8.cn/2020/07/14/ci-cd/","title":"","type":null},"content":[{"type":"text","text":"微服務自動化部署CI/CD","attrs":{}}]}]}]}],"attrs":{}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"歡迎關注","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/38/385163791aa296c4cc23be10afa5631d.jpeg","alt":null,"title":"","style":[{"key":"width","value":"25%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/86/860561234baac42a49dac2ea24239882.jpeg","alt":null,"title":"","style":[{"key":"width","value":"25%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/31/31b11ee840215b57487aa0269453e572.jpeg","alt":null,"title":"","style":[{"key":"width","value":"25%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}}]}