基於k8s的多節點部署

基於k8s的多節點部署

k8s多節點的部署

• 在master01上面操作：

[root@master1 ~]# cd /root/k8s/
[root@master1 k8s]# ls
  apiserver.sh                     k8s-cert
  cfssl.sh                         kubeconfig
  controller-manager.sh            kubernetes
  etcd-cert                        kubernetes-server-linux-amd64.tar.gz
  etcd.sh                          master.zip
  etcd-v3.3.10-linux-amd64         scheduler.sh
  etcd-v3.3.10-linux-amd64.tar.gz
  
[root@master1 k8s]# mkdir dashboard
[root@master1 k8s]# cd dashboard/

[root@master1 dashboard]# ls
dashboard-configmap.yaml   dashboard-rbac.yaml    dashboard-service.yaml
dashboard-controller.yaml  dashboard-secret.yaml  k8s-admin.yaml
注意：以上5個yaml文件是官方文檔， k8s-admin.yaml是我自己創建的

• 創建pod資源

kubectl run nginx --images=nginx
使用yaml文件進行創建：kubectl create -f yaml文件

首先，使用先創建dashboard-rbac.yaml
[root@master1 dashboard]# cat dashboard-rbac.yaml 
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
    addonmanager.kubernetes.io/mode: Reconcile
  name: kubernetes-dashboard-minimal
  namespace: kube-system
rules:
  # Allow Dashboard to get, update and delete Dashboard exclusive secrets.
- apiGroups: [""]
  resources: ["secrets"]
  resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs"]
  verbs: ["get", "update", "delete"]
  # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
- apiGroups: [""]
  resources: ["configmaps"]
  resourceNames: ["kubernetes-dashboard-settings"]
  verbs: ["get", "update"]
  # Allow Dashboard to get metrics from heapster.
- apiGroups: [""]
  resources: ["services"]
  resourceNames: ["heapster"]
  verbs: ["proxy"]
- apiGroups: [""]
  resources: ["services/proxy"]
  resourceNames: ["heapster", "http:heapster:", "https:heapster:"]
  verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: kubernetes-dashboard-minimal
  namespace: kube-system
  labels:
    k8s-app: kubernetes-dashboard
    addonmanager.kubernetes.io/mode: Reconcile
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: kubernetes-dashboard-minimal
subjects:
- kind: ServiceAccount
  name: kubernetes-dashboard
  namespace: kube-system

[root@master1 dashboard]# kubectl create -f dashboard-rbac.yaml    #創建角色
role.rbac.authorization.k8s.io/kubernetes-dashboard-minimal created
rolebinding.rbac.authorization.k8s.io/kubernetes-dashboard-minimal created

接下來創建dashboard-secret.yaml
[root@master1 dashboard]# cat dashboard-secret.yaml 
apiVersion: v1
kind: Secret
metadata:
  labels:
    k8s-app: kubernetes-dashboard
    # Allows editing resource and makes sure it is created first.
    addonmanager.kubernetes.io/mode: EnsureExists
  name: kubernetes-dashboard-certs
  namespace: kube-system
type: Opaque
---
apiVersion: v1
kind: Secret
metadata:
  labels:
    k8s-app: kubernetes-dashboard
    # Allows editing resource and makes sure it is created first.
    addonmanager.kubernetes.io/mode: EnsureExists
  name: kubernetes-dashboard-key-holder
  namespace: kube-system
type: Opaque

[root@master1 dashboard]# kubectl create -f dashboard-secret.yaml  #創建安全驗證
secret/kubernetes-dashboard-certs created
secret/kubernetes-dashboard-key-holder created 

其次，就是dashboard-configmap.yaml

[root@master1 dashboard]# cat dashboard-configmap.yaml 
apiVersion: v1
kind: ConfigMap
metadata:
  labels:
    k8s-app: kubernetes-dashboard
    # Allows editing resource and makes sure it is created first.
    addonmanager.kubernetes.io/mode: EnsureExists
  name: kubernetes-dashboard-settings
  namespace: kube-system

[root@master1 dashboard]# kubectl create -f dashboard-configmap.yaml   #創建配置
configmap/kubernetes-dashboard-settings created

然後，創建dashboard-controller.yaml
[root@master1 dashboard]# cat dashboard-controller.yaml 
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    k8s-app: kubernetes-dashboard
    addonmanager.kubernetes.io/mode: Reconcile
  name: kubernetes-dashboard
  namespace: kube-system
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: kubernetes-dashboard
  namespace: kube-system
  labels:
    k8s-app: kubernetes-dashboard
    kubernetes.io/cluster-service: "true"
    addonmanager.kubernetes.io/mode: Reconcile
spec:
  selector:
    matchLabels:
      k8s-app: kubernetes-dashboard
  template:
    metadata:
      labels:
        k8s-app: kubernetes-dashboard
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ''
        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
    spec:
      priorityClassName: system-cluster-critical
      containers:
      - name: kubernetes-dashboard
        image: siriuszg/kubernetes-dashboard-amd64:v1.8.3
        resources:
          limits:
            cpu: 100m
            memory: 300Mi
          requests:
            cpu: 50m
            memory: 100Mi
        ports:
        - containerPort: 8443
          protocol: TCP
        args:
          # PLATFORM-SPECIFIC ARGS HERE
          - --auto-generate-certificates
        volumeMounts:
        - name: kubernetes-dashboard-certs
          mountPath: /certs
        - name: tmp-volume
          mountPath: /tmp
        livenessProbe:
          httpGet:
            scheme: HTTPS
            path: /
            port: 8443
          initialDelaySeconds: 30
          timeoutSeconds: 30
      volumes:
      - name: kubernetes-dashboard-certs
        secret:
          secretName: kubernetes-dashboard-certs
      - name: tmp-volume
        emptyDir: {}
      serviceAccountName: kubernetes-dashboard
      tolerations:
      - key: "CriticalAddonsOnly"
        operator: "Exists"
        
[root@master1 dashboard]# kubectl create -f dashboard-controller.yaml
serviceaccount/kubernetes-dashboard created
deployment.apps/kubernetes-dashboard created

最後，創建dashboard-service.yaml
[root@master1 dashboard]# cat dashboard-service.yaml 
apiVersion: v1
kind: Service
metadata:
  name: kubernetes-dashboard
  namespace: kube-system
  labels:
    k8s-app: kubernetes-dashboard
    kubernetes.io/cluster-service: "true"
    addonmanager.kubernetes.io/mode: Reconcile
spec:
  type: NodePort
  selector:
    k8s-app: kubernetes-dashboard
  ports:
  - port: 443
    targetPort: 8443
    nodePort: 30001

[root@master1 dashboard]# kubectl create -f dashboard-service.yaml 
service/kubernetes-dashboard created

具體查看服務安裝信息
kubectl get kind名 -n 指定namesapce

• 查看創建的資源情況

[root@master1 dashboard]# kubectl get service -n kube-system
NAME                   TYPE       CLUSTER-IP   EXTERNAL-IP   PORT(S)         AGE
kubernetes-dashboard   NodePort   10.0.0.110   <none>        443:30001/TCP   5m13s
#可以看到對外提供訪問的端口是30001

• 查看資源創建在哪個node節點上

// -o wide可以查看所在的node節點
[root@master1 dashboard]# kubectl get pods -n kube-system -o wide
NAME                                    READY   STATUS    RESTARTS   AGE     IP           NODE            NOMINATED NODE
kubernetes-dashboard-65f974f565-6b98h   1/1     Running   0          7m33s   172.17.2.2   192.168.73.64   <none>

• 用谷歌瀏覽器訪問：https://192.168.73.64:30001/

按F12進入開發者模式

谷歌瀏覽器需要tls證書才能訪問，谷歌瀏覽器製作自簽證書

master01上操作
[root@localhost dashboard]# vim dashboard-cert.sh
  cat > dashboard-csr.json <<EOF
  {
     "CN": "Dashboard",
     "hosts": [],
     "key": {
         "algo": "rsa",
         "size": 2048
     },
     "names": [
         {
             "C": "CN",
             "L": "BeiJing",
             "ST": "BeiJing"
         }
     ]
  }
  EOF
  K8S_CA=$1
  cfssl gencert -ca=$K8S_CA/ca.pem -ca-key=$K8S_CA/ca-key.pem -config=$K8S_CA/ca-config.json -profile=kubernetes dashboard-csr.json | cfssljson -bare dashboard
  kubectl delete secret kubernetes-dashboard-certs -n kube-system
  kubectl create secret generic kubernetes-dashboard-certs --from-file=./ -n kube-system

• 生成兩個證書

[root@localhost dashboard]# bash dashboard-cert.sh /root/k8s/k8s-cert/
2020/05/08 10:47:04 [INFO] generate received request
2020/05/08 10:47:04 [INFO] received CSR
2020/05/08 10:47:04 [INFO] generating key: rsa-2048
2020/05/08 10:47:04 [INFO] encoded CSR
2020/05/08 10:47:04 [INFO] signed certificate with serial number 429560330602860800444046234554433478232541705712
2020/05/08 10:47:04 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
secret "kubernetes-dashboard-certs" deleted
secret/kubernetes-dashboard-certs created

[root@localhost dashboard]# vim dashboard-controller.yaml
在 args目錄下，添加生成的兩個證書的路徑：
 args:
          # PLATFORM-SPECIFIC ARGS HERE
          - --auto-generate-certificates
          - --tls-key-file=dashboard-key.pem
          - --tls-cert-file=dashboard.pem

• 重新部署服務

[root@localhost dashboard]#  kubectl apply -f dashboard-controller.yaml  #修改之後要更新
Warning: kubectl apply should be used on resource created by either kubectl create --save-config or kubectl apply
serviceaccount/kubernetes-dashboard configured
Warning: kubectl apply should be used on resource created by either kubectl create --save-config or kubectl apply
deployment.apps/kubernetes-dashboard configured
#重新部署很有可能把pod資源分配到另一個node節點上
[root@localhost dashboard]# kubectl get pods -n kube-system -o wide
[root@master1 dashboard]# kubectl get pods -n kube-system -o wide
NAME                                    READY   STATUS              RESTARTS   AGE   IP           NODE            NOMINATED NODE
kubernetes-dashboard-65f974f565-6b98h   1/1     Running             0          25m   172.17.2.2   192.168.73.64   <none>
kubernetes-dashboard-7dffbccd68-2gxmw   0/1     ContainerCreating   0          39s   <none>       192.168.73.63   <none>
[root@master1 dashboard]# kubectl get pods -n kube-system -o wide
NAME                                    READY   STATUS              RESTARTS   AGE   IP           NODE            NOMINATED NODE
kubernetes-dashboard-65f974f565-6b98h   1/1     Running             0          25m   172.17.2.2   192.168.73.64   <none>
kubernetes-dashboard-7dffbccd68-2gxmw   0/1     ContainerCreating   0          57s   <none>       192.168.73.63   <none>
[root@master1 dashboard]# kubectl get pods -n kube-system -o wide
NAME                                    READY   STATUS    RESTARTS   AGE   IP            NODE            NOMINATED NODE
kubernetes-dashboard-7dffbccd68-2gxmw   1/1     Running   0          89s   172.17.21.3   192.168.73.63   <none>
#從以上的過程可以看出，當我們重新部署dashboard-controller

• .再次訪問：https://192.168.73.63:30001/

輸入變化過後的IP地址

點擊高級，並且繼續前往

前往過後，點擊令牌

• 生成令牌

[root@master1 dashboard]# kubectl create -f k8s-admin.yaml
serviceaccount/dashboard-admin created
clusterrolebinding.rbac.authorization.k8s.io/dashboard-admin created

• k8s-admin.yaml文件內容

apiVersion: v1
kind: ServiceAccount
metadata:
  name: dashboard-admin
  namespace: kube-system
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: dashboard-admin
subjects:
  - kind: ServiceAccount
    name: dashboard-admin
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: cluster-admin
  apiGroup: rbac.authorization.k8s.io

• 查看生成的資源的名字

[root@master1 dashboard]# kubectl get secret -n kube-system
NAME                               TYPE                                  DATA   AGE
dashboard-admin-token-bx5kj        kubernetes.io/service-account-token   3      48s
default-token-twb85                kubernetes.io/service-account-token   3      2d23h
kubernetes-dashboard-certs         Opaque                                11     20m
kubernetes-dashboard-key-holder    Opaque                                2      44m
kubernetes-dashboard-token-f59fm   kubernetes.io/service-account-token   3      41m
查看令牌
[root@master1 dashboard]# kubectl describe secret dashboard-admin-token-bx5kj -n kube-system
Name:         dashboard-admin-token-bx5kj
Namespace:    kube-system
Labels:       <none>
Annotations:  kubernetes.io/service-account.name: dashboard-admin
              kubernetes.io/service-account.uid: f4887ff9-90d8-11ea-83dd-000c29c632b8

Type:  kubernetes.io/service-account-token

Data
====
ca.crt:     1359 bytes
namespace:  11 bytes
token:      eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJkYXNoYm9hcmQtYWRtaW4tdG9rZW4tYng1a2oiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoiZGFzaGJvYXJkLWFkbWluIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQudWlkIjoiZjQ4ODdmZjktOTBkOC0xMWVhLTgzZGQtMDAwYzI5YzYzMmI4Iiwic3ViIjoic3lzdGVtOnNlcnZpY2VhY2NvdW50Omt1YmUtc3lzdGVtOmRhc2hib2FyZC1hZG1pbiJ9.QzYmj70T3Nkg7lblMrZ1u6uHc6MJrMLj1oxMDxpD3G3mJIugALfiPMK2P1hA_zTuf8NkOvVCSgfkMjZ47t-FisqNbi3aA8d08DYAMmSEXMXLwtqlgtXtTh8k6_sedGxVE9tDekVZJvP8hHWQI5F4dnsl8RUEYkTgPsPK36gBaDJEJuM5OT-d2klgHneDpcySZnhxMBaBkvPW_QHkVYDimkh_J41JZW0ASOsyvOgN7Cvu6eK5Rlmo773ZOmOTKiOa2VZqKUx5NyKYMxfw6ag-RVA-4st5kAtEZeXm7Bw2nx4yNv1Rjeik3PKUAZZNGnzgoFxOf02XlRAngL3X9qCXwQ

• 將token令牌複製到剛剛的dashboard中的token登錄的地方，登錄過後就進入到k8s的web頁面

• 查看命名空間