kubernetes v1.18.2 二進制部署 ipv4 kube-apiserver 部署

簽發 kubernetes 證書

# 設置證書環境變量
# 設置證書使用時間87600h 10年
export EXPIRY_TIME="87600h"
# kube-apiserver 服務器IP 如果外部訪問K8s 集羣使用VIP ip 請在下面添加vip ip
export K8S_APISERVER_VIP="\"192.168.2.175\",\"192.168.2.176\",\"192.168.2.177\""
# kubernetes 服務 IP (一般是 SERVICE_CIDR 中第一個IP)
export CLUSTER_KUBERNETES_SVC_IP="10.66.0.1"
# 設置集羣參數
export CLUSTER_NAME=kubernetes
export KUBE_API=https://192.168.2.175:5443
# K8S 外部IP 這裏高可用使用本地環回IP
export K8S_VIP_DOMAIN=127.0.0.1
export K8S_SSL="\"${K8S_VIP_DOMAIN}\""
#證書所需要的配置參數
export CERT_ST="GuangDong"
export CERT_L="GuangZhou"
export CERT_O="k8s"
export CERT_OU="Qist"
export CERT_PROFILE="kubernetes"
# 生成 EncryptionConfig 所需的加密 key
export ENCRYPTION_KEY=$(head -c 32 /dev/urandom | base64)
# 設置工作目錄
export  HOST_PATH=`pwd`

# etcd 如果已經創建可以不用重複創建
# 創建etcd K8S 證書json 存放目錄
mkdir -p ${HOST_PATH}/cfssl/{k8s,etcd}
# 創建簽發證書存放目錄
mkdir -p ${HOST_PATH}/cfssl/pki/{k8s,etcd}
# CA 配置文件用於配置根證書的使用場景 (profile) 和具體參數 (usage，過期時間、服務端認證、客戶端認證、加密等)，後續在簽名其它證書時需要指定特定場景。
cat << EOF | tee ${HOST_PATH}/cfssl/ca-config.json
{
  "signing": {
    "default": {
      "expiry": "${EXPIRY_TIME}"
    },
    "profiles": {
      "${CERT_PROFILE}": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "${EXPIRY_TIME}"
      }
    }
  }
}
EOF
# 創建 Kubernetes CA 配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/k8s/k8s-ca-csr.json
{
  "CN": "$CLUSTER_NAME",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
            "C": "CN",
            "ST": "$CERT_ST",
            "L": "$CERT_L",
            "O": "$CERT_O",
            "OU": "$CERT_OU"
    }
  ],
 "ca": {
  "expiry": "${EXPIRY_TIME}"
  }
}
EOF
# 生成 Kubernetes CA 證書和私鑰
cfssl gencert -initca ${HOST_PATH}/cfssl/k8s/k8s-ca-csr.json | \
    cfssljson -bare ${HOST_PATH}/cfssl/pki/k8s/k8s-ca   
# 創建 Kubernetes API Server 配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/k8s/k8s-apiserver.json
{
  "CN": "$CLUSTER_NAME",
  "hosts": [
    ${K8S_APISERVER_VIP},
    "${CLUSTER_KUBERNETES_SVC_IP}", 
    ${K8S_SSL},
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.${CLUSTER_DNS_DOMAIN}"    
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
            "C": "CN",
            "ST": "$CERT_ST",
            "L": "$CERT_L",
            "O": "$CERT_O",
            "OU": "$CERT_OU"
    }
  ]
}
EOF
# 生成 Kubernetes API Server 證書和私鑰
cfssl gencert \
    -ca=${HOST_PATH}/cfssl/pki/k8s/k8s-ca.pem \
    -ca-key=${HOST_PATH}/cfssl/pki/k8s/k8s-ca-key.pem \
    -config=${HOST_PATH}/cfssl/ca-config.json \
    -profile=${CERT_PROFILE} \
    ${HOST_PATH}/cfssl/k8s/k8s-apiserver.json | \
    cfssljson -bare ${HOST_PATH}/cfssl/pki/k8s/k8s-server
# 創建 Kubernetes webhook 證書配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/k8s/aggregator.json
{
  "CN": "aggregator",
  "hosts": [""], 
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
            "C": "CN",
            "ST": "$CERT_ST",
            "L": "$CERT_L",
            "O": "$CERT_O",
            "OU": "$CERT_OU"
    }
  ]
}
EOF
# 生成 Kubernetes webhook 證書和私鑰
cfssl gencert \
    -ca=${HOST_PATH}/cfssl/pki/k8s/k8s-ca.pem \
    -ca-key=${HOST_PATH}/cfssl/pki/k8s/k8s-ca-key.pem \
    -config=${HOST_PATH}/cfssl/ca-config.json \
    -profile=${CERT_PROFILE} \
    ${HOST_PATH}/cfssl/k8s/aggregator.json | \
    cfssljson -bare ${HOST_PATH}/cfssl/pki/k8s/aggregator
# 創建admin管理員 配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/k8s/k8s-apiserver-admin.json
{
  "CN": "admin",
  "hosts": [""], 
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
            "ST": "$CERT_ST",
            "L": "$CERT_L",
      "O": "system:masters",
      "OU": "Kubernetes-manual"
    }
  ]
}
EOF
# 生成 Kubernetes admin管理員證書  
cfssl gencert -ca=${HOST_PATH}/cfssl/pki/k8s/k8s-ca.pem \
      -ca-key=${HOST_PATH}/cfssl/pki/k8s/k8s-ca-key.pem \
      -config=${HOST_PATH}/cfssl/ca-config.json \
      -profile=${CERT_PROFILE} \
      ${HOST_PATH}/cfssl/k8s/k8s-apiserver-admin.json | \
     cfssljson -bare ${HOST_PATH}/cfssl/pki/k8s/k8s-apiserver-admin
# 分發生成的證書到所有需要部署kube-apiserver 節點
ssh 192.168.2.175 mkdir -p /apps/k8s/ssl/{k8s,etcd}
ssh 192.168.2.176 mkdir -p /apps/k8s/ssl/{k8s,etcd}
ssh 192.168.2.177 mkdir -p /apps/k8s/ssl/{k8s,etcd}
# 分發文件
scp -r ./cfssl/pki/k8s/* 192.168.2.175:/apps/k8s/ssl/k8s
scp -r ./cfssl/pki/k8s/* 192.168.2.176:/apps/k8s/ssl/k8s
scp -r ./cfssl/pki/k8s/* 192.168.2.177:/apps/k8s/ssl/k8s
# etcd 客戶端連接證書分發
scp -r ./cfssl/pki/etcd/etcd-client* 192.168.2.175:/apps/k8s/ssl/etcd
scp -r ./cfssl/pki/etcd/etcd-client* 192.168.2.176:/apps/k8s/ssl/etcd
scp -r ./cfssl/pki/etcd/etcd-client* 192.168.2.177:/apps/k8s/ssl/etcd
scp -r ./cfssl/pki/etcd/etcd-ca.pem 192.168.2.175:/apps/k8s/ssl/etcd
scp -r ./cfssl/pki/etcd/etcd-ca.pem 192.168.2.176:/apps/k8s/ssl/etcd
scp -r ./cfssl/pki/etcd/etcd-ca.pem 192.168.2.177:/apps/k8s/ssl/etcd

kube-apiserver 二進制文件準備

wget https://storage.googleapis.com/kubernetes-release/release/v1.18.2/kubernetes-server-linux-amd64.tar.gz
# 解壓下載好文件
tar -xvf  kubernetes-server-linux-amd64.tar.gz
# 創建二進制遠程存放目錄
ssh 192.168.2.175 mkdir -p /apps/k8s/bin
ssh 192.168.2.176 mkdir -p /apps/k8s/bin
ssh 192.168.2.177 mkdir -p /apps/k8s/bin
# 分發解壓好二進制文件
cd kubernetes/server/bin/
scp -r kube-apiserver 192.168.2.175:/apps/k8s/bin
scp -r kube-apiserver 192.168.2.176:/apps/k8s/bin
scp -r kube-apiserver 192.168.2.177:/apps/k8s/bin

kube-apiserver 配置文件準備

# 生成encryption-config.yaml
cat << EOF | tee ${HOST_PATH}/encryption-config.yaml
kind: EncryptionConfig
apiVersion: v1
resources:
  - resources:
      - secrets
    providers:
      - aescbc:
          keys:
            - name: key1
              secret: ${ENCRYPTION_KEY}
      - identity: {}
EOF
# 創建審計策略文件
cat << EOF | tee ${HOST_PATH}/audit-policy.yaml
apiVersion: audit.k8s.io/v1beta1
kind: Policy
rules:
  # The following requests were manually identified as high-volume and low-risk, so drop them.
  - level: None
    resources:
      - group: ""
        resources:
          - endpoints
          - services
          - services/status
    users:
      - 'system:kube-proxy'
    verbs:
      - watch

  - level: None
    resources:
      - group: ""
        resources:
          - nodes
          - nodes/status
    userGroups:
      - 'system:nodes'
    verbs:
      - get

  - level: None
    namespaces:
      - kube-system
    resources:
      - group: ""
        resources:
          - endpoints
    users:
      - 'system:kube-controller-manager'
      - 'system:kube-scheduler'
      - 'system:serviceaccount:kube-system:endpoint-controller'
    verbs:
      - get
      - update

  - level: None
    resources:
      - group: ""
        resources:
          - namespaces
          - namespaces/status
          - namespaces/finalize
    users:
      - 'system:apiserver'
    verbs:
      - get

  # Don't log HPA fetching metrics.
  - level: None
    resources:
      - group: metrics.k8s.io
    users:
      - 'system:kube-controller-manager'
    verbs:
      - get
      - list

  # Don't log these read-only URLs.
  - level: None
    nonResourceURLs:
      - '/healthz*'
      - /version
      - '/swagger*'

  # Don't log events requests.
  - level: None
    resources:
      - group: ""
        resources:
          - events

  # node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes
  - level: Request
    omitStages:
      - RequestReceived
    resources:
      - group: ""
        resources:
          - nodes/status
          - pods/status
    users:
      - kubelet
      - 'system:node-problem-detector'
      - 'system:serviceaccount:kube-system:node-problem-detector'
    verbs:
      - update
      - patch

  - level: Request
    omitStages:
      - RequestReceived
    resources:
      - group: ""
        resources:
          - nodes/status
          - pods/status
    userGroups:
      - 'system:nodes'
    verbs:
      - update
      - patch

  # deletecollection calls can be large, don't log responses for expected namespace deletions
  - level: Request
    omitStages:
      - RequestReceived
    users:
      - 'system:serviceaccount:kube-system:namespace-controller'
    verbs:
      - deletecollection

  # Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data,
  # so only log at the Metadata level.
  - level: Metadata
    omitStages:
      - RequestReceived
    resources:
      - group: ""
        resources:
          - secrets
          - configmaps
      - group: authentication.k8s.io
        resources:
          - tokenreviews
  # Get repsonses can be large; skip them.
  - level: Request
    omitStages:
      - RequestReceived
    resources:
      - group: ""
      - group: admissionregistration.k8s.io
      - group: apiextensions.k8s.io
      - group: apiregistration.k8s.io
      - group: apps
      - group: authentication.k8s.io
      - group: authorization.k8s.io
      - group: autoscaling
      - group: batch
      - group: certificates.k8s.io
      - group: extensions
      - group: metrics.k8s.io
      - group: networking.k8s.io
      - group: policy
      - group: rbac.authorization.k8s.io
      - group: scheduling.k8s.io
      - group: settings.k8s.io
      - group: storage.k8s.io
    verbs:
      - get
      - list
      - watch

  # Default level for known APIs
  - level: RequestResponse
    omitStages:
      - RequestReceived
    resources:
      - group: ""
      - group: admissionregistration.k8s.io
      - group: apiextensions.k8s.io
      - group: apiregistration.k8s.io
      - group: apps
      - group: authentication.k8s.io
      - group: authorization.k8s.io
      - group: autoscaling
      - group: batch
      - group: certificates.k8s.io
      - group: extensions
      - group: metrics.k8s.io
      - group: networking.k8s.io
      - group: policy
      - group: rbac.authorization.k8s.io
      - group: scheduling.k8s.io
      - group: settings.k8s.io
      - group: storage.k8s.io

  # Default level for all other requests.
  - level: Metadata
    omitStages:
      - RequestReceived
EOF
# 創建遠程配置目錄
ssh 192.168.2.175 mkdir -p /apps/k8s/config
ssh 192.168.2.176 mkdir -p /apps/k8s/config
ssh 192.168.2.177 mkdir -p /apps/k8s/config
# 分發配置到遠程目錄
scp -r {audit-policy.yaml,encryption-config.yaml} 192.168.2.175:/apps/k8s/config
scp -r {audit-policy.yaml,encryption-config.yaml} 192.168.2.176:/apps/k8s/config
scp -r {audit-policy.yaml,encryption-config.yaml} 192.168.2.177:/apps/k8s/config
# 創建kube-apiserver 啓動配置文件
ssh 192.168.2.175 mkdir -p /apps/k8s/conf
ssh 192.168.2.176 mkdir -p /apps/k8s/conf
ssh 192.168.2.177 mkdir -p /apps/k8s/conf
# 192.168.2.175   配置
ssh 192.168.2.175 
cat << EOF | tee /apps/k8s/conf/kube-apiserver
KUBE_APISERVER_OPTS="--logtostderr=false \\
        --bind-address=0.0.0.0 \\
        --advertise-address=192.168.2.175 \\
        --secure-port=5443 \\
        --insecure-port=0 \\
        --service-cluster-ip-range=10.66.0.0/16 \\
        --service-node-port-range=30000-65535 \\
        --etcd-cafile=/apps/k8s/ssl/etcd/etcd-ca.pem \\
        --etcd-certfile=/apps/k8s/ssl/etcd/etcd-client.pem \\
        --etcd-keyfile=/apps/k8s/ssl/etcd/etcd-client-key.pem \\
        --etcd-prefix=/registry \\
        --etcd-servers=https://192.168.2.175:2379,https://192.168.2.176:2379,https://192.168.2.177:2379 \\
        --client-ca-file=/apps/k8s/ssl/k8s/k8s-ca.pem \\
        --tls-cert-file=/apps/k8s/ssl/k8s/k8s-server.pem \\
        --tls-private-key-file=/apps/k8s/ssl/k8s/k8s-server-key.pem \\
        --kubelet-client-certificate=/apps/k8s/ssl/k8s/k8s-server.pem \\
        --kubelet-client-key=/apps/k8s/ssl/k8s/k8s-server-key.pem \\
        --service-account-key-file=/apps/k8s/ssl/k8s/k8s-ca.pem \\
        --requestheader-client-ca-file=/apps/k8s/ssl/k8s/k8s-ca.pem \\
        --proxy-client-cert-file=/apps/k8s/ssl/k8s/aggregator.pem \\
        --proxy-client-key-file=/apps/k8s/ssl/k8s/aggregator-key.pem \\
        --requestheader-allowed-names=aggregator \\
        --requestheader-group-headers=X-Remote-Group \\
        --requestheader-extra-headers-prefix=X-Remote-Extra- \\
        --requestheader-username-headers=X-Remote-User \\
        --enable-aggregator-routing=true \\
        --anonymous-auth=false \\
        --experimental-encryption-provider-config=/apps/k8s/config/encryption-config.yaml \\
        --enable-admission-plugins=DefaultStorageClass,DefaultTolerationSeconds,LimitRanger,NamespaceExists,NamespaceLifecycle,NodeRestriction,PodNodeSelector,PersistentVolumeClaimResize,PodPreset,PodTolerationRestriction,ResourceQuota,ServiceAccount,StorageObjectInUseProtection,MutatingAdmissionWebhook,ValidatingAdmissionWebhook \\
        --disable-admission-plugins=DenyEscalatingExec,ExtendedResourceToleration,ImagePolicyWebhook,LimitPodHardAntiAffinityTopology,NamespaceAutoProvision,Priority,EventRateLimit,PodSecurityPolicy \\
        --cors-allowed-origins=.* \\
        --enable-swagger-ui \\
        --runtime-config=api/all=true \\
        --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \\
        --authorization-mode=Node,RBAC \\
        --allow-privileged=true \\
        --apiserver-count=1 \\
        --audit-dynamic-configuration \\
        --audit-log-maxage=30 \\
        --audit-log-maxbackup=3 \\
        --audit-log-maxsize=100 \\
        --audit-log-truncate-enabled \\
        --audit-policy-file=/apps/k8s/config/audit-policy.yaml \\
        --audit-log-path=/apps/k8s/log/api-server-audit.log \\
        --profiling \\
        --kubelet-https \\
        --event-ttl=1h \\
        --feature-gates=DynamicAuditing=true,ServiceTopology=true,EndpointSlice=true \\
        --enable-bootstrap-token-auth=true \\
        --alsologtostderr=true \\
        --log-dir=/apps/k8s/log \\
        --v=2 \\
        --endpoint-reconciler-type=lease \\
        --max-mutating-requests-inflight=500 \\
        --max-requests-inflight=1500 \\
        --target-ram-mb=600"
EOF
# 192.168.2.176 配置
ssh 192.168.2.176
cat << EOF | tee /apps/k8s/conf/kube-apiserver
KUBE_APISERVER_OPTS="--logtostderr=false \\
        --bind-address=0.0.0.0 \\
        --advertise-address=192.168.2.176 \\
        --secure-port=5443 \\
        --insecure-port=0 \\
        --service-cluster-ip-range=10.66.0.0/16 \\
        --service-node-port-range=30000-65535 \\
        --etcd-cafile=/apps/k8s/ssl/etcd/etcd-ca.pem \\
        --etcd-certfile=/apps/k8s/ssl/etcd/etcd-client.pem \\
        --etcd-keyfile=/apps/k8s/ssl/etcd/etcd-client-key.pem \\
        --etcd-prefix=/registry \\
        --etcd-servers=https://192.168.2.175:2379,https://192.168.2.176:2379,https://192.168.2.177:2379 \\
        --client-ca-file=/apps/k8s/ssl/k8s/k8s-ca.pem \\
        --tls-cert-file=/apps/k8s/ssl/k8s/k8s-server.pem \\
        --tls-private-key-file=/apps/k8s/ssl/k8s/k8s-server-key.pem \\
        --kubelet-client-certificate=/apps/k8s/ssl/k8s/k8s-server.pem \\
        --kubelet-client-key=/apps/k8s/ssl/k8s/k8s-server-key.pem \\
        --service-account-key-file=/apps/k8s/ssl/k8s/k8s-ca.pem \\
        --requestheader-client-ca-file=/apps/k8s/ssl/k8s/k8s-ca.pem \\
        --proxy-client-cert-file=/apps/k8s/ssl/k8s/aggregator.pem \\
        --proxy-client-key-file=/apps/k8s/ssl/k8s/aggregator-key.pem \\
        --requestheader-allowed-names=aggregator \\
        --requestheader-group-headers=X-Remote-Group \\
        --requestheader-extra-headers-prefix=X-Remote-Extra- \\
        --requestheader-username-headers=X-Remote-User \\
        --enable-aggregator-routing=true \\
        --anonymous-auth=false \\
        --experimental-encryption-provider-config=/apps/k8s/config/encryption-config.yaml \\
        --enable-admission-plugins=DefaultStorageClass,DefaultTolerationSeconds,LimitRanger,NamespaceExists,NamespaceLifecycle,NodeRestriction,PodNodeSelector,PersistentVolumeClaimResize,PodPreset,PodTolerationRestriction,ResourceQuota,ServiceAccount,StorageObjectInUseProtection,MutatingAdmissionWebhook,ValidatingAdmissionWebhook \\
        --disable-admission-plugins=DenyEscalatingExec,ExtendedResourceToleration,ImagePolicyWebhook,LimitPodHardAntiAffinityTopology,NamespaceAutoProvision,Priority,EventRateLimit,PodSecurityPolicy \\
        --cors-allowed-origins=.* \\
        --enable-swagger-ui \\
        --runtime-config=api/all=true \\
        --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \\
        --authorization-mode=Node,RBAC \\
        --allow-privileged=true \\
        --apiserver-count=1 \\
        --audit-dynamic-configuration \\
        --audit-log-maxage=30 \\
        --audit-log-maxbackup=3 \\
        --audit-log-maxsize=100 \\
        --audit-log-truncate-enabled \\
        --audit-policy-file=/apps/k8s/config/audit-policy.yaml \\
        --audit-log-path=/apps/k8s/log/api-server-audit.log \\
        --profiling \\
        --kubelet-https \\
        --event-ttl=1h \\
        --feature-gates=DynamicAuditing=true,ServiceTopology=true,EndpointSlice=true \\
        --enable-bootstrap-token-auth=true \\
        --alsologtostderr=true \\
        --log-dir=/apps/k8s/log \\
        --v=2 \\
        --endpoint-reconciler-type=lease \\
        --max-mutating-requests-inflight=500 \\
        --max-requests-inflight=1500 \\
        --target-ram-mb=600"
EOF
# 192.168.2.177 配置
ssh 192.168.2.177
cat << EOF | tee /apps/k8s/conf/kube-apiserver
KUBE_APISERVER_OPTS="--logtostderr=false \\
        --bind-address=0.0.0.0 \\
        --advertise-address=192.168.2.177 \\
        --secure-port=5443 \\
        --insecure-port=0 \\
        --service-cluster-ip-range=10.66.0.0/16 \\
        --service-node-port-range=30000-65535 \\
        --etcd-cafile=/apps/k8s/ssl/etcd/etcd-ca.pem \\
        --etcd-certfile=/apps/k8s/ssl/etcd/etcd-client.pem \\
        --etcd-keyfile=/apps/k8s/ssl/etcd/etcd-client-key.pem \\
        --etcd-prefix=/registry \\
        --etcd-servers=https://192.168.2.175:2379,https://192.168.2.176:2379,https://192.168.2.177:2379 \\
        --client-ca-file=/apps/k8s/ssl/k8s/k8s-ca.pem \\
        --tls-cert-file=/apps/k8s/ssl/k8s/k8s-server.pem \\
        --tls-private-key-file=/apps/k8s/ssl/k8s/k8s-server-key.pem \\
        --kubelet-client-certificate=/apps/k8s/ssl/k8s/k8s-server.pem \\
        --kubelet-client-key=/apps/k8s/ssl/k8s/k8s-server-key.pem \\
        --service-account-key-file=/apps/k8s/ssl/k8s/k8s-ca.pem \\
        --requestheader-client-ca-file=/apps/k8s/ssl/k8s/k8s-ca.pem \\
        --proxy-client-cert-file=/apps/k8s/ssl/k8s/aggregator.pem \\
        --proxy-client-key-file=/apps/k8s/ssl/k8s/aggregator-key.pem \\
        --requestheader-allowed-names=aggregator \\
        --requestheader-group-headers=X-Remote-Group \\
        --requestheader-extra-headers-prefix=X-Remote-Extra- \\
        --requestheader-username-headers=X-Remote-User \\
        --enable-aggregator-routing=true \\
        --anonymous-auth=false \\
        --experimental-encryption-provider-config=/apps/k8s/config/encryption-config.yaml \\
        --enable-admission-plugins=DefaultStorageClass,DefaultTolerationSeconds,LimitRanger,NamespaceExists,NamespaceLifecycle,NodeRestriction,PodNodeSelector,PersistentVolumeClaimResize,PodPreset,PodTolerationRestriction,ResourceQuota,ServiceAccount,StorageObjectInUseProtection,MutatingAdmissionWebhook,ValidatingAdmissionWebhook \\
        --disable-admission-plugins=DenyEscalatingExec,ExtendedResourceToleration,ImagePolicyWebhook,LimitPodHardAntiAffinityTopology,NamespaceAutoProvision,Priority,EventRateLimit,PodSecurityPolicy \\
        --cors-allowed-origins=.* \\
        --enable-swagger-ui \\
        --runtime-config=api/all=true \\
        --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \\
        --authorization-mode=Node,RBAC \\
        --allow-privileged=true \\
        --apiserver-count=1 \\
        --audit-dynamic-configuration \\
        --audit-log-maxage=30 \\
        --audit-log-maxbackup=3 \\
        --audit-log-maxsize=100 \\
        --audit-log-truncate-enabled \\
        --audit-policy-file=/apps/k8s/config/audit-policy.yaml \\
        --audit-log-path=/apps/k8s/log/api-server-audit.log \\
        --profiling \\
        --kubelet-https \\
        --event-ttl=1h \\
        --feature-gates=DynamicAuditing=true,ServiceTopology=true,EndpointSlice=true \\
        --enable-bootstrap-token-auth=true \\
        --alsologtostderr=true \\
        --log-dir=/apps/k8s/log \\
        --v=2 \\
        --endpoint-reconciler-type=lease \\
        --max-mutating-requests-inflight=500 \\
        --max-requests-inflight=1500 \\
        --target-ram-mb=600"
EOF

創建apiserver systemd文件

cat << EOF | tee kube-apiserver.service
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes

[Service]
Type=notify
LimitNOFILE=65535
LimitNPROC=65535
LimitCORE=infinity
LimitMEMLOCK=infinity

EnvironmentFile=-/apps/k8s/conf/kube-apiserver
ExecStart=/apps/k8s/bin/kube-apiserver \$KUBE_APISERVER_OPTS
Restart=on-failure
RestartSec=5
User=k8s 
[Install]
WantedBy=multi-user.target
EOF
# 上傳啓動文件到服務器
scp kube-apiserver.service 192.168.2.175:/usr/lib/systemd/system
scp kube-apiserver.service 192.168.2.176:/usr/lib/systemd/system
scp kube-apiserver.service 192.168.2.176:/usr/lib/systemd/system

kube-apiserver 啓動準備

# 創建文件目錄
ssh 192.168.2.175 mkdir -p /apps/k8s/{log,kubelet-plugins,conf} && mkdir -p /apps/k8s/kubelet-plugins/volume
ssh 192.168.2.176 mkdir -p /apps/k8s/{log,kubelet-plugins,conf} && mkdir -p /apps/k8s/kubelet-plugins/volume
ssh 192.168.2.177 mkdir -p /apps/k8s/{log,kubelet-plugins,conf} && mkdir -p /apps/k8s/kubelet-plugins/volume
# 創建k8s 用戶
ssh  192.168.2.175 useradd k8s -s /sbin/nologin -M
ssh  192.168.2.176 useradd k8s -s /sbin/nologin -M
ssh  192.168.2.177 useradd k8s -s /sbin/nologin -M
# 給/apps/k8s k8s 用戶權限
ssh  192.168.2.175 chown -R k8s:root /apps/k8s
ssh  192.168.2.176 chown -R k8s:root /apps/k8s
ssh  192.168.2.177 chown -R k8s:root /apps/k8s

kube-apiserver 啓動

# 刷新service
ssh  192.168.2.175 systemctl daemon-reload
ssh  192.168.2.176 systemctl daemon-reload
ssh  192.168.2.177 systemctl daemon-reload
# 設置開機啓動
ssh  192.168.2.175 systemctl enable kube-apiserver.service
ssh  192.168.2.176 systemctl enable kube-apiserver.service 
ssh  192.168.2.177 systemctl enable kube-apiserver.service 
# 啓動 kube-apiserver
ssh  192.168.2.175 systemctl  start kube-apiserver.service
ssh  192.168.2.176 systemctl  start kube-apiserver.service
ssh  192.168.2.177 systemctl  start kube-apiserver.service
# 查看啓動狀態
ssh  192.168.2.175 systemctl  status kube-apiserver.service
ssh  192.168.2.176 systemctl  status kube-apiserver.service 
ssh  192.168.2.177 systemctl  status kube-apiserver.service

驗證kube-apiserver 是否啓動成功

#創建 kubeconfig 文件夾
mkdir -p ${HOST_PATH}/kubeconfig
# 創建admin管理員登錄kubeconfig
# 設置集羣參數
kubectl config set-cluster ${CLUSTER_NAME} \
--certificate-authority=${HOST_PATH}/cfssl/pki/k8s/k8s-ca.pem \
--embed-certs=true  \
--server=${KUBE_API} \
--kubeconfig=${HOST_PATH}/kubeconfig/admin.kubeconfig
# 設置客戶端認證參數
kubectl config set-credentials admin \
 --client-certificate=${HOST_PATH}/cfssl/pki/k8s/k8s-apiserver-admin.pem \
 --client-key=${HOST_PATH}/cfssl/pki/k8s/k8s-apiserver-admin-key.pem \
 --embed-certs=true \
 --kubeconfig=${HOST_PATH}/kubeconfig/admin.kubeconfig
# 設置上下文參數 
kubectl config set-context ${CLUSTER_NAME} \
--cluster=${CLUSTER_NAME} \
--user=admin \
--namespace=kube-system \
--kubeconfig=${HOST_PATH}/kubeconfig/admin.kubeconfig
# 設置默認上下文
kubectl config use-context ${CLUSTER_NAME} --kubeconfig=${HOST_PATH}/kubeconfig/admin.kubeconfig
# 創建當前家目錄.kube 目錄
mkdir -p ~/.kube
cp ${HOST_PATH}/kubeconfig/admin.kubeconfig ~/.kube/config
# cp 二進制kubectl 到 /bin 目錄
cp kubectl  /bin
# 驗證集羣
root@Qist:/tmp/sss# kubectl get cs
NAME                 STATUS      MESSAGE                                                                                     ERROR
controller-manager   Unhealthy   Get http://127.0.0.1:10252/healthz: dial tcp 127.0.0.1:10252: connect: connection refused
scheduler            Unhealthy   Get http://127.0.0.1:10251/healthz: dial tcp 127.0.0.1:10251: connect: connection refused
etcd-1               Healthy     {"health":"true"}
etcd-0               Healthy     {"health":"true"}
etcd-2               Healthy     {"health":"true"}
root@Qist:/tmp/sss# kubectl cluster-info
Kubernetes master is running at https://192.168.2.175:5443

To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'.
集羣能夠正常訪問 kube-apiserver 部署正常