一、服務器環境準備
二、服務器環境初始化(在所有節點操作)
1、修改 /etc/hosts 文件
[root@localhost ~]# cat >> /etc/hosts <<EOF
192.168.10.102 master
192.168.10.103 node1
192.168.10.104 node2
192.168.10.105 node3
192.168.10.106 node4
EOF
2、關閉 selinux
[root@localhost ~]# sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config
3、關閉系統防火牆
[root@master ~]# systemctl stop firewalld
[root@master ~]# systemctl disable firewalld
Removed symlink /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.
[root@master ~]# systemctl mask firewalld
Created symlink from /etc/systemd/system/firewalld.service to /dev/null.
4、配置 kubernetes yum源
[root@master ~]# cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
EOF
5、關閉系統swap(Kubernetes 1.8開始要求)
[root@master ~]# swapoff -a
[root@master ~]# yes | cp /etc/fstab /etc/fstab_bak
[root@master ~]# cat /etc/fstab_bak |grep -v swap > /etc/fstab
6、同步時間
[root@master ~]# ntpdate -u ntp.api.bz
7、升級系統內核到最新
[root@master ~]# grub2-set-default 0 && grub2-mkconfig -o /etc/grub2.cfg
[root@master ~]# grubby --default-kernel
[root@master ~]# grubby --args="user_namespace.enable=1" --update-kernel="$(grubby --default-kernel)"
重啓系統後確認內核是否爲最新,確認內核版本後開啓IPVS(如果內核未升級,刪掉ip_vs_fo)
[root@master ~]# uname -a
Linux master 3.10.0-957.21.3.el7.x86_64 #1 SMP Tue Jun 18 16:35:19 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux
[root@master ~]# cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
ipvs_modules="ip_vs ip_vs_lc ip_vs_wlc ip_vs_rr ip_vs_wrr ip_vs_lblc ip_vs_lblcr ip_vs_dh ip_vs_sh ip_vs_fo ip_vs_nq ip_vs_sed ip_vs_ftp nf_conntrack"
for kernel_module in \${ipvs_modules}; do
/sbin/modinfo -F filename \${kernel_module} > /dev/null 2>&1
if [ $? -eq 0 ]; then
/sbin/modprobe \${kernel_module}
fi
done
EOF
[root@master ~]# chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep ip_vs
[root@master ~]# sysctl -p
8、所有服務器配置 k8s 內核參數
[root@master ~]# cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_keepalive_probes = 10
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1
net.ipv6.conf.lo.disable_ipv6 = 1
net.ipv4.neigh.default.gc_stale_time = 120
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.default.rp_filter = 0
net.ipv4.conf.default.arp_announce = 2
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_announce = 2
net.ipv4.ip_forward = 1
net.ipv4.tcp_max_tw_buckets = 5000
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 1024
net.ipv4.tcp_synack_retries = 2
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.netfilter.nf_conntrack_max = 2310720
fs.inotify.max_user_watches=89100
fs.may_detach_mounts = 1
fs.file-max = 52706963
fs.nr_open = 52706963
net.bridge.bridge-nf-call-arptables = 1
vm.swappiness = 0
vm.overcommit_memory=1
vm.panic_on_oom=0
EOF
[root@master ~]# sysctl --system
注意:如果centos7添加bridge-nf-call-ip6tables出現No such file or directory,執行一下 modprobe br_netfilter 即可解決
9、設置 yum 緩存路徑
[root@master ~]# mkdir -p /wdata/yum/cache
[root@master ~]# cat /etc/yum.conf
[main]
# cachedir=/var/cache/yum/$basearch/$releasever
cachedir=/wdata/yum/cache
keepcache=0
debuglevel=2
logfile=/var/log/yum.log
exactarch=1
obsoletes=1
gpgcheck=1
plugins=1
installonly_limit=5
bugtracker_url=http://bugs.centos.org/set_project.php?project_id=23&ref=http://bugs.centos.org/bug_report_page.php?category=yum
distroverpkg=centos-release
# This is the default, if you make this bigger yum won't see if the metadata
# is newer on the remote and so you'll "gain" the bandwidth of not having to
# download the new metadata and "pay" for it by yum not having correct
# information.
# It is esp. important, to have correct metadata, for distributions like
# Fedora which don't keep old packages around. If you don't like this checking
# interupting your command line usage, it's much better to have something
# manually check the metadata once an hour (yum-updatesd will do this).
# metadata_expire=90m
# PUT YOUR REPOS HERE OR IN separate files named file.repo
# in /etc/yum.repos.d
10、安裝相關工具
[root@master ~]# yum -y install net-tools ntpdate lrzsz socat ipvsadm wget
11、配置免密登錄(在master上執行)
[root@master ~]# ssh-keygen
[root@master ~]# for i in {103..106};do ssh-copy-id -i ~/.ssh/id_rsa.pub 192.168.10.$i;done
三、kubernetes 集羣部署
1、配置 docker yum源
[root@master ~]# yum install -y yum-utils device-mapper-persistent-data lvm2
[root@master ~]# yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
[root@master ~]# yum clean all
[root@master ~]# yum makecache fast
2、查看 docker 版本,並安裝
[root@master ~]# yum list docker-ce --showduplicates | sort -r
已加載插件:fastestmirror
可安裝的軟件包
* updates: mirrors.cn99.com
Loading mirror speeds from cached hostfile
* extras: mirrors.nju.edu.cn
* epel: fedora.cs.nctu.edu.tw
docker-ce.x86_64 3:19.03.1-3.el7 docker-ce-stable
docker-ce.x86_64 3:19.03.0-3.el7 docker-ce-stable
docker-ce.x86_64 3:18.09.8-3.el7 docker-ce-stable
docker-ce.x86_64 3:18.09.7-3.el7 docker-ce-stable
docker-ce.x86_64 3:18.09.6-3.el7 docker-ce-stable
docker-ce.x86_64 3:18.09.5-3.el7 docker-ce-stable
docker-ce.x86_64 3:18.09.4-3.el7 docker-ce-stable
docker-ce.x86_64 3:18.09.3-3.el7 docker-ce-stable
docker-ce.x86_64 3:18.09.2-3.el7 docker-ce-stable
docker-ce.x86_64 3:18.09.1-3.el7 docker-ce-stable
docker-ce.x86_64 3:18.09.0-3.el7 docker-ce-stable
docker-ce.x86_64 18.06.3.ce-3.el7 docker-ce-stable
docker-ce.x86_64 18.06.2.ce-3.el7 docker-ce-stable
docker-ce.x86_64 18.06.1.ce-3.el7 docker-ce-stable
docker-ce.x86_64 18.06.0.ce-3.el7 docker-ce-stable
docker-ce.x86_64 18.03.1.ce-1.el7.centos docker-ce-stable
docker-ce.x86_64 18.03.0.ce-1.el7.centos docker-ce-stable
docker-ce.x86_64 17.12.1.ce-1.el7.centos docker-ce-stable
docker-ce.x86_64 17.12.0.ce-1.el7.centos docker-ce-stable
docker-ce.x86_64 17.09.1.ce-1.el7.centos docker-ce-stable
docker-ce.x86_64 17.09.0.ce-1.el7.centos docker-ce-stable
docker-ce.x86_64 17.06.2.ce-1.el7.centos docker-ce-stable
docker-ce.x86_64 17.06.1.ce-1.el7.centos docker-ce-stable
docker-ce.x86_64 17.06.0.ce-1.el7.centos docker-ce-stable
docker-ce.x86_64 17.03.3.ce-1.el7 docker-ce-stable
docker-ce.x86_64 17.03.2.ce-1.el7.centos docker-ce-stable
docker-ce.x86_64 17.03.1.ce-1.el7.centos docker-ce-stable
docker-ce.x86_64 17.03.0.ce-1.el7.centos docker-ce-stable
* base: mirror1.ku.ac.th
[root@master ~]# yum -y install docker-ce-18.09.8-3.el7
[root@master ~]# systemctl enable --now docker
[root@master ~]# systemctl status docker
3、yum 安裝 kubelet、kubeadm、kubectl
[root@master ~]# yum install -y kubelet kubeadm kubectl --disableexcludes=kubernetes
[root@master ~]# systemctl enable --now kubelet
4、配置Cgroup驅動程序
[root@master ~]# mkdir /etc/docker
[root@master ~]# cat > /etc/docker/daemon.json <<EOF
{
"exec-opts": ["native.cgroupdriver=systemd"],
"log-driver": "json-file",
"log-opts": {
"max-size": "100m"
},
"storage-driver": "overlay2",
"storage-opts": [
"overlay2.override_kernel_check=true"
]
}
EOF
[root@master ~]# mkdir -p /etc/systemd/system/docker.service.d
[root@master ~]# systemctl daemon-reload
[root@master ~]# systemctl restart docker
5、鏡像拉取
[root@master ~]# kubeadm config images pull
[config/images] Pulled k8s.gcr.io/kube-apiserver:v1.15.1
[config/images] Pulled k8s.gcr.io/kube-controller-manager:v1.15.1
[config/images] Pulled k8s.gcr.io/kube-scheduler:v1.15.1
[config/images] Pulled k8s.gcr.io/kube-proxy:v1.15.1
[config/images] Pulled k8s.gcr.io/pause:3.1
[config/images] Pulled k8s.gcr.io/etcd:3.3.10
[config/images] Pulled k8s.gcr.io/coredns:1.3.1
6、初始化集羣
[root@master ~]# kubeadm init \
--apiserver-advertise-address=192.168.10.102 \
--kubernetes-version v1.15.0 \
--service-cidr=10.1.0.0/16 \
--pod-network-cidr=10.244.0.0/16
[init] Using Kubernetes version: v1.15.0
[preflight] Running pre-flight checks
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Activating the kubelet service
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Generating "ca" certificate and key
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [master kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local] and IPs [10.1.0.1 192.168.10.102]
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Generating "etcd/ca" certificate and key
[certs] Generating "etcd/peer" certificate and key
[certs] etcd/peer serving cert is signed for DNS names [master localhost] and IPs [192.168.10.102 127.0.0.1 ::1]
[certs] Generating "apiserver-etcd-client" certificate and key
[certs] Generating "etcd/server" certificate and key
[certs] etcd/server serving cert is signed for DNS names [master localhost] and IPs [192.168.10.102 127.0.0.1 ::1]
[certs] Generating "etcd/healthcheck-client" certificate and key
[certs] Generating "front-proxy-ca" certificate and key
[certs] Generating "front-proxy-client" certificate and key
[certs] Generating "sa" key and public key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[apiclient] All control plane components are healthy after 22.502507 seconds
[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config-1.15" in namespace kube-system with the configuration for the kubelets in the cluster
[upload-certs] Skipping phase. Please see --upload-certs
[mark-control-plane] Marking the node master as control-plane by adding the label "node-role.kubernetes.io/master=''"
[mark-control-plane] Marking the node master as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule]
[bootstrap-token] Using token: j63zfk.i6nkik0sxr8rncny
[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace
[addons] Applied essential addon: CoreDNS
[addons] Applied essential addon: kube-proxy
Your Kubernetes control-plane has initialized successfully!
To start using your cluster, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
Then you can join any number of worker nodes by running the following on each as root:
kubeadm join 192.168.10.102:6443 --token j63zfk.i6nkik0sxr8rncny \
--discovery-token-ca-cert-hash sha256:75ebf4f00fbd65dc6cb7d4c2af1fb6455c9daab700523a716626028c6ee63c87
通過上面的信息輸出,我們可以看到以下幾條關鍵內容
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml" 生成kubelet配置文件
[certs] Using certificateDir folder "/etc/kubernetes/pki" 生成相關證書目錄及證書文件
[kubeconfig] Using kubeconfig folder "/etc/kubernetes" 生成kubeconfig目錄及kubeconfig文件
[bootstrap-token] Using token: j63zfk.i6nkik0sxr8rncny 生成在集羣中添加節點的token
配置用戶訪問集羣的命令
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
節點加入集羣的方式
kubeadm join 192.168.10.102:6443 --token j63zfk.i6nkik0sxr8rncny \
--discovery-token-ca-cert-hash sha256:75ebf4f00fbd65dc6cb7d4c2af1fb6455c9daab700523a716626028c6ee63c87
7、其他 4 個節點加入到集羣,分別在其他節點上執行即可
kubeadm join 192.168.10.102:6443 --token j63zfk.i6nkik0sxr8rncny \
--discovery-token-ca-cert-hash sha256:75ebf4f00fbd65dc6cb7d4c2af1fb6455c9daab700523a716626028c6ee63c87
檢查節點是否成功加入
[root@master ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
master NotReady master 37m v1.15.1
node1 NotReady <none> 9m11s v1.15.1
node2 NotReady <none> 9m8s v1.15.1
node3 NotReady <none> 8m46s v1.15.1
node4 NotReady <none> 8m58s v1.15.1
8、檢查集羣狀態
[root@master kubernetes]# kubectl get cs
NAME STATUS MESSAGE ERROR
scheduler Healthy ok
controller-manager Healthy ok
etcd-0 Healthy {"health":"true"}
問題回顧:
如果以上步驟有一部出現錯誤,則可以將集羣進行清理,在進行初始化,清理命令如下
kubeadm reset
ifconfig cni0 down
ip link delete cni0
ifconfig flannel.1 down
ip link delete flannel.1
rm -rf /var/lib/cni/
四、安裝 Pod Network
1、安裝 flannel
[root@master ~]# mkdir /wdata/kubernetes && cd /wdata/kubernetes
[root@master kubernetes]# wget -c https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
2、修改 kube-flannel.yml 文件,加入通訊網卡(如果是多網卡的情況下)
[root@master kubernetes]# vim kube-flannel.yml
containers:
- name: kube-flannel
image: quay.io/coreos/flannel:v0.11.0-amd64
command:
- /opt/bin/flanneld
args:
- --ip-masq
- --kube-subnet-mgr
- --iface=ens33
3、安裝
[root@master kubernetes]# kubectl apply -f kube-flannel.yml
podsecuritypolicy.policy/psp.flannel.unprivileged created
clusterrole.rbac.authorization.k8s.io/flannel configured
clusterrolebinding.rbac.authorization.k8s.io/flannel unchanged
serviceaccount/flannel created
configmap/kube-flannel-cfg created
daemonset.apps/kube-flannel-ds-amd64 created
daemonset.apps/kube-flannel-ds-arm64 created
daemonset.apps/kube-flannel-ds-arm created
daemonset.apps/kube-flannel-ds-ppc64le created
daemonset.apps/kube-flannel-ds-s390x created
4、驗證
[root@master kubernetes]# kubectl get pod --all-namespaces -o wide
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
kube-system coredns-5c98db65d4-8zvl7 1/1 Running 0 99m 10.244.1.3 node1 <none> <none>
kube-system coredns-5c98db65d4-j69dq 1/1 Running 0 99m 10.244.1.2 node1 <none> <none>
kube-system etcd-master 1/1 Running 0 99m 192.168.10.102 master <none> <none>
kube-system kube-apiserver-master 1/1 Running 0 99m 192.168.10.102 master <none> <none>
kube-system kube-controller-manager-master 1/1 Running 0 99m 192.168.10.102 master <none> <none>
kube-system kube-flannel-ds-amd64-6w6jl 1/1 Running 0 4m33s 192.168.10.105 node3 <none> <none>
kube-system kube-flannel-ds-amd64-d2mm2 1/1 Running 0 3m53s 192.168.10.102 master <none> <none>
kube-system kube-flannel-ds-amd64-s4wzt 1/1 Running 0 3m9s 192.168.10.106 node4 <none> <none>
kube-system kube-flannel-ds-amd64-w5b42 1/1 Running 0 5m44s 192.168.10.103 node1 <none> <none>
kube-system kube-flannel-ds-amd64-zs9kc 1/1 Running 0 5m8s 192.168.10.104 node2 <none> <none>
kube-system kube-proxy-9j669 1/1 Running 0 99m 192.168.10.102 master <none> <none>
kube-system kube-proxy-qgxv8 1/1 Running 0 71m 192.168.10.106 node4 <none> <none>
kube-system kube-proxy-r72vg 1/1 Running 0 71m 192.168.10.105 node3 <none> <none>
kube-system kube-proxy-tvwwj 1/1 Running 0 71m 192.168.10.103 node1 <none> <none>
kube-system kube-proxy-vg4l7 1/1 Running 0 71m 192.168.10.104 node2 <none> <none>
kube-system kube-scheduler-master 1/1 Running 0 99m 192.168.10.102 master <none> <none>
kube-system kube-controller-manager-master 1/1 Running 0 93m 192.168.10.102 master <none> <none>
說明:需要確保所有節點全部爲 Running 狀態
如果發現某個 pod 狀態錯誤,則可以執行 kubectl --namespace=kube-system describe pod <pod_name> 來查看錯誤信息。例如:
[root@master kubernetes]# kubectl --namespace=kube-system describe pod coredns-5c98db65d4-8zvl7
Name: coredns-5c98db65d4-8zvl7
Namespace: kube-system
Priority: 2000000000
Priority Class Name: system-cluster-critical
Node: node1/192.168.10.103
Start Time: Wed, 31 Jul 2019 18:25:31 +0800
Labels: k8s-app=kube-dns
pod-template-hash=5c98db65d4
Annotations: <none>
Status: Running
IP: 10.244.1.4
Controlled By: ReplicaSet/coredns-5c98db65d4
Containers:
coredns:
Container ID: docker://c6508701e1ee37481d0ebe05bbf3a1e466378394ca42361e219e0e4410841926
Image: k8s.gcr.io/coredns:1.3.1
Image ID: docker-pullable://k8s.gcr.io/coredns@sha256:02382353821b12c21b062c59184e227e001079bb13ebd01f9d3270ba0fcbf1e4
Ports: 53/UDP, 53/TCP, 9153/TCP
Host Ports: 0/UDP, 0/TCP, 0/TCP
Args:
-conf
/etc/coredns/Corefile
State: Running
Started: Wed, 31 Jul 2019 20:34:24 +0800
Last State: Terminated
Reason: Completed
Exit Code: 0
Started: Wed, 31 Jul 2019 18:25:41 +0800
Finished: Wed, 31 Jul 2019 20:33:20 +0800
Ready: True
Restart Count: 1
Limits:
memory: 170Mi
Requests:
cpu: 100m
memory: 70Mi
Liveness: http-get http://:8080/health delay=60s timeout=5s period=10s #success=1 #failure=5
Readiness: http-get http://:8080/health delay=0s timeout=1s period=10s #success=1 #failure=3
Environment: <none>
Mounts:
/etc/coredns from config-volume (ro)
/var/run/secrets/kubernetes.io/serviceaccount from coredns-token-lj2p9 (ro)
Conditions:
Type Status
Initialized True
Ready True
ContainersReady True
PodScheduled True
Volumes:
config-volume:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: coredns
Optional: false
coredns-token-lj2p9:
Type: Secret (a volume populated by a Secret)
SecretName: coredns-token-lj2p9
Optional: false
QoS Class: Burstable
Node-Selectors: beta.kubernetes.io/os=linux
Tolerations: CriticalAddonsOnly
node-role.kubernetes.io/master:NoSchedule
node.kubernetes.io/not-ready:NoExecute for 300s
node.kubernetes.io/unreachable:NoExecute for 300s
Events: <none>
5、測試 DNS
執行 kubectl run curl --image=radial/busyboxplus:curl -it ,進入查看能否正常解析
[root@master kubernetes]# kubectl run curl --image=radial/busyboxplus:curl -it
kubectl run --generator=deployment/apps.v1 is DEPRECATED and will be removed in a future version. Use kubectl run --generator=run-pod/v1 or kubectl create instead.
If you don't see a command prompt, try pressing enter.
[ root@curl-6bf6db5c4f-q7vp7:/ ]$ nslookup kubernetes.default
Server: 10.1.0.10
Address 1: 10.1.0.10 kube-dns.kube-system.svc.cluster.local
Name: kubernetes.default
Address 1: 10.1.0.1 kubernetes.default.svc.cluster.local
[ root@curl-6bf6db5c4f-q7vp7:/ ]$
如上所示,證明 DNS 正常
五、kube-proxy 開啓 IPVS
1、修改 ConfigMap 的 kube-system/kube-proxy 中的 config.conf,mode:"ipvs"
[root@master kubernetes]# kubectl edit cm kube-proxy -n kube-system
2、重啓各個節點的 kube-proxy pod
[root@master kubernetes]# kubectl get pod -n kube-system | grep kube-proxy | awk '{system("kubectl delete pod "$1" -n kube-system")}'
pod "kube-proxy-9j669" deleted
pod "kube-proxy-qgxv8" deleted
pod "kube-proxy-r72vg" deleted
pod "kube-proxy-tvwwj" deleted
pod "kube-proxy-vg4l7" deleted
3、查看各個節點的 kube-proxy pod 狀態
[root@master kubernetes]# kubectl get pod -n kube-system | grep kube-proxy
kube-proxy-7hzkl 1/1 Running 0 46s
kube-proxy-jtmc9 1/1 Running 0 42s
kube-proxy-kzjjn 1/1 Running 0 36s
kube-proxy-mbk7q 1/1 Running 0 44s
kube-proxy-pp4ms 1/1 Running 0 60s
4、查看任意 kube-proxy pod 的 ipvs 模塊是否開啓
[root@master ~]# kubectl logs kube-proxy-zwmlp -n kube-system
I0731 12:34:18.663297 1 server_others.go:170] Using ipvs Proxier.
W0731 12:34:18.663535 1 proxier.go:401] IPVS scheduler not specified, use rr by default
I0731 12:34:18.663688 1 server.go:534] Version: v1.15.0
I0731 12:34:18.673416 1 conntrack.go:100] Set sysctl 'net/netfilter/nf_conntrack_max' to 131072
I0731 12:34:18.673445 1 conntrack.go:52] Setting nf_conntrack_max to 131072
I0731 12:34:18.674576 1 conntrack.go:83] Setting conntrack hashsize to 32768
I0731 12:34:18.680325 1 conntrack.go:100] Set sysctl 'net/netfilter/nf_conntrack_tcp_timeout_established' to 86400
I0731 12:34:18.680387 1 conntrack.go:100] Set sysctl 'net/netfilter/nf_conntrack_tcp_timeout_close_wait' to 3600
I0731 12:34:18.681385 1 config.go:187] Starting service config controller
I0731 12:34:18.681407 1 controller_utils.go:1029] Waiting for caches to sync for service config controller
I0731 12:34:18.681473 1 config.go:96] Starting endpoints config controller
I0731 12:34:18.681483 1 controller_utils.go:1029] Waiting for caches to sync for endpoints config controller
I0731 12:34:18.781898 1 controller_utils.go:1036] Caches are synced for service config controller
I0731 12:34:18.781953 1 controller_utils.go:1036] Caches are synced for endpoints config controller
如上所示,出現 Using ipvs Proxier ,則表示 ipvs 模塊開啓成功。如果出現以下錯誤,則可能是系統加載了 ipvs 模塊後沒有重啓,重啓以下服務器可解決。
[root@master kubernetes]# kubectl logs kube-proxy-zwmlp -n kube-system
W0731 12:31:04.233204 1 proxier.go:513] Failed to load kernel module ip_vs with modprobe. You can ignore this message when kube-proxy is running inside container without mounting /lib/modules
W0731 12:31:04.234176 1 proxier.go:513] Failed to load kernel module ip_vs_rr with modprobe. You can ignore this message when kube-proxy is running inside container without mounting /lib/modules
W0731 12:31:04.234956 1 proxier.go:513] Failed to load kernel module ip_vs_wrr with modprobe. You can ignore this message when kube-proxy is running inside container without mounting /lib/modules
W0731 12:31:04.235798 1 proxier.go:513] Failed to load kernel module ip_vs_sh with modprobe. You can ignore this message when kube-proxy is running inside container without mounting /lib/modules
W0731 12:31:04.243905 1 proxier.go:513] Failed to load kernel module ip_vs with modprobe. You can ignore this message when kube-proxy is running inside container without mounting /lib/modules
W0731 12:31:04.245174 1 proxier.go:513] Failed to load kernel module ip_vs_rr with modprobe. You can ignore this message when kube-proxy is running inside container without mounting /lib/modules
W0731 12:31:04.246018 1 proxier.go:513] Failed to load kernel module ip_vs_wrr with modprobe. You can ignore this message when kube-proxy is running inside container without mounting /lib/modules
W0731 12:31:04.246854 1 proxier.go:513] Failed to load kernel module ip_vs_sh with modprobe. You can ignore this message when kube-proxy is running inside container without mounting /lib/modules
E0731 12:31:04.249862 1 server_others.go:259] can't determine whether to use ipvs proxy, error: IPVS proxier will not be used because the following required kernel modules are not loaded: [ip_vs ip_vs_rr ip_vs_wrr ip_vs_sh]
I0731 12:31:04.258287 1 server_others.go:143] Using iptables Proxier.
I0731 12:31:04.260120 1 server.go:534] Version: v1.15.0
I0731 12:31:04.267653 1 conntrack.go:52] Setting nf_conntrack_max to 131072
I0731 12:31:04.267812 1 config.go:96] Starting endpoints config controller
I0731 12:31:04.267831 1 controller_utils.go:1029] Waiting for caches to sync for endpoints config controller
I0731 12:31:04.267901 1 config.go:187] Starting service config controller
I0731 12:31:04.267909 1 controller_utils.go:1029] Waiting for caches to sync for service config controller
I0731 12:31:04.368357 1 controller_utils.go:1036] Caches are synced for endpoints config controller
I0731 12:31:04.368416 1 controller_utils.go:1036] Caches are synced for service config controller
5、查看 ipvs 規則
[root@master ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 10.1.0.1:443 rr
-> 192.168.10.102:6443 Masq 1 1 0
TCP 10.1.0.10:53 rr
-> 10.244.1.4:53 Masq 1 0 0
-> 10.244.1.5:53 Masq 1 0 0
TCP 10.1.0.10:9153 rr
-> 10.244.1.4:9153 Masq 1 0 0
-> 10.244.1.5:9153 Masq 1 0 0
UDP 10.1.0.10:53 rr
-> 10.244.1.4:53 Masq 1 0 0
-> 10.244.1.5:53 Masq 1 0 0
六、部署 Kubernetes 包管理器組件 Helm
1、Helm 安裝
[root@master ~]# cd /wdata/kubernetes
[root@master ~]# wget https://storage.googleapis.com/kubernetes-helm/helm-v2.14.0-linux-amd64.tar.gz
[root@master kubernetes]# tar -zxvf helm-v2.14.0-linux-amd64.tar.gz && cd linux-amd64
linux-amd64/
linux-amd64/tiller
linux-amd64/README.md
linux-amd64/LICENSE
linux-amd64/helm
[root@master linux-amd64]# cp helm /usr/local/bin
爲了安裝服務端 tiller,還需要服務器上配置好kubectl工具和kubeconfig文件,確保kubectl工具可以在這臺機器上訪問apiserver且正常使用。 這裏已經在 master 節點上配置好了kubectl。
2、編寫 tiller.yaml 文件
[root@master kubernetes]# vim tiller.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: tiller
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: tiller
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: tiller
namespace: kube-system
3、執行以下命令安裝 Helm
[root@master kubernetes]# kubectl create -f tiller.yaml
serviceaccount/tiller created
clusterrolebinding.rbac.authorization.k8s.io/tiller created
4、使用 Helm 部署 tiller
[root@master kubernetes]# helm init --service-account tiller --skip-refresh
Creating /root/.helm
Creating /root/.helm/repository
Creating /root/.helm/repository/cache
Creating /root/.helm/repository/local
Creating /root/.helm/plugins
Creating /root/.helm/starters
Creating /root/.helm/cache/archive
Creating /root/.helm/repository/repositories.yaml
Adding stable repo with URL: https://kubernetes-charts.storage.googleapis.com
Adding local repo with URL: http://127.0.0.1:8879/charts
$HELM_HOME has been configured at /root/.helm.
Tiller (the Helm server-side component) has been installed into your Kubernetes Cluster.
Please note: by default, Tiller is deployed with an insecure 'allow unauthenticated users' policy.
To prevent this, run `helm init` with the --tiller-tls-verify flag.
For more information on securing your installation see: https://docs.helm.sh/using_helm/#securing-your-helm-installation
tiller 默認被部署在 kubernetes 集羣中的 kube-system 這個 namespace 下,查看 tiller 運行狀態:
[root@master kubernetes]# kubectl get pod -n kube-system -l app=helm
NAME READY STATUS RESTARTS AGE
tiller-deploy-5744948f4b-8rfgr 1/1 Running 0 54s
5、查看 Helm 版本:
[root@master kubernetes]# helm version
Client: &version.Version{SemVer:"v2.14.0", GitCommit:"05811b84a3f93603dd6c2fcfe57944dfa7ab7fd0", GitTreeState:"clean"}
Server: &version.Version{SemVer:"v2.14.0", GitCommit:"05811b84a3f93603dd6c2fcfe57944dfa7ab7fd0", GitTreeState:"clean"}
如上顯示,客戶端和服務端版本均爲 2.14.0
注意:由於某些原因需要網絡可以訪問 gcr.io 和 kubernetes-charts.storage.googleapis.com ,如果無法訪問,可以通過 helm init -service-account tiller -tiller-image <your-docker-registry>/tiller:v2.14.0 -skip-refresh 使用私有鏡像倉庫中的tiller鏡像
七、使用 Helm 部署 Nginx Ingress
爲了便於將集羣中的服務暴露到集羣外部,從集羣外部訪問,接下來使用Helm將Nginx Ingress部署到Kubernetes上。 Nginx Ingress Controller被部署在Kubernetes的邊緣節點上,關於Kubernetes邊緣節點的高可用相關的內容可以查看《Kubernetes Ingress實戰(五):Bare metal環境下Kubernetes Ingress邊緣節點的高可用(基於IPVS)》
1、我們將 node3 和 node4 同時作爲邊緣節點,打上 Label
[root@master kubernetes]# kubectl label node node3 node-role.kubernetes.io/edge=
node/node3 labeled
[root@master kubernetes]# kubectl label node node4 node-role.kubernetes.io/edge=
node/node4 labeled
[root@master kubernetes]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
master Ready master 4h17m v1.15.1
node1 Ready <none> 3h49m v1.15.1
node2 Ready <none> 3h49m v1.15.1
node3 Ready edge 3h49m v1.15.1
node4 Ready edge 3h49m v1.15.1
2、創建 stable/nginx-ingress chart 的值文件 ingress-nginx.yaml
[root@master kubernetes]# vim ingress-nginx.yaml
controller:
replicaCount: 2
service:
externalIPs:
- 192.168.10.101
nodeSelector:
node-role.kubernetes.io/edge: ''
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: app
operator: In
values:
- nginx-ingress
- key: component
operator: In
values:
- controller
topologyKey: kubernetes.io/hostname
tolerations:
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
defaultBackend:
nodeSelector:
node-role.kubernetes.io/edge: ''
tolerations:
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
nginx ingress controller 的副本數 replicaCount爲 2,將被調度到 node3 和 node4 這兩個邊緣節點上。externalIPs 指定的 192.168.10.101 爲 VIP,將綁定到 kube-proxy kube-ipvs0 網卡上。
3、從可用圖表資料庫更新本地圖表信息
[root@master kubernetes]# helm repo update
Hang tight while we grab the latest from your chart repositories...
...Skip local chart repository
...Successfully got an update from the "stable" chart repository
Update Complete.
4、安裝 nginx-ingress
[root@master kubernetes]# helm install stable/nginx-ingress \
> -n nginx-ingress \
> --namespace ingress-nginx \
> -f ingress-nginx.yaml
NAME: nginx-ingress
LAST DEPLOYED: Wed Jul 31 21:46:58 2019
NAMESPACE: ingress-nginx
STATUS: DEPLOYED
RESOURCES:
==> v1/Pod(related)
NAME READY STATUS RESTARTS AGE
nginx-ingress-controller-578b5f54b5-gcf4s 0/1 ContainerCreating 0 0s
nginx-ingress-controller-578b5f54b5-lfkfw 0/1 ContainerCreating 0 0s
nginx-ingress-default-backend-85d99989c9-7jtzj 0/1 ContainerCreating 0 0s
==> v1/Service
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
nginx-ingress-controller LoadBalancer 10.1.151.211 192.168.10.101 80:30877/TCP,443:32129/TCP 0s
nginx-ingress-default-backend ClusterIP 10.1.76.40 <none> 80/TCP 0s
==> v1/ServiceAccount
NAME SECRETS AGE
nginx-ingress 1 0s
==> v1beta1/ClusterRole
NAME AGE
nginx-ingress 0s
==> v1beta1/ClusterRoleBinding
NAME AGE
nginx-ingress 0s
==> v1beta1/Deployment
NAME READY UP-TO-DATE AVAILABLE AGE
nginx-ingress-controller 0/2 2 0 0s
nginx-ingress-default-backend 0/1 1 0 0s
==> v1beta1/PodDisruptionBudget
NAME MIN AVAILABLE MAX UNAVAILABLE ALLOWED DISRUPTIONS AGE
nginx-ingress-controller 1 N/A 0 0s
==> v1beta1/Role
NAME AGE
nginx-ingress 0s
==> v1beta1/RoleBinding
NAME AGE
nginx-ingress 0s
NOTES:
The nginx-ingress controller has been installed.
It may take a few minutes for the LoadBalancer IP to be available.
You can watch the status by running 'kubectl --namespace ingress-nginx get services -o wide -w nginx-ingress-controller'
An example Ingress that makes use of the controller:
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
annotations:
kubernetes.io/ingress.class: nginx
name: example
namespace: foo
spec:
rules:
- host: www.example.com
http:
paths:
- backend:
serviceName: exampleService
servicePort: 80
path: /
# This section is only required if TLS is to be enabled for the Ingress
tls:
- hosts:
- www.example.com
secretName: example-tls
If TLS is enabled for the Ingress, a Secret containing the certificate and key must also be provided:
apiVersion: v1
kind: Secret
metadata:
name: example-tls
namespace: foo
data:
tls.crt: <base64 encoded cert>
tls.key: <base64 encoded key>
type: kubernetes.io/tls
5、查看 ingress-nginx 運行狀態
[root@master kubernetes]# kubectl get pod -n ingress-nginx -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx-ingress-controller-578b5f54b5-gcf4s 1/1 Running 0 3m9s 10.244.4.3 node3 <none> <none>
nginx-ingress-controller-578b5f54b5-lfkfw 1/1 Running 0 3m9s 10.244.3.2 node4 <none> <none>
nginx-ingress-default-backend-85d99989c9-7jtzj 1/1 Running 0 3m9s 10.244.3.4 node4 <none> <none>
[root@master kubernetes]# kubectl get pod --all-namespaces -o wide
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
default curl-6bf6db5c4f-q7vp7 1/1 Running 2 116m 10.244.2.3 node2 <none> <none>
ingress-nginx nginx-ingress-controller-578b5f54b5-gcf4s 1/1 Running 0 8m11s 10.244.4.3 node3 <none> <none>
ingress-nginx nginx-ingress-controller-578b5f54b5-lfkfw 1/1 Running 0 8m11s 10.244.3.2 node4 <none> <none>
ingress-nginx nginx-ingress-default-backend-85d99989c9-7jtzj 1/1 Running 0 8m11s 10.244.3.4 node4 <none> <none>
kube-system coredns-5c98db65d4-8zvl7 1/1 Running 1 4h37m 10.244.1.4 node1 <none> <none>
kube-system coredns-5c98db65d4-j69dq 1/1 Running 1 4h37m 10.244.1.5 node1 <none> <none>
kube-system etcd-master 1/1 Running 1 4h36m 192.168.10.102 master <none> <none>
kube-system kube-apiserver-master 1/1 Running 1 4h36m 192.168.10.102 master <none> <none>
kube-system kube-controller-manager-master 1/1 Running 1 4h36m 192.168.10.102 master <none> <none>
kube-system kube-flannel-ds-amd64-6w6jl 1/1 Running 1 3h1m 192.168.10.105 node3 <none> <none>
kube-system kube-flannel-ds-amd64-d2mm2 1/1 Running 1 3h1m 192.168.10.102 master <none> <none>
kube-system kube-flannel-ds-amd64-s4wzt 1/1 Running 1 3h 192.168.10.106 node4 <none> <none>
kube-system kube-flannel-ds-amd64-w5b42 1/1 Running 1 3h2m 192.168.10.103 node1 <none> <none>
kube-system kube-flannel-ds-amd64-zs9kc 1/1 Running 1 3h2m 192.168.10.104 node2 <none> <none>
kube-system kube-proxy-7bgqf 1/1 Running 1 84m 192.168.10.106 node4 <none> <none>
kube-system kube-proxy-7r5vp 1/1 Running 1 84m 192.168.10.104 node2 <none> <none>
kube-system kube-proxy-n6ln7 1/1 Running 1 84m 192.168.10.103 node1 <none> <none>
kube-system kube-proxy-zgl4x 1/1 Running 1 84m 192.168.10.102 master <none> <none>
kube-system kube-proxy-zwmlp 1/1 Running 1 84m 192.168.10.105 node3 <none> <none>
kube-system kube-scheduler-master 1/1 Running 1 4h36m 192.168.10.102 master <none> <none>
kube-system tiller-deploy-5744948f4b-8rfgr 1/1 Running 0 37m 10.244.4.2 node3 <none> <none>
如果訪問 http://192.168.10.101 返回 default backend,則部署完成,如下圖所示:
6、啓動一個測試實例
[root@master kubernetes]# kubectl run nginx --image=nginx --replicas=3
kubectl run --generator=deployment/apps.v1 is DEPRECATED and will be removed in a future version. Use kubectl run --generator=run-pod/v1 or kubectl create instead.
deployment.apps/nginx created
[root@master kubernetes]# kubectl get pod
NAME READY STATUS RESTARTS AGE
curl-6bf6db5c4f-q7vp7 1/1 Running 2 15h
nginx-7bb7cd8db5-kmc4g 1/1 Running 0 2m50s
nginx-7bb7cd8db5-qrqsv 1/1 Running 0 2m50s
nginx-7bb7cd8db5-s6frr 1/1 Running 0 2m50s
[root@master kubernetes]# kubectl expose deployment nginx --port=88 --target-port=80 --type=NodePort
service/nginx exposed
[root@master kubernetes]# kubectl get svc nginx
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
nginx NodePort 10.1.233.149 <none> 88:32229/TCP 15s
[root@master kubernetes]# kubectl get all
NAME READY STATUS RESTARTS AGE
pod/curl-6bf6db5c4f-q7vp7 1/1 Running 2 15h
pod/nginx-7bb7cd8db5-kmc4g 1/1 Running 0 4m12s
pod/nginx-7bb7cd8db5-qrqsv 1/1 Running 0 4m12s
pod/nginx-7bb7cd8db5-s6frr 1/1 Running 0 4m12s
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/kubernetes ClusterIP 10.1.0.1 <none> 443/TCP 18h
service/nginx NodePort 10.1.233.149 <none> 88:32229/TCP 25s
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/curl 1/1 1 1 15h
deployment.apps/nginx 3/3 3 3 4m12s
NAME DESIRED CURRENT READY AGE
replicaset.apps/curl-6bf6db5c4f 1 1 1 15h
replicaset.apps/nginx-7bb7cd8db5 3 3 3 4m12s
如以上輸出可以看出,該 nginx 的實例對外端口爲32229,我們在瀏覽器中訪問 http://192.168.10.102:32229/ ,如下圖:
7、安裝 kubernetes UI 界面
下載 kubernetes-dashboard.yaml 文件
[root@master kubernetes]# wget https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.1/src/deploy/recommended/kubernetes-dashboard.yaml
--2019-08-01 12:20:21-- https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.1/src/deploy/recommended/kubernetes-dashboard.yaml
正在解析主機 raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...
正在連接 raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... 已連接。
已發出 HTTP 請求,正在等待迴應... 200 OK
長度:4577 (4.5K) [text/plain]
正在保存至: “kubernetes-dashboard.yaml”
100%[====================================================================================================================================================================================================================================>] 4,577 --.-K/s 用時 0s
2019-08-01 12:20:27 (79.7 MB/s) - 已保存 “kubernetes-dashboard.yaml” [4577/4577])
編輯 kubernetes-dashboard.yaml 文件,將 apiserver 地址修改爲自己的服務器 IP 地址
[root@master kubernetes]# vim kubernetes-dashboard.yaml
args:
- --auto-generate-certificates
# Uncomment the following line to manually specify Kubernetes API server Host
# If not specified, Dashboard will attempt to auto discover the API server and connect
# to it. Uncomment only if the default does not work.
- --apiserver-host=https://192,168.10.102:6443
volumeMounts:
- name: kubernetes-dashboard-certs
mountPath: /certs
# Create on-disk volume to store exec logs
- mountPath: /tmp
name: tmp-volume
安裝 kubernetes-dashboard.yaml
[root@master kubernetes]# kubectl apply -f kubernetes-dashboard.yaml
secret/kubernetes-dashboard-certs created
serviceaccount/kubernetes-dashboard created
role.rbac.authorization.k8s.io/kubernetes-dashboard-minimal created
rolebinding.rbac.authorization.k8s.io/kubernetes-dashboard-minimal created
deployment.apps/kubernetes-dashboard created
service/kubernetes-dashboard created
查看是否安裝成功
[root@master kubernetes]# kubectl get -f kubernetes-dashboard.yaml
NAME TYPE DATA AGE
secret/kubernetes-dashboard-certs Opaque 0 7m6s
NAME SECRETS AGE
serviceaccount/kubernetes-dashboard 1 7m6s
NAME AGE
role.rbac.authorization.k8s.io/kubernetes-dashboard-minimal 7m6s
NAME AGE
rolebinding.rbac.authorization.k8s.io/kubernetes-dashboard-minimal 7m6s
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/kubernetes-dashboard 1/1 1 1 7m6s
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/kubernetes-dashboard ClusterIP 10.1.133.139 <none> 443/TCP 7m6s
查看 pod 狀態
[root@master kubernetes]# kubectl get pods --all-namespaces
NAMESPACE NAME READY STATUS RESTARTS AGE
default curl-6bf6db5c4f-q7vp7 1/1 Running 2 16h
default nginx-7bb7cd8db5-kmc4g 1/1 Running 0 34m
default nginx-7bb7cd8db5-qrqsv 1/1 Running 0 34m
default nginx-7bb7cd8db5-s6frr 1/1 Running 0 34m
ingress-nginx nginx-ingress-controller-578b5f54b5-gcf4s 1/1 Running 0 14h
ingress-nginx nginx-ingress-controller-578b5f54b5-lfkfw 1/1 Running 0 14h
ingress-nginx nginx-ingress-default-backend-85d99989c9-7jtzj 1/1 Running 0 14h
kube-system coredns-5c98db65d4-8zvl7 1/1 Running 1 19h
kube-system coredns-5c98db65d4-j69dq 1/1 Running 1 19h
kube-system etcd-master 1/1 Running 1 19h
kube-system kube-apiserver-master 1/1 Running 1 19h
kube-system kube-controller-manager-master 1/1 Running 1 19h
kube-system kube-flannel-ds-amd64-6w6jl 1/1 Running 1 17h
kube-system kube-flannel-ds-amd64-d2mm2 1/1 Running 1 17h
kube-system kube-flannel-ds-amd64-s4wzt 1/1 Running 1 17h
kube-system kube-flannel-ds-amd64-w5b42 1/1 Running 1 17h
kube-system kube-flannel-ds-amd64-zs9kc 1/1 Running 1 17h
kube-system kube-proxy-7bgqf 1/1 Running 1 15h
kube-system kube-proxy-7r5vp 1/1 Running 1 15h
kube-system kube-proxy-n6ln7 1/1 Running 1 15h
kube-system kube-proxy-zgl4x 1/1 Running 1 15h
kube-system kube-proxy-zwmlp 1/1 Running 1 15h
kube-system kube-scheduler-master 1/1 Running 1 19h
kube-system kubernetes-dashboard-7b89455d8-zf4jk 0/1 CrashLoopBackOff 5 4m56s
kube-system kubernetes-dashboard-7d75c474bb-wstl9 1/1 Running 0 11m
kube-system tiller-deploy-5744948f4b-8rfgr 1/1 Running 0 15h
發現 kubernetes-dashboard-7b89455d8-zf4jk 一直處於 CrashLoopBackOff 狀態,通過以下命令查看命名空間的日誌
[root@master kubernetes]# kubectl describe pod kubernetes-dashboard-7b89455d8-zf4jk --namespace=kube-system
Name: kubernetes-dashboard-7b89455d8-zf4jk
Namespace: kube-system
Priority: 0
Node: node4/192.168.10.106
Start Time: Thu, 01 Aug 2019 12:22:43 +0800
Labels: k8s-app=kubernetes-dashboard
pod-template-hash=7b89455d8
Annotations: <none>
Status: Running
IP: 10.244.3.6
Controlled By: ReplicaSet/kubernetes-dashboard-7b89455d8
Containers:
kubernetes-dashboard:
Container ID: docker://984f622e19baf8bd7ecd0b6c0ac5439282ebd91e44e84fb74ea525e11c75a326
Image: k8s.gcr.io/kubernetes-dashboard-amd64:v1.10.1
Image ID: docker-pullable://k8s.gcr.io/kubernetes-dashboard-amd64@sha256:0ae6b69432e78069c5ce2bcde0fe409c5c4d6f0f4d9cd50a17974fea38898747
Port: 8443/TCP
Host Port: 0/TCP
Args:
--auto-generate-certificates
--apiserver-host=https://192,168.10.102:6443
State: Waiting
Reason: CrashLoopBackOff
Last State: Terminated
Reason: Error
Exit Code: 1
Started: Thu, 01 Aug 2019 12:26:00 +0800
Finished: Thu, 01 Aug 2019 12:26:00 +0800
Ready: False
Restart Count: 5
Liveness: http-get https://:8443/ delay=30s timeout=30s period=10s #success=1 #failure=3
Environment: <none>
Mounts:
/certs from kubernetes-dashboard-certs (rw)
/tmp from tmp-volume (rw)
/var/run/secrets/kubernetes.io/serviceaccount from kubernetes-dashboard-token-zqthw (ro)
Conditions:
Type Status
Initialized True
Ready False
ContainersReady False
PodScheduled True
Volumes:
kubernetes-dashboard-certs:
Type: Secret (a volume populated by a Secret)
SecretName: kubernetes-dashboard-certs
Optional: false
tmp-volume:
Type: EmptyDir (a temporary directory that shares a pod's lifetime)
Medium:
SizeLimit: <unset>
kubernetes-dashboard-token-zqthw:
Type: Secret (a volume populated by a Secret)
SecretName: kubernetes-dashboard-token-zqthw
Optional: false
QoS Class: BestEffort
Node-Selectors: <none>
Tolerations: node-role.kubernetes.io/master:NoSchedule
node.kubernetes.io/not-ready:NoExecute for 300s
node.kubernetes.io/unreachable:NoExecute for 300s
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Scheduled 5m47s default-scheduler Successfully assigned kube-system/kubernetes-dashboard-7b89455d8-zf4jk to node4
Normal Pulling 5m46s kubelet, localhost Pulling image "k8s.gcr.io/kubernetes-dashboard-amd64:v1.10.1"
Normal Pulled 5m29s kubelet, localhost Successfully pulled image "k8s.gcr.io/kubernetes-dashboard-amd64:v1.10.1"
Normal Started 4m43s (x4 over 5m29s) kubelet, localhost Started container kubernetes-dashboard
Normal Created 3m55s (x5 over 5m29s) kubelet, localhost Created container kubernetes-dashboard
Normal Pulled 3m55s (x4 over 5m29s) kubelet, localhost Container image "k8s.gcr.io/kubernetes-dashboard-amd64:v1.10.1" already present on machine
Warning BackOff 35s (x28 over 5m28s) kubelet, localhost Back-off restarting failed container
[root@master kubernetes]# kubectl logs kubernetes-dashboard-7b89455d8-zf4jk --namespace=kube-system
2019/08/01 04:33:50 Starting overwatch
2019/08/01 04:33:50 Using apiserver-host location: https://192,168.10.102:6443
2019/08/01 04:33:50 Skipping in-cluster config
2019/08/01 04:33:50 Using random key for csrf signing
2019/08/01 04:33:50 Error while initializing connection to Kubernetes apiserver. This most likely means that the cluster is misconfigured (e.g., it has invalid apiserver certificates or service account's configuration) or the --apiserver-host param points to a server that does not exist. Reason: Get https://192,168.10.102:6443/version: dial tcp: lookup 192,168.10.102: no such host
Refer to our FAQ and wiki pages for more information: https://github.com/kubernetes/dashboard/wiki/FAQ
通過上述日誌得知,我們在修改 kubernetes-dashboard.yaml 文件中 apiserver 地址的時候,IP地址寫錯了,修改後重新之心即可。
8、登錄 kubernetes UI
http://192.168.10.101:8080/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/#/login
9、獲取登錄token
[root@master1 kubernetes]# kubectl -n kube-system get secret | grep kubernetes-dashboard-token
kubernetes-dashboard-token-5sfkz kubernetes.io/service-account-token 3 2m37s
[root@master1 kubernetes]# kubectl describe -n kube-system secret/kubernetes-dashboard-token-5sfkz
Name: kubernetes-dashboard-token-5sfkz
Namespace: kube-system
Labels: <none>
Annotations: kubernetes.io/service-account.name: kubernetes-dashboard
kubernetes.io/service-account.uid: 035b148d-0fd1-4d6d-9ae5-95cf9d57eb3f
Type: kubernetes.io/service-account-token
Data
====
ca.crt: 1025 bytes
namespace: 11 bytes
token: eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJrdWJlcm5ldGVzLWRhc2hib2FyZC10b2tlbi01c2ZreiIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjAzNWIxNDhkLTBmZDEtNGQ2ZC05YWU1LTk1Y2Y5ZDU3ZWIzZiIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDprdWJlLXN5c3RlbTprdWJlcm5ldGVzLWRhc2hib2FyZCJ9.IL-JOSmQTUjGcVDQ2zWu7O4olsFtme_11XQu3hoezwxS03NM-BNILgRfeJHGSheKsKb-ZlgfCsswKynLLLFzzWPK2wYixnE2osdmadKba7kV7-fhvH1LseMSBZeSwhdOGor73JXd2xc2KyEFo3HdhhFly-q4I2rzq73uUaz010iFM9s2dCnxGgzu4JcLhmFKBeenrYJMVCE1NuazG69C5vugpvY1FCT5w4Y0XFUk5uzYYfSi7bbNKWq7U7iAZwX6Z76CoicqeAy-9MCUwFdVNNOIOcWh287PHmM4gIeyXOzeX3IfR8IHfNilljxmOzcOUwjxDg375k3UsIy9WE4LIA
上面 token 字段即爲登錄 token ,將其複製,粘貼進瀏覽器中的 token 下,可正常登錄
八、問題總結
1、訪問 apiserver 是報錯 forbidden: User "system:anonymous" cannot get path "/" ,如下圖所示:
出現該問題是因爲 Kubernetes API Server 新增了 --anonymous-auth 選項,允許匿名請求訪問 secure port 。沒有被其他 authentication 方法拒絕的請求即 Anonymous requests, 這樣的匿名請求的 username 爲 system:anonymous , 歸屬的組爲 system:unauthenticated。並且該選線是默認的。這樣一來,當採用 chrome 瀏覽器訪問 dashboard UI 或者 apiserver 時很可能無法彈出用戶名、密碼輸入對話框,導致後續authorization失敗。爲了保證用戶名、密碼輸入對話框的彈出,需要將 --anonymous-auth 設置爲 false。
如果 apiserver 是手動安裝的,則修改 /etc/kubernetes/apiserver 文件,在 KUBE_API_ARGS="" 中加入 --anonymous-auth=false
vim /etc/kubernetes/apiserver
KUBE_API_ARGS="--anonymous-auth=false"
如果 apiserver 是通過 kubeadm 安裝的,則修改 /etc/kubernetes/manifests/kube-apiserver.yaml 文件,在 command 下加入 - --anonymous-auth=false
[root@master ~]# vim /etc/kubernetes/manifests/kube-apiserver.yaml
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
labels:
component: kube-apiserver
tier: control-plane
name: kube-apiserver
namespace: kube-system
spec:
containers:
- command:
- kube-apiserver
- --advertise-address=192.168.10.102
- --allow-privileged=true
- --authorization-mode=Node,RBAC
- --client-ca-file=/etc/kubernetes/pki/ca.crt
- --enable-admission-plugins=NodeRestriction
- --enable-bootstrap-token-auth=true
- --etcd-cafile=/etc/kubernetes/pki/etcd/ca.crt
- --etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt
- --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key
- --etcd-servers=https://127.0.0.1:2379
- --insecure-port=0
- --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt
- --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key
- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
- --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt
- --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
- --requestheader-allowed-names=front-proxy-client
- --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
- --requestheader-extra-headers-prefix=X-Remote-Extra-
- --requestheader-group-headers=X-Remote-Group
- --requestheader-username-headers=X-Remote-User
- --secure-port=6443
- --service-account-key-file=/etc/kubernetes/pki/sa.pub
- --service-cluster-ip-range=10.1.0.0/16
- --tls-cert-file=/etc/kubernetes/pki/apiserver.crt
- --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
- --anonymous-auth=false
image: k8s.gcr.io/kube-apiserver:v1.15.0
imagePullPolicy: IfNotPresent
經過上面的操作之後重啓 apiserver 後可恢復正常。