丟棄haproxy+Keepalived 使用內核級LVS實現K8S master高可用

使用lvscare 監控後端節點

# 使用centos7 
# lvscare 完成後端master 檢查發現錯誤摘掉後端IP 恢復正常重新加入負載
# 使用sealyun 開源的LVScare 項目
# 項目地址：https://github.com/sealyun/LVScare
# haproxy 在流量很大節點數很多的時候佔用cpu非常高整個性能也不會太高。

編譯LVScare 項目

1、go環境的搭建
 #下載go 二進制包 
cd /usr/local/src/
 wget https://dl.google.com/go/go1.14.linux-amd64.tar.gz
# 解壓 go 二進制包
tar -xvf go1.14.linux-amd64.tar.gz 
# 複製解壓完成的go 目錄到 /usr/local/
mv go ../
# 配置環境變量
vi /etc/profile
export GOPATH=/root/go
export GOBIN=/root/go/bin
PATH=$PATH:/usr/local/go/bin:$HOME/bin:$GOBIN
export PATH
# 生效環境變量
source  /etc/profile
#驗證go 是否安裝正常
go   version
[root@localhost ]# go   version
go version go1.14 linux/amd64
#　安裝git
yum install git -y
# 編譯LVScare
go get github.com/sealyun/lvscare 
# 編譯完成尋找編譯結果
which  lvscare
[root@localhost src]# which  lvscare
/root/go/bin/lvscare

準備配置文件及啓動文件

# 創建配置文件
mkdir -p /apps/lvscare/{bin,conf}
cat << EOF | tee /apps/lvscare/conf/lvscare
LVSCARE_OPTS="care \\
              --vs 10.10.10.10:9443 \\
              --rs 192.168.2.10:6443 \\
              --rs 192.168.2.11:6443 \\             
              --rs 192.168.2.12:6443 \\
              --health-path / \\
              --health-schem https"
EOF
#　配置項說明：
# --vs　vip IP 勁量使用一個陌生ip 新的端口
#  --rs 後端服務器IP 負載端口號
# --health-path　路徑
#  health-schem http https 
#  tcp 負載可以刪除 health-path  health-schem
# 創建啓動文件
cat << EOF | tee /usr/lib/systemd/system/lvscare.service
[Unit]
Description=lvscare 
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
LimitNOFILE=1024000
LimitNPROC=1024000
LimitCORE=infinity
LimitMEMLOCK=infinity
EnvironmentFile=-/apps/lvscare/conf/lvscare
ExecStart=/apps/lvscare/bin/lvscare \$LVSCARE_OPTS
Restart=on-failure
KillMode=process
[Install]
WantedBy=multi-user.target
EOF

修改kube-proxy

# 修改 kube-proxy ipvs 模式時會全局刪除ipvs規則所以添加過濾IP
# 修改的參數ipvs-exclude-cidrs=10.10.10.10/32 
# kubeadm 方式部署
# 由於修改configmaps kube-proxy 一直啓動不了所以改爲修改啓動參數
 kubectl -n kube-system edit configmaps kube-proxy
# 修改內容
    ipvs:
      excludeCIDRs:
      - "10.10.10.10/32" vip ip
#二進制方式部署修改內容
# 修改內容
        - --logtostderr=true
        - --v=4
        - --feature-gates=SupportIPVSProxyMode=true
        - --masquerade-all=true
        - --proxy-mode=ipvs
        - --ipvs-min-sync-period=5s
        - --ipvs-sync-period=5s
        - --ipvs-scheduler=rr
        - --cluster-cidr=10.244.0.0/16
        - --metrics-bind-address=0.0.0.0
        - --ipvs-exclude-cidrs=10.10.10.10/32 vip ip

啓動LVScare

# 設置開機啓動 
systemctl enable lvscare.service
# 啓動 lvscare
systemctl start lvscare.service
# 查看啓動狀態
systemctl status lvscare.service
[root@localhost apps]# systemctl status lvscare.service
● lvscare.service - lvscare Kubelet
   Loaded: loaded (/usr/lib/systemd/system/lvscare.service; enabled; vendor preset: disabled)
   Active: active (running) since Fri 2020-03-13 15:57:35 CST; 5min ago
 Main PID: 118865 (lvscare)
    Tasks: 12 (limit: 204628)
   Memory: 24.4M
   CGroup: /system.slice/lvscare.service
           └─118865 /apps/lvscare/bin/lvscare care --vs 10.10.10.10:9443 --rs 192.168.2.10:6443 --rs 192.168.2.11:6443 --rs 192.168.2.12:6443 --health-path / --health-schem https

Mar 13 16:02:40 localhost.localdomain lvscare[118865]: check realserver ip: 192.168.2.11, port %!s(uint16=6443)
Mar 13 16:02:40 localhost.localdomain lvscare[118865]: check realserver ip: 192.168.2.12, port %!s(uint16=6443)
Mar 13 16:02:45 localhost.localdomain lvscare[118865]: check svc ip: 10.96.0.10, port 53
Mar 13 16:02:45 localhost.localdomain lvscare[118865]: check svc ip: 10.10.10.10, port 9443
Mar 13 16:02:45 localhost.localdomain lvscare[118865]: check realserver ip: 192.168.2.12, port %!s(uint16=6443)
Mar 13 16:02:45 localhost.localdomain lvscare[118865]: check realserver ip: 192.168.2.11, port %!s(uint16=6443)
Mar 13 16:02:45 localhost.localdomain lvscare[118865]: check realserver ip: 192.168.2.10, port %!s(uint16=6443)
Mar 13 16:02:45 localhost.localdomain lvscare[118865]: check realserver ip: 192.168.2.12, port %!s(uint16=6443)
Mar 13 16:02:45 localhost.localdomain lvscare[118865]: check realserver ip: 192.168.2.11, port %!s(uint16=6443)
Mar 13 16:02:45 localhost.localdomain lvscare[118865]: check realserver ip: 192.168.2.12, port %!s(uint16=6443)

驗證LVScare

[root@localhost apps]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.10.10.10:9443 rr
  -> 192.168.2.10:6443            Masq    1      0          0
  -> 192.168.2.11:6443            Masq    1      0          0
  -> 192.168.2.12:6443            Masq    1      0          0
curl -k https://10.10.10.10:9443
[root@localhost apps]# curl -k https://10.10.10.10:9443
{
  "kind": "Status",
  "apiVersion": "v1",
  "metadata": {

  },
  "status": "Failure",
  "message": "forbidden: User \"system:anonymous\" cannot get path \"/\"",
  "reason": "Forbidden",
  "details": {

  },
  "code": 403
}[root@localhost apps]#
能正常返回數據 lvs 正常
# 驗證LVScare 是否能夠計算刪除故障節點及故障節點恢復自動添加
[root@localhost apps]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.10.10.10:9443 rr
  -> 192.168.2.10:6443            Masq    1      0          0
  -> 192.168.2.11:6443            Masq    1      0          0
  -> 192.168.2.12:6443            Masq    1      0          0
    6443 端口後端正常
    #關閉 192.168.2.12   kube-apiserver
    service kube-apiserver stop  
    [root@k8s-master3 ~]# ps -ef | grep  kube-apiserver
root     3770208 3769875  0 16:07 pts/0    00:00:00 grep --color=auto kube-apiserver

# 192.168.2.12:6443 已經刪除
啓動 192.168.2.12   kube-apiserver 
[root@k8s-master3 ~]# service kube-apiserver start
Redirecting to /bin/systemctl start kube-apiserver.service
[root@k8s-master3 ~]# ps -ef | grep  kube-apiserver
root     3771436       1 99 16:09 ?        00:00:02 /apps/k8s/bin/kube-apiserver -

# 節點已經恢復

最後說明：

# 記得在每個node 節點部署lvscare 
# vip ip 子網是32的只能本機訪問