搭建LB負載均衡和keepalived
一、環境優化
LB1
[root@localhost ~]# hostnamectl set-hostname lb1
[root@localhost ~]# su //修改主機名
[root@lb1 ~]# systemctl stop NetworkManager
//關閉NetworkManage服務
[root@lb1 ~]# systemctl disable NetworkManager
Removed symlink /etc/systemd/system/multi-user.target.wants/NetworkManager.service.
Removed symlink /etc/systemd/system/dbus-org.freedesktop.nm-dispatcher.service.
Removed symlink /etc/systemd/system/network-online.target.wants/NetworkManager-wait-online.service.
[root@lb1 ~]# setenforce //關閉增強型安全功能
[root@lb1 ~]# sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
[root@lb1 ~]# iptables -F //清空防火牆策略LB2
[root@localhost ~]# hostnamectl set-hostname lb2
[root@localhost ~]# su
[root@lb2 ~]# systemctl stop NetworkManager
[root@lb2 ~]# systemctl disable NetworkManager
Removed symlink /etc/systemd/system/multi-user.target.wants/NetworkManager.service.
Removed symlink /etc/systemd/system/dbus-org.freedesktop.nm-dispatcher.service.
Removed symlink /etc/systemd/system/network-online.target.wants/NetworkManager-wait-online.service.
[root@lb2 ~]# setenforce 0
[root@lb2 ~]# sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
[root@lb2 ~]# iptables -F2.在兩個LB服務器上安裝nginx
[root@lb1 ~]# echo -e '[nginx]\nname=nginx.repo\nbaseurl=http://nginx.org/packages/centos/7/$basearch/\ngpgcheck=0' > /etc/yum.repos.d/nginx.repo
[root@lb1 ~]# yum makecache
[root@lb1 ~]# yum install nginx -y[root@lb2 ~]# echo -e '[nginx]\nname=nginx.repo\nbaseurl=http://nginx.org/packages/centos/7/$basearch/\ngpgcheck=0' > /etc/yum.repos.d/nginx.repo
[root@lb2 ~]# yum makecache
[root@lb2 ~]# yum install nginx -y3.在兩個LB服務器添加四層轉發upstream
以LB1爲例
[root@lb1 ~]# vim /etc/nginx/nginx.conf
events {
worker_connections 1024;
}
stream {
log_format main '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';
access_log /var/log/nginx/k8s-access.log main;
upstream k8s-apiserver {
server 192.168.191.134:6443;
server 192.168.191.133:6443;
#兩個master地址，apiserver端口號6443
}
server {
listen 6443;
proxy_pass k8s-apiserver;
}
}
http {
[root@lb1 ~]# nginx -t
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful4.開啓nginx服務（以LB1爲例）
[root@lb1 ~]# systemctl start nginx
[root@lb1 ~]# systemctl status nginx
● nginx.service - nginx - high performance web server
Loaded: loaded (/usr/lib/systemd/system/nginx.service; disabled; vendor preset: disabled)
Active: active (running) since Sat 2020-05-03 13:02:50 CST; 5s ago
Docs: http://nginx.org/en/docs/
Process: 29485 ExecStart=/usr/sbin/nginx -c /etc/nginx/nginx.conf (code=exited, status=0/SUCCESS)
Main PID: 29488 (nginx)
Tasks: 2
CGroup: /system.slice/nginx.service
├─29488 nginx: master process /usr/sbin/nginx -c /etc/nginx/nginx.conf
└─29489 nginx: worker process
May 03 13:02:50 lb1 systemd[1]: Starting nginx - high performance web server...
May 03 13:02:50 lb1 systemd[1]: Started nginx - high performance web server.
[root@lb1 ~]# systemctl enable nginx
Created symlink from /etc/systemd/system/multi-user.target.wants/nginx.service to /usr/lib/systemd/system/nginx.service.5.本地驗證
二、部署Keepalived
以LB1爲例
1.安裝Keepalived
[root@lb1 ~]# yum install keepalived -y2.修改Keepalived配置文件.
[root@lb1 ~]# mkdir /abc
[root@lb1 ~]# mount.cifs //192.168.0.88/linuxs /abc
Password for root@//192.168.0.88/linuxs:
[root@lb1 ~]# cp /abc/k8s/keepalived.conf /etc/keepalived/keepalived.conf
cp: overwrite ‘/etc/keepalived/keepalived.conf’? y[root@lb1 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
# 接收郵件地址
notification_email {
[email protected]
[email protected]
[email protected]
}
# 郵件發送地址
notification_email_from [email protected]
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id NGINX_MASTER
}
vrrp_script check_nginx {
script "/etc/check_nginx.sh" #這個配置文件後面會編輯
}
vrrp_instance VI_1 {
state MASTER
interface ens32 #指定物理網口
virtual_router_id 51 # VRRP 路由 ID實例，每個實例是唯一的
priority 100 # 優先級，備服務器設置 90
advert_int 1 # 指定VRRP 心跳包通告間隔時間，默認1秒
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.191.135/24 #指定虛擬IP
}
track_script { #監控腳本
check_nginx
}
}LB2虛擬路由IP不要一致，state爲BACKUP，其他一樣
vrrp_instance VI_1 {
state BACKUP
interface ens32
virtual_router_id 52
priority 903.編輯nginx腳本
[root@lb1 ~]# vim /etc/nginx/check_nginx.sh
count=$(ps -ef |grep nginx |egrep -cv "grep|$$")
#變量是建廠nginx是否開啓，如果沒有開啓，那麼就關閉keepalived
if [ "$count" -eq 0 ];then
/etc/init.d/keepalived stop
fi
[root@lb1 ~]# chmod +x /etc/nginx/check_nginx.sh4.開啓LB1的Keepalived服務，LB2backup狀態
[root@lb1 ~]# systemctl start keepalived.service 5.使用ip a命令可以查看到虛擬IP
[root@lb1 ~]# ip a
2: ens32: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:ef:81:b6 brd ff:ff:ff:ff:ff:ff
inet 192.168.191.130/24 brd 192.168.191.255 scope global noprefixroute dynamic ens32
valid_lft 5355394sec preferred_lft 5355394sec
inet 192.168.191.135/24 scope global secondary ens32三、將k8s中的node節點關於apiserver地址指向爲vip
1.node節點通過master找vip
[root@node01 ~]# cd /k8s/cfg/
[root@node01 cfg]# ls
bootstrap.kubeconfig kubelet.config kube-proxy
kubelet kubelet.kubeconfig kube-proxy.kubeconfig
[root@node01 cfg]# vim bootstrap.kubeconfig
server: https://192.168.191.133:6443
[root@node01 cfg]# vim kubelet.kubeconfig
server: https://192.168.191.133:6443
[root@node01 cfg]# vim kube-proxy.kubeconfig
server: https://192.168.191.133:64432.重啓kubelet、proxy服務
[root@node01 cfg]# systemctl restart kubelet.service
[root@node01 cfg]# systemctl restart kube-proxy.service 3.替換完成自檢
[root@node01 cfg]# grep 100 *
bootstrap.kubeconfig: server: https://192.168.191.133:6443
kubelet.kubeconfig: server: https://192.168.191.133:6443
kube-proxy.kubeconfig: server: https://192.168.191.133:64434.驗證apiserver飄逸地址
先在lb1節點關掉nginx，再在lb2查看虛擬IP是否生效，若是檢測到nginx發現關閉，keepalived會自動關閉
[root@lb1 ~]# pkill nginx
[root@lb1 ~]# ps -ef |grep nginx |egrep -cv "grep|$$"
0
[root@lb1 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens32: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:ef:81:b6 brd ff:ff:ff:ff:ff:ff
inet 192.168.191.130/24 brd 192.168.191.255 scope global noprefixroute dynamic ens32
valid_lft 5354179sec preferred_lft 5354179sec
inet6 fe80::d8f:d3dc:3ef7:446/64 scope link noprefixroute
valid_lft forever preferred_lft forever此時vip.不在LB1上，再查看LB2
[root@lb2 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens32: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:df:af:4e brd ff:ff:ff:ff:ff:ff
inet 192.168.191.132/24 brd 192.168.247.255 scope global noprefixroute dynamic ens32
valid_lft 5354144sec preferred_lft 5354144sec
inet 192.168.191.133/24 scope global secondary ens333.重啓LB1上的nginx 在查看在線vip回到了LB1上
[root@lb1 ~]# systemctl restart nginx
[root@lb1 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens32: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:ef:81:b6 brd ff:ff:ff:ff:ff:ff
inet 192.168.191.132/24 brd 192.168.191.255 scope global noprefixroute dynamic ens32
valid_lft 5354038sec preferred_lft 5354038sec
inet 192.168.247.100/24 scope global secondary ens33四、創建pod測試一下
1.此時node節點docker狀態爲
node1
[root@node01 cfg]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
39f034a2f24e centos:7 "/bin/bash" 3 days ago Up 3 days beautiful_jennings
[root@node01 cfg]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
centos 7 5e35e350aded 5 months ago 203MBnode2
[root@node02 cfg]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
centos 7 5e35e350aded 5 months ago 203MB
[root@node02 cfg]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
fea29d0ff39b centos:7 "/bin/bash" 3 days ago 2.使用kublet創建pod
在集羣中運行一個指定的鏡像
[root@master1 cfg]# kubectl run nginx --image=nginx
kubectl run --generator=deployment/apps.v1beta1 is DEPRECATED and will be removed in a future version. Use kubectl create instead.
deployment.apps/nginx created
[root@master1 cfg]# kubectl get pods
NAME READY STATUS RESTARTS AGE
nginx-dbddb74b8-sx4m6 1/1 Running 0 49spod在run運行狀態前，還有一個containercreating創建狀態
kubectl controls the Kubernetes cluster manager.
Find more information at: https://kubernetes.io/docs/reference/kubectl/overview/
Basic Commands (Beginner):
create 從文件或stdin創建資源。
expose 使用 replication controller, service, deployment 或者 pod 並暴露它作爲一個 新的 Kubernetes Service
run 在集羣中運行一個指定的鏡像
set 爲 objects 設置一個指定的特徵
Basic Commands (Intermediate):
explain 查看資源的文檔
get 顯示一個或更多 resources
edit 在服務器上編輯一個資源
delete 按文件名、stdin、資源和名稱刪除資源，或按資源和標籤選擇器刪除資源
Deploy Commands:
rollout 管理資源的推出
scale 爲 Deployment, ReplicaSet, Replication Controller 或者 Job 設置一個新的副本數量
autoscale 自動調整一個 Deployment, ReplicaSet, 或者 ReplicationController 的副本數量
Cluster Management Commands:
certificate 修改 certificate 資源.
cluster-info 顯示集羣信息
top Display Resource (CPU/Memory/Storage) usage.
cordon 標記 node 爲 unschedulable
uncordon 標記 node 爲 schedulable
drain Drain node in preparation for maintenance
taint 更新一個或者多個 node 上的 taints
Troubleshooting and Debugging Commands:
describe 顯示一個指定 resource 或者 group 的 resources 詳情
logs 輸出容器在 pod 中的日誌
attach Attach 到一個運行中的 container
exec 在一個 container 中執行一個命令
port-forward Forward one or more local ports to a pod
proxy 運行一個 proxy 到 Kubernetes API server
cp 複製 files 和 directories 到 containers 和從容器中複製 files 和 directories.
auth Inspect authorization
Advanced Commands:
apply 通過文件名或標準輸入流(stdin)對資源進行配置
patch 使用 strategic merge patch 更新一個資源的 field(s)
replace 通過 filename 或者 stdin替換一個資源
wait Experimental: Wait for a specific condition on one or many resources.
convert 在不同的 API versions 轉換配置文件
Settings Commands:
label 更新在這個資源上的 labels
annotate 更新一個資源的註解
completion Output shell completion code for the specified shell (bash or zsh)
Other Commands:
alpha Commands for features in alpha
api-resources Print the supported API resources on the server
api-versions Print the supported API versions on the server, in the form of "group/version"
config 修改 kubeconfig 文件
plugin Provides utilities for interacting with plugins.
version 輸出 client 和 server 的版本信息
Usage:
kubectl [flags] [options]
Use "kubectl <command> --help" for more information about a given command.
Use "kubectl options" for a list of global command-line options (applies to all commands).3.查看pod網絡，這也可以查看出此pod被部署到哪個node上
[root@master1 cfg]# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE
nginx-dbddb74b8-sx4m6 1/1 Running 0 16m 172.17.42.3 192.168.191.131 <none>4.此時在node2節點上有三個容器，一個剛剛創建的，一個是容器倉庫，還有一個是之前測試flannel
[root@node02 cfg]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
6eff0af2c578 nginx "nginx -g 'daemon of…" 16 minutes ago Up 16 minutes k8s_nginx_nginx-dbddb74b8-sx4m6_default_cd5a2ea4-8c68-11ea-a668-000c29db840b_0
c4ca11690aa1 registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64:3.0 "/pause" 16 minutes ago Up 16 minutes k8s_POD_nginx-dbddb74b8-sx4m6_default_cd5a2ea4-8c68-11ea-a668-000c29db840b_0
fea29d0ff39b centos:7 "/bin/bash" 3 days ago Up 3 days kind_burnell
[root@node02 cfg]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
nginx latest 602e111c06b6 8 days ago 127MB
centos 7 5e35e350aded 5 months ago 203MB
registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64 3.0 99e59f495ffa 3 years ago 747kB在node2節點上可以直接訪問nginx
[root@node02 cfg]# curl 172.17.42.3
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
body {
width: 35em;
margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif;
}
</style>此時再次查看容器的日誌
[root@master1 cfg]# kubectl logs nginx-dbddb74b8-sx4m6
172.17.42.1 - - [02/May/2020:11:52:45 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.29.0" "-"