1.
基礎概念
statefulset,可以翻譯成有狀態的設定.
和deployment的對比
deployment部署創建的pod是無狀態的,重新調度pod,pod名字hostname,啓動pod順序刪除pod順序都是隨機的.deployment使用的是共享存儲,所有pod共用一個存儲卷.
statefulset部署創建的pod是有狀態的,重新調度pod,pod名字hostname保持固定不變,啓動pod順序刪除pod順序都可以根據定義的順序有序執行操作,有序的動態更新,statefulset使用的存儲,不是共用一個存儲卷,一個pod對應一個存儲卷(pv).pod重新創建調度掛載的存儲卷保持不變.
2.
statefulset實現用到的技術概念
headless service
headless service不需要配置clusterIP,對應的每一個pod都會有相應的dns域名.因此可以爲每個pod生成可解析的dns記錄.
statefulset
statefulset 用來部署管理pod資源
volumeClaimTemplates
實現每個pod對應一個pv存儲卷
3.
測試環境準備
kubernets集羣必須部署dns,能正常解析.
kubernetes集羣必須配置storageclass,能夠實現動態創建pv.
我這裏用的是nfs存儲環境.
4.
headless service和service解析對比
service
[root@k8s-master1 ~]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
httpd-svc NodePort 10.254.33.250 <none> 80:8768/TCP 92d
kubernetes ClusterIP 10.254.0.1 <none> 443/TCP 92d注意下面解析用到dns的ip
[root@k8s-master1 ~]# kubectl get svc -n kube-system
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kube-dns ClusterIP 10.254.0.2 <none> 53/UDP,53/TCP 92d[root@k8s-master1 ~]# kubectl describe svc httpd-svc
Name: httpd-svc
Namespace: default
Labels: <none>
Annotations: kubectl.kubernetes.io/last-applied-configuration={"apiVersion":"v1","kind":"Service","metadata":{"annotations":{},"name":"httpd-svc","namespace":"default"},"spec":{"ports":[{"port":80}],"selector":{"a...
Selector: app=httpd-app
Type: NodePort
IP: 10.254.33.250
Port: <unset> 80/TCP
TargetPort: 80/TCP
NodePort: <unset> 8768/TCP
Endpoints: 172.30.37.7:80
Session Affinity: None
External Traffic Policy: Cluster
Events: <none>
[root@k8s-master1 ~]#隨便進個namespace是default的pod用nslookup解析
[root@k8s-master1 ~]# kubectl exec -it nfs-client-provisioner-65bf6bd464-qdzcj nslookup httpd-svc.default.svc.cluster.local 10.254.0.2
Server: 10.254.0.2
Address 1: 10.254.0.2 kube-dns.kube-system.svc.cluster.local
Name: httpd-svc.default.svc.cluster.local
Address 1: 10.254.33.250 httpd-svc.default.svc.cluster.local
[root@k8s-master1 ~]#headless service
[root@k8s-master1 test]# cat headless-svc-test.yaml
apiVersion: v1
kind: Service
metadata:
name: headless-svc
labels:
app: headless-svc
spec:
ports:
- port: 80
name: myweb
selector:
app: headless-pod
clusterIP: None
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: headless-test
spec:
replicas: 3
selector:
matchLabels:
app: headless-pod
template:
metadata:
labels:
app: headless-pod
spec:
containers:
- image: httpd
name: myhttpd
ports:
- containerPort: 80
[root@k8s-master1 test]#[root@k8s-master1 test]# kubectl describe svc headless-svc
Name: headless-svc
Namespace: default
Labels: app=headless-svc
Annotations: kubectl.kubernetes.io/last-applied-configuration={"apiVersion":"v1","kind":"Service","metadata":{"annotations":{},"labels":{"app":"headless-svc"},"name":"headless-svc","namespace":"default"},"spec":{"...
Selector: app=headless-pod
Type: ClusterIP
IP: None
Port: myweb 80/TCP
TargetPort: 80/TCP
Endpoints: 172.30.65.4:80,172.30.65.5:80,172.30.81.6:80
Session Affinity: None
Events: <none>
[root@k8s-master1 test]#隨便進個namespace是default的pod用nslookup解析
[root@k8s-master1 test]# kubectl exec -it nfs-client-provisioner-65bf6bd464-qdzcj nslookup headless-svc.default.svc.cluster.local 10.254.0.2
Server: 10.254.0.2
Address 1: 10.254.0.2 kube-dns.kube-system.svc.cluster.local
Name: headless-svc.default.svc.cluster.local
Address 1: 172.30.65.5 172-30-65-5.headless-svc.default.svc.cluster.local
Address 2: 172.30.65.4 172-30-65-4.headless-svc.default.svc.cluster.local
Address 3: 172.30.81.6 172-30-81-6.headless-svc.default.svc.cluster.local
[root@k8s-master1 test]#對比可知:
配置headless service 和service的區別只是加一個clusterIP: None
dns解析的不同
service 解析返回svc的ip和dns域名
headless service 解析返回pod的ip和dns域名
headless service解析能夠返回pod的hostname.
statefulset需要固定pod name,hostname,必須要能解析pod的hostname.因此使用headless service.
5.
statefulset配置測試
[root@k8s-master1 test]# cat statefulset-test.yaml
apiVersion: v1
kind: Service
metadata:
name: headless-svc
labels:
app: headless-svc
spec:
ports:
- port: 80
name: myweb
selector:
app: headless-pod
clusterIP: None
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: statefulset-test
spec:
serviceName: headless-svc
replicas: 3
selector:
matchLabels:
app: headless-pod
template:
metadata:
labels:
app: headless-pod
spec:
containers:
- image: httpd
name: myhttpd
ports:
- containerPort: 80
name: httpd
[root@k8s-master1 test]#[root@k8s-master1 test]# kubectl describe svc headless-svc
Name: headless-svc
Namespace: default
Labels: app=headless-svc
Annotations: kubectl.kubernetes.io/last-applied-configuration={"apiVersion":"v1","kind":"Service","metadata":{"annotations":{},"labels":{"app":"headless-svc"},"name":"headless-svc","namespace":"default"},"spec":{"...
Selector: app=headless-pod
Type: ClusterIP
IP: None
Port: myweb 80/TCP
TargetPort: 80/TCP
Endpoints: 172.30.65.4:80,172.30.65.5:80,172.30.81.6:80
Session Affinity: None
Events: <none>[root@k8s-master1 test]# kubectl get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE
statefulset-test-0 1/1 Running 0 4m 172.30.65.4 k8s-master3
statefulset-test-1 1/1 Running 0 4m 172.30.81.6 k8s-master2
statefulset-test-2 1/1 Running 0 4m 172.30.65.5 k8s-master3hostname
[root@k8s-master1 test]# kubectl exec -it statefulset-test-0 hostname
statefulset-test-0
[root@k8s-master1 test]# kubectl exec -it statefulset-test-1 hostname
statefulset-test-1
[root@k8s-master1 test]# kubectl exec -it statefulset-test-2 hostname
statefulset-test-2解析
[root@k8s-master1 test]# kubectl exec -it nfs-client-provisioner-65bf6bd464-qdzcj nslookup headless-svc.default.svc.cluster.local 10.254.0.2
Server: 10.254.0.2
Address 1: 10.254.0.2 kube-dns.kube-system.svc.cluster.local
Name: headless-svc.default.svc.cluster.local
Address 1: 172.30.65.5 statefulset-test-2.headless-svc.default.svc.cluster.local
Address 2: 172.30.65.4 statefulset-test-0.headless-svc.default.svc.cluster.local
Address 3: 172.30.81.6 statefulset-test-1.headless-svc.default.svc.cluster.local
[root@k8s-master1 test]#statefulset部署和上面的head services使用deployment部署對比
pod名字是按照順序生成,從0開始每次+1命名pod
創建pod,也是按照順序一個一個創建,沒有像deployment部署那樣批量一次性全部創建.
測試
刪除pod
[root@k8s-master1 test]# kubectl delete pod statefulset-test-0
pod "statefulset-test-0" deleted[root@k8s-master1 test]# kubectl get pod
NAME READY STATUS RESTARTS AGE
statefulset-test-0 0/1 Terminating 0 19m
statefulset-test-1 1/1 Running 0 19m
statefulset-test-2 1/1 Running 0 19m重新自動創建pod
[root@k8s-master1 test]# kubectl get pod
NAME READY STATUS RESTARTS AGE
statefulset-test-0 0/1 ContainerCreating 0 0s
statefulset-test-1 1/1 Running 0 19m
statefulset-test-2 1/1 Running 0 19m[root@k8s-master1 test]# kubectl get pod
NAME READY STATUS RESTARTS AGE
statefulset-test-0 1/1 Running 0 4s
statefulset-test-1 1/1 Running 0 20m
statefulset-test-2 1/1 Running 0 19m[root@k8s-master1 test]# kubectl exec -it statefulset-test-0 hostname
statefulset-test-0
[root@k8s-master1 test]#可見重新創建pod,pod的名字和hostname沒有改變.
把replicas改成1
[root@k8s-master1 test]# kubectl apply -f statefulset-test.yaml
service "headless-svc" unchanged
statefulset.apps "statefulset-test" configured
[root@k8s-master1 test]# kubectl get pod
NAME READY STATUS RESTARTS AGE
statefulset-test-0 1/1 Running 0 4m
statefulset-test-1 1/1 Running 0 24m
statefulset-test-2 0/1 Terminating 0 24m
[root@k8s-master1 test]# kubectl get pod
NAME READY STATUS RESTARTS AGE
statefulset-test-0 1/1 Running 0 4m
statefulset-test-1 0/1 Terminating 0 24m
[root@k8s-master1 test]# kubectl get pod
NAME READY STATUS RESTARTS AGE
statefulset-test-0 1/1 Running 0 5mpod按照順序一個一個刪除
把replicas改回3
[root@k8s-master1 test]# kubectl apply -f statefulset-test.yaml
service "headless-svc" unchanged
statefulset.apps "statefulset-test" configured
[root@k8s-master1 test]# kubectl get pod
NAME READY STATUS RESTARTS AGE
statefulset-test-0 1/1 Running 0 6m
statefulset-test-1 0/1 ContainerCreating 0 4s
[root@k8s-master1 test]# kubectl get pod
NAME READY STATUS RESTARTS AGE
statefulset-test-0 1/1 Running 0 6m
statefulset-test-1 1/1 Running 0 8s
statefulset-test-2 0/1 ContainerCreating 0 1s
[root@k8s-master1 test]# kubectl get pod
NAME READY STATUS RESTARTS AGE
statefulset-test-0 1/1 Running 0 6m
statefulset-test-1 1/1 Running 0 14s
statefulset-test-2 1/1 Running 0 7s
[root@k8s-master1 test]#
[root@k8s-master1 test]# kubectl exec -it statefulset-test-2 hostname
statefulset-test-2
[root@k8s-master1 test]#pod按照順序順序一個一個創建,並且pod名字和hostname沒有改變
6.
statefulset結合volumeClaimTemplates測試
[root@k8s-master1 test]# cat statefulset-test.yaml
apiVersion: v1
kind: Service
metadata:
name: headless-svc
labels:
app: headless-svc
spec:
ports:
- port: 80
name: myweb
selector:
app: headless-pod
clusterIP: None
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: statefulset-test
spec:
serviceName: headless-svc
replicas: 3
selector:
matchLabels:
app: headless-pod
template:
metadata:
labels:
app: headless-pod
spec:
containers:
- image: httpd
name: myhttpd
ports:
- containerPort: 80
name: httpd
volumeMounts:
- mountPath: /mnt
name: test
volumeClaimTemplates:
- metadata:
name: test
annotations:
volume.beta.kubernetes.io/storage-class: managed-nfs-storage
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 100Mi注意格式,多個空格少個空格都會失敗.
annotations:
volume.beta.kubernetes.io/storage-class: managed-nfs-storage這是指定storageclass
[root@k8s-master1 test]# kubectl get pv,pvc
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
persistentvolume/pvc-a1c6928d-aebc-11e9-8580-000c291d7023 100Mi RWO Delete Bound default/test-statefulset-test-0 managed-nfs-storage 1m
persistentvolume/pvc-a94813df-aebc-11e9-8580-000c291d7023 100Mi RWO Delete Bound default/test-statefulset-test-1 managed-nfs-storage 1m
persistentvolume/pvc-b234f5a7-aebc-11e9-8580-000c291d7023 100Mi RWO Delete Bound default/test-statefulset-test-2 managed-nfs-storage 49s
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
persistentvolumeclaim/test-statefulset-test-0 Bound pvc-a1c6928d-aebc-11e9-8580-000c291d7023 100Mi RWO managed-nfs-storage 1m
persistentvolumeclaim/test-statefulset-test-1 Bound pvc-a94813df-aebc-11e9-8580-000c291d7023 100Mi RWO managed-nfs-storage 1m
persistentvolumeclaim/test-statefulset-test-2 Bound pvc-b234f5a7-aebc-11e9-8580-000c291d7023 100Mi RWO managed-nfs-storage 1m
[root@k8s-master1 test]#檢索pod對應的pv卷,截取volumes部分對比
[root@k8s-master1 test]# kubectl describe pod statefulset-test-0
Name: statefulset-test-0
Namespace: default
Volumes:
test:
Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
ClaimName: test-statefulset-test-0
ReadOnly: false
[root@k8s-master1 test]# kubectl describe pod statefulset-test-1
Name: statefulset-test-1
Namespace: default
Volumes:
test:
Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
ClaimName: test-statefulset-test-1
ReadOnly: false
[root@k8s-master1 test]# kubectl describe pod statefulset-test-2
Name: statefulset-test-2
Namespace: default
Volumes:
test:
Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
ClaimName: test-statefulset-test-2
ReadOnly: false可以看到一個pod對應使用了一個pv卷.
檢索nfs server存取目錄,可以看到新建的pvc目錄
[root@k8s-master3 k8s]# ls
default-test-statefulset-test-0-pvc-a1c6928d-aebc-11e9-8580-000c291d7023
default-test-statefulset-test-1-pvc-a94813df-aebc-11e9-8580-000c291d7023
default-test-statefulset-test-2-pvc-b234f5a7-aebc-11e9-8580-000c291d7023
[root@k8s-master3 k8s]#測試
刪除pod,看看能重新對應原來的pvc卷嗎？
[root@k8s-master1 test]# kubectl get pod
NAME READY STATUS RESTARTS AGE
statefulset-test-0 1/1 Running 0 12m
statefulset-test-1 1/1 Running 0 11m
statefulset-test-2 1/1 Running 0 11m在statefulset-test-1和statefulset-test-2 新建個文件存儲在pvc
[root@k8s-master1 test]# kubectl exec -it statefulset-test-1 touch /mnt/t1
[root@k8s-master1 test]# kubectl exec -it statefulset-test-2 touch /mnt/t2檢索pvc卷
[root@k8s-master3 k8s]# cd default-test-statefulset-test-1-pvc-a94813df-aebc-11e9-8580-000c291d7023/
[root@k8s-master3 default-test-statefulset-test-1-pvc-a94813df-aebc-11e9-8580-000c291d7023]# ls
t1
[root@k8s-master3 default-test-statefulset-test-1-pvc-a94813df-aebc-11e9-8580-000c291d7023]# cd ..
[root@k8s-master3 k8s]# cd default-test-statefulset-test-2-pvc-b234f5a7-aebc-11e9-8580-000c291d7023/
[root@k8s-master3 default-test-statefulset-test-2-pvc-b234f5a7-aebc-11e9-8580-000c291d7023]# ls
t2刪除這兩個pod
[root@k8s-master1 test]# kubectl delete pod statefulset-test-1 statefulset-test-2
pod "statefulset-test-1" deleted
pod "statefulset-test-2" deleted刪除後,pod自動創建創建
再檢索重新創建的pod
[root@k8s-master1 test]# kubectl describe pod statefulset-test-1
Name: statefulset-test-1
Namespace: default
Volumes:
test:
Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
ClaimName: test-statefulset-test-1
ReadOnly: false
[root@k8s-master1 test]# kubectl describe pod statefulset-test-2
Name: statefulset-test-2
Namespace: default
Volumes:
test:
Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
ClaimName: test-statefulset-test-2
ReadOnly: false可以看到pod命令hostname和存儲卷全部都和刪除前保持一樣.
[root@k8s-master1 test]# kubectl exec -it statefulset-test-1 ls /mnt
t1
[root@k8s-master1 test]# kubectl exec -it statefulset-test-2 ls /mnt
t2
[root@k8s-master1 test]#前面創建的保存在存儲卷的文件也保持不變.
7.
留下的一個疑問:
deployment部署,svc通過kube-proxy結合ipvs和iptables實現訪問後端pod.
statefulset部署,svc是如何實現訪問後端pod呢?規則是什麼?