Kubernetes的Pod網絡設置
網絡插件加載前
k8s.io/kubernetes/cmd/kubelet/kubelet.go:
if err := app.Run(s, nil); err != nil {
k8s.io/kubernetes/cmd/kubelet/app/server.go:
if err := run(s, kubeDeps); err != nil {
k8s.io/kubernetes/cmd/kubelet/app/server.go, run():
kubeDeps, err = UnsecuredKubeletDeps(s)
k8s.io/kubernetes/cmd/kubelet/app/server.go, UnsecuredKubeletDeps():
NetworkPlugins: ProbeNetworkPlugins(s.NetworkPluginDir, s.CNIConfDir, s.CNIBinDir),
k8s.io/kubernetes/cmd/kubelet/app/plugins.go:
// ProbeNetworkPlugins collects all compiled-in plugins
func ProbeNetworkPlugins(pluginDir, cniConfDir, cniBinDir string) []network.NetworkPlugin {
allPlugins := []network.NetworkPlugin{}
// for backwards-compat, allow pluginDir as a source of CNI config files
if cniConfDir == "" {
cniConfDir = pluginDir
}
// for each existing plugin, add to the list
allPlugins = append(allPlugins, cni.ProbeNetworkPlugins(cniConfDir, cniBinDir)...)
allPlugins = append(allPlugins, kubenet.NewPlugin(pluginDir))
return allPlugins
}
網絡插件加載
k8s.io/kubernetes/pkg/kubelet/network/cni/cni.go:
func ProbeNetworkPlugins(pluginDir, binDir string) []network.NetworkPlugin {
return probeNetworkPluginsWithVendorCNIDirPrefix(pluginDir, binDir, "")
}
k8s.io/kubernetes/pkg/kubelet/network/cni/cni.go，probeNetworkPluginsWithVendorCNIDirPrefix():
func probeNetworkPluginsWithVendorCNIDirPrefix(pluginDir, binDir, vendorCNIDirPrefix string) []network.NetworkPlugin {
if binDir == "" {
binDir = DefaultCNIDir
}
plugin := &cniNetworkPlugin{
defaultNetwork: nil,
loNetwork: getLoNetwork(binDir, vendorCNIDirPrefix),
execer: utilexec.New(),
pluginDir: pluginDir,
binDir: binDir,
vendorCNIDirPrefix: vendorCNIDirPrefix,
}
// sync NetworkConfig in best effort during probing.
plugin.syncNetworkConfig()
return []network.NetworkPlugin{plugin}
}
讀取cni配置文件，設置默認網絡
k8s.io/kubernetes/pkg/kubelet/network/cni/cni.go:
func (plugin *cniNetworkPlugin) syncNetworkConfig() {
network, err := getDefaultCNINetwork(plugin.pluginDir, plugin.binDir, plugin.vendorCNIDirPrefix)
if err != nil {
glog.Warningf("Unable to update cni config: %s", err)
return
}
plugin.setDefaultNetwork(network)
}
k8s.io/kubernetes/pkg/kubelet/network/cni/cni.go, getDefaultCNINetwork():
//從pluginDir目錄中讀取所有.conf文件
files, err := libcni.ConfFiles(pluginDir)
...
for _, confFile := range files {
conf, err := libcni.ConfFromFile(confFile)
if err != nil {
glog.Warningf("Error loading CNI config file %s: %v", confFile, err)
continue
}
// Search for vendor-specific plugins as well as default plugins in the CNI codebase.
vendorDir := vendorCNIDir(vendorCNIDirPrefix, conf.Network.Type)
cninet := &libcni.CNIConfig{
Path: []string{binDir, vendorDir},
}
network := &cniNetwork{name: conf.Network.Name, NetworkConfig: conf, CNIConfig: cninet}
return network, nil
}
...
配置文件的格式爲:
type NetConf struct {
Name string `json:"name,omitempty"`
Type string `json:"type,omitempty"`
IPAM struct {
Type string `json:"type,omitempty"`
} `json:"ipam,omitempty"`
DNS DNS `json:"dns"`
}
type DNS struct {
Nameservers []string `json:"nameservers,omitempty"`
Domain string `json:"domain,omitempty"`
Search []string `json:"search,omitempty"`
Options []string `json:"options,omitempty"`
}
網絡插件初始化
前面的過程結束後，kubeDeps.NetworkPlugins中就設置好了指定的插件。
在k8s.io/kubernetes/pkg/kubelet/kubelet.go，NewMainKubelet()中:
if plug, err := network.InitNetworkPlugin(kubeDeps.NetworkPlugins,
kubeCfg.NetworkPluginName,
&criNetworkHost{&networkHost{klet}, &network.NoopPortMappingGetter{}},
klet.hairpinMode,
klet.nonMasqueradeCIDR,
int(kubeCfg.NetworkPluginMTU)); err != nil {
k8s.io/kubernetes/pkg/kubelet/network/plugins.go, InitNetworkPlugin()
chosenPlugin := pluginMap[networkPluginName]
if chosenPlugin != nil {
err := chosenPlugin.Init(host, hairpinMode, nonMasqueradeCIDR, mtu)
直接調用的cniNetworkPlugin的Init()函數:
func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode, nonMasqueradeCIDR string, mtu int) error {
var err error
plugin.nsenterPath, err = plugin.execer.LookPath("nsenter")
if err != nil {
return err
}
plugin.host = host
plugin.syncNetworkConfig()
return nil
}
網絡插件的使用
cniNetworkPlugin的定義:
--cniNetworkPlugin : struct
[fields]
-binDir : string
-defaultNetwork : *cniNetwork
-execer : utilexec.Interface
-host : network.Host
-loNetwork : *cniNetwork
-nsenterPath : string //二進制文件nsenter的路徑
-pluginDir : string
-vendorCNIDirPrefix : string
[embedded]
+network.NoopNetworkPlugin : network.NoopNetworkPlugin
+sync.RWMutex : sync.RWMutex
[methods]
+GetPodNetworkStatus(namespace string, name string, id kubecontainer.ContainerID) : *network.PodNetworkStatus, error
+Init(host network.Host, hairpinMode componentconfig.HairpinMode, nonMasqueradeCIDR string, mtu int) : error
+Name() : string
+SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations map[string]string) : error
+Status() : error
+TearDownPod(namespace string, name string, id kubecontainer.ContainerID) : error
-checkInitialized() : error
-getDefaultNetwork() : *cniNetwork
-setDefaultNetwork(n *cniNetwork)
-syncNetworkConfig()
Init()用於初始化，setUpPod用於設置容器的網絡，重點關注setUpPod。
SetUpPod()
SetUpPod()的參數分別是namespace,容器的name, pause容器的ID，註解。
這裏必須要說明一下，k8s中的pod至少是包含兩個容器的，其中一個容器作爲infra容器，同一個pod中的其它容器和infra容器共享一個網絡ns。
創建pod的時候，kubelet首先創建infra容器，得到infra容器的ID，然後創建其它容器。
k8s.io/kubernetes/pkg/kubelet/dockertools/docker_manager.go:
// If we should create infra container then we do it first.
podInfraContainerID := containerChanges.InfraContainerId
if containerChanges.StartInfraContainer && (len(containerChanges.ContainersToStart) > 0) {
glog.V(4).Infof("Creating pod infra container for %q", format.Pod(pod))
startContainerResult := kubecontainer.NewSyncResult(kubecontainer.StartContainer, PodInfraContainerName)
result.AddSyncResult(startContainerResult)
var msg string
podInfraContainerID, err, msg = dm.createPodInfraContainer(pod)
if err != nil {
startContainerResult.Fail(err, msg)
glog.Errorf("Failed to create pod infra container: %v; Skipping pod %q: %s", err, format.Pod(pod), msg)
return
}
setupNetworkResult := kubecontainer.NewSyncResult(kubecontainer.SetupNetwork, kubecontainer.GetPodFullName(pod))
result.AddSyncResult(setupNetworkResult)
if !kubecontainer.IsHostNetworkPod(pod) {
if err := dm.network.SetUpPod(pod.Namespace, pod.Name, podInfraContainerID.ContainerID(), pod.Annotations); err != nil {
setupNetworkResult.Fail(kubecontainer.ErrSetupNetwork, err.Error())
glog.Error(err)
// Delete infra container
......
所以在SetUpPod中設置好podInfraContainerID的網絡即可。
將容器加入指定網絡的實現
每個plugin都有一個類型爲defaultNetwork的成員cniNetwork,k8s.io/kubernetes/pkg/kubelet/network/cni/cni.go:
defaultNetwork *cniNetwork
加入、推出網絡都是調用defaultNetwork的成員函數，k8s.io/kubernetes/pkg/kubelet/network/cni/cni.go
func (plugin *cniNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations map[string]string) error {
if err := plugin.checkInitialized(); err != nil {
return err
}
netnsPath, err := plugin.host.GetNetNS(id.ID)
if err != nil {
return fmt.Errorf("CNI failed to retrieve network namespace path: %v", err)
}
_, err = plugin.loNetwork.addToNetwork(name, namespace, id, netnsPath)
if err != nil {
glog.Errorf("Error while adding to cni lo network: %s", err)
return err
}
_, err = plugin.getDefaultNetwork().addToNetwork(name, namespace, id, netnsPath)
if err != nil {
glog.Errorf("Error while adding to cni network: %s", err)
return err
}
return err
}
k8s.io/kubernetes/pkg/kubelet/network/cni/cni.go，addToNetwork:
netconf, cninet := network.NetworkConfig, network.CNIConfig
glog.V(4).Infof("About to run with conf.Network.Type=%v", netconf.Network.Type)
res, err := cninet.AddNetwork(netconf, rt)
可以看到最終使用的是成員CNIConfig的AddNetwork()完成的。
defaultNetwork在k8s.io/kubernetes/pkg/kubelet/network/cni/cni.go，getDefaultCNINetwork()中創建:
vendorDir := vendorCNIDir(vendorCNIDirPrefix, conf.Network.Type)
cninet := &libcni.CNIConfig{
Path: []string{binDir, vendorDir},
}
network := &cniNetwork{name: conf.Network.Name, NetworkConfig: conf, CNIConfig: cninet}
return network, nil
cninet的類型是libcni.CNIConfig:
libcni.CNIConfig
k8s.io/kubernetes/vendor/github.com/containernetworking/cni/libcni/api.go:
func (c *CNIConfig) AddNetwork(net *NetworkConfig, rt *RuntimeConf) (*types.Result, error) {
pluginPath, err := invoke.FindInPath(net.Network.Type, c.Path)
if err != nil {
return nil, err
}
return invoke.ExecPluginWithResult(pluginPath, net.Bytes, c.args("ADD", rt))
}
invoke.FindInPath在c.Path目錄下尋找名爲net.Network.Type的文件，返回文件的完整路徑pluginPath
最後，直接使用plugin的子命令ADD，將容器添加到指定網絡中。