kubelet grpc實現

kubelet grpc

kubelet中用到grpc主要是操作runtime，kubelet截止到目前的版本1.16.0支持多種runtime：Docker，CRI-O，Containerd，Other CRI runtimes: frakti。這裏主要講的是docker runtime的grcp實現，實現的大致流程如下：kubelet在啓動的時候，封裝了關於runtime的操作的grpc server（通過docker的client）和client端，然後啓動server端，完成server端的啓動之後，在初始化對應server端的client，封裝在kubelet的runtimeService和imageService中，後續對pod相關的容器的操作，都是通過runtimeService和imageService去操作的完成的，至於爲什麼kubelet不直接調用docker的client去實現container相關的功能，而是封裝一層grpc server和client，其主要目的是爲了適配後端不同的runtime。

grpc server 接口定義

type RuntimeServiceServer interface {
	Version(context.Context, *VersionRequest) (*VersionResponse, error)
	RunPodSandbox(context.Context, *RunPodSandboxRequest) (*RunPodSandboxResponse, error)
	StopPodSandbox(context.Context, *StopPodSandboxRequest) (*StopPodSandboxResponse, error)
	RemovePodSandbox(context.Context, *RemovePodSandboxRequest) (*RemovePodSandboxResponse, error)
	PodSandboxStatus(context.Context, *PodSandboxStatusRequest) (*PodSandboxStatusResponse, error)
	ListPodSandbox(context.Context, *ListPodSandboxRequest) (*ListPodSandboxResponse, error)
	CreateContainer(context.Context, *CreateContainerRequest) (*CreateContainerResponse, error)
	StartContainer(context.Context, *StartContainerRequest) (*StartContainerResponse, error)
	StopContainer(context.Context, *StopContainerRequest) (*StopContainerResponse, error)
	RemoveContainer(context.Context, *RemoveContainerRequest) (*RemoveContainerResponse, error)
	ListContainers(context.Context, *ListContainersRequest) (*ListContainersResponse, error)
	ContainerStatus(context.Context, *ContainerStatusRequest) (*ContainerStatusResponse, error)
	UpdateContainerResources(context.Context, *UpdateContainerResourcesRequest) (*UpdateContainerResourcesResponse, error)
	ReopenContainerLog(context.Context, *ReopenContainerLogRequest) (*ReopenContainerLogResponse, error)
	ExecSync(context.Context, *ExecSyncRequest) (*ExecSyncResponse, error)
	Exec(context.Context, *ExecRequest) (*ExecResponse, error)
	Attach(context.Context, *AttachRequest) (*AttachResponse, error)
	PortForward(context.Context, *PortForwardRequest) (*PortForwardResponse, error)
	ContainerStats(context.Context, *ContainerStatsRequest) (*ContainerStatsResponse, error)
	ListContainerStats(context.Context, *ListContainerStatsRequest) (*ListContainerStatsResponse, error)
	UpdateRuntimeConfig(context.Context, *UpdateRuntimeConfigRequest) (*UpdateRuntimeConfigResponse, error)
	Status(context.Context, *StatusRequest) (*StatusResponse, error)
}


type ImageServiceServer interface {
	ListImages(context.Context, *ListImagesRequest) (*ListImagesResponse, error)
	ImageStatus(context.Context, *ImageStatusRequest) (*ImageStatusResponse, error)
	PullImage(context.Context, *PullImageRequest) (*PullImageResponse, error)
	RemoveImage(context.Context, *RemoveImageRequest) (*RemoveImageResponse, error)
	ImageFsInfo(context.Context, *ImageFsInfoRequest) (*ImageFsInfoResponse, error)
}

相關的接口定義以及grpc的代碼所在位置：k8s.io/cri-api/pkg/apis/runtime/v1alpha2/api.pb.go

grpc server 接口實現

grpc server的代碼實現主要在以下幾個文件中：
k8s.io/kubernetes/pkg/kubelet/kuberuntime/kuberuntime_image.go實現接口 ImageServiceServer

k8s.io/kubernetes/pkg/kubelet/dockershim/docker_sandbox.go
k8s.io/kubernetes/pkg/kubelet/dockershim/docker_container.go
k8s.io/kubernetes/pkg/kubelet/dockershim/docker_streaming.go
k8s.io/kubernetes/pkg/kubelet/dockershim/docker_logs.go
實現RuntimeServiceServer接口

grpc server/client 實例化以及使用

func NewMainKubelet(...) (*Kubelet, error){
	...
	switch containerRuntime {
	case kubetypes.DockerContainerRuntime:
		// Create and start the CRI shim running as a grpc server.
		streamingConfig := getStreamingConfig(kubeCfg, kubeDeps, crOptions)
		ds, err := dockershim.NewDockerService(kubeDeps.DockerClientConfig, crOptions.PodSandboxImage, streamingConfig,
			&pluginSettings, runtimeCgroups, kubeCfg.CgroupDriver, crOptions.DockershimRootDirectory, !crOptions.RedirectContainerStreaming)
		if err != nil {
			return nil, err
		}
		if crOptions.RedirectContainerStreaming {
			klet.criHandler = ds
		}

		// The unix socket for kubelet <-> dockershim communication.
		klog.V(5).Infof("RemoteRuntimeEndpoint: %q, RemoteImageEndpoint: %q",
			remoteRuntimeEndpoint,
			remoteImageEndpoint)
		klog.V(2).Infof("Starting the GRPC server for the docker CRI shim.")
		server := dockerremote.NewDockerServer(remoteRuntimeEndpoint, ds)
		// Start 方法會吧ds的所有方法註冊到grpc的server端
		if err := server.Start(); err != nil {
			return nil, err
		}

		// Create dockerLegacyService when the logging driver is not supported.
		supported, err := ds.IsCRISupportedLogDriver()
		if err != nil {
			return nil, err
		}
		if !supported {
			klet.dockerLegacyService = ds
			legacyLogProvider = ds
		}
	case kubetypes.RemoteContainerRuntime:
		// No-op.
		break
	default:
		return nil, fmt.Errorf("unsupported CRI runtime: %q", containerRuntime)
	}
	// 這裏是grpc的client端的實例的實現
	runtimeService, imageService, err := getRuntimeAndImageServices(remoteRuntimeEndpoint, remoteImageEndpoint, kubeCfg.RuntimeRequestTimeout)
	if err != nil {
		return nil, err
	}
	klet.runtimeService = runtimeService
	...
	runtime, err := kuberuntime.NewKubeGenericRuntimeManager(
		kubecontainer.FilterEventRecorder(kubeDeps.Recorder),
		klet.livenessManager,
		seccompProfileRoot,
		containerRefManager,
		machineInfo,
		klet,
		kubeDeps.OSInterface,
		klet,
		httpClient,
		imageBackOff,
		kubeCfg.SerializeImagePulls,
		float32(kubeCfg.RegistryPullQPS),
		int(kubeCfg.RegistryBurst),
		kubeCfg.CPUCFSQuota,
		kubeCfg.CPUCFSQuotaPeriod,
		runtimeService,
		imageService,
		kubeDeps.ContainerManager.InternalContainerLifecycle(),
		legacyLogProvider,
		klet.runtimeClassManager,
	)
	...
}

在實例化kubelet類型struct的時候，會根據runtime的類型去創建對應的dockerService，然後創建的對應的關於runtime所有接口實現的grpc server端，並監聽grpc鏈接請求，提供runtime操作的服務，到這裏grpc server端的實例化已經完成。

緊接着是初始化kubelet的runtimeService和imageService，實際就是創建對應的grpc的client端。這個client端在後續的多個manager中都會用到。比如：syncPod後，需要創建對應的容器就是最終就是調用runtimeService和imageService去創建容器。