Merge pull request #44197 from Random-Liu/dockershim-only-mode

Automatic merge from submit-queue

Add dockershim only mode

This PR added a `experimental-dockershim` hidden flag in kubelet to run dockershim only.

We introduce this flag mainly for cri validation test. In the future we should compile dockershim into another binary.

@yujuhong @feiskyer @xlgao-zju 
/cc @kubernetes/sig-node-pr-reviews

cmd/kubelet/app/BUILD

@@ -55,6 +55,8 @@ go_library(
         "//pkg/kubelet/cm:go_default_library",
         "//pkg/kubelet/config:go_default_library",
         "//pkg/kubelet/container:go_default_library",
+        "//pkg/kubelet/dockershim:go_default_library",
+        "//pkg/kubelet/dockershim/remote:go_default_library",
         "//pkg/kubelet/dockertools:go_default_library",
         "//pkg/kubelet/eviction:go_default_library",
         "//pkg/kubelet/eviction/api:go_default_library",
@@ -62,6 +64,7 @@ go_library(
         "//pkg/kubelet/network/cni:go_default_library",
         "//pkg/kubelet/network/kubenet:go_default_library",
         "//pkg/kubelet/server:go_default_library",
+        "//pkg/kubelet/server/streaming:go_default_library",
         "//pkg/kubelet/types:go_default_library",
         "//pkg/kubelet/util/csr:go_default_library",
         "//pkg/util/configz:go_default_library",

cmd/kubelet/app/options/options.go

@@ -296,6 +296,8 @@ func (c *kubeletConfiguration) addFlags(fs *pflag.FlagSet) {
 	// implementation.
 	fs.BoolVar(&c.EnableCRI, "enable-cri", c.EnableCRI, "Enable the Container Runtime Interface (CRI) integration. If --container-runtime is set to \"remote\", Kubelet will communicate with the runtime/image CRI server listening on the endpoint specified by --remote-runtime-endpoint/--remote-image-endpoint. If --container-runtime is set to \"docker\", Kubelet will launch a in-process CRI server on behalf of docker, and communicate over a default endpoint. If --container-runtime is \"rkt\", the flag will be ignored because rkt integration doesn't support CRI yet. [default=true]")
 	fs.MarkDeprecated("enable-cri", "The non-CRI implementation will be deprecated and removed in a future version.")
+	fs.BoolVar(&c.ExperimentalDockershim, "experimental-dockershim", c.ExperimentalDockershim, "Enable dockershim only mode. In this mode, kubelet will only start dockershim without any other functionalities. This flag only serves test purpose, please do not use it unless you are conscious of what you are doing. [default=false]")
+	fs.MarkHidden("experimental-dockershim")
 
 	fs.StringVar(&c.RemoteRuntimeEndpoint, "container-runtime-endpoint", c.RemoteRuntimeEndpoint, "[Experimental] The unix socket endpoint of remote runtime service. The endpoint is used only when CRI integration is enabled (--enable-cri)")
 	fs.StringVar(&c.RemoteImageEndpoint, "image-service-endpoint", c.RemoteImageEndpoint, "[Experimental] The unix socket endpoint of remote image service. If not specified, it will be the same with container-runtime-endpoint by default. The endpoint is used only when CRI integration is enabled (--enable-cri)")

cmd/kubelet/app/server.go

@@ -26,6 +26,7 @@ import (
 	"net"
 	"net/http"
 	_ "net/http/pprof"
+	"net/url"
 	"os"
 	"path"
 	"strconv"
@@ -70,10 +71,13 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/cm"
 	"k8s.io/kubernetes/pkg/kubelet/config"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+	"k8s.io/kubernetes/pkg/kubelet/dockershim"
+	dockerremote "k8s.io/kubernetes/pkg/kubelet/dockershim/remote"
 	"k8s.io/kubernetes/pkg/kubelet/dockertools"
 	"k8s.io/kubernetes/pkg/kubelet/eviction"
 	evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api"
 	"k8s.io/kubernetes/pkg/kubelet/server"
+	"k8s.io/kubernetes/pkg/kubelet/server/streaming"
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
 	"k8s.io/kubernetes/pkg/util/configz"
 	"k8s.io/kubernetes/pkg/util/flock"
@@ -929,3 +933,72 @@ func parseResourceList(m componentconfig.ConfigurationMap) (v1.ResourceList, err
 	}
 	return rl, nil
 }
+
+// RunDockershim only starts the dockershim in current process. This is only used for cri validate testing purpose
+// TODO(random-liu): Move this to a separate binary.
+func RunDockershim(c *componentconfig.KubeletConfiguration) error {
+	// Create docker client.
+	dockerClient := dockertools.ConnectToDockerOrDie(c.DockerEndpoint, c.RuntimeRequestTimeout.Duration,
+		c.ImagePullProgressDeadline.Duration)
+
+	// Initialize docker exec handler.
+	var dockerExecHandler dockertools.ExecHandler
+	switch c.DockerExecHandlerName {
+	case "native":
+		dockerExecHandler = &dockertools.NativeExecHandler{}
+	case "nsenter":
+		dockerExecHandler = &dockertools.NsenterExecHandler{}
+	default:
+		glog.Warningf("Unknown Docker exec handler %q; defaulting to native", c.DockerExecHandlerName)
+		dockerExecHandler = &dockertools.NativeExecHandler{}
+	}
+
+	// Initialize network plugin settings.
+	binDir := c.CNIBinDir
+	if binDir == "" {
+		binDir = c.NetworkPluginDir
+	}
+	pluginSettings := dockershim.NetworkPluginSettings{
+		HairpinMode:       componentconfig.HairpinMode(c.HairpinMode),
+		NonMasqueradeCIDR: c.NonMasqueradeCIDR,
+		PluginName:        c.NetworkPluginName,
+		PluginConfDir:     c.CNIConfDir,
+		PluginBinDir:      binDir,
+		MTU:               int(c.NetworkPluginMTU),
+	}
+
+	// Initialize streaming configuration. (Not using TLS now)
+	streamingConfig := &streaming.Config{
+		// Use a relative redirect (no scheme or host).
+		BaseURL:                         &url.URL{Path: "/cri/"},
+		StreamIdleTimeout:               c.StreamingConnectionIdleTimeout.Duration,
+		StreamCreationTimeout:           streaming.DefaultConfig.StreamCreationTimeout,
+		SupportedRemoteCommandProtocols: streaming.DefaultConfig.SupportedRemoteCommandProtocols,
+		SupportedPortForwardProtocols:   streaming.DefaultConfig.SupportedPortForwardProtocols,
+	}
+
+	ds, err := dockershim.NewDockerService(dockerClient, c.SeccompProfileRoot, c.PodInfraContainerImage,
+		streamingConfig, &pluginSettings, c.RuntimeCgroups, c.CgroupDriver, dockerExecHandler)
+	if err != nil {
+		return err
+	}
+	if err := ds.Start(); err != nil {
+		return err
+	}
+
+	// The unix socket for kubelet <-> dockershim communication.
+	ep := c.RemoteRuntimeEndpoint
+	if len(ep) == 0 {
+		ep = "/var/run/dockershim.sock"
+	}
+
+	glog.V(2).Infof("Starting the GRPC server for the docker CRI shim.")
+	server := dockerremote.NewDockerServer(ep, ds)
+	if err := server.Start(); err != nil {
+		return err
+	}
+
+	// Start the streaming server
+	addr := net.JoinHostPort(c.Address, strconv.Itoa(int(c.Port)))
+	return http.ListenAndServe(addr, ds)
+}

cmd/kubelet/kubelet.go

@@ -45,6 +45,13 @@ func main() {
 
 	verflag.PrintAndExitIfRequested()
 
+	if s.ExperimentalDockershim {
+		if err := app.RunDockershim(&s.KubeletConfiguration); err != nil {
+			fmt.Fprintf(os.Stderr, "error: %v\n", err)
+			os.Exit(1)
+		}
+	}
+
 	if err := app.Run(s, nil); err != nil {
 		fmt.Fprintf(os.Stderr, "error: %v\n", err)
 		os.Exit(1)

hack/verify-flags/known-flags.txt

@@ -236,6 +236,7 @@ experimental-bootstrap-kubeconfig
 experimental-bootstrap-token-auth
 experimental-check-node-capabilities-before-mount
 experimental-cri
+experimental-dockershim
 experimental-fail-swap-on
 experimental-kernel-memcg-notification
 experimental-keystone-ca-file

pkg/apis/componentconfig/types.go

@@ -468,6 +468,9 @@ type KubeletConfiguration struct {
 	// Enable Container Runtime Interface (CRI) integration.
 	// +optional
 	EnableCRI bool
+	// Enable dockershim only mode.
+	// +optional
+	ExperimentalDockershim bool
 	// TODO(#34726:1.8.0): Remove the opt-in for failing when swap is enabled.
 	// Tells the Kubelet to fail to start if swap is enabled on the node.
 	ExperimentalFailSwapOn bool

pkg/apis/componentconfig/v1alpha1/defaults.go

@@ -421,6 +421,9 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) {
 	if obj.EnableCRI == nil {
 		obj.EnableCRI = boolVar(true)
 	}
+	if obj.ExperimentalDockershim == nil {
+		obj.ExperimentalDockershim = boolVar(false)
+	}
 }
 
 func boolVar(b bool) *bool {

pkg/apis/componentconfig/v1alpha1/types.go

@@ -523,6 +523,9 @@ type KubeletConfiguration struct {
 	// Enable Container Runtime Interface (CRI) integration.
 	// +optional
 	EnableCRI *bool `json:"enableCRI,omitempty"`
+	// Enable dockershim only mode.
+	// +optional
+	ExperimentalDockershim *bool `json:"experimentalDockershim,omitempty"`
 	// TODO(#34726:1.8.0): Remove the opt-in for failing when swap is enabled.
 	// Tells the Kubelet to fail to start if swap is enabled on the node.
 	ExperimentalFailSwapOn bool `json:"experimentalFailSwapOn,omitempty"`

pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go

@@ -420,6 +420,9 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_componentconfig_KubeletConfigu
 	if err := v1.Convert_Pointer_bool_To_bool(&in.EnableCRI, &out.EnableCRI, s); err != nil {
 		return err
 	}
+	if err := v1.Convert_Pointer_bool_To_bool(&in.ExperimentalDockershim, &out.ExperimentalDockershim, s); err != nil {
+		return err
+	}
 	out.ExperimentalFailSwapOn = in.ExperimentalFailSwapOn
 	out.ExperimentalCheckNodeCapabilitiesBeforeMount = in.ExperimentalCheckNodeCapabilitiesBeforeMount
 	out.KeepTerminatedPodVolumes = in.KeepTerminatedPodVolumes
@@ -617,6 +620,9 @@ func autoConvert_componentconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigu
 	if err := v1.Convert_bool_To_Pointer_bool(&in.EnableCRI, &out.EnableCRI, s); err != nil {
 		return err
 	}
+	if err := v1.Convert_bool_To_Pointer_bool(&in.ExperimentalDockershim, &out.ExperimentalDockershim, s); err != nil {
+		return err
+	}
 	out.ExperimentalFailSwapOn = in.ExperimentalFailSwapOn
 	out.ExperimentalCheckNodeCapabilitiesBeforeMount = in.ExperimentalCheckNodeCapabilitiesBeforeMount
 	out.KeepTerminatedPodVolumes = in.KeepTerminatedPodVolumes

pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go

@@ -298,6 +298,11 @@ func DeepCopy_v1alpha1_KubeletConfiguration(in interface{}, out interface{}, c *
 			*out = new(bool)
 			**out = **in
 		}
+		if in.ExperimentalDockershim != nil {
+			in, out := &in.ExperimentalDockershim, &out.ExperimentalDockershim
+			*out = new(bool)
+			**out = **in
+		}
 		if in.SystemReserved != nil {
 			in, out := &in.SystemReserved, &out.SystemReserved
 			*out = make(map[string]string)

pkg/generated/openapi/zz_generated.openapi.go

@@ -13964,6 +13964,13 @@ func GetOpenAPIDefinitions(ref openapi.ReferenceCallback) map[string]openapi.Ope
 								Format:      "",
 							},
 						},
+						"experimentalDockershim": {
+							SchemaProps: spec.SchemaProps{
+								Description: "Enable dockershim only mode.",
+								Type:        []string{"boolean"},
+								Format:      "",
+							},
+						},
 						"experimentalFailSwapOn": {
 							SchemaProps: spec.SchemaProps{
 								Description: "Tells the Kubelet to fail to start if swap is enabled on the node.",