Add --image-service-endpoint flag (#8279)

* Add --image-service-endpoint flag

Problem:
External container runtime can be set but image service endpoint is unchanged
and also is not exposed as a flag. This is useful for using containerd
snapshotters outside of the ones that have built-in support like
stargz-snapshotter.

Solution:
Add a flag --image-service-endpoint and also default image service endpoint to
container runtime endpoint if set.

Signed-off-by: Edgar Lee <edgarhinshunlee@gmail.com>

pkg/agent/config/config.go

@@ -497,6 +497,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 		Docker:                   envInfo.Docker,
 		SELinux:                  envInfo.EnableSELinux,
 		ContainerRuntimeEndpoint: envInfo.ContainerRuntimeEndpoint,
+		ImageServiceEndpoint:     envInfo.ImageServiceEndpoint,
 		MultiClusterCIDR:         controlConfig.MultiClusterCIDR,
 		FlannelBackend:           controlConfig.FlannelBackend,
 		FlannelIPv6Masq:          controlConfig.FlannelIPv6Masq,
@@ -528,24 +529,30 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	nodeConfig.Containerd.Config = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "config.toml")
 	nodeConfig.Containerd.Root = filepath.Join(envInfo.DataDir, "agent", "containerd")
 	nodeConfig.CRIDockerd.Root = filepath.Join(envInfo.DataDir, "agent", "cri-dockerd")
-	if !nodeConfig.Docker && nodeConfig.ContainerRuntimeEndpoint == "" {
+	if !nodeConfig.Docker {
-		switch nodeConfig.AgentConfig.Snapshotter {
+		if nodeConfig.ImageServiceEndpoint != "" {
-		case "overlayfs":
+			nodeConfig.AgentConfig.ImageServiceSocket = nodeConfig.ImageServiceEndpoint
-			if err := containerd.OverlaySupported(nodeConfig.Containerd.Root); err != nil {
+		} else if nodeConfig.ContainerRuntimeEndpoint == "" {
-				return nil, errors.Wrapf(err, "\"overlayfs\" snapshotter cannot be enabled for %q, try using \"fuse-overlayfs\" or \"native\"",
+			switch nodeConfig.AgentConfig.Snapshotter {
-					nodeConfig.Containerd.Root)
+			case "overlayfs":
+				if err := containerd.OverlaySupported(nodeConfig.Containerd.Root); err != nil {
+					return nil, errors.Wrapf(err, "\"overlayfs\" snapshotter cannot be enabled for %q, try using \"fuse-overlayfs\" or \"native\"",
+						nodeConfig.Containerd.Root)
+				}
+			case "fuse-overlayfs":
+				if err := containerd.FuseoverlayfsSupported(nodeConfig.Containerd.Root); err != nil {
+					return nil, errors.Wrapf(err, "\"fuse-overlayfs\" snapshotter cannot be enabled for %q, try using \"native\"",
+						nodeConfig.Containerd.Root)
+				}
+			case "stargz":
+				if err := containerd.StargzSupported(nodeConfig.Containerd.Root); err != nil {
+					return nil, errors.Wrapf(err, "\"stargz\" snapshotter cannot be enabled for %q, try using \"overlayfs\" or \"native\"",
+						nodeConfig.Containerd.Root)
+				}
+				nodeConfig.AgentConfig.ImageServiceSocket = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
 			}
-		case "fuse-overlayfs":
+		} else {
-			if err := containerd.FuseoverlayfsSupported(nodeConfig.Containerd.Root); err != nil {
+			nodeConfig.AgentConfig.ImageServiceSocket = nodeConfig.ContainerRuntimeEndpoint
-				return nil, errors.Wrapf(err, "\"fuse-overlayfs\" snapshotter cannot be enabled for %q, try using \"native\"",
-					nodeConfig.Containerd.Root)
-			}
-		case "stargz":
-			if err := containerd.StargzSupported(nodeConfig.Containerd.Root); err != nil {
-				return nil, errors.Wrapf(err, "\"stargz\" snapshotter cannot be enabled for %q, try using \"overlayfs\" or \"native\"",
-					nodeConfig.Containerd.Root)
-			}
-			nodeConfig.AgentConfig.ImageServiceSocket = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
 		}
 	}
 	nodeConfig.Containerd.Opt = filepath.Join(envInfo.DataDir, "agent", "containerd")

pkg/agent/run_linux.go

@@ -37,5 +37,9 @@ func setupCriCtlConfig(cfg cmds.Agent, nodeConfig *config.Node) error {
 	}
 
 	crp := "runtime-endpoint: " + cre + "\n"
+	ise := nodeConfig.ImageServiceEndpoint
+	if ise != "" && ise != cre {
+		crp += "image-endpoint: " + cre + "\n"
+	}
 	return os.WriteFile(agentConfDir+"/crictl.yaml", []byte(crp), 0600)
 }

pkg/agent/run_windows.go

@@ -39,5 +39,9 @@ func setupCriCtlConfig(cfg cmds.Agent, nodeConfig *config.Node) error {
 	}
 
 	crp := "runtime-endpoint: " + cre + "\n"
+	ise := nodeConfig.ImageServiceEndpoint
+	if ise != "" && ise != cre {
+		crp += "image-endpoint: " + cre + "\n"
+	}
 	return os.WriteFile(filepath.Join(agentConfDir, "crictl.yaml"), []byte(crp), 0600)
 }

pkg/cli/cmds/agent.go

@@ -27,6 +27,7 @@ type Agent struct {
 	Snapshotter              string
 	Docker                   bool
 	ContainerRuntimeEndpoint string
+	ImageServiceEndpoint     string
 	FlannelIface             string
 	FlannelConf              string
 	FlannelCniConfFile       string
@@ -114,6 +115,11 @@ var (
 		Usage:       "(agent/runtime) Disable embedded containerd and use the CRI socket at the given path; when used with --docker this sets the docker socket path",
 		Destination: &AgentConfig.ContainerRuntimeEndpoint,
 	}
+	ImageServiceEndpointFlag = &cli.StringFlag{
+		Name:        "image-service-endpoint",
+		Usage:       "(agent/runtime) Disable embedded containerd image service and use remote image service socket at the given path. If not specified, defaults to --container-runtime-endpoint.",
+		Destination: &AgentConfig.ImageServiceEndpoint,
+	}
 	PrivateRegistryFlag = &cli.StringFlag{
 		Name:        "private-registry",
 		Usage:       "(agent/runtime) Private registry configuration file",
@@ -247,6 +253,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
 			LBServerPortFlag,
 			ProtectKernelDefaultsFlag,
 			CRIEndpointFlag,
+			ImageServiceEndpointFlag,
 			PauseImageFlag,
 			SnapshotterFlag,
 			PrivateRegistryFlag,

pkg/cli/cmds/server.go

@@ -497,6 +497,7 @@ var ServerFlags = []cli.Flag{
 	ImageCredProvConfigFlag,
 	DockerFlag,
 	CRIEndpointFlag,
+	ImageServiceEndpointFlag,
 	PauseImageFlag,
 	SnapshotterFlag,
 	PrivateRegistryFlag,

pkg/daemons/config/types.go

@@ -37,6 +37,7 @@ const (
 type Node struct {
 	Docker                   bool
 	ContainerRuntimeEndpoint string
+	ImageServiceEndpoint     string
 	NoFlannel                bool
 	SELinux                  bool
 	MultiClusterCIDR         bool