From fe18b1fce98d4b36b3ff58c5b525de2f18de1e10 Mon Sep 17 00:00:00 2001 From: Edgar Lee Date: Wed, 27 Sep 2023 16:20:50 -0400 Subject: [PATCH] Add --image-service-endpoint flag (#8279) * Add --image-service-endpoint flag Problem: External container runtime can be set but image service endpoint is unchanged and also is not exposed as a flag. This is useful for using containerd snapshotters outside of the ones that have built-in support like stargz-snapshotter. Solution: Add a flag --image-service-endpoint and also default image service endpoint to container runtime endpoint if set. Signed-off-by: Edgar Lee --- pkg/agent/config/config.go | 41 ++++++++++++++++++++++--------------- pkg/agent/run_linux.go | 4 ++++ pkg/agent/run_windows.go | 4 ++++ pkg/cli/cmds/agent.go | 7 +++++++ pkg/cli/cmds/server.go | 1 + pkg/daemons/config/types.go | 1 + 6 files changed, 41 insertions(+), 17 deletions(-) diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index 0798642c06..852bb22029 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -497,6 +497,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N Docker: envInfo.Docker, SELinux: envInfo.EnableSELinux, ContainerRuntimeEndpoint: envInfo.ContainerRuntimeEndpoint, + ImageServiceEndpoint: envInfo.ImageServiceEndpoint, MultiClusterCIDR: controlConfig.MultiClusterCIDR, FlannelBackend: controlConfig.FlannelBackend, FlannelIPv6Masq: controlConfig.FlannelIPv6Masq, @@ -528,24 +529,30 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N nodeConfig.Containerd.Config = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "config.toml") nodeConfig.Containerd.Root = filepath.Join(envInfo.DataDir, "agent", "containerd") nodeConfig.CRIDockerd.Root = filepath.Join(envInfo.DataDir, "agent", "cri-dockerd") - if !nodeConfig.Docker && nodeConfig.ContainerRuntimeEndpoint == "" { - switch nodeConfig.AgentConfig.Snapshotter { - case "overlayfs": - if err := containerd.OverlaySupported(nodeConfig.Containerd.Root); err != nil { - return nil, errors.Wrapf(err, "\"overlayfs\" snapshotter cannot be enabled for %q, try using \"fuse-overlayfs\" or \"native\"", - nodeConfig.Containerd.Root) - } - case "fuse-overlayfs": - if err := containerd.FuseoverlayfsSupported(nodeConfig.Containerd.Root); err != nil { - return nil, errors.Wrapf(err, "\"fuse-overlayfs\" snapshotter cannot be enabled for %q, try using \"native\"", - nodeConfig.Containerd.Root) - } - case "stargz": - if err := containerd.StargzSupported(nodeConfig.Containerd.Root); err != nil { - return nil, errors.Wrapf(err, "\"stargz\" snapshotter cannot be enabled for %q, try using \"overlayfs\" or \"native\"", - nodeConfig.Containerd.Root) + if !nodeConfig.Docker { + if nodeConfig.ImageServiceEndpoint != "" { + nodeConfig.AgentConfig.ImageServiceSocket = nodeConfig.ImageServiceEndpoint + } else if nodeConfig.ContainerRuntimeEndpoint == "" { + switch nodeConfig.AgentConfig.Snapshotter { + case "overlayfs": + if err := containerd.OverlaySupported(nodeConfig.Containerd.Root); err != nil { + return nil, errors.Wrapf(err, "\"overlayfs\" snapshotter cannot be enabled for %q, try using \"fuse-overlayfs\" or \"native\"", + nodeConfig.Containerd.Root) + } + case "fuse-overlayfs": + if err := containerd.FuseoverlayfsSupported(nodeConfig.Containerd.Root); err != nil { + return nil, errors.Wrapf(err, "\"fuse-overlayfs\" snapshotter cannot be enabled for %q, try using \"native\"", + nodeConfig.Containerd.Root) + } + case "stargz": + if err := containerd.StargzSupported(nodeConfig.Containerd.Root); err != nil { + return nil, errors.Wrapf(err, "\"stargz\" snapshotter cannot be enabled for %q, try using \"overlayfs\" or \"native\"", + nodeConfig.Containerd.Root) + } + nodeConfig.AgentConfig.ImageServiceSocket = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock" } - nodeConfig.AgentConfig.ImageServiceSocket = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock" + } else { + nodeConfig.AgentConfig.ImageServiceSocket = nodeConfig.ContainerRuntimeEndpoint } } nodeConfig.Containerd.Opt = filepath.Join(envInfo.DataDir, "agent", "containerd") diff --git a/pkg/agent/run_linux.go b/pkg/agent/run_linux.go index 4eadf870f5..caa4c2bae4 100644 --- a/pkg/agent/run_linux.go +++ b/pkg/agent/run_linux.go @@ -37,5 +37,9 @@ func setupCriCtlConfig(cfg cmds.Agent, nodeConfig *config.Node) error { } crp := "runtime-endpoint: " + cre + "\n" + ise := nodeConfig.ImageServiceEndpoint + if ise != "" && ise != cre { + crp += "image-endpoint: " + cre + "\n" + } return os.WriteFile(agentConfDir+"/crictl.yaml", []byte(crp), 0600) } diff --git a/pkg/agent/run_windows.go b/pkg/agent/run_windows.go index a8dc605be2..e2ff622983 100644 --- a/pkg/agent/run_windows.go +++ b/pkg/agent/run_windows.go @@ -39,5 +39,9 @@ func setupCriCtlConfig(cfg cmds.Agent, nodeConfig *config.Node) error { } crp := "runtime-endpoint: " + cre + "\n" + ise := nodeConfig.ImageServiceEndpoint + if ise != "" && ise != cre { + crp += "image-endpoint: " + cre + "\n" + } return os.WriteFile(filepath.Join(agentConfDir, "crictl.yaml"), []byte(crp), 0600) } diff --git a/pkg/cli/cmds/agent.go b/pkg/cli/cmds/agent.go index 433f0162cf..f3dcafc523 100644 --- a/pkg/cli/cmds/agent.go +++ b/pkg/cli/cmds/agent.go @@ -27,6 +27,7 @@ type Agent struct { Snapshotter string Docker bool ContainerRuntimeEndpoint string + ImageServiceEndpoint string FlannelIface string FlannelConf string FlannelCniConfFile string @@ -114,6 +115,11 @@ var ( Usage: "(agent/runtime) Disable embedded containerd and use the CRI socket at the given path; when used with --docker this sets the docker socket path", Destination: &AgentConfig.ContainerRuntimeEndpoint, } + ImageServiceEndpointFlag = &cli.StringFlag{ + Name: "image-service-endpoint", + Usage: "(agent/runtime) Disable embedded containerd image service and use remote image service socket at the given path. If not specified, defaults to --container-runtime-endpoint.", + Destination: &AgentConfig.ImageServiceEndpoint, + } PrivateRegistryFlag = &cli.StringFlag{ Name: "private-registry", Usage: "(agent/runtime) Private registry configuration file", @@ -247,6 +253,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command { LBServerPortFlag, ProtectKernelDefaultsFlag, CRIEndpointFlag, + ImageServiceEndpointFlag, PauseImageFlag, SnapshotterFlag, PrivateRegistryFlag, diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index c5717e06f5..617aa43fc2 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -497,6 +497,7 @@ var ServerFlags = []cli.Flag{ ImageCredProvConfigFlag, DockerFlag, CRIEndpointFlag, + ImageServiceEndpointFlag, PauseImageFlag, SnapshotterFlag, PrivateRegistryFlag, diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index dec2906f72..3c50733d26 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -37,6 +37,7 @@ const ( type Node struct { Docker bool ContainerRuntimeEndpoint string + ImageServiceEndpoint string NoFlannel bool SELinux bool MultiClusterCIDR bool