Update containerd config schema to version 3

Ref: https://github.com/containerd/containerd/blob/release/2.0/docs/cri/config.md

Since this is a breaking change, add support for a new v3 template file. If no v3 template is present, fall back to checking for the legacy v2 template and render the old structure.

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

pkg/agent/config/config.go

@@ -682,7 +682,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	applyContainerdStateAndAddress(nodeConfig)
 	applyCRIDockerdAddress(nodeConfig)
 	applyContainerdQoSClassConfigFileIfPresent(envInfo, &nodeConfig.Containerd)
-	nodeConfig.Containerd.Template = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "config.toml.tmpl")
+	nodeConfig.Containerd.Template = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd")
 
 	if envInfo.BindAddress != "" {
 		nodeConfig.AgentConfig.ListenAddress = envInfo.BindAddress

pkg/agent/containerd/config.go

@@ -21,19 +21,48 @@ import (
 
 type HostConfigs map[string]templates.HostConfig
 
+type templateGeneration struct {
+	version  int
+	filename string
+	base     string
+}
+
+var templateGenerations = []templateGeneration{
+	{
+		version:  3,
+		filename: "config-v3.toml.tmpl",
+		base:     templates.ContainerdConfigTemplateV3,
+	},
+	{
+		version:  2,
+		filename: "config.toml.tmpl",
+		base:     templates.ContainerdConfigTemplate,
+	},
+}
+
 // writeContainerdConfig renders and saves config.toml from the filled template
 func writeContainerdConfig(cfg *config.Node, containerdConfig templates.ContainerdConfig) error {
-	var containerdTemplate string
-	containerdTemplateBytes, err := os.ReadFile(cfg.Containerd.Template)
-	if err == nil {
-		logrus.Infof("Using containerd template at %s", cfg.Containerd.Template)
-		containerdTemplate = string(containerdTemplateBytes)
-	} else if os.IsNotExist(err) {
-		containerdTemplate = templates.ContainerdConfigTemplate
-	} else {
-		return err
+	// use v3 template by default
+	userTemplate := templates.ContainerdConfigTemplateV3
+	baseTemplate := templates.ContainerdConfigTemplateV3
+	cfg.Containerd.ConfigVersion = 3
+
+	// check for user templates
+	for _, tg := range templateGenerations {
+		path := filepath.Join(cfg.Containerd.Template, tg.filename)
+		b, err := os.ReadFile(path)
+		if err == nil {
+			logrus.Infof("Using containerd config template at %s", path)
+			baseTemplate = tg.base
+			userTemplate = string(b)
+			cfg.Containerd.ConfigVersion = tg.version
+			break
+		} else if !os.IsNotExist(err) {
+			return err
+		}
 	}
-	parsedTemplate, err := templates.ParseTemplateFromConfig(containerdTemplate, containerdConfig)
+
+	parsedTemplate, err := templates.ParseTemplateFromConfig(userTemplate, baseTemplate, containerdConfig)
 	if err != nil {
 		return err
 	}

pkg/agent/containerd/config_linux.go

@@ -31,9 +31,16 @@ func getContainerdArgs(cfg *config.Node) []string {
 	args := []string{
 		"containerd",
 		"-c", cfg.Containerd.Config,
-		"-a", cfg.Containerd.Address,
-		"--state", cfg.Containerd.State,
-		"--root", cfg.Containerd.Root,
+	}
+
+	// The legacy version 2 linux containerd config template did not include
+	// address/state/root settings, so they need to be passed on the command line.
+	if cfg.Containerd.ConfigVersion < 3 {
+		args = append(args,
+			"-a", cfg.Containerd.Address,
+			"--state", cfg.Containerd.State,
+			"--root", cfg.Containerd.Root,
+		)
 	}
 	return args
 }

pkg/agent/containerd/config_test.go

@@ -1471,14 +1471,24 @@ func Test_UnitGetHostConfigs(t *testing.T) {
 				t.Fatalf("failed to parse %s: %v\n", registriesFile, err)
 			}
 
+			// This is an odd mishmash of linux and windows stuff just to excercise all the template bits
 			nodeConfig := &config.Node{
+				DefaultRuntime: "runhcs-wcow-process",
 				Containerd: config.Containerd{
 					Registry: tempDir + "/hosts.d",
+					Config:   tempDir + "/config.toml",
+					Template: tempDir,
+					Address:  "/run/k3s/containerd/containerd.sock",
+					Root:     "/var/lib/rancher/k3s/agent/containerd",
+					Opt:      "/var/lib/rancher/k3s/agent/containerd",
+					State:    "/run/k3s/containerd",
 				},
 				AgentConfig: config.Agent{
 					ImageServiceSocket: "containerd-stargz-grpc.sock",
 					Registry:           registry.Registry,
 					Snapshotter:        "stargz",
+					CNIBinDir:          "/var/lib/rancher/k3s/data/cni",
+					CNIConfDir:         "/var/lib/rancher/k3s/agent/etc/cni/net.d",
 				},
 			}
 
@@ -1498,20 +1508,39 @@ func Test_UnitGetHostConfigs(t *testing.T) {
 
 			// Confirm that hosts.toml renders properly for all registries
 			for host, config := range got {
-				hostsTemplate, err := templates.ParseHostsTemplateFromConfig(templates.HostsTomlTemplate, config)
+				hostsToml, err := templates.ParseHostsTemplateFromConfig(templates.HostsTomlTemplate, config)
 				assert.NoError(t, err, "ParseHostTemplateFromConfig for %s", host)
-				t.Logf("%s/hosts.d/%s/hosts.toml\n%s", tempDir, host, hostsTemplate)
+				t.Logf("%s/hosts.d/%s/hosts.toml\n%s", tempDir, host, hostsToml)
 			}
 
-			// Confirm that the main containerd config.toml renders properly
-			containerdConfig := templates.ContainerdConfig{
-				NodeConfig:            nodeConfig,
-				PrivateRegistryConfig: registry.Registry,
-				Program:               "k3s",
+			for _, template := range []string{"config.toml.tmpl", "config-v3.toml.tmpl"} {
+				t.Run(template, func(t *testing.T) {
+					templateFile := filepath.Join(tempDir, template)
+					err = os.WriteFile(templateFile, []byte(`{{ template "base" . }}`), 0600)
+					assert.NoError(t, err, "Write Template")
+
+					// Confirm that the main containerd config.toml renders properly
+					containerdConfig := templates.ContainerdConfig{
+						NodeConfig:            nodeConfig,
+						PrivateRegistryConfig: registry.Registry,
+						Program:               "k3s",
+						ExtraRuntimes: map[string]templates.ContainerdRuntimeConfig{
+							"runhcs-wcow-process": templates.ContainerdRuntimeConfig{
+								RuntimeType: "io.containerd.runhcs.v1",
+							},
+							"wasmtime": templates.ContainerdRuntimeConfig{
+								RuntimeType: "io.containerd.wasmtime.v1",
+								BinaryName:  "containerd-shim-wasmtime-v1",
+							},
+						},
+					}
+					err = writeContainerdConfig(nodeConfig, containerdConfig)
+					assert.NoError(t, err, "ParseTemplateFromConfig")
+					configToml, err := os.ReadFile(nodeConfig.Containerd.Config)
+					assert.NoError(t, err, "ReadFile "+nodeConfig.Containerd.Config)
+					t.Logf("%s\n%s", nodeConfig.Containerd.Config, configToml)
+				})
 			}
-			configTemplate, err := templates.ParseTemplateFromConfig(templates.ContainerdConfigTemplate, containerdConfig)
-			assert.NoError(t, err, "ParseTemplateFromConfig")
-			t.Logf("%s/config.toml\n%s", tempDir, configTemplate)
 		})
 	}
 }

pkg/agent/containerd/config_windows.go

@@ -18,6 +18,8 @@ func getContainerdArgs(cfg *config.Node) []string {
 		"containerd",
 		"-c", cfg.Containerd.Config,
 	}
+	// The legacy version 2 windows containerd config template did include
+	// address/state/root settings, so they do not need to be passed on the command line.
 	return args
 }
 
@@ -28,11 +30,11 @@ func SetupContainerdConfig(cfg *config.Node) error {
 		logrus.Warn("SELinux isn't supported on windows")
 	}
 
+	cfg.DefaultRuntime = "runhcs-wcow-process"
+	cfg.AgentConfig.Snapshotter = "windows"
 	containerdConfig := templates.ContainerdConfig{
 		NodeConfig:            cfg,
 		DisableCgroup:         true,
-		SystemdCgroup:         false,
-		IsRunningInUserNS:     false,
 		PrivateRegistryConfig: cfg.AgentConfig.Registry,
 		NoDefaultEndpoint:     cfg.Containerd.NoDefault,
 	}

pkg/agent/templates/templates.go

@@ -1,8 +1,11 @@
 package templates
 
 import (
+	"bufio"
 	"bytes"
+	"io"
 	"net/url"
+	"strings"
 	"text/template"
 
 	"github.com/rancher/wharfie/pkg/registries"
@@ -44,6 +47,7 @@ type HostConfig struct {
 
 var HostsTomlHeader = "# File generated by " + version.Program + ". DO NOT EDIT.\n"
 
+// This hosts.toml template is used by both Linux and Windows nodes
 const HostsTomlTemplate = `
 {{- /* */ -}}
 # File generated by {{ .Program }}. DO NOT EDIT.
@@ -91,21 +95,157 @@ skip_verify = true
 {{ end -}}
 `
 
-func ParseTemplateFromConfig(templateBuffer string, config interface{}) (string, error) {
+// This version 3 config template is used by both Linux and Windows nodes
+const ContainerdConfigTemplateV3 = `
+{{- /* */ -}}
+# File generated by {{ .Program }}. DO NOT EDIT. Use config.toml.tmpl instead.
+version = 3
+root = {{ printf "%q" .NodeConfig.Containerd.Root }}
+state = {{ printf "%q" .NodeConfig.Containerd.State }}
+
+[grpc]
+  address = {{ deschemify .NodeConfig.Containerd.Address | printf "%q" }}
+
+[plugins.'io.containerd.internal.v1.opt']
+  path = {{ printf "%q" .NodeConfig.Containerd.Opt }}
+
+[plugins.'io.containerd.grpc.v1.cri']
+  stream_server_address = "127.0.0.1"
+  stream_server_port = "10010"
+
+[plugins.'io.containerd.cri.v1.runtime']
+  enable_selinux = {{ .NodeConfig.SELinux }}
+  enable_unprivileged_ports = {{ .EnableUnprivileged }}
+  enable_unprivileged_icmp = {{ .EnableUnprivileged }}
+  device_ownership_from_security_context = {{ .NonrootDevices }}
+
+{{ if .DisableCgroup}}
+  disable_cgroup = true
+{{ end }}
+
+{{ if .IsRunningInUserNS }}
+  disable_apparmor = true
+  restrict_oom_score_adj = true
+{{ end }}
+
+{{ with .NodeConfig.AgentConfig.Snapshotter }}
+[plugins.'io.containerd.cri.v1.images']
+  snapshotter = "{{ . }}"
+  disable_snapshot_annotations = {{ if eq . "stargz" }}false{{else}}true{{end}}
+{{ end }}
+
+{{ with .NodeConfig.AgentConfig.PauseImage }}
+[plugins.'io.containerd.cri.v1.images'.pinned_images]
+  sandbox = "{{ . }}"
+{{ end }}
+
+[plugins.'io.containerd.cri.v1.images'.registry]
+  config_path = {{ printf "%q" .NodeConfig.Containerd.Registry }}
+
+{{ if not .NodeConfig.NoFlannel }}
+[plugins.'io.containerd.cri.v1.runtime'.cni]
+  bin_dir = {{ printf "%q" .NodeConfig.AgentConfig.CNIBinDir }}
+  conf_dir = {{ printf "%q" .NodeConfig.AgentConfig.CNIConfDir }}
+{{ end }}
+
+{{ if or .NodeConfig.Containerd.BlockIOConfig .NodeConfig.Containerd.RDTConfig }}
+[plugins.'io.containerd.service.v1.tasks-service']
+  {{ with .NodeConfig.Containerd.BlockIOConfig }}blockio_config_file = {{ printf "%q" . }}{{ end }}
+  {{ with .NodeConfig.Containerd.RDTConfig }}rdt_config_file = {{ printf "%q" . }}{{ end }}
+{{ end }}
+
+{{ with .NodeConfig.DefaultRuntime }}
+[plugins.'io.containerd.cri.v1.runtime'.containerd]
+  default_runtime_name = "{{ . }}"
+{{ end }}
+
+{{ with .SystemdCgroup }}
+[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runc]
+  runtime_type = "io.containerd.runc.v2"
+
+[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runc.options]
+  SystemdCgroup = {{ . }}
+{{ end }}
+
+{{ range $k, $v := .ExtraRuntimes }}
+[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.'{{ $k }}']
+  runtime_type = "{{$v.RuntimeType}}"
+{{ with $v.BinaryName}}
+[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.'{{ $k }}'.options]
+  BinaryName = {{ printf "%q" . }}
+  SystemdCgroup = {{ $.SystemdCgroup }}
+{{ end }}
+{{ end }}
+
+{{ if .PrivateRegistryConfig }}
+{{ range $k, $v := .PrivateRegistryConfig.Configs }}
+{{ with $v.Auth }}
+[plugins.'io.containerd.grpc.v1.cri'.registry.configs.'{{ $k }}'.auth]
+  {{ with .Username }}username = {{ printf "%q" . }}{{ end }}
+  {{ with .Password }}password = {{ printf "%q" . }}{{ end }}
+  {{ with .Auth }}auth = {{ printf "%q" . }}{{ end }}
+  {{ with .IdentityToken }}identitytoken = {{ printf "%q" . }}{{ end }}
+{{ end }}
+{{ end }}
+{{ end }}
+
+{{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }}
+{{ with .NodeConfig.AgentConfig.ImageServiceSocket }}
+[plugins.'io.containerd.snapshotter.v1.stargz']
+  cri_keychain_image_service_path = {{ printf "%q" . }}
+
+[plugins.'io.containerd.snapshotter.v1.stargz'.cri_keychain]
+  enable_keychain = true
+{{ end }}
+
+[plugins.'io.containerd.snapshotter.v1.stargz'.registry]
+  config_path = {{ printf "%q" .NodeConfig.Containerd.Registry }}
+
+{{ if .PrivateRegistryConfig }}
+{{ range $k, $v := .PrivateRegistryConfig.Configs }}
+{{ with $v.Auth }}
+[plugins.'io.containerd.snapshotter.v1.stargz'.registry.configs.'{{ $k }}'.auth]
+  {{ with .Username }}username = {{ printf "%q" . }}{{ end }}
+  {{ with .Password }}password = {{ printf "%q" . }}{{ end }}
+  {{ with .Auth }}auth = {{ printf "%q" . }}{{ end }}
+  {{ with .IdentityToken }}identitytoken = {{ printf "%q" . }}{{ end }}
+{{ end }}
+{{ end }}
+{{ end }}
+{{ end }}
+`
+
+func ParseTemplateFromConfig(userTemplate, baseTemplate string, config interface{}) (string, error) {
 	out := new(bytes.Buffer)
-	t := template.Must(template.New("compiled_template").Funcs(templateFuncs).Parse(templateBuffer))
-	template.Must(t.New("base").Parse(ContainerdConfigTemplate))
+	t := template.Must(template.New("compiled_template").Funcs(templateFuncs).Parse(userTemplate))
+	template.Must(t.New("base").Parse(baseTemplate))
 	if err := t.Execute(out, config); err != nil {
 		return "", err
 	}
-	return out.String(), nil
+	return trimEmpty(out)
 }
 
-func ParseHostsTemplateFromConfig(templateBuffer string, config interface{}) (string, error) {
+func ParseHostsTemplateFromConfig(userTemplate string, config interface{}) (string, error) {
 	out := new(bytes.Buffer)
-	t := template.Must(template.New("compiled_template").Funcs(templateFuncs).Parse(templateBuffer))
+	t := template.Must(template.New("compiled_template").Funcs(templateFuncs).Parse(userTemplate))
 	if err := t.Execute(out, config); err != nil {
 		return "", err
 	}
-	return out.String(), nil
+	return trimEmpty(out)
+}
+
+// trimEmpty removes excess empty lines from the rendered template
+func trimEmpty(r io.Reader) (string, error) {
+	builder := strings.Builder{}
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.TrimSpace(line) != "" {
+			if strings.HasPrefix(line, "[") {
+				builder.WriteString("\n")
+			}
+			builder.WriteString(line + "\n")
+		}
+	}
+	return builder.String(), scanner.Err()
 }

pkg/agent/templates/templates_linux.go

@@ -3,9 +3,11 @@
 package templates
 
 import (
+	"encoding/json"
 	"text/template"
 )
 
+// This version 2 config template is only used by Linux nodes
 const ContainerdConfigTemplate = `
 {{- /* */ -}}
 # File generated by {{ .Program }}. DO NOT EDIT. Use config.toml.tmpl instead.
@@ -110,4 +112,8 @@ var templateFuncs = template.FuncMap{
 	"deschemify": func(s string) string {
 		return s
 	},
+	"toJson": func(v interface{}) string {
+		output, _ := json.Marshal(v)
+		return string(output)
+	},
 }

pkg/agent/templates/templates_windows.go

@@ -4,11 +4,13 @@
 package templates
 
 import (
+	"encoding/json"
 	"net/url"
 	"strings"
 	"text/template"
 )
 
+// This version 2 config template is only used by Windows nodes
 const ContainerdConfigTemplate = `
 {{- /* */ -}}
 # File generated by {{ .Program }}. DO NOT EDIT. Use config.toml.tmpl instead.
@@ -155,4 +157,8 @@ var templateFuncs = template.FuncMap{
 		}
 		return s
 	},
+	"toJson": func(v interface{}) string {
+		output, _ := json.Marshal(v)
+		return string(output)
+	},
 }

pkg/daemons/config/types.go

@@ -92,6 +92,7 @@ type Containerd struct {
 	NonrootDevices bool
 	SELinux        bool
 	Debug          bool
+	ConfigVersion  int
 }
 
 type CRIDockerd struct {