diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index 2e5efd1a4d..fcec2d7cdc 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -682,7 +682,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N applyContainerdStateAndAddress(nodeConfig) applyCRIDockerdAddress(nodeConfig) applyContainerdQoSClassConfigFileIfPresent(envInfo, &nodeConfig.Containerd) - nodeConfig.Containerd.Template = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "config.toml.tmpl") + nodeConfig.Containerd.Template = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd") if envInfo.BindAddress != "" { nodeConfig.AgentConfig.ListenAddress = envInfo.BindAddress diff --git a/pkg/agent/containerd/config.go b/pkg/agent/containerd/config.go index 18d0c5be47..095d78ba31 100644 --- a/pkg/agent/containerd/config.go +++ b/pkg/agent/containerd/config.go @@ -21,19 +21,48 @@ import ( type HostConfigs map[string]templates.HostConfig +type templateGeneration struct { + version int + filename string + base string +} + +var templateGenerations = []templateGeneration{ + { + version: 3, + filename: "config-v3.toml.tmpl", + base: templates.ContainerdConfigTemplateV3, + }, + { + version: 2, + filename: "config.toml.tmpl", + base: templates.ContainerdConfigTemplate, + }, +} + // writeContainerdConfig renders and saves config.toml from the filled template func writeContainerdConfig(cfg *config.Node, containerdConfig templates.ContainerdConfig) error { - var containerdTemplate string - containerdTemplateBytes, err := os.ReadFile(cfg.Containerd.Template) - if err == nil { - logrus.Infof("Using containerd template at %s", cfg.Containerd.Template) - containerdTemplate = string(containerdTemplateBytes) - } else if os.IsNotExist(err) { - containerdTemplate = templates.ContainerdConfigTemplate - } else { - return err + // use v3 template by default + userTemplate := templates.ContainerdConfigTemplateV3 + baseTemplate := templates.ContainerdConfigTemplateV3 + cfg.Containerd.ConfigVersion = 3 + + // check for user templates + for _, tg := range templateGenerations { + path := filepath.Join(cfg.Containerd.Template, tg.filename) + b, err := os.ReadFile(path) + if err == nil { + logrus.Infof("Using containerd config template at %s", path) + baseTemplate = tg.base + userTemplate = string(b) + cfg.Containerd.ConfigVersion = tg.version + break + } else if !os.IsNotExist(err) { + return err + } } - parsedTemplate, err := templates.ParseTemplateFromConfig(containerdTemplate, containerdConfig) + + parsedTemplate, err := templates.ParseTemplateFromConfig(userTemplate, baseTemplate, containerdConfig) if err != nil { return err } diff --git a/pkg/agent/containerd/config_linux.go b/pkg/agent/containerd/config_linux.go index b5219aee4e..edd27d7a88 100644 --- a/pkg/agent/containerd/config_linux.go +++ b/pkg/agent/containerd/config_linux.go @@ -31,9 +31,16 @@ func getContainerdArgs(cfg *config.Node) []string { args := []string{ "containerd", "-c", cfg.Containerd.Config, - "-a", cfg.Containerd.Address, - "--state", cfg.Containerd.State, - "--root", cfg.Containerd.Root, + } + + // The legacy version 2 linux containerd config template did not include + // address/state/root settings, so they need to be passed on the command line. + if cfg.Containerd.ConfigVersion < 3 { + args = append(args, + "-a", cfg.Containerd.Address, + "--state", cfg.Containerd.State, + "--root", cfg.Containerd.Root, + ) } return args } diff --git a/pkg/agent/containerd/config_test.go b/pkg/agent/containerd/config_test.go index 98a948a224..95238ec2c3 100644 --- a/pkg/agent/containerd/config_test.go +++ b/pkg/agent/containerd/config_test.go @@ -1471,14 +1471,24 @@ func Test_UnitGetHostConfigs(t *testing.T) { t.Fatalf("failed to parse %s: %v\n", registriesFile, err) } + // This is an odd mishmash of linux and windows stuff just to excercise all the template bits nodeConfig := &config.Node{ + DefaultRuntime: "runhcs-wcow-process", Containerd: config.Containerd{ Registry: tempDir + "/hosts.d", + Config: tempDir + "/config.toml", + Template: tempDir, + Address: "/run/k3s/containerd/containerd.sock", + Root: "/var/lib/rancher/k3s/agent/containerd", + Opt: "/var/lib/rancher/k3s/agent/containerd", + State: "/run/k3s/containerd", }, AgentConfig: config.Agent{ ImageServiceSocket: "containerd-stargz-grpc.sock", Registry: registry.Registry, Snapshotter: "stargz", + CNIBinDir: "/var/lib/rancher/k3s/data/cni", + CNIConfDir: "/var/lib/rancher/k3s/agent/etc/cni/net.d", }, } @@ -1498,20 +1508,39 @@ func Test_UnitGetHostConfigs(t *testing.T) { // Confirm that hosts.toml renders properly for all registries for host, config := range got { - hostsTemplate, err := templates.ParseHostsTemplateFromConfig(templates.HostsTomlTemplate, config) + hostsToml, err := templates.ParseHostsTemplateFromConfig(templates.HostsTomlTemplate, config) assert.NoError(t, err, "ParseHostTemplateFromConfig for %s", host) - t.Logf("%s/hosts.d/%s/hosts.toml\n%s", tempDir, host, hostsTemplate) + t.Logf("%s/hosts.d/%s/hosts.toml\n%s", tempDir, host, hostsToml) } - // Confirm that the main containerd config.toml renders properly - containerdConfig := templates.ContainerdConfig{ - NodeConfig: nodeConfig, - PrivateRegistryConfig: registry.Registry, - Program: "k3s", + for _, template := range []string{"config.toml.tmpl", "config-v3.toml.tmpl"} { + t.Run(template, func(t *testing.T) { + templateFile := filepath.Join(tempDir, template) + err = os.WriteFile(templateFile, []byte(`{{ template "base" . }}`), 0600) + assert.NoError(t, err, "Write Template") + + // Confirm that the main containerd config.toml renders properly + containerdConfig := templates.ContainerdConfig{ + NodeConfig: nodeConfig, + PrivateRegistryConfig: registry.Registry, + Program: "k3s", + ExtraRuntimes: map[string]templates.ContainerdRuntimeConfig{ + "runhcs-wcow-process": templates.ContainerdRuntimeConfig{ + RuntimeType: "io.containerd.runhcs.v1", + }, + "wasmtime": templates.ContainerdRuntimeConfig{ + RuntimeType: "io.containerd.wasmtime.v1", + BinaryName: "containerd-shim-wasmtime-v1", + }, + }, + } + err = writeContainerdConfig(nodeConfig, containerdConfig) + assert.NoError(t, err, "ParseTemplateFromConfig") + configToml, err := os.ReadFile(nodeConfig.Containerd.Config) + assert.NoError(t, err, "ReadFile "+nodeConfig.Containerd.Config) + t.Logf("%s\n%s", nodeConfig.Containerd.Config, configToml) + }) } - configTemplate, err := templates.ParseTemplateFromConfig(templates.ContainerdConfigTemplate, containerdConfig) - assert.NoError(t, err, "ParseTemplateFromConfig") - t.Logf("%s/config.toml\n%s", tempDir, configTemplate) }) } } diff --git a/pkg/agent/containerd/config_windows.go b/pkg/agent/containerd/config_windows.go index c12b88c84d..71b9ded0a0 100644 --- a/pkg/agent/containerd/config_windows.go +++ b/pkg/agent/containerd/config_windows.go @@ -18,6 +18,8 @@ func getContainerdArgs(cfg *config.Node) []string { "containerd", "-c", cfg.Containerd.Config, } + // The legacy version 2 windows containerd config template did include + // address/state/root settings, so they do not need to be passed on the command line. return args } @@ -28,11 +30,11 @@ func SetupContainerdConfig(cfg *config.Node) error { logrus.Warn("SELinux isn't supported on windows") } + cfg.DefaultRuntime = "runhcs-wcow-process" + cfg.AgentConfig.Snapshotter = "windows" containerdConfig := templates.ContainerdConfig{ NodeConfig: cfg, DisableCgroup: true, - SystemdCgroup: false, - IsRunningInUserNS: false, PrivateRegistryConfig: cfg.AgentConfig.Registry, NoDefaultEndpoint: cfg.Containerd.NoDefault, } diff --git a/pkg/agent/templates/templates.go b/pkg/agent/templates/templates.go index 9a66b90074..0177f17348 100644 --- a/pkg/agent/templates/templates.go +++ b/pkg/agent/templates/templates.go @@ -1,8 +1,11 @@ package templates import ( + "bufio" "bytes" + "io" "net/url" + "strings" "text/template" "github.com/rancher/wharfie/pkg/registries" @@ -44,6 +47,7 @@ type HostConfig struct { var HostsTomlHeader = "# File generated by " + version.Program + ". DO NOT EDIT.\n" +// This hosts.toml template is used by both Linux and Windows nodes const HostsTomlTemplate = ` {{- /* */ -}} # File generated by {{ .Program }}. DO NOT EDIT. @@ -91,21 +95,157 @@ skip_verify = true {{ end -}} ` -func ParseTemplateFromConfig(templateBuffer string, config interface{}) (string, error) { +// This version 3 config template is used by both Linux and Windows nodes +const ContainerdConfigTemplateV3 = ` +{{- /* */ -}} +# File generated by {{ .Program }}. DO NOT EDIT. Use config.toml.tmpl instead. +version = 3 +root = {{ printf "%q" .NodeConfig.Containerd.Root }} +state = {{ printf "%q" .NodeConfig.Containerd.State }} + +[grpc] + address = {{ deschemify .NodeConfig.Containerd.Address | printf "%q" }} + +[plugins.'io.containerd.internal.v1.opt'] + path = {{ printf "%q" .NodeConfig.Containerd.Opt }} + +[plugins.'io.containerd.grpc.v1.cri'] + stream_server_address = "127.0.0.1" + stream_server_port = "10010" + +[plugins.'io.containerd.cri.v1.runtime'] + enable_selinux = {{ .NodeConfig.SELinux }} + enable_unprivileged_ports = {{ .EnableUnprivileged }} + enable_unprivileged_icmp = {{ .EnableUnprivileged }} + device_ownership_from_security_context = {{ .NonrootDevices }} + +{{ if .DisableCgroup}} + disable_cgroup = true +{{ end }} + +{{ if .IsRunningInUserNS }} + disable_apparmor = true + restrict_oom_score_adj = true +{{ end }} + +{{ with .NodeConfig.AgentConfig.Snapshotter }} +[plugins.'io.containerd.cri.v1.images'] + snapshotter = "{{ . }}" + disable_snapshot_annotations = {{ if eq . "stargz" }}false{{else}}true{{end}} +{{ end }} + +{{ with .NodeConfig.AgentConfig.PauseImage }} +[plugins.'io.containerd.cri.v1.images'.pinned_images] + sandbox = "{{ . }}" +{{ end }} + +[plugins.'io.containerd.cri.v1.images'.registry] + config_path = {{ printf "%q" .NodeConfig.Containerd.Registry }} + +{{ if not .NodeConfig.NoFlannel }} +[plugins.'io.containerd.cri.v1.runtime'.cni] + bin_dir = {{ printf "%q" .NodeConfig.AgentConfig.CNIBinDir }} + conf_dir = {{ printf "%q" .NodeConfig.AgentConfig.CNIConfDir }} +{{ end }} + +{{ if or .NodeConfig.Containerd.BlockIOConfig .NodeConfig.Containerd.RDTConfig }} +[plugins.'io.containerd.service.v1.tasks-service'] + {{ with .NodeConfig.Containerd.BlockIOConfig }}blockio_config_file = {{ printf "%q" . }}{{ end }} + {{ with .NodeConfig.Containerd.RDTConfig }}rdt_config_file = {{ printf "%q" . }}{{ end }} +{{ end }} + +{{ with .NodeConfig.DefaultRuntime }} +[plugins.'io.containerd.cri.v1.runtime'.containerd] + default_runtime_name = "{{ . }}" +{{ end }} + +{{ with .SystemdCgroup }} +[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + +[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runc.options] + SystemdCgroup = {{ . }} +{{ end }} + +{{ range $k, $v := .ExtraRuntimes }} +[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.'{{ $k }}'] + runtime_type = "{{$v.RuntimeType}}" +{{ with $v.BinaryName}} +[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.'{{ $k }}'.options] + BinaryName = {{ printf "%q" . }} + SystemdCgroup = {{ $.SystemdCgroup }} +{{ end }} +{{ end }} + +{{ if .PrivateRegistryConfig }} +{{ range $k, $v := .PrivateRegistryConfig.Configs }} +{{ with $v.Auth }} +[plugins.'io.containerd.grpc.v1.cri'.registry.configs.'{{ $k }}'.auth] + {{ with .Username }}username = {{ printf "%q" . }}{{ end }} + {{ with .Password }}password = {{ printf "%q" . }}{{ end }} + {{ with .Auth }}auth = {{ printf "%q" . }}{{ end }} + {{ with .IdentityToken }}identitytoken = {{ printf "%q" . }}{{ end }} +{{ end }} +{{ end }} +{{ end }} + +{{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }} +{{ with .NodeConfig.AgentConfig.ImageServiceSocket }} +[plugins.'io.containerd.snapshotter.v1.stargz'] + cri_keychain_image_service_path = {{ printf "%q" . }} + +[plugins.'io.containerd.snapshotter.v1.stargz'.cri_keychain] + enable_keychain = true +{{ end }} + +[plugins.'io.containerd.snapshotter.v1.stargz'.registry] + config_path = {{ printf "%q" .NodeConfig.Containerd.Registry }} + +{{ if .PrivateRegistryConfig }} +{{ range $k, $v := .PrivateRegistryConfig.Configs }} +{{ with $v.Auth }} +[plugins.'io.containerd.snapshotter.v1.stargz'.registry.configs.'{{ $k }}'.auth] + {{ with .Username }}username = {{ printf "%q" . }}{{ end }} + {{ with .Password }}password = {{ printf "%q" . }}{{ end }} + {{ with .Auth }}auth = {{ printf "%q" . }}{{ end }} + {{ with .IdentityToken }}identitytoken = {{ printf "%q" . }}{{ end }} +{{ end }} +{{ end }} +{{ end }} +{{ end }} +` + +func ParseTemplateFromConfig(userTemplate, baseTemplate string, config interface{}) (string, error) { out := new(bytes.Buffer) - t := template.Must(template.New("compiled_template").Funcs(templateFuncs).Parse(templateBuffer)) - template.Must(t.New("base").Parse(ContainerdConfigTemplate)) + t := template.Must(template.New("compiled_template").Funcs(templateFuncs).Parse(userTemplate)) + template.Must(t.New("base").Parse(baseTemplate)) if err := t.Execute(out, config); err != nil { return "", err } - return out.String(), nil + return trimEmpty(out) } -func ParseHostsTemplateFromConfig(templateBuffer string, config interface{}) (string, error) { +func ParseHostsTemplateFromConfig(userTemplate string, config interface{}) (string, error) { out := new(bytes.Buffer) - t := template.Must(template.New("compiled_template").Funcs(templateFuncs).Parse(templateBuffer)) + t := template.Must(template.New("compiled_template").Funcs(templateFuncs).Parse(userTemplate)) if err := t.Execute(out, config); err != nil { return "", err } - return out.String(), nil + return trimEmpty(out) +} + +// trimEmpty removes excess empty lines from the rendered template +func trimEmpty(r io.Reader) (string, error) { + builder := strings.Builder{} + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := scanner.Text() + if strings.TrimSpace(line) != "" { + if strings.HasPrefix(line, "[") { + builder.WriteString("\n") + } + builder.WriteString(line + "\n") + } + } + return builder.String(), scanner.Err() } diff --git a/pkg/agent/templates/templates_linux.go b/pkg/agent/templates/templates_linux.go index dffce1737c..65b6b1933a 100644 --- a/pkg/agent/templates/templates_linux.go +++ b/pkg/agent/templates/templates_linux.go @@ -3,9 +3,11 @@ package templates import ( + "encoding/json" "text/template" ) +// This version 2 config template is only used by Linux nodes const ContainerdConfigTemplate = ` {{- /* */ -}} # File generated by {{ .Program }}. DO NOT EDIT. Use config.toml.tmpl instead. @@ -110,4 +112,8 @@ var templateFuncs = template.FuncMap{ "deschemify": func(s string) string { return s }, + "toJson": func(v interface{}) string { + output, _ := json.Marshal(v) + return string(output) + }, } diff --git a/pkg/agent/templates/templates_windows.go b/pkg/agent/templates/templates_windows.go index 5cccd7e43d..826ec0becd 100644 --- a/pkg/agent/templates/templates_windows.go +++ b/pkg/agent/templates/templates_windows.go @@ -4,11 +4,13 @@ package templates import ( + "encoding/json" "net/url" "strings" "text/template" ) +// This version 2 config template is only used by Windows nodes const ContainerdConfigTemplate = ` {{- /* */ -}} # File generated by {{ .Program }}. DO NOT EDIT. Use config.toml.tmpl instead. @@ -155,4 +157,8 @@ var templateFuncs = template.FuncMap{ } return s }, + "toJson": func(v interface{}) string { + output, _ := json.Marshal(v) + return string(output) + }, } diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 2de46082ce..2410ec4918 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -92,6 +92,7 @@ type Containerd struct { NonrootDevices bool SELinux bool Debug bool + ConfigVersion int } type CRIDockerd struct {