diff --git a/pkg/agent/containerd/config.go b/pkg/agent/containerd/config.go index 57c9c91259..ac2c5254ee 100644 --- a/pkg/agent/containerd/config.go +++ b/pkg/agent/containerd/config.go @@ -13,9 +13,12 @@ import ( "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/spegel" "github.com/k3s-io/k3s/pkg/version" + "github.com/rancher/wharfie/pkg/registries" "github.com/sirupsen/logrus" ) +type HostConfigs map[string]templates.HostConfig + // writeContainerdConfig renders and saves config.toml from the filled template func writeContainerdConfig(cfg *config.Node, containerdConfig templates.ContainerdConfig) error { var containerdTemplate string @@ -39,79 +42,163 @@ func writeContainerdConfig(cfg *config.Node, containerdConfig templates.Containe // writeContainerdHosts merges registry mirrors/configs, and renders and saves hosts.toml from the filled template func writeContainerdHosts(cfg *config.Node, containerdConfig templates.ContainerdConfig) error { mirrorAddr := net.JoinHostPort(spegel.DefaultRegistry.InternalAddress, spegel.DefaultRegistry.RegistryPort) - registry := containerdConfig.PrivateRegistryConfig + hosts := getHostConfigs(containerdConfig.PrivateRegistryConfig, containerdConfig.NoDefaultEndpoint, mirrorAddr) + + // Clean up previous configuration templates + os.RemoveAll(cfg.Containerd.Registry) + + // Write out new templates + for host, config := range hosts { + hostDir := filepath.Join(cfg.Containerd.Registry, host) + hostsFile := filepath.Join(hostDir, "hosts.toml") + hostsTemplate, err := templates.ParseHostsTemplateFromConfig(templates.HostsTomlTemplate, config) + if err != nil { + return err + } + if err := os.MkdirAll(hostDir, 0700); err != nil { + return err + } + if err := util2.WriteFile(hostsFile, hostsTemplate); err != nil { + return err + } + } + + return nil +} + +// getHostConfigs merges the registry mirrors/configs into HostConfig template structs +func getHostConfigs(registry *registries.Registry, noDefaultEndpoint bool, mirrorAddr string) HostConfigs { hosts := map[string]templates.HostConfig{} + // create endpoints for mirrors for host, mirror := range registry.Mirrors { - defaultHost, _ := docker.DefaultHost(host) config := templates.HostConfig{ - Host: defaultHost, Program: version.Program, } - if host == "*" { - host = "_default" - config.Host = "" - } else if containerdConfig.NoDefaultEndpoint { - config.Host = "" + if uri, _, err := normalizeEndpointAddress(host, mirrorAddr); err == nil { + config.DefaultEndpoint = uri.String() } + // TODO: rewrites are currently copied from the mirror settings into each endpoint. // In the future, we should allow for per-endpoint rewrites, instead of expecting // all mirrors to have the same structure. This will require changes to the registries.yaml // structure, which is defined in rancher/wharfie. for _, endpoint := range mirror.Endpoints { - if endpointURL, err := url.Parse(endpoint); err == nil { - re := templates.RegistryEndpoint{ - OverridePath: endpointURL.Path != "" && endpointURL.Path != "/" && !strings.HasSuffix(endpointURL.Path, "/v2"), - Config: registry.Configs[endpointURL.Host], - Rewrites: mirror.Rewrites, - URI: endpoint, - } + uri, override, err := normalizeEndpointAddress(endpoint, mirrorAddr) + if err != nil { + logrus.Warnf("Ignoring invalid endpoint URL %s for %s: %v", endpoint, host, err) + } else { + var rewrites map[string]string // Do not apply rewrites to the embedded registry endpoint - if endpointURL.Host == mirrorAddr { - re.Rewrites = nil + if uri.Host != mirrorAddr { + rewrites = mirror.Rewrites } - config.Endpoints = append(config.Endpoints, re) + config.Endpoints = append(config.Endpoints, templates.RegistryEndpoint{ + Config: registry.Configs[uri.Host], + Rewrites: rewrites, + OverridePath: override, + URI: uri.String(), + }) } } + + if host == "*" { + host = "_default" + } hosts[host] = config } + // create endpoints for registries using default endpoints for host, registry := range registry.Configs { config, ok := hosts[host] if !ok { config = templates.HostConfig{ Program: version.Program, } + if uri, _, err := normalizeEndpointAddress(host, mirrorAddr); err == nil { + config.DefaultEndpoint = uri.String() + } } + // If there is config for this host but no endpoints, inject the config for the default endpoint. if len(config.Endpoints) == 0 { - config.Endpoints = []templates.RegistryEndpoint{ - { + uri, _, err := normalizeEndpointAddress(host, mirrorAddr) + if err != nil { + logrus.Warnf("Ignoring invalid endpoint URL %s for %s: %v", host, host, err) + } else { + config.Endpoints = append(config.Endpoints, templates.RegistryEndpoint{ Config: registry, - URI: "https://" + host, - }, + URI: uri.String(), + }) } } + + if host == "*" { + host = "_default" + } hosts[host] = config } - // Clean up previous configuration templates - os.RemoveAll(cfg.Containerd.Registry) - - // Write out new templates + // Clean up hosts and default endpoints where resulting config leaves only defaults for host, config := range hosts { - hostDir := filepath.Join(cfg.Containerd.Registry, host) - hostsFile := filepath.Join(hostDir, "hosts.toml") - hostsTemplate, err := templates.ParseHostsTemplateFromConfig(templates.HostsTomlTemplate, config) - if err != nil { - return err + // if default endpoint is disabled, or this is the wildcard host, delete the default endpoint + if noDefaultEndpoint || host == "_default" { + config.DefaultEndpoint = "" + hosts[host] = config } - if err := os.MkdirAll(hostDir, 0700); err != nil { - return err + if l := len(config.Endpoints); l > 0 { + if ep := config.Endpoints[l-1]; ep.URI == config.DefaultEndpoint { + // if the last endpoint is the default endpoint + if ep.Config.Auth == nil && ep.Config.TLS == nil && len(ep.Rewrites) == 0 { + // if has no config, delete this host to use the default config + delete(hosts, host) + } else { + // if it has config, delete the default endpoint + config.DefaultEndpoint = "" + hosts[host] = config + } + } + } else { + // if this host has no endpoints, delete this host to use the default config + delete(hosts, host) } - if err := util2.WriteFile(hostsFile, hostsTemplate); err != nil { - return err + } + + return hosts +} + +// normalizeEndpointAddress normalizes the endpoint address. +// If successful, it returns the URL, and a bool indicating if the endpoint path should be overridden. +// If unsuccessful, an error is returned. +// Scheme and hostname logic should match containerd: +// https://github.com/containerd/containerd/blob/v1.7.13/remotes/docker/config/hosts.go#L99-L131 +func normalizeEndpointAddress(endpoint, mirrorAddr string) (*url.URL, bool, error) { + // Ensure that the endpoint address has a scheme so that the URL is parsed properly + if !strings.Contains(endpoint, "://") { + endpoint = "//" + endpoint + } + endpointURL, err := url.Parse(endpoint) + if err != nil { + return nil, false, err + } + port := endpointURL.Port() + + // set default scheme, if not provided + if endpointURL.Scheme == "" { + // localhost on odd ports defaults to http, unless it's the embedded mirror + if docker.IsLocalhost(endpointURL.Host) && port != "" && port != "443" && endpointURL.Host != mirrorAddr { + endpointURL.Scheme = "http" + } else { + endpointURL.Scheme = "https" } } + endpointURL.Host, _ = docker.DefaultHost(endpointURL.Host) - return nil + switch endpointURL.Path { + case "", "/", "/v2": + // If the path is empty, /, or /v2, use the default path. + endpointURL.Path = "/v2" + return endpointURL, false, nil + } + + return endpointURL, true, nil } diff --git a/pkg/agent/containerd/config_test.go b/pkg/agent/containerd/config_test.go new file mode 100644 index 0000000000..c52d18d071 --- /dev/null +++ b/pkg/agent/containerd/config_test.go @@ -0,0 +1,1110 @@ +package containerd + +import ( + "net" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/k3s-io/k3s/pkg/agent/templates" + "github.com/k3s-io/k3s/pkg/daemons/config" + "github.com/k3s-io/k3s/pkg/spegel" + "github.com/rancher/wharfie/pkg/registries" + "github.com/stretchr/testify/assert" +) + +func Test_UnitGetHostConfigs(t *testing.T) { + type args struct { + registryContent string + noDefaultEndpoint bool + mirrorAddr string + } + tests := []struct { + name string + args args + want HostConfigs + }{ + { + name: "no registries", + want: HostConfigs{}, + }, + { + name: "registry with default endpoint", + args: args{ + registryContent: ` + mirrors: + docker.io: + `, + }, + want: HostConfigs{}, + }, + { + name: "registry with default endpoint - embedded registry", + args: args{ + mirrorAddr: "127.0.0.1:6443", + registryContent: ` + mirrors: + docker.io: + `, + }, + want: HostConfigs{ + "docker.io": templates.HostConfig{ + DefaultEndpoint: "https://registry-1.docker.io/v2", + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://127.0.0.1:6443/v2", + Config: registries.RegistryConfig{ + TLS: ®istries.TLSConfig{ + CAFile: "server-ca", + KeyFile: "client-key", + CertFile: "client-cert", + }, + }, + }, + }, + }, + "127.0.0.1:6443": templates.HostConfig{ + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://127.0.0.1:6443/v2", + Config: registries.RegistryConfig{ + TLS: ®istries.TLSConfig{ + CAFile: "server-ca", + KeyFile: "client-key", + CertFile: "client-cert", + }, + }, + }, + }, + }, + }, + }, + { + name: "registry with default endpoint and creds", + args: args{ + registryContent: ` + mirrors: + docker.io: + configs: + docker.io: + auth: + username: user + password: pass + `, + }, + want: HostConfigs{ + "docker.io": templates.HostConfig{ + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://registry-1.docker.io/v2", + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", + }, + }, + }, + }, + }, + }, + }, + { + name: "registry with only creds", + args: args{ + registryContent: ` + configs: + docker.io: + auth: + username: user + password: pass + `, + }, + want: HostConfigs{ + "docker.io": templates.HostConfig{ + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://registry-1.docker.io/v2", + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", + }, + }, + }, + }, + }, + }, + }, + { + name: "private registry with default endpoint", + args: args{ + registryContent: ` + mirrors: + registry.example.com: + `, + }, + want: HostConfigs{}, + }, + { + name: "private registry with default endpoint and creds", + args: args{ + registryContent: ` + mirrors: + registry.example.com: + configs: + registry.example.com: + auth: + username: user + password: pass + `, + }, + want: HostConfigs{ + "registry.example.com": templates.HostConfig{ + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://registry.example.com/v2", + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", + }, + }, + }, + }, + }, + }, + }, + { + name: "private registry with only creds", + args: args{ + registryContent: ` + configs: + registry.example.com: + auth: + username: user + password: pass + `, + }, + want: HostConfigs{ + "registry.example.com": templates.HostConfig{ + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://registry.example.com/v2", + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", + }, + }, + }, + }, + }, + }, + }, + { + name: "registry with mirror endpoint - full URL with override path", + args: args{ + registryContent: ` + mirrors: + docker.io: + endpoint: + - https://registry.example.com/prefix/v2 + `, + }, + want: HostConfigs{ + "docker.io": templates.HostConfig{ + DefaultEndpoint: "https://registry-1.docker.io/v2", + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + OverridePath: true, + URI: "https://registry.example.com/prefix/v2", + }, + }, + }, + }, + }, + { + name: "registry with mirror endpoint - hostname only with override path", + args: args{ + registryContent: ` + mirrors: + docker.io: + endpoint: + - registry.example.com/prefix/v2 + `, + }, + want: HostConfigs{ + "docker.io": templates.HostConfig{ + DefaultEndpoint: "https://registry-1.docker.io/v2", + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + OverridePath: true, + URI: "https://registry.example.com/prefix/v2", + }, + }, + }, + }, + }, + { + name: "registry with mirror endpoint - hostname only with default path", + args: args{ + registryContent: ` + mirrors: + docker.io: + endpoint: + - registry.example.com/v2 + `, + }, + want: HostConfigs{ + "docker.io": templates.HostConfig{ + DefaultEndpoint: "https://registry-1.docker.io/v2", + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://registry.example.com/v2", + }, + }, + }, + }, + }, + { + name: "registry with mirror endpoint - full URL", + args: args{ + registryContent: ` + mirrors: + docker.io: + endpoint: + - https://registry.example.com/v2 + `, + }, + want: HostConfigs{ + "docker.io": templates.HostConfig{ + DefaultEndpoint: "https://registry-1.docker.io/v2", + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://registry.example.com/v2", + }, + }, + }, + }, + }, + { + name: "registry with mirror endpoint - URL without path", + args: args{ + registryContent: ` + mirrors: + docker.io: + endpoint: + - https://registry.example.com + `, + }, + want: HostConfigs{ + "docker.io": templates.HostConfig{ + DefaultEndpoint: "https://registry-1.docker.io/v2", + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://registry.example.com/v2", + }, + }, + }, + }, + }, + { + name: "registry with mirror endpoint - hostname only", + args: args{ + registryContent: ` + mirrors: + docker.io: + endpoint: + - registry.example.com + `, + }, + want: HostConfigs{ + "docker.io": templates.HostConfig{ + DefaultEndpoint: "https://registry-1.docker.io/v2", + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://registry.example.com/v2", + }, + }, + }, + }, + }, + { + name: "registry with mirror endpoint - hostname and port only", + args: args{ + registryContent: ` + mirrors: + docker.io: + endpoint: + - registry.example.com:443 + `, + }, + want: HostConfigs{ + "docker.io": templates.HostConfig{ + DefaultEndpoint: "https://registry-1.docker.io/v2", + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://registry.example.com:443/v2", + }, + }, + }, + }, + }, + { + name: "registry with mirror endpoint - ip address only", + args: args{ + registryContent: ` + mirrors: + docker.io: + endpoint: + - 1.2.3.4 + `, + }, + want: HostConfigs{ + "docker.io": templates.HostConfig{ + DefaultEndpoint: "https://registry-1.docker.io/v2", + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://1.2.3.4/v2", + }, + }, + }, + }, + }, + { + name: "registry with mirror endpoint - ip and port only", + args: args{ + registryContent: ` + mirrors: + docker.io: + endpoint: + - 1.2.3.4:443 + `, + }, + want: HostConfigs{ + "docker.io": templates.HostConfig{ + DefaultEndpoint: "https://registry-1.docker.io/v2", + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://1.2.3.4:443/v2", + }, + }, + }, + }, + }, + { + name: "registry with mirror endpoint - localhost and port only", + args: args{ + registryContent: ` + mirrors: + docker.io: + endpoint: + - localhost:5000 + `, + }, + want: HostConfigs{ + "docker.io": templates.HostConfig{ + DefaultEndpoint: "https://registry-1.docker.io/v2", + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "http://localhost:5000/v2", + }, + }, + }, + }, + }, + { + name: "registry with mirror endpoint - localhost and port with scheme", + args: args{ + registryContent: ` + mirrors: + docker.io: + endpoint: + - https://localhost:5000 + `, + }, + want: HostConfigs{ + "docker.io": templates.HostConfig{ + DefaultEndpoint: "https://registry-1.docker.io/v2", + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://localhost:5000/v2", + }, + }, + }, + }, + }, + { + name: "registry with mirror endpoint - loopback ip and port only", + args: args{ + registryContent: ` + mirrors: + docker.io: + endpoint: + - 127.0.0.1:5000 + `, + }, + want: HostConfigs{ + "docker.io": templates.HostConfig{ + DefaultEndpoint: "https://registry-1.docker.io/v2", + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "http://127.0.0.1:5000/v2", + }, + }, + }, + }, + }, + { + name: "registry with mirror endpoint - loopback ip and port with scheme", + args: args{ + registryContent: ` + mirrors: + docker.io: + endpoint: + - https://127.0.0.1:5000 + `, + }, + want: HostConfigs{ + "docker.io": templates.HostConfig{ + DefaultEndpoint: "https://registry-1.docker.io/v2", + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://127.0.0.1:5000/v2", + }, + }, + }, + }, + }, + { + name: "registry with mirror endpoint and mirror creds", + args: args{ + registryContent: ` + mirrors: + docker.io: + endpoint: + - https://registry.example.com/v2 + configs: + registry.example.com: + auth: + username: user + password: pass + `, + }, + want: HostConfigs{ + "docker.io": templates.HostConfig{ + DefaultEndpoint: "https://registry-1.docker.io/v2", + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://registry.example.com/v2", + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", + }, + }, + }, + }, + }, + "registry.example.com": templates.HostConfig{ + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://registry.example.com/v2", + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", + }, + }, + }, + }, + }, + }, + }, + { + name: "registry with mirror endpoint and mirror creds - override path with v2", + args: args{ + registryContent: ` + mirrors: + docker.io: + endpoint: + - https://registry.example.com/prefix/v2 + registry.example.com: + endpoint: + - https://registry.example.com/prefix/v2 + configs: + registry.example.com: + auth: + username: user + password: pass + `, + }, + want: HostConfigs{ + "docker.io": templates.HostConfig{ + DefaultEndpoint: "https://registry-1.docker.io/v2", + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + OverridePath: true, + URI: "https://registry.example.com/prefix/v2", + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", + }, + }, + }, + }, + }, + "registry.example.com": templates.HostConfig{ + DefaultEndpoint: "https://registry.example.com/v2", + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + OverridePath: true, + URI: "https://registry.example.com/prefix/v2", + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", + }, + }, + }, + }, + }, + }, + }, + { + name: "registry with mirror endpoint and mirror creds - override path without v2", + args: args{ + registryContent: ` + mirrors: + docker.io: + endpoint: + - https://registry.example.com/project/registry + registry.example.com: + endpoint: + - https://registry.example.com/project/registry + configs: + registry.example.com: + auth: + username: user + password: pass + `, + }, + want: HostConfigs{ + "docker.io": templates.HostConfig{ + DefaultEndpoint: "https://registry-1.docker.io/v2", + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + OverridePath: true, + URI: "https://registry.example.com/project/registry", + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", + }, + }, + }, + }, + }, + "registry.example.com": templates.HostConfig{ + DefaultEndpoint: "https://registry.example.com/v2", + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + OverridePath: true, + URI: "https://registry.example.com/project/registry", + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", + }, + }, + }, + }, + }, + }, + }, + { + name: "registry with mirror endpoint and mirror creds - no default endpoint", + args: args{ + noDefaultEndpoint: true, + registryContent: ` + mirrors: + docker.io: + endpoint: + - https://registry.example.com/v2 + configs: + registry.example.com: + auth: + username: user + password: pass + `, + }, + want: HostConfigs{ + "docker.io": templates.HostConfig{ + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://registry.example.com/v2", + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", + }, + }, + }, + }, + }, + "registry.example.com": templates.HostConfig{ + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://registry.example.com/v2", + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", + }, + }, + }, + }, + }, + }, + }, + { + name: "registry with mirror endpoint and mirror creds - embedded registry", + args: args{ + mirrorAddr: "127.0.0.1:6443", + registryContent: ` + mirrors: + docker.io: + endpoint: + - https://registry.example.com/v2 + configs: + registry.example.com: + auth: + username: user + password: pass + `, + }, + want: HostConfigs{ + "docker.io": templates.HostConfig{ + DefaultEndpoint: "https://registry-1.docker.io/v2", + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://127.0.0.1:6443/v2", + Config: registries.RegistryConfig{ + TLS: ®istries.TLSConfig{ + CAFile: "server-ca", + KeyFile: "client-key", + CertFile: "client-cert", + }, + }, + }, + templates.RegistryEndpoint{ + URI: "https://registry.example.com/v2", + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", + }, + }, + }, + }, + }, + "registry.example.com": templates.HostConfig{ + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://registry.example.com/v2", + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", + }, + }, + }, + }, + }, + "127.0.0.1:6443": templates.HostConfig{ + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://127.0.0.1:6443/v2", + Config: registries.RegistryConfig{ + TLS: ®istries.TLSConfig{ + CAFile: "server-ca", + KeyFile: "client-key", + CertFile: "client-cert", + }, + }, + }, + }, + }, + }, + }, + { + name: "registry with mirror endpoint and mirror creds - embedded registry with rewrites", + args: args{ + mirrorAddr: "127.0.0.1:6443", + registryContent: ` + mirrors: + docker.io: + endpoint: + - https://registry.example.com/v2 + rewrite: + "^rancher/(.*)": "docker/rancher-images/$1" + configs: + registry.example.com: + auth: + username: user + password: pass + `, + }, + want: HostConfigs{ + "docker.io": templates.HostConfig{ + DefaultEndpoint: "https://registry-1.docker.io/v2", + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://127.0.0.1:6443/v2", + Config: registries.RegistryConfig{ + TLS: ®istries.TLSConfig{ + CAFile: "server-ca", + KeyFile: "client-key", + CertFile: "client-cert", + }, + }, + }, + templates.RegistryEndpoint{ + URI: "https://registry.example.com/v2", + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", + }, + }, + Rewrites: map[string]string{ + "^rancher/(.*)": "docker/rancher-images/$1", + }, + }, + }, + }, + "registry.example.com": templates.HostConfig{ + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://registry.example.com/v2", + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", + }, + }, + }, + }, + }, + "127.0.0.1:6443": templates.HostConfig{ + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://127.0.0.1:6443/v2", + Config: registries.RegistryConfig{ + TLS: ®istries.TLSConfig{ + CAFile: "server-ca", + KeyFile: "client-key", + CertFile: "client-cert", + }, + }, + }, + }, + }, + }, + }, + { + name: "registry with mirror endpoint and mirror creds - embedded registry and no default endpoint", + args: args{ + mirrorAddr: "127.0.0.1:6443", + noDefaultEndpoint: true, + registryContent: ` + mirrors: + docker.io: + endpoint: + - https://registry.example.com/v2 + configs: + registry.example.com: + auth: + username: user + password: pass + `, + }, + want: HostConfigs{ + "docker.io": templates.HostConfig{ + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://127.0.0.1:6443/v2", + Config: registries.RegistryConfig{ + TLS: ®istries.TLSConfig{ + CAFile: "server-ca", + KeyFile: "client-key", + CertFile: "client-cert", + }, + }, + }, + templates.RegistryEndpoint{ + URI: "https://registry.example.com/v2", + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", + }, + }, + }, + }, + }, + "registry.example.com": templates.HostConfig{ + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://registry.example.com/v2", + Config: registries.RegistryConfig{ + Auth: ®istries.AuthConfig{ + Username: "user", + Password: "pass", + }, + }, + }, + }, + }, + "127.0.0.1:6443": templates.HostConfig{ + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://127.0.0.1:6443/v2", + Config: registries.RegistryConfig{ + TLS: ®istries.TLSConfig{ + CAFile: "server-ca", + KeyFile: "client-key", + CertFile: "client-cert", + }, + }, + }, + }, + }, + }, + }, + { + name: "wildcard mirror endpoint - full URL", + args: args{ + registryContent: ` + mirrors: + "*": + endpoint: + - https://registry.example.com/v2 + `, + }, + want: HostConfigs{ + "_default": templates.HostConfig{ + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://registry.example.com/v2", + }, + }, + }, + }, + }, + { + name: "wildcard mirror endpoint - full URL, embedded registry", + args: args{ + mirrorAddr: "127.0.0.1:6443", + registryContent: ` + mirrors: + "*": + endpoint: + - https://registry.example.com/v2 + `, + }, + want: HostConfigs{ + "_default": templates.HostConfig{ + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + // note that the embedded registry mirror is NOT listed as an endpoint. + // individual registries must be enabled for mirroring by name. + templates.RegistryEndpoint{ + URI: "https://registry.example.com/v2", + }, + }, + }, + "127.0.0.1:6443": templates.HostConfig{ + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://127.0.0.1:6443/v2", + Config: registries.RegistryConfig{ + TLS: ®istries.TLSConfig{ + CAFile: "server-ca", + KeyFile: "client-key", + CertFile: "client-cert", + }, + }, + }, + }, + }, + }, + }, + { + name: "localhost registry - default https endpoint on unspecified port", + args: args{ + registryContent: ` + mirrors: + "localhost": + `, + }, + want: HostConfigs{}, + }, + { + name: "localhost registry - default https endpoint on https port", + args: args{ + registryContent: ` + mirrors: + "localhost:443": + `, + }, + want: HostConfigs{}, + }, + { + name: "localhost registry - default http endpoint on odd port", + args: args{ + registryContent: ` + mirrors: + "localhost:5000": + `, + }, + want: HostConfigs{}, + }, + { + name: "localhost registry - default http endpoint on http port", + args: args{ + registryContent: ` + mirrors: + "localhost:80": + `, + }, + want: HostConfigs{}, + }, + { + name: "localhost registry - default http endpoint on odd port, embedded registry", + args: args{ + mirrorAddr: "127.0.0.1:6443", + registryContent: ` + mirrors: + "localhost:5000": + `, + }, + want: HostConfigs{ + // localhost registries are not handled by the embedded registry mirror. + "127.0.0.1:6443": templates.HostConfig{ + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://127.0.0.1:6443/v2", + Config: registries.RegistryConfig{ + TLS: ®istries.TLSConfig{ + CAFile: "server-ca", + KeyFile: "client-key", + CertFile: "client-cert", + }, + }, + }, + }, + }, + }, + }, + { + name: "localhost registry - https endpoint on odd port with tls verification disabled", + args: args{ + registryContent: ` + mirrors: + localhost:5000: + endpoint: + - https://localhost:5000 + configs: + localhost:5000: + tls: + insecure_skip_verify: true + `, + }, + want: HostConfigs{ + "localhost:5000": templates.HostConfig{ + DefaultEndpoint: "http://localhost:5000/v2", + Program: "k3s", + Endpoints: []templates.RegistryEndpoint{ + templates.RegistryEndpoint{ + URI: "https://localhost:5000/v2", + Config: registries.RegistryConfig{ + TLS: ®istries.TLSConfig{ + InsecureSkipVerify: true, + }, + }, + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // replace tabs from the inline yaml with spaces; yaml doesn't support tabs for indentation. + tt.args.registryContent = strings.ReplaceAll(tt.args.registryContent, "\t", " ") + registriesFile := filepath.Join(t.TempDir(), "registries.yaml") + os.WriteFile(registriesFile, []byte(tt.args.registryContent), 0644) + + registry, err := registries.GetPrivateRegistries(registriesFile) + if err != nil { + t.Fatalf("failed to parse %s: %v\n", registriesFile, err) + } + + // set up embedded registry, if enabled for the test + if tt.args.mirrorAddr != "" { + conf := spegel.DefaultRegistry + conf.ServerCAFile = "server-ca" + conf.ClientKeyFile = "client-key" + conf.ClientCertFile = "client-cert" + conf.InternalAddress, conf.RegistryPort, _ = net.SplitHostPort(tt.args.mirrorAddr) + conf.InjectMirror(&config.Node{AgentConfig: config.Agent{Registry: registry.Registry}}) + } + + got := getHostConfigs(registry.Registry, tt.args.noDefaultEndpoint, tt.args.mirrorAddr) + assert.Equal(t, tt.want, got, "getHostConfigs()") + }) + } +} diff --git a/pkg/agent/templates/templates.go b/pkg/agent/templates/templates.go index d920693d35..de273f26c1 100644 --- a/pkg/agent/templates/templates.go +++ b/pkg/agent/templates/templates.go @@ -34,15 +34,15 @@ type RegistryEndpoint struct { } type HostConfig struct { - Host string - Program string - Endpoints []RegistryEndpoint + DefaultEndpoint string + Program string + Endpoints []RegistryEndpoint } const HostsTomlTemplate = ` {{- /* */ -}} # File generated by {{ .Program }}. DO NOT EDIT. -{{ if .Host }}server = "https://{{ .Host }}"{{ end }} +{{ if .DefaultEndpoint }}server = "{{ .DefaultEndpoint }}"{{ end }} {{ range $e := .Endpoints -}} [host."{{ $e.URI }}"] diff --git a/pkg/spegel/registry.go b/pkg/spegel/registry.go index df378750db..403b96f9fb 100644 --- a/pkg/spegel/registry.go +++ b/pkg/spegel/registry.go @@ -3,6 +3,7 @@ package spegel import ( "net" + "github.com/containerd/containerd/remotes/docker" "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/rancher/wharfie/pkg/registries" ) @@ -11,6 +12,7 @@ import ( // to all configured registries. func (c *Config) InjectMirror(nodeConfig *config.Node) error { mirrorAddr := net.JoinHostPort(c.InternalAddress, c.RegistryPort) + mirrorURL := "https://" + mirrorAddr + "/v2" registry := nodeConfig.AgentConfig.Registry if registry.Configs == nil { @@ -28,11 +30,15 @@ func (c *Config) InjectMirror(nodeConfig *config.Node) error { registry.Mirrors = map[string]registries.Mirror{} } for host, mirror := range registry.Mirrors { - if host != "*" { - mirror.Endpoints = append([]string{"https://" + mirrorAddr}, mirror.Endpoints...) + // Don't handle wildcard or local registry entries + if host != "*" && !docker.IsLocalhost(host) { + mirror.Endpoints = append([]string{mirrorURL}, mirror.Endpoints...) registry.Mirrors[host] = mirror } } + registry.Mirrors[mirrorAddr] = registries.Mirror{ + Endpoints: []string{mirrorURL}, + } return nil }