github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Fix additional corner cases in registries handling 

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>
pull/9649/head
Brad Davidson 2024-02-02 02:51:13 +00:00 committed by Brad Davidson
parent 29c73e6965
commit b164d7a270
3 changed files with 564 additions and 267 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

167
pkg/agent/containerd/config.go
View File

@ -1,6 +1,7 @@
package containerd
import (
"fmt"
"net"
"net/url"
"os"
@ -70,65 +71,60 @@ func writeContainerdHosts(cfg *config.Node, containerdConfig templates.Container
func getHostConfigs(registry *registries.Registry, noDefaultEndpoint bool, mirrorAddr string) HostConfigs {
hosts := map[string]templates.HostConfig{}
// create config for default endpoints
for host, config := range registry.Configs {
if c, err := defaultHostConfig(host, mirrorAddr, config); err != nil {
logrus.Errorf("Failed to generate config for registry %s: %v", host, err)
} else {
if host == "*" {
host = "_default"
}
hosts[host] = *c
}
}
// create endpoints for mirrors
for host, mirror := range registry.Mirrors {
config := templates.HostConfig{
Program: version.Program,
}
if uri, _, err := normalizeEndpointAddress(host, mirrorAddr); err == nil {
config.DefaultEndpoint = uri.String()
// create the default config, if it wasn't explicitly mentioned in the config section
config, ok := hosts[host]
if !ok {
if c, err := defaultHostConfig(host, mirrorAddr, configForHost(registry.Configs, host)); err != nil {
logrus.Errorf("Failed to generate config for registry %s: %v", host, err)
continue
} else {
if host == "*" || noDefaultEndpoint {
c.Default = nil
}
config = *c
}
}
// TODO: rewrites are currently copied from the mirror settings into each endpoint.
// In the future, we should allow for per-endpoint rewrites, instead of expecting
// all mirrors to have the same structure. This will require changes to the registries.yaml
// structure, which is defined in rancher/wharfie.
for _, endpoint := range mirror.Endpoints {
uri, override, err := normalizeEndpointAddress(endpoint, mirrorAddr)
for i, endpoint := range mirror.Endpoints {
registryName, url, override, err := normalizeEndpointAddress(endpoint, mirrorAddr)
if err != nil {
logrus.Warnf("Ignoring invalid endpoint URL %s for %s: %v", endpoint, host, err)
logrus.Warnf("Ignoring invalid endpoint URL %d=%s for %s: %v", i, endpoint, host, err)
} else {
var rewrites map[string]string
// Do not apply rewrites to the embedded registry endpoint
if uri.Host != mirrorAddr {
if url.Host != mirrorAddr {
rewrites = mirror.Rewrites
}
config.Endpoints = append(config.Endpoints, templates.RegistryEndpoint{
Config: registry.Configs[uri.Host],
ep := templates.RegistryEndpoint{
Config: configForHost(registry.Configs, registryName),
Rewrites: rewrites,
OverridePath: override,
URI: uri.String(),
})
}
}
if host == "*" {
host = "_default"
}
hosts[host] = config
}
// create endpoints for registries using default endpoints
for host, registry := range registry.Configs {
config, ok := hosts[host]
if !ok {
config = templates.HostConfig{
Program: version.Program,
}
if uri, _, err := normalizeEndpointAddress(host, mirrorAddr); err == nil {
config.DefaultEndpoint = uri.String()
}
}
// If there is config for this host but no endpoints, inject the config for the default endpoint.
if len(config.Endpoints) == 0 {
uri, _, err := normalizeEndpointAddress(host, mirrorAddr)
if err != nil {
logrus.Warnf("Ignoring invalid endpoint URL %s for %s: %v", host, host, err)
} else {
config.Endpoints = append(config.Endpoints, templates.RegistryEndpoint{
Config: registry,
URI: uri.String(),
})
URL: url,
}
if i+1 == len(mirror.Endpoints) && endpointURLEqual(config.Default, &ep) {
// if the last endpoint is the default endpoint, move it there
config.Default = &ep
} else {
config.Endpoints = append(config.Endpoints, ep)
}
}
}
@ -140,25 +136,8 @@ func getHostConfigs(registry *registries.Registry, noDefaultEndpoint bool, mirro
// Clean up hosts and default endpoints where resulting config leaves only defaults
for host, config := range hosts {
// if default endpoint is disabled, or this is the wildcard host, delete the default endpoint
if noDefaultEndpoint || host == "_default" {
config.DefaultEndpoint = ""
hosts[host] = config
}
if l := len(config.Endpoints); l > 0 {
if ep := config.Endpoints[l-1]; ep.URI == config.DefaultEndpoint {
// if the last endpoint is the default endpoint
if ep.Config.Auth == nil && ep.Config.TLS == nil && len(ep.Rewrites) == 0 {
// if has no config, delete this host to use the default config
delete(hosts, host)
} else {
// if it has config, delete the default endpoint
config.DefaultEndpoint = ""
hosts[host] = config
}
}
} else {
// if this host has no endpoints, delete this host to use the default config
// if this host has no endpoints and the default has no config, delete this host
if len(config.Endpoints) == 0 && !endpointHasConfig(config.Default) {
delete(hosts, host)
}
}
@ -167,18 +146,18 @@ func getHostConfigs(registry *registries.Registry, noDefaultEndpoint bool, mirro
}
// normalizeEndpointAddress normalizes the endpoint address.
// If successful, it returns the URL, and a bool indicating if the endpoint path should be overridden.
// If successful, it returns the registry name, URL, and a bool indicating if the endpoint path should be overridden.
// If unsuccessful, an error is returned.
// Scheme and hostname logic should match containerd:
// https://github.com/containerd/containerd/blob/v1.7.13/remotes/docker/config/hosts.go#L99-L131
func normalizeEndpointAddress(endpoint, mirrorAddr string) (*url.URL, bool, error) {
func normalizeEndpointAddress(endpoint, mirrorAddr string) (string, *url.URL, bool, error) {
// Ensure that the endpoint address has a scheme so that the URL is parsed properly
if !strings.Contains(endpoint, "://") {
endpoint = "//" + endpoint
}
endpointURL, err := url.Parse(endpoint)
if err != nil {
return nil, false, err
return "", nil, false, err
}
port := endpointURL.Port()
@ -191,14 +170,66 @@ func normalizeEndpointAddress(endpoint, mirrorAddr string) (*url.URL, bool, erro
endpointURL.Scheme = "https"
}
}
endpointURL.Host, _ = docker.DefaultHost(endpointURL.Host)
registry := endpointURL.Host
endpointURL.Host, _ = docker.DefaultHost(registry)
// This is the reverse of the DefaultHost normalization
if endpointURL.Host == "registry-1.docker.io" {
registry = "docker.io"
}
switch endpointURL.Path {
case "", "/", "/v2":
// If the path is empty, /, or /v2, use the default path.
endpointURL.Path = "/v2"
return endpointURL, false, nil
return registry, endpointURL, false, nil
}
return endpointURL, true, nil
return registry, endpointURL, true, nil
}
func defaultHostConfig(host, mirrorAddr string, config registries.RegistryConfig) (*templates.HostConfig, error) {
_, url, _, err := normalizeEndpointAddress(host, mirrorAddr)
if err != nil {
return nil, fmt.Errorf("invalid endpoint URL %s for %s: %v", host, host, err)
}
if host == "*" {
url = nil
}
return &templates.HostConfig{
Program: version.Program,
Default: &templates.RegistryEndpoint{
URL: url,
Config: config,
},
}, nil
}
func configForHost(configs map[string]registries.RegistryConfig, host string) registries.RegistryConfig {
// check for config under modified hostname. If the hostname is unmodified, or there is no config for
// the modified hostname, return the config for the default hostname.
if h, _ := docker.DefaultHost(host); h != host {
if c, ok := configs[h]; ok {
return c
}
}
return configs[host]
}
// endpointURLEqual compares endpoint URL strings
func endpointURLEqual(a, b *templates.RegistryEndpoint) bool {
var au, bu string
if a != nil && a.URL != nil {
au = a.URL.String()
}
if b != nil && b.URL != nil {
bu = b.URL.String()
}
return au == bu
}
func endpointHasConfig(ep *templates.RegistryEndpoint) bool {
if ep != nil {
return ep.OverridePath || ep.Config.Auth != nil || ep.Config.TLS != nil || len(ep.Rewrites) > 0
}
return false
}

633
pkg/agent/containerd/config_test.go
View File

@ -2,6 +2,7 @@ package containerd
import (
"net"
"net/url"
"os"
"path/filepath"
"strings"
@ -11,9 +12,22 @@ import (
"github.com/k3s-io/k3s/pkg/daemons/config"
"github.com/k3s-io/k3s/pkg/spegel"
"github.com/rancher/wharfie/pkg/registries"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
)
func init() {
logrus.SetLevel(logrus.DebugLevel)
}
func u(s string) *url.URL {
u, err := url.Parse(s)
if err != nil {
panic(err)
}
return u
}
func Test_UnitGetHostConfigs(t *testing.T) {
type args struct {
registryContent string
@ -39,6 +53,18 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
want: HostConfigs{},
},
{
name: "registry with default endpoint explicitly listed",
args: args{
registryContent: `
mirrors:
docker.io:
endpoint:
- docker.io
`,
},
want: HostConfigs{},
},
{
name: "registry with default endpoint - embedded registry",
args: args{
@ -50,11 +76,13 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("https://registry-1.docker.io/v2"),
},
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://127.0.0.1:6443/v2",
URL: u("https://127.0.0.1:6443/v2"),
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
@ -67,15 +95,13 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
"127.0.0.1:6443": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://127.0.0.1:6443/v2",
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
KeyFile: "client-key",
CertFile: "client-cert",
},
Default: &templates.RegistryEndpoint{
URL: u("https://127.0.0.1:6443/v2"),
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
KeyFile: "client-key",
CertFile: "client-cert",
},
},
},
@ -98,20 +124,49 @@ func Test_UnitGetHostConfigs(t *testing.T) {
want: HostConfigs{
"docker.io": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://registry-1.docker.io/v2",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
Default: &templates.RegistryEndpoint{
URL: u("https://registry-1.docker.io/v2"),
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
},
},
},
{
name: "registry with default endpoint explicitly listed and creds",
args: args{
registryContent: `
mirrors:
docker.io:
endpoint:
- docker.io
configs:
docker.io:
auth:
username: user
password: pass
`,
},
want: HostConfigs{
"docker.io": templates.HostConfig{
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("https://registry-1.docker.io/v2"),
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
},
},
},
{
name: "registry with only creds",
args: args{
@ -126,14 +181,12 @@ func Test_UnitGetHostConfigs(t *testing.T) {
want: HostConfigs{
"docker.io": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://registry-1.docker.io/v2",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
Default: &templates.RegistryEndpoint{
URL: u("https://registry-1.docker.io/v2"),
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
@ -166,14 +219,12 @@ func Test_UnitGetHostConfigs(t *testing.T) {
want: HostConfigs{
"registry.example.com": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://registry.example.com/v2",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
Default: &templates.RegistryEndpoint{
URL: u("https://registry.example.com/v2"),
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
@ -194,14 +245,12 @@ func Test_UnitGetHostConfigs(t *testing.T) {
want: HostConfigs{
"registry.example.com": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://registry.example.com/v2",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
Default: &templates.RegistryEndpoint{
URL: u("https://registry.example.com/v2"),
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
@ -220,12 +269,14 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("https://registry-1.docker.io/v2"),
},
Endpoints: []templates.RegistryEndpoint{
{
OverridePath: true,
URI: "https://registry.example.com/prefix/v2",
URL: u("https://registry.example.com/prefix/v2"),
},
},
},
@ -243,12 +294,14 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("https://registry-1.docker.io/v2"),
},
Endpoints: []templates.RegistryEndpoint{
{
OverridePath: true,
URI: "https://registry.example.com/prefix/v2",
URL: u("https://registry.example.com/prefix/v2"),
},
},
},
@ -266,11 +319,13 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("https://registry-1.docker.io/v2"),
},
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://registry.example.com/v2",
URL: u("https://registry.example.com/v2"),
},
},
},
@ -288,11 +343,13 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("https://registry-1.docker.io/v2"),
},
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://registry.example.com/v2",
URL: u("https://registry.example.com/v2"),
},
},
},
@ -310,11 +367,13 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("https://registry-1.docker.io/v2"),
},
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://registry.example.com/v2",
URL: u("https://registry.example.com/v2"),
},
},
},
@ -332,11 +391,13 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("https://registry-1.docker.io/v2"),
},
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://registry.example.com/v2",
URL: u("https://registry.example.com/v2"),
},
},
},
@ -354,11 +415,13 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("https://registry-1.docker.io/v2"),
},
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://registry.example.com:443/v2",
URL: u("https://registry.example.com:443/v2"),
},
},
},
@ -376,11 +439,13 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("https://registry-1.docker.io/v2"),
},
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://1.2.3.4/v2",
URL: u("https://1.2.3.4/v2"),
},
},
},
@ -398,11 +463,13 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("https://registry-1.docker.io/v2"),
},
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://1.2.3.4:443/v2",
URL: u("https://1.2.3.4:443/v2"),
},
},
},
@ -420,11 +487,13 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("https://registry-1.docker.io/v2"),
},
Endpoints: []templates.RegistryEndpoint{
{
URI: "http://localhost:5000/v2",
URL: u("http://localhost:5000/v2"),
},
},
},
@ -442,11 +511,13 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("https://registry-1.docker.io/v2"),
},
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://localhost:5000/v2",
URL: u("https://localhost:5000/v2"),
},
},
},
@ -464,11 +535,13 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("https://registry-1.docker.io/v2"),
},
Endpoints: []templates.RegistryEndpoint{
{
URI: "http://127.0.0.1:5000/v2",
URL: u("http://127.0.0.1:5000/v2"),
},
},
},
@ -486,11 +559,13 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("https://registry-1.docker.io/v2"),
},
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://127.0.0.1:5000/v2",
URL: u("https://127.0.0.1:5000/v2"),
},
},
},
@ -513,11 +588,13 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("https://registry-1.docker.io/v2"),
},
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://registry.example.com/v2",
URL: u("https://registry.example.com/v2"),
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
@ -529,14 +606,12 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
"registry.example.com": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://registry.example.com/v2",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
Default: &templates.RegistryEndpoint{
URL: u("https://registry.example.com/v2"),
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
@ -563,12 +638,14 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("https://registry-1.docker.io/v2"),
},
Endpoints: []templates.RegistryEndpoint{
{
OverridePath: true,
URI: "https://registry.example.com/prefix/v2",
URL: u("https://registry.example.com/prefix/v2"),
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
@ -579,12 +656,20 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
},
"registry.example.com": templates.HostConfig{
DefaultEndpoint: "https://registry.example.com/v2",
Program: "k3s",
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("https://registry.example.com/v2"),
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
Endpoints: []templates.RegistryEndpoint{
{
OverridePath: true,
URI: "https://registry.example.com/prefix/v2",
URL: u("https://registry.example.com/prefix/v2"),
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
@ -616,12 +701,14 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("https://registry-1.docker.io/v2"),
},
Endpoints: []templates.RegistryEndpoint{
{
OverridePath: true,
URI: "https://registry.example.com/project/registry",
URL: u("https://registry.example.com/project/registry"),
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
@ -632,12 +719,20 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
},
"registry.example.com": templates.HostConfig{
DefaultEndpoint: "https://registry.example.com/v2",
Program: "k3s",
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("https://registry.example.com/v2"),
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
Endpoints: []templates.RegistryEndpoint{
{
OverridePath: true,
URI: "https://registry.example.com/project/registry",
URL: u("https://registry.example.com/project/registry"),
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
@ -670,7 +765,7 @@ func Test_UnitGetHostConfigs(t *testing.T) {
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://registry.example.com/v2",
URL: u("https://registry.example.com/v2"),
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
@ -682,14 +777,12 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
"registry.example.com": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://registry.example.com/v2",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
Default: &templates.RegistryEndpoint{
URL: u("https://registry.example.com/v2"),
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
@ -714,11 +807,13 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("https://registry-1.docker.io/v2"),
},
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://127.0.0.1:6443/v2",
URL: u("https://127.0.0.1:6443/v2"),
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
@ -728,7 +823,7 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
},
{
URI: "https://registry.example.com/v2",
URL: u("https://registry.example.com/v2"),
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
@ -740,29 +835,25 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
"registry.example.com": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://registry.example.com/v2",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
Default: &templates.RegistryEndpoint{
URL: u("https://registry.example.com/v2"),
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
},
"127.0.0.1:6443": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://127.0.0.1:6443/v2",
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
KeyFile: "client-key",
CertFile: "client-cert",
},
Default: &templates.RegistryEndpoint{
URL: u("https://127.0.0.1:6443/v2"),
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
KeyFile: "client-key",
CertFile: "client-cert",
},
},
},
@ -789,11 +880,13 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
want: HostConfigs{
"docker.io": templates.HostConfig{
DefaultEndpoint: "https://registry-1.docker.io/v2",
Program: "k3s",
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("https://registry-1.docker.io/v2"),
},
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://127.0.0.1:6443/v2",
URL: u("https://127.0.0.1:6443/v2"),
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
@ -803,7 +896,7 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
},
{
URI: "https://registry.example.com/v2",
URL: u("https://registry.example.com/v2"),
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
@ -818,29 +911,25 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
"registry.example.com": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://registry.example.com/v2",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
Default: &templates.RegistryEndpoint{
URL: u("https://registry.example.com/v2"),
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
},
"127.0.0.1:6443": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://127.0.0.1:6443/v2",
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
KeyFile: "client-key",
CertFile: "client-cert",
},
Default: &templates.RegistryEndpoint{
URL: u("https://127.0.0.1:6443/v2"),
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
KeyFile: "client-key",
CertFile: "client-cert",
},
},
},
@ -869,7 +958,7 @@ func Test_UnitGetHostConfigs(t *testing.T) {
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://127.0.0.1:6443/v2",
URL: u("https://127.0.0.1:6443/v2"),
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
@ -879,7 +968,7 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
},
{
URI: "https://registry.example.com/v2",
URL: u("https://registry.example.com/v2"),
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
@ -891,23 +980,53 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
"registry.example.com": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://registry.example.com/v2",
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
Default: &templates.RegistryEndpoint{
URL: u("https://registry.example.com/v2"),
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
},
},
},
"127.0.0.1:6443": templates.HostConfig{
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("https://127.0.0.1:6443/v2"),
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
KeyFile: "client-key",
CertFile: "client-cert",
},
},
},
},
},
},
{
name: "registry with mirror endpoint - embedded registry, default endpoint explicitly listed",
args: args{
mirrorAddr: "127.0.0.1:6443",
registryContent: `
mirrors:
docker.io:
endpoint:
- registry.example.com
- registry.example.org
- docker.io
`,
},
want: HostConfigs{
"docker.io": templates.HostConfig{
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("https://registry-1.docker.io/v2"),
},
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://127.0.0.1:6443/v2",
URL: u("https://127.0.0.1:6443/v2"),
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
@ -916,6 +1035,79 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
},
},
{
URL: u("https://registry.example.com/v2"),
},
{
URL: u("https://registry.example.org/v2"),
},
},
},
"127.0.0.1:6443": templates.HostConfig{
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("https://127.0.0.1:6443/v2"),
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
KeyFile: "client-key",
CertFile: "client-cert",
},
},
},
},
},
},
{
name: "registry with mirror endpoint - embedded registry and no default endpoint, default endpoint explicitly listed",
args: args{
mirrorAddr: "127.0.0.1:6443",
noDefaultEndpoint: true,
registryContent: `
mirrors:
docker.io:
endpoint:
- registry.example.com
- registry.example.org
- docker.io
`,
},
want: HostConfigs{
"docker.io": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
{
URL: u("https://127.0.0.1:6443/v2"),
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
KeyFile: "client-key",
CertFile: "client-cert",
},
},
},
{
URL: u("https://registry.example.com/v2"),
},
{
URL: u("https://registry.example.org/v2"),
},
{
URL: u("https://registry-1.docker.io/v2"),
},
},
},
"127.0.0.1:6443": templates.HostConfig{
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("https://127.0.0.1:6443/v2"),
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
KeyFile: "client-key",
CertFile: "client-cert",
},
},
},
},
},
@ -935,7 +1127,7 @@ func Test_UnitGetHostConfigs(t *testing.T) {
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://registry.example.com/v2",
URL: u("https://registry.example.com/v2"),
},
},
},
@ -959,21 +1151,49 @@ func Test_UnitGetHostConfigs(t *testing.T) {
// note that the embedded registry mirror is NOT listed as an endpoint.
// individual registries must be enabled for mirroring by name.
{
URI: "https://registry.example.com/v2",
URL: u("https://registry.example.com/v2"),
},
},
},
"127.0.0.1:6443": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://127.0.0.1:6443/v2",
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
KeyFile: "client-key",
CertFile: "client-cert",
},
Default: &templates.RegistryEndpoint{
URL: u("https://127.0.0.1:6443/v2"),
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
KeyFile: "client-key",
CertFile: "client-cert",
},
},
},
},
},
},
{
name: "wildcard config",
args: args{
registryContent: `
configs:
"*":
auth:
username: user
password: pass
tls:
insecure_skip_verify: true
`,
},
want: HostConfigs{
"_default": {
Program: "k3s",
Default: &templates.RegistryEndpoint{
Config: registries.RegistryConfig{
Auth: &registries.AuthConfig{
Username: "user",
Password: "pass",
},
TLS: &registries.TLSConfig{
InsecureSkipVerify: true,
},
},
},
@ -1033,15 +1253,13 @@ func Test_UnitGetHostConfigs(t *testing.T) {
// localhost registries are not handled by the embedded registry mirror.
"127.0.0.1:6443": templates.HostConfig{
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://127.0.0.1:6443/v2",
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
KeyFile: "client-key",
CertFile: "client-cert",
},
Default: &templates.RegistryEndpoint{
URL: u("https://127.0.0.1:6443/v2"),
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
CAFile: "server-ca",
KeyFile: "client-key",
CertFile: "client-cert",
},
},
},
@ -1064,11 +1282,18 @@ func Test_UnitGetHostConfigs(t *testing.T) {
},
want: HostConfigs{
"localhost:5000": templates.HostConfig{
DefaultEndpoint: "http://localhost:5000/v2",
Program: "k3s",
Default: &templates.RegistryEndpoint{
URL: u("http://localhost:5000/v2"),
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
InsecureSkipVerify: true,
},
},
},
Program: "k3s",
Endpoints: []templates.RegistryEndpoint{
{
URI: "https://localhost:5000/v2",
URL: u("https://localhost:5000/v2"),
Config: registries.RegistryConfig{
TLS: &registries.TLSConfig{
InsecureSkipVerify: true,
@ -1085,8 +1310,10 @@ func Test_UnitGetHostConfigs(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
// replace tabs from the inline yaml with spaces; yaml doesn't support tabs for indentation.
tt.args.registryContent = strings.ReplaceAll(tt.args.registryContent, "\t", " ")
registriesFile := filepath.Join(t.TempDir(), "registries.yaml")
tempDir := t.TempDir()
registriesFile := filepath.Join(tempDir, "registries.yaml")
os.WriteFile(registriesFile, []byte(tt.args.registryContent), 0644)
t.Logf("%s:\n%s", registriesFile, tt.args.registryContent)
registry, err := registries.GetPrivateRegistries(registriesFile)
if err != nil {
@ -1103,8 +1330,30 @@ func Test_UnitGetHostConfigs(t *testing.T) {
conf.InjectMirror(&config.Node{AgentConfig: config.Agent{Registry: registry.Registry}})
}
// Generate config template struct for all hosts
got := getHostConfigs(registry.Registry, tt.args.noDefaultEndpoint, tt.args.mirrorAddr)
assert.Equal(t, tt.want, got, "getHostConfigs()")
// Confirm that hosts.toml renders properly for all registries
for host, config := range got {
hostsTemplate, err := templates.ParseHostsTemplateFromConfig(templates.HostsTomlTemplate, config)
assert.NoError(t, err, "ParseHostTemplateFromConfig for %s", host)
t.Logf("%s/hosts.d/%s/hosts.toml\n%s", tempDir, host, hostsTemplate)
}
// Confirm that the main containerd config.toml renders properly
containerdConfig := templates.ContainerdConfig{
NodeConfig: &config.Node{
Containerd: config.Containerd{
Registry: tempDir + "/hosts.d",
},
},
PrivateRegistryConfig: registry.Registry,
Program: "k3s",
}
configTemplate, err := templates.ParseTemplateFromConfig(templates.ContainerdConfigTemplate, containerdConfig)
assert.NoError(t, err, "ParseTemplateFromConfig")
t.Logf("%s/config.toml\n%s", tempDir, configTemplate)
})
}
}

31
pkg/agent/templates/templates.go
View File

@ -2,6 +2,7 @@ package templates
import (
"bytes"
"net/url"
"text/template"
"github.com/rancher/wharfie/pkg/registries"
@ -28,24 +29,40 @@ type ContainerdConfig struct {
type RegistryEndpoint struct {
OverridePath bool
URI string
URL *url.URL
Rewrites map[string]string
Config registries.RegistryConfig
}
type HostConfig struct {
DefaultEndpoint string
Program string
Endpoints []RegistryEndpoint
Default *RegistryEndpoint
Program string
Endpoints []RegistryEndpoint
}
const HostsTomlTemplate = `
{{- /* */ -}}
# File generated by {{ .Program }}. DO NOT EDIT.
{{ if .DefaultEndpoint }}server = "{{ .DefaultEndpoint }}"{{ end }}
{{ with $e := .Default }}
{{- if $e.URL }}
server = "{{ $e.URL }}"
capabilities = ["pull", "resolve", "push"]
{{ end }}
{{- if $e.Config.TLS }}
{{- if $e.Config.TLS.CAFile }}
ca = [{{ printf "%q" $e.Config.TLS.CAFile }}]
{{- end }}
{{- if or $e.Config.TLS.CertFile $e.Config.TLS.KeyFile }}
client = [[{{ printf "%q" $e.Config.TLS.CertFile }}, {{ printf "%q" $e.Config.TLS.KeyFile }}]]
{{- end }}
{{- if $e.Config.TLS.InsecureSkipVerify }}
skip_verify = true
{{- end }}
{{ end }}
{{ end }}
{{ range $e := .Endpoints -}}
[host."{{ $e.URI }}"]
[host."{{ $e.URL }}"]
capabilities = ["pull", "resolve"]
{{- if $e.OverridePath }}
override_path = true
@ -62,7 +79,7 @@ const HostsTomlTemplate = `
{{- end }}
{{ end }}
{{- if $e.Rewrites }}
[host."{{ $e.URI }}".rewrite]
[host."{{ $e.URL }}".rewrite]
{{- range $pattern, $replace := $e.Rewrites }}
"{{ $pattern }}" = "{{ $replace }}"
{{- end }}