github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Fix issues with certs.d template generation

Browse Source 
* Fix issue with bare host or IP as endpoint
* Fix issue with localhost registries not defaulting to http.
* Move the registry template prep to a separate function,
  and adds tests of that function so that we can ensure we're
  generating the correct content.

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>
pull/9332/head
Brad Davidson 10 months ago committed by Brad Davidson
parent 6d77b7a920
commit 29848dea3d
4 changed files with 1245 additions and 42 deletions
Show Stats Download Patch File Download Diff File

159
pkg/agent/containerd/config.go
Unescape View File

@ -13,9 +13,12 @@ import (
"github.com/k3s-io/k3s/pkg/daemons/config"
"github.com/k3s-io/k3s/pkg/spegel"
"github.com/k3s-io/k3s/pkg/version"
"github.com/rancher/wharfie/pkg/registries"
"github.com/sirupsen/logrus"
)
type HostConfigs map[string]templates.HostConfig
// writeContainerdConfig renders and saves config.toml from the filled template
func writeContainerdConfig(cfg *config.Node, containerdConfig templates.ContainerdConfig) error {
var containerdTemplate string
@ -39,79 +42,163 @@ func writeContainerdConfig(cfg *config.Node, containerdConfig templates.Containe
// writeContainerdHosts merges registry mirrors/configs, and renders and saves hosts.toml from the filled template
func writeContainerdHosts(cfg *config.Node, containerdConfig templates.ContainerdConfig) error {
mirrorAddr := net.JoinHostPort(spegel.DefaultRegistry.InternalAddress, spegel.DefaultRegistry.RegistryPort)
registry := containerdConfig.PrivateRegistryConfig
hosts := getHostConfigs(containerdConfig.PrivateRegistryConfig, containerdConfig.NoDefaultEndpoint, mirrorAddr)
// Clean up previous configuration templates
os.RemoveAll(cfg.Containerd.Registry)
// Write out new templates
for host, config := range hosts {
hostDir := filepath.Join(cfg.Containerd.Registry, host)
hostsFile := filepath.Join(hostDir, "hosts.toml")
hostsTemplate, err := templates.ParseHostsTemplateFromConfig(templates.HostsTomlTemplate, config)
if err != nil {
return err
}
if err := os.MkdirAll(hostDir, 0700); err != nil {
return err
}
if err := util2.WriteFile(hostsFile, hostsTemplate); err != nil {
return err
}
}
return nil
}
// getHostConfigs merges the registry mirrors/configs into HostConfig template structs
func getHostConfigs(registry *registries.Registry, noDefaultEndpoint bool, mirrorAddr string) HostConfigs {
hosts := map[string]templates.HostConfig{}
// create endpoints for mirrors
for host, mirror := range registry.Mirrors {
defaultHost, _ := docker.DefaultHost(host)
config := templates.HostConfig{
Host: defaultHost,
Program: version.Program,
}
if host == "*" {
host = "_default"
config.Host = ""
} else if containerdConfig.NoDefaultEndpoint {
config.Host = ""
if uri, _, err := normalizeEndpointAddress(host, mirrorAddr); err == nil {
config.DefaultEndpoint = uri.String()
}
// TODO: rewrites are currently copied from the mirror settings into each endpoint.
// In the future, we should allow for per-endpoint rewrites, instead of expecting
// all mirrors to have the same structure. This will require changes to the registries.yaml
// structure, which is defined in rancher/wharfie.
for _, endpoint := range mirror.Endpoints {
if endpointURL, err := url.Parse(endpoint); err == nil {
re := templates.RegistryEndpoint{
OverridePath: endpointURL.Path != "" && endpointURL.Path != "/" && !strings.HasSuffix(endpointURL.Path, "/v2"),
Config: registry.Configs[endpointURL.Host],
Rewrites: mirror.Rewrites,
URI: endpoint,
}
uri, override, err := normalizeEndpointAddress(endpoint, mirrorAddr)
if err != nil {
logrus.Warnf("Ignoring invalid endpoint URL %s for %s: %v", endpoint, host, err)
} else {
var rewrites map[string]string
// Do not apply rewrites to the embedded registry endpoint
if endpointURL.Host == mirrorAddr {
re.Rewrites = nil
if uri.Host != mirrorAddr {
rewrites = mirror.Rewrites
}
config.Endpoints = append(config.Endpoints, re)
config.Endpoints = append(config.Endpoints, templates.RegistryEndpoint{
Config: registry.Configs[uri.Host],
Rewrites: rewrites,
OverridePath: override,
URI: uri.String(),
})
}
}
if host == "*" {
host = "_default"
}
hosts[host] = config
}
// create endpoints for registries using default endpoints
for host, registry := range registry.Configs {
config, ok := hosts[host]
if !ok {
config = templates.HostConfig{
Program: version.Program,
}
if uri, _, err := normalizeEndpointAddress(host, mirrorAddr); err == nil {
config.DefaultEndpoint = uri.String()
}
}
// If there is config for this host but no endpoints, inject the config for the default endpoint.
if len(config.Endpoints) == 0 {
config.Endpoints = []templates.RegistryEndpoint{
{
uri, _, err := normalizeEndpointAddress(host, mirrorAddr)
if err != nil {
logrus.Warnf("Ignoring invalid endpoint URL %s for %s: %v", host, host, err)
} else {
config.Endpoints = append(config.Endpoints, templates.RegistryEndpoint{
Config: registry,
URI: "https://" + host,
},
URI: uri.String(),
})
}
}
if host == "*" {
host = "_default"
}
hosts[host] = config
}
// Clean up previous configuration templates
os.RemoveAll(cfg.Containerd.Registry)
// Write out new templates
// Clean up hosts and default endpoints where resulting config leaves only defaults
for host, config := range hosts {
hostDir := filepath.Join(cfg.Containerd.Registry, host)
hostsFile := filepath.Join(hostDir, "hosts.toml")
hostsTemplate, err := templates.ParseHostsTemplateFromConfig(templates.HostsTomlTemplate, config)
if err != nil {
return err
// if default endpoint is disabled, or this is the wildcard host, delete the default endpoint
if noDefaultEndpoint || host == "_default" {
config.DefaultEndpoint = ""
hosts[host] = config
}
if err := os.MkdirAll(hostDir, 0700); err != nil {
return err
if l := len(config.Endpoints); l > 0 {
if ep := config.Endpoints[l-1]; ep.URI == config.DefaultEndpoint {
// if the last endpoint is the default endpoint
if ep.Config.Auth == nil && ep.Config.TLS == nil && len(ep.Rewrites) == 0 {
// if has no config, delete this host to use the default config
delete(hosts, host)
} else {
// if it has config, delete the default endpoint
config.DefaultEndpoint = ""
hosts[host] = config
}
}
} else {
// if this host has no endpoints, delete this host to use the default config
delete(hosts, host)
}
if err := util2.WriteFile(hostsFile, hostsTemplate); err != nil {
return err
}
return hosts
}
// normalizeEndpointAddress normalizes the endpoint address.
// If successful, it returns the URL, and a bool indicating if the endpoint path should be overridden.
// If unsuccessful, an error is returned.
// Scheme and hostname logic should match containerd:
// https://github.com/containerd/containerd/blob/v1.7.13/remotes/docker/config/hosts.go#L99-L131
func normalizeEndpointAddress(endpoint, mirrorAddr string) (*url.URL, bool, error) {
// Ensure that the endpoint address has a scheme so that the URL is parsed properly
if !strings.Contains(endpoint, "://") {
endpoint = "//" + endpoint
}
endpointURL, err := url.Parse(endpoint)
if err != nil {
return nil, false, err
}
port := endpointURL.Port()
// set default scheme, if not provided
if endpointURL.Scheme == "" {
// localhost on odd ports defaults to http, unless it's the embedded mirror
if docker.IsLocalhost(endpointURL.Host) && port != "" && port != "443" && endpointURL.Host != mirrorAddr {
endpointURL.Scheme = "http"
} else {
endpointURL.Scheme = "https"
}
}
endpointURL.Host, _ = docker.DefaultHost(endpointURL.Host)
return nil
switch endpointURL.Path {
case "", "/", "/v2":
// If the path is empty, /, or /v2, use the default path.
endpointURL.Path = "/v2"
return endpointURL, false, nil
}
return endpointURL, true, nil
}

1110
pkg/agent/containerd/config_test.go
View File

File diff suppressed because it is too large Load Diff

8
pkg/agent/templates/templates.go
Unescape View File

@ -34,15 +34,15 @@ type RegistryEndpoint struct {
}
type HostConfig struct {
Host string
Program string
Endpoints []RegistryEndpoint
DefaultEndpoint string
Program string
Endpoints []RegistryEndpoint
}
const HostsTomlTemplate = `
{{- /* */ -}}
# File generated by {{ .Program }}. DO NOT EDIT.
{{ if .Host }}server = "https://{{ .Host }}"{{ end }}
{{ if .DefaultEndpoint }}server = "{{ .DefaultEndpoint }}"{{ end }}
{{ range $e := .Endpoints -}}
[host."{{ $e.URI }}"]

10
pkg/spegel/registry.go
Unescape View File

@ -3,6 +3,7 @@ package spegel
import (
"net"
"github.com/containerd/containerd/remotes/docker"
"github.com/k3s-io/k3s/pkg/daemons/config"
"github.com/rancher/wharfie/pkg/registries"
)
@ -11,6 +12,7 @@ import (
// to all configured registries.
func (c *Config) InjectMirror(nodeConfig *config.Node) error {
mirrorAddr := net.JoinHostPort(c.InternalAddress, c.RegistryPort)
mirrorURL := "https://" + mirrorAddr + "/v2"
registry := nodeConfig.AgentConfig.Registry
if registry.Configs == nil {
@ -28,11 +30,15 @@ func (c *Config) InjectMirror(nodeConfig *config.Node) error {
registry.Mirrors = map[string]registries.Mirror{}
}
for host, mirror := range registry.Mirrors {
if host != "*" {
mirror.Endpoints = append([]string{"https://" + mirrorAddr}, mirror.Endpoints...)
// Don't handle wildcard or local registry entries
if host != "*" && !docker.IsLocalhost(host) {
mirror.Endpoints = append([]string{mirrorURL}, mirror.Endpoints...)
registry.Mirrors[host] = mirror
}
}
registry.Mirrors[mirrorAddr] = registries.Mirror{
Endpoints: []string{mirrorURL},
}
return nil
}

Write Preview
Loading…
Cancel
Save