Merge pull request #867 from galal-hussein/private_reg

Add private registry support to containerd

go.mod

@@ -112,6 +112,7 @@ require (
 	golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3
 	google.golang.org/grpc v1.23.0
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
+	gopkg.in/yaml.v2 v2.2.2
 	k8s.io/api v0.0.0
 	k8s.io/apimachinery v0.0.0
 	k8s.io/apiserver v0.0.0

pkg/agent/config/config.go

@@ -397,6 +397,7 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 
 	nodeConfig.AgentConfig.NodeTaints = envInfo.Taints
 	nodeConfig.AgentConfig.NodeLabels = envInfo.Labels
+	nodeConfig.AgentConfig.PrivateRegistry = envInfo.PrivateRegistry
 
 	return nodeConfig, nil
 }

pkg/agent/containerd/containerd.go

@@ -20,6 +20,7 @@ import (
 	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
+	yaml "gopkg.in/yaml.v2"
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
 	"k8s.io/kubernetes/pkg/kubelet/util"
 )
@@ -158,10 +159,15 @@ func preloadImages(cfg *config.Node) error {
 }
 
 func setupContainerdConfig(ctx context.Context, cfg *config.Node) error {
+	privRegistries, err := getPrivateRegistries(ctx, cfg)
+	if err != nil {
+		return err
+	}
 	var containerdTemplate string
 	containerdConfig := templates.ContainerdConfig{
 		NodeConfig:            cfg,
 		IsRunningInUserNS:     system.RunningInUserNS(),
+		PrivateRegistryConfig: privRegistries,
 	}
 
 	containerdTemplateBytes, err := ioutil.ReadFile(cfg.Containerd.Template)
@@ -180,3 +186,19 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error {
 
 	return util2.WriteFile(cfg.Containerd.Config, parsedTemplate)
 }
+
+func getPrivateRegistries(ctx context.Context, cfg *config.Node) (*templates.Registry, error) {
+	privRegistries := &templates.Registry{}
+	privRegistryFile, err := ioutil.ReadFile(cfg.AgentConfig.PrivateRegistry)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	logrus.Infof("Using registry config file at %s", cfg.AgentConfig.PrivateRegistry)
+	if err := yaml.Unmarshal([]byte(privRegistryFile), &privRegistries); err != nil {
+		return nil, err
+	}
+	return privRegistries, nil
+}

pkg/agent/templates/registry.go

@@ -0,0 +1,54 @@
+package templates
+
+// Mirror contains the config related to the registry mirror
+type Mirror struct {
+	// Endpoints are endpoints for a namespace. CRI plugin will try the endpoints
+	// one by one until a working one is found. The endpoint must be a valid url
+	// with host specified.
+	// The scheme, host and path from the endpoint URL will be used.
+	Endpoints []string `toml:"endpoint" yaml:"endpoint"`
+}
+
+// AuthConfig contains the config related to authentication to a specific registry
+type AuthConfig struct {
+	// Username is the username to login the registry.
+	Username string `toml:"username" yaml:"username"`
+	// Password is the password to login the registry.
+	Password string `toml:"password" yaml:"password"`
+	// Auth is a base64 encoded string from the concatenation of the username,
+	// a colon, and the password.
+	Auth string `toml:"auth" yaml:"auth"`
+	// IdentityToken is used to authenticate the user and get
+	// an access token for the registry.
+	IdentityToken string `toml:"identitytoken" yaml:"identity_token"`
+}
+
+// TLSConfig contains the CA/Cert/Key used for a registry
+type TLSConfig struct {
+	CAFile   string `toml:"ca_file" yaml:"ca_file"`
+	CertFile string `toml:"cert_file" yaml:"cert_file"`
+	KeyFile  string `toml:"key_file" yaml:"key_file"`
+}
+
+// Registry is registry settings configured
+type Registry struct {
+	// Mirrors are namespace to mirror mapping for all namespaces.
+	Mirrors map[string]Mirror `toml:"mirrors" yaml:"mirrors"`
+	// Configs are configs for each registry.
+	// The key is the FDQN or IP of the registry.
+	Configs map[string]RegistryConfig `toml:"configs" yaml:"configs"`
+
+	// Auths are registry endpoint to auth config mapping. The registry endpoint must
+	// be a valid url with host specified.
+	// DEPRECATED: Use Configs instead. Remove in containerd 1.4.
+	Auths map[string]AuthConfig `toml:"auths" yaml:"auths"`
+}
+
+// RegistryConfig contains configuration used to communicate with the registry.
+type RegistryConfig struct {
+	// Auth contains information to authenticate to the registry.
+	Auth *AuthConfig `toml:"auth" yaml:"auth"`
+	// TLS is a pair of CA/Cert/Key which then are used when creating the transport
+	// that communicates with the registry.
+	TLS *TLSConfig `toml:"tls" yaml:"tls"`
+}

pkg/agent/templates/templates.go

@@ -10,6 +10,7 @@ import (
 type ContainerdConfig struct {
 	NodeConfig            *config.Node
 	IsRunningInUserNS     bool
+	PrivateRegistryConfig *Registry
 }
 
 const ContainerdConfigTemplate = `
@@ -35,6 +36,31 @@ sandbox_image = "{{ .NodeConfig.AgentConfig.PauseImage }}"
     bin_dir = "{{ .NodeConfig.AgentConfig.CNIBinDir }}"
     conf_dir = "{{ .NodeConfig.AgentConfig.CNIConfDir }}"
 {{ end -}}
+
+{{ if .PrivateRegistryConfig }}
+{{ if .PrivateRegistryConfig.Mirrors }}
+[plugins.cri.registry.mirrors]{{end}}
+{{range $k, $v := .PrivateRegistryConfig.Mirrors }}
+[plugins.cri.registry.mirrors."{{$k}}"]
+	endpoint = [{{range $i, $j := $v.Endpoints}}{{if $i}}, {{end}}{{printf "%q" .}}{{end}}]
+{{end}}
+
+{{range $k, $v := .PrivateRegistryConfig.Configs }}
+{{ if $v.Auth }}
+[plugins.cri.registry.configs."{{$k}}".auth]
+	{{ if $v.Auth.Username }}username = "{{ $v.Auth.Username }}"{{end}}
+	{{ if $v.Auth.Password }}password = "{{ $v.Auth.Password }}"{{end}}
+	{{ if $v.Auth.Auth }}auth = "{{ $v.Auth.Auth }}"{{end}}
+	{{ if $v.Auth.IdentityToken }}identity_token = "{{ $v.Auth.IdentityToken }}"{{end}}
+{{end}}
+{{ if $v.TLS }}
+[plugins.cri.registry.configs."{{$k}}".tls]
+	{{ if $v.TLS.CAFile }}ca_file = "{{ $v.TLS.CAFile }}"{{end}}
+	{{ if $v.TLS.CertFile }}cert_file = "{{ $v.TLS.CertFile }}"{{end}}
+	{{ if $v.TLS.KeyFile }}key_file = "{{ $v.TLS.KeyFile }}"{{end}}
+{{end}}
+{{end}}
+{{end}}
 `
 
 func ParseTemplateFromConfig(templateBuffer string, config interface{}) (string, error) {

pkg/cli/cmds/agent.go

@@ -30,6 +30,7 @@ type Agent struct {
 	ExtraKubeProxyArgs cli.StringSlice
 	Labels             cli.StringSlice
 	Taints             cli.StringSlice
+	PrivateRegistry    string
 }
 
 type AgentShared struct {
@@ -106,6 +107,12 @@ var (
 		Usage: "(agent) Registering kubelet with set of labels",
 		Value: &AgentConfig.Labels,
 	}
+	PrivateRegistryFlag = cli.StringFlag{
+		Name:        "private-registry",
+		Usage:       "(agent) Private registry configuration file",
+		Destination: &AgentConfig.PrivateRegistry,
+		Value:       "/etc/rancher/k3s/registries.yaml",
+	}
 )
 
 func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
@@ -167,6 +174,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
 			ExtraKubeProxyArgs,
 			NodeLabels,
 			NodeTaints,
+			PrivateRegistryFlag,
 		},
 	}
 }

pkg/cli/cmds/server.go

@@ -218,6 +218,7 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
 			ExtraKubeProxyArgs,
 			NodeLabels,
 			NodeTaints,
+			PrivateRegistryFlag,
 		},
 	}
 }

pkg/daemons/config/types.go

@@ -76,6 +76,7 @@ type Agent struct {
 	NodeLabels          []string
 	IPSECPSK            string
 	StrongSwanDir       string
+	PrivateRegistry     string
 }
 
 type Control struct {