Ensure cluster-signing CA files contain only a single CA cert

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>
(cherry picked from commit 0919ec6755)

pkg/daemons/config/types.go

@@ -286,6 +286,10 @@ type ControlRuntime struct {
 	ClientKubeAPIKey  string
 	NodePasswdFile    string
 
+	SigningClientCA   string
+	SigningServerCA   string
+	ServiceCurrentKey string
+
 	KubeConfigAdmin           string
 	KubeConfigController      string
 	KubeConfigScheduler       string

pkg/daemons/control/deps/deps.go

@@ -10,6 +10,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io/fs"
 	"net"
 	"os"
 	"path/filepath"
@@ -30,6 +31,7 @@ import (
 	"k8s.io/apiserver/pkg/apis/apiserver"
 	apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
 	"k8s.io/apiserver/pkg/authentication/user"
+	"k8s.io/client-go/util/keyutil"
 )
 
 const (
@@ -110,6 +112,10 @@ func CreateRuntimeCertFiles(config *config.Control) {
 	runtime.PasswdFile = filepath.Join(config.DataDir, "cred", "passwd")
 	runtime.NodePasswdFile = filepath.Join(config.DataDir, "cred", "node-passwd")
 
+	runtime.SigningClientCA = filepath.Join(config.DataDir, "tls", "client-ca.nochain.crt")
+	runtime.SigningServerCA = filepath.Join(config.DataDir, "tls", "server-ca.nochain.crt")
+	runtime.ServiceCurrentKey = filepath.Join(config.DataDir, "tls", "service.current.key")
+
 	runtime.KubeConfigAdmin = filepath.Join(config.DataDir, "cred", "admin.kubeconfig")
 	runtime.KubeConfigController = filepath.Join(config.DataDir, "cred", "controller.kubeconfig")
 	runtime.KubeConfigScheduler = filepath.Join(config.DataDir, "cred", "scheduler.kubeconfig")
@@ -315,6 +321,18 @@ func genClientCerts(config *config.Control) error {
 		return err
 	}
 
+	certs, err := certutil.CertsFromFile(runtime.ClientCA)
+	if err != nil {
+		return err
+	}
+
+	// If our CA certs are signed by a root or intermediate CA, ClientCA will contain a chain.
+	// The controller-manager's signer wants just a single cert, not a full chain; so create a file
+	// that is guaranteed to contain only a single certificate.
+	if err := certutil.WriteCert(runtime.SigningClientCA, certutil.EncodeCertPEM(certs[0])); err != nil {
+		return err
+	}
+
 	factory := getSigningCertFactory(regen, nil, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, runtime.ClientCA, runtime.ClientCAKey)
 
 	var certGen bool
@@ -486,7 +504,24 @@ func createServerSigningCertKey(config *config.Control) (bool, error) {
 		}
 		return true, nil
 	}
-	return createSigningCertKey(version.Program+"-server", runtime.ServerCA, runtime.ServerCAKey)
+	regen, err := createSigningCertKey(version.Program+"-server", runtime.ServerCA, runtime.ServerCAKey)
+	if err != nil {
+		return regen, err
+	}
+
+	// If our CA certs are signed by a root or intermediate CA, ServerCA will contain a chain.
+	// The controller-manager's signer wants just a single cert, not a full chain; so create a file
+	// that is guaranteed to contain only a single certificate.
+	certs, err := certutil.CertsFromFile(runtime.ServerCA)
+	if err != nil {
+		return regen, err
+	}
+
+	if err := certutil.WriteCert(runtime.SigningServerCA, certutil.EncodeCertPEM(certs[0])); err != nil {
+		return regen, err
+	}
+
+	return regen, nil
 }
 
 func addSANs(altNames *certutil.AltNames, sans []string) {
@@ -612,17 +647,35 @@ func exists(files ...string) bool {
 }
 
 func genServiceAccount(runtime *config.ControlRuntime) error {
-	_, keyErr := os.Stat(runtime.ServiceKey)
+	if _, err := os.Stat(runtime.ServiceKey); err != nil {
-	if keyErr == nil {
+		if !errors.Is(err, fs.ErrNotExist) {
-		return nil
+			return err
+		}
+		key, err := certutil.NewPrivateKey()
+		if err != nil {
+			return err
+		}
+		if err := certutil.WriteKey(runtime.ServiceKey, certutil.EncodePrivateKeyPEM(key)); err != nil {
+			return err
+		}
 	}
 
-	key, err := certutil.NewPrivateKey()
+	// When rotating the ServiceAccount signing key, it is necessary to keep the old keys in ServiceKey so that
+	// old ServiceAccount tokens can be validated during the switchover process. The first key in the file
+	// should be the current key used to sign ServiceAccount tokens; others are old keys used for verification
+	// only. Create a file containing just the first key in the list, which will be used to configure the
+	// signing controller.
+	key, err := keyutil.PrivateKeyFromFile(runtime.ServiceKey)
 	if err != nil {
 		return err
 	}
 
-	return certutil.WriteKey(runtime.ServiceKey, certutil.EncodePrivateKeyPEM(key))
+	keyData, err := keyutil.MarshalPrivateKeyToPEM(key)
+	if err != nil {
+		return err
+	}
+
+	return certutil.WriteKey(runtime.ServiceCurrentKey, keyData)
 }
 
 func createSigningCertKey(prefix, certFile, keyFile string) (bool, error) {

pkg/daemons/control/server.go

@@ -94,7 +94,7 @@ func controllerManager(ctx context.Context, cfg *config.Control) error {
 		"kubeconfig":                       runtime.KubeConfigController,
 		"authorization-kubeconfig":         runtime.KubeConfigController,
 		"authentication-kubeconfig":        runtime.KubeConfigController,
-		"service-account-private-key-file": runtime.ServiceKey,
+		"service-account-private-key-file": runtime.ServiceCurrentKey,
 		"allocate-node-cidrs":              "true",
 		"service-cluster-ip-range":         util.JoinIPNets(cfg.ServiceIPRanges),
 		"cluster-cidr":                     util.JoinIPNets(cfg.ClusterIPRanges),
@@ -103,13 +103,13 @@ func controllerManager(ctx context.Context, cfg *config.Control) error {
 		"bind-address":                     cfg.Loopback(false),
 		"secure-port":                      "10257",
 		"use-service-account-credentials":  "true",
-		"cluster-signing-kube-apiserver-client-cert-file": runtime.ClientCA,
+		"cluster-signing-kube-apiserver-client-cert-file": runtime.SigningClientCA,
 		"cluster-signing-kube-apiserver-client-key-file":  runtime.ClientCAKey,
-		"cluster-signing-kubelet-client-cert-file":        runtime.ClientCA,
+		"cluster-signing-kubelet-client-cert-file":        runtime.SigningClientCA,
 		"cluster-signing-kubelet-client-key-file":         runtime.ClientCAKey,
-		"cluster-signing-kubelet-serving-cert-file":       runtime.ServerCA,
+		"cluster-signing-kubelet-serving-cert-file":       runtime.SigningServerCA,
 		"cluster-signing-kubelet-serving-key-file":        runtime.ServerCAKey,
-		"cluster-signing-legacy-unknown-cert-file":        runtime.ServerCA,
+		"cluster-signing-legacy-unknown-cert-file":        runtime.SigningServerCA,
 		"cluster-signing-legacy-unknown-key-file":         runtime.ServerCAKey,
 	}
 	if cfg.NoLeaderElect {
@@ -159,7 +159,7 @@ func apiServer(ctx context.Context, cfg *config.Control) error {
 	argsMap["cert-dir"] = certDir
 	argsMap["allow-privileged"] = "true"
 	argsMap["authorization-mode"] = strings.Join([]string{modes.ModeNode, modes.ModeRBAC}, ",")
-	argsMap["service-account-signing-key-file"] = runtime.ServiceKey
+	argsMap["service-account-signing-key-file"] = runtime.ServiceCurrentKey
 	argsMap["service-cluster-ip-range"] = util.JoinIPNets(cfg.ServiceIPRanges)
 	argsMap["service-node-port-range"] = cfg.ServiceNodePortRange.String()
 	argsMap["advertise-port"] = strconv.Itoa(cfg.AdvertisePort)