Merge pull request #50832 from nckturner/external-ca

Automatic merge from submit-queue (batch tested with PRs 50832, 51119, 51636, 48921, 51712)

kubeadm: Add support for using an external CA whose key is never stored in the cluster

We allow a kubeadm user to use an external CA by checking to see if ca.key is missing and skipping cert checks and kubeconfig generation if ca.key is missing.  We also pass an empty arg --cluster-signing-key-file="" to kube controller manager so that the csr signer doesn't start. 



**What this PR does / why we need it**:

This PR allows the kubeadm certs phase and kubeconfig phase to be skipped if the ca.key is missing but all other certs are present.  

**Which issue this PR fixes** : 

Fixes kubernetes/kubeadm/issues/280

**Special notes for your reviewer**:

@luxas @mikedanese @fabriziopandini 

**Release note**:

```release-note
kubeadm: Add support for using an external CA whose key is never stored in the cluster
```

cmd/kubeadm/app/cmd/init.go

@@ -277,14 +277,20 @@ func (i *Init) Run(out io.Writer) error {
 
 	adminKubeConfigPath := filepath.Join(kubeConfigDir, kubeadmconstants.AdminKubeConfigFileName)
 
-	// PHASE 1: Generate certificates
-	if err := certsphase.CreatePKIAssets(i.cfg); err != nil {
-		return err
-	}
+	if res, _ := certsphase.UsingExternalCA(i.cfg); !res {
 
-	// PHASE 2: Generate kubeconfig files for the admin and the kubelet
-	if err := kubeconfigphase.CreateInitKubeConfigFiles(kubeConfigDir, i.cfg); err != nil {
-		return err
+		// PHASE 1: Generate certificates
+		if err := certsphase.CreatePKIAssets(i.cfg); err != nil {
+			return err
+		}
+
+		// PHASE 2: Generate kubeconfig files for the admin and the kubelet
+		if err := kubeconfigphase.CreateInitKubeConfigFiles(kubeConfigDir, i.cfg); err != nil {
+			return err
+		}
+
+	} else {
+		fmt.Println("[externalca] No ca.key detected, but all other certificates are available, so using external CA mode.  Will not generate certs or kubeconfig.")
 	}
 
 	// Temporarily set cfg.CertificatesDir to the "real value" when writing controlplane manifests

cmd/kubeadm/app/phases/certs/certs.go

@@ -21,6 +21,8 @@ import (
 	"crypto/x509"
 	"fmt"
 	"net"
+	"os"
+	"path/filepath"
 
 	certutil "k8s.io/client-go/util/cert"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
@@ -398,6 +400,110 @@ func writeKeyFilesIfNotExist(pkiDir string, baseName string, key *rsa.PrivateKey
 	return nil
 }
 
+type certKeyLocation struct {
+	pkiDir     string
+	caBaseName string
+	baseName   string
+	uxName     string
+}
+
+// UsingExternalCA determines whether the user is relying on an external CA.  We currently implicitly determine this is the case when the CA Cert
+// is present but the CA Key is not. This allows us to, e.g., skip generating certs or not start the csr signing controller.
+func UsingExternalCA(cfg *kubeadmapi.MasterConfiguration) (bool, error) {
+
+	if err := validateCACert(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName, "", "CA"}); err != nil {
+		return false, err
+	}
+
+	caKeyPath := filepath.Join(cfg.CertificatesDir, kubeadmconstants.CAKeyName)
+	if _, err := os.Stat(caKeyPath); !os.IsNotExist(err) {
+		return false, fmt.Errorf("ca.key exists")
+	}
+
+	if err := validateSignedCert(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName, kubeadmconstants.APIServerCertAndKeyBaseName, "API server"}); err != nil {
+		return false, err
+	}
+
+	if err := validateSignedCert(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName, kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName, "API server kubelet client"}); err != nil {
+		return false, err
+	}
+
+	if err := validatePrivatePublicKey(certKeyLocation{cfg.CertificatesDir, "", kubeadmconstants.ServiceAccountKeyBaseName, "service account"}); err != nil {
+		return false, err
+	}
+
+	if err := validateCACertAndKey(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertAndKeyBaseName, "", "front-proxy CA"}); err != nil {
+		return false, err
+	}
+
+	if err := validateSignedCert(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertAndKeyBaseName, kubeadmconstants.FrontProxyClientCertAndKeyBaseName, "front-proxy client"}); err != nil {
+		return false, err
+	}
+
+	return true, nil
+}
+
+// validateCACert tries to load a x509 certificate from pkiDir and validates that it is a CA
+func validateCACert(l certKeyLocation) error {
+	// Check CA Cert
+	caCert, err := pkiutil.TryLoadCertFromDisk(l.pkiDir, l.caBaseName)
+	if err != nil {
+		return fmt.Errorf("failure loading certificate for %s: %v", l.uxName, err)
+	}
+
+	// Check if cert is a CA
+	if !caCert.IsCA {
+		return fmt.Errorf("certificate %s is not a CA", l.uxName)
+	}
+	return nil
+}
+
+// validateCACertAndKey tries to load a x509 certificate and private key from pkiDir,
+// and validates that the cert is a CA
+func validateCACertAndKey(l certKeyLocation) error {
+	if err := validateCACert(l); err != nil {
+		return err
+	}
+
+	_, err := pkiutil.TryLoadKeyFromDisk(l.pkiDir, l.caBaseName)
+	if err != nil {
+		return fmt.Errorf("failure loading key for %s: %v", l.uxName, err)
+	}
+	return nil
+}
+
+// validateSignedCert tries to load a x509 certificate and private key from pkiDir and validates
+// that the cert is signed by a given CA
+func validateSignedCert(l certKeyLocation) error {
+	// Try to load CA
+	caCert, err := pkiutil.TryLoadCertFromDisk(l.pkiDir, l.caBaseName)
+	if err != nil {
+		return fmt.Errorf("failure loading certificate authorithy for %s: %v", l.uxName, err)
+	}
+
+	// Try to load key and signed certificate
+	signedCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(l.pkiDir, l.baseName)
+	if err != nil {
+		return fmt.Errorf("failure loading certificate for %s: %v", l.uxName, err)
+	}
+
+	// Check if the cert is signed by the CA
+	if err := signedCert.CheckSignatureFrom(caCert); err != nil {
+		return fmt.Errorf("certificate %s is not signed by corresponding CA", l.uxName)
+	}
+	return nil
+}
+
+// validatePrivatePublicKey tries to load a private key from pkiDir
+func validatePrivatePublicKey(l certKeyLocation) error {
+	// Try to load key
+	_, _, err := pkiutil.TryLoadPrivatePublicKeyFromDisk(l.pkiDir, l.baseName)
+	if err != nil {
+		return fmt.Errorf("failure loading key for %s: %v", l.uxName, err)
+	}
+	return nil
+}
+
 // getAltNames builds an AltNames object for to be used when generating apiserver certificate
 func getAltNames(cfg *kubeadmapi.MasterConfiguration) (*certutil.AltNames, error) {
 

cmd/kubeadm/app/phases/certs/certs_test.go

@@ -19,14 +19,15 @@ package certs
 import (
 	"crypto/rsa"
 	"crypto/x509"
+	"fmt"
 	"net"
 	"os"
+	"path/filepath"
 	"testing"
 
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil"
-
 	testutil "k8s.io/kubernetes/cmd/kubeadm/test"
 	certstestutil "k8s.io/kubernetes/cmd/kubeadm/test/certs"
 )
@@ -381,6 +382,151 @@ func TestNewFrontProxyClientCertAndKey(t *testing.T) {
 	certstestutil.AssertCertificateHasClientAuthUsage(t, frontProxyClientCert)
 }
 
+func TestUsingExternalCA(t *testing.T) {
+
+	tests := []struct {
+		setupFuncs []func(cfg *kubeadmapi.MasterConfiguration) error
+		expected   bool
+	}{
+		{
+			setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{
+				CreatePKIAssets,
+			},
+			expected: false,
+		},
+		{
+			setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{
+				CreatePKIAssets,
+				deleteCAKey,
+			},
+			expected: true,
+		},
+	}
+
+	for _, test := range tests {
+		dir := testutil.SetupTempDir(t)
+		defer os.RemoveAll(dir)
+
+		cfg := &kubeadmapi.MasterConfiguration{
+			API:             kubeadmapi.API{AdvertiseAddress: "1.2.3.4"},
+			Networking:      kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
+			NodeName:        "valid-hostname",
+			CertificatesDir: dir,
+		}
+
+		for _, f := range test.setupFuncs {
+			if err := f(cfg); err != nil {
+				t.Errorf("error executing setup function: %v", err)
+			}
+		}
+
+		if val, _ := UsingExternalCA(cfg); val != test.expected {
+			t.Errorf("UsingExternalCA did not match expected: %v", test.expected)
+		}
+	}
+}
+
+func TestValidateMethods(t *testing.T) {
+
+	tests := []struct {
+		name            string
+		setupFuncs      []func(cfg *kubeadmapi.MasterConfiguration) error
+		validateFunc    func(l certKeyLocation) error
+		loc             certKeyLocation
+		expectedSuccess bool
+	}{
+		{
+			name: "validateCACert",
+			setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{
+				CreateCACertAndKeyfiles,
+			},
+			validateFunc:    validateCACert,
+			loc:             certKeyLocation{caBaseName: "ca", baseName: "", uxName: "CA"},
+			expectedSuccess: true,
+		},
+		{
+			name: "validateCACertAndKey (files present)",
+			setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{
+				CreateCACertAndKeyfiles,
+			},
+			validateFunc:    validateCACertAndKey,
+			loc:             certKeyLocation{caBaseName: "ca", baseName: "", uxName: "CA"},
+			expectedSuccess: true,
+		},
+		{
+			name: "validateCACertAndKey (key missing)",
+			setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{
+				CreatePKIAssets,
+				deleteCAKey,
+			},
+			validateFunc:    validateCACertAndKey,
+			loc:             certKeyLocation{caBaseName: "ca", baseName: "", uxName: "CA"},
+			expectedSuccess: false,
+		},
+		{
+			name: "validateSignedCert",
+			setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{
+				CreateCACertAndKeyfiles,
+				CreateAPIServerCertAndKeyFiles,
+			},
+			validateFunc:    validateSignedCert,
+			loc:             certKeyLocation{caBaseName: "ca", baseName: "apiserver", uxName: "apiserver"},
+			expectedSuccess: true,
+		},
+		{
+			name: "validatePrivatePublicKey",
+			setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{
+				CreateServiceAccountKeyAndPublicKeyFiles,
+			},
+			validateFunc:    validatePrivatePublicKey,
+			loc:             certKeyLocation{baseName: "sa", uxName: "service account"},
+			expectedSuccess: true,
+		},
+	}
+
+	for _, test := range tests {
+
+		dir := testutil.SetupTempDir(t)
+		defer os.RemoveAll(dir)
+		test.loc.pkiDir = dir
+
+		cfg := &kubeadmapi.MasterConfiguration{
+			API:             kubeadmapi.API{AdvertiseAddress: "1.2.3.4"},
+			Networking:      kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
+			NodeName:        "valid-hostname",
+			CertificatesDir: dir,
+		}
+
+		fmt.Println("Testing", test.name)
+
+		for _, f := range test.setupFuncs {
+			if err := f(cfg); err != nil {
+				t.Errorf("error executing setup function: %v", err)
+			}
+		}
+
+		err := test.validateFunc(test.loc)
+		if test.expectedSuccess && err != nil {
+			t.Errorf("expected success, error executing validateFunc: %v, %v", test.name, err)
+		} else if !test.expectedSuccess && err == nil {
+			t.Errorf("expected failure, no error executing validateFunc: %v", test.name)
+		}
+	}
+}
+
+func deleteCAKey(cfg *kubeadmapi.MasterConfiguration) error {
+	if err := os.Remove(filepath.Join(cfg.CertificatesDir, "ca.key")); err != nil {
+		return fmt.Errorf("failed removing ca.key: %v", err)
+	}
+	return nil
+}
+
+func assertIsCa(t *testing.T, cert *x509.Certificate) {
+	if !cert.IsCA {
+		t.Error("cert is not a valida CA")
+	}
+}
+
 func TestCreateCertificateFilesMethods(t *testing.T) {
 
 	var tests = []struct {

cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers.go

@@ -200,6 +200,35 @@ func TryLoadKeyFromDisk(pkiPath, name string) (*rsa.PrivateKey, error) {
 	return key, nil
 }
 
+// TryLoadPrivatePublicKeyFromDisk tries to load the key from the disk and validates that it is valid
+func TryLoadPrivatePublicKeyFromDisk(pkiPath, name string) (*rsa.PrivateKey, *rsa.PublicKey, error) {
+	privateKeyPath := pathForKey(pkiPath, name)
+
+	// Parse the private key from a file
+	privKey, err := certutil.PrivateKeyFromFile(privateKeyPath)
+	if err != nil {
+		return nil, nil, fmt.Errorf("couldn't load the private key file %s: %v", privateKeyPath, err)
+	}
+
+	publicKeyPath := pathForPublicKey(pkiPath, name)
+
+	// Parse the public key from a file
+	pubKeys, err := certutil.PublicKeysFromFile(publicKeyPath)
+	if err != nil {
+		return nil, nil, fmt.Errorf("couldn't load the public key file %s: %v", publicKeyPath, err)
+	}
+
+	// Allow RSA format only
+	k, ok := privKey.(*rsa.PrivateKey)
+	if !ok {
+		return nil, nil, fmt.Errorf("the private key file %s isn't in RSA format", privateKeyPath)
+	}
+
+	p := pubKeys[0].(*rsa.PublicKey)
+
+	return k, p, nil
+}
+
 func pathsForCertAndKey(pkiPath, name string) (string, string) {
 	return pathForCert(pkiPath, name), pathForKey(pkiPath, name)
 }

cmd/kubeadm/app/phases/controlplane/BUILD

@@ -16,6 +16,7 @@ go_test(
     deps = [
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
+        "//cmd/kubeadm/app/phases/certs:go_default_library",
         "//cmd/kubeadm/test:go_default_library",
         "//pkg/util/version:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
@@ -33,6 +34,7 @@ go_library(
         "//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/app/images:go_default_library",
+        "//cmd/kubeadm/app/phases/certs:go_default_library",
         "//cmd/kubeadm/app/util:go_default_library",
         "//cmd/kubeadm/app/util/staticpod:go_default_library",
         "//pkg/kubeapiserver/authorizer/modes:go_default_library",

cmd/kubeadm/app/phases/controlplane/manifests.go

@@ -29,6 +29,7 @@ import (
 	kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/images"
+	certphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
 	authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
@@ -204,6 +205,7 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *versio
 
 // getControllerManagerCommand builds the right controller manager command from the given config object and version
 func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.Version) []string {
+
 	defaultArguments := map[string]string{
 		"address":                          "127.0.0.1",
 		"leader-elect":                     "true",
@@ -216,6 +218,13 @@ func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion
 		"controllers":                      "*,bootstrapsigner,tokencleaner",
 	}
 
+	// If using external CA, pass empty string to controller manager instead of ca.key/ca.crt path,
+	// so that the csrsigning controller fails to start
+	if res, _ := certphase.UsingExternalCA(cfg); res {
+		defaultArguments["cluster-signing-key-file"] = ""
+		defaultArguments["cluster-signing-cert-file"] = ""
+	}
+
 	command := []string{"kube-controller-manager"}
 	command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.ControllerManagerExtraArgs)...)
 

cmd/kubeadm/app/phases/controlplane/manifests_test.go

@@ -26,6 +26,7 @@ import (
 
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
 	"k8s.io/kubernetes/pkg/util/version"
 
 	testutil "k8s.io/kubernetes/cmd/kubeadm/test"
@@ -461,6 +462,88 @@ func TestGetControllerManagerCommand(t *testing.T) {
 	}
 }
 
+func TestGetControllerManagerCommandExternalCA(t *testing.T) {
+
+	tests := []struct {
+		cfg             *kubeadmapi.MasterConfiguration
+		caKeyPresent    bool
+		expectedArgFunc func(dir string) []string
+	}{
+		{
+			cfg: &kubeadmapi.MasterConfiguration{
+				KubernetesVersion: "v1.7.0",
+				API:               kubeadmapi.API{AdvertiseAddress: "1.2.3.4"},
+				Networking:        kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
+				NodeName:          "valid-hostname",
+			},
+			caKeyPresent: false,
+			expectedArgFunc: func(tmpdir string) []string {
+				return []string{
+					"kube-controller-manager",
+					"--address=127.0.0.1",
+					"--leader-elect=true",
+					"--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
+					"--root-ca-file=" + tmpdir + "/ca.crt",
+					"--service-account-private-key-file=" + tmpdir + "/sa.key",
+					"--cluster-signing-cert-file=",
+					"--cluster-signing-key-file=",
+					"--use-service-account-credentials=true",
+					"--controllers=*,bootstrapsigner,tokencleaner",
+				}
+			},
+		},
+		{
+			cfg: &kubeadmapi.MasterConfiguration{
+				KubernetesVersion: "v1.7.0",
+				API:               kubeadmapi.API{AdvertiseAddress: "1.2.3.4"},
+				Networking:        kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
+				NodeName:          "valid-hostname",
+			},
+			caKeyPresent: true,
+			expectedArgFunc: func(tmpdir string) []string {
+				return []string{
+					"kube-controller-manager",
+					"--address=127.0.0.1",
+					"--leader-elect=true",
+					"--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
+					"--root-ca-file=" + tmpdir + "/ca.crt",
+					"--service-account-private-key-file=" + tmpdir + "/sa.key",
+					"--cluster-signing-cert-file=" + tmpdir + "/ca.crt",
+					"--cluster-signing-key-file=" + tmpdir + "/ca.key",
+					"--use-service-account-credentials=true",
+					"--controllers=*,bootstrapsigner,tokencleaner",
+				}
+			},
+		},
+	}
+
+	for _, test := range tests {
+		// Create temp folder for the test case
+		tmpdir := testutil.SetupTempDir(t)
+		defer os.RemoveAll(tmpdir)
+		test.cfg.CertificatesDir = tmpdir
+
+		if err := certs.CreatePKIAssets(test.cfg); err != nil {
+			t.Errorf("failed creating pki assets: %v", err)
+		}
+
+		// delete ca.key if test.caKeyPresent is false
+		if !test.caKeyPresent {
+			if err := os.Remove(filepath.Join(test.cfg.CertificatesDir, "ca.key")); err != nil {
+				t.Errorf("failed removing ca.key: %v", err)
+			}
+		}
+
+		actual := getControllerManagerCommand(test.cfg, version.MustParseSemantic(test.cfg.KubernetesVersion))
+		expected := test.expectedArgFunc(tmpdir)
+		sort.Strings(actual)
+		sort.Strings(expected)
+		if !reflect.DeepEqual(actual, expected) {
+			t.Errorf("failed getControllerManagerCommand:\nexpected:\n%v\nsaw:\n%v", expected, actual)
+		}
+	}
+}
+
 func TestGetSchedulerCommand(t *testing.T) {
 	var tests = []struct {
 		cfg      *kubeadmapi.MasterConfiguration