Merge pull request #62948 from fabriziopandini/kubeadm-ha-phases

Automatic merge from submit-queue (batch tested with PRs 65105, 62948). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

make kubeadm phases ready for join master

**What this PR does / why we need it**:
This PR implements one of the actions defined by https://github.com/kubernetes/kubeadm/issues/751 (checklist form implementing HA in kubeadm).

With this PR, kubeadm phases implements methods that will be used by the `kubeadm join --master`workflow, and more in detail:
- kubeconfig phase implements a new method for creating kubeconfig required files (nb. with respect to init, the kubelet.conf file should not be created because it will generated by the TLS bootstrap process)
- certs phase implements a new method for checking the pki provided by the users (all the certificates are present, the API server certificate is properly configured)

**Special notes for your reviewer**:
/CC @timothysc @kubernetes/sig-cluster-lifecycle-pr-reviews 

**Release note**:
```release-note
NONE
```

cmd/kubeadm/app/phases/certs/certs.go

@@ -621,6 +621,25 @@ type certKeyLocation struct {
 	uxName     string
 }
 
+// SharedCertificateExists verifies if the shared certificates - the certificates that must be
+// equal across masters: ca.key, ca.crt, sa.key, sa.pub
+func SharedCertificateExists(cfg *kubeadmapi.InitConfiguration) (bool, error) {
+
+	if err := validateCACertAndKey(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName, "", "CA"}); err != nil {
+		return false, err
+	}
+
+	if err := validatePrivatePublicKey(certKeyLocation{cfg.CertificatesDir, "", kubeadmconstants.ServiceAccountKeyBaseName, "service account"}); err != nil {
+		return false, err
+	}
+
+	if err := validateCACertAndKey(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertAndKeyBaseName, "", "front-proxy CA"}); err != nil {
+		return false, err
+	}
+
+	return true, nil
+}
+
 // UsingExternalCA determines whether the user is relying on an external CA.  We currently implicitly determine this is the case
 // when both the CA Cert and the front proxy CA Cert are present but the CA Key and front proxy CA Key are not.
 // This allows us to, e.g., skip generating certs or not start the csr signing controller.

cmd/kubeadm/app/phases/certs/certs_test.go

@@ -457,6 +457,77 @@ func TestNewFrontProxyClientCertAndKey(t *testing.T) {
 	certstestutil.AssertCertificateHasClientAuthUsage(t, frontProxyClientCert)
 }
 
+func TestSharedCertificateExists(t *testing.T) {
+
+	var tests = []struct {
+		setupFunc     func(cfg *kubeadmapi.InitConfiguration)
+		expectedError bool
+	}{
+		{ // expected certs exist, pass
+			setupFunc: func(cfg *kubeadmapi.InitConfiguration) {
+				CreateCACertAndKeyFiles(cfg)
+				CreateServiceAccountKeyAndPublicKeyFiles(cfg)
+				CreateFrontProxyCACertAndKeyFiles(cfg)
+			},
+			expectedError: false,
+		},
+		{ // expected ca.crt missing
+			setupFunc: func(cfg *kubeadmapi.InitConfiguration) {
+				// start from the condition created by the previous tests
+				os.Remove(filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName))
+			},
+			expectedError: true,
+		},
+		{ // expected sa.key missing
+			setupFunc: func(cfg *kubeadmapi.InitConfiguration) {
+				// start from the condition created by the previous tests
+				CreateCACertAndKeyFiles(cfg)
+				os.Remove(filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPublicKeyName))
+			},
+			expectedError: true,
+		},
+		{ // expected front-proxy.crt missing
+			setupFunc: func(cfg *kubeadmapi.InitConfiguration) {
+				// start from the condition created by the previous tests
+				CreateServiceAccountKeyAndPublicKeyFiles(cfg)
+				os.Remove(filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertName))
+			},
+			expectedError: true,
+		},
+	}
+
+	tmpdir := testutil.SetupTempDir(t)
+	defer os.RemoveAll(tmpdir)
+
+	cfg := &kubeadmapi.InitConfiguration{
+		CertificatesDir: tmpdir,
+	}
+
+	for _, test := range tests {
+		// executes setup func (if necessary)
+		if test.setupFunc != nil {
+			test.setupFunc(cfg)
+		}
+
+		// executes create func
+		ret, err := SharedCertificateExists(cfg)
+
+		if !test.expectedError && err != nil {
+			t.Errorf("error SharedCertificateExists failed when not expected to fail: %v", err)
+			continue
+		} else if test.expectedError && err == nil {
+			t.Error("error SharedCertificateExists didn't failed when expected")
+			continue
+		} else if test.expectedError {
+			continue
+		}
+
+		if ret != (err == nil) {
+			t.Errorf("error SharedCertificateExists returned %v when expected to return %v", ret, err == nil)
+		}
+	}
+}
+
 func TestUsingExternalCA(t *testing.T) {
 
 	tests := []struct {

cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go

@@ -73,6 +73,21 @@ func CreateInitKubeConfigFiles(outDir string, cfg *kubeadmapi.InitConfiguration)
 	)
 }
 
+// CreateJoinMasterKubeConfigFiles will create and write to disk the kubeconfig files required by kubeadm
+// join --master workflow, plus the admin kubeconfig file to be deployed on the new master; the
+// kubelet.conf file must not be created when joining master nodes because it will be created and signed by
+// the kubelet TLS bootstrap process.
+// If any kubeconfig files already exists, it used only if evaluated equal; otherwise an error is returned.
+func CreateJoinMasterKubeConfigFiles(outDir string, cfg *kubeadmapi.InitConfiguration) error {
+	return createKubeConfigFiles(
+		outDir,
+		cfg,
+		kubeadmconstants.AdminKubeConfigFileName,
+		kubeadmconstants.ControllerManagerKubeConfigFileName,
+		kubeadmconstants.SchedulerKubeConfigFileName,
+	)
+}
+
 // CreateAdminKubeConfigFile create a kubeconfig file for the admin to use and for kubeadm itself.
 // If the kubeconfig file already exists, it is used only if evaluated equal; otherwise an error is returned.
 func CreateAdminKubeConfigFile(outDir string, cfg *kubeadmapi.InitConfiguration) error {

cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go

@@ -271,6 +271,14 @@ func TestCreateKubeconfigFilesAndWrappers(t *testing.T) {
 				kubeadmconstants.SchedulerKubeConfigFileName,
 			},
 		},
+		{ // Test CreateJoinMasterKubeConfigFiles (wrapper to createKubeConfigFile)
+			createKubeConfigFunction: CreateJoinMasterKubeConfigFiles,
+			expectedFiles: []string{
+				kubeadmconstants.AdminKubeConfigFileName,
+				kubeadmconstants.ControllerManagerKubeConfigFileName,
+				kubeadmconstants.SchedulerKubeConfigFileName,
+			},
+		},
 		{ // Test CreateAdminKubeConfigFile (wrapper to createKubeConfigFile)
 			createKubeConfigFunction: CreateAdminKubeConfigFile,
 			expectedFiles:            []string{kubeadmconstants.AdminKubeConfigFileName},

cmd/kubeadm/app/phases/uploadconfig/BUILD

@@ -15,7 +15,9 @@ go_library(
         "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/app/util/apiclient:go_default_library",
         "//cmd/kubeadm/app/util/config:go_default_library",
+        "//pkg/apis/rbac/v1:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/api/rbac/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
     ],

cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go

@@ -20,12 +20,21 @@ import (
 	"fmt"
 
 	"k8s.io/api/core/v1"
+	rbac "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
 	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
+	rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1"
+)
+
+const (
+	// BootstrapDiscoveryClusterRoleName sets the name for the ClusterRole that allows
+	// the bootstrap tokens to access the kubeadm-config ConfigMap during the node bootstrap/discovery
+	// phase for additional master nodes
+	BootstrapDiscoveryClusterRoleName = "kubeadm:bootstrap-discovery-kubeadm-config"
 )
 
 // UploadConfiguration saves the InitConfiguration used for later reference (when upgrading for instance)
@@ -53,7 +62,7 @@ func UploadConfiguration(cfg *kubeadmapi.InitConfiguration, client clientset.Int
 		return err
 	}
 
-	return apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{
+	err = apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      kubeadmconstants.InitConfigurationConfigMap,
 			Namespace: metav1.NamespaceSystem,
@@ -62,4 +71,41 @@ func UploadConfiguration(cfg *kubeadmapi.InitConfiguration, client clientset.Int
 			kubeadmconstants.InitConfigurationConfigMapKey: string(cfgYaml),
 		},
 	})
+	if err != nil {
+		return err
+	}
+
+	// Ensure that the BootstrapDiscoveryClusterRole exists
+	err = apiclient.CreateOrUpdateRole(client, &rbac.Role{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      BootstrapDiscoveryClusterRoleName,
+			Namespace: metav1.NamespaceSystem,
+		},
+		Rules: []rbac.PolicyRule{
+			rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(kubeadmconstants.InitConfigurationConfigMap).RuleOrDie(),
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	// Binds the BootstrapDiscoveryClusterRole to all the bootstrap tokens
+	// that are members of the system:bootstrappers:kubeadm:default-node-token group
+	return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      BootstrapDiscoveryClusterRoleName,
+			Namespace: metav1.NamespaceSystem,
+		},
+		RoleRef: rbac.RoleRef{
+			APIGroup: rbac.GroupName,
+			Kind:     "Role",
+			Name:     BootstrapDiscoveryClusterRoleName,
+		},
+		Subjects: []rbac.Subject{
+			{
+				Kind: rbac.GroupKind,
+				Name: kubeadmconstants.NodeBootstrapTokenAuthGroup,
+			},
+		},
+	})
 }