From f9000a3f8a9cb766b91e74a6bd0da4b1363c750c Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Thu, 12 Jul 2018 16:45:12 +0200 Subject: [PATCH] kubeadm-ha-phases --- cmd/kubeadm/app/phases/certs/certs.go | 19 +++++ cmd/kubeadm/app/phases/certs/certs_test.go | 71 +++++++++++++++++++ .../app/phases/kubeconfig/kubeconfig.go | 15 ++++ .../app/phases/kubeconfig/kubeconfig_test.go | 8 +++ cmd/kubeadm/app/phases/uploadconfig/BUILD | 2 + .../app/phases/uploadconfig/uploadconfig.go | 48 ++++++++++++- 6 files changed, 162 insertions(+), 1 deletion(-) diff --git a/cmd/kubeadm/app/phases/certs/certs.go b/cmd/kubeadm/app/phases/certs/certs.go index e4cbb54735..40cdd41eca 100644 --- a/cmd/kubeadm/app/phases/certs/certs.go +++ b/cmd/kubeadm/app/phases/certs/certs.go @@ -621,6 +621,25 @@ type certKeyLocation struct { uxName string } +// SharedCertificateExists verifies if the shared certificates - the certificates that must be +// equal across masters: ca.key, ca.crt, sa.key, sa.pub +func SharedCertificateExists(cfg *kubeadmapi.InitConfiguration) (bool, error) { + + if err := validateCACertAndKey(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName, "", "CA"}); err != nil { + return false, err + } + + if err := validatePrivatePublicKey(certKeyLocation{cfg.CertificatesDir, "", kubeadmconstants.ServiceAccountKeyBaseName, "service account"}); err != nil { + return false, err + } + + if err := validateCACertAndKey(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertAndKeyBaseName, "", "front-proxy CA"}); err != nil { + return false, err + } + + return true, nil +} + // UsingExternalCA determines whether the user is relying on an external CA. We currently implicitly determine this is the case // when both the CA Cert and the front proxy CA Cert are present but the CA Key and front proxy CA Key are not. // This allows us to, e.g., skip generating certs or not start the csr signing controller. diff --git a/cmd/kubeadm/app/phases/certs/certs_test.go b/cmd/kubeadm/app/phases/certs/certs_test.go index ca63002912..7da7bd9c9d 100644 --- a/cmd/kubeadm/app/phases/certs/certs_test.go +++ b/cmd/kubeadm/app/phases/certs/certs_test.go @@ -457,6 +457,77 @@ func TestNewFrontProxyClientCertAndKey(t *testing.T) { certstestutil.AssertCertificateHasClientAuthUsage(t, frontProxyClientCert) } +func TestSharedCertificateExists(t *testing.T) { + + var tests = []struct { + setupFunc func(cfg *kubeadmapi.InitConfiguration) + expectedError bool + }{ + { // expected certs exist, pass + setupFunc: func(cfg *kubeadmapi.InitConfiguration) { + CreateCACertAndKeyFiles(cfg) + CreateServiceAccountKeyAndPublicKeyFiles(cfg) + CreateFrontProxyCACertAndKeyFiles(cfg) + }, + expectedError: false, + }, + { // expected ca.crt missing + setupFunc: func(cfg *kubeadmapi.InitConfiguration) { + // start from the condition created by the previous tests + os.Remove(filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName)) + }, + expectedError: true, + }, + { // expected sa.key missing + setupFunc: func(cfg *kubeadmapi.InitConfiguration) { + // start from the condition created by the previous tests + CreateCACertAndKeyFiles(cfg) + os.Remove(filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPublicKeyName)) + }, + expectedError: true, + }, + { // expected front-proxy.crt missing + setupFunc: func(cfg *kubeadmapi.InitConfiguration) { + // start from the condition created by the previous tests + CreateServiceAccountKeyAndPublicKeyFiles(cfg) + os.Remove(filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertName)) + }, + expectedError: true, + }, + } + + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + + cfg := &kubeadmapi.InitConfiguration{ + CertificatesDir: tmpdir, + } + + for _, test := range tests { + // executes setup func (if necessary) + if test.setupFunc != nil { + test.setupFunc(cfg) + } + + // executes create func + ret, err := SharedCertificateExists(cfg) + + if !test.expectedError && err != nil { + t.Errorf("error SharedCertificateExists failed when not expected to fail: %v", err) + continue + } else if test.expectedError && err == nil { + t.Error("error SharedCertificateExists didn't failed when expected") + continue + } else if test.expectedError { + continue + } + + if ret != (err == nil) { + t.Errorf("error SharedCertificateExists returned %v when expected to return %v", ret, err == nil) + } + } +} + func TestUsingExternalCA(t *testing.T) { tests := []struct { diff --git a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go index 725683fec0..8c5335ae86 100644 --- a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go +++ b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go @@ -73,6 +73,21 @@ func CreateInitKubeConfigFiles(outDir string, cfg *kubeadmapi.InitConfiguration) ) } +// CreateJoinMasterKubeConfigFiles will create and write to disk the kubeconfig files required by kubeadm +// join --master workflow, plus the admin kubeconfig file to be deployed on the new master; the +// kubelet.conf file must not be created when joining master nodes because it will be created and signed by +// the kubelet TLS bootstrap process. +// If any kubeconfig files already exists, it used only if evaluated equal; otherwise an error is returned. +func CreateJoinMasterKubeConfigFiles(outDir string, cfg *kubeadmapi.InitConfiguration) error { + return createKubeConfigFiles( + outDir, + cfg, + kubeadmconstants.AdminKubeConfigFileName, + kubeadmconstants.ControllerManagerKubeConfigFileName, + kubeadmconstants.SchedulerKubeConfigFileName, + ) +} + // CreateAdminKubeConfigFile create a kubeconfig file for the admin to use and for kubeadm itself. // If the kubeconfig file already exists, it is used only if evaluated equal; otherwise an error is returned. func CreateAdminKubeConfigFile(outDir string, cfg *kubeadmapi.InitConfiguration) error { diff --git a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go index 925dc32fe2..fe14d5ecb9 100644 --- a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go +++ b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go @@ -271,6 +271,14 @@ func TestCreateKubeconfigFilesAndWrappers(t *testing.T) { kubeadmconstants.SchedulerKubeConfigFileName, }, }, + { // Test CreateJoinMasterKubeConfigFiles (wrapper to createKubeConfigFile) + createKubeConfigFunction: CreateJoinMasterKubeConfigFiles, + expectedFiles: []string{ + kubeadmconstants.AdminKubeConfigFileName, + kubeadmconstants.ControllerManagerKubeConfigFileName, + kubeadmconstants.SchedulerKubeConfigFileName, + }, + }, { // Test CreateAdminKubeConfigFile (wrapper to createKubeConfigFile) createKubeConfigFunction: CreateAdminKubeConfigFile, expectedFiles: []string{kubeadmconstants.AdminKubeConfigFileName}, diff --git a/cmd/kubeadm/app/phases/uploadconfig/BUILD b/cmd/kubeadm/app/phases/uploadconfig/BUILD index 2aa89ed7cd..7786368fbd 100644 --- a/cmd/kubeadm/app/phases/uploadconfig/BUILD +++ b/cmd/kubeadm/app/phases/uploadconfig/BUILD @@ -15,7 +15,9 @@ go_library( "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/util/apiclient:go_default_library", "//cmd/kubeadm/app/util/config:go_default_library", + "//pkg/apis/rbac/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/api/rbac/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", ], diff --git a/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go b/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go index ae1b17b684..0377ec54c6 100644 --- a/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go +++ b/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go @@ -20,12 +20,21 @@ import ( "fmt" "k8s.io/api/core/v1" + rbac "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" + rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1" +) + +const ( + // BootstrapDiscoveryClusterRoleName sets the name for the ClusterRole that allows + // the bootstrap tokens to access the kubeadm-config ConfigMap during the node bootstrap/discovery + // phase for additional master nodes + BootstrapDiscoveryClusterRoleName = "kubeadm:bootstrap-discovery-kubeadm-config" ) // UploadConfiguration saves the InitConfiguration used for later reference (when upgrading for instance) @@ -53,7 +62,7 @@ func UploadConfiguration(cfg *kubeadmapi.InitConfiguration, client clientset.Int return err } - return apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{ + err = apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: kubeadmconstants.InitConfigurationConfigMap, Namespace: metav1.NamespaceSystem, @@ -62,4 +71,41 @@ func UploadConfiguration(cfg *kubeadmapi.InitConfiguration, client clientset.Int kubeadmconstants.InitConfigurationConfigMapKey: string(cfgYaml), }, }) + if err != nil { + return err + } + + // Ensure that the BootstrapDiscoveryClusterRole exists + err = apiclient.CreateOrUpdateRole(client, &rbac.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: BootstrapDiscoveryClusterRoleName, + Namespace: metav1.NamespaceSystem, + }, + Rules: []rbac.PolicyRule{ + rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(kubeadmconstants.InitConfigurationConfigMap).RuleOrDie(), + }, + }) + if err != nil { + return err + } + + // Binds the BootstrapDiscoveryClusterRole to all the bootstrap tokens + // that are members of the system:bootstrappers:kubeadm:default-node-token group + return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: BootstrapDiscoveryClusterRoleName, + Namespace: metav1.NamespaceSystem, + }, + RoleRef: rbac.RoleRef{ + APIGroup: rbac.GroupName, + Kind: "Role", + Name: BootstrapDiscoveryClusterRoleName, + }, + Subjects: []rbac.Subject{ + { + Kind: rbac.GroupKind, + Name: kubeadmconstants.NodeBootstrapTokenAuthGroup, + }, + }, + }) }