Merge pull request #51956 from luxas/kubeadm_upgrade_bootstraptokens

Automatic merge from submit-queue (batch tested with PRs 51956, 50708)

kubeadm: Upgrade Bootstrap Tokens to beta when upgrading to v1.8

**What this PR does / why we need it**:

Makes sure the v1.7 -> v1.8 upgrade works regarding the Bootstrap Token alpha -> beta graduation.
Not much have to be done, but some LoC are needed to preserve the behaivor

**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #

**Special notes for your reviewer**:

**Release note**:

```release-note
NONE
```
@kubernetes/sig-cluster-lifecycle-pr-reviews

cmd/kubeadm/app/cmd/init.go

@@ -379,7 +379,7 @@ func (i *Init) Run(out io.Writer) error {
 		return err
 	}
 	// Create RBAC rules that makes the bootstrap tokens able to post CSRs
-	if err := nodebootstraptokenphase.AllowBootstrapTokensToPostCSRs(client); err != nil {
+	if err := nodebootstraptokenphase.AllowBootstrapTokensToPostCSRs(client, k8sVersion); err != nil {
 		return err
 	}
 	// Create RBAC rules that makes the bootstrap tokens able to get their CSRs approved automatically

cmd/kubeadm/app/cmd/phases/bootstraptoken.go

@@ -100,7 +100,10 @@ func NewSubCmdNodeBootstrapTokenPostCSRs(kubeConfigFile *string) *cobra.Command
 			client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile)
 			kubeadmutil.CheckErr(err)
 
-			err = node.AllowBootstrapTokensToPostCSRs(client)
+			clusterVersion, err := getClusterVersion(client)
+			kubeadmutil.CheckErr(err)
+
+			err = node.AllowBootstrapTokensToPostCSRs(client, clusterVersion)
 			kubeadmutil.CheckErr(err)
 		},
 	}

cmd/kubeadm/app/constants/BUILD

@@ -32,4 +32,5 @@ go_test(
     name = "go_default_test",
     srcs = ["constants_test.go"],
     library = ":go_default_library",
+    deps = ["//pkg/util/version:go_default_library"],
 )

cmd/kubeadm/app/constants/constants.go

@@ -122,9 +122,11 @@ const (
 	// KubeConfigVolumeName specifies the name for the Volume that is used for injecting the kubeconfig to talk securely to the api server for a control plane component if applicable
 	KubeConfigVolumeName = "kubeconfig"
 
-	// NodeBootstrapTokenAuthGroup specifies which group a Node Bootstrap Token should be authenticated in
+	// V17NodeBootstrapTokenAuthGroup specifies which group a Node Bootstrap Token should be authenticated in, in v1.7
-	// TODO: This should be changed in the v1.8 dev cycle to a node-BT-specific group instead of the generic Bootstrap Token group that is used now
+	V17NodeBootstrapTokenAuthGroup = "system:bootstrappers"
-	NodeBootstrapTokenAuthGroup = "system:bootstrappers"
+
+	// V18NodeBootstrapTokenAuthGroup specifies which group a Node Bootstrap Token should be authenticated in, in v1.8
+	V18NodeBootstrapTokenAuthGroup = "system:bootstrappers:kubeadm:default-node-token"
 
 	// DefaultCIImageRepository points to image registry where CI uploads images from ci-cross build job
 	DefaultCIImageRepository = "gcr.io/kubernetes-ci-images"
@@ -198,3 +200,11 @@ func CreateTempDirForKubeadm(dirName string) (string, error) {
 	}
 	return tempDir, nil
 }
+
+// GetNodeBootstrapTokenAuthGroup gets the bootstrap token auth group conditionally based on version
+func GetNodeBootstrapTokenAuthGroup(k8sVersion *version.Version) string {
+	if k8sVersion.AtLeast(UseEnableBootstrapTokenAuthFlagVersion) {
+		return V18NodeBootstrapTokenAuthGroup
+	}
+	return V17NodeBootstrapTokenAuthGroup
+}

cmd/kubeadm/app/constants/constants_test.go

@@ -18,6 +18,8 @@ package constants
 
 import (
 	"testing"
+
+	"k8s.io/kubernetes/pkg/util/version"
 )
 
 func TestGetStaticPodDirectory(t *testing.T) {
@@ -110,3 +112,48 @@ func TestAddSelfHostedPrefix(t *testing.T) {
 		}
 	}
 }
+
+func TestGetNodeBootstrapTokenAuthGroup(t *testing.T) {
+	var tests = []struct {
+		k8sVersion, expected string
+	}{
+		{
+			k8sVersion: "v1.7.0",
+			expected:   "system:bootstrappers",
+		},
+		{
+			k8sVersion: "v1.7.8",
+			expected:   "system:bootstrappers",
+		},
+		{
+			k8sVersion: "v1.8.0-alpha.3",
+			expected:   "system:bootstrappers",
+		},
+		{
+			k8sVersion: "v1.8.0-beta.0",
+			expected:   "system:bootstrappers:kubeadm:default-node-token",
+		},
+		{
+			k8sVersion: "v1.8.0-rc.1",
+			expected:   "system:bootstrappers:kubeadm:default-node-token",
+		},
+		{
+			k8sVersion: "v1.8.0",
+			expected:   "system:bootstrappers:kubeadm:default-node-token",
+		},
+		{
+			k8sVersion: "v1.8.9",
+			expected:   "system:bootstrappers:kubeadm:default-node-token",
+		},
+	}
+	for _, rt := range tests {
+		actual := GetNodeBootstrapTokenAuthGroup(version.MustParseSemantic(rt.k8sVersion))
+		if actual != rt.expected {
+			t.Errorf(
+				"failed GetNodeBootstrapTokenAuthGroup:\n\texpected: %s\n\t  actual: %s",
+				rt.expected,
+				actual,
+			)
+		}
+	}
+}

cmd/kubeadm/app/phases/bootstraptoken/node/tlsbootstrap.go

@@ -38,12 +38,12 @@ const (
 	// CSRAutoApprovalClusterRoleName defines the name of the auto-bootstrapped ClusterRole for making the csrapprover controller auto-approve the CSR
 	// TODO: This value should be defined in an other, generic authz package instead of here
 	CSRAutoApprovalClusterRoleName = "system:certificates.k8s.io:certificatesigningrequests:nodeclient"
-	// NodeAutoApproveBootstrap defines the name of the ClusterRoleBinding that makes the csrapprover approve node CSRs
+	// NodeAutoApproveBootstrapClusterRoleBinding defines the name of the ClusterRoleBinding that makes the csrapprover approve node CSRs
-	NodeAutoApproveBootstrap = "kubeadm:node-autoapprove-bootstrap"
+	NodeAutoApproveBootstrapClusterRoleBinding = "kubeadm:node-autoapprove-bootstrap"
 )
 
 // AllowBootstrapTokensToPostCSRs creates RBAC rules in a way the makes Node Bootstrap Tokens able to post CSRs
-func AllowBootstrapTokensToPostCSRs(client clientset.Interface) error {
+func AllowBootstrapTokensToPostCSRs(client clientset.Interface, k8sVersion *version.Version) error {
 
 	fmt.Println("[bootstraptoken] Configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials")
 
@@ -59,7 +59,7 @@ func AllowBootstrapTokensToPostCSRs(client clientset.Interface) error {
 		Subjects: []rbac.Subject{
 			{
 				Kind: rbac.GroupKind,
-				Name: constants.NodeBootstrapTokenAuthGroup,
+				Name: constants.GetNodeBootstrapTokenAuthGroup(k8sVersion),
 			},
 		},
 	})
@@ -89,7 +89,7 @@ func AutoApproveNodeBootstrapTokens(client clientset.Interface, k8sVersion *vers
 	// Always create this kubeadm-specific binding though
 	return apiclient.CreateOrUpdateClusterRoleBinding(client, &rbac.ClusterRoleBinding{
 		ObjectMeta: metav1.ObjectMeta{
-			Name: NodeAutoApproveBootstrap,
+			Name: NodeAutoApproveBootstrapClusterRoleBinding,
 		},
 		RoleRef: rbac.RoleRef{
 			APIGroup: rbac.GroupName,
@@ -99,7 +99,7 @@ func AutoApproveNodeBootstrapTokens(client clientset.Interface, k8sVersion *vers
 		Subjects: []rbac.Subject{
 			{
 				Kind: "Group",
-				Name: constants.NodeBootstrapTokenAuthGroup,
+				Name: constants.GetNodeBootstrapTokenAuthGroup(k8sVersion),
 			},
 		},
 	})

cmd/kubeadm/app/phases/upgrade/BUILD

@@ -8,6 +8,7 @@ go_library(
         "health.go",
         "policy.go",
         "postupgrade.go",
+        "postupgrade_v17_v18.go",
         "prepull.go",
         "selfhosted.go",
         "staticpods.go",
@@ -32,12 +33,14 @@ go_library(
         "//cmd/kubeadm/app/util/apiclient:go_default_library",
         "//cmd/kubeadm/app/util/config:go_default_library",
         "//pkg/api:go_default_library",
+        "//pkg/bootstrap/api:go_default_library",
         "//pkg/util/version:go_default_library",
         "//pkg/version:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",

cmd/kubeadm/app/phases/upgrade/postupgrade.go

@@ -41,10 +41,27 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.MasterC
 		errs = append(errs, err)
 	}
 
-	// Create/update RBAC rules that makes the bootstrap tokens able to post CSRs
+	// Handle Bootstrap Tokens graduating to from alpha to beta in the v1.7 -> v1.8 upgrade
-	if err := nodebootstraptoken.AllowBootstrapTokensToPostCSRs(client); err != nil {
+	// That transition requires two minor changes
+
+	// Remove the old ClusterRoleBinding for approving if it already exists due to the reasons outlined in the comment below
+	if err := deleteOldApprovalClusterRoleBindingIfExists(client, k8sVersion); err != nil {
 		errs = append(errs, err)
 	}
+	// Upgrade the Bootstrap Tokens' authentication group
+	if err := upgradeBootstrapTokens(client, k8sVersion); err != nil {
+		errs = append(errs, err)
+	}
+	// Upgrade the cluster-info RBAC rules
+	if err := deleteWronglyNamedClusterInfoRBACRules(client, k8sVersion); err != nil {
+		errs = append(errs, err)
+	}
+
+	// Create/update RBAC rules that makes the bootstrap tokens able to post CSRs
+	if err := nodebootstraptoken.AllowBootstrapTokensToPostCSRs(client, k8sVersion); err != nil {
+		errs = append(errs, err)
+	}
+
 	// Create/update RBAC rules that makes the bootstrap tokens able to get their CSRs approved automatically
 	if err := nodebootstraptoken.AutoApproveNodeBootstrapTokens(client, k8sVersion); err != nil {
 		errs = append(errs, err)

cmd/kubeadm/app/phases/upgrade/postupgrade_v17_v18.go

@@ -0,0 +1,152 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package upgrade
+
+import (
+	"bytes"
+	"fmt"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/util/errors"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo"
+	nodebootstraptoken "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node"
+	"k8s.io/kubernetes/pkg/api"
+	bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api"
+	"k8s.io/kubernetes/pkg/util/version"
+)
+
+const (
+	oldClusterInfoRole = "system:bootstrap-signer-clusterinfo"
+)
+
+// deleteOldApprovalClusterRoleBindingIfExists exists because the roleRef of the NodeAutoApproveBootstrapClusterRoleBinding changed between
+// v1.7 and v1.8, and roleRef updates are not possible. So in order to change that binding's roleRef, we have to delete it if it already exists
+// TODO: When the v1.9 cycle starts, we can remove this logic, as the kubeadm v1.9 CLI doesn't support upgrading from v1.7
+func deleteOldApprovalClusterRoleBindingIfExists(client clientset.Interface, k8sVersion *version.Version) error {
+
+	// Gate this upgrade behavior for new clusters above v1.9.0-alpha.3 where this change took place
+	if k8sVersion.AtLeast(constants.MinimumCSRAutoApprovalClusterRolesVersion) {
+
+		err := client.RbacV1beta1().ClusterRoleBindings().Delete(nodebootstraptoken.NodeAutoApproveBootstrapClusterRoleBinding, &metav1.DeleteOptions{})
+		// If the binding was not found, happily continue
+		if apierrors.IsNotFound(err) {
+			return nil
+		}
+		// If an unexpected error occurred, return it
+		if err != nil {
+			return err
+		}
+	}
+	// The binding was successfully deleted
+	return nil
+}
+
+// deleteWronglyNamedClusterInfoRBACRules exists because the cluster-info Role's name changed from "system:bootstrap-signer-clusterinfo" in v1.7 to
+// "kubeadm:bootstrap-signer-clusterinfo" in v1.8. It was incorrectly prefixed "system:" in v1.7
+// The old, incorrectly-named Role should be removed and roleRef updates on the binding are not possible. So in order to change that binding's roleRef,
+// we have to delete it if it already exists
+// TODO: When the v1.9 cycle starts, we can remove this logic, as the kubeadm v1.9 CLI doesn't support upgrading from v1.7
+func deleteWronglyNamedClusterInfoRBACRules(client clientset.Interface, k8sVersion *version.Version) error {
+	// Gate this upgrade behavior for new clusters above v1.8.0-beta.0 where this change took place
+	if k8sVersion.AtLeast(constants.UseEnableBootstrapTokenAuthFlagVersion) {
+
+		if err := removeOldRole(client); err != nil {
+			return err
+		}
+		if err := removeOldRoleBinding(client); err != nil {
+			return err
+		}
+	}
+	// The binding was successfully deleted
+	return nil
+}
+
+func removeOldRole(client clientset.Interface) error {
+	err := client.RbacV1beta1().Roles(metav1.NamespacePublic).Delete(oldClusterInfoRole, &metav1.DeleteOptions{})
+	// If the binding was not found, happily continue
+	if apierrors.IsNotFound(err) {
+		return nil
+	}
+	// If an unexpected error occurred, return it
+	if err != nil {
+		return err
+	}
+	// The role was successfully deleted
+	return nil
+}
+
+func removeOldRoleBinding(client clientset.Interface) error {
+	err := client.RbacV1beta1().RoleBindings(metav1.NamespacePublic).Delete(clusterinfo.BootstrapSignerClusterRoleName, &metav1.DeleteOptions{})
+	// If the binding was not found, happily continue
+	if apierrors.IsNotFound(err) {
+		return nil
+	}
+	// If an unexpected error occurred, return it
+	if err != nil {
+		return err
+	}
+	// The binding was successfully removed
+	return nil
+}
+
+// upgradeBootstrapTokens handles the transition from alpha bootstrap tokens to beta. There isn't much that is changing,
+// but the group that a Bootstrap Token authenticates as changes from "system:bootstrappers" (alpha) in v1.7 to
+// "system:bootstrappers:kubeadm:default-node-token" (beta). To handle this transition correctly, the RBAC bindings earlier
+// bound to "system:bootstrappers" are now bound to "system:bootstrappers:kubeadm:default-node-token". To make v1.7 tokens
+// still valid in v1.8; this code makes sure that all tokens that were used for authentication in v1.7 have the right group
+// bound to it in v1.8.
+// TODO: When the v1.9 cycle starts, we can remove this logic, as the kubeadm v1.9 CLI doesn't support upgrading from v1.7
+func upgradeBootstrapTokens(client clientset.Interface, k8sVersion *version.Version) error {
+
+	// Gate this upgrade behavior for new clusters above v1.8.0-beta.0; where this BT change took place
+	if k8sVersion.AtLeast(constants.UseEnableBootstrapTokenAuthFlagVersion) {
+
+		tokenSelector := fields.SelectorFromSet(
+			map[string]string{
+				api.SecretTypeField: string(bootstrapapi.SecretTypeBootstrapToken),
+			},
+		)
+		listOptions := metav1.ListOptions{
+			FieldSelector: tokenSelector.String(),
+		}
+
+		secrets, err := client.CoreV1().Secrets(metav1.NamespaceSystem).List(listOptions)
+		if err != nil {
+			return fmt.Errorf("failed to list bootstrap tokens: %v", err)
+		}
+
+		errs := []error{}
+		for _, secret := range secrets.Items {
+			// If this Bootstrap Token is used for authentication, the permissions it had in v1.7 should be preserved
+			if bytes.Equal(secret.Data[bootstrapapi.BootstrapTokenUsageAuthentication], []byte("true")) {
+
+				secret.Data[bootstrapapi.BootstrapTokenExtraGroupsKey] = []byte(constants.GetNodeBootstrapTokenAuthGroup(k8sVersion))
+
+				// Update the Bootstrap Token Secret
+				if _, err := client.CoreV1().Secrets(metav1.NamespaceSystem).Update(&secret); err != nil {
+					errs = append(errs, err)
+				}
+			}
+		}
+		return errors.NewAggregate(errs)
+	}
+	return nil
+}