diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 509e05afc2..0ceaf7f19d 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -379,7 +379,7 @@ func (i *Init) Run(out io.Writer) error { return err } // Create RBAC rules that makes the bootstrap tokens able to post CSRs - if err := nodebootstraptokenphase.AllowBootstrapTokensToPostCSRs(client); err != nil { + if err := nodebootstraptokenphase.AllowBootstrapTokensToPostCSRs(client, k8sVersion); err != nil { return err } // Create RBAC rules that makes the bootstrap tokens able to get their CSRs approved automatically diff --git a/cmd/kubeadm/app/cmd/phases/bootstraptoken.go b/cmd/kubeadm/app/cmd/phases/bootstraptoken.go index 40a5f1d0a0..f398838f20 100644 --- a/cmd/kubeadm/app/cmd/phases/bootstraptoken.go +++ b/cmd/kubeadm/app/cmd/phases/bootstraptoken.go @@ -100,7 +100,10 @@ func NewSubCmdNodeBootstrapTokenPostCSRs(kubeConfigFile *string) *cobra.Command client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile) kubeadmutil.CheckErr(err) - err = node.AllowBootstrapTokensToPostCSRs(client) + clusterVersion, err := getClusterVersion(client) + kubeadmutil.CheckErr(err) + + err = node.AllowBootstrapTokensToPostCSRs(client, clusterVersion) kubeadmutil.CheckErr(err) }, } diff --git a/cmd/kubeadm/app/constants/BUILD b/cmd/kubeadm/app/constants/BUILD index 1e7597d46f..5b6884926a 100644 --- a/cmd/kubeadm/app/constants/BUILD +++ b/cmd/kubeadm/app/constants/BUILD @@ -32,4 +32,5 @@ go_test( name = "go_default_test", srcs = ["constants_test.go"], library = ":go_default_library", + deps = ["//pkg/util/version:go_default_library"], ) diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index ea0ccd705e..4e04af2d23 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -122,9 +122,11 @@ const ( // KubeConfigVolumeName specifies the name for the Volume that is used for injecting the kubeconfig to talk securely to the api server for a control plane component if applicable KubeConfigVolumeName = "kubeconfig" - // NodeBootstrapTokenAuthGroup specifies which group a Node Bootstrap Token should be authenticated in - // TODO: This should be changed in the v1.8 dev cycle to a node-BT-specific group instead of the generic Bootstrap Token group that is used now - NodeBootstrapTokenAuthGroup = "system:bootstrappers" + // V17NodeBootstrapTokenAuthGroup specifies which group a Node Bootstrap Token should be authenticated in, in v1.7 + V17NodeBootstrapTokenAuthGroup = "system:bootstrappers" + + // V18NodeBootstrapTokenAuthGroup specifies which group a Node Bootstrap Token should be authenticated in, in v1.8 + V18NodeBootstrapTokenAuthGroup = "system:bootstrappers:kubeadm:default-node-token" // DefaultCIImageRepository points to image registry where CI uploads images from ci-cross build job DefaultCIImageRepository = "gcr.io/kubernetes-ci-images" @@ -198,3 +200,11 @@ func CreateTempDirForKubeadm(dirName string) (string, error) { } return tempDir, nil } + +// GetNodeBootstrapTokenAuthGroup gets the bootstrap token auth group conditionally based on version +func GetNodeBootstrapTokenAuthGroup(k8sVersion *version.Version) string { + if k8sVersion.AtLeast(UseEnableBootstrapTokenAuthFlagVersion) { + return V18NodeBootstrapTokenAuthGroup + } + return V17NodeBootstrapTokenAuthGroup +} diff --git a/cmd/kubeadm/app/constants/constants_test.go b/cmd/kubeadm/app/constants/constants_test.go index 29cffa2abb..70f491b72f 100644 --- a/cmd/kubeadm/app/constants/constants_test.go +++ b/cmd/kubeadm/app/constants/constants_test.go @@ -18,6 +18,8 @@ package constants import ( "testing" + + "k8s.io/kubernetes/pkg/util/version" ) func TestGetStaticPodDirectory(t *testing.T) { @@ -110,3 +112,48 @@ func TestAddSelfHostedPrefix(t *testing.T) { } } } + +func TestGetNodeBootstrapTokenAuthGroup(t *testing.T) { + var tests = []struct { + k8sVersion, expected string + }{ + { + k8sVersion: "v1.7.0", + expected: "system:bootstrappers", + }, + { + k8sVersion: "v1.7.8", + expected: "system:bootstrappers", + }, + { + k8sVersion: "v1.8.0-alpha.3", + expected: "system:bootstrappers", + }, + { + k8sVersion: "v1.8.0-beta.0", + expected: "system:bootstrappers:kubeadm:default-node-token", + }, + { + k8sVersion: "v1.8.0-rc.1", + expected: "system:bootstrappers:kubeadm:default-node-token", + }, + { + k8sVersion: "v1.8.0", + expected: "system:bootstrappers:kubeadm:default-node-token", + }, + { + k8sVersion: "v1.8.9", + expected: "system:bootstrappers:kubeadm:default-node-token", + }, + } + for _, rt := range tests { + actual := GetNodeBootstrapTokenAuthGroup(version.MustParseSemantic(rt.k8sVersion)) + if actual != rt.expected { + t.Errorf( + "failed GetNodeBootstrapTokenAuthGroup:\n\texpected: %s\n\t actual: %s", + rt.expected, + actual, + ) + } + } +} diff --git a/cmd/kubeadm/app/phases/bootstraptoken/node/tlsbootstrap.go b/cmd/kubeadm/app/phases/bootstraptoken/node/tlsbootstrap.go index 079204a015..82125f5604 100644 --- a/cmd/kubeadm/app/phases/bootstraptoken/node/tlsbootstrap.go +++ b/cmd/kubeadm/app/phases/bootstraptoken/node/tlsbootstrap.go @@ -38,12 +38,12 @@ const ( // CSRAutoApprovalClusterRoleName defines the name of the auto-bootstrapped ClusterRole for making the csrapprover controller auto-approve the CSR // TODO: This value should be defined in an other, generic authz package instead of here CSRAutoApprovalClusterRoleName = "system:certificates.k8s.io:certificatesigningrequests:nodeclient" - // NodeAutoApproveBootstrap defines the name of the ClusterRoleBinding that makes the csrapprover approve node CSRs - NodeAutoApproveBootstrap = "kubeadm:node-autoapprove-bootstrap" + // NodeAutoApproveBootstrapClusterRoleBinding defines the name of the ClusterRoleBinding that makes the csrapprover approve node CSRs + NodeAutoApproveBootstrapClusterRoleBinding = "kubeadm:node-autoapprove-bootstrap" ) // AllowBootstrapTokensToPostCSRs creates RBAC rules in a way the makes Node Bootstrap Tokens able to post CSRs -func AllowBootstrapTokensToPostCSRs(client clientset.Interface) error { +func AllowBootstrapTokensToPostCSRs(client clientset.Interface, k8sVersion *version.Version) error { fmt.Println("[bootstraptoken] Configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials") @@ -59,7 +59,7 @@ func AllowBootstrapTokensToPostCSRs(client clientset.Interface) error { Subjects: []rbac.Subject{ { Kind: rbac.GroupKind, - Name: constants.NodeBootstrapTokenAuthGroup, + Name: constants.GetNodeBootstrapTokenAuthGroup(k8sVersion), }, }, }) @@ -89,7 +89,7 @@ func AutoApproveNodeBootstrapTokens(client clientset.Interface, k8sVersion *vers // Always create this kubeadm-specific binding though return apiclient.CreateOrUpdateClusterRoleBinding(client, &rbac.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ - Name: NodeAutoApproveBootstrap, + Name: NodeAutoApproveBootstrapClusterRoleBinding, }, RoleRef: rbac.RoleRef{ APIGroup: rbac.GroupName, @@ -99,7 +99,7 @@ func AutoApproveNodeBootstrapTokens(client clientset.Interface, k8sVersion *vers Subjects: []rbac.Subject{ { Kind: "Group", - Name: constants.NodeBootstrapTokenAuthGroup, + Name: constants.GetNodeBootstrapTokenAuthGroup(k8sVersion), }, }, }) diff --git a/cmd/kubeadm/app/phases/upgrade/BUILD b/cmd/kubeadm/app/phases/upgrade/BUILD index bcbad557ea..f4c3fcfe4f 100644 --- a/cmd/kubeadm/app/phases/upgrade/BUILD +++ b/cmd/kubeadm/app/phases/upgrade/BUILD @@ -8,6 +8,7 @@ go_library( "health.go", "policy.go", "postupgrade.go", + "postupgrade_v17_v18.go", "prepull.go", "selfhosted.go", "staticpods.go", @@ -32,12 +33,14 @@ go_library( "//cmd/kubeadm/app/util/apiclient:go_default_library", "//cmd/kubeadm/app/util/config:go_default_library", "//pkg/api:go_default_library", + "//pkg/bootstrap/api:go_default_library", "//pkg/util/version:go_default_library", "//pkg/version:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/api/extensions/v1beta1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade.go b/cmd/kubeadm/app/phases/upgrade/postupgrade.go index 1b19d74925..0751e9a6bc 100644 --- a/cmd/kubeadm/app/phases/upgrade/postupgrade.go +++ b/cmd/kubeadm/app/phases/upgrade/postupgrade.go @@ -41,10 +41,27 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.MasterC errs = append(errs, err) } - // Create/update RBAC rules that makes the bootstrap tokens able to post CSRs - if err := nodebootstraptoken.AllowBootstrapTokensToPostCSRs(client); err != nil { + // Handle Bootstrap Tokens graduating to from alpha to beta in the v1.7 -> v1.8 upgrade + // That transition requires two minor changes + + // Remove the old ClusterRoleBinding for approving if it already exists due to the reasons outlined in the comment below + if err := deleteOldApprovalClusterRoleBindingIfExists(client, k8sVersion); err != nil { errs = append(errs, err) } + // Upgrade the Bootstrap Tokens' authentication group + if err := upgradeBootstrapTokens(client, k8sVersion); err != nil { + errs = append(errs, err) + } + // Upgrade the cluster-info RBAC rules + if err := deleteWronglyNamedClusterInfoRBACRules(client, k8sVersion); err != nil { + errs = append(errs, err) + } + + // Create/update RBAC rules that makes the bootstrap tokens able to post CSRs + if err := nodebootstraptoken.AllowBootstrapTokensToPostCSRs(client, k8sVersion); err != nil { + errs = append(errs, err) + } + // Create/update RBAC rules that makes the bootstrap tokens able to get their CSRs approved automatically if err := nodebootstraptoken.AutoApproveNodeBootstrapTokens(client, k8sVersion); err != nil { errs = append(errs, err) diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade_v17_v18.go b/cmd/kubeadm/app/phases/upgrade/postupgrade_v17_v18.go new file mode 100644 index 0000000000..5465a24b41 --- /dev/null +++ b/cmd/kubeadm/app/phases/upgrade/postupgrade_v17_v18.go @@ -0,0 +1,152 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "bytes" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/util/errors" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo" + nodebootstraptoken "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node" + "k8s.io/kubernetes/pkg/api" + bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" + "k8s.io/kubernetes/pkg/util/version" +) + +const ( + oldClusterInfoRole = "system:bootstrap-signer-clusterinfo" +) + +// deleteOldApprovalClusterRoleBindingIfExists exists because the roleRef of the NodeAutoApproveBootstrapClusterRoleBinding changed between +// v1.7 and v1.8, and roleRef updates are not possible. So in order to change that binding's roleRef, we have to delete it if it already exists +// TODO: When the v1.9 cycle starts, we can remove this logic, as the kubeadm v1.9 CLI doesn't support upgrading from v1.7 +func deleteOldApprovalClusterRoleBindingIfExists(client clientset.Interface, k8sVersion *version.Version) error { + + // Gate this upgrade behavior for new clusters above v1.9.0-alpha.3 where this change took place + if k8sVersion.AtLeast(constants.MinimumCSRAutoApprovalClusterRolesVersion) { + + err := client.RbacV1beta1().ClusterRoleBindings().Delete(nodebootstraptoken.NodeAutoApproveBootstrapClusterRoleBinding, &metav1.DeleteOptions{}) + // If the binding was not found, happily continue + if apierrors.IsNotFound(err) { + return nil + } + // If an unexpected error occurred, return it + if err != nil { + return err + } + } + // The binding was successfully deleted + return nil +} + +// deleteWronglyNamedClusterInfoRBACRules exists because the cluster-info Role's name changed from "system:bootstrap-signer-clusterinfo" in v1.7 to +// "kubeadm:bootstrap-signer-clusterinfo" in v1.8. It was incorrectly prefixed "system:" in v1.7 +// The old, incorrectly-named Role should be removed and roleRef updates on the binding are not possible. So in order to change that binding's roleRef, +// we have to delete it if it already exists +// TODO: When the v1.9 cycle starts, we can remove this logic, as the kubeadm v1.9 CLI doesn't support upgrading from v1.7 +func deleteWronglyNamedClusterInfoRBACRules(client clientset.Interface, k8sVersion *version.Version) error { + // Gate this upgrade behavior for new clusters above v1.8.0-beta.0 where this change took place + if k8sVersion.AtLeast(constants.UseEnableBootstrapTokenAuthFlagVersion) { + + if err := removeOldRole(client); err != nil { + return err + } + if err := removeOldRoleBinding(client); err != nil { + return err + } + } + // The binding was successfully deleted + return nil +} + +func removeOldRole(client clientset.Interface) error { + err := client.RbacV1beta1().Roles(metav1.NamespacePublic).Delete(oldClusterInfoRole, &metav1.DeleteOptions{}) + // If the binding was not found, happily continue + if apierrors.IsNotFound(err) { + return nil + } + // If an unexpected error occurred, return it + if err != nil { + return err + } + // The role was successfully deleted + return nil +} + +func removeOldRoleBinding(client clientset.Interface) error { + err := client.RbacV1beta1().RoleBindings(metav1.NamespacePublic).Delete(clusterinfo.BootstrapSignerClusterRoleName, &metav1.DeleteOptions{}) + // If the binding was not found, happily continue + if apierrors.IsNotFound(err) { + return nil + } + // If an unexpected error occurred, return it + if err != nil { + return err + } + // The binding was successfully removed + return nil +} + +// upgradeBootstrapTokens handles the transition from alpha bootstrap tokens to beta. There isn't much that is changing, +// but the group that a Bootstrap Token authenticates as changes from "system:bootstrappers" (alpha) in v1.7 to +// "system:bootstrappers:kubeadm:default-node-token" (beta). To handle this transition correctly, the RBAC bindings earlier +// bound to "system:bootstrappers" are now bound to "system:bootstrappers:kubeadm:default-node-token". To make v1.7 tokens +// still valid in v1.8; this code makes sure that all tokens that were used for authentication in v1.7 have the right group +// bound to it in v1.8. +// TODO: When the v1.9 cycle starts, we can remove this logic, as the kubeadm v1.9 CLI doesn't support upgrading from v1.7 +func upgradeBootstrapTokens(client clientset.Interface, k8sVersion *version.Version) error { + + // Gate this upgrade behavior for new clusters above v1.8.0-beta.0; where this BT change took place + if k8sVersion.AtLeast(constants.UseEnableBootstrapTokenAuthFlagVersion) { + + tokenSelector := fields.SelectorFromSet( + map[string]string{ + api.SecretTypeField: string(bootstrapapi.SecretTypeBootstrapToken), + }, + ) + listOptions := metav1.ListOptions{ + FieldSelector: tokenSelector.String(), + } + + secrets, err := client.CoreV1().Secrets(metav1.NamespaceSystem).List(listOptions) + if err != nil { + return fmt.Errorf("failed to list bootstrap tokens: %v", err) + } + + errs := []error{} + for _, secret := range secrets.Items { + // If this Bootstrap Token is used for authentication, the permissions it had in v1.7 should be preserved + if bytes.Equal(secret.Data[bootstrapapi.BootstrapTokenUsageAuthentication], []byte("true")) { + + secret.Data[bootstrapapi.BootstrapTokenExtraGroupsKey] = []byte(constants.GetNodeBootstrapTokenAuthGroup(k8sVersion)) + + // Update the Bootstrap Token Secret + if _, err := client.CoreV1().Secrets(metav1.NamespaceSystem).Update(&secret); err != nil { + errs = append(errs, err) + } + } + } + return errors.NewAggregate(errs) + } + return nil +}