From de2ef8f0c7c0d0909106f4d197674ccf1903625a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Tue, 6 Jun 2017 07:47:18 +0300 Subject: [PATCH] kubeadm: Make kubeadm use the right CSR approver for the right version --- cmd/kubeadm/app/cmd/BUILD | 2 +- cmd/kubeadm/app/cmd/defaults.go | 13 ++++----- cmd/kubeadm/app/constants/BUILD | 5 +++- cmd/kubeadm/app/constants/constants.go | 13 +++++++-- cmd/kubeadm/app/master/BUILD | 1 + cmd/kubeadm/app/master/manifests.go | 16 ++++++----- cmd/kubeadm/app/master/manifests_test.go | 36 +++++++++++++++++++----- cmd/kubeadm/app/master/selfhosted.go | 11 ++++++-- 8 files changed, 67 insertions(+), 30 deletions(-) diff --git a/cmd/kubeadm/app/cmd/BUILD b/cmd/kubeadm/app/cmd/BUILD index 023d9e1116..67c5c0a3fe 100644 --- a/cmd/kubeadm/app/cmd/BUILD +++ b/cmd/kubeadm/app/cmd/BUILD @@ -45,8 +45,8 @@ go_library( "//pkg/printers:go_default_library", "//pkg/util/i18n:go_default_library", "//pkg/util/initsystem:go_default_library", + "//pkg/util/version:go_default_library", "//pkg/version:go_default_library", - "//vendor/github.com/blang/semver:go_default_library", "//vendor/github.com/ghodss/yaml:go_default_library", "//vendor/github.com/renstrom/dedent:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", diff --git a/cmd/kubeadm/app/cmd/defaults.go b/cmd/kubeadm/app/cmd/defaults.go index 2eb3e23ae6..e1de14af4a 100644 --- a/cmd/kubeadm/app/cmd/defaults.go +++ b/cmd/kubeadm/app/cmd/defaults.go @@ -25,12 +25,9 @@ import ( kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" tokenutil "k8s.io/kubernetes/cmd/kubeadm/app/util/token" - - "github.com/blang/semver" + "k8s.io/kubernetes/pkg/util/version" ) -var minK8sVersion = semver.MustParse(kubeadmconstants.MinimumControlPlaneVersion) - func setInitDynamicDefaults(cfg *kubeadmapi.MasterConfiguration) error { // Choose the right address for the API Server to advertise. If the advertise address is localhost or 0.0.0.0, the default interface's IP address is used @@ -48,13 +45,13 @@ func setInitDynamicDefaults(cfg *kubeadmapi.MasterConfiguration) error { } cfg.KubernetesVersion = ver - // Omit the "v" in the beginning, otherwise semver will fail - k8sVersion, err := semver.Parse(cfg.KubernetesVersion[1:]) + // Parse the given kubernetes version and make sure it's higher than the lowest supported + k8sVersion, err := version.ParseSemantic(cfg.KubernetesVersion) if err != nil { return fmt.Errorf("couldn't parse kubernetes version %q: %v", cfg.KubernetesVersion, err) } - if k8sVersion.LT(minK8sVersion) { - return fmt.Errorf("this version of kubeadm only supports deploying clusters with the control plane version >= v%s. Current version: %s", kubeadmconstants.MinimumControlPlaneVersion, cfg.KubernetesVersion) + if k8sVersion.LessThan(kubeadmconstants.MinimumControlPlaneVersion) { + return fmt.Errorf("this version of kubeadm only supports deploying clusters with the control plane version >= %s. Current version: %s", kubeadmconstants.MinimumControlPlaneVersion.String(), cfg.KubernetesVersion) } fmt.Printf("[init] Using Kubernetes version: %s\n", cfg.KubernetesVersion) diff --git a/cmd/kubeadm/app/constants/BUILD b/cmd/kubeadm/app/constants/BUILD index 0ad7f34d74..240377285f 100644 --- a/cmd/kubeadm/app/constants/BUILD +++ b/cmd/kubeadm/app/constants/BUILD @@ -11,7 +11,10 @@ go_library( name = "go_default_library", srcs = ["constants.go"], tags = ["automanaged"], - deps = ["//vendor/k8s.io/client-go/pkg/api/v1:go_default_library"], + deps = [ + "//pkg/util/version:go_default_library", + "//vendor/k8s.io/client-go/pkg/api/v1:go_default_library", + ], ) filegroup( diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index b2c35f1e4a..58b453a845 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -21,6 +21,7 @@ import ( "time" "k8s.io/client-go/pkg/api/v1" + "k8s.io/kubernetes/pkg/util/version" ) const ( @@ -58,9 +59,6 @@ const ( ControllerManagerKubeConfigFileName = "controller-manager.conf" SchedulerKubeConfigFileName = "scheduler.conf" - // Important: a "v"-prefix shouldn't exist here; semver doesn't allow that - MinimumControlPlaneVersion = "1.6.0-beta.3" - // Some well-known users and groups in the core Kubernetes authorization system ControllerManagerUser = "system:kube-controller-manager" @@ -109,4 +107,13 @@ var ( // DefaultTokenUsages specifies the default functions a token will get DefaultTokenUsages = []string{"signing", "authentication"} + + // MinimumControlPlaneVersion specifies the minimum control plane version kubeadm can deploy + MinimumControlPlaneVersion = version.MustParseSemantic("v1.6.0") + + // MinimumCSRSARApproverVersion specifies the minimum kubernetes version that can be used for enabling the new-in-v1.7 CSR approver based on a SubjectAccessReview + MinimumCSRSARApproverVersion = version.MustParseSemantic("v1.7.0-beta.0") + + // MinimumAPIAggregationVersion specifies the minimum kubernetes version that can be used enabling the API aggregation in the apiserver and the front proxy flags + MinimumAPIAggregationVersion = version.MustParseSemantic("v1.7.0-alpha.1") ) diff --git a/cmd/kubeadm/app/master/BUILD b/cmd/kubeadm/app/master/BUILD index 5b77e80dd6..0fb6f9adcc 100644 --- a/cmd/kubeadm/app/master/BUILD +++ b/cmd/kubeadm/app/master/BUILD @@ -22,6 +22,7 @@ go_library( "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/images:go_default_library", "//cmd/kubeadm/app/util/kubeconfig:go_default_library", + "//pkg/bootstrap/api:go_default_library", "//pkg/kubeapiserver/authorizer/modes:go_default_library", "//pkg/kubectl/cmd/util:go_default_library", "//pkg/util/version:go_default_library", diff --git a/cmd/kubeadm/app/master/manifests.go b/cmd/kubeadm/app/master/manifests.go index f438f2775a..5752ca984b 100644 --- a/cmd/kubeadm/app/master/manifests.go +++ b/cmd/kubeadm/app/master/manifests.go @@ -33,6 +33,7 @@ import ( kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/images" + bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/util/version" @@ -53,10 +54,6 @@ const ( kubeProxy = "kube-proxy" ) -var ( - v170 = version.MustParseSemantic("v1.7.0-alpha.0") -) - // WriteStaticPodManifests builds manifest objects based on user provided configuration and then dumps it to disk // where kubelet will pick and schedule them. func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration) error { @@ -97,7 +94,7 @@ func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration) error { kubeControllerManager: componentPod(api.Container{ Name: kubeControllerManager, Image: images.GetCoreImage(images.KubeControllerManagerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage), - Command: getControllerManagerCommand(cfg, false), + Command: getControllerManagerCommand(cfg, false, k8sVersion), VolumeMounts: volumeMounts, LivenessProbe: componentProbe(10252, "/healthz", api.URISchemeHTTP), Resources: componentResources("200m"), @@ -349,7 +346,7 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool, k "requestheader-client-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertName), "requestheader-allowed-names": "front-proxy-client", } - if k8sVersion.AtLeast(v170) { + if k8sVersion.AtLeast(kubeadmconstants.MinimumAPIAggregationVersion) { // add options which allow the kube-apiserver to act as a front-proxy to aggregated API servers defaultArguments["proxy-client-cert-file"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientCertName) defaultArguments["proxy-client-key-file"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientKeyName) @@ -409,7 +406,7 @@ func getEtcdCommand(cfg *kubeadmapi.MasterConfiguration) []string { return command } -func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool) []string { +func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool, k8sVersion *version.Version) []string { var command []string // self-hosted controller-manager needs to wait on a lock @@ -428,6 +425,11 @@ func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted "use-service-account-credentials": "true", "controllers": "*,bootstrapsigner,tokencleaner", } + if k8sVersion.LessThan(kubeadmconstants.MinimumCSRSARApproverVersion) { + // enable the former CSR group approver for v1.6 clusters. + // TODO(luxas): Remove this once we're targeting v1.8 at HEAD + defaultArguments["insecure-experimental-approve-all-kubelet-csrs-for-group"] = bootstrapapi.BootstrapGroup + } command = getComponentBaseCommand(controllerManager) command = append(command, getExtraParameters(cfg.ControllerManagerExtraArgs, defaultArguments)...) diff --git a/cmd/kubeadm/app/master/manifests_test.go b/cmd/kubeadm/app/master/manifests_test.go index de8cf267ad..6be970f32d 100644 --- a/cmd/kubeadm/app/master/manifests_test.go +++ b/cmd/kubeadm/app/master/manifests_test.go @@ -667,7 +667,8 @@ func TestGetControllerManagerCommand(t *testing.T) { }{ { cfg: &kubeadmapi.MasterConfiguration{ - CertificatesDir: testCertsDir, + CertificatesDir: testCertsDir, + KubernetesVersion: "v1.7.0", }, expected: []string{ "kube-controller-manager", @@ -684,8 +685,28 @@ func TestGetControllerManagerCommand(t *testing.T) { }, { cfg: &kubeadmapi.MasterConfiguration{ - CloudProvider: "foo", - CertificatesDir: testCertsDir, + CertificatesDir: testCertsDir, + KubernetesVersion: "v1.6.4", + }, + expected: []string{ + "kube-controller-manager", + "--address=127.0.0.1", + "--leader-elect=true", + "--kubeconfig=" + kubeadmapi.GlobalEnvParams.KubernetesDir + "/controller-manager.conf", + "--root-ca-file=" + testCertsDir + "/ca.crt", + "--service-account-private-key-file=" + testCertsDir + "/sa.key", + "--cluster-signing-cert-file=" + testCertsDir + "/ca.crt", + "--cluster-signing-key-file=" + testCertsDir + "/ca.key", + "--use-service-account-credentials=true", + "--controllers=*,bootstrapsigner,tokencleaner", + "--insecure-experimental-approve-all-kubelet-csrs-for-group=system:bootstrappers", + }, + }, + { + cfg: &kubeadmapi.MasterConfiguration{ + CloudProvider: "foo", + CertificatesDir: testCertsDir, + KubernetesVersion: "v1.7.0", }, expected: []string{ "kube-controller-manager", @@ -703,8 +724,9 @@ func TestGetControllerManagerCommand(t *testing.T) { }, { cfg: &kubeadmapi.MasterConfiguration{ - Networking: kubeadm.Networking{PodSubnet: "bar"}, - CertificatesDir: testCertsDir, + Networking: kubeadm.Networking{PodSubnet: "bar"}, + CertificatesDir: testCertsDir, + KubernetesVersion: "v1.7.0", }, expected: []string{ "kube-controller-manager", @@ -724,7 +746,7 @@ func TestGetControllerManagerCommand(t *testing.T) { } for _, rt := range tests { - actual := getControllerManagerCommand(rt.cfg, false) + actual := getControllerManagerCommand(rt.cfg, false, version.MustParseSemantic(rt.cfg.KubernetesVersion)) sort.Strings(actual) sort.Strings(rt.expected) if !reflect.DeepEqual(actual, rt.expected) { @@ -935,7 +957,7 @@ func TestVersionCompare(t *testing.T) { "v1.7.1", } for _, v := range versions { - if !version.MustParseSemantic(v).AtLeast(v170) { + if !version.MustParseSemantic(v).AtLeast(kubeadmconstants.MinimumAPIAggregationVersion) { t.Errorf("err") } } diff --git a/cmd/kubeadm/app/master/selfhosted.go b/cmd/kubeadm/app/master/selfhosted.go index d3aea59f50..8331ee3676 100644 --- a/cmd/kubeadm/app/master/selfhosted.go +++ b/cmd/kubeadm/app/master/selfhosted.go @@ -124,7 +124,12 @@ func launchSelfHostedAPIServer(cfg *kubeadmapi.MasterConfiguration, client *clie func launchSelfHostedControllerManager(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error { start := time.Now() - ctrlMgr := getControllerManagerDeployment(cfg, volumes, volumeMounts) + kubeVersion, err := version.ParseSemantic(cfg.KubernetesVersion) + if err != nil { + return err + } + + ctrlMgr := getControllerManagerDeployment(cfg, volumes, volumeMounts, kubeVersion) if _, err := client.Extensions().Deployments(metav1.NamespaceSystem).Create(&ctrlMgr); err != nil { return fmt.Errorf("failed to create self-hosted %q deployment [%v]", kubeControllerManager, err) } @@ -232,7 +237,7 @@ func getAPIServerDS(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, vo return ds } -func getControllerManagerDeployment(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, volumeMounts []v1.VolumeMount) ext.Deployment { +func getControllerManagerDeployment(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, volumeMounts []v1.VolumeMount, kubeVersion *version.Version) ext.Deployment { d := ext.Deployment{ TypeMeta: metav1.TypeMeta{ APIVersion: "extensions/v1beta1", @@ -268,7 +273,7 @@ func getControllerManagerDeployment(cfg *kubeadmapi.MasterConfiguration, volumes { Name: "self-hosted-" + kubeControllerManager, Image: images.GetCoreImage(images.KubeControllerManagerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage), - Command: getControllerManagerCommand(cfg, true), + Command: getControllerManagerCommand(cfg, true, kubeVersion), VolumeMounts: volumeMounts, LivenessProbe: componentProbe(10252, "/healthz", v1.URISchemeHTTP), Resources: componentResources("200m"),