From 8654217b121c8e5050c58c59a6dd3f814b660b0d Mon Sep 17 00:00:00 2001 From: xilabao Date: Mon, 27 Feb 2017 08:41:34 +0800 Subject: [PATCH] Make the CLI arguments for the control plane overridable --- cmd/kubeadm/app/apis/kubeadm/types.go | 4 + .../app/apis/kubeadm/v1alpha1/types.go | 4 + cmd/kubeadm/app/constants/constants.go | 3 + cmd/kubeadm/app/master/manifests.go | 98 ++++++++++------ cmd/kubeadm/app/master/manifests_test.go | 105 ++++++++++++------ 5 files changed, 143 insertions(+), 71 deletions(-) diff --git a/cmd/kubeadm/app/apis/kubeadm/types.go b/cmd/kubeadm/app/apis/kubeadm/types.go index 188e44ce86..8e4a3be87a 100644 --- a/cmd/kubeadm/app/apis/kubeadm/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/types.go @@ -44,6 +44,10 @@ type MasterConfiguration struct { // controller manager are managed by Kubernetes itself. This option is likely to // become the default in the future. SelfHosted bool + + APIServerExtraArgs map[string]string + ControllerManagerExtraArgs map[string]string + SchedulerExtraArgs map[string]string } type API struct { diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go index b662f6120a..99131d607b 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go @@ -34,6 +34,10 @@ type MasterConfiguration struct { // controller manager are managed by Kubernetes itself. This option is likely to // become the default in the future. SelfHosted bool `json:"selfHosted"` + + APIServerExtraArgs map[string]string `json:"apiServerExtraArgs"` + ControllerManagerExtraArgs map[string]string `json:"controllerManagerExtraArgs"` + SchedulerExtraArgs map[string]string `json:"schedulerExtraArgs"` } type API struct { diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index 36e8fa9de5..b92d3efaff 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -97,6 +97,9 @@ const ( // MinExternalEtcdVersion indicates minimum external etcd version which kubeadm supports MinExternalEtcdVersion = "3.0.14" + + // DefaultAdmissionControl specifies the default admission control options that will be used + DefaultAdmissionControl = "NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds" ) var ( diff --git a/cmd/kubeadm/app/master/manifests.go b/cmd/kubeadm/app/master/manifests.go index 8118d9ff70..e86a3080a5 100644 --- a/cmd/kubeadm/app/master/manifests.go +++ b/cmd/kubeadm/app/master/manifests.go @@ -304,30 +304,33 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool) [ command = []string{"/usr/bin/flock", "--exclusive", "--timeout=30", "/var/lock/api-server.lock"} } - command = append(getComponentBaseCommand(apiServer), - "--insecure-bind-address=127.0.0.1", - "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds", - "--service-cluster-ip-range="+cfg.Networking.ServiceSubnet, - "--service-account-key-file="+getCertFilePath(kubeadmconstants.ServiceAccountPublicKeyName), - "--client-ca-file="+getCertFilePath(kubeadmconstants.CACertName), - "--tls-cert-file="+getCertFilePath(kubeadmconstants.APIServerCertName), - "--tls-private-key-file="+getCertFilePath(kubeadmconstants.APIServerKeyName), - "--kubelet-client-certificate="+getCertFilePath(kubeadmconstants.APIServerKubeletClientCertName), - "--kubelet-client-key="+getCertFilePath(kubeadmconstants.APIServerKubeletClientKeyName), - "--token-auth-file="+kubeadmapi.GlobalEnvParams.HostPKIPath+"/tokens.csv", - fmt.Sprintf("--secure-port=%d", cfg.API.Port), - "--allow-privileged", - "--storage-backend=etcd3", - "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", + defaultArguments := map[string]string{ + "insecure-bind-address": "127.0.0.1", + "admission-control": kubeadmconstants.DefaultAdmissionControl, + "service-cluster-ip-range": cfg.Networking.ServiceSubnet, + "service-account-key-file": getCertFilePath(kubeadmconstants.ServiceAccountPublicKeyName), + "client-ca-file": getCertFilePath(kubeadmconstants.CACertName), + "tls-cert-file": getCertFilePath(kubeadmconstants.APIServerCertName), + "tls-private-key-file": getCertFilePath(kubeadmconstants.APIServerKeyName), + "kubelet-client-certificate": getCertFilePath(kubeadmconstants.APIServerKubeletClientCertName), + "kubelet-client-key": getCertFilePath(kubeadmconstants.APIServerKubeletClientKeyName), + "token-auth-file": path.Join(kubeadmapi.GlobalEnvParams.HostPKIPath, kubeadmconstants.CSVTokenFileName), + "secure-port": fmt.Sprintf("%d", cfg.API.Port), + "allow-privileged": "true", + "storage-backend": "etcd3", + "kubelet-preferred-address-types": "InternalIP,ExternalIP,Hostname", + // add options to configure the front proxy. Without the generated client cert, this will never be useable // so add it unconditionally with recommended values - "--requestheader-username-headers=X-Remote-User", - "--requestheader-group-headers=X-Remote-Group", - "--requestheader-extra-headers-prefix=X-Remote-Extra-", - "--requestheader-client-ca-file="+getCertFilePath(kubeadmconstants.FrontProxyCACertName), - "--requestheader-allowed-names=front-proxy-client", - ) + "requestheader-username-headers": "X-Remote-User", + "requestheader-group-headers": "X-Remote-Group", + "requestheader-extra-headers-prefix": "X-Remote-Extra-", + "requestheader-client-ca-file": getCertFilePath(kubeadmconstants.FrontProxyCACertName), + "requestheader-allowed-names": "front-proxy-client", + } + command = getComponentBaseCommand(apiServer) + command = append(command, getExtraParameters(cfg.APIServerExtraArgs, defaultArguments)...) command = append(command, getAuthzParameters(cfg.AuthorizationMode)...) // Use first address we are given @@ -376,17 +379,20 @@ func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted command = []string{"/usr/bin/flock", "--exclusive", "--timeout=30", "/var/lock/controller-manager.lock"} } - command = append(getComponentBaseCommand(controllerManager), - "--address=127.0.0.1", - "--leader-elect", - "--kubeconfig="+path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.ControllerManagerKubeConfigFileName), - "--root-ca-file="+getCertFilePath(kubeadmconstants.CACertName), - "--service-account-private-key-file="+getCertFilePath(kubeadmconstants.ServiceAccountPrivateKeyName), - "--cluster-signing-cert-file="+getCertFilePath(kubeadmconstants.CACertName), - "--cluster-signing-key-file="+getCertFilePath(kubeadmconstants.CAKeyName), - "--insecure-experimental-approve-all-kubelet-csrs-for-group="+kubeadmconstants.CSVTokenBootstrapGroup, - "--use-service-account-credentials", - ) + defaultArguments := map[string]string{ + "address": "127.0.0.1", + "leader-elect": "true", + "kubeconfig": path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.ControllerManagerKubeConfigFileName), + "root-ca-file": getCertFilePath(kubeadmconstants.CACertName), + "service-account-private-key-file": getCertFilePath(kubeadmconstants.ServiceAccountPrivateKeyName), + "cluster-signing-cert-file": getCertFilePath(kubeadmconstants.CACertName), + "cluster-signing-key-file": getCertFilePath(kubeadmconstants.CAKeyName), + "insecure-experimental-approve-all-kubelet-csrs-for-group": kubeadmconstants.CSVTokenBootstrapGroup, + "use-service-account-credentials": "true", + } + + command = getComponentBaseCommand(controllerManager) + command = append(command, getExtraParameters(cfg.ControllerManagerExtraArgs, defaultArguments)...) if cfg.CloudProvider != "" { command = append(command, "--cloud-provider="+cfg.CloudProvider) @@ -414,11 +420,14 @@ func getSchedulerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool) [ command = []string{"/usr/bin/flock", "--exclusive", "--timeout=30", "/var/lock/api-server.lock"} } - command = append(getComponentBaseCommand(scheduler), - "--address=127.0.0.1", - "--leader-elect", - "--kubeconfig="+path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.SchedulerKubeConfigFileName), - ) + defaultArguments := map[string]string{ + "address": "127.0.0.1", + "leader-elect": "true", + "kubeconfig": path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.SchedulerKubeConfigFileName), + } + + command = getComponentBaseCommand(scheduler) + command = append(command, getExtraParameters(cfg.SchedulerExtraArgs, defaultArguments)...) return command } @@ -472,3 +481,18 @@ func getAuthzParameters(authzMode string) []string { } return command } + +func getExtraParameters(overrides map[string]string, defaults map[string]string) []string { + var command []string + for k, v := range overrides { + if len(v) > 0 { + command = append(command, fmt.Sprintf("--%s=%s", k, v)) + } + } + for k, v := range defaults { + if _, overrideExists := overrides[k]; !overrideExists { + command = append(command, fmt.Sprintf("--%s=%s", k, v)) + } + } + return command +} diff --git a/cmd/kubeadm/app/master/manifests_test.go b/cmd/kubeadm/app/master/manifests_test.go index c9e56c8f6a..2f47173820 100644 --- a/cmd/kubeadm/app/master/manifests_test.go +++ b/cmd/kubeadm/app/master/manifests_test.go @@ -20,6 +20,8 @@ import ( "fmt" "io/ioutil" "os" + "reflect" + "sort" "testing" "k8s.io/apimachinery/pkg/util/intstr" @@ -380,7 +382,7 @@ func TestGetAPIServerCommand(t *testing.T) { "--kubelet-client-key=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver-kubelet-client.key", "--token-auth-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/tokens.csv", fmt.Sprintf("--secure-port=%d", 123), - "--allow-privileged", + "--allow-privileged=true", "--storage-backend=etcd3", "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", "--requestheader-username-headers=X-Remote-User", @@ -410,7 +412,7 @@ func TestGetAPIServerCommand(t *testing.T) { "--kubelet-client-key=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver-kubelet-client.key", "--token-auth-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/tokens.csv", fmt.Sprintf("--secure-port=%d", 123), - "--allow-privileged", + "--allow-privileged=true", "--storage-backend=etcd3", "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", "--requestheader-username-headers=X-Remote-User", @@ -442,7 +444,7 @@ func TestGetAPIServerCommand(t *testing.T) { "--kubelet-client-key=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver-kubelet-client.key", "--token-auth-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/tokens.csv", fmt.Sprintf("--secure-port=%d", 123), - "--allow-privileged", + "--allow-privileged=true", "--storage-backend=etcd3", "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", "--requestheader-username-headers=X-Remote-User", @@ -460,14 +462,10 @@ func TestGetAPIServerCommand(t *testing.T) { for _, rt := range tests { actual := getAPIServerCommand(rt.cfg, false) - for i := range actual { - if actual[i] != rt.expected[i] { - t.Errorf( - "failed getAPIServerCommand:\n\texpected: %s\n\t actual: %s", - rt.expected[i], - actual[i], - ) - } + sort.Strings(actual) + sort.Strings(rt.expected) + if !reflect.DeepEqual(actual, rt.expected) { + t.Errorf("failed getAPIServerCommand:\nexpected:\n%v\nsaw:\n%v", rt.expected, actual) } } } @@ -482,14 +480,14 @@ func TestGetControllerManagerCommand(t *testing.T) { expected: []string{ "kube-controller-manager", "--address=127.0.0.1", - "--leader-elect", + "--leader-elect=true", "--kubeconfig=" + kubeadmapi.GlobalEnvParams.KubernetesDir + "/controller-manager.conf", "--root-ca-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.crt", "--service-account-private-key-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/sa.key", "--cluster-signing-cert-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.crt", "--cluster-signing-key-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.key", "--insecure-experimental-approve-all-kubelet-csrs-for-group=kubeadm:kubelet-bootstrap", - "--use-service-account-credentials", + "--use-service-account-credentials=true", }, }, { @@ -497,14 +495,14 @@ func TestGetControllerManagerCommand(t *testing.T) { expected: []string{ "kube-controller-manager", "--address=127.0.0.1", - "--leader-elect", + "--leader-elect=true", "--kubeconfig=" + kubeadmapi.GlobalEnvParams.KubernetesDir + "/controller-manager.conf", "--root-ca-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.crt", "--service-account-private-key-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/sa.key", "--cluster-signing-cert-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.crt", "--cluster-signing-key-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.key", "--insecure-experimental-approve-all-kubelet-csrs-for-group=kubeadm:kubelet-bootstrap", - "--use-service-account-credentials", + "--use-service-account-credentials=true", "--cloud-provider=foo", }, }, @@ -513,14 +511,14 @@ func TestGetControllerManagerCommand(t *testing.T) { expected: []string{ "kube-controller-manager", "--address=127.0.0.1", - "--leader-elect", + "--leader-elect=true", "--kubeconfig=" + kubeadmapi.GlobalEnvParams.KubernetesDir + "/controller-manager.conf", "--root-ca-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.crt", "--service-account-private-key-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/sa.key", "--cluster-signing-cert-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.crt", "--cluster-signing-key-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.key", "--insecure-experimental-approve-all-kubelet-csrs-for-group=kubeadm:kubelet-bootstrap", - "--use-service-account-credentials", + "--use-service-account-credentials=true", "--allocate-node-cidrs=true", "--cluster-cidr=bar", }, @@ -529,14 +527,10 @@ func TestGetControllerManagerCommand(t *testing.T) { for _, rt := range tests { actual := getControllerManagerCommand(rt.cfg, false) - for i := range actual { - if actual[i] != rt.expected[i] { - t.Errorf( - "failed getControllerManagerCommand:\n\texpected: %s\n\t actual: %s", - rt.expected[i], - actual[i], - ) - } + sort.Strings(actual) + sort.Strings(rt.expected) + if !reflect.DeepEqual(actual, rt.expected) { + t.Errorf("failed getControllerManagerCommand:\nexpected:\n%v\nsaw:\n%v", rt.expected, actual) } } } @@ -551,7 +545,7 @@ func TestGetSchedulerCommand(t *testing.T) { expected: []string{ "kube-scheduler", "--address=127.0.0.1", - "--leader-elect", + "--leader-elect=true", "--kubeconfig=" + kubeadmapi.GlobalEnvParams.KubernetesDir + "/scheduler.conf", }, }, @@ -559,14 +553,10 @@ func TestGetSchedulerCommand(t *testing.T) { for _, rt := range tests { actual := getSchedulerCommand(rt.cfg, false) - for i := range actual { - if actual[i] != rt.expected[i] { - t.Errorf( - "failed getSchedulerCommand:\n\texpected: %s\n\t actual: %s", - rt.expected[i], - actual[i], - ) - } + sort.Strings(actual) + sort.Strings(rt.expected) + if !reflect.DeepEqual(actual, rt.expected) { + t.Errorf("failed getSchedulerCommand:\nexpected:\n%v\nsaw:\n%v", rt.expected, actual) } } } @@ -629,3 +619,50 @@ func TestGetAuthzParameters(t *testing.T) { } } } + +func TestGetExtraParameters(t *testing.T) { + var tests = []struct { + overrides map[string]string + defaults map[string]string + expected []string + }{ + { + overrides: map[string]string{ + "admission-control": "NamespaceLifecycle,LimitRanger", + }, + defaults: map[string]string{ + "admission-control": "NamespaceLifecycle", + "insecure-bind-address": "127.0.0.1", + "allow-privileged": "true", + }, + expected: []string{ + "--admission-control=NamespaceLifecycle,LimitRanger", + "--insecure-bind-address=127.0.0.1", + "--allow-privileged=true", + }, + }, + { + overrides: map[string]string{ + "admission-control": "NamespaceLifecycle,LimitRanger", + }, + defaults: map[string]string{ + "insecure-bind-address": "127.0.0.1", + "allow-privileged": "true", + }, + expected: []string{ + "--admission-control=NamespaceLifecycle,LimitRanger", + "--insecure-bind-address=127.0.0.1", + "--allow-privileged=true", + }, + }, + } + + for _, rt := range tests { + actual := getExtraParameters(rt.overrides, rt.defaults) + sort.Strings(actual) + sort.Strings(rt.expected) + if !reflect.DeepEqual(actual, rt.expected) { + t.Errorf("failed getExtraParameters:\nexpected:\n%v\nsaw:\n%v", rt.expected, actual) + } + } +}