Make the CLI arguments for the control plane overridable

cmd/kubeadm/app/apis/kubeadm/types.go

@@ -44,6 +44,10 @@ type MasterConfiguration struct {
 	// controller manager are managed by Kubernetes itself. This option is likely to
 	// become the default in the future.
 	SelfHosted bool
+
+	APIServerExtraArgs         map[string]string
+	ControllerManagerExtraArgs map[string]string
+	SchedulerExtraArgs         map[string]string
 }
 
 type API struct {

cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go

@@ -34,6 +34,10 @@ type MasterConfiguration struct {
 	// controller manager are managed by Kubernetes itself. This option is likely to
 	// become the default in the future.
 	SelfHosted bool `json:"selfHosted"`
+
+	APIServerExtraArgs         map[string]string `json:"apiServerExtraArgs"`
+	ControllerManagerExtraArgs map[string]string `json:"controllerManagerExtraArgs"`
+	SchedulerExtraArgs         map[string]string `json:"schedulerExtraArgs"`
 }
 
 type API struct {

cmd/kubeadm/app/constants/constants.go

@@ -97,6 +97,9 @@ const (
 
 	// MinExternalEtcdVersion indicates minimum external etcd version which kubeadm supports
 	MinExternalEtcdVersion = "3.0.14"
+
+	// DefaultAdmissionControl specifies the default admission control options that will be used
+	DefaultAdmissionControl = "NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds"
 )
 
 var (

cmd/kubeadm/app/master/manifests.go

@@ -304,30 +304,33 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool) [
 		command = []string{"/usr/bin/flock", "--exclusive", "--timeout=30", "/var/lock/api-server.lock"}
 	}
 
-	command = append(getComponentBaseCommand(apiServer),
+	defaultArguments := map[string]string{
-		"--insecure-bind-address=127.0.0.1",
+		"insecure-bind-address":           "127.0.0.1",
-		"--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds",
+		"admission-control":               kubeadmconstants.DefaultAdmissionControl,
-		"--service-cluster-ip-range="+cfg.Networking.ServiceSubnet,
+		"service-cluster-ip-range":        cfg.Networking.ServiceSubnet,
-		"--service-account-key-file="+getCertFilePath(kubeadmconstants.ServiceAccountPublicKeyName),
+		"service-account-key-file":        getCertFilePath(kubeadmconstants.ServiceAccountPublicKeyName),
-		"--client-ca-file="+getCertFilePath(kubeadmconstants.CACertName),
+		"client-ca-file":                  getCertFilePath(kubeadmconstants.CACertName),
-		"--tls-cert-file="+getCertFilePath(kubeadmconstants.APIServerCertName),
+		"tls-cert-file":                   getCertFilePath(kubeadmconstants.APIServerCertName),
-		"--tls-private-key-file="+getCertFilePath(kubeadmconstants.APIServerKeyName),
+		"tls-private-key-file":            getCertFilePath(kubeadmconstants.APIServerKeyName),
-		"--kubelet-client-certificate="+getCertFilePath(kubeadmconstants.APIServerKubeletClientCertName),
+		"kubelet-client-certificate":      getCertFilePath(kubeadmconstants.APIServerKubeletClientCertName),
-		"--kubelet-client-key="+getCertFilePath(kubeadmconstants.APIServerKubeletClientKeyName),
+		"kubelet-client-key":              getCertFilePath(kubeadmconstants.APIServerKubeletClientKeyName),
-		"--token-auth-file="+kubeadmapi.GlobalEnvParams.HostPKIPath+"/tokens.csv",
+		"token-auth-file":                 path.Join(kubeadmapi.GlobalEnvParams.HostPKIPath, kubeadmconstants.CSVTokenFileName),
-		fmt.Sprintf("--secure-port=%d", cfg.API.Port),
+		"secure-port":                     fmt.Sprintf("%d", cfg.API.Port),
-		"--allow-privileged",
+		"allow-privileged":                "true",
-		"--storage-backend=etcd3",
+		"storage-backend":                 "etcd3",
-		"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
+		"kubelet-preferred-address-types": "InternalIP,ExternalIP,Hostname",
+
 		// add options to configure the front proxy.  Without the generated client cert, this will never be useable
 		// so add it unconditionally with recommended values
-		"--requestheader-username-headers=X-Remote-User",
+		"requestheader-username-headers":     "X-Remote-User",
-		"--requestheader-group-headers=X-Remote-Group",
+		"requestheader-group-headers":        "X-Remote-Group",
-		"--requestheader-extra-headers-prefix=X-Remote-Extra-",
+		"requestheader-extra-headers-prefix": "X-Remote-Extra-",
-		"--requestheader-client-ca-file="+getCertFilePath(kubeadmconstants.FrontProxyCACertName),
+		"requestheader-client-ca-file":       getCertFilePath(kubeadmconstants.FrontProxyCACertName),
-		"--requestheader-allowed-names=front-proxy-client",
+		"requestheader-allowed-names":        "front-proxy-client",
-	)
+	}
 
+	command = getComponentBaseCommand(apiServer)
+	command = append(command, getExtraParameters(cfg.APIServerExtraArgs, defaultArguments)...)
 	command = append(command, getAuthzParameters(cfg.AuthorizationMode)...)
 
 	// Use first address we are given
@@ -376,17 +379,20 @@ func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted
 		command = []string{"/usr/bin/flock", "--exclusive", "--timeout=30", "/var/lock/controller-manager.lock"}
 	}
 
-	command = append(getComponentBaseCommand(controllerManager),
+	defaultArguments := map[string]string{
-		"--address=127.0.0.1",
+		"address":                                                  "127.0.0.1",
-		"--leader-elect",
+		"leader-elect":                                             "true",
-		"--kubeconfig="+path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.ControllerManagerKubeConfigFileName),
+		"kubeconfig":                                               path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.ControllerManagerKubeConfigFileName),
-		"--root-ca-file="+getCertFilePath(kubeadmconstants.CACertName),
+		"root-ca-file":                                             getCertFilePath(kubeadmconstants.CACertName),
-		"--service-account-private-key-file="+getCertFilePath(kubeadmconstants.ServiceAccountPrivateKeyName),
+		"service-account-private-key-file":                         getCertFilePath(kubeadmconstants.ServiceAccountPrivateKeyName),
-		"--cluster-signing-cert-file="+getCertFilePath(kubeadmconstants.CACertName),
+		"cluster-signing-cert-file":                                getCertFilePath(kubeadmconstants.CACertName),
-		"--cluster-signing-key-file="+getCertFilePath(kubeadmconstants.CAKeyName),
+		"cluster-signing-key-file":                                 getCertFilePath(kubeadmconstants.CAKeyName),
-		"--insecure-experimental-approve-all-kubelet-csrs-for-group="+kubeadmconstants.CSVTokenBootstrapGroup,
+		"insecure-experimental-approve-all-kubelet-csrs-for-group": kubeadmconstants.CSVTokenBootstrapGroup,
-		"--use-service-account-credentials",
+		"use-service-account-credentials":                          "true",
-	)
+	}
+
+	command = getComponentBaseCommand(controllerManager)
+	command = append(command, getExtraParameters(cfg.ControllerManagerExtraArgs, defaultArguments)...)
 
 	if cfg.CloudProvider != "" {
 		command = append(command, "--cloud-provider="+cfg.CloudProvider)
@@ -414,11 +420,14 @@ func getSchedulerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool) [
 		command = []string{"/usr/bin/flock", "--exclusive", "--timeout=30", "/var/lock/api-server.lock"}
 	}
 
-	command = append(getComponentBaseCommand(scheduler),
+	defaultArguments := map[string]string{
-		"--address=127.0.0.1",
+		"address":      "127.0.0.1",
-		"--leader-elect",
+		"leader-elect": "true",
-		"--kubeconfig="+path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.SchedulerKubeConfigFileName),
+		"kubeconfig":   path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.SchedulerKubeConfigFileName),
-	)
+	}
+
+	command = getComponentBaseCommand(scheduler)
+	command = append(command, getExtraParameters(cfg.SchedulerExtraArgs, defaultArguments)...)
 
 	return command
 }
@@ -472,3 +481,18 @@ func getAuthzParameters(authzMode string) []string {
 	}
 	return command
 }
+
+func getExtraParameters(overrides map[string]string, defaults map[string]string) []string {
+	var command []string
+	for k, v := range overrides {
+		if len(v) > 0 {
+			command = append(command, fmt.Sprintf("--%s=%s", k, v))
+		}
+	}
+	for k, v := range defaults {
+		if _, overrideExists := overrides[k]; !overrideExists {
+			command = append(command, fmt.Sprintf("--%s=%s", k, v))
+		}
+	}
+	return command
+}

cmd/kubeadm/app/master/manifests_test.go

@@ -20,6 +20,8 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"reflect"
+	"sort"
 	"testing"
 
 	"k8s.io/apimachinery/pkg/util/intstr"
@@ -380,7 +382,7 @@ func TestGetAPIServerCommand(t *testing.T) {
 				"--kubelet-client-key=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver-kubelet-client.key",
 				"--token-auth-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/tokens.csv",
 				fmt.Sprintf("--secure-port=%d", 123),
-				"--allow-privileged",
+				"--allow-privileged=true",
 				"--storage-backend=etcd3",
 				"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
 				"--requestheader-username-headers=X-Remote-User",
@@ -410,7 +412,7 @@ func TestGetAPIServerCommand(t *testing.T) {
 				"--kubelet-client-key=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver-kubelet-client.key",
 				"--token-auth-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/tokens.csv",
 				fmt.Sprintf("--secure-port=%d", 123),
-				"--allow-privileged",
+				"--allow-privileged=true",
 				"--storage-backend=etcd3",
 				"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
 				"--requestheader-username-headers=X-Remote-User",
@@ -442,7 +444,7 @@ func TestGetAPIServerCommand(t *testing.T) {
 				"--kubelet-client-key=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver-kubelet-client.key",
 				"--token-auth-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/tokens.csv",
 				fmt.Sprintf("--secure-port=%d", 123),
-				"--allow-privileged",
+				"--allow-privileged=true",
 				"--storage-backend=etcd3",
 				"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
 				"--requestheader-username-headers=X-Remote-User",
@@ -460,14 +462,10 @@ func TestGetAPIServerCommand(t *testing.T) {
 
 	for _, rt := range tests {
 		actual := getAPIServerCommand(rt.cfg, false)
-		for i := range actual {
+		sort.Strings(actual)
-			if actual[i] != rt.expected[i] {
+		sort.Strings(rt.expected)
-				t.Errorf(
+		if !reflect.DeepEqual(actual, rt.expected) {
-					"failed getAPIServerCommand:\n\texpected: %s\n\t  actual: %s",
+			t.Errorf("failed getAPIServerCommand:\nexpected:\n%v\nsaw:\n%v", rt.expected, actual)
-					rt.expected[i],
-					actual[i],
-				)
-			}
 		}
 	}
 }
@@ -482,14 +480,14 @@ func TestGetControllerManagerCommand(t *testing.T) {
 			expected: []string{
 				"kube-controller-manager",
 				"--address=127.0.0.1",
-				"--leader-elect",
+				"--leader-elect=true",
 				"--kubeconfig=" + kubeadmapi.GlobalEnvParams.KubernetesDir + "/controller-manager.conf",
 				"--root-ca-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.crt",
 				"--service-account-private-key-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/sa.key",
 				"--cluster-signing-cert-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.crt",
 				"--cluster-signing-key-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.key",
 				"--insecure-experimental-approve-all-kubelet-csrs-for-group=kubeadm:kubelet-bootstrap",
-				"--use-service-account-credentials",
+				"--use-service-account-credentials=true",
 			},
 		},
 		{
@@ -497,14 +495,14 @@ func TestGetControllerManagerCommand(t *testing.T) {
 			expected: []string{
 				"kube-controller-manager",
 				"--address=127.0.0.1",
-				"--leader-elect",
+				"--leader-elect=true",
 				"--kubeconfig=" + kubeadmapi.GlobalEnvParams.KubernetesDir + "/controller-manager.conf",
 				"--root-ca-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.crt",
 				"--service-account-private-key-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/sa.key",
 				"--cluster-signing-cert-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.crt",
 				"--cluster-signing-key-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.key",
 				"--insecure-experimental-approve-all-kubelet-csrs-for-group=kubeadm:kubelet-bootstrap",
-				"--use-service-account-credentials",
+				"--use-service-account-credentials=true",
 				"--cloud-provider=foo",
 			},
 		},
@@ -513,14 +511,14 @@ func TestGetControllerManagerCommand(t *testing.T) {
 			expected: []string{
 				"kube-controller-manager",
 				"--address=127.0.0.1",
-				"--leader-elect",
+				"--leader-elect=true",
 				"--kubeconfig=" + kubeadmapi.GlobalEnvParams.KubernetesDir + "/controller-manager.conf",
 				"--root-ca-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.crt",
 				"--service-account-private-key-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/sa.key",
 				"--cluster-signing-cert-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.crt",
 				"--cluster-signing-key-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.key",
 				"--insecure-experimental-approve-all-kubelet-csrs-for-group=kubeadm:kubelet-bootstrap",
-				"--use-service-account-credentials",
+				"--use-service-account-credentials=true",
 				"--allocate-node-cidrs=true",
 				"--cluster-cidr=bar",
 			},
@@ -529,14 +527,10 @@ func TestGetControllerManagerCommand(t *testing.T) {
 
 	for _, rt := range tests {
 		actual := getControllerManagerCommand(rt.cfg, false)
-		for i := range actual {
+		sort.Strings(actual)
-			if actual[i] != rt.expected[i] {
+		sort.Strings(rt.expected)
-				t.Errorf(
+		if !reflect.DeepEqual(actual, rt.expected) {
-					"failed getControllerManagerCommand:\n\texpected: %s\n\t  actual: %s",
+			t.Errorf("failed getControllerManagerCommand:\nexpected:\n%v\nsaw:\n%v", rt.expected, actual)
-					rt.expected[i],
-					actual[i],
-				)
-			}
 		}
 	}
 }
@@ -551,7 +545,7 @@ func TestGetSchedulerCommand(t *testing.T) {
 			expected: []string{
 				"kube-scheduler",
 				"--address=127.0.0.1",
-				"--leader-elect",
+				"--leader-elect=true",
 				"--kubeconfig=" + kubeadmapi.GlobalEnvParams.KubernetesDir + "/scheduler.conf",
 			},
 		},
@@ -559,14 +553,10 @@ func TestGetSchedulerCommand(t *testing.T) {
 
 	for _, rt := range tests {
 		actual := getSchedulerCommand(rt.cfg, false)
-		for i := range actual {
+		sort.Strings(actual)
-			if actual[i] != rt.expected[i] {
+		sort.Strings(rt.expected)
-				t.Errorf(
+		if !reflect.DeepEqual(actual, rt.expected) {
-					"failed getSchedulerCommand:\n\texpected: %s\n\t  actual: %s",
+			t.Errorf("failed getSchedulerCommand:\nexpected:\n%v\nsaw:\n%v", rt.expected, actual)
-					rt.expected[i],
-					actual[i],
-				)
-			}
 		}
 	}
 }
@@ -629,3 +619,50 @@ func TestGetAuthzParameters(t *testing.T) {
 		}
 	}
 }
+
+func TestGetExtraParameters(t *testing.T) {
+	var tests = []struct {
+		overrides map[string]string
+		defaults  map[string]string
+		expected  []string
+	}{
+		{
+			overrides: map[string]string{
+				"admission-control": "NamespaceLifecycle,LimitRanger",
+			},
+			defaults: map[string]string{
+				"admission-control":     "NamespaceLifecycle",
+				"insecure-bind-address": "127.0.0.1",
+				"allow-privileged":      "true",
+			},
+			expected: []string{
+				"--admission-control=NamespaceLifecycle,LimitRanger",
+				"--insecure-bind-address=127.0.0.1",
+				"--allow-privileged=true",
+			},
+		},
+		{
+			overrides: map[string]string{
+				"admission-control": "NamespaceLifecycle,LimitRanger",
+			},
+			defaults: map[string]string{
+				"insecure-bind-address": "127.0.0.1",
+				"allow-privileged":      "true",
+			},
+			expected: []string{
+				"--admission-control=NamespaceLifecycle,LimitRanger",
+				"--insecure-bind-address=127.0.0.1",
+				"--allow-privileged=true",
+			},
+		},
+	}
+
+	for _, rt := range tests {
+		actual := getExtraParameters(rt.overrides, rt.defaults)
+		sort.Strings(actual)
+		sort.Strings(rt.expected)
+		if !reflect.DeepEqual(actual, rt.expected) {
+			t.Errorf("failed getExtraParameters:\nexpected:\n%v\nsaw:\n%v", rt.expected, actual)
+		}
+	}
+}