Merge pull request #42064 from luxas/kubeadm_beta_init_ux

Automatic merge from submit-queue (batch tested with PRs 42128, 42064, 42253, 42309, 42322)

kubeadm: Rename some flags for beta UI and fixup some logic

**What this PR does / why we need it**:

In this PR:
 - `--api-advertise-addresses` becomes `--apiserver-advertise-address`
   - The API Server's logic here is that if the address is `0.0.0.0`, it chooses the host's default interface's address. kubeadm here uses exactly the same logic. This arg is then passed to `--advertise-address`, and the API Server will advertise that one for the service VIP.
 - `--api-port` becomes `--apiserver-bind-port` for clarity

ref the meeting notes: https://docs.google.com/document/d/1deJYPIF4LmhGjDVaqrswErIrV7mtwJgovtLnPCDxP7U/edit#

**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #

**Special notes for your reviewer**:

**Release note**:

```release-note
```
@jbeda @dmmcquay @pires @lukemarsden @dgoodwin @mikedanese

cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go

@@ -28,7 +28,8 @@ func KubeadmFuzzerFuncs(t apitesting.TestingCommon) []interface{} {
 		func(obj *kubeadm.MasterConfiguration, c fuzz.Continue) {
 			c.FuzzNoCustom(obj)
 			obj.KubernetesVersion = "v10"
-			obj.API.Port = 20
+			obj.API.BindPort = 20
+			obj.API.AdvertiseAddress = "foo"
 			obj.Networking.ServiceSubnet = "foo"
 			obj.Networking.DNSDomain = "foo"
 			obj.AuthorizationMode = "foo"

cmd/kubeadm/app/apis/kubeadm/types.go

@@ -51,9 +51,9 @@ type MasterConfiguration struct {
 }
 
 type API struct {
-	AdvertiseAddresses []string
-	ExternalDNSNames   []string
-	Port               int32
+	AdvertiseAddress string
+	ExternalDNSNames []string
+	BindPort         int32
 }
 
 type Discovery struct {

cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go

@@ -47,8 +47,8 @@ func SetDefaults_MasterConfiguration(obj *MasterConfiguration) {
 		obj.KubernetesVersion = DefaultKubernetesVersion
 	}
 
-	if obj.API.Port == 0 {
-		obj.API.Port = DefaultAPIBindPort
+	if obj.API.BindPort == 0 {
+		obj.API.BindPort = DefaultAPIBindPort
 	}
 
 	if obj.Networking.ServiceSubnet == "" {

cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go

@@ -41,9 +41,10 @@ type MasterConfiguration struct {
 }
 
 type API struct {
-	AdvertiseAddresses []string `json:"advertiseAddresses"`
-	ExternalDNSNames   []string `json:"externalDNSNames"`
-	Port               int32    `json:"port"`
+	// The address for the API server to advertise.
+	AdvertiseAddress string   `json:"advertiseAddress"`
+	ExternalDNSNames []string `json:"externalDNSNames"`
+	BindPort         int32    `json:"bindPort"`
 }
 
 type Discovery struct {

cmd/kubeadm/app/cmd/defaults.go

@@ -18,7 +18,7 @@ package cmd
 
 import (
 	"fmt"
-	"strconv"
+	"net"
 
 	netutil "k8s.io/apimachinery/pkg/util/net"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
@@ -35,14 +35,14 @@ var (
 )
 
 func setInitDynamicDefaults(cfg *kubeadmapi.MasterConfiguration) error {
-	// Auto-detect the IP
-	if len(cfg.API.AdvertiseAddresses) == 0 {
-		ip, err := netutil.ChooseHostInterface()
-		if err != nil {
-			return err
-		}
-		cfg.API.AdvertiseAddresses = []string{ip.String()}
+
+	// Choose the right address for the API Server to advertise. If the advertise address is localhost or 0.0.0.0, the default interface's IP address is used
+	// This is the same logic as the API Server uses
+	ip, err := netutil.ChooseBindAddress(net.ParseIP(cfg.API.AdvertiseAddress))
+	if err != nil {
+		return err
 	}
+	cfg.API.AdvertiseAddress = ip.String()
 
 	// Validate version argument
 	ver, err := kubeadmutil.KubernetesReleaseVersion(cfg.KubernetesVersion)
@@ -89,7 +89,7 @@ func setInitDynamicDefaults(cfg *kubeadmapi.MasterConfiguration) error {
 
 		// If there aren't any addresses specified, default to the first advertised address which can be user-provided or the default network interface's IP address
 		if len(cfg.Discovery.Token.Addresses) == 0 {
-			cfg.Discovery.Token.Addresses = []string{cfg.API.AdvertiseAddresses[0] + ":" + strconv.Itoa(kubeadmapiext.DefaultDiscoveryBindPort)}
+			cfg.Discovery.Token.Addresses = []string{fmt.Sprintf("%s:%d", cfg.API.AdvertiseAddress, kubeadmapiext.DefaultDiscoveryBindPort)}
 		}
 	}
 

cmd/kubeadm/app/cmd/init.go

@@ -80,13 +80,13 @@ func NewCmdInit(out io.Writer) *cobra.Command {
 		},
 	}
 
-	cmd.PersistentFlags().StringSliceVar(
-		&cfg.API.AdvertiseAddresses, "api-advertise-addresses", cfg.API.AdvertiseAddresses,
-		"The IP addresses to advertise, in case autodetection fails",
+	cmd.PersistentFlags().StringVar(
+		&cfg.API.AdvertiseAddress, "apiserver-advertise-address", cfg.API.AdvertiseAddress,
+		"The IP address the API Server will advertise it's listening on. 0.0.0.0 means the default network interface's address.",
 	)
 	cmd.PersistentFlags().Int32Var(
-		&cfg.API.Port, "api-port", cfg.API.Port,
-		"Port for API to bind to",
+		&cfg.API.BindPort, "apiserver-bind-port", cfg.API.BindPort,
+		"Port for the API Server to bind to",
 	)
 	cmd.PersistentFlags().StringSliceVar(
 		&cfg.API.ExternalDNSNames, "api-external-dns-names", cfg.API.ExternalDNSNames,
@@ -189,7 +189,7 @@ func (i *Init) Run(out io.Writer) error {
 	// TODO this is not great, but there is only one address we can use here
 	// so we'll pick the first one, there is much of chance to have an empty
 	// slice by the time this gets called
-	masterEndpoint := fmt.Sprintf("https://%s:%d", i.cfg.API.AdvertiseAddresses[0], i.cfg.API.Port)
+	masterEndpoint := fmt.Sprintf("https://%s:%d", i.cfg.API.AdvertiseAddress, i.cfg.API.BindPort)
 	err = kubeconfigphase.CreateInitKubeConfigFiles(masterEndpoint, kubeadmapi.GlobalEnvParams.HostPKIPath, kubeadmapi.GlobalEnvParams.KubernetesDir)
 	if err != nil {
 		return err

cmd/kubeadm/app/master/discovery.go

@@ -46,18 +46,13 @@ const (
 
 func encodeKubeDiscoverySecretData(dcfg *kubeadmapi.TokenDiscovery, apicfg kubeadmapi.API, caCert *x509.Certificate) map[string][]byte {
 	var (
-		data         = map[string][]byte{}
-		endpointList = []string{}
-		tokenMap     = map[string]string{}
+		data     = map[string][]byte{}
+		tokenMap = map[string]string{}
 	)
 
-	for _, addr := range apicfg.AdvertiseAddresses {
-		endpointList = append(endpointList, fmt.Sprintf("https://%s:%d", addr, apicfg.Port))
-	}
-
 	tokenMap[dcfg.ID] = dcfg.Secret
 
-	data["endpoint-list.json"], _ = json.Marshal(endpointList)
+	data["endpoint-list.json"], _ = json.Marshal([]string{fmt.Sprintf("https://%s:%d", apicfg.AdvertiseAddress, apicfg.BindPort)})
 	data["token-map.json"], _ = json.Marshal(tokenMap)
 	data["ca.pem"] = certutil.EncodeCertPEM(caCert)
 

cmd/kubeadm/app/master/manifests.go

@@ -315,7 +315,7 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool) [
 		"kubelet-client-certificate":      getCertFilePath(kubeadmconstants.APIServerKubeletClientCertName),
 		"kubelet-client-key":              getCertFilePath(kubeadmconstants.APIServerKubeletClientKeyName),
 		"token-auth-file":                 path.Join(kubeadmapi.GlobalEnvParams.HostPKIPath, kubeadmconstants.CSVTokenFileName),
-		"secure-port":                     fmt.Sprintf("%d", cfg.API.Port),
+		"secure-port":                     fmt.Sprintf("%d", cfg.API.BindPort),
 		"allow-privileged":                "true",
 		"storage-backend":                 "etcd3",
 		"kubelet-preferred-address-types": "InternalIP,ExternalIP,Hostname",
@@ -332,13 +332,10 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool) [
 	command = append(command, getExtraParameters(cfg.APIServerExtraArgs, defaultArguments)...)
 	command = append(command, getAuthzParameters(cfg.AuthorizationMode)...)
 
-	// Use first address we are given
-	if len(cfg.API.AdvertiseAddresses) > 0 {
-		if selfHosted {
-			command = append(command, "--advertise-address=$(POD_IP)")
-		} else {
-			command = append(command, fmt.Sprintf("--advertise-address=%s", cfg.API.AdvertiseAddresses[0]))
-		}
+	if selfHosted {
+		command = append(command, "--advertise-address=$(POD_IP)")
+	} else {
+		command = append(command, fmt.Sprintf("--advertise-address=%s", cfg.API.AdvertiseAddress))
 	}
 
 	// Check if the user decided to use an external etcd cluster

cmd/kubeadm/app/master/manifests_test.go

@@ -380,7 +380,7 @@ func TestGetAPIServerCommand(t *testing.T) {
 	}{
 		{
 			cfg: &kubeadmapi.MasterConfiguration{
-				API:        kubeadm.API{Port: 123},
+				API:        kubeadm.API{BindPort: 123, AdvertiseAddress: "1.2.3.4"},
 				Networking: kubeadm.Networking{ServiceSubnet: "bar"},
 			},
 			expected: []string{
@@ -405,12 +405,13 @@ func TestGetAPIServerCommand(t *testing.T) {
 				"--requestheader-client-ca-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/front-proxy-ca.crt",
 				"--requestheader-allowed-names=front-proxy-client",
 				"--authorization-mode=RBAC",
+				"--advertise-address=1.2.3.4",
 				"--etcd-servers=http://127.0.0.1:2379",
 			},
 		},
 		{
 			cfg: &kubeadmapi.MasterConfiguration{
-				API:        kubeadm.API{Port: 123, AdvertiseAddresses: []string{"foo"}},
+				API:        kubeadm.API{BindPort: 123, AdvertiseAddress: "4.3.2.1"},
 				Networking: kubeadm.Networking{ServiceSubnet: "bar"},
 			},
 			expected: []string{
@@ -435,13 +436,13 @@ func TestGetAPIServerCommand(t *testing.T) {
 				"--requestheader-client-ca-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/front-proxy-ca.crt",
 				"--requestheader-allowed-names=front-proxy-client",
 				"--authorization-mode=RBAC",
-				"--advertise-address=foo",
+				"--advertise-address=4.3.2.1",
 				"--etcd-servers=http://127.0.0.1:2379",
 			},
 		},
 		{
 			cfg: &kubeadmapi.MasterConfiguration{
-				API:        kubeadm.API{Port: 123},
+				API:        kubeadm.API{BindPort: 123, AdvertiseAddress: "4.3.2.1"},
 				Networking: kubeadm.Networking{ServiceSubnet: "bar"},
 				Etcd:       kubeadm.Etcd{CertFile: "fiz", KeyFile: "faz"},
 			},
@@ -467,6 +468,7 @@ func TestGetAPIServerCommand(t *testing.T) {
 				"--requestheader-client-ca-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/front-proxy-ca.crt",
 				"--requestheader-allowed-names=front-proxy-client",
 				"--authorization-mode=RBAC",
+				"--advertise-address=4.3.2.1",
 				"--etcd-servers=http://127.0.0.1:2379",
 				"--etcd-certfile=fiz",
 				"--etcd-keyfile=faz",

cmd/kubeadm/app/phases/addons/addons.go

@@ -38,7 +38,7 @@ import (
 func CreateEssentialAddons(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset) error {
 	proxyConfigMapBytes, err := kubeadmutil.ParseTemplate(KubeProxyConfigMap, struct{ MasterEndpoint string }{
 		// Fetch this value from the kubeconfig file
-		MasterEndpoint: fmt.Sprintf("https://%s:%d", cfg.API.AdvertiseAddresses[0], cfg.API.Port),
+		MasterEndpoint: fmt.Sprintf("https://%s:%d", cfg.API.AdvertiseAddress, cfg.API.BindPort),
 	})
 	if err != nil {
 		return fmt.Errorf("error when parsing kube-proxy configmap template: %v", err)

cmd/kubeadm/app/phases/certs/certs.go

@@ -57,14 +57,6 @@ func CreatePKIAssets(cfg *kubeadmapi.MasterConfiguration, pkiDir string) error {
 	altNames.DNSNames = append(cfg.API.ExternalDNSNames, hostname)
 	altNames.DNSNames = append(altNames.DNSNames, internalAPIServerFQDN...)
 
-	// then, add all IP addresses we're bound to
-	for _, a := range cfg.API.AdvertiseAddresses {
-		if ip := net.ParseIP(a); ip != nil {
-			altNames.IPs = append(altNames.IPs, ip)
-		} else {
-			return fmt.Errorf("could not parse ip %q", a)
-		}
-	}
 	// and lastly, extract the internal IP address for the API server
 	_, n, err := net.ParseCIDR(cfg.Networking.ServiceSubnet)
 	if err != nil {
@@ -75,7 +67,7 @@ func CreatePKIAssets(cfg *kubeadmapi.MasterConfiguration, pkiDir string) error {
 		return fmt.Errorf("unable to allocate IP address for the API server from the given CIDR (%q) [%v]", &cfg.Networking.ServiceSubnet, err)
 	}
 
-	altNames.IPs = append(altNames.IPs, internalAPIServerVirtualIP)
+	altNames.IPs = append(altNames.IPs, internalAPIServerVirtualIP, net.ParseIP(cfg.API.AdvertiseAddress))
 
 	var caCert *x509.Certificate
 	var caKey *rsa.PrivateKey

cmd/kubeadm/app/phases/certs/certs_test.go

@@ -45,7 +45,7 @@ func TestCreatePKIAssets(t *testing.T) {
 		{
 			// CIDR too small
 			cfg: &kubeadmapi.MasterConfiguration{
-				API:        kubeadmapi.API{AdvertiseAddresses: []string{"10.0.0.1"}},
+				API:        kubeadmapi.API{AdvertiseAddress: "1.2.3.4"},
 				Networking: kubeadmapi.Networking{ServiceSubnet: "10.0.0.1/1"},
 			},
 			expected: false,
@@ -53,14 +53,14 @@ func TestCreatePKIAssets(t *testing.T) {
 		{
 			// CIDR invalid
 			cfg: &kubeadmapi.MasterConfiguration{
-				API:        kubeadmapi.API{AdvertiseAddresses: []string{"10.0.0.1"}},
+				API:        kubeadmapi.API{AdvertiseAddress: "1.2.3.4"},
 				Networking: kubeadmapi.Networking{ServiceSubnet: "invalid"},
 			},
 			expected: false,
 		},
 		{
 			cfg: &kubeadmapi.MasterConfiguration{
-				API:        kubeadmapi.API{AdvertiseAddresses: []string{"10.0.0.1"}},
+				API:        kubeadmapi.API{AdvertiseAddress: "1.2.3.4"},
 				Networking: kubeadmapi.Networking{ServiceSubnet: "10.0.0.1/24"},
 			},
 			expected: true,

cmd/kubeadm/app/phases/certs/doc.go

@@ -22,7 +22,6 @@ package certs
 
 	INPUTS:
 		From MasterConfiguration
-			.API.AdvertiseAddresses is needed for knowing which IPs the certs should be signed for
 			.API.ExternalDNSNames is needed for knowing which DNS names the certs should be signed for
 			.Networking.DNSDomain is needed for knowing which DNS name the internal kubernetes service has
 			.Networking.ServiceSubnet is needed for knowing which IP the internal kubernetes service is going to point to

cmd/kubeadm/app/phases/kubeconfig/doc.go

@@ -22,7 +22,7 @@ package kubeconfig
 
 	INPUTS:
 		From MasterConfiguration
-			(.API.AdvertiseAddresses is currently needed for knowing the) MasterAPIEndpoint is required so the KubeConfig file knows where to find the master
+			The Master API Server endpoint (AdvertiseAddress + BindPort) is required so the KubeConfig file knows where to find the master
 			The KubernetesDir path is required for knowing where to put the KubeConfig files
 			The PKIPath is required for knowing where all certificates should be stored
 
@@ -30,4 +30,6 @@ package kubeconfig
 		Files to KubernetesDir (default /etc/kubernetes):
 		 - admin.conf
 		 - kubelet.conf
+		 - scheduler.conf
+		 - controller-manager.conf
 */

cmd/kubeadm/app/preflight/checks.go

@@ -486,12 +486,12 @@ func RunInitMasterChecks(cfg *kubeadmapi.MasterConfiguration) error {
 		HostnameCheck{},
 		ServiceCheck{Service: "kubelet", CheckIfActive: false},
 		ServiceCheck{Service: "docker", CheckIfActive: true},
-		FirewalldCheck{ports: []int{int(cfg.API.Port), 10250}},
-		PortOpenCheck{port: int(cfg.API.Port)},
+		FirewalldCheck{ports: []int{int(cfg.API.BindPort), 10250}},
+		PortOpenCheck{port: int(cfg.API.BindPort)},
 		PortOpenCheck{port: 10250},
 		PortOpenCheck{port: 10251},
 		PortOpenCheck{port: 10252},
-		HTTPProxyCheck{Proto: "https", Host: cfg.API.AdvertiseAddresses[0], Port: int(cfg.API.Port)},
+		HTTPProxyCheck{Proto: "https", Host: cfg.API.AdvertiseAddress, Port: int(cfg.API.BindPort)},
 		DirAvailableCheck{Path: filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "manifests")},
 		DirAvailableCheck{Path: "/var/lib/kubelet"},
 		FileContentCheck{Path: bridgenf, Content: []byte{'1'}},

cmd/kubeadm/app/preflight/checks_test.go

@@ -180,7 +180,7 @@ func TestRunInitMasterChecks(t *testing.T) {
 	}{
 		{
 			cfg: &kubeadmapi.MasterConfiguration{
-				API: kubeadm.API{AdvertiseAddresses: []string{"foo"}},
+				API: kubeadm.API{AdvertiseAddress: "foo"},
 			},
 			expected: false,
 		},

hack/verify-flags/known-flags.txt

@@ -11,15 +11,15 @@ allowed-not-ready-nodes
 allow-missing-template-keys
 allow-privileged
 anonymous-auth
-api-advertise-addresses
 api-burst
 api-external-dns-names
-api-port
 api-prefix
 api-rate
 api-server-advertise-address
+apiserver-advertise-address
 apiserver-arg-overrides
 apiserver-arg-overrides
+apiserver-bind-port
 apiserver-count
 apiserver-count
 apiserver-count