Merge pull request #34573 from errordeveloper/fix-early-deployment-issue

Automatic merge from submit-queue

Test API more extensivelly before declaring readiness 

**What this PR does / why we need it**:

It's possible that first deployment kubeadm creates will hit `deployments.extensions "kube-discovery" is forbidden: not yet ready to handle request`, which comes from NamespaceLifecycle admission controller and has something to do with cache. According to @derekwaynecarr, we need to create a namespace-scoped resource to really check for this. I didn't want to make a check with deployment of whatever comes first right now, and decided to have explicit step for this in `apiclient.go`.

**Which issue this PR fixes**: fixes #34411

**Special notes for your reviewer**: @kubernetes/sig-cluster-lifecycle 

**Release note**:
```release-note
NONE
```

cmd/kubeadm/app/images/images.go

@@ -34,6 +34,7 @@ const (
 	KubeDNSImage         = "kube-dns"
 	KubeDNSmasqImage     = "dnsmasq"
 	KubeExechealthzImage = "exechealthz"
+	Pause                = "pause"
 
 	gcrPrefix   = "gcr.io/google_containers"
 	etcdVersion = "2.2.5"
@@ -41,6 +42,7 @@ const (
 	kubeDNSVersion     = "1.7"
 	dnsmasqVersion     = "1.3"
 	exechealthzVersion = "1.1"
+	pauseVersion       = "3.0"
 )
 
 func GetCoreImage(image string, cfg *kubeadmapi.MasterConfiguration, overrideImage string) string {
@@ -62,5 +64,6 @@ func GetAddonImage(image string) string {
 		KubeDNSImage:         fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kubedns", runtime.GOARCH, kubeDNSVersion),
 		KubeDNSmasqImage:     fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-dnsmasq", runtime.GOARCH, dnsmasqVersion),
 		KubeExechealthzImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "exechealthz", runtime.GOARCH, exechealthzVersion),
+		Pause:                fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "pause", runtime.GOARCH, pauseVersion),
 	}[image]
 }

cmd/kubeadm/app/master/apiclient.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"time"
 
+	"k8s.io/kubernetes/cmd/kubeadm/app/images"
 	"k8s.io/kubernetes/pkg/api"
 	apierrs "k8s.io/kubernetes/pkg/api/errors"
 	unversionedapi "k8s.io/kubernetes/pkg/api/unversioned"
@@ -96,6 +97,8 @@ func CreateClientAndWaitForAPI(adminConfig *clientcmdapi.Config) (*clientset.Cli
 		return true, nil
 	})
 
+	createDummyDeployment(client)
+
 	return client, nil
 }
 
@@ -219,3 +222,41 @@ func SetMasterNodeAffinity(meta *api.ObjectMeta) {
 	}
 	meta.Annotations[api.AffinityAnnotationKey] = string(affinityAnnotation)
 }
+
+func createDummyDeployment(client *clientset.Clientset) {
+	fmt.Println("<master/apiclient> attempting a test deployment")
+	dummyDeployment := NewDeployment("dummy", 1, api.PodSpec{
+		SecurityContext: &api.PodSecurityContext{HostNetwork: true},
+		Containers: []api.Container{{
+			Name:  "dummy",
+			Image: images.GetAddonImage("pause"),
+		}},
+	})
+
+	wait.PollInfinite(apiCallRetryInterval, func() (bool, error) {
+		// TODO: we should check the error, as some cases may be fatal
+		if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(dummyDeployment); err != nil {
+			fmt.Printf("<master/apiclient> failed to create test deployment [%v] (will retry)", err)
+			return false, nil
+		}
+		return true, nil
+	})
+
+	wait.PollInfinite(apiCallRetryInterval, func() (bool, error) {
+		d, err := client.Extensions().Deployments(api.NamespaceSystem).Get("dummy")
+		if err != nil {
+			fmt.Printf("<master/apiclient> failed to get test deployment [%v] (will retry)", err)
+			return false, nil
+		}
+		if d.Status.AvailableReplicas < 1 {
+			return false, nil
+		}
+		return true, nil
+	})
+
+	fmt.Println("<master/apiclient> test deployment succeeded")
+
+	if err := client.Extensions().Deployments(api.NamespaceSystem).Delete("dummy", &api.DeleteOptions{}); err != nil {
+		fmt.Printf("<master/apiclient> failed to delete test deployment [%v] (will ignore)", err)
+	}
+}