Merge pull request #8696 from derekwaynecarr/force_namespace_creation

Force explicit namespace provision, update e2e for failures

cluster/aws/config-default.sh

@@ -75,7 +75,7 @@ DNS_DOMAIN="cluster.local"
 DNS_REPLICAS=1
 
 # Admission Controllers to invoke prior to persisting objects in cluster
-ADMISSION_CONTROL=NamespaceLifecycle,NamespaceAutoProvision,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota
+ADMISSION_CONTROL=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota
 
 # Optional: Enable/disable public IP assignment for minions.
 # Important Note: disable only if you have setup a NAT instance for internet access and configured appropriate routes!

cluster/aws/config-test.sh

@@ -72,7 +72,7 @@ DNS_DOMAIN="cluster.local"
 DNS_REPLICAS=1
 
 # Admission Controllers to invoke prior to persisting objects in cluster
-ADMISSION_CONTROL=NamespaceLifecycle,NamespaceAutoProvision,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota
+ADMISSION_CONTROL=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota
 
 # Optional: Enable/disable public IP assignment for minions.
 # Important Note: disable only if you have setup a NAT instance for internet access and configured appropriate routes!

cluster/azure/config-default.sh

@@ -49,4 +49,4 @@ ELASTICSEARCH_LOGGING_REPLICAS=1
 ENABLE_CLUSTER_MONITORING="${KUBE_ENABLE_CLUSTER_MONITORING:-true}"
 
 # Admission Controllers to invoke prior to persisting objects in cluster
-ADMISSION_CONTROL=NamespaceLifecycle,NamespaceAutoProvision,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota
+ADMISSION_CONTROL=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota

cluster/gce/config-default.sh

@@ -77,4 +77,4 @@ DNS_DOMAIN="cluster.local"
 DNS_REPLICAS=1
 
 # Admission Controllers to invoke prior to persisting objects in cluster
-ADMISSION_CONTROL=NamespaceLifecycle,NamespaceAutoProvision,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota
+ADMISSION_CONTROL=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota

cluster/gce/config-test.sh

@@ -75,4 +75,4 @@ DNS_SERVER_IP="10.0.0.10"
 DNS_DOMAIN="cluster.local"
 DNS_REPLICAS=1
 
-ADMISSION_CONTROL=NamespaceAutoProvision,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota
+ADMISSION_CONTROL=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota

cluster/vagrant/config-default.sh

@@ -50,7 +50,7 @@ MASTER_USER=vagrant
 MASTER_PASSWD=vagrant
 
 # Admission Controllers to invoke prior to persisting objects in cluster
-ADMISSION_CONTROL=NamespaceLifecycle,NamespaceAutoProvision,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota
+ADMISSION_CONTROL=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota
 
 # Optional: Install node monitoring.
 ENABLE_NODE_MONITORING=true

plugin/pkg/admission/namespace/exists/admission.go

@@ -19,6 +19,7 @@ package exists
 import (
 	"fmt"
 	"io"
+	"time"
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/admission"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
@@ -73,7 +74,14 @@ func (e *exists) Admit(a admission.Attributes) (err error) {
 	if exists {
 		return nil
 	}
-	return admission.NewForbidden(a, fmt.Errorf("Namespace %s does not exist", a.GetNamespace()))
+
+	// in case of latency in our caches, make a call direct to storage to verify that it truly exists or not
+	_, err = e.client.Namespaces().Get(a.GetNamespace())
+	if err != nil {
+		return admission.NewForbidden(a, fmt.Errorf("Namespace %s does not exist", a.GetNamespace()))
+	}
+
+	return nil
 }
 
 // NewExists creates a new namespace exists admission control handler
@@ -90,7 +98,7 @@ func NewExists(c client.Interface) admission.Interface {
 		},
 		&api.Namespace{},
 		store,
-		0,
+		5*time.Minute,
 	)
 	reflector.Run()
 	return &exists{

test/e2e/framework.go

@@ -63,6 +63,10 @@ func (f *Framework) beforeEach() {
 	Expect(err).NotTo(HaveOccurred())
 
 	f.Namespace = namespace
+
+	By("Waiting for a default service account to be provisioned in namespace")
+	err = waitForDefaultServiceAccountInNamespace(c, namespace.Name)
+	Expect(err).NotTo(HaveOccurred())
 }
 
 // afterEach deletes the namespace, after reading its events.

test/e2e/pods.go

@@ -34,11 +34,22 @@ import (
 	. "github.com/onsi/gomega"
 )
 
+// createNamespaceIfDoesNotExist ensures that the namespace with specified name exists, or returns an error
+func createNamespaceIfDoesNotExist(c *client.Client, name string) (*api.Namespace, error) {
+	namespace, err := c.Namespaces().Get(name)
+	if err != nil {
+		namespace, err = c.Namespaces().Create(&api.Namespace{ObjectMeta: api.ObjectMeta{Name: name}})
+	}
+	return namespace, err
+}
+
 func runLivenessTest(c *client.Client, podDescr *api.Pod, expectRestart bool) {
 	ns := "e2e-test-" + string(util.NewUUID())
+	_, err := createNamespaceIfDoesNotExist(c, ns)
+	expectNoError(err, fmt.Sprintf("creating namespace %s", ns))
 
 	By(fmt.Sprintf("Creating pod %s in namespace %s", podDescr.Name, ns))
-	_, err := c.Pods(ns).Create(podDescr)
+	_, err = c.Pods(ns).Create(podDescr)
 	expectNoError(err, fmt.Sprintf("creating pod %s", podDescr.Name))
 
 	// At the end of the test, clean up by removing the pod.
@@ -85,10 +96,13 @@ func runLivenessTest(c *client.Client, podDescr *api.Pod, expectRestart bool) {
 // testHostIP tests that a pod gets a host IP
 func testHostIP(c *client.Client, pod *api.Pod) {
 	ns := "e2e-test-" + string(util.NewUUID())
+	_, err := createNamespaceIfDoesNotExist(c, ns)
+	expectNoError(err, fmt.Sprintf("creating namespace %s", ns))
+
 	podClient := c.Pods(ns)
 	By("creating pod")
 	defer podClient.Delete(pod.Name, nil)
-	_, err := podClient.Create(pod)
+	_, err = podClient.Create(pod)
 	if err != nil {
 		Fail(fmt.Sprintf("Failed to create pod: %v", err))
 	}

test/e2e/util.go

@@ -58,6 +58,13 @@ const (
 
 	// How often to poll pods.
 	podPoll = 5 * time.Second
+
+	// service accounts are provisioned after namespace creation
+	// a service account is required to support pod creation in a namespace as part of admission control
+	serviceAccountProvisionTimeout = 2 * time.Minute
+
+	// How often to poll for service accounts
+	serviceAccountPoll = 5 * time.Second
 )
 
 type CloudConfig struct {
@@ -205,6 +212,20 @@ func waitForPodsRunningReady(ns string, minPods int, timeout time.Duration) erro
 	return fmt.Errorf("Not all pods in namespace '%s' running and ready within %v", ns, timeout)
 }
 
+func waitForServiceAccountInNamespace(c *client.Client, ns, serviceAccountName string, poll, timeout time.Duration) error {
+	Logf("Waiting up to %v for service account %s to be provisioned in ns %s", timeout, serviceAccountName, ns)
+	for start := time.Now(); time.Since(start) < timeout; time.Sleep(poll) {
+		_, err := c.ServiceAccounts(ns).Get(serviceAccountName)
+		if err != nil {
+			Logf("Get service account %s in ns %s failed, ignoring for %v: %v", serviceAccountName, ns, poll, err)
+			continue
+		}
+		Logf("Service account %s in ns %s found. (%v)", serviceAccountName, ns, time.Since(start))
+		return nil
+	}
+	return fmt.Errorf("Service account %s in namespace %s not ready within %v", serviceAccountName, ns, timeout)
+}
+
 func waitForPodCondition(c *client.Client, ns, podName, desc string, poll, timeout time.Duration, condition podCondition) error {
 	Logf("Waiting up to %v for pod %s status to be %s", timeout, podName, desc)
 	for start := time.Now(); time.Since(start) < timeout; time.Sleep(poll) {
@@ -223,6 +244,13 @@ func waitForPodCondition(c *client.Client, ns, podName, desc string, poll, timeo
 	return fmt.Errorf("gave up waiting for pod '%s' to be '%s' after %v", podName, desc, timeout)
 }
 
+// waitForDefaultServiceAccountInNamespace waits for the default service account to be provisioned
+// the default service account is what is associated with pods when they do not specify a service account
+// as a result, pods are not able to be provisioned in a namespace until the service account is provisioned
+func waitForDefaultServiceAccountInNamespace(c *client.Client, namespace string) error {
+	return waitForServiceAccountInNamespace(c, namespace, "default", serviceAccountPoll, serviceAccountProvisionTimeout)
+}
+
 // createNS should be used by every test, note that we append a common prefix to the provided test name.
 func createTestingNS(baseName string, c *client.Client) (*api.Namespace, error) {
 	namespaceObj := &api.Namespace{