Add granular networking e2e tests

This change adds 2 new e2e tests to test that:

 - a pod can reach another pod on the same node
 - a pod can reach another pod on another node

The intention is to provide a more granular indication of networking
failure than the existing intra-pod test for the benefit of network
plugin developers.

test/e2e/networking.go

@@ -17,6 +17,7 @@ limitations under the License.
 package e2e
 
 import (
+	"errors"
 	"fmt"
 	"net/http"
 	"strings"
@@ -51,32 +52,7 @@ var _ = Describe("Networking", func() {
 
 	It("should provide Internet connection for containers [Conformance]", func() {
 		By("Running container which tries to wget google.com")
-		podName := "wget-test"
-		contName := "wget-test-container"
-		pod := &api.Pod{
-			TypeMeta: unversioned.TypeMeta{
-				Kind: "Pod",
-			},
-			ObjectMeta: api.ObjectMeta{
-				Name: podName,
-			},
-			Spec: api.PodSpec{
-				Containers: []api.Container{
-					{
-						Name:    contName,
-						Image:   "gcr.io/google_containers/busybox:1.24",
-						Command: []string{"wget", "-s", "google.com"},
-					},
-				},
-				RestartPolicy: api.RestartPolicyNever,
-			},
-		}
-		_, err := f.Client.Pods(f.Namespace.Name).Create(pod)
-		expectNoError(err)
-		defer f.Client.Pods(f.Namespace.Name).Delete(podName, nil)
-
-		By("Verify that the pod succeed")
-		expectNoError(waitForPodSuccessInNamespace(f.Client, podName, contName, f.Namespace.Name))
+		expectNoError(checkConnectivityToHost(f, "", "wget-test", "google.com"))
 	})
 
 	// First test because it has no dependencies on variables created later on.
@@ -136,16 +112,9 @@ var _ = Describe("Networking", func() {
 
 		By("Creating a webserver (pending) pod on each node")
 
-		nodes := ListSchedulableNodesOrDie(f.Client)
-		// previous tests may have cause failures of some nodes. Let's skip
-		// 'Not Ready' nodes, just in case (there is no need to fail the test).
-		filterNodes(nodes, func(node api.Node) bool {
-			return !node.Spec.Unschedulable && isNodeConditionSetAsExpected(&node, api.NodeReady, true)
-		})
+		nodes, err := getReadyNodes(f)
+		expectNoError(err)
 
-		if len(nodes.Items) == 0 {
-			Failf("No Ready nodes found.")
-		}
 		if len(nodes.Items) == 1 {
 			// in general, the test requires two nodes. But for local development, often a one node cluster
 			// is created, for simplicity and speed. (see issue #10012). We permit one-node test
@@ -241,6 +210,49 @@ var _ = Describe("Networking", func() {
 		Expect(string(body)).To(Equal("pass"))
 	})
 
+	// Marked with [Flaky] until the tests prove themselves stable.
+	Describe("[Flaky] Granular Checks", func() {
+
+		It("should function for pod communication on a single node", func() {
+
+			By("Picking a node")
+			nodes, err := getReadyNodes(f)
+			expectNoError(err)
+			node := nodes.Items[0]
+
+			By("Creating a webserver pod")
+			podName := "same-node-webserver"
+			defer f.Client.Pods(f.Namespace.Name).Delete(podName, nil)
+			ip := launchWebserverPod(f, podName, node.Name)
+
+			By("Checking that the webserver is accessible from a pod on the same node")
+			expectNoError(checkConnectivityToHost(f, node.Name, "same-node-wget", ip))
+		})
+
+		It("should function for pod communication between nodes", func() {
+
+			podClient := f.Client.Pods(f.Namespace.Name)
+
+			By("Picking multiple nodes")
+			nodes, err := getReadyNodes(f)
+			expectNoError(err)
+
+			if len(nodes.Items) == 1 {
+				Skipf("The test requires two Ready nodes on %s, but found just one.", testContext.Provider)
+			}
+
+			node1 := nodes.Items[0]
+			node2 := nodes.Items[1]
+
+			By("Creating a webserver pod")
+			podName := "different-node-webserver"
+			defer podClient.Delete(podName, nil)
+			ip := launchWebserverPod(f, podName, node1.Name)
+
+			By("Checking that the webserver is accessible from a pod on a different node")
+			expectNoError(checkConnectivityToHost(f, node2.Name, "different-node-wget", ip))
+		})
+	})
 })
 
 func LaunchNetTestPodPerNode(f *Framework, nodes *api.NodeList, name, version string) []string {
@@ -282,3 +294,81 @@ func LaunchNetTestPodPerNode(f *Framework, nodes *api.NodeList, name, version st
 	}
 	return podNames
 }
+
+func getReadyNodes(f *Framework) (nodes *api.NodeList, err error) {
+	nodes = ListSchedulableNodesOrDie(f.Client)
+	// previous tests may have cause failures of some nodes. Let's skip
+	// 'Not Ready' nodes, just in case (there is no need to fail the test).
+	filterNodes(nodes, func(node api.Node) bool {
+		return !node.Spec.Unschedulable && isNodeConditionSetAsExpected(&node, api.NodeReady, true)
+	})
+
+	if len(nodes.Items) == 0 {
+		return nil, errors.New("No Ready nodes found.")
+	}
+	return nodes, nil
+}
+
+func launchWebserverPod(f *Framework, podName, nodeName string) (ip string) {
+	containerName := fmt.Sprintf("%s-container", podName)
+	port := 8080
+	pod := &api.Pod{
+		TypeMeta: unversioned.TypeMeta{
+			Kind: "Pod",
+		},
+		ObjectMeta: api.ObjectMeta{
+			Name: podName,
+		},
+		Spec: api.PodSpec{
+			Containers: []api.Container{
+				{
+					Name:  containerName,
+					Image: "gcr.io/google_containers/porter:cd5cb5791ebaa8641955f0e8c2a9bed669b1eaab",
+					Env:   []api.EnvVar{{Name: fmt.Sprintf("SERVE_PORT_%d", port), Value: "foo"}},
+					Ports: []api.ContainerPort{{ContainerPort: port}},
+				},
+			},
+			NodeName:      nodeName,
+			RestartPolicy: api.RestartPolicyNever,
+		},
+	}
+	podClient := f.Client.Pods(f.Namespace.Name)
+	_, err := podClient.Create(pod)
+	expectNoError(err)
+	expectNoError(f.WaitForPodRunning(podName))
+	createdPod, err := podClient.Get(podName)
+	expectNoError(err)
+	ip = fmt.Sprintf("%s:%d", createdPod.Status.PodIP, port)
+	Logf("Target pod IP:port is %s", ip)
+	return
+}
+
+func checkConnectivityToHost(f *Framework, nodeName, podName, host string) error {
+	contName := fmt.Sprintf("%s-container", podName)
+	pod := &api.Pod{
+		TypeMeta: unversioned.TypeMeta{
+			Kind: "Pod",
+		},
+		ObjectMeta: api.ObjectMeta{
+			Name: podName,
+		},
+		Spec: api.PodSpec{
+			Containers: []api.Container{
+				{
+					Name:    contName,
+					Image:   "gcr.io/google_containers/busybox:1.24",
+					Command: []string{"wget", "-s", host},
+				},
+			},
+			NodeName:      nodeName,
+			RestartPolicy: api.RestartPolicyNever,
+		},
+	}
+	podClient := f.Client.Pods(f.Namespace.Name)
+	_, err := podClient.Create(pod)
+	if err != nil {
+		return err
+	}
+	defer podClient.Delete(podName, nil)
+	return waitForPodSuccessInNamespace(f.Client, podName, contName, f.Namespace.Name)
+}