no-snat test

Test checks that Pods can communicate with each other in the same
cluster without SNAT.

hack/verify-flags/exceptions.txt

@@ -163,6 +163,7 @@ test/e2e/common/projected.go:			Command: []string{"/mt", "--break_on_expected_co
 test/e2e/common/secrets.go:						Command: []string{"/mt", "--break_on_expected_content=false", "--retry_time=120", "--file_content_in_loop=/etc/secret-volumes/create/data-1"},
 test/e2e/common/secrets.go:						Command: []string{"/mt", "--break_on_expected_content=false", "--retry_time=120", "--file_content_in_loop=/etc/secret-volumes/delete/data-1"},
 test/e2e/common/secrets.go:						Command: []string{"/mt", "--break_on_expected_content=false", "--retry_time=120", "--file_content_in_loop=/etc/secret-volumes/update/data-3"},
+test/e2e/no-snat.go:	node_ip := v1.EnvVar{
 test/e2e_node/container_manager_test.go:		return fmt.Errorf("expected pid %d's oom_score_adj to be %d; found %d", pid, expectedOOMScoreAdj, oomScore)
 test/e2e_node/container_manager_test.go:		return fmt.Errorf("expected pid %d's oom_score_adj to be < %d; found %d", pid, expectedMaxOOMScoreAdj, oomScore)
 test/e2e_node/container_manager_test.go:		return fmt.Errorf("expected pid %d's oom_score_adj to be >= %d; found %d", pid, expectedMinOOMScoreAdj, oomScore)

test/e2e/BUILD

@@ -85,6 +85,7 @@ go_library(
         "network_policy.go",
         "networking.go",
         "networking_perf.go",
+        "no-snat.go",
         "nodeoutofdisk.go",
         "nvidia-gpus.go",
         "pod_gc.go",

test/e2e/no-snat.go

@@ -0,0 +1,258 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+	// . "github.com/onsi/gomega"
+)
+
+const (
+	testPodPort  = 8080
+	testPodImage = "gcr.io/google_containers/no-snat-test-amd64:1.0.1"
+
+	testProxyPort  = 31235 // Firewall rule allows external traffic on ports 30000-32767. I just picked a random one.
+	testProxyImage = "gcr.io/google_containers/no-snat-test-proxy-amd64:1.0.1"
+)
+
+var (
+	testPod = v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "no-snat-test",
+			Labels: map[string]string{
+				"no-snat-test": "",
+			},
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:  "no-snat-test",
+					Image: testPodImage,
+					Args:  []string{"--port", strconv.Itoa(testPodPort)},
+					Env: []v1.EnvVar{
+						{
+							Name:      "POD_IP",
+							ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "status.podIP"}},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	testProxyPod = v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "no-snat-test-proxy",
+		},
+		Spec: v1.PodSpec{
+			HostNetwork: true,
+			Containers: []v1.Container{
+				{
+					Name:  "no-snat-test-proxy",
+					Image: testProxyImage,
+					Args:  []string{"--port", strconv.Itoa(testProxyPort)},
+					Ports: []v1.ContainerPort{
+						{
+							ContainerPort: testProxyPort,
+							HostPort:      testProxyPort,
+						},
+					},
+				},
+			},
+		},
+	}
+)
+
+// Produces a pod spec that passes nip as NODE_IP env var using downward API
+func newTestPod(nodename string, nip string) *v1.Pod {
+	pod := testPod
+	node_ip := v1.EnvVar{
+		Name:  "NODE_IP",
+		Value: nip,
+	}
+	pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, node_ip)
+	pod.Spec.NodeName = nodename
+	return &pod
+}
+
+func newTestProxyPod(nodename string) *v1.Pod {
+	pod := testProxyPod
+	pod.Spec.NodeName = nodename
+	return &pod
+}
+
+func getIP(iptype v1.NodeAddressType, node *v1.Node) (string, error) {
+	for _, addr := range node.Status.Addresses {
+		if addr.Type == iptype {
+			return addr.Address, nil
+		}
+	}
+	return "", fmt.Errorf("did not find %s on Node", iptype)
+}
+
+func getSchedulable(nodes []v1.Node) (*v1.Node, error) {
+	for _, node := range nodes {
+		if node.Spec.Unschedulable == false {
+			return &node, nil
+		}
+	}
+	return nil, fmt.Errorf("all Nodes were unschedulable")
+}
+
+func checknosnatURL(proxy, pip string, ips []string) string {
+	return fmt.Sprintf("http://%s/checknosnat?target=%s&ips=%s", proxy, pip, strings.Join(ips, ","))
+}
+
+// This test verifies that a Pod on each node in a cluster can talk to Pods on every other node without SNAT.
+// We use the [Feature:NoSNAT] tag so that most jobs will skip this test by default.
+var _ = framework.KubeDescribe("NoSNAT [Feature:NoSNAT] [Slow]", func() {
+	f := framework.NewDefaultFramework("no-snat-test")
+	It("Should be able to send traffic between Pods without SNAT", func() {
+		cs := f.ClientSet
+		pc := cs.Core().Pods(f.Namespace.Name)
+		nc := cs.Core().Nodes()
+
+		By("creating a test pod on each Node")
+		nodes, err := nc.List(metav1.ListOptions{})
+		framework.ExpectNoError(err)
+		if len(nodes.Items) == 0 {
+			framework.ExpectNoError(fmt.Errorf("no Nodes in the cluster"))
+		}
+		for _, node := range nodes.Items {
+			// find the Node's internal ip address to feed to the Pod
+			inIP, err := getIP(v1.NodeInternalIP, &node)
+			framework.ExpectNoError(err)
+
+			// target Pod at Node and feed Pod Node's InternalIP
+			pod := newTestPod(node.Name, inIP)
+			_, err = pc.Create(pod)
+			framework.ExpectNoError(err)
+		}
+
+		// In some (most?) scenarios, the test harness doesn't run in the same network as the Pods,
+		// which means it can't query Pods using their cluster-internal IPs. To get around this,
+		// we create a Pod in a Node's host network, and have that Pod serve on a specific port of that Node.
+		// We can then ask this proxy Pod to query the internal endpoints served by the test Pods.
+
+		// Find the first schedulable node; masters are marked unschedulable. We don't put the proxy on the master
+		// because in some (most?) deployments firewall rules don't allow external traffic to hit ports 30000-32767
+		// on the master, but do allow this on the nodes.
+		node, err := getSchedulable(nodes.Items)
+		framework.ExpectNoError(err)
+		By("creating a no-snat-test-proxy Pod on Node " + node.Name + " port " + strconv.Itoa(testProxyPort) +
+			" so we can target our test Pods through this Node's ExternalIP")
+
+		extIP, err := getIP(v1.NodeExternalIP, node)
+		framework.ExpectNoError(err)
+		proxyNodeIP := extIP + ":" + strconv.Itoa(testProxyPort)
+
+		_, err = pc.Create(newTestProxyPod(node.Name))
+		framework.ExpectNoError(err)
+
+		By("waiting for all of the no-snat-test pods to be scheduled and running")
+		err = wait.PollImmediate(10*time.Second, 1*time.Minute, func() (bool, error) {
+			pods, err := pc.List(metav1.ListOptions{LabelSelector: "no-snat-test"})
+			if err != nil {
+				return false, err
+			}
+
+			// check all pods are running
+			for _, pod := range pods.Items {
+				if pod.Status.Phase != v1.PodRunning {
+					if pod.Status.Phase != v1.PodPending {
+						return false, fmt.Errorf("expected pod to be in phase \"Pending\" or \"Running\"")
+					}
+					return false, nil // pod is still pending
+				}
+			}
+			return true, nil // all pods are running
+		})
+		framework.ExpectNoError(err)
+
+		By("waiting for the no-snat-test-proxy Pod to be scheduled and running")
+		err = wait.PollImmediate(10*time.Second, 1*time.Minute, func() (bool, error) {
+			pod, err := pc.Get("no-snat-test-proxy", metav1.GetOptions{})
+			if err != nil {
+				return false, err
+			}
+			if pod.Status.Phase != v1.PodRunning {
+				if pod.Status.Phase != v1.PodPending {
+					return false, fmt.Errorf("expected pod to be in phase \"Pending\" or \"Running\"")
+				}
+				return false, nil // pod is still pending
+			}
+			return true, nil // pod is running
+		})
+		framework.ExpectNoError(err)
+
+		By("sending traffic from each pod to the others and checking that SNAT does not occur")
+		pods, err := pc.List(metav1.ListOptions{LabelSelector: "no-snat-test"})
+		framework.ExpectNoError(err)
+
+		// collect pod IPs
+		podIPs := []string{}
+		for _, pod := range pods.Items {
+			podIPs = append(podIPs, pod.Status.PodIP+":"+strconv.Itoa(testPodPort))
+		}
+
+		// hit the /checknosnat endpoint on each Pod, tell each Pod to check all the other Pods
+		// this test is O(n^2) but it doesn't matter because we only run this test on small clusters (~3 nodes)
+		errs := []string{}
+		client := http.Client{
+			Timeout: 5 * time.Minute,
+		}
+		for _, pip := range podIPs {
+			ips := []string{}
+			for _, ip := range podIPs {
+				if ip == pip {
+					continue
+				}
+				ips = append(ips, ip)
+			}
+			// hit /checknosnat on pip, via proxy
+			resp, err := client.Get(checknosnatURL(proxyNodeIP, pip, ips))
+			framework.ExpectNoError(err)
+
+			// check error code on the response, if 500 record the body, which will describe the error
+			if resp.StatusCode == 500 {
+				body, err := ioutil.ReadAll(resp.Body)
+				framework.ExpectNoError(err)
+				errs = append(errs, string(body))
+			}
+			resp.Body.Close()
+		}
+
+		// report the errors all at the end
+		if len(errs) > 0 {
+			str := strings.Join(errs, "\n")
+			err := fmt.Errorf("/checknosnat failed in the following cases:\n%s", str)
+			framework.ExpectNoError(err)
+		}
+	})
+})

test/images/BUILD

@@ -28,6 +28,8 @@ filegroup(
         "//test/images/net:all-srcs",
         "//test/images/netexec:all-srcs",
         "//test/images/network-tester:all-srcs",
+        "//test/images/no-snat-test:all-srcs",
+        "//test/images/no-snat-test-proxy:all-srcs",
         "//test/images/port-forward-tester:all-srcs",
         "//test/images/porter:all-srcs",
         "//test/images/resource-consumer:all-srcs",

test/images/no-snat-test-proxy/BUILD

@@ -0,0 +1,39 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "no-snat-test-proxy",
+    library = ":go_default_library",
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    tags = ["automanaged"],
+    deps = [
+        "//vendor/github.com/spf13/pflag:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/logs:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

test/images/no-snat-test-proxy/Dockerfile

@@ -0,0 +1,20 @@
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM alpine:3.5
+
+ADD no-snat-test-proxy /usr/bin/no-snat-test-proxy
+RUN chmod +x /usr/bin/no-snat-test-proxy
+
+ENTRYPOINT ["/usr/bin/no-snat-test-proxy"]

test/images/no-snat-test-proxy/Makefile

@@ -0,0 +1,29 @@
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REGISTRY?=gcr.io/google_containers
+REGISTRY_TAG?=1.0.1
+ARCH?=amd64
+NAME?=no-snat-test-proxy
+
+build:
+	go build --ldflags '-linkmode external -extldflags "-static"' -o $(NAME) main.go
+	docker build -t $(REGISTRY)/$(NAME)-$(ARCH):$(REGISTRY_TAG) .
+	rm $(NAME)
+
+push: build
+	gcloud docker -- push $(REGISTRY)/$(NAME)-$(ARCH):$(REGISTRY_TAG)
+
+all: build
+.PHONY: build push

test/images/no-snat-test-proxy/main.go

@@ -0,0 +1,101 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/spf13/pflag"
+	"k8s.io/apiserver/pkg/util/flag"
+	"k8s.io/apiserver/pkg/util/logs"
+)
+
+// This Pod's /checknosnat takes `target` and `ips` arguments, and queries {target}/checknosnat?ips={ips}
+
+type MasqTestProxy struct {
+	Port string
+}
+
+func NewMasqTestProxy() *MasqTestProxy {
+	return &MasqTestProxy{
+		Port: "31235",
+	}
+}
+
+func (m *MasqTestProxy) AddFlags(fs *pflag.FlagSet) {
+	fs.StringVar(&m.Port, "port", m.Port, "The port to serve /checknosnat endpoint on.")
+}
+
+func main() {
+	m := NewMasqTestProxy()
+	m.AddFlags(pflag.CommandLine)
+
+	flag.InitFlags()
+	logs.InitLogs()
+	defer logs.FlushLogs()
+
+	if err := m.Run(); err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+}
+
+func (m *MasqTestProxy) Run() error {
+	// register handler
+	http.HandleFunc("/checknosnat", checknosnat)
+
+	// spin up the server
+	return http.ListenAndServe(":"+m.Port, nil)
+}
+
+type handler func(http.ResponseWriter, *http.Request)
+
+func joinErrors(errs []error, sep string) string {
+	strs := make([]string, len(errs))
+	for i, err := range errs {
+		strs[i] = err.Error()
+	}
+	return strings.Join(strs, sep)
+}
+
+func checknosnatURL(pip, ips string) string {
+	return fmt.Sprintf("http://%s/checknosnat?ips=%s", pip, ips)
+}
+
+func checknosnat(w http.ResponseWriter, req *http.Request) {
+	url := checknosnatURL(req.URL.Query().Get("target"), req.URL.Query().Get("ips"))
+	resp, err := http.Get(url)
+	if err != nil {
+		w.WriteHeader(500)
+		fmt.Fprintf(w, "error querying %q, err: %v", url, err)
+	} else {
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			w.WriteHeader(500)
+			fmt.Fprintf(w, "error reading body of response from %q, err: %v", url, err)
+		} else {
+			// Respond the same status code and body as /checknosnat on the internal Pod
+			w.WriteHeader(resp.StatusCode)
+			w.Write(body)
+		}
+	}
+	resp.Body.Close()
+}

test/images/no-snat-test/BUILD

@@ -0,0 +1,39 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "no-snat-test",
+    library = ":go_default_library",
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    tags = ["automanaged"],
+    deps = [
+        "//vendor/github.com/spf13/pflag:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/logs:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

test/images/no-snat-test/Dockerfile

@@ -0,0 +1,20 @@
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM alpine:3.5
+
+ADD no-snat-test /usr/bin/no-snat-test
+RUN chmod +x /usr/bin/no-snat-test
+
+ENTRYPOINT ["/usr/bin/no-snat-test"]

test/images/no-snat-test/Makefile

@@ -0,0 +1,29 @@
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REGISTRY?=gcr.io/google_containers
+REGISTRY_TAG?=1.0.1
+ARCH?=amd64
+NAME?=no-snat-test
+
+build:
+	go build --ldflags '-linkmode external -extldflags "-static"' -o $(NAME) main.go
+	docker build -t $(REGISTRY)/$(NAME)-$(ARCH):$(REGISTRY_TAG) .
+	rm $(NAME)
+
+push: build
+	gcloud docker -- push $(REGISTRY)/$(NAME)-$(ARCH):$(REGISTRY_TAG)
+
+all: build
+.PHONY: build push

test/images/no-snat-test/main.go

@@ -0,0 +1,153 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/spf13/pflag"
+	"k8s.io/apiserver/pkg/util/flag"
+	"k8s.io/apiserver/pkg/util/logs"
+)
+
+// ip = target for /whoami query
+// rip = returned ip
+// pip = this pod's ip
+// nip = this node's ip
+
+type MasqTester struct {
+	Port string
+}
+
+func NewMasqTester() *MasqTester {
+	return &MasqTester{
+		Port: "8080",
+	}
+}
+
+func (m *MasqTester) AddFlags(fs *pflag.FlagSet) {
+	fs.StringVar(&m.Port, "port", m.Port, "The port to serve /checknosnat and /whoami endpoints on.")
+}
+
+func main() {
+	m := NewMasqTester()
+	m.AddFlags(pflag.CommandLine)
+
+	flag.InitFlags()
+	logs.InitLogs()
+	defer logs.FlushLogs()
+
+	if err := m.Run(); err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+}
+
+func (m *MasqTester) Run() error {
+	// pip is the current pod's IP and nip is the current node's IP
+	// pull the pip and nip out of the env
+	pip, ok := os.LookupEnv("POD_IP")
+	if !ok {
+		return fmt.Errorf("POD_IP env var was not present in the environment")
+	}
+	nip, ok := os.LookupEnv("NODE_IP")
+	if !ok {
+		return fmt.Errorf("NODE_IP env var was not present in the environment")
+	}
+
+	// validate that pip and nip are ip addresses.
+	if net.ParseIP(pip) == nil {
+		return fmt.Errorf("POD_IP env var contained %q, which is not an IP address", pip)
+	}
+	if net.ParseIP(nip) == nil {
+		return fmt.Errorf("NODE_IP env var contained %q, which is not an IP address", nip)
+	}
+
+	// register handlers
+	http.HandleFunc("/whoami", whoami)
+	http.HandleFunc("/checknosnat", mkChecknosnat(pip, nip))
+
+	// spin up the server
+	return http.ListenAndServe(":"+m.Port, nil)
+}
+
+type handler func(http.ResponseWriter, *http.Request)
+
+func joinErrors(errs []error, sep string) string {
+	strs := make([]string, len(errs))
+	for i, err := range errs {
+		strs[i] = err.Error()
+	}
+	return strings.Join(strs, sep)
+}
+
+// Builds checknosnat handler, using pod and node ip of current location
+func mkChecknosnat(pip string, nip string) handler {
+	// Queries /whoami for each provided ip, resp 200 if all resp bodies match this Pod's ip, 500 otherwise
+	return func(w http.ResponseWriter, req *http.Request) {
+		errs := []error{}
+		ips := strings.Split(req.URL.Query().Get("ips"), ",")
+		for _, ip := range ips {
+			if err := check(ip, pip, nip); err != nil {
+				errs = append(errs, err)
+			}
+		}
+		if len(errs) > 0 {
+			w.WriteHeader(500)
+			fmt.Fprintf(w, "%s", joinErrors(errs, ", "))
+			return
+		}
+		w.WriteHeader(200)
+	}
+}
+
+// Writes the req.RemoteAddr into the response, req.RemoteAddr is the address of the incoming connection
+func whoami(w http.ResponseWriter, req *http.Request) {
+	fmt.Fprintf(w, "%s", req.RemoteAddr)
+}
+
+// Queries ip/whoami and compares response to pip, uses nip to differentiate SNAT from other potential failure modes
+func check(ip string, pip string, nip string) error {
+	url := fmt.Sprintf("http://%s/whoami", ip)
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	rips := strings.Split(string(body), ":")
+	if rips == nil || len(rips) == 0 {
+		return fmt.Errorf("Invalid returned ip %q from %q", string(body), url)
+	}
+	rip := rips[0]
+	if rip != pip {
+		if rip == nip {
+			return fmt.Errorf("Returned ip %q != my Pod ip %q, == my Node ip %q - SNAT", rip, pip, nip)
+		} else {
+			return fmt.Errorf("Returned ip %q != my Pod ip %q or my Node ip %q - SNAT to unexpected ip (possible SNAT through unexpected interface on the way into another node)", rip, pip, nip)
+		}
+	}
+	return nil
+}