diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 58897ee3eb..c6b30a527d 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -1615,6 +1615,11 @@ func NewKubectlCommand(args ...string) *kubectlBuilder { return b } +func (b *kubectlBuilder) WithEnv(env []string) *kubectlBuilder { + b.cmd.Env = env + return b +} + func (b *kubectlBuilder) WithTimeout(t <-chan time.Time) *kubectlBuilder { b.timeout = t return b diff --git a/test/e2e/kubectl.go b/test/e2e/kubectl.go index c71801a6a7..d6b0e26826 100644 --- a/test/e2e/kubectl.go +++ b/test/e2e/kubectl.go @@ -23,10 +23,11 @@ import ( "fmt" "io" "io/ioutil" + "log" "mime/multipart" "net" "net/http" - "net/url" + "net/http/httptest" "os" "os/exec" "path" @@ -37,6 +38,7 @@ import ( "strings" "time" + "github.com/elazarl/goproxy" "github.com/ghodss/yaml" "k8s.io/kubernetes/pkg/api" @@ -298,233 +300,32 @@ var _ = framework.KubeDescribe("Kubectl client", func() { framework.Failf("--host variable must be set to the full URI to the api server on e2e run.") } - // Make sure the apiServer is set to what kubectl requires - apiServer := framework.TestContext.Host - apiServerUrl, err := url.Parse(apiServer) - if err != nil { - framework.Failf("Unable to parse URL %s. Error=%s", apiServer, err) - } - apiServerUrl.Scheme = "https" - if !strings.Contains(apiServer, ":443") { - apiServerUrl.Host = apiServerUrl.Host + ":443" - } - apiServer = apiServerUrl.String() - - // Build the static kubectl - By("Finding a static kubectl for upload") - testStaticKubectlPath, err := findBinary("kubectl", "linux/amd64") - if err != nil { - framework.Logf("No kubectl found: %v.\nAttempting a local build...", err) - // Fall back to trying to build a local static kubectl - kubectlContainerPath := path.Join(framework.TestContext.RepoRoot, "/examples/kubectl-container/") - if _, err := os.Stat(path.Join(framework.TestContext.RepoRoot, "hack/build-go.sh")); err != nil { - framework.Failf("Can't build static kubectl due to missing hack/build-go.sh. Error=%s", err) - } - By("Building a static kubectl for upload") - staticKubectlBuild := exec.Command("make", "-C", kubectlContainerPath) - if out, err := staticKubectlBuild.Output(); err != nil { - framework.Failf("Unable to create static kubectl. Error=%s, Output=%q", err, out) - } - // Verify the static kubectl path - testStaticKubectlPath = path.Join(kubectlContainerPath, "kubectl") - _, err := os.Stat(testStaticKubectlPath) - if err != nil { - framework.Failf("static kubectl path could not be found in %s. Error=%s", testStaticKubectlPath, err) - } - } - By(fmt.Sprintf("Using the kubectl in %s", testStaticKubectlPath)) - - // Verify the kubeconfig path - kubeConfigFilePath := framework.TestContext.KubeConfig - _, err = os.Stat(kubeConfigFilePath) - if err != nil { - framework.Failf("kube config path could not be accessed. Error=%s", err) - } - // start exec-proxy-tester container - netexecPod := framework.ReadOrDie("test/images/netexec/pod.yaml") - // Add "validate=false" if the server version is less than 1.2. - // More details: https://github.com/kubernetes/kubernetes/issues/22884. - validateFlag := "--validate=true" - gte, err := framework.ServerVersionGTE(podProbeParametersVersion, c) - if err != nil { - framework.Failf("Failed to get server version: %v", err) - } - if !gte { - validateFlag = "--validate=false" - } - framework.RunKubectlOrDieInput(string(netexecPod[:]), "create", "-f", "-", fmt.Sprintf("--namespace=%v", ns), validateFlag) - framework.CheckPodsRunningReady(c, ns, []string{netexecContainer}, framework.PodStartTimeout) - // Clean up - defer cleanupKubectlInputs(string(netexecPod[:]), ns, netexecPodSelector) - defer func() { - if CurrentGinkgoTestDescription().Failed { - logs, err := framework.GetPodLogs(c, ns, "netexec", "netexec") - framework.Logf("Logs for netexec pod(error: %s): %s", err, logs) - } - }() - // Upload kubeconfig - type NetexecOutput struct { - Output string `json:"output"` - Error string `json:"error"` - } - - var uploadConfigOutput NetexecOutput - // Upload the kubeconfig file - By("uploading kubeconfig to netexec") - pipeConfigReader, postConfigBodyWriter, err := newStreamingUpload(kubeConfigFilePath) - if err != nil { - framework.Failf("unable to create streaming upload. Error: %s", err) - } - - subResourceProxyAvailable, err := framework.ServerVersionGTE(framework.SubResourcePodProxyVersion, c) - if err != nil { - framework.Failf("Unable to determine server version. Error: %s", err) - } - - var resp []byte - if subResourceProxyAvailable { - resp, err = c.Post(). - Namespace(ns). - Name("netexec"). - Resource("pods"). - SubResource("proxy"). - Suffix("upload"). - SetHeader("Content-Type", postConfigBodyWriter.FormDataContentType()). - Body(pipeConfigReader). - Do().Raw() - } else { - resp, err = c.Post(). - Prefix("proxy"). - Namespace(ns). - Name("netexec"). - Resource("pods"). - Suffix("upload"). - SetHeader("Content-Type", postConfigBodyWriter.FormDataContentType()). - Body(pipeConfigReader). - Do().Raw() - } - if err != nil { - framework.Failf("Unable to upload kubeconfig to the remote exec server due to error: %s", err) - } - - if err := json.Unmarshal(resp, &uploadConfigOutput); err != nil { - framework.Failf("Unable to read the result from the netexec server. Error: %s", err) - } - kubecConfigRemotePath := uploadConfigOutput.Output - - // Upload - pipeReader, postBodyWriter, err := newStreamingUpload(testStaticKubectlPath) - if err != nil { - framework.Failf("unable to create streaming upload. Error: %s", err) - } - - By("uploading kubectl to netexec") - var uploadOutput NetexecOutput - // Upload the kubectl binary - if subResourceProxyAvailable { - resp, err = c.Post(). - Namespace(ns). - Name("netexec"). - Resource("pods"). - SubResource("proxy"). - Suffix("upload"). - SetHeader("Content-Type", postBodyWriter.FormDataContentType()). - Body(pipeReader). - Do().Raw() - } else { - resp, err = c.Post(). - Prefix("proxy"). - Namespace(ns). - Name("netexec"). - Resource("pods"). - Suffix("upload"). - SetHeader("Content-Type", postBodyWriter.FormDataContentType()). - Body(pipeReader). - Do().Raw() - } - if err != nil { - framework.Failf("Unable to upload kubectl binary to the remote exec server due to error: %s", err) - } - - if err := json.Unmarshal(resp, &uploadOutput); err != nil { - framework.Failf("Unable to read the result from the netexec server. Error: %s", err) - } - uploadBinaryName := uploadOutput.Output - // Verify that we got the expected response back in the body - if !strings.HasPrefix(uploadBinaryName, "/uploads/") { - framework.Failf("Unable to upload kubectl binary to remote exec server. /uploads/ not in response. Response: %s", uploadBinaryName) - } - - By("Starting goproxy pods using different strategies.") - goproxyPodContents := framework.ReadOrDie("test/images/goproxy/pod.yaml") - framework.RunKubectlOrDieInput(string(goproxyPodContents[:]), "create", "-f", "-", fmt.Sprintf("--namespace=%v", ns)) - framework.CheckPodsRunningReady(c, ns, []string{goproxyContainer}, framework.PodStartTimeout) + By("Starting goproxy") + testSrv, proxyLogs := startLocalProxy() + defer testSrv.Close() + proxyAddr := testSrv.URL for _, proxyVar := range []string{"https_proxy", "HTTPS_PROXY"} { - By("Running kubectl in netexec via an HTTP proxy using " + proxyVar) - - // get the proxy address - goproxyPod, err := c.Pods(ns).Get(goproxyContainer) - if err != nil { - framework.Failf("Unable to get the goproxy pod. Error: %s", err) - } - proxyAddr := fmt.Sprintf("http://%s:8080", goproxyPod.Status.PodIP) - - By("Running kubectl in netexec via an HTTP proxy using " + proxyVar) - shellCommand := fmt.Sprintf("%s=%s .%s --kubeconfig=%s --server=%s --namespace=%s exec nginx echo running in container", - proxyVar, proxyAddr, uploadBinaryName, kubecConfigRemotePath, apiServer, ns) - framework.Logf("About to remote exec: %v", shellCommand) - // Execute kubectl on remote exec server. - var netexecShellOutput []byte - if subResourceProxyAvailable { - netexecShellOutput, err = c.Post(). - Namespace(ns). - Name("netexec"). - Resource("pods"). - SubResource("proxy"). - Suffix("shell"). - Param("shellCommand", shellCommand). - Do().Raw() - } else { - netexecShellOutput, err = c.Post(). - Prefix("proxy"). - Namespace(ns). - Name("netexec"). - Resource("pods"). - Suffix("shell"). - Param("shellCommand", shellCommand). - Do().Raw() - } - if err != nil { - framework.Failf("Unable to execute kubectl binary on the remote exec server due to error: %s", err) - } - - var netexecOuput NetexecOutput - if err := json.Unmarshal(netexecShellOutput, &netexecOuput); err != nil { - framework.Failf("Unable to read the result from the netexec server. Error: %s", err) - } - - // Get (and print!) the proxy logs here, so - // they'll be present in case the below check - // fails the test, to help diagnose #19500 if - // it recurs. - proxyLog := framework.RunKubectlOrDie("log", "goproxy", fmt.Sprintf("--namespace=%v", ns)) + proxyLogs.Reset() + By("Running kubectl via an HTTP proxy using " + proxyVar) + output := framework.NewKubectlCommand(fmt.Sprintf("--namespace=%s", ns), "exec", "nginx", "echo", "running", "in", "container"). + WithEnv(append(os.Environ(), fmt.Sprintf("%s=%s", proxyVar, proxyAddr))). + ExecOrDie() // Verify we got the normal output captured by the exec server expectedExecOutput := "running in container\n" - if netexecOuput.Output != expectedExecOutput { - framework.Failf("Unexpected kubectl exec output. Wanted %q, got %q", expectedExecOutput, netexecOuput.Output) + if output != expectedExecOutput { + framework.Failf("Unexpected kubectl exec output. Wanted %q, got %q", expectedExecOutput, output) } // Verify the proxy server logs saw the connection expectedProxyLog := fmt.Sprintf("Accepting CONNECT to %s", strings.TrimRight(strings.TrimLeft(framework.TestContext.Host, "https://"), "/api")) + proxyLog := proxyLogs.String() if !strings.Contains(proxyLog, expectedProxyLog) { framework.Failf("Missing expected log result on proxy server for %s. Expected: %q, got %q", proxyVar, expectedProxyLog, proxyLog) } } - // Clean up the goproxyPod - cleanupKubectlInputs(string(goproxyPodContents[:]), ns, goproxyPodSelector) }) It("should support inline execution and attach", func() { @@ -1656,3 +1457,11 @@ func findBinary(binName string, platform string) (string, error) { } return binPath, fmt.Errorf("Could not find %v for %v", binName, platform) } + +func startLocalProxy() (srv *httptest.Server, logs *bytes.Buffer) { + logs = &bytes.Buffer{} + p := goproxy.NewProxyHttpServer() + p.Verbose = true + p.Logger = log.New(logs, "", 0) + return httptest.NewServer(p), logs +}