From 49e1baa313e46a67aba07db11abc2fe5c3123e1d Mon Sep 17 00:00:00 2001 From: Phillip Wittrock Date: Tue, 10 Nov 2015 15:42:07 -0800 Subject: [PATCH] Test runner harness for node e2e tests --- hack/test-go.sh | 1 + hack/verify-flags/known-flags.txt | 3 + test/e2e_node/doc.go | 17 +++ test/e2e_node/e2e_node_suite_test.go | 45 +++++++ test/e2e_node/gcloud/gcloud.go | 187 +++++++++++++++++++++++++++ test/e2e_node/kubelet_test.go | 53 ++++++++ test/e2e_node/runner/run_e2e.go | 154 ++++++++++++++++++++++ 7 files changed, 460 insertions(+) create mode 100644 test/e2e_node/doc.go create mode 100644 test/e2e_node/e2e_node_suite_test.go create mode 100644 test/e2e_node/gcloud/gcloud.go create mode 100644 test/e2e_node/kubelet_test.go create mode 100644 test/e2e_node/runner/run_e2e.go diff --git a/hack/test-go.sh b/hack/test-go.sh index 3311f2afeb..2406878eb0 100755 --- a/hack/test-go.sh +++ b/hack/test-go.sh @@ -37,6 +37,7 @@ kube::test::find_dirs() { -o -path './release/*' \ -o -path './target/*' \ -o -path './test/e2e/*' \ + -o -path './test/e2e_node/*' \ -o -path './test/integration/*' \ \) -prune \ \) -name '*_test.go' -print0 | xargs -0n1 dirname | sed 's|^\./||' | sort -u diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 8e9587303a..53b5dde55a 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -13,6 +13,7 @@ allow-privileged api-burst api-prefix api-rate +api-server-host api-server-port api-servers api-token @@ -143,6 +144,7 @@ kubelet-certificate-authority kubelet-client-certificate kubelet-client-key kubelet-docker-endpoint +kubelet-host kubelet-host-network-sources kubelet-https kubelet-network-plugin @@ -154,6 +156,7 @@ kubelet-sync-frequency kubelet-timeout kube-master kubernetes-service-node-port +k8s-build-output label-columns last-release-pr leave-stdin-open diff --git a/test/e2e_node/doc.go b/test/e2e_node/doc.go new file mode 100644 index 0000000000..c5e79c4b28 --- /dev/null +++ b/test/e2e_node/doc.go @@ -0,0 +1,17 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_node diff --git a/test/e2e_node/e2e_node_suite_test.go b/test/e2e_node/e2e_node_suite_test.go new file mode 100644 index 0000000000..b4a230c5fc --- /dev/null +++ b/test/e2e_node/e2e_node_suite_test.go @@ -0,0 +1,45 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_node + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "flag" + "testing" +) + +var kubeletHost = flag.String("kubelet-host", "localhost", "Host address of the kubelet") +var kubeletPort = flag.Int("kubelet-port", 10250, "Kubelet port") + +var apiServerHost = flag.String("api-server-host", "localhost", "Host address of the api server") +var apiServerPort = flag.Int("api-server-port", 8080, "Api server port") + +func TestE2eNode(t *testing.T) { + flag.Parse() + RegisterFailHandler(Fail) + RunSpecs(t, "E2eNode Suite") +} + +// Setup the kubelet on the node +var _ = BeforeSuite(func() { +}) + +// Tear down the kubelet on the node +var _ = AfterSuite(func() { +}) diff --git a/test/e2e_node/gcloud/gcloud.go b/test/e2e_node/gcloud/gcloud.go new file mode 100644 index 0000000000..d073bc981a --- /dev/null +++ b/test/e2e_node/gcloud/gcloud.go @@ -0,0 +1,187 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gcloud + +import ( + "errors" + "fmt" + "math/rand" + "os/exec" + + "net" + "net/http" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/golang/glog" +) + +var freePortRegexp = regexp.MustCompile(".+:([0-9]+)") + +type TearDown func() + +type GCloudClient interface { + CopyAndWaitTillHealthy(sudo bool, remotePort string, timeout time.Duration, healthUrl string, bin string, args ...string) (*CmdHandle, error) +} + +type gCloudClientImpl struct { + host string + zone string +} + +type RunResult struct { + out []byte + err error + cmd string +} + +type CmdHandle struct { + TearDown TearDown + Output chan RunResult + LPort string +} + +func NewGCloudClient(host string, zone string) GCloudClient { + return &gCloudClientImpl{host, zone} +} + +func (gc *gCloudClientImpl) Command(cmd string, moreargs ...string) ([]byte, error) { + args := append([]string{"compute", "ssh"}) + if gc.zone != "" { + args = append(args, "--zone", gc.zone) + } + args = append(args, gc.host, "--", cmd) + args = append(args, moreargs...) + glog.V(2).Infof("Command gcloud %s", strings.Join(args, " ")) + return exec.Command("gcloud", args...).CombinedOutput() +} + +func (gc *gCloudClientImpl) TunnelCommand(sudo bool, lPort string, rPort string, cmd string, moreargs ...string) ([]byte, error) { + tunnelStr := fmt.Sprintf("-L %s:localhost:%s", lPort, rPort) + args := []string{"compute", "ssh"} + if gc.zone != "" { + args = append(args, "--zone", gc.zone) + } + args = append(args, "--ssh-flag", tunnelStr, gc.host, "--") + if sudo { + args = append(args, "sudo") + } + args = append(args, cmd) + args = append(args, moreargs...) + glog.V(2).Infof("Command gcloud %s", strings.Join(args, " ")) + return exec.Command("gcloud", args...).CombinedOutput() +} + +func (gc *gCloudClientImpl) CopyToHost(from string, to string) ([]byte, error) { + rto := fmt.Sprintf("%s:%s", gc.host, to) + args := []string{"compute", "copy-files"} + if gc.zone != "" { + args = append(args, "--zone", gc.zone) + } + args = append(args, from, rto) + glog.V(2).Infof("Command gcloud %s", strings.Join(args, " ")) + return exec.Command("gcloud", args...).CombinedOutput() +} + +func (gc *gCloudClientImpl) CopyAndRun(sudo bool, remotePort string, bin string, args ...string) *CmdHandle { + h := &CmdHandle{} + h.Output = make(chan RunResult) + + rand.Seed(time.Now().UnixNano()) + + // Define where we will copy the temp binary + tDir := fmt.Sprintf("/tmp/gcloud-e2e-%d", rand.Int31()) + _, f := filepath.Split(bin) + cmd := filepath.Join(tDir, f) + h.LPort = getLocalPort() + + h.TearDown = func() { + out, err := gc.Command("sudo", "pkill", f) + if err != nil { + h.Output <- RunResult{out, err, fmt.Sprintf("pkill %s", cmd)} + return + } + out, err = gc.Command("rm", "-rf", tDir) + if err != nil { + h.Output <- RunResult{out, err, fmt.Sprintf("rm -rf %s", tDir)} + return + } + } + + // Create the tmp directory + out, err := gc.Command("mkdir", "-p", tDir) + if err != nil { + glog.Errorf("mkdir failed %v", err) + h.Output <- RunResult{out, err, fmt.Sprintf("mkdir -p %s", tDir)} + return h + } + + // Copy the binary + out, err = gc.CopyToHost(bin, tDir) + if err != nil { + glog.Errorf("copy-files failed %v", err) + h.Output <- RunResult{out, err, fmt.Sprintf("copy-files %s %s", bin, tDir)} + return h + } + + // Do the setup + go func() { + // Start the process + out, err = gc.TunnelCommand(sudo, h.LPort, remotePort, cmd, args...) + if err != nil { + glog.Errorf("command failed %v", err) + h.Output <- RunResult{out, err, fmt.Sprintf("%s %s", cmd, strings.Join(args, " "))} + return + } + }() + return h +} + +func (gc *gCloudClientImpl) CopyAndWaitTillHealthy( + sudo bool, + remotePort string, timeout time.Duration, healthUrl string, bin string, args ...string) (*CmdHandle, error) { + h := gc.CopyAndRun(sudo, remotePort, bin, args...) + eTime := time.Now().Add(timeout) + done := false + for eTime.After(time.Now()) && !done { + select { + case r := <-h.Output: + glog.V(2).Infof("Error running %s Output:\n%s Error:\n%v", r.cmd, r.out, r.err) + return h, r.err + case <-time.After(2 * time.Second): + resp, err := http.Get(fmt.Sprintf("http://localhost:%s/%s", h.LPort, healthUrl)) + if err == nil && resp.StatusCode == http.StatusOK { + done = true + break + } + } + } + if !done { + return h, errors.New(fmt.Sprintf("Timeout waiting for service to be healthy at http://localhost:%s/%s", h.LPort, healthUrl)) + } + glog.Info("Healthz Success") + return h, nil +} + +// GetLocalPort returns a free local port that can be used for ssh tunneling +func getLocalPort() string { + l, _ := net.Listen("tcp", ":0") + defer l.Close() + return freePortRegexp.FindStringSubmatch(l.Addr().String())[1] +} diff --git a/test/e2e_node/kubelet_test.go b/test/e2e_node/kubelet_test.go new file mode 100644 index 0000000000..c07fe35940 --- /dev/null +++ b/test/e2e_node/kubelet_test.go @@ -0,0 +1,53 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_node + +import ( + "fmt" + "io/ioutil" + "net/http" + + "github.com/golang/glog" + . "github.com/onsi/ginkgo" +) + +var _ = Describe("Kubelet", func() { + BeforeEach(func() { + // Setup the client to talk to the kubelet + }) + + Describe("checking kubelet status", func() { + Context("when retrieving the node status", func() { + It("should have the container version", func() { + + // TODO: This is just a place holder, write a real test here + resp, err := http.Get(fmt.Sprintf("http://%s:%d/api/v2.0/attributes", *kubeletHost, *kubeletPort)) + if err != nil { + glog.Errorf("Error: %v", err) + return + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + glog.Errorf("Error: %v", err) + return + } + glog.Infof("Resp: %s", body) + }) + }) + }) +}) diff --git a/test/e2e_node/runner/run_e2e.go b/test/e2e_node/runner/run_e2e.go new file mode 100644 index 0000000000..b4f4fc2519 --- /dev/null +++ b/test/e2e_node/runner/run_e2e.go @@ -0,0 +1,154 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bufio" + "flag" + "fmt" + "os" + "os/exec" + "strings" + "sync" + "time" + + "runtime" + + "github.com/golang/glog" + "k8s.io/kubernetes/test/e2e_node/gcloud" + "path/filepath" +) + +type RunFunc func(host string, port string) ([]byte, error) + +type Result struct { + host string + output []byte + err error +} + +var u = sync.WaitGroup{} +var zone = flag.String("zone", "", "gce zone the hosts live in") +var hosts = flag.String("hosts", "", "hosts to test") +var wait = flag.Bool("wait", false, "if true, wait for input before running tests") +var kubeOutputRelPath = flag.String("k8s-build-output", "_output/local/bin/linux/amd64", "Where k8s binary files are written") + +var kubeRoot = "" + +const buildScriptRelPath = "hack/build-go.sh" +const ginkoTestRelPath = "test/e2e_node" +const healthyTimeoutDuration = time.Minute * 3 + +func main() { + flag.Parse() + if *hosts == "" { + glog.Fatalf("Must specific --hosts flag") + } + + // Figure out the kube root + _, path, _, _ := runtime.Caller(0) + kubeRoot, _ = filepath.Split(path) + kubeRoot = strings.Split(kubeRoot, "/test/e2e_node")[0] + + // Build the go code + out, err := exec.Command(filepath.Join(kubeRoot, buildScriptRelPath)).CombinedOutput() + if err != nil { + glog.Fatalf("Failed to build go packages %s: %v", out, err) + } + + // Copy kubelet to each host and run test + if *wait { + u.Add(1) + } + + w := sync.WaitGroup{} + for _, h := range strings.Split(*hosts, ",") { + w.Add(1) + go func(host string) { + out, err := runTests(host) + if err != nil { + glog.Infof("Failure Finished Test Suite %s %v", out, err) + } else { + glog.Infof("Success Finished Test Suite %s", out) + } + w.Done() + }(h) + } + + // Maybe wait for user input before running tests + if *wait { + WaitForUser() + } + + // Wait for the tests to finish + w.Wait() + glog.Infof("All hosts finished") +} + +func WaitForUser() { + scanner := bufio.NewScanner(os.Stdin) + fmt.Printf("Enter \"y\" to run tests\n") + for scanner.Scan() { + if strings.ToUpper(scanner.Text()) != "Y\n" { + break + } + fmt.Printf("Enter \"y\" to run tests\n") + } + u.Done() +} + +func runTests(host string) ([]byte, error) { + c := gcloud.NewGCloudClient(host, *zone) + // TODO(pwittrock): Come up with something better for bootstrapping the environment. + etcdBin := filepath.Join(kubeRoot, "third_party/etcd/etcd") + eh, err := c.CopyAndWaitTillHealthy(false, "4001", healthyTimeoutDuration, "v2/keys/", etcdBin) + defer func() { eh.TearDown() }() + if err != nil { + return nil, fmt.Errorf("Host %s failed to run command %v", host, err) + } + + apiBin := filepath.Join(kubeRoot, *kubeOutputRelPath, "kube-apiserver") + ah, err := c.CopyAndWaitTillHealthy( + true, "8080", healthyTimeoutDuration, "healthz", apiBin, "--service-cluster-ip-range", + "10.0.0.1/24", "--insecure-bind-address", "0.0.0.0", "--etcd-servers", "http://localhost:4001", + "--cluster-name", "kubernetes", "--v", "2", "--kubelet-port", "10250") + defer func() { ah.TearDown() }() + if err != nil { + return nil, fmt.Errorf("Host %s failed to run command %v", host, err) + } + + kubeletBin := filepath.Join(kubeRoot, *kubeOutputRelPath, "kubelet") + kh, err := c.CopyAndWaitTillHealthy( + true, "4194", healthyTimeoutDuration, "healthz", kubeletBin, "--api-servers", "http://localhost:8080", + "--logtostderr", "--address", "0.0.0.0", "--port", "10250") + defer func() { kh.TearDown() }() + if err != nil { + return nil, fmt.Errorf("Host %s failed to run command %v", host, err) + } + + // Run the tests + glog.Infof("Kubelet healthy on host %s", host) + glog.Infof("Kubelet host %s tunnel running on port %s", host, ah.LPort) + u.Wait() + glog.Infof("Running ginkgo tests against host %s", host) + ginkoTests := filepath.Join(kubeRoot, ginkoTestRelPath) + return exec.Command( + "ginkgo", ginkoTests, "--", + "--kubelet-host", "localhost", "--kubelet-port", kh.LPort, + "--api-server-host", "localhost", "--api-server-port", kh.LPort, + "-logtostderr").CombinedOutput() +}