From ad37e2654edbc98b1a2b6b40dc41b454902e5c0d Mon Sep 17 00:00:00 2001 From: Phillip Wittrock Date: Thu, 3 Dec 2015 14:27:51 -0800 Subject: [PATCH] Node e2e test fixes: - Improve documentation and method naming - Fix command this is run remotely - Never reschedule the busybox logging test pod since it is supposed to terminate - Update log test condition retrylogic to correctly retry instead of failing the test - localhost -> 127.0.0.1 to work on coreos - give name to etcd to work on coreos - allow using full hostname for nodename for coreos --- test/e2e_node/e2e_node_suite_test.go | 2 + test/e2e_node/gcloud/gcloud.go | 69 +++++++++++++++------------- test/e2e_node/kubelet_test.go | 13 ++++-- test/e2e_node/runner/run_e2e.go | 25 +++++----- test/e2e_node/util.go | 21 +++++++-- 5 files changed, 76 insertions(+), 54 deletions(-) diff --git a/test/e2e_node/e2e_node_suite_test.go b/test/e2e_node/e2e_node_suite_test.go index 177074a6a3..67d2bbb8e9 100644 --- a/test/e2e_node/e2e_node_suite_test.go +++ b/test/e2e_node/e2e_node_suite_test.go @@ -14,6 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ +// To run tests in this suite +// `$ ginkgo -- --node-name node-e2e-test-1 --api-server-address --logtostderr` package e2e_node import ( diff --git a/test/e2e_node/gcloud/gcloud.go b/test/e2e_node/gcloud/gcloud.go index 6289704065..6d5ed940c5 100644 --- a/test/e2e_node/gcloud/gcloud.go +++ b/test/e2e_node/gcloud/gcloud.go @@ -34,10 +34,10 @@ import ( var freePortRegexp = regexp.MustCompile(".+:([0-9]+)") -type TearDown func() +type TearDown func() *RunResult type GCloudClient interface { - CopyAndWaitTillHealthy( + RunAndWaitTillHealthy( sudo bool, copyBin bool, remotePort string, timeout time.Duration, healthUrl string, bin string, args ...string) (*CmdHandle, error) } @@ -102,7 +102,7 @@ func (gc *gCloudClientImpl) CopyToHost(from string, to string) ([]byte, error) { return exec.Command("gcloud", args...).CombinedOutput() } -func (gc *gCloudClientImpl) CopyAndRun( +func (gc *gCloudClientImpl) Run( sudo bool, copyBin bool, remotePort string, bin string, args ...string) *CmdHandle { h := &CmdHandle{} @@ -119,54 +119,57 @@ func (gc *gCloudClientImpl) CopyAndRun( } h.LPort = getLocalPort() - h.TearDown = func() { + h.TearDown = func() *RunResult { out, err := gc.Command("sudo", "pkill", f) if err != nil { - h.Output <- RunResult{out, err, fmt.Sprintf("pkill %s", cmd)} - return + return &RunResult{out, err, fmt.Sprintf("pkill %s", f)} } out, err = gc.Command("rm", "-rf", tDir) if err != nil { - h.Output <- RunResult{out, err, fmt.Sprintf("rm -rf %s", tDir)} - return + return &RunResult{out, err, fmt.Sprintf("rm -rf %s", tDir)} } + return &RunResult{} } - // Create the tmp directory - out, err := gc.Command("mkdir", "-p", tDir) - if err != nil { - glog.Errorf("mkdir failed %v", err) - h.Output <- RunResult{out, err, fmt.Sprintf("mkdir -p %s", tDir)} - return h - } - - // Copy the binary - if copyBin { - out, err = gc.CopyToHost(bin, tDir) - if err != nil { - glog.Errorf("copy-files failed %v", err) - h.Output <- RunResult{out, err, fmt.Sprintf("copy-files %s %s", bin, tDir)} - return h - } - } - - // Do the setup + // Run the commands in a Go fn so that this method doesn't block when writing to a channel + // to report an error go func() { - // Start the process - out, err = gc.TunnelCommand(sudo, h.LPort, remotePort, tDir, fmt.Sprintf("./%s", f), args...) + // Create the tmp directory + out, err := gc.Command("mkdir", "-p", tDir) if err != nil { - glog.Errorf("command failed %v", err) - h.Output <- RunResult{out, err, fmt.Sprintf("%s %s", f, strings.Join(args, " "))} + glog.Errorf("mkdir failed %v %s", err, out) + h.Output <- RunResult{out, err, fmt.Sprintf("mkdir -p %s", tDir)} return } + + // Copy the binary + if copyBin { + out, err = gc.CopyToHost(bin, tDir) + if err != nil { + glog.Errorf("copy-files failed %v %s", err, out) + h.Output <- RunResult{out, err, fmt.Sprintf("copy-files %s %s", bin, tDir)} + return + } + } + + // Do the setup + go func() { + // Start the process + out, err = gc.TunnelCommand(sudo, h.LPort, remotePort, tDir, cmd, args...) + if err != nil { + glog.Errorf("command failed %v %s", err, out) + h.Output <- RunResult{out, err, fmt.Sprintf("%s %s", cmd, strings.Join(args, " "))} + return + } + }() }() return h } -func (gc *gCloudClientImpl) CopyAndWaitTillHealthy( +func (gc *gCloudClientImpl) RunAndWaitTillHealthy( sudo bool, copyBin bool, remotePort string, timeout time.Duration, healthUrl string, bin string, args ...string) (*CmdHandle, error) { - h := gc.CopyAndRun(sudo, copyBin, remotePort, bin, args...) + h := gc.Run(sudo, copyBin, remotePort, bin, args...) eTime := time.Now().Add(timeout) done := false for eTime.After(time.Now()) && !done { diff --git a/test/e2e_node/kubelet_test.go b/test/e2e_node/kubelet_test.go index 0e449ac689..3a7e5bea2c 100644 --- a/test/e2e_node/kubelet_test.go +++ b/test/e2e_node/kubelet_test.go @@ -43,13 +43,16 @@ var _ = Describe("Kubelet", func() { Namespace: api.NamespaceDefault, }, Spec: api.PodSpec{ + // Force the Pod to schedule to the node without a scheduler running NodeName: *nodeName, + // Don't restart the Pod since it is expected to exit + RestartPolicy: api.RestartPolicyNever, Containers: []api.Container{ { - Image: "busybox", + Image: "gcr.io/google_containers/busybox", Name: "busybox", Command: []string{"echo", "'Hello World'"}, - ImagePullPolicy: "IfNotPresent", + ImagePullPolicy: api.PullIfNotPresent, }, }, }, @@ -59,7 +62,7 @@ var _ = Describe("Kubelet", func() { }) It("it should print the output to logs", func() { - errs := Retry(time.Minute*3, time.Second*2, cl, func(cl *client.Client) error { + errs := Retry(time.Minute, time.Second*4, func() error { rc, err := cl.Pods(api.NamespaceDefault).GetLogs("busybox", &api.PodLogOptions{}).Stream() if err != nil { return err @@ -67,7 +70,9 @@ var _ = Describe("Kubelet", func() { defer rc.Close() buf := new(bytes.Buffer) buf.ReadFrom(rc) - Expect(buf.String()).To(Equal("'Hello World'\n")) + if buf.String() != "'Hello World'\n" { + return fmt.Errorf("Expected %s to match 'Hello World'", buf.String()) + } return nil }) Expect(errs).To(BeEmpty(), fmt.Sprintf("Failed to get Logs")) diff --git a/test/e2e_node/runner/run_e2e.go b/test/e2e_node/runner/run_e2e.go index 29c3db2239..9303126ec5 100644 --- a/test/e2e_node/runner/run_e2e.go +++ b/test/e2e_node/runner/run_e2e.go @@ -83,9 +83,9 @@ func main() { go func(host string) { out, err := runTests(host) if err != nil { - glog.Infof("Failure Finished Test Suite %s %v", out, err) + glog.Infof("Failure Finished Host %s Test Suite %s %v", host, out, err) } else { - glog.Infof("Success Finished Test Suite %s", out) + glog.Infof("Success Finished Host %s Test Suite %s", host, out) } w.Done() }(h) @@ -113,20 +113,21 @@ func WaitForUser() { u.Done() } -func runTests(host string) ([]byte, error) { +func runTests(fullhost string) ([]byte, error) { + host := strings.Split(fullhost, ".")[0] c := gcloud.NewGCloudClient(host, *zone) // TODO(pwittrock): Come up with something better for bootstrapping the environment. - eh, err := c.CopyAndWaitTillHealthy( - false, false, "4001", healthyTimeoutDuration, "v2/keys/", "etcd", "--data-dir", "./") + eh, err := c.RunAndWaitTillHealthy( + false, false, "4001", healthyTimeoutDuration, "v2/keys/", "etcd", "--data-dir", "./", "--name", "e2e-node") defer func() { eh.TearDown() }() if err != nil { return nil, fmt.Errorf("Host %s failed to run command %v", host, err) } apiBin := filepath.Join(kubeRoot, *kubeOutputRelPath, "kube-apiserver") - ah, err := c.CopyAndWaitTillHealthy( + ah, err := c.RunAndWaitTillHealthy( true, true, "8080", healthyTimeoutDuration, "healthz", apiBin, "--service-cluster-ip-range", - "10.0.0.1/24", "--insecure-bind-address", "0.0.0.0", "--etcd-servers", "http://localhost:4001", + "10.0.0.1/24", "--insecure-bind-address", "0.0.0.0", "--etcd-servers", "http://127.0.0.1:4001", "--cluster-name", "kubernetes", "--v", "2", "--kubelet-port", "10250") defer func() { ah.TearDown() }() if err != nil { @@ -134,8 +135,8 @@ func runTests(host string) ([]byte, error) { } kubeletBin := filepath.Join(kubeRoot, *kubeOutputRelPath, "kubelet") - kh, err := c.CopyAndWaitTillHealthy( - true, true, "10255", healthyTimeoutDuration, "healthz", kubeletBin, "--api-servers", "http://localhost:8080", + kh, err := c.RunAndWaitTillHealthy( + true, true, "10255", healthyTimeoutDuration, "healthz", kubeletBin, "--api-servers", "http://127.0.0.1:8080", "--logtostderr", "--address", "0.0.0.0", "--port", "10250") defer func() { kh.TearDown() }() if err != nil { @@ -150,8 +151,8 @@ func runTests(host string) ([]byte, error) { ginkoTests := filepath.Join(kubeRoot, ginkoTestRelPath) return exec.Command( "ginkgo", ginkoTests, "--", - "--kubelet-address", fmt.Sprintf("http://localhost:%s", kh.LPort), - "--api-server-address", fmt.Sprintf("http://localhost:%s", ah.LPort), - "--node-name", host, + "--kubelet-address", fmt.Sprintf("http://127.0.0.1:%s", kh.LPort), + "--api-server-address", fmt.Sprintf("http://127.0.0.1:%s", ah.LPort), + "--node-name", fullhost, "-logtostderr").CombinedOutput() } diff --git a/test/e2e_node/util.go b/test/e2e_node/util.go index 0082acda22..abea474aef 100644 --- a/test/e2e_node/util.go +++ b/test/e2e_node/util.go @@ -18,16 +18,27 @@ package e2e_node import ( "time" - - client "k8s.io/kubernetes/pkg/client/unversioned" ) -type RetryFn func(cl *client.Client) error +// RetryFn represents a retryable test condition. It returns an error if the condition is not met +// otherwise returns nil for success. +type RetryFn func() error -func Retry(maxWait time.Duration, wait time.Duration, cl *client.Client, retry RetryFn) []error { +// Retry retries the RetryFn for a maximum of maxWait time. The wait duration is waited between +// retries. If the success condition is not met in maxWait time, the list of encountered errors +// is returned. If successful returns an empty list. +// Example: +// Expect(Retry(time.Minute*1, time.Second*2, func() error { +// if success { +// return nil +// } else { +// return errors.New("Failed") +// } +// }).To(BeNil(), fmt.Sprintf("Failed")) +func Retry(maxWait time.Duration, wait time.Duration, retry RetryFn) []error { errs := []error{} for start := time.Now(); time.Now().Before(start.Add(maxWait)); { - if err := retry(cl); err != nil { + if err := retry(); err != nil { errs = append(errs, err) } else { return []error{}