From 0c7dce77dd21c5a7bfafcda03efc7bd2578e1029 Mon Sep 17 00:00:00 2001 From: Filipe Brandenburger Date: Thu, 5 Feb 2015 16:44:58 -0800 Subject: [PATCH] Turn the hack/e2e.go commands into standalone scripts. This gets rid of bashWrap() and of running bash -s passing it a script on stdin. Also get rid of most of the runBash*() helpers. In particular, `go run hack/e2e.go -build` will now work when it needs to ask whether it is OK to download a large docker image, it used to fail since stdin was not available for the bash script using the `read` command. Tested by running a full `hack/e2e-test.sh` run including the build stage without the docker image loaded, which used to fail before this change. --- hack/e2e-internal/build-release.sh | 34 +++++++++++ hack/e2e-internal/e2e-cluster-size.sh | 36 ++++++++++++ hack/e2e-internal/e2e-down.sh | 34 +++++++++++ hack/e2e-internal/e2e-push.sh | 34 +++++++++++ hack/e2e-internal/e2e-status.sh | 34 +++++++++++ hack/e2e-internal/e2e-up.sh | 35 +++++++++++ hack/e2e-internal/e2e-watch-events.sh | 34 +++++++++++ hack/e2e.go | 83 +++++++-------------------- hack/lib/golang.sh | 1 + 9 files changed, 264 insertions(+), 61 deletions(-) create mode 100755 hack/e2e-internal/build-release.sh create mode 100755 hack/e2e-internal/e2e-cluster-size.sh create mode 100755 hack/e2e-internal/e2e-down.sh create mode 100755 hack/e2e-internal/e2e-push.sh create mode 100755 hack/e2e-internal/e2e-status.sh create mode 100755 hack/e2e-internal/e2e-up.sh create mode 100755 hack/e2e-internal/e2e-watch-events.sh diff --git a/hack/e2e-internal/build-release.sh b/hack/e2e-internal/build-release.sh new file mode 100755 index 0000000000..7e388fa8a3 --- /dev/null +++ b/hack/e2e-internal/build-release.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. + +: ${KUBE_VERSION_ROOT:=${KUBE_ROOT}} +: ${KUBECTL:="${KUBE_VERSION_ROOT}/cluster/kubectl.sh"} +: ${KUBE_CONFIG_FILE:="config-test.sh"} + +export KUBECTL KUBE_CONFIG_FILE + +source "${KUBE_ROOT}/cluster/kube-env.sh" +source "${KUBE_VERSION_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh" + +prepare-e2e + +test-build-release diff --git a/hack/e2e-internal/e2e-cluster-size.sh b/hack/e2e-internal/e2e-cluster-size.sh new file mode 100755 index 0000000000..c8309f73b1 --- /dev/null +++ b/hack/e2e-internal/e2e-cluster-size.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. + +: ${KUBE_VERSION_ROOT:=${KUBE_ROOT}} +: ${KUBECTL:="${KUBE_VERSION_ROOT}/cluster/kubectl.sh"} +: ${KUBE_CONFIG_FILE:="config-test.sh"} + +export KUBECTL KUBE_CONFIG_FILE + +source "${KUBE_ROOT}/cluster/kube-env.sh" +source "${KUBE_VERSION_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh" + +prepare-e2e + +while true; do + ${KUBECTL} --watch-only get events +done diff --git a/hack/e2e-internal/e2e-down.sh b/hack/e2e-internal/e2e-down.sh new file mode 100755 index 0000000000..2d2ab6c23c --- /dev/null +++ b/hack/e2e-internal/e2e-down.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. + +: ${KUBE_VERSION_ROOT:=${KUBE_ROOT}} +: ${KUBECTL:="${KUBE_VERSION_ROOT}/cluster/kubectl.sh"} +: ${KUBE_CONFIG_FILE:="config-test.sh"} + +export KUBECTL KUBE_CONFIG_FILE + +source "${KUBE_ROOT}/cluster/kube-env.sh" +source "${KUBE_VERSION_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh" + +prepare-e2e + +test-teardown diff --git a/hack/e2e-internal/e2e-push.sh b/hack/e2e-internal/e2e-push.sh new file mode 100755 index 0000000000..4c0a13b50c --- /dev/null +++ b/hack/e2e-internal/e2e-push.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. + +: ${KUBE_VERSION_ROOT:=${KUBE_ROOT}} +: ${KUBECTL:="${KUBE_VERSION_ROOT}/cluster/kubectl.sh"} +: ${KUBE_CONFIG_FILE:="config-test.sh"} + +export KUBECTL KUBE_CONFIG_FILE + +source "${KUBE_ROOT}/cluster/kube-env.sh" +source "${KUBE_VERSION_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh" + +prepare-e2e + +"${KUBE_VERSION_ROOT}/cluster/kube-push.sh" diff --git a/hack/e2e-internal/e2e-status.sh b/hack/e2e-internal/e2e-status.sh new file mode 100755 index 0000000000..e21a391691 --- /dev/null +++ b/hack/e2e-internal/e2e-status.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. + +: ${KUBE_VERSION_ROOT:=${KUBE_ROOT}} +: ${KUBECTL:="${KUBE_VERSION_ROOT}/cluster/kubectl.sh"} +: ${KUBE_CONFIG_FILE:="config-test.sh"} + +export KUBECTL KUBE_CONFIG_FILE + +source "${KUBE_ROOT}/cluster/kube-env.sh" +source "${KUBE_VERSION_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh" + +prepare-e2e + +${KUBECTL} version diff --git a/hack/e2e-internal/e2e-up.sh b/hack/e2e-internal/e2e-up.sh new file mode 100755 index 0000000000..0f72540156 --- /dev/null +++ b/hack/e2e-internal/e2e-up.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. + +: ${KUBE_VERSION_ROOT:=${KUBE_ROOT}} +: ${KUBECTL:="${KUBE_VERSION_ROOT}/cluster/kubectl.sh"} +: ${KUBE_CONFIG_FILE:="config-test.sh"} + +export KUBECTL KUBE_CONFIG_FILE + +source "${KUBE_ROOT}/cluster/kube-env.sh" +source "${KUBE_VERSION_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh" + +prepare-e2e + +"${KUBE_VERSION_ROOT}/cluster/kube-up.sh" +test-setup diff --git a/hack/e2e-internal/e2e-watch-events.sh b/hack/e2e-internal/e2e-watch-events.sh new file mode 100755 index 0000000000..84886e5f97 --- /dev/null +++ b/hack/e2e-internal/e2e-watch-events.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. + +: ${KUBE_VERSION_ROOT:=${KUBE_ROOT}} +: ${KUBECTL:="${KUBE_VERSION_ROOT}/cluster/kubectl.sh"} +: ${KUBE_CONFIG_FILE:="config-test.sh"} + +export KUBECTL KUBE_CONFIG_FILE + +source "${KUBE_ROOT}/cluster/kube-env.sh" +source "${KUBE_VERSION_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh" + +prepare-e2e + +${KUBECTL} get minions --no-headers | wc -l diff --git a/hack/e2e.go b/hack/e2e.go index bfdeccd526..414b2da01f 100644 --- a/hack/e2e.go +++ b/hack/e2e.go @@ -45,7 +45,6 @@ var ( test = flag.Bool("test", false, "Run Ginkgo tests.") root = flag.String("root", absOrDie(filepath.Clean(filepath.Join(path.Base(os.Args[0]), ".."))), "Root directory of kubernetes repository.") verbose = flag.Bool("v", false, "If true, print all command output.") - trace_bash = flag.Bool("trace-bash", false, "If true, pass -x to bash to trace all bash commands") checkVersionSkew = flag.Bool("check_version_skew", true, ""+ "By default, verify that client and server have exact version match. "+ "You can explicitly set to false if you're, e.g., testing client changes "+ @@ -101,7 +100,11 @@ func main() { } if *build { - if !runBash("build-release", `test-build-release`) { + // The build-release script needs stdin to ask the user whether + // it's OK to download the docker image. + cmd := exec.Command(path.Join(*root, "hack/e2e-internal/build-release.sh")) + cmd.Stdin = os.Stdin + if !finishRunning("build-release", cmd) { log.Fatal("Error building. Aborting.") } } @@ -138,7 +141,7 @@ func main() { log.Fatal("Error starting e2e cluster. Aborting.") } } else if *push { - if !runBash("push", path.Join(versionRoot, "/cluster/kube-push.sh")) { + if !finishRunning("push", exec.Command(path.Join(*root, "hack/e2e-internal/e2e-push.sh"))) { log.Fatal("Error pushing e2e cluster. Aborting.") } } @@ -146,7 +149,8 @@ func main() { failure := false switch { case *ctlCmd != "": - failure = !runBash("'kubectl "+*ctlCmd+"'", "$KUBECTL "+*ctlCmd) + ctlArgs := strings.Fields(*ctlCmd) + failure = !finishRunning("'kubectl "+*ctlCmd+"'", exec.Command(path.Join(versionRoot, "cluster/kubectl.sh"), ctlArgs...)) case *test: failure = Test() } @@ -161,7 +165,7 @@ func main() { } func TearDown() bool { - return runBash("teardown", "test-teardown") + return finishRunning("teardown", exec.Command(path.Join(*root, "hack/e2e-internal/e2e-down.sh"))) } // Up brings an e2e cluster up, recreating it if one is already running. @@ -173,15 +177,13 @@ func Up() bool { } } - return runBash("up", path.Join(versionRoot, "/cluster/kube-up.sh; test-setup;")) + return finishRunning("up", exec.Command(path.Join(*root, "hack/e2e-internal/e2e-up.sh"))) } // Ensure that the cluster is large engough to run the e2e tests. func ValidateClusterSize() { // Check that there are at least 3 minions running - res, stdout, _ := runBashWithOutputs( - "validate cluster size", - "cluster/kubectl.sh get minions --no-headers | wc -l") + res, stdout, _ := finishRunningWithOutputs("validate cluster size", exec.Command(path.Join(*root, "hack/e2e-internal/e2e-watch-events.sh"))) if !res { log.Fatal("Could not get nodes to validate cluster size") } @@ -198,7 +200,7 @@ func ValidateClusterSize() { // Is the e2e cluster up? func IsUp() bool { - return runBash("get status", `$KUBECTL version`) + return finishRunning("get status", exec.Command(path.Join(*root, "hack/e2e-internal/e2e-status.sh"))) } // PrepareVersion makes sure that the specified release version is locally @@ -241,7 +243,7 @@ func PrepareVersion(version string) (string, error) { } out.Close() } - if !runRawBash("untarRelease", fmt.Sprintf("tar -C %s -zxf %s --strip-components=1", localReleaseDir, localReleaseTar)) { + if !finishRunning("untarRelease", exec.Command("tar", "-C", localReleaseDir, "-zxf", localReleaseTar, "--strip-components=1")) { log.Fatal("Failed to untar release. Aborting.") } // Now that we have the binaries saved locally, use the path to the untarred @@ -258,7 +260,7 @@ func shuffleStrings(strings []string, r *rand.Rand) { } func Test() bool { - defer runBashUntil("watchEvents", "while true; do $KUBECTL --watch-only get events; done")() + defer runBashUntil("watchEvents", exec.Command(filepath.Join(*root, "hack/e2e-internal/e2e-watch-events.sh")))() if !IsUp() { log.Fatal("Testing requested, but e2e cluster not up!") @@ -266,38 +268,13 @@ func Test() bool { ValidateClusterSize() - return runBash("Ginkgo tests", filepath.Join(*root, "hack", "ginkgo-e2e.sh")) + return finishRunning("Ginkgo tests", exec.Command(filepath.Join(*root, "hack/ginkgo-e2e.sh"))) } // All nonsense below is temporary until we have go versions of these things. -// Runs the provided bash without wrapping it in any kubernetes-specific gunk. -func runRawBashWithOutputs(stepName, bash string) (bool, string, string) { - cmd := exec.Command("bash", "-s") - if *trace_bash { - cmd.Args = append(cmd.Args, "-x") - } - cmd.Stdin = strings.NewReader(bash) - return finishRunning(stepName, cmd) -} - -func runRawBash(stepName, bashFragment string) bool { - result, _, _ := runRawBashWithOutputs(stepName, bashFragment) - return result -} - -func runBashWithOutputs(stepName, bashFragment string) (bool, string, string) { - return runRawBashWithOutputs(stepName, bashWrap(bashFragment)) -} - -func runBash(stepName, bashFragment string) bool { - return runRawBash(stepName, bashWrap(bashFragment)) -} - // call the returned anonymous function to stop. -func runBashUntil(stepName, bashFragment string) func() { - cmd := exec.Command("bash", "-s") - cmd.Stdin = strings.NewReader(bashWrap(bashFragment)) +func runBashUntil(stepName string, cmd *exec.Cmd) func() { log.Printf("Running in background: %v", stepName) stdout, stderr := bytes.NewBuffer(nil), bytes.NewBuffer(nil) cmd.Stdout, cmd.Stderr = stdout, stderr @@ -313,7 +290,7 @@ func runBashUntil(stepName, bashFragment string) func() { } } -func finishRunning(stepName string, cmd *exec.Cmd) (bool, string, string) { +func finishRunningWithOutputs(stepName string, cmd *exec.Cmd) (bool, string, string) { log.Printf("Running: %v", stepName) stdout, stderr := bytes.NewBuffer(nil), bytes.NewBuffer(nil) if *verbose { @@ -344,6 +321,11 @@ func finishRunning(stepName string, cmd *exec.Cmd) (bool, string, string) { return true, string(stdout.Bytes()), string(stderr.Bytes()) } +func finishRunning(stepName string, cmd *exec.Cmd) bool { + result, _, _ := finishRunningWithOutputs(stepName, cmd) + return result +} + func printBashOutputs(headerprefix, lineprefix, stdout, stderr string, escape bool) { // The |'s (plus appropriate prefixing) are to make this look // "YAMLish" to the Jenkins TAP plugin: @@ -398,24 +380,3 @@ func kubectlArgs() string { } return "" } - -func bashWrap(cmd string) string { - return ` -set -o errexit -set -o nounset -set -o pipefail - -export KUBE_CONFIG_FILE="config-test.sh" - -# TODO(jbeda): This will break on usage if there is a space in -# ${KUBE_ROOT}. Convert to an array? Or an exported function? -export KUBECTL="` + versionRoot + `/cluster/kubectl.sh` + kubectlArgs() + `" - -source "` + *root + `/cluster/kube-env.sh" -source "` + versionRoot + `/cluster/${KUBERNETES_PROVIDER}/util.sh" - -prepare-e2e - -` + cmd + ` -` -} diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh index d81f7e6c46..12c7aa27a7 100644 --- a/hack/lib/golang.sh +++ b/hack/lib/golang.sh @@ -55,6 +55,7 @@ readonly KUBE_TEST_PORTABLE=( contrib/for-tests/network-tester/service.json hack/e2e.go hack/e2e-suite + hack/e2e-internal hack/ginkgo-e2e.sh )