diff --git a/test/e2e/kubectl.go b/test/e2e/kubectl.go index 805abd43b8..af9de8fa92 100644 --- a/test/e2e/kubectl.go +++ b/test/e2e/kubectl.go @@ -17,16 +17,14 @@ limitations under the License. package e2e import ( - "bytes" "encoding/json" + "errors" "fmt" - "os/exec" "path/filepath" "strings" "time" "github.com/GoogleCloudPlatform/kubernetes/pkg/client" - "github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd" . "github.com/onsi/ginkgo" ) @@ -65,7 +63,7 @@ var _ = Describe("kubectl", func() { By("creating a replication controller") runKubectl("create", "-f", nautilusPath) - validateController(c, nautilusImage, 2) + validateController(c, nautilusImage, 2, "update-demo", updateDemoSelector, getUDData("nautilus.jpg")) }) It("should scale a replication controller", func() { @@ -73,13 +71,13 @@ var _ = Describe("kubectl", func() { By("creating a replication controller") runKubectl("create", "-f", nautilusPath) - validateController(c, nautilusImage, 2) + validateController(c, nautilusImage, 2, "update-demo", updateDemoSelector, getUDData("nautilus.jpg")) By("scaling down the replication controller") runKubectl("resize", "rc", "update-demo-nautilus", "--replicas=1") - validateController(c, nautilusImage, 1) + validateController(c, nautilusImage, 1, "update-demo", updateDemoSelector, getUDData("nautilus.jpg")) By("scaling up the replication controller") runKubectl("resize", "rc", "update-demo-nautilus", "--replicas=2") - validateController(c, nautilusImage, 2) + validateController(c, nautilusImage, 2, "update-demo", updateDemoSelector, getUDData("nautilus.jpg")) }) It("should do a rolling update of a replication controller", func() { @@ -88,10 +86,10 @@ var _ = Describe("kubectl", func() { By("creating the initial replication controller") runKubectl("create", "-f", nautilusPath) - validateController(c, nautilusImage, 2) + validateController(c, nautilusImage, 2, "update-demo", updateDemoSelector, getUDData("nautilus.jpg")) By("rollingupdate to new replication controller") runKubectl("rollingupdate", "update-demo-nautilus", "--update-period=1s", "-f", kittenPath) - validateController(c, kittenImage, 2) + validateController(c, kittenImage, 2, "update-demo", updateDemoSelector, getUDData("kitten.jpg")) }) }) @@ -153,131 +151,38 @@ func makeRequestToGuestbook(c *client.Client, cmd, value string) (string, error) return string(result), err } -func cleanup(filePath string, selectors ...string) { - By("using stop to clean up resources") - runKubectl("stop", "-f", filePath) - - for _, selector := range selectors { - resources := runKubectl("get", "pods,rc,se", "-l", selector, "--no-headers") - if resources != "" { - Failf("Resources left running after stop:\n%s", resources) - } - } -} - -func validateController(c *client.Client, image string, replicas int) { - - getPodsTemplate := "--template={{range.items}}{{.id}} {{end}}" - - // NB: kubectl adds the "exists" function to the standard template functions. - // This lets us check to see if the "running" entry exists for each of the containers - // we care about. Exists will never return an error and it's safe to check a chain of - // things, any one of which may not exist. In the below template, all of info, - // containername, and running might be nil, so the normal index function isn't very - // helpful. - // This template is unit-tested in kubectl, so if you change it, update the unit test. - // - // You can read about the syntax here: http://golang.org/pkg/text/template/ - getContainerStateTemplate := fmt.Sprintf(`--template={{and (exists . "currentState" "info" "%s" "state" "running")}}`, updateDemoContainer) - - getImageTemplate := fmt.Sprintf(`--template={{(index .currentState.info "%s").image}}`, updateDemoContainer) - - By(fmt.Sprintf("waiting for all containers in %s pods to come up.", updateDemoSelector)) - for start := time.Now(); time.Since(start) < podStartTimeout; time.Sleep(5 * time.Second) { - getPodsOutput := runKubectl("get", "pods", "-o", "template", getPodsTemplate, "-l", updateDemoSelector) - pods := strings.Fields(getPodsOutput) - if numPods := len(pods); numPods != replicas { - By(fmt.Sprintf("Replicas for %s: expected=%d actual=%d", updateDemoSelector, replicas, numPods)) - continue - } - var runningPods []string - for _, podID := range pods { - running := runKubectl("get", "pods", podID, "-o", "template", getContainerStateTemplate) - if running == "false" { - Logf("%s is created but not running", podID) - continue - } - - currentImage := runKubectl("get", "pods", podID, "-o", "template", getImageTemplate) - if currentImage != image { - Logf("%s is created but running wrong image; expected: %s, actual: %s", podID, image, currentImage) - continue - } - - data, err := getData(c, podID) - if err != nil { - Logf("%s is running right image but fetching data failed: %v", podID, err) - continue - } - if strings.Contains(data.image, image) { - Logf("%s is running right image but fetched data has the wrong info: %s", podID, data) - continue - } - - Logf("%s is verified up and running", podID) - runningPods = append(runningPods, podID) - } - if len(runningPods) == replicas { - return - } - } - Failf("Timed out after %v seconds waiting for %s pods to reach valid state", podStartTimeout.Seconds(), updateDemoSelector) -} - type updateDemoData struct { - image string `json:"image"` + Image string } -func getData(c *client.Client, podID string) (*updateDemoData, error) { - body, err := c.Get(). - Prefix("proxy"). - Resource("pods"). - Name(podID). - Suffix("data.json"). - Do(). - Raw() - if err != nil { - return nil, err - } - Logf("got data: %s", body) - var data updateDemoData - err = json.Unmarshal(body, &data) - return &data, err -} +// getUDData creates a validator function based on the input string (i.e. kitten.jpg). +// For example, if you send "kitten.jpg", this function veridies that the image jpg = kitten.jpg +// in the container's json field. +func getUDData(jpgExpected string) func(*client.Client, string) error { -func kubectlCmd(args ...string) *exec.Cmd { - defaultArgs := []string{} - if testContext.kubeConfig != "" { - defaultArgs = append(defaultArgs, "--"+clientcmd.RecommendedConfigPathFlag+"="+testContext.kubeConfig) - } else { - defaultArgs = append(defaultArgs, "--"+clientcmd.FlagAuthPath+"="+testContext.authConfig) - if testContext.certDir != "" { - defaultArgs = append(defaultArgs, - fmt.Sprintf("--certificate-authority=%s", filepath.Join(testContext.certDir, "ca.crt")), - fmt.Sprintf("--client-certificate=%s", filepath.Join(testContext.certDir, "kubecfg.crt")), - fmt.Sprintf("--client-key=%s", filepath.Join(testContext.certDir, "kubecfg.key"))) + // getUDData validates data.json in the update-demo (returns nil if data is ok). + return func(c *client.Client, podID string) error { + Logf("validating pod %s", podID) + body, err := c.Get(). + Prefix("proxy"). + Resource("pods"). + Name(podID). + Suffix("data.json"). + Do(). + Raw() + if err != nil { + return err + } + Logf("got data: %s", body) + var data updateDemoData + if err := json.Unmarshal(body, &data); err != nil { + return err + } + Logf("Unmarshalled json jpg/img => %s , expecting %s .", data, jpgExpected) + if strings.Contains(data.Image, jpgExpected) { + return nil + } else { + return errors.New(fmt.Sprintf("data served up in container is innaccurate, %s didn't contain %s", data, jpgExpected)) } } - kubectlArgs := append(defaultArgs, args...) - // TODO: Remove this once gcloud writes a proper entry in the kubeconfig file. - if testContext.provider == "gke" { - kubectlArgs = append(kubectlArgs, "--server="+testContext.host) - } - cmd := exec.Command("kubectl", kubectlArgs...) - Logf("Running '%s %s'", cmd.Path, strings.Join(cmd.Args, " ")) - return cmd -} - -func runKubectl(args ...string) string { - var stdout, stderr bytes.Buffer - cmd := kubectlCmd(args...) - cmd.Stdout, cmd.Stderr = &stdout, &stderr - - if err := cmd.Run(); err != nil { - Failf("Error running %v:\nCommand stdout:\n%v\nstderr:\n%v\n", cmd, cmd.Stdout, cmd.Stderr) - return "" - } - Logf(stdout.String()) - // TODO: trimspace should be unnecessary after switching to use kubectl binary directly - return strings.TrimSpace(stdout.String()) } diff --git a/test/e2e/util.go b/test/e2e/util.go index dcc7c7f876..ca63c53cce 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -17,16 +17,20 @@ limitations under the License. package e2e import ( + "bytes" "fmt" "math/rand" + "os/exec" "path/filepath" "strconv" + "strings" "time" "github.com/GoogleCloudPlatform/kubernetes/pkg/api" "github.com/GoogleCloudPlatform/kubernetes/pkg/client" "github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd" "github.com/GoogleCloudPlatform/kubernetes/pkg/clientauth" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -182,3 +186,119 @@ func randomSuffix() string { func expectNoError(err error, explain ...interface{}) { ExpectWithOffset(1, err).NotTo(HaveOccurred(), explain...) } + +func cleanup(filePath string, selectors ...string) { + By("using stop to clean up resources") + runKubectl("stop", "-f", filePath) + + for _, selector := range selectors { + resources := runKubectl("get", "pods,rc,se", "-l", selector, "--no-headers") + if resources != "" { + Failf("Resources left running after stop:\n%s", resources) + } + } +} + +// validatorFn is the function which is individual tests will implement. +// we may want it to return more than just an error, at some point. +type validatorFn func(c *client.Client, podID string) error + +// validateController is a generic mechanism for testing RC's that are running. +// It takes a container name, a test name, and a validator function which is plugged in by a specific test. +// "containername": this is grepped for. +// "containerImage" : this is the name of the image we expect to be launched. Not to confuse w/ images (kitten.jpg) which are validated. +// "testname": which gets bubbled up to the logging/failure messages if errors happen. +// "validator" function: This function is given a podID and a client, and it can do some specific validations that way. +func validateController(c *client.Client, containerImage string, replicas int, containername string, testname string, validator validatorFn) { + getPodsTemplate := "--template={{range.items}}{{.id}} {{end}}" + // NB: kubectl adds the "exists" function to the standard template functions. + // This lets us check to see if the "running" entry exists for each of the containers + // we care about. Exists will never return an error and it's safe to check a chain of + // things, any one of which may not exist. In the below template, all of info, + // containername, and running might be nil, so the normal index function isn't very + // helpful. + // This template is unit-tested in kubectl, so if you change it, update the unit test. + // You can read about the syntax here: http://golang.org/pkg/text/template/. + getContainerStateTemplate := fmt.Sprintf(`--template={{and (exists . "currentState" "info" "%s" "state" "running")}}`, containername) + + getImageTemplate := fmt.Sprintf(`--template={{(index .currentState.info "%s").image}}`, containername) + + By(fmt.Sprintf("waiting for all containers in %s pods to come up.", testname)) //testname should be selector + for start := time.Now(); time.Since(start) < podStartTimeout; time.Sleep(5 * time.Second) { + getPodsOutput := runKubectl("get", "pods", "-o", "template", getPodsTemplate, "-l", testname) + pods := strings.Fields(getPodsOutput) + if numPods := len(pods); numPods != replicas { + By(fmt.Sprintf("Replicas for %s: expected=%d actual=%d", testname, replicas, numPods)) + continue + } + var runningPods []string + for _, podID := range pods { + running := runKubectl("get", "pods", podID, "-o", "template", getContainerStateTemplate) + if running == "false" { + Logf("%s is created but not running", podID) + continue + } + + currentImage := runKubectl("get", "pods", podID, "-o", "template", getImageTemplate) + if currentImage != containerImage { + Logf("%s is created but running wrong image; expected: %s, actual: %s", podID, containerImage, currentImage) + continue + } + + // Call the generic validator function here. + // This might validate for example, that (1) getting a url works and (2) url is serving correct content. + if err := validator(c, podID); err != nil { + Logf("%s is running right image but validator function failed: %v", podID, err) + continue + } + + Logf("%s is verified up and running", podID) + runningPods = append(runningPods, podID) + } + // If we reach here, then all our checks passed. + if len(runningPods) == replicas { + return + } + } + // Reaching here means that one of more checks failed multiple times. Assuming its not a race condition, something is broken. + Failf("Timed out after %v seconds waiting for %s pods to reach valid state", podStartTimeout.Seconds(), testname) +} + +// kubectlCmd runs the kubectl executable. +func kubectlCmd(args ...string) *exec.Cmd { + defaultArgs := []string{} + if testContext.kubeConfig != "" { + defaultArgs = append(defaultArgs, "--"+clientcmd.RecommendedConfigPathFlag+"="+testContext.kubeConfig) + } else { + defaultArgs = append(defaultArgs, "--"+clientcmd.FlagAuthPath+"="+testContext.authConfig) + if testContext.certDir != "" { + defaultArgs = append(defaultArgs, + fmt.Sprintf("--certificate-authority=%s", filepath.Join(testContext.certDir, "ca.crt")), + fmt.Sprintf("--client-certificate=%s", filepath.Join(testContext.certDir, "kubecfg.crt")), + fmt.Sprintf("--client-key=%s", filepath.Join(testContext.certDir, "kubecfg.key"))) + } + } + kubectlArgs := append(defaultArgs, args...) + // TODO: Remove this once gcloud writes a proper entry in the kubeconfig file. + if testContext.provider == "gke" { + kubectlArgs = append(kubectlArgs, "--server="+testContext.host) + } + //TODO: the "kubectl" path string might be worth externalizing into an (optional) ginko arg. + cmd := exec.Command("kubectl", kubectlArgs...) + Logf("Running '%s %s'", cmd.Path, strings.Join(cmd.Args, " ")) + return cmd +} + +func runKubectl(args ...string) string { + var stdout, stderr bytes.Buffer + cmd := kubectlCmd(args...) + cmd.Stdout, cmd.Stderr = &stdout, &stderr + + if err := cmd.Run(); err != nil { + Failf("Error running %v:\nCommand stdout:\n%v\nstderr:\n%v\n", cmd, cmd.Stdout, cmd.Stderr) + return "" + } + Logf(stdout.String()) + // TODO: trimspace should be unnecessary after switching to use kubectl binary directly + return strings.TrimSpace(stdout.String()) +}