mirror of https://github.com/k3s-io/k3s
Revert 'Merge pull request #6008 from GoogleCloudPlatform/revert-5773-e2e_kubectl_util' with additional clean up, disambiguate the container name from the json contents.
jayunit100
jayunit100
parent
3b0db99692
commit
ef1508867d
|
|
@ -17,16 +17,14 @@ limitations under the License.
|
|||
package e2e
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
)
|
||||
|
|
@ -65,7 +63,7 @@ var _ = Describe("kubectl", func() {
|
|||
|
||||
By("creating a replication controller")
|
||||
runKubectl("create", "-f", nautilusPath)
|
||||
validateController(c, nautilusImage, 2)
|
||||
validateController(c, nautilusImage, 2, "update-demo", updateDemoSelector, getUDData("nautilus.jpg"))
|
||||
})
|
||||
|
||||
It("should scale a replication controller", func() {
|
||||
|
|
@ -73,13 +71,13 @@ var _ = Describe("kubectl", func() {
|
|||
|
||||
By("creating a replication controller")
|
||||
runKubectl("create", "-f", nautilusPath)
|
||||
validateController(c, nautilusImage, 2)
|
||||
validateController(c, nautilusImage, 2, "update-demo", updateDemoSelector, getUDData("nautilus.jpg"))
|
||||
By("scaling down the replication controller")
|
||||
runKubectl("resize", "rc", "update-demo-nautilus", "--replicas=1")
|
||||
validateController(c, nautilusImage, 1)
|
||||
validateController(c, nautilusImage, 1, "update-demo", updateDemoSelector, getUDData("nautilus.jpg"))
|
||||
By("scaling up the replication controller")
|
||||
runKubectl("resize", "rc", "update-demo-nautilus", "--replicas=2")
|
||||
validateController(c, nautilusImage, 2)
|
||||
validateController(c, nautilusImage, 2, "update-demo", updateDemoSelector, getUDData("nautilus.jpg"))
|
||||
})
|
||||
|
||||
It("should do a rolling update of a replication controller", func() {
|
||||
|
|
@ -88,10 +86,10 @@ var _ = Describe("kubectl", func() {
|
|||
|
||||
By("creating the initial replication controller")
|
||||
runKubectl("create", "-f", nautilusPath)
|
||||
validateController(c, nautilusImage, 2)
|
||||
validateController(c, nautilusImage, 2, "update-demo", updateDemoSelector, getUDData("nautilus.jpg"))
|
||||
By("rollingupdate to new replication controller")
|
||||
runKubectl("rollingupdate", "update-demo-nautilus", "--update-period=1s", "-f", kittenPath)
|
||||
validateController(c, kittenImage, 2)
|
||||
validateController(c, kittenImage, 2, "update-demo", updateDemoSelector, getUDData("kitten.jpg"))
|
||||
})
|
||||
})
|
||||
|
||||
|
|
@ -153,131 +151,38 @@ func makeRequestToGuestbook(c *client.Client, cmd, value string) (string, error)
|
|||
return string(result), err
|
||||
}
|
||||
|
||||
func cleanup(filePath string, selectors ...string) {
|
||||
By("using stop to clean up resources")
|
||||
runKubectl("stop", "-f", filePath)
|
||||
|
||||
for _, selector := range selectors {
|
||||
resources := runKubectl("get", "pods,rc,se", "-l", selector, "--no-headers")
|
||||
if resources != "" {
|
||||
Failf("Resources left running after stop:\n%s", resources)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func validateController(c *client.Client, image string, replicas int) {
|
||||
|
||||
getPodsTemplate := "--template={{range.items}}{{.id}} {{end}}"
|
||||
|
||||
// NB: kubectl adds the "exists" function to the standard template functions.
|
||||
// This lets us check to see if the "running" entry exists for each of the containers
|
||||
// we care about. Exists will never return an error and it's safe to check a chain of
|
||||
// things, any one of which may not exist. In the below template, all of info,
|
||||
// containername, and running might be nil, so the normal index function isn't very
|
||||
// helpful.
|
||||
// This template is unit-tested in kubectl, so if you change it, update the unit test.
|
||||
//
|
||||
// You can read about the syntax here: http://golang.org/pkg/text/template/
|
||||
getContainerStateTemplate := fmt.Sprintf(`--template={{and (exists . "currentState" "info" "%s" "state" "running")}}`, updateDemoContainer)
|
||||
|
||||
getImageTemplate := fmt.Sprintf(`--template={{(index .currentState.info "%s").image}}`, updateDemoContainer)
|
||||
|
||||
By(fmt.Sprintf("waiting for all containers in %s pods to come up.", updateDemoSelector))
|
||||
for start := time.Now(); time.Since(start) < podStartTimeout; time.Sleep(5 * time.Second) {
|
||||
getPodsOutput := runKubectl("get", "pods", "-o", "template", getPodsTemplate, "-l", updateDemoSelector)
|
||||
pods := strings.Fields(getPodsOutput)
|
||||
if numPods := len(pods); numPods != replicas {
|
||||
By(fmt.Sprintf("Replicas for %s: expected=%d actual=%d", updateDemoSelector, replicas, numPods))
|
||||
continue
|
||||
}
|
||||
var runningPods []string
|
||||
for _, podID := range pods {
|
||||
running := runKubectl("get", "pods", podID, "-o", "template", getContainerStateTemplate)
|
||||
if running == "false" {
|
||||
Logf("%s is created but not running", podID)
|
||||
continue
|
||||
}
|
||||
|
||||
currentImage := runKubectl("get", "pods", podID, "-o", "template", getImageTemplate)
|
||||
if currentImage != image {
|
||||
Logf("%s is created but running wrong image; expected: %s, actual: %s", podID, image, currentImage)
|
||||
continue
|
||||
}
|
||||
|
||||
data, err := getData(c, podID)
|
||||
if err != nil {
|
||||
Logf("%s is running right image but fetching data failed: %v", podID, err)
|
||||
continue
|
||||
}
|
||||
if strings.Contains(data.image, image) {
|
||||
Logf("%s is running right image but fetched data has the wrong info: %s", podID, data)
|
||||
continue
|
||||
}
|
||||
|
||||
Logf("%s is verified up and running", podID)
|
||||
runningPods = append(runningPods, podID)
|
||||
}
|
||||
if len(runningPods) == replicas {
|
||||
return
|
||||
}
|
||||
}
|
||||
Failf("Timed out after %v seconds waiting for %s pods to reach valid state", podStartTimeout.Seconds(), updateDemoSelector)
|
||||
}
|
||||
|
||||
type updateDemoData struct {
|
||||
image string `json:"image"`
|
||||
Image string
|
||||
}
|
||||
|
||||
func getData(c *client.Client, podID string) (*updateDemoData, error) {
|
||||
body, err := c.Get().
|
||||
Prefix("proxy").
|
||||
Resource("pods").
|
||||
Name(podID).
|
||||
Suffix("data.json").
|
||||
Do().
|
||||
Raw()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
Logf("got data: %s", body)
|
||||
var data updateDemoData
|
||||
err = json.Unmarshal(body, &data)
|
||||
return &data, err
|
||||
}
|
||||
// getUDData creates a validator function based on the input string (i.e. kitten.jpg).
|
||||
// For example, if you send "kitten.jpg", this function veridies that the image jpg = kitten.jpg
|
||||
// in the container's json field.
|
||||
func getUDData(jpgExpected string) func(*client.Client, string) error {
|
||||
|
||||
func kubectlCmd(args ...string) *exec.Cmd {
|
||||
defaultArgs := []string{}
|
||||
if testContext.kubeConfig != "" {
|
||||
defaultArgs = append(defaultArgs, "--"+clientcmd.RecommendedConfigPathFlag+"="+testContext.kubeConfig)
|
||||
} else {
|
||||
defaultArgs = append(defaultArgs, "--"+clientcmd.FlagAuthPath+"="+testContext.authConfig)
|
||||
if testContext.certDir != "" {
|
||||
defaultArgs = append(defaultArgs,
|
||||
fmt.Sprintf("--certificate-authority=%s", filepath.Join(testContext.certDir, "ca.crt")),
|
||||
fmt.Sprintf("--client-certificate=%s", filepath.Join(testContext.certDir, "kubecfg.crt")),
|
||||
fmt.Sprintf("--client-key=%s", filepath.Join(testContext.certDir, "kubecfg.key")))
|
||||
// getUDData validates data.json in the update-demo (returns nil if data is ok).
|
||||
return func(c *client.Client, podID string) error {
|
||||
Logf("validating pod %s", podID)
|
||||
body, err := c.Get().
|
||||
Prefix("proxy").
|
||||
Resource("pods").
|
||||
Name(podID).
|
||||
Suffix("data.json").
|
||||
Do().
|
||||
Raw()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
Logf("got data: %s", body)
|
||||
var data updateDemoData
|
||||
if err := json.Unmarshal(body, &data); err != nil {
|
||||
return err
|
||||
}
|
||||
Logf("Unmarshalled json jpg/img => %s , expecting %s .", data, jpgExpected)
|
||||
if strings.Contains(data.Image, jpgExpected) {
|
||||
return nil
|
||||
} else {
|
||||
return errors.New(fmt.Sprintf("data served up in container is innaccurate, %s didn't contain %s", data, jpgExpected))
|
||||
}
|
||||
}
|
||||
kubectlArgs := append(defaultArgs, args...)
|
||||
// TODO: Remove this once gcloud writes a proper entry in the kubeconfig file.
|
||||
if testContext.provider == "gke" {
|
||||
kubectlArgs = append(kubectlArgs, "--server="+testContext.host)
|
||||
}
|
||||
cmd := exec.Command("kubectl", kubectlArgs...)
|
||||
Logf("Running '%s %s'", cmd.Path, strings.Join(cmd.Args, " "))
|
||||
return cmd
|
||||
}
|
||||
|
||||
func runKubectl(args ...string) string {
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd := kubectlCmd(args...)
|
||||
cmd.Stdout, cmd.Stderr = &stdout, &stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
Failf("Error running %v:\nCommand stdout:\n%v\nstderr:\n%v\n", cmd, cmd.Stdout, cmd.Stderr)
|
||||
return ""
|
||||
}
|
||||
Logf(stdout.String())
|
||||
// TODO: trimspace should be unnecessary after switching to use kubectl binary directly
|
||||
return strings.TrimSpace(stdout.String())
|
||||
}
|
||||
|
|
|
|||
120
test/e2e/util.go
120
test/e2e/util.go
|
|
@ -17,16 +17,20 @@ limitations under the License.
|
|||
package e2e
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/clientauth"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
|
@ -182,3 +186,119 @@ func randomSuffix() string {
|
|||
func expectNoError(err error, explain ...interface{}) {
|
||||
ExpectWithOffset(1, err).NotTo(HaveOccurred(), explain...)
|
||||
}
|
||||
|
||||
func cleanup(filePath string, selectors ...string) {
|
||||
By("using stop to clean up resources")
|
||||
runKubectl("stop", "-f", filePath)
|
||||
|
||||
for _, selector := range selectors {
|
||||
resources := runKubectl("get", "pods,rc,se", "-l", selector, "--no-headers")
|
||||
if resources != "" {
|
||||
Failf("Resources left running after stop:\n%s", resources)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// validatorFn is the function which is individual tests will implement.
|
||||
// we may want it to return more than just an error, at some point.
|
||||
type validatorFn func(c *client.Client, podID string) error
|
||||
|
||||
// validateController is a generic mechanism for testing RC's that are running.
|
||||
// It takes a container name, a test name, and a validator function which is plugged in by a specific test.
|
||||
// "containername": this is grepped for.
|
||||
// "containerImage" : this is the name of the image we expect to be launched. Not to confuse w/ images (kitten.jpg) which are validated.
|
||||
// "testname": which gets bubbled up to the logging/failure messages if errors happen.
|
||||
// "validator" function: This function is given a podID and a client, and it can do some specific validations that way.
|
||||
func validateController(c *client.Client, containerImage string, replicas int, containername string, testname string, validator validatorFn) {
|
||||
getPodsTemplate := "--template={{range.items}}{{.id}} {{end}}"
|
||||
// NB: kubectl adds the "exists" function to the standard template functions.
|
||||
// This lets us check to see if the "running" entry exists for each of the containers
|
||||
// we care about. Exists will never return an error and it's safe to check a chain of
|
||||
// things, any one of which may not exist. In the below template, all of info,
|
||||
// containername, and running might be nil, so the normal index function isn't very
|
||||
// helpful.
|
||||
// This template is unit-tested in kubectl, so if you change it, update the unit test.
|
||||
// You can read about the syntax here: http://golang.org/pkg/text/template/.
|
||||
getContainerStateTemplate := fmt.Sprintf(`--template={{and (exists . "currentState" "info" "%s" "state" "running")}}`, containername)
|
||||
|
||||
getImageTemplate := fmt.Sprintf(`--template={{(index .currentState.info "%s").image}}`, containername)
|
||||
|
||||
By(fmt.Sprintf("waiting for all containers in %s pods to come up.", testname)) //testname should be selector
|
||||
for start := time.Now(); time.Since(start) < podStartTimeout; time.Sleep(5 * time.Second) {
|
||||
getPodsOutput := runKubectl("get", "pods", "-o", "template", getPodsTemplate, "-l", testname)
|
||||
pods := strings.Fields(getPodsOutput)
|
||||
if numPods := len(pods); numPods != replicas {
|
||||
By(fmt.Sprintf("Replicas for %s: expected=%d actual=%d", testname, replicas, numPods))
|
||||
continue
|
||||
}
|
||||
var runningPods []string
|
||||
for _, podID := range pods {
|
||||
running := runKubectl("get", "pods", podID, "-o", "template", getContainerStateTemplate)
|
||||
if running == "false" {
|
||||
Logf("%s is created but not running", podID)
|
||||
continue
|
||||
}
|
||||
|
||||
currentImage := runKubectl("get", "pods", podID, "-o", "template", getImageTemplate)
|
||||
if currentImage != containerImage {
|
||||
Logf("%s is created but running wrong image; expected: %s, actual: %s", podID, containerImage, currentImage)
|
||||
continue
|
||||
}
|
||||
|
||||
// Call the generic validator function here.
|
||||
// This might validate for example, that (1) getting a url works and (2) url is serving correct content.
|
||||
if err := validator(c, podID); err != nil {
|
||||
Logf("%s is running right image but validator function failed: %v", podID, err)
|
||||
continue
|
||||
}
|
||||
|
||||
Logf("%s is verified up and running", podID)
|
||||
runningPods = append(runningPods, podID)
|
||||
}
|
||||
// If we reach here, then all our checks passed.
|
||||
if len(runningPods) == replicas {
|
||||
return
|
||||
}
|
||||
}
|
||||
// Reaching here means that one of more checks failed multiple times. Assuming its not a race condition, something is broken.
|
||||
Failf("Timed out after %v seconds waiting for %s pods to reach valid state", podStartTimeout.Seconds(), testname)
|
||||
}
|
||||
|
||||
// kubectlCmd runs the kubectl executable.
|
||||
func kubectlCmd(args ...string) *exec.Cmd {
|
||||
defaultArgs := []string{}
|
||||
if testContext.kubeConfig != "" {
|
||||
defaultArgs = append(defaultArgs, "--"+clientcmd.RecommendedConfigPathFlag+"="+testContext.kubeConfig)
|
||||
} else {
|
||||
defaultArgs = append(defaultArgs, "--"+clientcmd.FlagAuthPath+"="+testContext.authConfig)
|
||||
if testContext.certDir != "" {
|
||||
defaultArgs = append(defaultArgs,
|
||||
fmt.Sprintf("--certificate-authority=%s", filepath.Join(testContext.certDir, "ca.crt")),
|
||||
fmt.Sprintf("--client-certificate=%s", filepath.Join(testContext.certDir, "kubecfg.crt")),
|
||||
fmt.Sprintf("--client-key=%s", filepath.Join(testContext.certDir, "kubecfg.key")))
|
||||
}
|
||||
}
|
||||
kubectlArgs := append(defaultArgs, args...)
|
||||
// TODO: Remove this once gcloud writes a proper entry in the kubeconfig file.
|
||||
if testContext.provider == "gke" {
|
||||
kubectlArgs = append(kubectlArgs, "--server="+testContext.host)
|
||||
}
|
||||
//TODO: the "kubectl" path string might be worth externalizing into an (optional) ginko arg.
|
||||
cmd := exec.Command("kubectl", kubectlArgs...)
|
||||
Logf("Running '%s %s'", cmd.Path, strings.Join(cmd.Args, " "))
|
||||
return cmd
|
||||
}
|
||||
|
||||
func runKubectl(args ...string) string {
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd := kubectlCmd(args...)
|
||||
cmd.Stdout, cmd.Stderr = &stdout, &stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
Failf("Error running %v:\nCommand stdout:\n%v\nstderr:\n%v\n", cmd, cmd.Stdout, cmd.Stderr)
|
||||
return ""
|
||||
}
|
||||
Logf(stdout.String())
|
||||
// TODO: trimspace should be unnecessary after switching to use kubectl binary directly
|
||||
return strings.TrimSpace(stdout.String())
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue