From 4ddb160bd01ded13f9322ed28598251e098a1294 Mon Sep 17 00:00:00 2001 From: liang chenye Date: Fri, 6 May 2016 15:30:16 +0800 Subject: [PATCH] e2e-node test: check runtime ready/restartcount/status Signed-off-by: liang chenye --- test/e2e_node/container.go | 74 ++++- test/e2e_node/image_conformance_test.go | 2 +- test/e2e_node/runtime_conformance_test.go | 338 ++++++++++++++++------ 3 files changed, 315 insertions(+), 99 deletions(-) diff --git a/test/e2e_node/container.go b/test/e2e_node/container.go index 3d6925f52d..0a300fabaf 100644 --- a/test/e2e_node/container.go +++ b/test/e2e_node/container.go @@ -23,17 +23,22 @@ import ( "k8s.io/kubernetes/pkg/api" apierrs "k8s.io/kubernetes/pkg/api/errors" client "k8s.io/kubernetes/pkg/client/unversioned" + "k8s.io/kubernetes/pkg/util" "github.com/onsi/gomega/format" "github.com/onsi/gomega/types" ) -//One pod one container +// One pod one container type ConformanceContainer struct { - Container api.Container - Client *client.Client - Phase api.PodPhase - NodeName string + Container api.Container + Client *client.Client + RestartPolicy api.RestartPolicy + Volumes []api.Volume + NodeName string + Namespace string + + podName string } type ConformanceContainerEqualMatcher struct { @@ -63,36 +68,37 @@ func (matcher *ConformanceContainerEqualMatcher) NegatedFailureMessage(actual in } func (cc *ConformanceContainer) Create() error { + cc.podName = cc.Container.Name + string(util.NewUUID()) pod := &api.Pod{ ObjectMeta: api.ObjectMeta{ - //Same with the container name - Name: cc.Container.Name, - Namespace: api.NamespaceDefault, + Name: cc.podName, + Namespace: cc.Namespace, }, Spec: api.PodSpec{ NodeName: cc.NodeName, - RestartPolicy: api.RestartPolicyNever, + RestartPolicy: cc.RestartPolicy, Containers: []api.Container{ cc.Container, }, + Volumes: cc.Volumes, }, } - _, err := cc.Client.Pods(api.NamespaceDefault).Create(pod) + _, err := cc.Client.Pods(cc.Namespace).Create(pod) return err } //Same with 'delete' func (cc *ConformanceContainer) Stop() error { - return cc.Client.Pods(api.NamespaceDefault).Delete(cc.Container.Name, &api.DeleteOptions{}) + return cc.Client.Pods(cc.Namespace).Delete(cc.podName, &api.DeleteOptions{}) } func (cc *ConformanceContainer) Delete() error { - return cc.Client.Pods(api.NamespaceDefault).Delete(cc.Container.Name, &api.DeleteOptions{}) + return cc.Client.Pods(cc.Namespace).Delete(cc.podName, &api.DeleteOptions{}) } func (cc *ConformanceContainer) Get() (ConformanceContainer, error) { - pod, err := cc.Client.Pods(api.NamespaceDefault).Get(cc.Container.Name) + pod, err := cc.Client.Pods(cc.Namespace).Get(cc.podName) if err != nil { return ConformanceContainer{}, err } @@ -101,15 +107,51 @@ func (cc *ConformanceContainer) Get() (ConformanceContainer, error) { if containers == nil || len(containers) != 1 { return ConformanceContainer{}, errors.New("Failed to get container") } - return ConformanceContainer{containers[0], cc.Client, pod.Status.Phase, pod.Spec.NodeName}, nil + return ConformanceContainer{containers[0], cc.Client, pod.Spec.RestartPolicy, pod.Spec.Volumes, pod.Spec.NodeName, cc.Namespace, cc.podName}, nil +} + +func (cc *ConformanceContainer) GetStatus() (api.ContainerStatus, api.PodPhase, error) { + pod, err := cc.Client.Pods(cc.Namespace).Get(cc.podName) + if err != nil { + return api.ContainerStatus{}, api.PodUnknown, err + } + + statuses := pod.Status.ContainerStatuses + if len(statuses) != 1 { + return api.ContainerStatus{}, api.PodUnknown, errors.New("Failed to get container status") + } + return statuses[0], pod.Status.Phase, nil } func (cc *ConformanceContainer) Present() (bool, error) { - _, err := cc.Client.Pods(api.NamespaceDefault).Get(cc.Container.Name) + _, err := cc.Client.Pods(cc.Namespace).Get(cc.podName) if err == nil { return true, nil - } else if err != nil && apierrs.IsNotFound(err) { + } + if apierrs.IsNotFound(err) { return false, nil } return false, err } + +type ContainerState uint32 + +const ( + ContainerStateWaiting ContainerState = 1 << iota + ContainerStateRunning + ContainerStateTerminated + ContainerStateUnknown +) + +func GetContainerState(state api.ContainerState) ContainerState { + if state.Waiting != nil { + return ContainerStateWaiting + } + if state.Running != nil { + return ContainerStateRunning + } + if state.Terminated != nil { + return ContainerStateTerminated + } + return ContainerStateUnknown +} diff --git a/test/e2e_node/image_conformance_test.go b/test/e2e_node/image_conformance_test.go index 838145a490..7b5ab4005d 100644 --- a/test/e2e_node/image_conformance_test.go +++ b/test/e2e_node/image_conformance_test.go @@ -31,7 +31,7 @@ const ( imagePullInterval = time.Second * 15 ) -var _ = Describe("Container Conformance Test", func() { +var _ = Describe("Image Container Conformance Test", func() { var cl *client.Client BeforeEach(func() { diff --git a/test/e2e_node/runtime_conformance_test.go b/test/e2e_node/runtime_conformance_test.go index 174c5bfb56..02ce3afcd2 100644 --- a/test/e2e_node/runtime_conformance_test.go +++ b/test/e2e_node/runtime_conformance_test.go @@ -17,6 +17,10 @@ limitations under the License. package e2e_node import ( + "fmt" + "io/ioutil" + "os" + "path" "time" "k8s.io/kubernetes/pkg/api" @@ -28,11 +32,21 @@ import ( ) const ( - retryTimeout = time.Minute * 4 + retryTimeout = time.Minute * 5 pollInterval = time.Second * 5 ) -var _ = Describe("Container Runtime Conformance Test", func() { +type testStatus struct { + Name string + RestartPolicy api.RestartPolicy + Phase api.PodPhase + State ContainerState + RestartCountOper string + RestartCount int32 + Ready bool +} + +var _ = Describe("Container runtime Conformance Test", func() { var cl *client.Client BeforeEach(func() { @@ -40,100 +54,260 @@ var _ = Describe("Container Runtime Conformance Test", func() { cl = client.NewOrDie(&restclient.Config{Host: *apiServerAddress}) }) - Describe("container conformance blackbox test", func() { - Context("when running a container that terminates", func() { - var containerCase ConformanceContainer - - BeforeEach(func() { - containerCase = ConformanceContainer{ - Container: api.Container{ - Image: "gcr.io/google_containers/busybox", - Name: "busybox", - Command: []string{"sh", "-c", "env"}, - ImagePullPolicy: api.PullIfNotPresent, - }, - Client: cl, - Phase: api.PodSucceeded, - NodeName: *nodeName, - } - err := containerCase.Create() - Expect(err).NotTo(HaveOccurred()) - }) - - It("it should report its phase as 'succeeded' and get a same container [Conformance]", func() { - var container ConformanceContainer - var err error - Eventually(func() api.PodPhase { - container, err = containerCase.Get() - return container.Phase - }, retryTimeout, pollInterval).Should(Equal(api.PodSucceeded)) - Expect(err).NotTo(HaveOccurred()) - Expect(container).Should(CContainerEqual(containerCase)) - }) - - It("it should be possible to delete [Conformance]", func() { - err := containerCase.Delete() - Expect(err).NotTo(HaveOccurred()) - Eventually(func() bool { - isPresent, err := containerCase.Present() - return err == nil && !isPresent - }, retryTimeout, pollInterval).Should(BeTrue()) - }) - - AfterEach(func() { - containerCase.Delete() - Eventually(func() bool { - isPresent, err := containerCase.Present() - return err == nil && !isPresent - }, retryTimeout, pollInterval).Should(BeTrue()) - }) + Describe("container runtime conformance blackbox test", func() { + var testCContainers []ConformanceContainer + namespace := "runtime-conformance" + BeforeEach(func() { + testCContainers = []ConformanceContainer{} }) - Context("when running a container with invalid image", func() { - var containerCase ConformanceContainer - BeforeEach(func() { - containerCase = ConformanceContainer{ - Container: api.Container{ - Image: "foo.com/foo/foo", - Name: "foo", - Command: []string{"foo", "'Should not work'"}, - ImagePullPolicy: api.PullIfNotPresent, + + Context("when start a container that exits successfully", func() { + It("it should run with the expected status [Conformance]", func() { + testContainer := api.Container{ + Image: "gcr.io/google_containers/busybox", + VolumeMounts: []api.VolumeMount{ + { + MountPath: "/restart-count", + Name: "restart-count", + }, }, - Client: cl, - Phase: api.PodPending, - NodeName: *nodeName, + ImagePullPolicy: api.PullIfNotPresent, + } + testVolumes := []api.Volume{ + { + Name: "restart-count", + VolumeSource: api.VolumeSource{ + HostPath: &api.HostPathVolumeSource{ + Path: os.TempDir(), + }, + }, + }, + } + testCount := int32(3) + testStatuses := []testStatus{ + {"terminate-cmd-rpa", api.RestartPolicyAlways, api.PodRunning, ContainerStateWaiting | ContainerStateRunning | ContainerStateTerminated, ">", testCount, false}, + {"terminate-cmd-rpof", api.RestartPolicyOnFailure, api.PodSucceeded, ContainerStateTerminated, "==", testCount, false}, + {"terminate-cmd-rpn", api.RestartPolicyNever, api.PodSucceeded, ContainerStateTerminated, "==", 0, false}, } - err := containerCase.Create() - Expect(err).NotTo(HaveOccurred()) - }) - It("it should report its phase as 'pending' and get a same container [Conformance]", func() { - var container ConformanceContainer - var err error + for _, testStatus := range testStatuses { + tmpFile, err := ioutil.TempFile("", "restartCount") + Expect(err).NotTo(HaveOccurred()) + defer os.Remove(tmpFile.Name()) + + // It fails in the first three runs and succeeds after that. + tmpCmd := fmt.Sprintf("echo 'hello' >> /restart-count/%s ; test $(wc -l /restart-count/%s| awk {'print $1'}) -ge %d", path.Base(tmpFile.Name()), path.Base(tmpFile.Name()), testCount+1) + testContainer.Name = testStatus.Name + testContainer.Command = []string{"sh", "-c", tmpCmd} + terminateContainer := ConformanceContainer{ + Container: testContainer, + Client: cl, + RestartPolicy: testStatus.RestartPolicy, + Volumes: testVolumes, + NodeName: *nodeName, + Namespace: namespace, + } + err = terminateContainer.Create() + Expect(err).NotTo(HaveOccurred()) + testCContainers = append(testCContainers, terminateContainer) + + Eventually(func() api.PodPhase { + _, phase, _ := terminateContainer.GetStatus() + return phase + }, retryTimeout, pollInterval).ShouldNot(Equal(api.PodPending)) + + var status api.ContainerStatus + By("it should get the expected 'RestartCount'") + Eventually(func() int32 { + status, _, _ = terminateContainer.GetStatus() + return status.RestartCount + }, retryTimeout, pollInterval).Should(BeNumerically(testStatus.RestartCountOper, testStatus.RestartCount)) + + By("it should get the expected 'Ready' status") + Expect(status.Ready).To(Equal(testStatus.Ready)) + + By("it should get the expected 'State'") + Expect(GetContainerState(status.State) & testStatus.State).NotTo(Equal(0)) + + By("it should be possible to delete [Conformance]") + err = terminateContainer.Delete() + Expect(err).NotTo(HaveOccurred()) + Eventually(func() bool { + isPresent, err := terminateContainer.Present() + return err == nil && !isPresent + }, retryTimeout, pollInterval).Should(BeTrue()) + } + }) + }) + + Context("when start a container that keeps running", func() { + It("it should run with the expected status [Conformance]", func() { + testContainer := api.Container{ + Image: "gcr.io/google_containers/busybox", + Command: []string{"sh", "-c", "while true; do echo hello; sleep 1; done"}, + ImagePullPolicy: api.PullIfNotPresent, + } + testStatuses := []testStatus{ + {"loop-cmd-rpa", api.RestartPolicyAlways, api.PodRunning, ContainerStateRunning, "==", 0, true}, + {"loop-cmd-rpof", api.RestartPolicyOnFailure, api.PodRunning, ContainerStateRunning, "==", 0, true}, + {"loop-cmd-rpn", api.RestartPolicyNever, api.PodRunning, ContainerStateRunning, "==", 0, true}, + } + for _, testStatus := range testStatuses { + testContainer.Name = testStatus.Name + runningContainer := ConformanceContainer{ + Container: testContainer, + Client: cl, + RestartPolicy: testStatus.RestartPolicy, + NodeName: *nodeName, + Namespace: namespace, + } + err := runningContainer.Create() + Expect(err).NotTo(HaveOccurred()) + testCContainers = append(testCContainers, runningContainer) + + Eventually(func() api.PodPhase { + _, phase, _ := runningContainer.GetStatus() + return phase + }, retryTimeout, pollInterval).Should(Equal(api.PodRunning)) + + var status api.ContainerStatus + var phase api.PodPhase + Consistently(func() api.PodPhase { + status, phase, err = runningContainer.GetStatus() + return phase + }, retryTimeout, pollInterval).Should(Equal(testStatus.Phase)) + Expect(err).NotTo(HaveOccurred()) + + By("it should get the expected 'RestartCount'") + Expect(status.RestartCount).To(BeNumerically(testStatus.RestartCountOper, testStatus.RestartCount)) + + By("it should get the expected 'Ready' status") + Expect(status.Ready).To(Equal(testStatus.Ready)) + + By("it should get the expected 'State'") + Expect(GetContainerState(status.State) & testStatus.State).NotTo(Equal(0)) + + By("it should be possible to delete [Conformance]") + err = runningContainer.Delete() + Expect(err).NotTo(HaveOccurred()) + Eventually(func() bool { + isPresent, err := runningContainer.Present() + return err == nil && !isPresent + }, retryTimeout, pollInterval).Should(BeTrue()) + } + }) + }) + + Context("when start a container that exits failure", func() { + It("it should run with the expected status [Conformance]", func() { + testContainer := api.Container{ + Image: "gcr.io/google_containers/busybox", + Command: []string{"false"}, + ImagePullPolicy: api.PullIfNotPresent, + } + testStatuses := []testStatus{ + {"fail-cmd-rpa", api.RestartPolicyAlways, api.PodRunning, ContainerStateWaiting | ContainerStateRunning | ContainerStateTerminated, ">", 0, false}, + {"fail-cmd-rpof", api.RestartPolicyOnFailure, api.PodRunning, ContainerStateTerminated, ">", 0, false}, + {"fail-cmd-rpn", api.RestartPolicyNever, api.PodFailed, ContainerStateTerminated, "==", 0, false}, + } + for _, testStatus := range testStatuses { + testContainer.Name = testStatus.Name + failureContainer := ConformanceContainer{ + Container: testContainer, + Client: cl, + RestartPolicy: testStatus.RestartPolicy, + NodeName: *nodeName, + Namespace: namespace, + } + err := failureContainer.Create() + Expect(err).NotTo(HaveOccurred()) + testCContainers = append(testCContainers, failureContainer) + + Eventually(func() api.PodPhase { + _, phase, _ := failureContainer.GetStatus() + return phase + }, retryTimeout, pollInterval).ShouldNot(Equal(api.PodPending)) + + var status api.ContainerStatus + By("it should get the expected 'RestartCount'") + Eventually(func() int32 { + status, _, _ = failureContainer.GetStatus() + return status.RestartCount + }, retryTimeout, pollInterval).Should(BeNumerically(testStatus.RestartCountOper, testStatus.RestartCount)) + + By("it should get the expected 'Ready' status") + Expect(status.Ready).To(Equal(testStatus.Ready)) + + By("it should get the expected 'State'") + Expect(GetContainerState(status.State) & testStatus.State).NotTo(Equal(0)) + + By("it should be possible to delete [Conformance]") + err = failureContainer.Delete() + Expect(err).NotTo(HaveOccurred()) + Eventually(func() bool { + isPresent, err := failureContainer.Present() + return err == nil && !isPresent + }, retryTimeout, pollInterval).Should(BeTrue()) + } + }) + }) + + Context("when running a container with invalid image", func() { + It("it should run with the expected status [Conformance]", func() { + testContainer := api.Container{ + Image: "foo.com/foo/foo", + Command: []string{"false"}, + ImagePullPolicy: api.PullIfNotPresent, + } + testStatus := testStatus{"invalid-image-rpa", api.RestartPolicyAlways, api.PodPending, ContainerStateWaiting, "==", 0, false} + testContainer.Name = testStatus.Name + invalidImageContainer := ConformanceContainer{ + Container: testContainer, + Client: cl, + RestartPolicy: testStatus.RestartPolicy, + NodeName: *nodeName, + Namespace: namespace, + } + err := invalidImageContainer.Create() + Expect(err).NotTo(HaveOccurred()) + testCContainers = append(testCContainers, invalidImageContainer) + + var status api.ContainerStatus + var phase api.PodPhase + Consistently(func() api.PodPhase { - container, err = containerCase.Get() - return container.Phase - }, retryTimeout, pollInterval).Should(Equal(api.PodPending)) + if status, phase, err = invalidImageContainer.GetStatus(); err != nil { + return api.PodPending + } else { + return phase + } + }, retryTimeout, pollInterval).Should(Equal(testStatus.Phase)) Expect(err).NotTo(HaveOccurred()) - Expect(container).Should(CContainerEqual(containerCase)) - }) - It("it should be possible to delete [Conformance]", func() { - err := containerCase.Delete() + By("it should get the expected 'RestartCount'") + Expect(status.RestartCount).To(BeNumerically(testStatus.RestartCountOper, testStatus.RestartCount)) + + By("it should get the expected 'Ready' status") + Expect(status.Ready).To(Equal(testStatus.Ready)) + + By("it should get the expected 'State'") + Expect(GetContainerState(status.State) & testStatus.State).NotTo(Equal(0)) + + By("it should be possible to delete [Conformance]") + err = invalidImageContainer.Delete() Expect(err).NotTo(HaveOccurred()) Eventually(func() bool { - isPresent, err := containerCase.Present() + isPresent, err := invalidImageContainer.Present() return err == nil && !isPresent }, retryTimeout, pollInterval).Should(BeTrue()) }) + }) - AfterEach(func() { - containerCase.Delete() - Eventually(func() bool { - isPresent, err := containerCase.Present() - return err == nil && !isPresent - }, retryTimeout, pollInterval).Should(BeTrue()) - }) + AfterEach(func() { + for _, cc := range testCContainers { + cc.Delete() + } }) }) })