e2e-node test: check runtime ready/restartcount/status

Signed-off-by: liang chenye <liangchenye@huawei.com>

test/e2e_node/container.go

@@ -23,17 +23,22 @@ import (
 	"k8s.io/kubernetes/pkg/api"
 	apierrs "k8s.io/kubernetes/pkg/api/errors"
 	client "k8s.io/kubernetes/pkg/client/unversioned"
+	"k8s.io/kubernetes/pkg/util"
 
 	"github.com/onsi/gomega/format"
 	"github.com/onsi/gomega/types"
 )
 
-//One pod one container
+// One pod one container
 type ConformanceContainer struct {
-	Container api.Container
+	Container     api.Container
-	Client    *client.Client
+	Client        *client.Client
-	Phase     api.PodPhase
+	RestartPolicy api.RestartPolicy
-	NodeName  string
+	Volumes       []api.Volume
+	NodeName      string
+	Namespace     string
+
+	podName string
 }
 
 type ConformanceContainerEqualMatcher struct {
@@ -63,36 +68,37 @@ func (matcher *ConformanceContainerEqualMatcher) NegatedFailureMessage(actual in
 }
 
 func (cc *ConformanceContainer) Create() error {
+	cc.podName = cc.Container.Name + string(util.NewUUID())
 	pod := &api.Pod{
 		ObjectMeta: api.ObjectMeta{
-			//Same with the container name
+			Name:      cc.podName,
-			Name:      cc.Container.Name,
+			Namespace: cc.Namespace,
-			Namespace: api.NamespaceDefault,
 		},
 		Spec: api.PodSpec{
 			NodeName:      cc.NodeName,
-			RestartPolicy: api.RestartPolicyNever,
+			RestartPolicy: cc.RestartPolicy,
 			Containers: []api.Container{
 				cc.Container,
 			},
+			Volumes: cc.Volumes,
 		},
 	}
 
-	_, err := cc.Client.Pods(api.NamespaceDefault).Create(pod)
+	_, err := cc.Client.Pods(cc.Namespace).Create(pod)
 	return err
 }
 
 //Same with 'delete'
 func (cc *ConformanceContainer) Stop() error {
-	return cc.Client.Pods(api.NamespaceDefault).Delete(cc.Container.Name, &api.DeleteOptions{})
+	return cc.Client.Pods(cc.Namespace).Delete(cc.podName, &api.DeleteOptions{})
 }
 
 func (cc *ConformanceContainer) Delete() error {
-	return cc.Client.Pods(api.NamespaceDefault).Delete(cc.Container.Name, &api.DeleteOptions{})
+	return cc.Client.Pods(cc.Namespace).Delete(cc.podName, &api.DeleteOptions{})
 }
 
 func (cc *ConformanceContainer) Get() (ConformanceContainer, error) {
-	pod, err := cc.Client.Pods(api.NamespaceDefault).Get(cc.Container.Name)
+	pod, err := cc.Client.Pods(cc.Namespace).Get(cc.podName)
 	if err != nil {
 		return ConformanceContainer{}, err
 	}
@@ -101,15 +107,51 @@ func (cc *ConformanceContainer) Get() (ConformanceContainer, error) {
 	if containers == nil || len(containers) != 1 {
 		return ConformanceContainer{}, errors.New("Failed to get container")
 	}
-	return ConformanceContainer{containers[0], cc.Client, pod.Status.Phase, pod.Spec.NodeName}, nil
+	return ConformanceContainer{containers[0], cc.Client, pod.Spec.RestartPolicy, pod.Spec.Volumes, pod.Spec.NodeName, cc.Namespace, cc.podName}, nil
+}
+
+func (cc *ConformanceContainer) GetStatus() (api.ContainerStatus, api.PodPhase, error) {
+	pod, err := cc.Client.Pods(cc.Namespace).Get(cc.podName)
+	if err != nil {
+		return api.ContainerStatus{}, api.PodUnknown, err
+	}
+
+	statuses := pod.Status.ContainerStatuses
+	if len(statuses) != 1 {
+		return api.ContainerStatus{}, api.PodUnknown, errors.New("Failed to get container status")
+	}
+	return statuses[0], pod.Status.Phase, nil
 }
 
 func (cc *ConformanceContainer) Present() (bool, error) {
-	_, err := cc.Client.Pods(api.NamespaceDefault).Get(cc.Container.Name)
+	_, err := cc.Client.Pods(cc.Namespace).Get(cc.podName)
 	if err == nil {
 		return true, nil
-	} else if err != nil && apierrs.IsNotFound(err) {
+	}
+	if apierrs.IsNotFound(err) {
 		return false, nil
 	}
 	return false, err
 }
+
+type ContainerState uint32
+
+const (
+	ContainerStateWaiting ContainerState = 1 << iota
+	ContainerStateRunning
+	ContainerStateTerminated
+	ContainerStateUnknown
+)
+
+func GetContainerState(state api.ContainerState) ContainerState {
+	if state.Waiting != nil {
+		return ContainerStateWaiting
+	}
+	if state.Running != nil {
+		return ContainerStateRunning
+	}
+	if state.Terminated != nil {
+		return ContainerStateTerminated
+	}
+	return ContainerStateUnknown
+}

test/e2e_node/image_conformance_test.go

@@ -31,7 +31,7 @@ const (
 	imagePullInterval = time.Second * 15
 )
 
-var _ = Describe("Container Conformance Test", func() {
+var _ = Describe("Image Container Conformance Test", func() {
 	var cl *client.Client
 
 	BeforeEach(func() {

test/e2e_node/runtime_conformance_test.go

@@ -17,6 +17,10 @@ limitations under the License.
 package e2e_node
 
 import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
 	"time"
 
 	"k8s.io/kubernetes/pkg/api"
@@ -28,11 +32,21 @@ import (
 )
 
 const (
-	retryTimeout = time.Minute * 4
+	retryTimeout = time.Minute * 5
 	pollInterval = time.Second * 5
 )
 
-var _ = Describe("Container Runtime Conformance Test", func() {
+type testStatus struct {
+	Name             string
+	RestartPolicy    api.RestartPolicy
+	Phase            api.PodPhase
+	State            ContainerState
+	RestartCountOper string
+	RestartCount     int32
+	Ready            bool
+}
+
+var _ = Describe("Container runtime Conformance Test", func() {
 	var cl *client.Client
 
 	BeforeEach(func() {
@@ -40,100 +54,260 @@ var _ = Describe("Container Runtime Conformance Test", func() {
 		cl = client.NewOrDie(&restclient.Config{Host: *apiServerAddress})
 	})
 
-	Describe("container conformance blackbox test", func() {
+	Describe("container runtime conformance blackbox test", func() {
-		Context("when running a container that terminates", func() {
+		var testCContainers []ConformanceContainer
-			var containerCase ConformanceContainer
+		namespace := "runtime-conformance"
-
-			BeforeEach(func() {
-				containerCase = ConformanceContainer{
-					Container: api.Container{
-						Image:           "gcr.io/google_containers/busybox",
-						Name:            "busybox",
-						Command:         []string{"sh", "-c", "env"},
-						ImagePullPolicy: api.PullIfNotPresent,
-					},
-					Client:   cl,
-					Phase:    api.PodSucceeded,
-					NodeName: *nodeName,
-				}
-				err := containerCase.Create()
-				Expect(err).NotTo(HaveOccurred())
-			})
-
-			It("it should report its phase as 'succeeded' and get a same container [Conformance]", func() {
-				var container ConformanceContainer
-				var err error
-				Eventually(func() api.PodPhase {
-					container, err = containerCase.Get()
-					return container.Phase
-				}, retryTimeout, pollInterval).Should(Equal(api.PodSucceeded))
-				Expect(err).NotTo(HaveOccurred())
-				Expect(container).Should(CContainerEqual(containerCase))
-			})
-
-			It("it should be possible to delete [Conformance]", func() {
-				err := containerCase.Delete()
-				Expect(err).NotTo(HaveOccurred())
-				Eventually(func() bool {
-					isPresent, err := containerCase.Present()
-					return err == nil && !isPresent
-				}, retryTimeout, pollInterval).Should(BeTrue())
-			})
-
-			AfterEach(func() {
-				containerCase.Delete()
-				Eventually(func() bool {
-					isPresent, err := containerCase.Present()
-					return err == nil && !isPresent
-				}, retryTimeout, pollInterval).Should(BeTrue())
-			})
 
+		BeforeEach(func() {
+			testCContainers = []ConformanceContainer{}
 		})
-		Context("when running a container with invalid image", func() {
+
-			var containerCase ConformanceContainer
+		Context("when start a container that exits successfully", func() {
-			BeforeEach(func() {
+			It("it should run with the expected status [Conformance]", func() {
-				containerCase = ConformanceContainer{
+				testContainer := api.Container{
-					Container: api.Container{
+					Image: "gcr.io/google_containers/busybox",
-						Image:           "foo.com/foo/foo",
+					VolumeMounts: []api.VolumeMount{
-						Name:            "foo",
+						{
-						Command:         []string{"foo", "'Should not work'"},
+							MountPath: "/restart-count",
-						ImagePullPolicy: api.PullIfNotPresent,
+							Name:      "restart-count",
+						},
 					},
-					Client:   cl,
+					ImagePullPolicy: api.PullIfNotPresent,
-					Phase:    api.PodPending,
+				}
-					NodeName: *nodeName,
+				testVolumes := []api.Volume{
+					{
+						Name: "restart-count",
+						VolumeSource: api.VolumeSource{
+							HostPath: &api.HostPathVolumeSource{
+								Path: os.TempDir(),
+							},
+						},
+					},
+				}
+				testCount := int32(3)
+				testStatuses := []testStatus{
+					{"terminate-cmd-rpa", api.RestartPolicyAlways, api.PodRunning, ContainerStateWaiting | ContainerStateRunning | ContainerStateTerminated, ">", testCount, false},
+					{"terminate-cmd-rpof", api.RestartPolicyOnFailure, api.PodSucceeded, ContainerStateTerminated, "==", testCount, false},
+					{"terminate-cmd-rpn", api.RestartPolicyNever, api.PodSucceeded, ContainerStateTerminated, "==", 0, false},
 				}
-				err := containerCase.Create()
-				Expect(err).NotTo(HaveOccurred())
-			})
 
-			It("it should report its phase as 'pending' and get a same container [Conformance]", func() {
+				for _, testStatus := range testStatuses {
-				var container ConformanceContainer
+					tmpFile, err := ioutil.TempFile("", "restartCount")
-				var err error
+					Expect(err).NotTo(HaveOccurred())
+					defer os.Remove(tmpFile.Name())
+
+					// It fails in the first three runs and succeeds after that.
+					tmpCmd := fmt.Sprintf("echo 'hello' >> /restart-count/%s ; test $(wc -l /restart-count/%s| awk {'print $1'}) -ge %d", path.Base(tmpFile.Name()), path.Base(tmpFile.Name()), testCount+1)
+					testContainer.Name = testStatus.Name
+					testContainer.Command = []string{"sh", "-c", tmpCmd}
+					terminateContainer := ConformanceContainer{
+						Container:     testContainer,
+						Client:        cl,
+						RestartPolicy: testStatus.RestartPolicy,
+						Volumes:       testVolumes,
+						NodeName:      *nodeName,
+						Namespace:     namespace,
+					}
+					err = terminateContainer.Create()
+					Expect(err).NotTo(HaveOccurred())
+					testCContainers = append(testCContainers, terminateContainer)
+
+					Eventually(func() api.PodPhase {
+						_, phase, _ := terminateContainer.GetStatus()
+						return phase
+					}, retryTimeout, pollInterval).ShouldNot(Equal(api.PodPending))
+
+					var status api.ContainerStatus
+					By("it should get the expected 'RestartCount'")
+					Eventually(func() int32 {
+						status, _, _ = terminateContainer.GetStatus()
+						return status.RestartCount
+					}, retryTimeout, pollInterval).Should(BeNumerically(testStatus.RestartCountOper, testStatus.RestartCount))
+
+					By("it should get the expected 'Ready' status")
+					Expect(status.Ready).To(Equal(testStatus.Ready))
+
+					By("it should get the expected 'State'")
+					Expect(GetContainerState(status.State) & testStatus.State).NotTo(Equal(0))
+
+					By("it should be possible to delete [Conformance]")
+					err = terminateContainer.Delete()
+					Expect(err).NotTo(HaveOccurred())
+					Eventually(func() bool {
+						isPresent, err := terminateContainer.Present()
+						return err == nil && !isPresent
+					}, retryTimeout, pollInterval).Should(BeTrue())
+				}
+			})
+		})
+
+		Context("when start a container that keeps running", func() {
+			It("it should run with the expected status [Conformance]", func() {
+				testContainer := api.Container{
+					Image:           "gcr.io/google_containers/busybox",
+					Command:         []string{"sh", "-c", "while true; do echo hello; sleep 1; done"},
+					ImagePullPolicy: api.PullIfNotPresent,
+				}
+				testStatuses := []testStatus{
+					{"loop-cmd-rpa", api.RestartPolicyAlways, api.PodRunning, ContainerStateRunning, "==", 0, true},
+					{"loop-cmd-rpof", api.RestartPolicyOnFailure, api.PodRunning, ContainerStateRunning, "==", 0, true},
+					{"loop-cmd-rpn", api.RestartPolicyNever, api.PodRunning, ContainerStateRunning, "==", 0, true},
+				}
+				for _, testStatus := range testStatuses {
+					testContainer.Name = testStatus.Name
+					runningContainer := ConformanceContainer{
+						Container:     testContainer,
+						Client:        cl,
+						RestartPolicy: testStatus.RestartPolicy,
+						NodeName:      *nodeName,
+						Namespace:     namespace,
+					}
+					err := runningContainer.Create()
+					Expect(err).NotTo(HaveOccurred())
+					testCContainers = append(testCContainers, runningContainer)
+
+					Eventually(func() api.PodPhase {
+						_, phase, _ := runningContainer.GetStatus()
+						return phase
+					}, retryTimeout, pollInterval).Should(Equal(api.PodRunning))
+
+					var status api.ContainerStatus
+					var phase api.PodPhase
+					Consistently(func() api.PodPhase {
+						status, phase, err = runningContainer.GetStatus()
+						return phase
+					}, retryTimeout, pollInterval).Should(Equal(testStatus.Phase))
+					Expect(err).NotTo(HaveOccurred())
+
+					By("it should get the expected 'RestartCount'")
+					Expect(status.RestartCount).To(BeNumerically(testStatus.RestartCountOper, testStatus.RestartCount))
+
+					By("it should get the expected 'Ready' status")
+					Expect(status.Ready).To(Equal(testStatus.Ready))
+
+					By("it should get the expected 'State'")
+					Expect(GetContainerState(status.State) & testStatus.State).NotTo(Equal(0))
+
+					By("it should be possible to delete [Conformance]")
+					err = runningContainer.Delete()
+					Expect(err).NotTo(HaveOccurred())
+					Eventually(func() bool {
+						isPresent, err := runningContainer.Present()
+						return err == nil && !isPresent
+					}, retryTimeout, pollInterval).Should(BeTrue())
+				}
+			})
+		})
+
+		Context("when start a container that exits failure", func() {
+			It("it should run with the expected status [Conformance]", func() {
+				testContainer := api.Container{
+					Image:           "gcr.io/google_containers/busybox",
+					Command:         []string{"false"},
+					ImagePullPolicy: api.PullIfNotPresent,
+				}
+				testStatuses := []testStatus{
+					{"fail-cmd-rpa", api.RestartPolicyAlways, api.PodRunning, ContainerStateWaiting | ContainerStateRunning | ContainerStateTerminated, ">", 0, false},
+					{"fail-cmd-rpof", api.RestartPolicyOnFailure, api.PodRunning, ContainerStateTerminated, ">", 0, false},
+					{"fail-cmd-rpn", api.RestartPolicyNever, api.PodFailed, ContainerStateTerminated, "==", 0, false},
+				}
+				for _, testStatus := range testStatuses {
+					testContainer.Name = testStatus.Name
+					failureContainer := ConformanceContainer{
+						Container:     testContainer,
+						Client:        cl,
+						RestartPolicy: testStatus.RestartPolicy,
+						NodeName:      *nodeName,
+						Namespace:     namespace,
+					}
+					err := failureContainer.Create()
+					Expect(err).NotTo(HaveOccurred())
+					testCContainers = append(testCContainers, failureContainer)
+
+					Eventually(func() api.PodPhase {
+						_, phase, _ := failureContainer.GetStatus()
+						return phase
+					}, retryTimeout, pollInterval).ShouldNot(Equal(api.PodPending))
+
+					var status api.ContainerStatus
+					By("it should get the expected 'RestartCount'")
+					Eventually(func() int32 {
+						status, _, _ = failureContainer.GetStatus()
+						return status.RestartCount
+					}, retryTimeout, pollInterval).Should(BeNumerically(testStatus.RestartCountOper, testStatus.RestartCount))
+
+					By("it should get the expected 'Ready' status")
+					Expect(status.Ready).To(Equal(testStatus.Ready))
+
+					By("it should get the expected 'State'")
+					Expect(GetContainerState(status.State) & testStatus.State).NotTo(Equal(0))
+
+					By("it should be possible to delete [Conformance]")
+					err = failureContainer.Delete()
+					Expect(err).NotTo(HaveOccurred())
+					Eventually(func() bool {
+						isPresent, err := failureContainer.Present()
+						return err == nil && !isPresent
+					}, retryTimeout, pollInterval).Should(BeTrue())
+				}
+			})
+		})
+
+		Context("when running a container with invalid image", func() {
+			It("it should run with the expected status [Conformance]", func() {
+				testContainer := api.Container{
+					Image:           "foo.com/foo/foo",
+					Command:         []string{"false"},
+					ImagePullPolicy: api.PullIfNotPresent,
+				}
+				testStatus := testStatus{"invalid-image-rpa", api.RestartPolicyAlways, api.PodPending, ContainerStateWaiting, "==", 0, false}
+				testContainer.Name = testStatus.Name
+				invalidImageContainer := ConformanceContainer{
+					Container:     testContainer,
+					Client:        cl,
+					RestartPolicy: testStatus.RestartPolicy,
+					NodeName:      *nodeName,
+					Namespace:     namespace,
+				}
+				err := invalidImageContainer.Create()
+				Expect(err).NotTo(HaveOccurred())
+				testCContainers = append(testCContainers, invalidImageContainer)
+
+				var status api.ContainerStatus
+				var phase api.PodPhase
+
 				Consistently(func() api.PodPhase {
-					container, err = containerCase.Get()
+					if status, phase, err = invalidImageContainer.GetStatus(); err != nil {
-					return container.Phase
+						return api.PodPending
-				}, retryTimeout, pollInterval).Should(Equal(api.PodPending))
+					} else {
+						return phase
+					}
+				}, retryTimeout, pollInterval).Should(Equal(testStatus.Phase))
 				Expect(err).NotTo(HaveOccurred())
-				Expect(container).Should(CContainerEqual(containerCase))
-			})
 
-			It("it should be possible to delete [Conformance]", func() {
+				By("it should get the expected 'RestartCount'")
-				err := containerCase.Delete()
+				Expect(status.RestartCount).To(BeNumerically(testStatus.RestartCountOper, testStatus.RestartCount))
+
+				By("it should get the expected 'Ready' status")
+				Expect(status.Ready).To(Equal(testStatus.Ready))
+
+				By("it should get the expected 'State'")
+				Expect(GetContainerState(status.State) & testStatus.State).NotTo(Equal(0))
+
+				By("it should be possible to delete [Conformance]")
+				err = invalidImageContainer.Delete()
 				Expect(err).NotTo(HaveOccurred())
 				Eventually(func() bool {
-					isPresent, err := containerCase.Present()
+					isPresent, err := invalidImageContainer.Present()
 					return err == nil && !isPresent
 				}, retryTimeout, pollInterval).Should(BeTrue())
 			})
+		})
 
-			AfterEach(func() {
+		AfterEach(func() {
-				containerCase.Delete()
+			for _, cc := range testCContainers {
-				Eventually(func() bool {
+				cc.Delete()
-					isPresent, err := containerCase.Present()
+			}
-					return err == nil && !isPresent
-				}, retryTimeout, pollInterval).Should(BeTrue())
-			})
 		})
 	})
 })