From e90246d661869a5215a903678cad7b118d46ad48 Mon Sep 17 00:00:00 2001 From: Venil Noronha Date: Tue, 13 Mar 2018 16:25:33 -0700 Subject: [PATCH 1/2] Adds e2e test for the VMware vpxd restart scenario This commit adds a test to verify Volume access in a situation where VMware vCenter's vpxd service is down. The test mainly verifies Volume/file access before, during, and after restarting the vpxd service on the vCenter host. --- test/e2e/storage/vsphere/BUILD | 1 + test/e2e/storage/vsphere/vsphere_utils.go | 72 ++++++- .../vsphere/vsphere_volume_cluster_ds.go | 5 +- .../vsphere/vsphere_volume_master_restart.go | 10 +- .../vsphere/vsphere_volume_placement.go | 6 +- .../vsphere/vsphere_volume_vpxd_restart.go | 176 ++++++++++++++++++ 6 files changed, 255 insertions(+), 15 deletions(-) create mode 100644 test/e2e/storage/vsphere/vsphere_volume_vpxd_restart.go diff --git a/test/e2e/storage/vsphere/BUILD b/test/e2e/storage/vsphere/BUILD index c06139f25e..5e52c74b46 100644 --- a/test/e2e/storage/vsphere/BUILD +++ b/test/e2e/storage/vsphere/BUILD @@ -33,6 +33,7 @@ go_library( "vsphere_volume_ops_storm.go", "vsphere_volume_perf.go", "vsphere_volume_placement.go", + "vsphere_volume_vpxd_restart.go", "vsphere_volume_vsan_policy.go", ], importpath = "k8s.io/kubernetes/test/e2e/storage/vsphere", diff --git a/test/e2e/storage/vsphere/vsphere_utils.go b/test/e2e/storage/vsphere/vsphere_utils.go index 51a21b029b..d11da8fe21 100644 --- a/test/e2e/storage/vsphere/vsphere_utils.go +++ b/test/e2e/storage/vsphere/vsphere_utils.go @@ -23,6 +23,7 @@ import ( "time" "github.com/golang/glog" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vmware/govmomi/object" @@ -374,7 +375,7 @@ func getVSpherePodSpecWithVolumePaths(volumePaths []string, keyValuelabel map[st return pod } -func verifyFilesExistOnVSphereVolume(namespace string, podName string, filePaths []string) { +func verifyFilesExistOnVSphereVolume(namespace string, podName string, filePaths ...string) { for _, filePath := range filePaths { _, err := framework.RunKubectl("exec", fmt.Sprintf("--namespace=%s", namespace), podName, "--", "/bin/ls", filePath) Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to verify file: %q on the pod: %q", filePath, podName)) @@ -756,3 +757,72 @@ func GetReadySchedulableRandomNodeInfo() *NodeInfo { Expect(nodesInfo).NotTo(BeEmpty()) return nodesInfo[rand.Int()%len(nodesInfo)] } + +// invokeVCenterServiceControl invokes the given command for the given service +// via service-control on the given vCenter host over SSH. +func invokeVCenterServiceControl(command, service, host string) error { + sshCmd := fmt.Sprintf("service-control --%s %s", command, service) + framework.Logf("Invoking command %v on vCenter host %v", sshCmd, host) + result, err := framework.SSH(sshCmd, host, framework.TestContext.Provider) + if err != nil || result.Code != 0 { + framework.LogSSHResult(result) + return fmt.Errorf("couldn't execute command: %s on vCenter host: %v", sshCmd, err) + } + return nil +} + +// expectVolumeToBeAttached checks if the given Volume is attached to the given +// Node, else fails. +func expectVolumeToBeAttached(nodeName, volumePath string) { + isAttached, err := diskIsAttached(volumePath, nodeName) + Expect(err).NotTo(HaveOccurred()) + Expect(isAttached).To(BeTrue(), fmt.Sprintf("disk: %s is not attached with the node", volumePath)) +} + +// expectVolumesToBeAttached checks if the given Volumes are attached to the +// corresponding set of Nodes, else fails. +func expectVolumesToBeAttached(pods []*v1.Pod, volumePaths []string) { + for i, pod := range pods { + nodeName := pod.Spec.NodeName + volumePath := volumePaths[i] + By(fmt.Sprintf("Verifying that volume %v is attached to node %v", volumePath, nodeName)) + expectVolumeToBeAttached(nodeName, volumePath) + } +} + +// expectFilesToBeAccessible checks if the given files are accessible on the +// corresponding set of Nodes, else fails. +func expectFilesToBeAccessible(namespace string, pods []*v1.Pod, filePaths []string) { + for i, pod := range pods { + podName := pod.Name + filePath := filePaths[i] + By(fmt.Sprintf("Verifying that file %v is accessible on pod %v", filePath, podName)) + verifyFilesExistOnVSphereVolume(namespace, podName, filePath) + } +} + +// writeContentToPodFile writes the given content to the specified file. +func writeContentToPodFile(namespace, podName, filePath, content string) error { + _, err := framework.RunKubectl("exec", fmt.Sprintf("--namespace=%s", namespace), podName, + "--", "/bin/sh", "-c", fmt.Sprintf("echo '%s' > %s", content, filePath)) + return err +} + +// expectFileContentToMatch checks if a given file contains the specified +// content, else fails. +func expectFileContentToMatch(namespace, podName, filePath, content string) { + _, err := framework.RunKubectl("exec", fmt.Sprintf("--namespace=%s", namespace), podName, + "--", "/bin/sh", "-c", fmt.Sprintf("grep '%s' %s", content, filePath)) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to match content of file: %q on the pod: %q", filePath, podName)) +} + +// expectFileContentsToMatch checks if the given contents match the ones present +// in corresponding files on respective Pods, else fails. +func expectFileContentsToMatch(namespace string, pods []*v1.Pod, filePaths []string, contents []string) { + for i, pod := range pods { + podName := pod.Name + filePath := filePaths[i] + By(fmt.Sprintf("Matching file content for %v on pod %v", filePath, podName)) + expectFileContentToMatch(namespace, podName, filePath, contents[i]) + } +} diff --git a/test/e2e/storage/vsphere/vsphere_volume_cluster_ds.go b/test/e2e/storage/vsphere/vsphere_volume_cluster_ds.go index 1e0092cecf..f6c4aa50b9 100644 --- a/test/e2e/storage/vsphere/vsphere_volume_cluster_ds.go +++ b/test/e2e/storage/vsphere/vsphere_volume_cluster_ds.go @@ -17,7 +17,6 @@ limitations under the License. package vsphere import ( - "fmt" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -98,9 +97,7 @@ var _ = utils.SIGDescribe("Volume Provisioning On Clustered Datastore [Feature:v nodeName := pod.Spec.NodeName By("Verifying volume is attached") - isAttached, err := diskIsAttached(volumePath, nodeName) - Expect(err).NotTo(HaveOccurred()) - Expect(isAttached).To(BeTrue(), fmt.Sprintf("disk: %s is not attached with the node: %v", volumePath, nodeName)) + expectVolumeToBeAttached(nodeName, volumePath) By("Deleting pod") err = framework.DeletePodWithWait(f, client, pod) diff --git a/test/e2e/storage/vsphere/vsphere_volume_master_restart.go b/test/e2e/storage/vsphere/vsphere_volume_master_restart.go index b8358267b5..437930cc85 100644 --- a/test/e2e/storage/vsphere/vsphere_volume_master_restart.go +++ b/test/e2e/storage/vsphere/vsphere_volume_master_restart.go @@ -103,10 +103,7 @@ var _ = utils.SIGDescribe("Volume Attach Verify [Feature:vsphere][Serial][Disrup nodeName := pod.Spec.NodeName By(fmt.Sprintf("Verify volume %s is attached to the pod %s", volumePath, nodeName)) - isAttached, err := diskIsAttached(volumePath, nodeName) - Expect(err).NotTo(HaveOccurred()) - Expect(isAttached).To(BeTrue(), fmt.Sprintf("disk: %s is not attached with the node", volumePath)) - + expectVolumeToBeAttached(nodeName, volumePath) } By("Restarting kubelet on master node") @@ -121,10 +118,9 @@ var _ = utils.SIGDescribe("Volume Attach Verify [Feature:vsphere][Serial][Disrup for i, pod := range pods { volumePath := volumePaths[i] nodeName := pod.Spec.NodeName + By(fmt.Sprintf("After master restart, verify volume %v is attached to the pod %v", volumePath, nodeName)) - isAttached, err := diskIsAttached(volumePaths[i], nodeName) - Expect(err).NotTo(HaveOccurred()) - Expect(isAttached).To(BeTrue(), fmt.Sprintf("disk: %s is not attached with the node", volumePath)) + expectVolumeToBeAttached(nodeName, volumePath) By(fmt.Sprintf("Deleting pod on node %s", nodeName)) err = framework.DeletePodWithWait(f, client, pod) diff --git a/test/e2e/storage/vsphere/vsphere_volume_placement.go b/test/e2e/storage/vsphere/vsphere_volume_placement.go index e276591d3e..654ee22714 100644 --- a/test/e2e/storage/vsphere/vsphere_volume_placement.go +++ b/test/e2e/storage/vsphere/vsphere_volume_placement.go @@ -320,9 +320,9 @@ var _ = utils.SIGDescribe("Volume Placement", func() { // Verify newly and previously created files present on the volume mounted on the pod By("Verify newly Created file and previously created files present on volume mounted on pod-A") - verifyFilesExistOnVSphereVolume(ns, podA.Name, podAFiles) + verifyFilesExistOnVSphereVolume(ns, podA.Name, podAFiles...) By("Verify newly Created file and previously created files present on volume mounted on pod-B") - verifyFilesExistOnVSphereVolume(ns, podB.Name, podBFiles) + verifyFilesExistOnVSphereVolume(ns, podB.Name, podBFiles...) By("Deleting pod-A") framework.ExpectNoError(framework.DeletePodWithWait(f, c, podA), "Failed to delete pod ", podA.Name) @@ -378,7 +378,7 @@ func createAndVerifyFilesOnVolume(namespace string, podname string, newEmptyfile // Verify newly and previously created files present on the volume mounted on the pod By(fmt.Sprintf("Verify newly Created file and previously created files present on volume mounted on: %v", podname)) - verifyFilesExistOnVSphereVolume(namespace, podname, filesToCheck) + verifyFilesExistOnVSphereVolume(namespace, podname, filesToCheck...) } func deletePodAndWaitForVolumeToDetach(f *framework.Framework, c clientset.Interface, pod *v1.Pod, nodeName string, volumePaths []string) { diff --git a/test/e2e/storage/vsphere/vsphere_volume_vpxd_restart.go b/test/e2e/storage/vsphere/vsphere_volume_vpxd_restart.go new file mode 100644 index 0000000000..973a5f4ffa --- /dev/null +++ b/test/e2e/storage/vsphere/vsphere_volume_vpxd_restart.go @@ -0,0 +1,176 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vsphere + +import ( + "fmt" + "strconv" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" +) + +/* + Test to verify that a volume remains attached through vpxd restart. + + For the number of schedulable nodes: + 1. Create a Volume with default options. + 2. Create a Pod with the created Volume. + 3. Verify that the Volume is attached. + 4. Create a file with random contents under the Volume's mount point on the Pod. + 5. Stop the vpxd service on the vCenter host. + 6. Verify that the file is accessible on the Pod and that it's contents match. + 7. Start the vpxd service on the vCenter host. + 8. Verify that the Volume remains attached, the file is accessible on the Pod, and that it's contents match. + 9. Delete the Pod and wait for the Volume to be detached. + 10. Delete the Volume. +*/ +var _ = utils.SIGDescribe("Verify Volume Attach Through vpxd Restart [Feature:vsphere][Serial][Disruptive]", func() { + f := framework.NewDefaultFramework("restart-vpxd") + + type node struct { + name string + kvLabels map[string]string + nodeInfo *NodeInfo + } + + const ( + labelKey = "vsphere_e2e_label_vpxd_restart" + vpxdServiceName = "vmware-vpxd" + ) + + var ( + client clientset.Interface + namespace string + vcNodesMap map[string][]node + ) + + BeforeEach(func() { + framework.SkipUnlessProviderIs("vsphere") + Bootstrap(f) + client = f.ClientSet + namespace = f.Namespace.Name + framework.ExpectNoError(framework.WaitForAllNodesSchedulable(client, framework.TestContext.NodeSchedulableTimeout)) + + nodes := framework.GetReadySchedulableNodesOrDie(client) + numNodes := len(nodes.Items) + if numNodes < 1 { + framework.Skipf("Requires at least %d nodes (not %d)", 1, len(nodes.Items)) + } + + vcNodesMap = make(map[string][]node) + for i := 0; i < numNodes; i++ { + nodeInfo := TestContext.NodeMapper.GetNodeInfo(nodes.Items[i].Name) + nodeName := nodes.Items[i].Name + nodeLabel := "vsphere_e2e_" + string(uuid.NewUUID()) + framework.AddOrUpdateLabelOnNode(client, nodeName, labelKey, nodeLabel) + + vcHost := nodeInfo.VSphere.Config.Hostname + vcNodesMap[vcHost] = append(vcNodesMap[vcHost], node{ + name: nodeName, + kvLabels: map[string]string{labelKey: nodeLabel}, + nodeInfo: nodeInfo, + }) + } + }) + + It("verify volume remains attached through vpxd restart", func() { + for vcHost, nodes := range vcNodesMap { + var ( + volumePaths []string + filePaths []string + fileContents []string + pods []*v1.Pod + ) + + framework.Logf("Testing for nodes on vCenter host: %s", vcHost) + + for i, node := range nodes { + By(fmt.Sprintf("Creating test vsphere volume %d", i)) + volumePath, err := node.nodeInfo.VSphere.CreateVolume(&VolumeOptions{}, node.nodeInfo.DataCenterRef) + Expect(err).NotTo(HaveOccurred()) + volumePaths = append(volumePaths, volumePath) + + By(fmt.Sprintf("Creating pod %d on node %v", i, node.name)) + podspec := getVSpherePodSpecWithVolumePaths([]string{volumePath}, node.kvLabels, nil) + pod, err := client.CoreV1().Pods(namespace).Create(podspec) + Expect(err).NotTo(HaveOccurred()) + + By(fmt.Sprintf("Waiting for pod %d to be ready", i)) + Expect(framework.WaitForPodNameRunningInNamespace(client, pod.Name, namespace)).To(Succeed()) + + pod, err = client.CoreV1().Pods(namespace).Get(pod.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + pods = append(pods, pod) + + nodeName := pod.Spec.NodeName + By(fmt.Sprintf("Verifying that volume %v is attached to node %v", volumePath, nodeName)) + expectVolumeToBeAttached(nodeName, volumePath) + + By(fmt.Sprintf("Creating a file with random content on the volume mounted on pod %d", i)) + filePath := fmt.Sprintf("/mnt/volume1/%v_vpxd_restart_test_%v.txt", namespace, strconv.FormatInt(time.Now().UnixNano(), 10)) + randomContent := fmt.Sprintf("Random Content -- %v", strconv.FormatInt(time.Now().UnixNano(), 10)) + err = writeContentToPodFile(namespace, pod.Name, filePath, randomContent) + Expect(err).NotTo(HaveOccurred()) + filePaths = append(filePaths, filePath) + fileContents = append(fileContents, randomContent) + } + + By("Stopping vpxd on the vCenter host") + vcAddress := vcHost + ":22" + err := invokeVCenterServiceControl("stop", vpxdServiceName, vcAddress) + Expect(err).NotTo(HaveOccurred(), "Unable to stop vpxd on the vCenter host") + + expectFilesToBeAccessible(namespace, pods, filePaths) + expectFileContentsToMatch(namespace, pods, filePaths, fileContents) + + By("Starting vpxd on the vCenter host") + err = invokeVCenterServiceControl("start", vpxdServiceName, vcAddress) + Expect(err).NotTo(HaveOccurred(), "Unable to start vpxd on the vCenter host") + + expectVolumesToBeAttached(pods, volumePaths) + expectFilesToBeAccessible(namespace, pods, filePaths) + expectFileContentsToMatch(namespace, pods, filePaths, fileContents) + + for i, node := range nodes { + pod := pods[i] + nodeName := pod.Spec.NodeName + volumePath := volumePaths[i] + + By(fmt.Sprintf("Deleting pod on node %s", nodeName)) + err = framework.DeletePodWithWait(f, client, pod) + Expect(err).NotTo(HaveOccurred()) + + By(fmt.Sprintf("Waiting for volume %s to be detached from node %s", volumePath, nodeName)) + err = waitForVSphereDiskToDetach(volumePath, nodeName) + Expect(err).NotTo(HaveOccurred()) + + By(fmt.Sprintf("Deleting volume %s", volumePath)) + err = node.nodeInfo.VSphere.DeleteVolume(volumePath, node.nodeInfo.DataCenterRef) + Expect(err).NotTo(HaveOccurred()) + } + } + }) +}) From d10179920347d3aba1dfb06aebc917207663ced1 Mon Sep 17 00:00:00 2001 From: Venil Noronha Date: Mon, 9 Apr 2018 11:23:58 -0700 Subject: [PATCH 2/2] Addresses review comments --- test/e2e/storage/vsphere/vsphere_volume_vpxd_restart.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e/storage/vsphere/vsphere_volume_vpxd_restart.go b/test/e2e/storage/vsphere/vsphere_volume_vpxd_restart.go index 973a5f4ffa..e30704ab3c 100644 --- a/test/e2e/storage/vsphere/vsphere_volume_vpxd_restart.go +++ b/test/e2e/storage/vsphere/vsphere_volume_vpxd_restart.go @@ -68,7 +68,9 @@ var _ = utils.SIGDescribe("Verify Volume Attach Through vpxd Restart [Feature:vs ) BeforeEach(func() { + // Requires SSH access to vCenter. framework.SkipUnlessProviderIs("vsphere") + Bootstrap(f) client = f.ClientSet namespace = f.Namespace.Name @@ -76,9 +78,7 @@ var _ = utils.SIGDescribe("Verify Volume Attach Through vpxd Restart [Feature:vs nodes := framework.GetReadySchedulableNodesOrDie(client) numNodes := len(nodes.Items) - if numNodes < 1 { - framework.Skipf("Requires at least %d nodes (not %d)", 1, len(nodes.Items)) - } + Expect(numNodes).NotTo(BeZero(), "No nodes are available for testing volume access through vpxd restart") vcNodesMap = make(map[string][]node) for i := 0; i < numNodes; i++ {