github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #61614 from vmware/vpxd-restart-test 

Automatic merge from submit-queue (batch tested with PRs 60197, 61614, 62074, 62071, 62301). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Adds e2e test for vpxd restart scenario in vSphere Cloud Provider

This PR adds a test to verify Volume access in a situation where VMware vCenter's `vpxd` service is down. The test mainly verifies Volume/file access before, during, and after restarting the `vpxd` service on the vCenter host.

See vmware/kubernetes#373 for more details.

**Reviewers note:** This PR was internally reviewed at vmware/kubernetes#468.

/cc @kubernetes/vmware
pull/8/head
Kubernetes Submit Queue 2018-04-10 08:56:19 -07:00 committed by GitHub
parent 60c6d0e0ba d101799203
commit cc826360ef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 255 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
test/e2e/storage/vsphere/BUILD
View File

@ -33,6 +33,7 @@ go_library(
"vsphere_volume_ops_storm.go",
"vsphere_volume_perf.go",
"vsphere_volume_placement.go",
"vsphere_volume_vpxd_restart.go",
"vsphere_volume_vsan_policy.go",
],
importpath = "k8s.io/kubernetes/test/e2e/storage/vsphere",

72
test/e2e/storage/vsphere/vsphere_utils.go
View File

@ -26,6 +26,7 @@ import (
"time"
"github.com/golang/glog"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vmware/govmomi/find"
"github.com/vmware/govmomi/object"
@ -371,7 +372,7 @@ func getVSpherePodSpecWithVolumePaths(volumePaths []string, keyValuelabel map[st
return pod
}
func verifyFilesExistOnVSphereVolume(namespace string, podName string, filePaths []string) {
func verifyFilesExistOnVSphereVolume(namespace string, podName string, filePaths ...string) {
for _, filePath := range filePaths {
_, err := framework.RunKubectl("exec", fmt.Sprintf("--namespace=%s", namespace), podName, "--", "/bin/ls", filePath)
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to verify file: %q on the pod: %q", filePath, podName))
@ -753,3 +754,72 @@ func GetReadySchedulableRandomNodeInfo() *NodeInfo {
Expect(nodesInfo).NotTo(BeEmpty())
return nodesInfo[rand.Int()%len(nodesInfo)]
}
// invokeVCenterServiceControl invokes the given command for the given service
// via service-control on the given vCenter host over SSH.
func invokeVCenterServiceControl(command, service, host string) error {
sshCmd := fmt.Sprintf("service-control --%s %s", command, service)
framework.Logf("Invoking command %v on vCenter host %v", sshCmd, host)
result, err := framework.SSH(sshCmd, host, framework.TestContext.Provider)
if err != nil || result.Code != 0 {
framework.LogSSHResult(result)
return fmt.Errorf("couldn't execute command: %s on vCenter host: %v", sshCmd, err)
}
return nil
}
// expectVolumeToBeAttached checks if the given Volume is attached to the given
// Node, else fails.
func expectVolumeToBeAttached(nodeName, volumePath string) {
isAttached, err := diskIsAttached(volumePath, nodeName)
Expect(err).NotTo(HaveOccurred())
Expect(isAttached).To(BeTrue(), fmt.Sprintf("disk: %s is not attached with the node", volumePath))
}
// expectVolumesToBeAttached checks if the given Volumes are attached to the
// corresponding set of Nodes, else fails.
func expectVolumesToBeAttached(pods []*v1.Pod, volumePaths []string) {
for i, pod := range pods {
nodeName := pod.Spec.NodeName
volumePath := volumePaths[i]
By(fmt.Sprintf("Verifying that volume %v is attached to node %v", volumePath, nodeName))
expectVolumeToBeAttached(nodeName, volumePath)
}
}
// expectFilesToBeAccessible checks if the given files are accessible on the
// corresponding set of Nodes, else fails.
func expectFilesToBeAccessible(namespace string, pods []*v1.Pod, filePaths []string) {
for i, pod := range pods {
podName := pod.Name
filePath := filePaths[i]
By(fmt.Sprintf("Verifying that file %v is accessible on pod %v", filePath, podName))
verifyFilesExistOnVSphereVolume(namespace, podName, filePath)
}
}
// writeContentToPodFile writes the given content to the specified file.
func writeContentToPodFile(namespace, podName, filePath, content string) error {
_, err := framework.RunKubectl("exec", fmt.Sprintf("--namespace=%s", namespace), podName,
"--", "/bin/sh", "-c", fmt.Sprintf("echo '%s' > %s", content, filePath))
return err
}
// expectFileContentToMatch checks if a given file contains the specified
// content, else fails.
func expectFileContentToMatch(namespace, podName, filePath, content string) {
_, err := framework.RunKubectl("exec", fmt.Sprintf("--namespace=%s", namespace), podName,
"--", "/bin/sh", "-c", fmt.Sprintf("grep '%s' %s", content, filePath))
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to match content of file: %q on the pod: %q", filePath, podName))
}
// expectFileContentsToMatch checks if the given contents match the ones present
// in corresponding files on respective Pods, else fails.
func expectFileContentsToMatch(namespace string, pods []*v1.Pod, filePaths []string, contents []string) {
for i, pod := range pods {
podName := pod.Name
filePath := filePaths[i]
By(fmt.Sprintf("Matching file content for %v on pod %v", filePath, podName))
expectFileContentToMatch(namespace, podName, filePath, contents[i])
}
}

5
test/e2e/storage/vsphere/vsphere_volume_cluster_ds.go
View File

@ -17,7 +17,6 @@ limitations under the License.
package vsphere
import (
"fmt"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -98,9 +97,7 @@ var _ = utils.SIGDescribe("Volume Provisioning On Clustered Datastore [Feature:v
nodeName := pod.Spec.NodeName
By("Verifying volume is attached")
isAttached, err := diskIsAttached(volumePath, nodeName)
Expect(err).NotTo(HaveOccurred())
Expect(isAttached).To(BeTrue(), fmt.Sprintf("disk: %s is not attached with the node: %v", volumePath, nodeName))
expectVolumeToBeAttached(nodeName, volumePath)
By("Deleting pod")
err = framework.DeletePodWithWait(f, client, pod)

10
test/e2e/storage/vsphere/vsphere_volume_master_restart.go
View File

@ -103,10 +103,7 @@ var _ = utils.SIGDescribe("Volume Attach Verify [Feature:vsphere][Serial][Disrup
nodeName := pod.Spec.NodeName
By(fmt.Sprintf("Verify volume %s is attached to the pod %s", volumePath, nodeName))
isAttached, err := diskIsAttached(volumePath, nodeName)
Expect(err).NotTo(HaveOccurred())
Expect(isAttached).To(BeTrue(), fmt.Sprintf("disk: %s is not attached with the node", volumePath))
expectVolumeToBeAttached(nodeName, volumePath)
}
By("Restarting kubelet on master node")
@ -121,10 +118,9 @@ var _ = utils.SIGDescribe("Volume Attach Verify [Feature:vsphere][Serial][Disrup
for i, pod := range pods {
volumePath := volumePaths[i]
nodeName := pod.Spec.NodeName
By(fmt.Sprintf("After master restart, verify volume %v is attached to the pod %v", volumePath, nodeName))
isAttached, err := diskIsAttached(volumePaths[i], nodeName)
Expect(err).NotTo(HaveOccurred())
Expect(isAttached).To(BeTrue(), fmt.Sprintf("disk: %s is not attached with the node", volumePath))
expectVolumeToBeAttached(nodeName, volumePath)
By(fmt.Sprintf("Deleting pod on node %s", nodeName))
err = framework.DeletePodWithWait(f, client, pod)

6
test/e2e/storage/vsphere/vsphere_volume_placement.go
View File

@ -320,9 +320,9 @@ var _ = utils.SIGDescribe("Volume Placement", func() {
// Verify newly and previously created files present on the volume mounted on the pod
By("Verify newly Created file and previously created files present on volume mounted on pod-A")
verifyFilesExistOnVSphereVolume(ns, podA.Name, podAFiles)
verifyFilesExistOnVSphereVolume(ns, podA.Name, podAFiles...)
By("Verify newly Created file and previously created files present on volume mounted on pod-B")
verifyFilesExistOnVSphereVolume(ns, podB.Name, podBFiles)
verifyFilesExistOnVSphereVolume(ns, podB.Name, podBFiles...)
By("Deleting pod-A")
framework.ExpectNoError(framework.DeletePodWithWait(f, c, podA), "Failed to delete pod ", podA.Name)
@ -378,7 +378,7 @@ func createAndVerifyFilesOnVolume(namespace string, podname string, newEmptyfile
// Verify newly and previously created files present on the volume mounted on the pod
By(fmt.Sprintf("Verify newly Created file and previously created files present on volume mounted on: %v", podname))
verifyFilesExistOnVSphereVolume(namespace, podname, filesToCheck)
verifyFilesExistOnVSphereVolume(namespace, podname, filesToCheck...)
}
func deletePodAndWaitForVolumeToDetach(f *framework.Framework, c clientset.Interface, pod *v1.Pod, nodeName string, volumePaths []string) {

176
test/e2e/storage/vsphere/vsphere_volume_vpxd_restart.go Normal file
View File

@ -0,0 +1,176 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package vsphere
import (
"fmt"
"strconv"
"time"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/uuid"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/storage/utils"
)
/*
Test to verify that a volume remains attached through vpxd restart.
For the number of schedulable nodes:
1. Create a Volume with default options.
2. Create a Pod with the created Volume.
3. Verify that the Volume is attached.
4. Create a file with random contents under the Volume's mount point on the Pod.
5. Stop the vpxd service on the vCenter host.
6. Verify that the file is accessible on the Pod and that it's contents match.
7. Start the vpxd service on the vCenter host.
8. Verify that the Volume remains attached, the file is accessible on the Pod, and that it's contents match.
9. Delete the Pod and wait for the Volume to be detached.
10. Delete the Volume.
*/
var _ = utils.SIGDescribe("Verify Volume Attach Through vpxd Restart [Feature:vsphere][Serial][Disruptive]", func() {
f := framework.NewDefaultFramework("restart-vpxd")
type node struct {
name string
kvLabels map[string]string
nodeInfo *NodeInfo
}
const (
labelKey = "vsphere_e2e_label_vpxd_restart"
vpxdServiceName = "vmware-vpxd"
)
var (
client clientset.Interface
namespace string
vcNodesMap map[string][]node
)
BeforeEach(func() {
// Requires SSH access to vCenter.
framework.SkipUnlessProviderIs("vsphere")
Bootstrap(f)
client = f.ClientSet
namespace = f.Namespace.Name
framework.ExpectNoError(framework.WaitForAllNodesSchedulable(client, framework.TestContext.NodeSchedulableTimeout))
nodes := framework.GetReadySchedulableNodesOrDie(client)
numNodes := len(nodes.Items)
Expect(numNodes).NotTo(BeZero(), "No nodes are available for testing volume access through vpxd restart")
vcNodesMap = make(map[string][]node)
for i := 0; i < numNodes; i++ {
nodeInfo := TestContext.NodeMapper.GetNodeInfo(nodes.Items[i].Name)
nodeName := nodes.Items[i].Name
nodeLabel := "vsphere_e2e_" + string(uuid.NewUUID())
framework.AddOrUpdateLabelOnNode(client, nodeName, labelKey, nodeLabel)
vcHost := nodeInfo.VSphere.Config.Hostname
vcNodesMap[vcHost] = append(vcNodesMap[vcHost], node{
name: nodeName,
kvLabels: map[string]string{labelKey: nodeLabel},
nodeInfo: nodeInfo,
})
}
})
It("verify volume remains attached through vpxd restart", func() {
for vcHost, nodes := range vcNodesMap {
var (
volumePaths []string
filePaths []string
fileContents []string
pods []*v1.Pod
)
framework.Logf("Testing for nodes on vCenter host: %s", vcHost)
for i, node := range nodes {
By(fmt.Sprintf("Creating test vsphere volume %d", i))
volumePath, err := node.nodeInfo.VSphere.CreateVolume(&VolumeOptions{}, node.nodeInfo.DataCenterRef)
Expect(err).NotTo(HaveOccurred())
volumePaths = append(volumePaths, volumePath)
By(fmt.Sprintf("Creating pod %d on node %v", i, node.name))
podspec := getVSpherePodSpecWithVolumePaths([]string{volumePath}, node.kvLabels, nil)
pod, err := client.CoreV1().Pods(namespace).Create(podspec)
Expect(err).NotTo(HaveOccurred())
By(fmt.Sprintf("Waiting for pod %d to be ready", i))
Expect(framework.WaitForPodNameRunningInNamespace(client, pod.Name, namespace)).To(Succeed())
pod, err = client.CoreV1().Pods(namespace).Get(pod.Name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
pods = append(pods, pod)
nodeName := pod.Spec.NodeName
By(fmt.Sprintf("Verifying that volume %v is attached to node %v", volumePath, nodeName))
expectVolumeToBeAttached(nodeName, volumePath)
By(fmt.Sprintf("Creating a file with random content on the volume mounted on pod %d", i))
filePath := fmt.Sprintf("/mnt/volume1/%v_vpxd_restart_test_%v.txt", namespace, strconv.FormatInt(time.Now().UnixNano(), 10))
randomContent := fmt.Sprintf("Random Content -- %v", strconv.FormatInt(time.Now().UnixNano(), 10))
err = writeContentToPodFile(namespace, pod.Name, filePath, randomContent)
Expect(err).NotTo(HaveOccurred())
filePaths = append(filePaths, filePath)
fileContents = append(fileContents, randomContent)
}
By("Stopping vpxd on the vCenter host")
vcAddress := vcHost + ":22"
err := invokeVCenterServiceControl("stop", vpxdServiceName, vcAddress)
Expect(err).NotTo(HaveOccurred(), "Unable to stop vpxd on the vCenter host")
expectFilesToBeAccessible(namespace, pods, filePaths)
expectFileContentsToMatch(namespace, pods, filePaths, fileContents)
By("Starting vpxd on the vCenter host")
err = invokeVCenterServiceControl("start", vpxdServiceName, vcAddress)
Expect(err).NotTo(HaveOccurred(), "Unable to start vpxd on the vCenter host")
expectVolumesToBeAttached(pods, volumePaths)
expectFilesToBeAccessible(namespace, pods, filePaths)
expectFileContentsToMatch(namespace, pods, filePaths, fileContents)
for i, node := range nodes {
pod := pods[i]
nodeName := pod.Spec.NodeName
volumePath := volumePaths[i]
By(fmt.Sprintf("Deleting pod on node %s", nodeName))
err = framework.DeletePodWithWait(f, client, pod)
Expect(err).NotTo(HaveOccurred())
By(fmt.Sprintf("Waiting for volume %s to be detached from node %s", volumePath, nodeName))
err = waitForVSphereDiskToDetach(volumePath, nodeName)
Expect(err).NotTo(HaveOccurred())
By(fmt.Sprintf("Deleting volume %s", volumePath))
err = node.nodeInfo.VSphere.DeleteVolume(volumePath, node.nodeInfo.DataCenterRef)
Expect(err).NotTo(HaveOccurred())
}
}
})
})