github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add GCE PD tests for windows cluster 

This PR is the first one to add a few GCE PD tests for windows cluster. Will
add more tests in later PRs
pull/564/head
Jing Xu 2019-02-27 10:22:35 -08:00
parent fb9fdb3360
commit 18be0a49e6
7 changed files with 210 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

164
test/e2e/framework/volume_util.go
View File

@ -41,6 +41,7 @@ package framework
import (
"fmt"
"path/filepath"
"strconv"
"time"
@ -397,13 +398,21 @@ func VolumeTestCleanup(f *Framework, config VolumeTestConfig) {
}
}
// Start a client pod using given VolumeSource (exported by startVolumeServer())
// TestVolumeClient start a client pod using given VolumeSource (exported by startVolumeServer())
// and check that the pod sees expected data, e.g. from the server pod.
// Multiple VolumeTests can be specified to mount multiple volumes to a single
// pod.
func TestVolumeClient(client clientset.Interface, config VolumeTestConfig, fsGroup *int64, fsType string, tests []VolumeTest) {
By(fmt.Sprint("starting ", config.Prefix, " client"))
By(fmt.Sprint("starting ", config.Prefix, "-client"))
var gracePeriod int64 = 1
var command string
if !NodeOSDistroIs("windows") {
command = "while true ; do cat /opt/0/index.html ; sleep 2 ; ls -altrh /opt/ ; sleep 2 ; done "
} else {
command = "while(1) {cat /opt/0/index.html ; sleep 2 ; ls /opt/; sleep 2}"
}
seLinuxOptions := &v1.SELinuxOptions{Level: "s0:c0,c1"}
clientPod := &v1.Pod{
TypeMeta: metav1.TypeMeta{
Kind: "Pod",
@ -419,34 +428,24 @@ func TestVolumeClient(client clientset.Interface, config VolumeTestConfig, fsGro
Containers: []v1.Container{
{
Name: config.Prefix + "-client",
Image: BusyBoxImage,
Image: GetTestImage(BusyBoxImage),
WorkingDir: "/opt",
// An imperative and easily debuggable container which reads vol contents for
// us to scan in the tests or by eye.
// We expect that /opt is empty in the minimal containers which we use in this test.
Command: []string{
"/bin/sh",
"-c",
"while true ; do cat /opt/0/index.html ; sleep 2 ; ls -altrh /opt/ ; sleep 2 ; done ",
},
Command: GenerateScriptCmd(command),
VolumeMounts: []v1.VolumeMount{},
},
},
TerminationGracePeriodSeconds: &gracePeriod,
SecurityContext: &v1.PodSecurityContext{
SELinuxOptions: &v1.SELinuxOptions{
Level: "s0:c0,c1",
},
},
Volumes: []v1.Volume{},
NodeName: config.ClientNodeName,
NodeSelector: config.NodeSelector,
SecurityContext: GeneratePodSecurityContext(fsGroup, seLinuxOptions),
Volumes: []v1.Volume{},
NodeName: config.ClientNodeName,
NodeSelector: config.NodeSelector,
},
}
podsNamespacer := client.CoreV1().Pods(config.Namespace)
clientPod.Spec.SecurityContext.FSGroup = fsGroup
for i, test := range tests {
volumeName := fmt.Sprintf("%s-%s-%d", config.Prefix, "volume", i)
clientPod.Spec.Containers[0].VolumeMounts = append(clientPod.Spec.Containers[0].VolumeMounts, v1.VolumeMount{
@ -461,30 +460,33 @@ func TestVolumeClient(client clientset.Interface, config VolumeTestConfig, fsGro
clientPod, err := podsNamespacer.Create(clientPod)
if err != nil {
Failf("Failed to create %s pod: %v", clientPod.Name, err)
}
ExpectNoError(WaitForPodRunningInNamespace(client, clientPod))
By("Checking that text file contents are perfect.")
for i, test := range tests {
fileName := fmt.Sprintf("/opt/%d/%s", i, test.File)
_, err = LookForStringInPodExec(config.Namespace, clientPod.Name, []string{"cat", fileName}, test.ExpectedContent, time.Minute)
commands := GenerateReadFileCmd(fileName)
_, err = LookForStringInPodExec(config.Namespace, clientPod.Name, commands, test.ExpectedContent, time.Minute)
ExpectNoError(err, "failed: finding the contents of the mounted file %s.", fileName)
}
if !NodeOSDistroIs("windows") {
if fsGroup != nil {
By("Checking fsGroup is correct.")
_, err = LookForStringInPodExec(config.Namespace, clientPod.Name, []string{"ls", "-ld", "/opt/0"}, strconv.Itoa(int(*fsGroup)), time.Minute)
ExpectNoError(err, "failed: getting the right privileges in the file %v", int(*fsGroup))
}
if fsGroup != nil {
By("Checking fsGroup is correct.")
_, err = LookForStringInPodExec(config.Namespace, clientPod.Name, []string{"ls", "-ld", "/opt/0"}, strconv.Itoa(int(*fsGroup)), time.Minute)
ExpectNoError(err, "failed: getting the right privileges in the file %v", int(*fsGroup))
}
if fsType != "" {
By("Checking fsType is correct.")
_, err = LookForStringInPodExec(config.Namespace, clientPod.Name, []string{"grep", " /opt/0 ", "/proc/mounts"}, fsType, time.Minute)
ExpectNoError(err, "failed: getting the right fsType %s", fsType)
if fsType != "" {
By("Checking fsType is correct.")
_, err = LookForStringInPodExec(config.Namespace, clientPod.Name, []string{"grep", " /opt/0 ", "/proc/mounts"}, fsType, time.Minute)
ExpectNoError(err, "failed: getting the right fsType %s", fsType)
}
}
}
// Insert index.html with given content into given volume. It does so by
// InjectHtml insert index.html with given content into given volume. It does so by
// starting and auxiliary pod which writes the file there.
// The volume must be writable.
func InjectHtml(client clientset.Interface, config VolumeTestConfig, fsGroup *int64, volume v1.VolumeSource, content string) {
@ -492,7 +494,7 @@ func InjectHtml(client clientset.Interface, config VolumeTestConfig, fsGroup *in
podClient := client.CoreV1().Pods(config.Namespace)
podName := fmt.Sprintf("%s-injector-%s", config.Prefix, rand.String(4))
volMountName := fmt.Sprintf("%s-volume-%s", config.Prefix, rand.String(4))
privileged := true
fileName := "/mnt/index.html"
injectPod := &v1.Pod{
TypeMeta: metav1.TypeMeta{
@ -509,18 +511,15 @@ func InjectHtml(client clientset.Interface, config VolumeTestConfig, fsGroup *in
Containers: []v1.Container{
{
Name: config.Prefix + "-injector",
Image: BusyBoxImage,
Command: []string{"/bin/sh"},
Args: []string{"-c", "echo '" + content + "' > /mnt/index.html && chmod o+rX /mnt /mnt/index.html"},
Image: GetTestImage(BusyBoxImage),
Command: GenerateWriteFileCmd(content, fileName),
VolumeMounts: []v1.VolumeMount{
{
Name: volMountName,
MountPath: "/mnt",
},
},
SecurityContext: &v1.SecurityContext{
Privileged: &privileged,
},
SecurityContext: GenerateSecurityContext(true),
},
},
SecurityContext: &v1.PodSecurityContext{
@ -559,3 +558,94 @@ func CreateGCEVolume() (*v1.PersistentVolumeSource, string) {
},
}, diskName
}
// GenerateScriptCmd generates the corresponding command lines to execute a command.
// Depending on the Node OS is Windows or linux, the command will use powershell or /bin/sh
func GenerateScriptCmd(command string) []string {
var commands []string
if !NodeOSDistroIs("windows") {
commands = []string{"/bin/sh", "-c", command}
} else {
commands = []string{"powershell", "/c", command}
}
return commands
}
// GenerateWriteFileCmd generates the corresponding command lines to write a file with the given content and file path.
// Depending on the Node OS is Windows or linux, the command will use powershell or /bin/sh
func GenerateWriteFileCmd(content, fullPath string) []string {
var commands []string
if !NodeOSDistroIs("windows") {
commands = []string{"/bin/sh", "-c", "echo '" + content + "' > " + fullPath}
} else {
commands = []string{"powershell", "/c", "echo '" + content + "' > " + fullPath}
}
return commands
}
// GenerateReadFileCmd generates the corresponding command lines to read from a file with the given file path.
// Depending on the Node OS is Windows or linux, the command will use powershell or /bin/sh
func GenerateReadFileCmd(fullPath string) []string {
var commands []string
if !NodeOSDistroIs("windows") {
commands = []string{"cat", fullPath}
} else {
commands = []string{"powershell", "/c", "type " + fullPath}
}
return commands
}
// GenerateWriteandExecuteScriptFileCmd generates the corresponding command lines to write a file with the given file path
// and also execute this file.
// Depending on the Node OS is Windows or linux, the command will use powershell or /bin/sh
func GenerateWriteandExecuteScriptFileCmd(content, fileName, filePath string) []string {
// for windows cluster, modify the Pod spec.
if NodeOSDistroIs("windows") {
scriptName := fmt.Sprintf("%s.ps1", fileName)
fullPath := filepath.Join(filePath, scriptName)
cmd := "echo \"" + content + "\" > " + fullPath + "; .\\" + fullPath
Logf("generated pod command %s", cmd)
return []string{"powershell", "/c", cmd}
} else {
scriptName := fmt.Sprintf("%s.sh", fileName)
fullPath := filepath.Join(filePath, scriptName)
cmd := fmt.Sprintf("echo \"%s\" > %s; chmod u+x %s; %s;", content, fullPath, fullPath, fullPath)
return []string{"/bin/sh", "-ec", cmd}
}
}
// GenerateSecurityContext generates the corresponding container security context with the given inputs
// If the Node OS is windows, currently we will ignore the inputs and return nil.
// TODO: Will modify it after windows has its own security context
func GenerateSecurityContext(privileged bool) *v1.SecurityContext {
if NodeOSDistroIs("windows") {
return nil
}
return &v1.SecurityContext{
Privileged: &privileged,
}
}
// GeneratePodSecurityContext generates the corresponding pod security context with the given inputs
// If the Node OS is windows, currently we will ignore the inputs and return nil.
// TODO: Will modify it after windows has its own security context
func GeneratePodSecurityContext(fsGroup *int64, seLinuxOptions *v1.SELinuxOptions) *v1.PodSecurityContext {
if NodeOSDistroIs("windows") {
return nil
}
return &v1.PodSecurityContext{
SELinuxOptions: seLinuxOptions,
FSGroup: fsGroup,
}
}
// GetTestImage returns the image name with the given input
// If the Node OS is windows, currently we return Nettest image for Windows node
// due to the issue of #https://github.com/kubernetes-sigs/windows-testing/pull/35.
func GetTestImage(image string) string {
if NodeOSDistroIs("windows") {
return imageutils.GetE2EImage(imageutils.Nettest)
}
return image
}

6
test/e2e/storage/drivers/csi.go
View File

@ -347,6 +347,9 @@ func (g *gcePDCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
if pattern.FsType == "xfs" {
framework.SkipUnlessNodeOSDistroIs("ubuntu", "custom")
}
if pattern.FeatureTag == "sig-windows" {
framework.Skipf("Skipping tests for windows since CSI does not support it yet")
}
}
func (g *gcePDCSIDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
@ -461,6 +464,9 @@ func (g *gcePDExternalCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPa
if pattern.FsType == "xfs" {
framework.SkipUnlessNodeOSDistroIs("ubuntu", "custom")
}
if pattern.FeatureTag == "sig-windows" {
framework.Skipf("Skipping tests for windows since CSI does not support it yet")
}
}
func (g *gcePDExternalCSIDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {

45
test/e2e/storage/drivers/in_tree.go
View File

@ -1104,23 +1104,32 @@ var _ testsuites.InlineVolumeTestDriver = &gcePdDriver{}
var _ testsuites.PreprovisionedPVTestDriver = &gcePdDriver{}
var _ testsuites.DynamicPVTestDriver = &gcePdDriver{}
// InitGceDriver returns gcePdDriver that implements TestDriver interface
// InitGcePdDriver returns gcePdDriver that implements TestDriver interface
func InitGcePdDriver() testsuites.TestDriver {
var supportedTypes sets.String
var capFsGroup bool
if framework.NodeOSDistroIs("windows") {
supportedTypes = sets.NewString("ntfs")
capFsGroup = false
} else {
supportedTypes = sets.NewString(
"", // Default fsType
"ext2",
"ext3",
"ext4",
"xfs",
)
capFsGroup = true
}
return &gcePdDriver{
driverInfo: testsuites.DriverInfo{
Name: "gcepd",
MaxFileSize: testpatterns.FileSizeMedium,
SupportedFsType: sets.NewString(
"", // Default fsType
"ext2",
"ext3",
"ext4",
"xfs",
),
Name: "gcepd",
MaxFileSize: testpatterns.FileSizeMedium,
SupportedFsType: supportedTypes,
SupportedMountOption: sets.NewString("debug", "nouid32"),
Capabilities: map[testsuites.Capability]bool{
testsuites.CapPersistence: true,
testsuites.CapFsGroup: true,
testsuites.CapFsGroup: capFsGroup,
testsuites.CapBlock: true,
testsuites.CapExec: true,
},
@ -1134,6 +1143,9 @@ func (g *gcePdDriver) GetDriverInfo() *testsuites.DriverInfo {
func (g *gcePdDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
framework.SkipUnlessProviderIs("gce", "gke")
if pattern.FeatureTag == "sig-windows" {
framework.SkipUnlessNodeOSDistroIs("windows")
}
}
func (g *gcePdDriver) GetVolumeSource(readOnly bool, fsType string, volume testsuites.TestVolume) *v1.VolumeSource {
@ -1183,11 +1195,18 @@ func (h *gcePdDriver) GetClaimSize() string {
}
func (g *gcePdDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
return &testsuites.PerTestConfig{
config := &testsuites.PerTestConfig{
Driver: g,
Prefix: "gcepd",
Framework: f,
}, func() {}
}
if framework.NodeOSDistroIs("windows") {
config.ClientNodeSelector = map[string]string{
"beta.kubernetes.io/os": "windows",
}
}
return config, func() {}
}
func (g *gcePdDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume {

24
test/e2e/storage/testpatterns/testpattern.go
View File

@ -145,6 +145,30 @@ var (
FsType: "xfs",
}
// Definitions for ntfs
// NtfsInlineVolume is TestPattern for "Inline-volume (ntfs)"
NtfsInlineVolume = TestPattern{
Name: "Inline-volume (ntfs)",
VolType: InlineVolume,
FsType: "ntfs",
FeatureTag: "sig-windows",
}
// NtfsPreprovisionedPV is TestPattern for "Pre-provisioned PV (ntfs)"
NtfsPreprovisionedPV = TestPattern{
Name: "Pre-provisioned PV (ntfs)",
VolType: PreprovisionedPV,
FsType: "ntfs",
FeatureTag: "sig-windows",
}
// NtfsDynamicPV is TestPattern for "Dynamic PV (xfs)"
NtfsDynamicPV = TestPattern{
Name: "Dynamic PV (ntfs)",
VolType: DynamicPV,
FsType: "ntfs",
FeatureTag: "sig-windows",
}
// Definitions for Filesystem volume mode
// FsVolModePreprovisionedPV is TestPattern for "Pre-provisioned PV (filesystem)"

3
test/e2e/storage/testsuites/base.go
View File

@ -133,6 +133,9 @@ func skipUnsupportedTest(driver TestDriver, pattern testpatterns.TestPattern) {
if pattern.FsType == "xfs" && framework.NodeOSDistroIs("gci") {
framework.Skipf("Distro doesn't support xfs -- skipping")
}
if pattern.FsType == "ntfs" && !framework.NodeOSDistroIs("windows") {
framework.Skipf("Distro %s doesn't support ntfs -- skipping", framework.TestContext.NodeOSDistro)
}
}
// 4. Check with driver specific logic

12
test/e2e/storage/testsuites/provisioning.go
View File

@ -35,7 +35,6 @@ import (
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/storage/testpatterns"
imageutils "k8s.io/kubernetes/test/utils/image"
)
// StorageClassTest represents parameters to be used by provisioning tests.
@ -70,6 +69,7 @@ func InitProvisioningTestSuite() TestSuite {
name: "provisioning",
testPatterns: []testpatterns.TestPattern{
testpatterns.DefaultFsDynamicPV,
testpatterns.NtfsDynamicPV,
},
},
}
@ -120,7 +120,7 @@ func (p *provisioningTestSuite) defineTests(driver TestDriver, pattern testpatte
l.config, l.testCleanup = driver.PrepareTest(f)
l.cs = l.config.Framework.ClientSet
claimSize := dDriver.GetClaimSize()
l.sc = dDriver.GetDynamicProvisionStorageClass(l.config, "")
l.sc = dDriver.GetDynamicProvisionStorageClass(l.config, pattern.FsType)
if l.sc == nil {
framework.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", dInfo.Name)
}
@ -472,6 +472,9 @@ func PVMultiNodeCheck(client clientset.Interface, claim *v1.PersistentVolumeClai
By(fmt.Sprintf("checking the created volume is readable and retains data on another node %+v", secondNode))
command = "grep 'hello world' /mnt/test/data"
if framework.NodeOSDistroIs("windows") {
command = "select-string 'hello world' /mnt/test/data"
}
pod = StartInPodWithVolume(client, claim.Namespace, claim.Name, "pvc-reader-node2", command, secondNode)
framework.ExpectNoError(framework.WaitForPodSuccessInNamespaceSlow(client, pod.Name, pod.Namespace))
runningPod, err = client.CoreV1().Pods(pod.Namespace).Get(pod.Name, metav1.GetOptions{})
@ -607,9 +610,8 @@ func StartInPodWithVolume(c clientset.Interface, ns, claimName, podName, command
Containers: []v1.Container{
{
Name: "volume-tester",
Image: imageutils.GetE2EImage(imageutils.BusyBox),
Command: []string{"/bin/sh"},
Args: []string{"-c", command},
Image: framework.GetTestImage(framework.BusyBoxImage),
Command: framework.GenerateScriptCmd(command),
VolumeMounts: []v1.VolumeMount{
{
Name: "my-volume",

20
test/e2e/storage/testsuites/volumes.go
View File

@ -23,7 +23,6 @@ package testsuites
import (
"fmt"
"path/filepath"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@ -63,6 +62,10 @@ func InitVolumesTestSuite() TestSuite {
testpatterns.XfsInlineVolume,
testpatterns.XfsPreprovisionedPV,
testpatterns.XfsDynamicPV,
// ntfs
testpatterns.NtfsInlineVolume,
testpatterns.NtfsPreprovisionedPV,
testpatterns.NtfsDynamicPV,
},
},
}
@ -105,7 +108,7 @@ func (t *volumesTestSuite) defineTests(driver TestDriver, pattern testpatterns.T
// registers its own BeforeEach which creates the namespace. Beware that it
// also registers an AfterEach which renders f unusable. Any code using
// f must run inside an It or Context callback.
f := framework.NewDefaultFramework("volumeio")
f := framework.NewDefaultFramework("volume")
init := func() {
l = local{}
@ -181,10 +184,9 @@ func testScriptInPod(
volName = "vol1"
)
suffix := generateSuffixForPodName(volumeType)
scriptName := fmt.Sprintf("test-%s.sh", suffix)
fullPath := filepath.Join(volPath, scriptName)
cmd := fmt.Sprintf("echo \"ls %s\" > %s; chmod u+x %s; %s", volPath, fullPath, fullPath, fullPath)
fileName := fmt.Sprintf("test-%s", suffix)
content := fmt.Sprintf("ls %s", volPath)
command := framework.GenerateWriteandExecuteScriptFileCmd(content, fileName, volPath)
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("exec-volume-test-%s", suffix),
@ -194,8 +196,8 @@ func testScriptInPod(
Containers: []v1.Container{
{
Name: fmt.Sprintf("exec-container-%s", suffix),
Image: imageutils.GetE2EImage(imageutils.Nginx),
Command: []string{"/bin/sh", "-ec", cmd},
Image: framework.GetTestImage(imageutils.GetE2EImage(imageutils.Nginx)),
Command: command,
VolumeMounts: []v1.VolumeMount{
{
Name: volName,
@ -215,7 +217,7 @@ func testScriptInPod(
},
}
By(fmt.Sprintf("Creating pod %s", pod.Name))
f.TestContainerOutput("exec-volume-test", pod, 0, []string{scriptName})
f.TestContainerOutput("exec-volume-test", pod, 0, []string{fileName})
By(fmt.Sprintf("Deleting pod %s", pod.Name))
err := framework.DeletePodWithWait(f, f.ClientSet, pod)