github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #69441 from mkimuram/provision 

Move minimum set of dynamic provisioning e2e test to testsuites
pull/58/head
k8s-ci-robot 2018-10-23 07:08:35 -07:00 committed by GitHub
parent 327a2bb871 4fcb36e205
commit 83d3f7033b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 650 additions and 513 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
test/e2e/storage/csi_volumes.go
View File

@ -31,6 +31,7 @@ import (
csiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1"
csiclient "k8s.io/csi-api/pkg/client/clientset/versioned"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/storage/testsuites"
"k8s.io/kubernetes/test/e2e/storage/utils"
imageutils "k8s.io/kubernetes/test/utils/image"
@ -49,7 +50,7 @@ const (
type csiTestDriver interface {
createCSIDriver()
cleanupCSIDriver()
createStorageClassTest(node v1.Node) storageClassTest
createStorageClassTest(node v1.Node) testsuites.StorageClassTest
}
var csiTestDrivers = map[string]func(f *framework.Framework, config framework.VolumeTestConfig) csiTestDriver{
@ -110,7 +111,7 @@ var _ = utils.SIGDescribe("CSI Volumes", func() {
claim := newClaim(t, ns.GetName(), "")
class := newStorageClass(t, ns.GetName(), "")
claim.Spec.StorageClassName = &class.ObjectMeta.Name
testDynamicProvisioning(t, cs, claim, class)
testsuites.TestDynamicProvisioning(t, cs, claim, class)
})
})
}
@ -187,7 +188,7 @@ var _ = utils.SIGDescribe("CSI Volumes", func() {
By("Checking if VolumeAttachment was created for the pod")
// Check that VolumeAttachment does not exist
handle := getVolumeHandle(cs, claim)
attachmentHash := sha256.Sum256([]byte(fmt.Sprintf("%s%s%s", handle, t.provisioner, node.Name)))
attachmentHash := sha256.Sum256([]byte(fmt.Sprintf("%s%s%s", handle, t.Provisioner, node.Name)))
attachmentName := fmt.Sprintf("csi-%x", attachmentHash)
_, err = cs.StorageV1beta1().VolumeAttachments().Get(attachmentName, metav1.GetOptions{})
if err != nil {
@ -242,7 +243,7 @@ func getVolumeHandle(cs clientset.Interface, claim *v1.PersistentVolumeClaim) st
return pv.Spec.CSI.VolumeHandle
}
func startPausePod(cs clientset.Interface, t storageClassTest, ns string) (*storagev1.StorageClass, *v1.PersistentVolumeClaim, *v1.Pod) {
func startPausePod(cs clientset.Interface, t testsuites.StorageClassTest, ns string) (*storagev1.StorageClass, *v1.PersistentVolumeClaim, *v1.Pod) {
class := newStorageClass(t, ns, "")
class, err := cs.StorageV1().StorageClasses().Create(class)
framework.ExpectNoError(err, "Failed to create class : %v", err)
@ -283,8 +284,8 @@ func startPausePod(cs clientset.Interface, t storageClassTest, ns string) (*stor
},
}
if len(t.nodeName) != 0 {
pod.Spec.NodeName = t.nodeName
if len(t.NodeName) != 0 {
pod.Spec.NodeName = t.NodeName
}
pod, err = cs.CoreV1().Pods(ns).Create(pod)
framework.ExpectNoError(err, "Failed to create pod: %v", err)
@ -311,14 +312,14 @@ func initCSIHostpath(f *framework.Framework, config framework.VolumeTestConfig)
}
}
func (h *hostpathCSIDriver) createStorageClassTest(node v1.Node) storageClassTest {
return storageClassTest{
name: "csi-hostpath",
provisioner: "csi-hostpath",
parameters: map[string]string{},
claimSize: "1Gi",
expectedSize: "1Gi",
nodeName: node.Name,
func (h *hostpathCSIDriver) createStorageClassTest(node v1.Node) testsuites.StorageClassTest {
return testsuites.StorageClassTest{
Name: "csi-hostpath",
Provisioner: "csi-hostpath",
Parameters: map[string]string{},
ClaimSize: "1Gi",
ExpectedSize: "1Gi",
NodeName: node.Name,
}
}
@ -379,14 +380,14 @@ func initCSIgcePD(f *framework.Framework, config framework.VolumeTestConfig) csi
}
}
func (g *gcePDCSIDriver) createStorageClassTest(node v1.Node) storageClassTest {
return storageClassTest{
name: "com.google.csi.gcepd",
provisioner: "com.google.csi.gcepd",
parameters: map[string]string{"type": "pd-standard"},
claimSize: "5Gi",
expectedSize: "5Gi",
nodeName: node.Name,
func (g *gcePDCSIDriver) createStorageClassTest(node v1.Node) testsuites.StorageClassTest {
return testsuites.StorageClassTest{
Name: "com.google.csi.gcepd",
Provisioner: "com.google.csi.gcepd",
Parameters: map[string]string{"type": "pd-standard"},
ClaimSize: "5Gi",
ExpectedSize: "5Gi",
NodeName: node.Name,
}
}

16
test/e2e/storage/drivers/base.go
View File

@ -81,11 +81,13 @@ type DriverInfo struct {
Name string // Name of the driver
FeatureTag string // FeatureTag for the driver
MaxFileSize int64 // Max file size to be tested for this driver
SupportedFsType sets.String // Map of string for supported fs type
IsPersistent bool // Flag to represent whether it provides persistency
IsFsGroupSupported bool // Flag to represent whether it supports fsGroup
IsBlockSupported bool // Flag to represent whether it supports Block Volume
MaxFileSize int64 // Max file size to be tested for this driver
SupportedFsType sets.String // Map of string for supported fs type
SupportedMountOption sets.String // Map of string for supported mount option
RequiredMountOption sets.String // Map of string for required mount option (Optional)
IsPersistent bool // Flag to represent whether it provides persistency
IsFsGroupSupported bool // Flag to represent whether it supports fsGroup
IsBlockSupported bool // Flag to represent whether it supports Block Volume
// Parameters below will be set inside test loop by using SetCommonDriverParameters.
// Drivers that implement TestDriver is required to set all the above parameters
@ -104,8 +106,8 @@ func GetDriverNameWithFeatureTags(driver TestDriver) string {
return fmt.Sprintf("[Driver: %s]%s", dInfo.Name, dInfo.FeatureTag)
}
// CreateVolume creates volume for test unless dynamicPV test
func CreateVolume(driver TestDriver, volType testpatterns.TestVolType) interface{} {
// Create Volume for test unless dynamicPV test
switch volType {
case testpatterns.InlineVolume:
fallthrough
@ -121,8 +123,8 @@ func CreateVolume(driver TestDriver, volType testpatterns.TestVolType) interface
return nil
}
// DeleteVolume deletes volume for test unless dynamicPV test
func DeleteVolume(driver TestDriver, volType testpatterns.TestVolType, testResource interface{}) {
// Delete Volume for test unless dynamicPV test
switch volType {
case testpatterns.InlineVolume:
fallthrough

24
test/e2e/storage/drivers/in_tree.go
View File

@ -88,9 +88,11 @@ func InitNFSDriver() TestDriver {
SupportedFsType: sets.NewString(
"", // Default fsType
),
IsPersistent: true,
IsFsGroupSupported: false,
IsBlockSupported: false,
SupportedMountOption: sets.NewString("proto=tcp", "nosuid"),
RequiredMountOption: sets.NewString("vers=4.1"),
IsPersistent: true,
IsFsGroupSupported: false,
IsBlockSupported: false,
},
}
}
@ -673,7 +675,7 @@ var _ TestDriver = &hostPathDriver{}
var _ PreprovisionedVolumeTestDriver = &hostPathDriver{}
var _ InlineVolumeTestDriver = &hostPathDriver{}
// InitHostpathDriver returns hostPathDriver that implements TestDriver interface
// InitHostPathDriver returns hostPathDriver that implements TestDriver interface
func InitHostPathDriver() TestDriver {
return &hostPathDriver{
driverInfo: DriverInfo{
@ -1118,9 +1120,10 @@ func InitGcePdDriver() TestDriver {
"ext4",
"xfs",
),
IsPersistent: true,
IsFsGroupSupported: true,
IsBlockSupported: true,
SupportedMountOption: sets.NewString("debug", "nouid32"),
IsPersistent: true,
IsFsGroupSupported: true,
IsBlockSupported: true,
},
}
}
@ -1460,9 +1463,10 @@ func InitAwsDriver() TestDriver {
"", // Default fsType
"ext3",
),
IsPersistent: true,
IsFsGroupSupported: true,
IsBlockSupported: true,
SupportedMountOption: sets.NewString("debug", "nouid32"),
IsPersistent: true,
IsFsGroupSupported: true,
IsBlockSupported: true,
},
}
}

7
test/e2e/storage/generic_persistent_volume-disruptive.go
View File

@ -23,6 +23,7 @@ import (
"k8s.io/api/core/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/storage/testsuites"
"k8s.io/kubernetes/test/e2e/storage/utils"
)
@ -85,9 +86,9 @@ var _ = utils.SIGDescribe("GenericPersistentVolume[Disruptive]", func() {
func createPodPVCFromSC(f *framework.Framework, c clientset.Interface, ns string) (*v1.Pod, *v1.PersistentVolumeClaim, *v1.PersistentVolume) {
var err error
test := storageClassTest{
name: "default",
claimSize: "2Gi",
test := testsuites.StorageClassTest{
Name: "default",
ClaimSize: "2Gi",
}
pvc := newClaim(test, ns, "default")
pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(pvc)

1
test/e2e/storage/in_tree_volumes.go
View File

@ -50,6 +50,7 @@ var testSuites = []func() testsuites.TestSuite{
testsuites.InitVolumeIOTestSuite,
testsuites.InitVolumeModeTestSuite,
testsuites.InitSubPathTestSuite,
testsuites.InitProvisioningTestSuite,
}
// This executes testSuites for in-tree volumes.

7
test/e2e/storage/mounted_volume_resize.go
View File

@ -31,6 +31,7 @@ import (
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/pkg/client/conditions"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/storage/testsuites"
"k8s.io/kubernetes/test/e2e/storage/utils"
)
@ -72,9 +73,9 @@ var _ = utils.SIGDescribe("Mounted volume expand[Slow]", func() {
isNodeLabeled = true
}
test := storageClassTest{
name: "default",
claimSize: "2Gi",
test := testsuites.StorageClassTest{
Name: "default",
ClaimSize: "2Gi",
}
resizableSc, err = createResizableStorageClass(test, ns, "resizing", c)
Expect(err).NotTo(HaveOccurred(), "Error creating resizable storage class")

5
test/e2e/storage/pvc_protection.go
View File

@ -26,6 +26,7 @@ import (
"k8s.io/kubernetes/pkg/util/slice"
volumeutil "k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/storage/testsuites"
"k8s.io/kubernetes/test/e2e/storage/utils"
)
@ -47,8 +48,8 @@ var _ = utils.SIGDescribe("PVC Protection", func() {
By("Creating a PVC")
suffix := "pvc-protection"
defaultSC := getDefaultStorageClassName(client)
testStorageClass := storageClassTest{
claimSize: "1Gi",
testStorageClass := testsuites.StorageClassTest{
ClaimSize: "1Gi",
}
pvc = newClaim(testStorageClass, nameSpace, suffix)
pvc.Spec.StorageClassName = &defaultSC

71
test/e2e/storage/regional_pd.go
View File

@ -38,6 +38,7 @@ import (
"k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/framework/providers/gce"
"k8s.io/kubernetes/test/e2e/storage/testsuites"
"k8s.io/kubernetes/test/e2e/storage/utils"
imageutils "k8s.io/kubernetes/test/utils/image"
)
@ -90,19 +91,19 @@ func testVolumeProvisioning(c clientset.Interface, ns string) {
// This test checks that dynamic provisioning can provision a volume
// that can be used to persist data among pods.
tests := []storageClassTest{
tests := []testsuites.StorageClassTest{
{
name: "HDD Regional PD on GCE/GKE",
cloudProviders: []string{"gce", "gke"},
provisioner: "kubernetes.io/gce-pd",
parameters: map[string]string{
Name: "HDD Regional PD on GCE/GKE",
CloudProviders: []string{"gce", "gke"},
Provisioner: "kubernetes.io/gce-pd",
Parameters: map[string]string{
"type": "pd-standard",
"zones": strings.Join(cloudZones, ","),
"replication-type": "regional-pd",
},
claimSize: "1.5Gi",
expectedSize: "2Gi",
pvCheck: func(volume *v1.PersistentVolume) error {
ClaimSize: "1.5Gi",
ExpectedSize: "2Gi",
PvCheck: func(volume *v1.PersistentVolume) error {
err := checkGCEPD(volume, "pd-standard")
if err != nil {
return err
@ -111,16 +112,16 @@ func testVolumeProvisioning(c clientset.Interface, ns string) {
},
},
{
name: "HDD Regional PD with auto zone selection on GCE/GKE",
cloudProviders: []string{"gce", "gke"},
provisioner: "kubernetes.io/gce-pd",
parameters: map[string]string{
Name: "HDD Regional PD with auto zone selection on GCE/GKE",
CloudProviders: []string{"gce", "gke"},
Provisioner: "kubernetes.io/gce-pd",
Parameters: map[string]string{
"type": "pd-standard",
"replication-type": "regional-pd",
},
claimSize: "1.5Gi",
expectedSize: "2Gi",
pvCheck: func(volume *v1.PersistentVolume) error {
ClaimSize: "1.5Gi",
ExpectedSize: "2Gi",
PvCheck: func(volume *v1.PersistentVolume) error {
err := checkGCEPD(volume, "pd-standard")
if err != nil {
return err
@ -138,7 +139,7 @@ func testVolumeProvisioning(c clientset.Interface, ns string) {
class := newStorageClass(test, ns, "" /* suffix */)
claim := newClaim(test, ns, "" /* suffix */)
claim.Spec.StorageClassName = &class.Name
testDynamicProvisioning(test, c, claim, class)
testsuites.TestDynamicProvisioning(test, c, claim, class)
}
}
@ -278,15 +279,15 @@ func testZonalFailover(c clientset.Interface, ns string) {
}
func testRegionalDelayedBinding(c clientset.Interface, ns string) {
test := storageClassTest{
name: "Regional PD storage class with waitForFirstConsumer test on GCE",
provisioner: "kubernetes.io/gce-pd",
parameters: map[string]string{
test := testsuites.StorageClassTest{
Name: "Regional PD storage class with waitForFirstConsumer test on GCE",
Provisioner: "kubernetes.io/gce-pd",
Parameters: map[string]string{
"type": "pd-standard",
"replication-type": "regional-pd",
},
claimSize: "2Gi",
delayBinding: true,
ClaimSize: "2Gi",
DelayBinding: true,
}
suffix := "delayed-regional"
@ -305,15 +306,15 @@ func testRegionalDelayedBinding(c clientset.Interface, ns string) {
}
func testRegionalAllowedTopologies(c clientset.Interface, ns string) {
test := storageClassTest{
name: "Regional PD storage class with allowedTopologies test on GCE",
provisioner: "kubernetes.io/gce-pd",
parameters: map[string]string{
test := testsuites.StorageClassTest{
Name: "Regional PD storage class with allowedTopologies test on GCE",
Provisioner: "kubernetes.io/gce-pd",
Parameters: map[string]string{
"type": "pd-standard",
"replication-type": "regional-pd",
},
claimSize: "2Gi",
expectedSize: "2Gi",
ClaimSize: "2Gi",
ExpectedSize: "2Gi",
}
suffix := "topo-regional"
@ -322,20 +323,20 @@ func testRegionalAllowedTopologies(c clientset.Interface, ns string) {
addAllowedTopologiesToStorageClass(c, class, zones)
claim := newClaim(test, ns, suffix)
claim.Spec.StorageClassName = &class.Name
pv := testDynamicProvisioning(test, c, claim, class)
pv := testsuites.TestDynamicProvisioning(test, c, claim, class)
checkZonesFromLabelAndAffinity(pv, sets.NewString(zones...), true)
}
func testRegionalAllowedTopologiesWithDelayedBinding(c clientset.Interface, ns string) {
test := storageClassTest{
name: "Regional PD storage class with allowedTopologies and waitForFirstConsumer test on GCE",
provisioner: "kubernetes.io/gce-pd",
parameters: map[string]string{
test := testsuites.StorageClassTest{
Name: "Regional PD storage class with allowedTopologies and waitForFirstConsumer test on GCE",
Provisioner: "kubernetes.io/gce-pd",
Parameters: map[string]string{
"type": "pd-standard",
"replication-type": "regional-pd",
},
claimSize: "2Gi",
delayBinding: true,
ClaimSize: "2Gi",
DelayBinding: true,
}
suffix := "topo-delayed-regional"

1
test/e2e/storage/testsuites/BUILD
View File

@ -4,6 +4,7 @@ go_library(
name = "go_default_library",
srcs = [
"base.go",
"provisioning.go",
"subpath.go",
"volume_io.go",
"volumemode.go",

7
test/e2e/storage/testsuites/base.go
View File

@ -34,7 +34,7 @@ import (
"k8s.io/kubernetes/test/e2e/storage/testpatterns"
)
// TestSuite represents an interface for a set of tests whchi works with TestDriver
// TestSuite represents an interface for a set of tests which works with TestDriver
type TestSuite interface {
// getTestSuiteInfo returns the TestSuiteInfo for this TestSuite
getTestSuiteInfo() TestSuiteInfo
@ -44,6 +44,7 @@ type TestSuite interface {
execTest(drivers.TestDriver, testpatterns.TestPattern)
}
// TestSuiteInfo represents a set of parameters for TestSuite
type TestSuiteInfo struct {
name string // name of the TestSuite
featureTag string // featureTag for the TestSuite
@ -132,7 +133,7 @@ type genericVolumeTestResource struct {
var _ TestResource = &genericVolumeTestResource{}
// SetupResource sets up genericVolumeTestResource
// setupResource sets up genericVolumeTestResource
func (r *genericVolumeTestResource) setupResource(driver drivers.TestDriver, pattern testpatterns.TestPattern) {
r.driver = driver
dInfo := driver.GetDriverInfo()
@ -186,7 +187,7 @@ func (r *genericVolumeTestResource) setupResource(driver drivers.TestDriver, pat
}
}
// CleanupResource clean up genericVolumeTestResource
// cleanupResource cleans up genericVolumeTestResource
func (r *genericVolumeTestResource) cleanupResource(driver drivers.TestDriver, pattern testpatterns.TestPattern) {
dInfo := driver.GetDriverInfo()
f := dInfo.Framework

367
test/e2e/storage/testsuites/provisioning.go Normal file
View File

@ -0,0 +1,367 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package testsuites
import (
"fmt"
"time"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
apierrs "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/api/core/v1"
storage "k8s.io/api/storage/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/storage/drivers"
"k8s.io/kubernetes/test/e2e/storage/testpatterns"
imageutils "k8s.io/kubernetes/test/utils/image"
)
// StorageClassTest represents parameters to be used by provisioning tests
type StorageClassTest struct {
Name string
CloudProviders []string
Provisioner string
Parameters map[string]string
DelayBinding bool
ClaimSize string
ExpectedSize string
PvCheck func(volume *v1.PersistentVolume) error
NodeName string
SkipWriteReadCheck bool
VolumeMode *v1.PersistentVolumeMode
}
type provisioningTestSuite struct {
tsInfo TestSuiteInfo
}
var _ TestSuite = &provisioningTestSuite{}
// InitProvisioningTestSuite returns provisioningTestSuite that implements TestSuite interface
func InitProvisioningTestSuite() TestSuite {
return &provisioningTestSuite{
tsInfo: TestSuiteInfo{
name: "provisioning",
testPatterns: []testpatterns.TestPattern{
testpatterns.DefaultFsDynamicPV,
},
},
}
}
func (p *provisioningTestSuite) getTestSuiteInfo() TestSuiteInfo {
return p.tsInfo
}
func (p *provisioningTestSuite) skipUnsupportedTest(pattern testpatterns.TestPattern, driver drivers.TestDriver) {
}
func createProvisioningTestInput(driver drivers.TestDriver, pattern testpatterns.TestPattern) (provisioningTestResource, provisioningTestInput) {
// Setup test resource for driver and testpattern
resource := provisioningTestResource{}
resource.setupResource(driver, pattern)
input := provisioningTestInput{
testCase: StorageClassTest{
ClaimSize: resource.claimSize,
ExpectedSize: resource.claimSize,
},
cs: driver.GetDriverInfo().Framework.ClientSet,
pvc: resource.pvc,
sc: resource.sc,
dInfo: driver.GetDriverInfo(),
}
if driver.GetDriverInfo().Config.ClientNodeName != "" {
input.testCase.NodeName = driver.GetDriverInfo().Config.ClientNodeName
}
return resource, input
}
func (p *provisioningTestSuite) execTest(driver drivers.TestDriver, pattern testpatterns.TestPattern) {
Context(getTestNameStr(p, pattern), func() {
var (
resource provisioningTestResource
input provisioningTestInput
needsCleanup bool
)
BeforeEach(func() {
needsCleanup = false
// Skip unsupported tests to avoid unnecessary resource initialization
skipUnsupportedTest(p, driver, pattern)
needsCleanup = true
// Create test input
resource, input = createProvisioningTestInput(driver, pattern)
})
AfterEach(func() {
if needsCleanup {
resource.cleanupResource(driver, pattern)
}
})
// Ginkgo's "Global Shared Behaviors" require arguments for a shared function
// to be a single struct and to be passed as a pointer.
// Please see https://onsi.github.io/ginkgo/#global-shared-behaviors for details.
testProvisioning(&input)
})
}
type provisioningTestResource struct {
driver drivers.TestDriver
claimSize string
sc *storage.StorageClass
pvc *v1.PersistentVolumeClaim
}
var _ TestResource = &provisioningTestResource{}
func (p *provisioningTestResource) setupResource(driver drivers.TestDriver, pattern testpatterns.TestPattern) {
// Setup provisioningTest resource
switch pattern.VolType {
case testpatterns.DynamicPV:
if dDriver, ok := driver.(drivers.DynamicPVTestDriver); ok {
p.sc = dDriver.GetDynamicProvisionStorageClass("")
if p.sc == nil {
framework.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", driver.GetDriverInfo().Name)
}
p.driver = driver
p.claimSize = "2Gi"
p.pvc = getClaim(p.claimSize, driver.GetDriverInfo().Framework.Namespace.Name)
p.pvc.Spec.StorageClassName = &p.sc.Name
framework.Logf("In creating storage class object and pvc object for driver - sc: %v, pvc: %v", p.sc, p.pvc)
}
default:
framework.Failf("Dynamic Provision test doesn't support: %s", pattern.VolType)
}
}
func (p *provisioningTestResource) cleanupResource(driver drivers.TestDriver, pattern testpatterns.TestPattern) {
}
type provisioningTestInput struct {
testCase StorageClassTest
cs clientset.Interface
pvc *v1.PersistentVolumeClaim
sc *storage.StorageClass
dInfo *drivers.DriverInfo
}
func testProvisioning(input *provisioningTestInput) {
It("should provision storage", func() {
TestDynamicProvisioning(input.testCase, input.cs, input.pvc, input.sc)
})
It("should provision storage with mount options", func() {
if input.dInfo.SupportedMountOption == nil {
framework.Skipf("Driver %q does not define supported mount option - skipping", input.dInfo.Name)
}
input.sc.MountOptions = input.dInfo.SupportedMountOption.Union(input.dInfo.RequiredMountOption).List()
TestDynamicProvisioning(input.testCase, input.cs, input.pvc, input.sc)
})
It("should provision storage with non-default reclaim policy Retain", func() {
retain := v1.PersistentVolumeReclaimRetain
input.sc.ReclaimPolicy = &retain
pv := TestDynamicProvisioning(input.testCase, input.cs, input.pvc, input.sc)
By(fmt.Sprintf("waiting for the provisioned PV %q to enter phase %s", pv.Name, v1.VolumeReleased))
framework.ExpectNoError(framework.WaitForPersistentVolumePhase(v1.VolumeReleased, input.cs, pv.Name, 1*time.Second, 30*time.Second))
By(fmt.Sprintf("deleting the PV %q", pv.Name))
framework.ExpectNoError(framework.DeletePersistentVolume(input.cs, pv.Name), "Failed to delete PV ", pv.Name)
framework.ExpectNoError(framework.WaitForPersistentVolumeDeleted(input.cs, pv.Name, 1*time.Second, 30*time.Second))
})
It("should create and delete block persistent volumes [Feature:BlockVolume]", func() {
if !input.dInfo.IsBlockSupported {
framework.Skipf("Driver %q does not support BlockVolume - skipping", input.dInfo.Name)
}
block := v1.PersistentVolumeBlock
input.testCase.VolumeMode = &block
input.testCase.SkipWriteReadCheck = true
input.pvc.Spec.VolumeMode = &block
TestDynamicProvisioning(input.testCase, input.cs, input.pvc, input.sc)
})
}
// TestDynamicProvisioning tests dynamic provisioning with specified StorageClassTest and storageClass
func TestDynamicProvisioning(t StorageClassTest, client clientset.Interface, claim *v1.PersistentVolumeClaim, class *storage.StorageClass) *v1.PersistentVolume {
var err error
if class != nil {
By("creating a StorageClass " + class.Name)
class, err = client.StorageV1().StorageClasses().Create(class)
Expect(err).NotTo(HaveOccurred())
defer func() {
framework.Logf("deleting storage class %s", class.Name)
framework.ExpectNoError(client.StorageV1().StorageClasses().Delete(class.Name, nil))
}()
}
By("creating a claim")
claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(claim)
Expect(err).NotTo(HaveOccurred())
defer func() {
framework.Logf("deleting claim %q/%q", claim.Namespace, claim.Name)
// typically this claim has already been deleted
err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, nil)
if err != nil && !apierrs.IsNotFound(err) {
framework.Failf("Error deleting claim %q. Error: %v", claim.Name, err)
}
}()
err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout)
Expect(err).NotTo(HaveOccurred())
By("checking the claim")
// Get new copy of the claim
claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
// Get the bound PV
pv, err := client.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
// Check sizes
expectedCapacity := resource.MustParse(t.ExpectedSize)
pvCapacity := pv.Spec.Capacity[v1.ResourceName(v1.ResourceStorage)]
Expect(pvCapacity.Value()).To(Equal(expectedCapacity.Value()), "pvCapacity is not equal to expectedCapacity")
requestedCapacity := resource.MustParse(t.ClaimSize)
claimCapacity := claim.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
Expect(claimCapacity.Value()).To(Equal(requestedCapacity.Value()), "claimCapacity is not equal to requestedCapacity")
// Check PV properties
By("checking the PV")
expectedAccessModes := []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}
Expect(pv.Spec.AccessModes).To(Equal(expectedAccessModes))
Expect(pv.Spec.ClaimRef.Name).To(Equal(claim.ObjectMeta.Name))
Expect(pv.Spec.ClaimRef.Namespace).To(Equal(claim.ObjectMeta.Namespace))
if class == nil {
Expect(pv.Spec.PersistentVolumeReclaimPolicy).To(Equal(v1.PersistentVolumeReclaimDelete))
} else {
Expect(pv.Spec.PersistentVolumeReclaimPolicy).To(Equal(*class.ReclaimPolicy))
Expect(pv.Spec.MountOptions).To(Equal(class.MountOptions))
}
if t.VolumeMode != nil {
Expect(pv.Spec.VolumeMode).NotTo(BeNil())
Expect(*pv.Spec.VolumeMode).To(Equal(*t.VolumeMode))
}
// Run the checker
if t.PvCheck != nil {
err = t.PvCheck(pv)
Expect(err).NotTo(HaveOccurred())
}
if !t.SkipWriteReadCheck {
// We start two pods:
// - The first writes 'hello word' to the /mnt/test (= the volume).
// - The second one runs grep 'hello world' on /mnt/test.
// If both succeed, Kubernetes actually allocated something that is
// persistent across pods.
By("checking the created volume is writable and has the PV's mount options")
command := "echo 'hello world' > /mnt/test/data"
// We give the first pod the secondary responsibility of checking the volume has
// been mounted with the PV's mount options, if the PV was provisioned with any
for _, option := range pv.Spec.MountOptions {
// Get entry, get mount options at 6th word, replace brackets with commas
command += fmt.Sprintf(" && ( mount | grep 'on /mnt/test' | awk '{print $6}' | sed 's/^(/,/; s/)$/,/' | grep -q ,%s, )", option)
}
runInPodWithVolume(client, claim.Namespace, claim.Name, t.NodeName, command)
By("checking the created volume is readable and retains data")
runInPodWithVolume(client, claim.Namespace, claim.Name, t.NodeName, "grep 'hello world' /mnt/test/data")
}
By(fmt.Sprintf("deleting claim %q/%q", claim.Namespace, claim.Name))
framework.ExpectNoError(client.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, nil))
// Wait for the PV to get deleted if reclaim policy is Delete. (If it's
// Retain, there's no use waiting because the PV won't be auto-deleted and
// it's expected for the caller to do it.) Technically, the first few delete
// attempts may fail, as the volume is still attached to a node because
// kubelet is slowly cleaning up the previous pod, however it should succeed
// in a couple of minutes. Wait 20 minutes to recover from random cloud
// hiccups.
if pv.Spec.PersistentVolumeReclaimPolicy == v1.PersistentVolumeReclaimDelete {
By(fmt.Sprintf("deleting the claim's PV %q", pv.Name))
framework.ExpectNoError(framework.WaitForPersistentVolumeDeleted(client, pv.Name, 5*time.Second, 20*time.Minute))
}
return pv
}
// runInPodWithVolume runs a command in a pod with given claim mounted to /mnt directory.
func runInPodWithVolume(c clientset.Interface, ns, claimName, nodeName, command string) {
pod := &v1.Pod{
TypeMeta: metav1.TypeMeta{
Kind: "Pod",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
GenerateName: "pvc-volume-tester-",
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "volume-tester",
Image: imageutils.GetE2EImage(imageutils.BusyBox),
Command: []string{"/bin/sh"},
Args: []string{"-c", command},
VolumeMounts: []v1.VolumeMount{
{
Name: "my-volume",
MountPath: "/mnt/test",
},
},
},
},
RestartPolicy: v1.RestartPolicyNever,
Volumes: []v1.Volume{
{
Name: "my-volume",
VolumeSource: v1.VolumeSource{
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
ClaimName: claimName,
ReadOnly: false,
},
},
},
},
},
}
if len(nodeName) != 0 {
pod.Spec.NodeName = nodeName
}
pod, err := c.CoreV1().Pods(ns).Create(pod)
framework.ExpectNoError(err, "Failed to create pod: %v", err)
defer func() {
framework.DeletePodOrFail(c, ns, pod.Name)
}()
framework.ExpectNoError(framework.WaitForPodSuccessInNamespaceSlow(c, pod.Name, pod.Namespace))
}

2
test/e2e/storage/utils/utils.go
View File

@ -338,7 +338,7 @@ func StartExternalProvisioner(c clientset.Interface, ns string, externalPluginNa
Containers: []v1.Container{
{
Name: "nfs-provisioner",
Image: "quay.io/kubernetes_incubator/nfs-provisioner:v2.1.0-k8s1.11",
Image: "quay.io/kubernetes_incubator/nfs-provisioner:v2.2.0-k8s1.12",
SecurityContext: &v1.SecurityContext{
Capabilities: &v1.Capabilities{
Add: []v1.Capability{"DAC_READ_SEARCH"},

9
test/e2e/storage/volume_expand.go
View File

@ -30,6 +30,7 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/storage/testsuites"
"k8s.io/kubernetes/test/e2e/storage/utils"
)
@ -54,9 +55,9 @@ var _ = utils.SIGDescribe("Volume expand [Slow]", func() {
c = f.ClientSet
ns = f.Namespace.Name
framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.TestContext.NodeSchedulableTimeout))
test := storageClassTest{
name: "default",
claimSize: "2Gi",
test := testsuites.StorageClassTest{
Name: "default",
ClaimSize: "2Gi",
}
resizableSc, err = createResizableStorageClass(test, ns, "resizing", c)
Expect(err).NotTo(HaveOccurred(), "Error creating resizable storage class")
@ -133,7 +134,7 @@ var _ = utils.SIGDescribe("Volume expand [Slow]", func() {
})
})
func createResizableStorageClass(t storageClassTest, ns string, suffix string, c clientset.Interface) (*storage.StorageClass, error) {
func createResizableStorageClass(t testsuites.StorageClassTest, ns string, suffix string, c clientset.Interface) (*storage.StorageClass, error) {
stKlass := newStorageClass(t, ns, suffix)
allowExpansion := true
stKlass.AllowVolumeExpansion = &allowExpansion

7
test/e2e/storage/volume_metrics.go
View File

@ -31,6 +31,7 @@ import (
kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/framework/metrics"
"k8s.io/kubernetes/test/e2e/storage/testsuites"
"k8s.io/kubernetes/test/e2e/storage/utils"
)
@ -52,9 +53,9 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
defaultScName := getDefaultStorageClassName(c)
verifyDefaultStorageClass(c, defaultScName, true)
test := storageClassTest{
name: "default",
claimSize: "2Gi",
test := testsuites.StorageClassTest{
Name: "default",
ClaimSize: "2Gi",
}
pvc = newClaim(test, ns, "default")

594
test/e2e/storage/volume_provisioning.go
View File

@ -47,133 +47,15 @@ import (
volumeutil "k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/framework/providers/gce"
"k8s.io/kubernetes/test/e2e/storage/testsuites"
"k8s.io/kubernetes/test/e2e/storage/utils"
imageutils "k8s.io/kubernetes/test/utils/image"
)
type storageClassTest struct {
name string
cloudProviders []string
provisioner string
parameters map[string]string
delayBinding bool
claimSize string
expectedSize string
pvCheck func(volume *v1.PersistentVolume) error
nodeName string
skipWriteReadCheck bool
volumeMode *v1.PersistentVolumeMode
}
const (
// Plugin name of the external provisioner
externalPluginName = "example.com/nfs"
)
func testDynamicProvisioning(t storageClassTest, client clientset.Interface, claim *v1.PersistentVolumeClaim, class *storage.StorageClass) *v1.PersistentVolume {
var err error
if class != nil {
By("creating a StorageClass " + class.Name)
class, err = client.StorageV1().StorageClasses().Create(class)
Expect(err).NotTo(HaveOccurred())
defer func() {
framework.Logf("deleting storage class %s", class.Name)
framework.ExpectNoError(client.StorageV1().StorageClasses().Delete(class.Name, nil))
}()
}
By("creating a claim")
claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(claim)
Expect(err).NotTo(HaveOccurred())
defer func() {
framework.Logf("deleting claim %q/%q", claim.Namespace, claim.Name)
// typically this claim has already been deleted
err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, nil)
if err != nil && !apierrs.IsNotFound(err) {
framework.Failf("Error deleting claim %q. Error: %v", claim.Name, err)
}
}()
err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout)
Expect(err).NotTo(HaveOccurred())
By("checking the claim")
// Get new copy of the claim
claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
// Get the bound PV
pv, err := client.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
// Check sizes
expectedCapacity := resource.MustParse(t.expectedSize)
pvCapacity := pv.Spec.Capacity[v1.ResourceName(v1.ResourceStorage)]
Expect(pvCapacity.Value()).To(Equal(expectedCapacity.Value()), "pvCapacity is not equal to expectedCapacity")
requestedCapacity := resource.MustParse(t.claimSize)
claimCapacity := claim.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
Expect(claimCapacity.Value()).To(Equal(requestedCapacity.Value()), "claimCapacity is not equal to requestedCapacity")
// Check PV properties
By("checking the PV")
expectedAccessModes := []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}
Expect(pv.Spec.AccessModes).To(Equal(expectedAccessModes))
Expect(pv.Spec.ClaimRef.Name).To(Equal(claim.ObjectMeta.Name))
Expect(pv.Spec.ClaimRef.Namespace).To(Equal(claim.ObjectMeta.Namespace))
if class == nil {
Expect(pv.Spec.PersistentVolumeReclaimPolicy).To(Equal(v1.PersistentVolumeReclaimDelete))
} else {
Expect(pv.Spec.PersistentVolumeReclaimPolicy).To(Equal(*class.ReclaimPolicy))
Expect(pv.Spec.MountOptions).To(Equal(class.MountOptions))
}
if t.volumeMode != nil {
Expect(pv.Spec.VolumeMode).NotTo(BeNil())
Expect(*pv.Spec.VolumeMode).To(Equal(*t.volumeMode))
}
// Run the checker
if t.pvCheck != nil {
err = t.pvCheck(pv)
Expect(err).NotTo(HaveOccurred())
}
if !t.skipWriteReadCheck {
// We start two pods:
// - The first writes 'hello word' to the /mnt/test (= the volume).
// - The second one runs grep 'hello world' on /mnt/test.
// If both succeed, Kubernetes actually allocated something that is
// persistent across pods.
By("checking the created volume is writable and has the PV's mount options")
command := "echo 'hello world' > /mnt/test/data"
// We give the first pod the secondary responsibility of checking the volume has
// been mounted with the PV's mount options, if the PV was provisioned with any
for _, option := range pv.Spec.MountOptions {
// Get entry, get mount options at 6th word, replace brackets with commas
command += fmt.Sprintf(" && ( mount | grep 'on /mnt/test' | awk '{print $6}' | sed 's/^(/,/; s/)$/,/' | grep -q ,%s, )", option)
}
runInPodWithVolume(client, claim.Namespace, claim.Name, t.nodeName, command)
By("checking the created volume is readable and retains data")
runInPodWithVolume(client, claim.Namespace, claim.Name, t.nodeName, "grep 'hello world' /mnt/test/data")
}
By(fmt.Sprintf("deleting claim %q/%q", claim.Namespace, claim.Name))
framework.ExpectNoError(client.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, nil))
// Wait for the PV to get deleted if reclaim policy is Delete. (If it's
// Retain, there's no use waiting because the PV won't be auto-deleted and
// it's expected for the caller to do it.) Technically, the first few delete
// attempts may fail, as the volume is still attached to a node because
// kubelet is slowly cleaning up the previous pod, however it should succeed
// in a couple of minutes. Wait 20 minutes to recover from random cloud
// hiccups.
if pv.Spec.PersistentVolumeReclaimPolicy == v1.PersistentVolumeReclaimDelete {
By(fmt.Sprintf("deleting the claim's PV %q", pv.Name))
framework.ExpectNoError(framework.WaitForPersistentVolumeDeleted(client, pv.Name, 5*time.Second, 20*time.Minute))
}
return pv
}
func testBindingWaitForFirstConsumer(client clientset.Interface, claim *v1.PersistentVolumeClaim, class *storage.StorageClass) (*v1.PersistentVolume, *v1.Node) {
var err error
@ -367,172 +249,172 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
// This test checks that dynamic provisioning can provision a volume
// that can be used to persist data among pods.
tests := []storageClassTest{
tests := []testsuites.StorageClassTest{
// GCE/GKE
{
name: "SSD PD on GCE/GKE",
cloudProviders: []string{"gce", "gke"},
provisioner: "kubernetes.io/gce-pd",
parameters: map[string]string{
Name: "SSD PD on GCE/GKE",
CloudProviders: []string{"gce", "gke"},
Provisioner: "kubernetes.io/gce-pd",
Parameters: map[string]string{
"type": "pd-ssd",
"zone": cloudZone,
},
claimSize: "1.5Gi",
expectedSize: "2Gi",
pvCheck: func(volume *v1.PersistentVolume) error {
ClaimSize: "1.5Gi",
ExpectedSize: "2Gi",
PvCheck: func(volume *v1.PersistentVolume) error {
return checkGCEPD(volume, "pd-ssd")
},
},
{
name: "HDD PD on GCE/GKE",
cloudProviders: []string{"gce", "gke"},
provisioner: "kubernetes.io/gce-pd",
parameters: map[string]string{
Name: "HDD PD on GCE/GKE",
CloudProviders: []string{"gce", "gke"},
Provisioner: "kubernetes.io/gce-pd",
Parameters: map[string]string{
"type": "pd-standard",
},
claimSize: "1.5Gi",
expectedSize: "2Gi",
pvCheck: func(volume *v1.PersistentVolume) error {
ClaimSize: "1.5Gi",
ExpectedSize: "2Gi",
PvCheck: func(volume *v1.PersistentVolume) error {
return checkGCEPD(volume, "pd-standard")
},
},
// AWS
{
name: "gp2 EBS on AWS",
cloudProviders: []string{"aws"},
provisioner: "kubernetes.io/aws-ebs",
parameters: map[string]string{
Name: "gp2 EBS on AWS",
CloudProviders: []string{"aws"},
Provisioner: "kubernetes.io/aws-ebs",
Parameters: map[string]string{
"type": "gp2",
"zone": cloudZone,
},
claimSize: "1.5Gi",
expectedSize: "2Gi",
pvCheck: func(volume *v1.PersistentVolume) error {
ClaimSize: "1.5Gi",
ExpectedSize: "2Gi",
PvCheck: func(volume *v1.PersistentVolume) error {
return checkAWSEBS(volume, "gp2", false)
},
},
{
name: "io1 EBS on AWS",
cloudProviders: []string{"aws"},
provisioner: "kubernetes.io/aws-ebs",
parameters: map[string]string{
Name: "io1 EBS on AWS",
CloudProviders: []string{"aws"},
Provisioner: "kubernetes.io/aws-ebs",
Parameters: map[string]string{
"type": "io1",
"iopsPerGB": "50",
},
claimSize: "3.5Gi",
expectedSize: "4Gi", // 4 GiB is minimum for io1
pvCheck: func(volume *v1.PersistentVolume) error {
ClaimSize: "3.5Gi",
ExpectedSize: "4Gi", // 4 GiB is minimum for io1
PvCheck: func(volume *v1.PersistentVolume) error {
return checkAWSEBS(volume, "io1", false)
},
},
{
name: "sc1 EBS on AWS",
cloudProviders: []string{"aws"},
provisioner: "kubernetes.io/aws-ebs",
parameters: map[string]string{
Name: "sc1 EBS on AWS",
CloudProviders: []string{"aws"},
Provisioner: "kubernetes.io/aws-ebs",
Parameters: map[string]string{
"type": "sc1",
},
claimSize: "500Gi", // minimum for sc1
expectedSize: "500Gi",
pvCheck: func(volume *v1.PersistentVolume) error {
ClaimSize: "500Gi", // minimum for sc1
ExpectedSize: "500Gi",
PvCheck: func(volume *v1.PersistentVolume) error {
return checkAWSEBS(volume, "sc1", false)
},
},
{
name: "st1 EBS on AWS",
cloudProviders: []string{"aws"},
provisioner: "kubernetes.io/aws-ebs",
parameters: map[string]string{
Name: "st1 EBS on AWS",
CloudProviders: []string{"aws"},
Provisioner: "kubernetes.io/aws-ebs",
Parameters: map[string]string{
"type": "st1",
},
claimSize: "500Gi", // minimum for st1
expectedSize: "500Gi",
pvCheck: func(volume *v1.PersistentVolume) error {
ClaimSize: "500Gi", // minimum for st1
ExpectedSize: "500Gi",
PvCheck: func(volume *v1.PersistentVolume) error {
return checkAWSEBS(volume, "st1", false)
},
},
{
name: "encrypted EBS on AWS",
cloudProviders: []string{"aws"},
provisioner: "kubernetes.io/aws-ebs",
parameters: map[string]string{
Name: "encrypted EBS on AWS",
CloudProviders: []string{"aws"},
Provisioner: "kubernetes.io/aws-ebs",
Parameters: map[string]string{
"encrypted": "true",
},
claimSize: "1Gi",
expectedSize: "1Gi",
pvCheck: func(volume *v1.PersistentVolume) error {
ClaimSize: "1Gi",
ExpectedSize: "1Gi",
PvCheck: func(volume *v1.PersistentVolume) error {
return checkAWSEBS(volume, "gp2", true)
},
},
// OpenStack generic tests (works on all OpenStack deployments)
{
name: "generic Cinder volume on OpenStack",
cloudProviders: []string{"openstack"},
provisioner: "kubernetes.io/cinder",
parameters: map[string]string{},
claimSize: "1.5Gi",
expectedSize: "2Gi",
pvCheck: nil, // there is currently nothing to check on OpenStack
Name: "generic Cinder volume on OpenStack",
CloudProviders: []string{"openstack"},
Provisioner: "kubernetes.io/cinder",
Parameters: map[string]string{},
ClaimSize: "1.5Gi",
ExpectedSize: "2Gi",
PvCheck: nil, // there is currently nothing to check on OpenStack
},
{
name: "Cinder volume with empty volume type and zone on OpenStack",
cloudProviders: []string{"openstack"},
provisioner: "kubernetes.io/cinder",
parameters: map[string]string{
Name: "Cinder volume with empty volume type and zone on OpenStack",
CloudProviders: []string{"openstack"},
Provisioner: "kubernetes.io/cinder",
Parameters: map[string]string{
"type": "",
"availability": "",
},
claimSize: "1.5Gi",
expectedSize: "2Gi",
pvCheck: nil, // there is currently nothing to check on OpenStack
ClaimSize: "1.5Gi",
ExpectedSize: "2Gi",
PvCheck: nil, // there is currently nothing to check on OpenStack
},
// vSphere generic test
{
name: "generic vSphere volume",
cloudProviders: []string{"vsphere"},
provisioner: "kubernetes.io/vsphere-volume",
parameters: map[string]string{},
claimSize: "1.5Gi",
expectedSize: "1.5Gi",
pvCheck: nil,
Name: "generic vSphere volume",
CloudProviders: []string{"vsphere"},
Provisioner: "kubernetes.io/vsphere-volume",
Parameters: map[string]string{},
ClaimSize: "1.5Gi",
ExpectedSize: "1.5Gi",
PvCheck: nil,
},
// Azure
{
name: "Azure disk volume with empty sku and location",
cloudProviders: []string{"azure"},
provisioner: "kubernetes.io/azure-disk",
parameters: map[string]string{},
claimSize: "1Gi",
expectedSize: "1Gi",
pvCheck: nil,
Name: "Azure disk volume with empty sku and location",
CloudProviders: []string{"azure"},
Provisioner: "kubernetes.io/azure-disk",
Parameters: map[string]string{},
ClaimSize: "1Gi",
ExpectedSize: "1Gi",
PvCheck: nil,
},
}
var betaTest *storageClassTest
var betaTest *testsuites.StorageClassTest
for i, t := range tests {
// Beware of clojure, use local variables instead of those from
// outer scope
test := t
if !framework.ProviderIs(test.cloudProviders...) {
framework.Logf("Skipping %q: cloud providers is not %v", test.name, test.cloudProviders)
if !framework.ProviderIs(test.CloudProviders...) {
framework.Logf("Skipping %q: cloud providers is not %v", test.Name, test.CloudProviders)
continue
}
// Remember the last supported test for subsequent test of beta API
betaTest = &test
By("Testing " + test.name)
By("Testing " + test.Name)
suffix := fmt.Sprintf("%d", i)
class := newStorageClass(test, ns, suffix)
claim := newClaim(test, ns, suffix)
claim.Spec.StorageClassName = &class.Name
testDynamicProvisioning(test, c, claim, class)
testsuites.TestDynamicProvisioning(test, c, claim, class)
}
// Run the last test with storage.k8s.io/v1beta1 on pvc
if betaTest != nil {
By("Testing " + betaTest.name + " with beta volume provisioning")
By("Testing " + betaTest.Name + " with beta volume provisioning")
class := newBetaStorageClass(*betaTest, "beta")
// we need to create the class manually, testDynamicProvisioning does not accept beta class
class, err := c.StorageV1beta1().StorageClasses().Create(class)
@ -541,67 +423,10 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
claim := newClaim(*betaTest, ns, "beta")
claim.Spec.StorageClassName = &(class.Name)
testDynamicProvisioning(*betaTest, c, claim, nil)
testsuites.TestDynamicProvisioning(*betaTest, c, claim, nil)
}
})
It("should provision storage with non-default reclaim policy Retain", func() {
framework.SkipUnlessProviderIs("gce", "gke")
test := storageClassTest{
name: "HDD PD on GCE/GKE",
cloudProviders: []string{"gce", "gke"},
provisioner: "kubernetes.io/gce-pd",
parameters: map[string]string{
"type": "pd-standard",
},
claimSize: "1Gi",
expectedSize: "1Gi",
pvCheck: func(volume *v1.PersistentVolume) error {
return checkGCEPD(volume, "pd-standard")
},
}
class := newStorageClass(test, ns, "reclaimpolicy")
retain := v1.PersistentVolumeReclaimRetain
class.ReclaimPolicy = &retain
claim := newClaim(test, ns, "reclaimpolicy")
claim.Spec.StorageClassName = &class.Name
pv := testDynamicProvisioning(test, c, claim, class)
By(fmt.Sprintf("waiting for the provisioned PV %q to enter phase %s", pv.Name, v1.VolumeReleased))
framework.ExpectNoError(framework.WaitForPersistentVolumePhase(v1.VolumeReleased, c, pv.Name, 1*time.Second, 30*time.Second))
By(fmt.Sprintf("deleting the storage asset backing the PV %q", pv.Name))
framework.ExpectNoError(framework.DeletePDWithRetry(pv.Spec.GCEPersistentDisk.PDName))
By(fmt.Sprintf("deleting the PV %q", pv.Name))
framework.ExpectNoError(framework.DeletePersistentVolume(c, pv.Name), "Failed to delete PV ", pv.Name)
framework.ExpectNoError(framework.WaitForPersistentVolumeDeleted(c, pv.Name, 1*time.Second, 30*time.Second))
})
It("should provision storage with mount options", func() {
framework.SkipUnlessProviderIs("gce", "gke")
test := storageClassTest{
name: "HDD PD on GCE/GKE",
cloudProviders: []string{"gce", "gke"},
provisioner: "kubernetes.io/gce-pd",
parameters: map[string]string{
"type": "pd-standard",
},
claimSize: "1Gi",
expectedSize: "1Gi",
pvCheck: func(volume *v1.PersistentVolume) error {
return checkGCEPD(volume, "pd-standard")
},
}
class := newStorageClass(test, ns, "mountoptions")
class.MountOptions = []string{"debug", "nouid32"}
claim := newClaim(test, ns, "mountoptions")
claim.Spec.StorageClassName = &class.Name
testDynamicProvisioning(test, c, claim, class)
})
It("should not provision a volume in an unmanaged GCE zone.", func() {
framework.SkipUnlessProviderIs("gce", "gke")
var suffix string = "unmananged"
@ -634,11 +459,11 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
}
By("Creating a StorageClass for the unmanaged zone")
test := storageClassTest{
name: "unmanaged_zone",
provisioner: "kubernetes.io/gce-pd",
parameters: map[string]string{"zone": unmanagedZone},
claimSize: "1Gi",
test := testsuites.StorageClassTest{
Name: "unmanaged_zone",
Provisioner: "kubernetes.io/gce-pd",
Parameters: map[string]string{"zone": unmanagedZone},
ClaimSize: "1Gi",
}
sc := newStorageClass(test, ns, suffix)
sc, err = c.StorageV1().StorageClasses().Create(sc)
@ -671,10 +496,10 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
const raceAttempts int = 100
var residualPVs []*v1.PersistentVolume
By(fmt.Sprintf("Creating and deleting PersistentVolumeClaims %d times", raceAttempts))
test := storageClassTest{
name: "deletion race",
provisioner: "", // Use a native one based on current cloud provider
claimSize: "1Gi",
test := testsuites.StorageClassTest{
Name: "deletion race",
Provisioner: "", // Use a native one based on current cloud provider
ClaimSize: "1Gi",
}
class := newStorageClass(test, ns, "race")
@ -822,18 +647,18 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
defer framework.DeletePodOrFail(c, ns, pod.Name)
By("creating a StorageClass")
test := storageClassTest{
name: "external provisioner test",
provisioner: externalPluginName,
claimSize: "1500Mi",
expectedSize: "1500Mi",
test := testsuites.StorageClassTest{
Name: "external provisioner test",
Provisioner: externalPluginName,
ClaimSize: "1500Mi",
ExpectedSize: "1500Mi",
}
class := newStorageClass(test, ns, "external")
claim := newClaim(test, ns, "external")
claim.Spec.StorageClassName = &(class.Name)
By("creating a claim with a external provisioning annotation")
testDynamicProvisioning(test, c, claim, class)
testsuites.TestDynamicProvisioning(test, c, claim, class)
})
})
@ -842,23 +667,23 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere", "azure")
By("creating a claim with no annotation")
test := storageClassTest{
name: "default",
claimSize: "2Gi",
expectedSize: "2Gi",
test := testsuites.StorageClassTest{
Name: "default",
ClaimSize: "2Gi",
ExpectedSize: "2Gi",
}
claim := newClaim(test, ns, "default")
testDynamicProvisioning(test, c, claim, nil)
testsuites.TestDynamicProvisioning(test, c, claim, nil)
})
// Modifying the default storage class can be disruptive to other tests that depend on it
It("should be disabled by changing the default annotation [Serial] [Disruptive]", func() {
framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere", "azure")
scName := getDefaultStorageClassName(c)
test := storageClassTest{
name: "default",
claimSize: "2Gi",
test := testsuites.StorageClassTest{
Name: "default",
ClaimSize: "2Gi",
}
By("setting the is-default StorageClass annotation to false")
@ -887,9 +712,9 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
It("should be disabled by removing the default annotation [Serial] [Disruptive]", func() {
framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere", "azure")
scName := getDefaultStorageClassName(c)
test := storageClassTest{
name: "default",
claimSize: "2Gi",
test := testsuites.StorageClassTest{
Name: "default",
ClaimSize: "2Gi",
}
By("removing the is-default StorageClass annotation")
@ -921,13 +746,13 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
pod := startGlusterDpServerPod(c, ns)
serverUrl := "https://" + pod.Status.PodIP + ":8081"
By("creating a StorageClass")
test := storageClassTest{
name: "Gluster Dynamic provisioner test",
provisioner: "kubernetes.io/glusterfs",
claimSize: "2Gi",
expectedSize: "2Gi",
parameters: map[string]string{"resturl": serverUrl},
skipWriteReadCheck: true,
test := testsuites.StorageClassTest{
Name: "Gluster Dynamic provisioner test",
Provisioner: "kubernetes.io/glusterfs",
ClaimSize: "2Gi",
ExpectedSize: "2Gi",
Parameters: map[string]string{"resturl": serverUrl},
SkipWriteReadCheck: true,
}
suffix := fmt.Sprintf("glusterdptest")
class := newStorageClass(test, ns, suffix)
@ -935,38 +760,18 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
By("creating a claim object with a suffix for gluster dynamic provisioner")
claim := newClaim(test, ns, suffix)
testDynamicProvisioning(test, c, claim, class)
testsuites.TestDynamicProvisioning(test, c, claim, class)
})
})
Describe("Block volume provisioning [Feature:BlockVolume]", func() {
It("should create and delete block persistent volumes", func() {
// TODO: add openstack once Cinder volume plugin supports block volumes
framework.SkipUnlessProviderIs("gce", "aws", "gke", "vsphere", "azure")
By("creating a claim with default class")
block := v1.PersistentVolumeBlock
test := storageClassTest{
name: "default",
claimSize: "2Gi",
expectedSize: "2Gi",
volumeMode: &block,
skipWriteReadCheck: true,
}
claim := newClaim(test, ns, "default")
claim.Spec.VolumeMode = &block
testDynamicProvisioning(test, c, claim, nil)
})
})
Describe("Invalid AWS KMS key", func() {
It("should report an error and create no PV", func() {
framework.SkipUnlessProviderIs("aws")
test := storageClassTest{
name: "AWS EBS with invalid KMS key",
provisioner: "kubernetes.io/aws-ebs",
claimSize: "2Gi",
parameters: map[string]string{"kmsKeyId": "arn:aws:kms:us-east-1:123456789012:key/55555555-5555-5555-5555-555555555555"},
test := testsuites.StorageClassTest{
Name: "AWS EBS with invalid KMS key",
Provisioner: "kubernetes.io/aws-ebs",
ClaimSize: "2Gi",
Parameters: map[string]string{"kmsKeyId": "arn:aws:kms:us-east-1:123456789012:key/55555555-5555-5555-5555-555555555555"},
}
By("creating a StorageClass")
@ -1008,25 +813,25 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
})
Describe("DynamicProvisioner delayed binding [Slow]", func() {
It("should create a persistent volume in the same zone as node after a pod mounting the claim is started", func() {
tests := []storageClassTest{
tests := []testsuites.StorageClassTest{
{
name: "Delayed binding EBS storage class test",
cloudProviders: []string{"aws"},
provisioner: "kubernetes.io/aws-ebs",
claimSize: "2Gi",
delayBinding: true,
Name: "Delayed binding EBS storage class test",
CloudProviders: []string{"aws"},
Provisioner: "kubernetes.io/aws-ebs",
ClaimSize: "2Gi",
DelayBinding: true,
},
{
name: "Delayed binding GCE PD storage class test",
cloudProviders: []string{"gce", "gke"},
provisioner: "kubernetes.io/gce-pd",
claimSize: "2Gi",
delayBinding: true,
Name: "Delayed binding GCE PD storage class test",
CloudProviders: []string{"gce", "gke"},
Provisioner: "kubernetes.io/gce-pd",
ClaimSize: "2Gi",
DelayBinding: true,
},
}
for _, test := range tests {
if !framework.ProviderIs(test.cloudProviders...) {
framework.Logf("Skipping %q: cloud providers is not %v", test.name, test.cloudProviders)
if !framework.ProviderIs(test.CloudProviders...) {
framework.Logf("Skipping %q: cloud providers is not %v", test.Name, test.CloudProviders)
continue
}
By("creating a claim with class with waitForFirstConsumer")
@ -1048,25 +853,25 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
})
Describe("DynamicProvisioner allowedTopologies", func() {
It("should create persistent volume in the zone specified in allowedTopologies of storageclass", func() {
tests := []storageClassTest{
tests := []testsuites.StorageClassTest{
{
name: "AllowedTopologies EBS storage class test",
cloudProviders: []string{"aws"},
provisioner: "kubernetes.io/aws-ebs",
claimSize: "2Gi",
expectedSize: "2Gi",
Name: "AllowedTopologies EBS storage class test",
CloudProviders: []string{"aws"},
Provisioner: "kubernetes.io/aws-ebs",
ClaimSize: "2Gi",
ExpectedSize: "2Gi",
},
{
name: "AllowedTopologies GCE PD storage class test",
cloudProviders: []string{"gce", "gke"},
provisioner: "kubernetes.io/gce-pd",
claimSize: "2Gi",
expectedSize: "2Gi",
Name: "AllowedTopologies GCE PD storage class test",
CloudProviders: []string{"gce", "gke"},
Provisioner: "kubernetes.io/gce-pd",
ClaimSize: "2Gi",
ExpectedSize: "2Gi",
},
}
for _, test := range tests {
if !framework.ProviderIs(test.cloudProviders...) {
framework.Logf("Skipping %q: cloud providers is not %v", test.name, test.cloudProviders)
if !framework.ProviderIs(test.CloudProviders...) {
framework.Logf("Skipping %q: cloud providers is not %v", test.Name, test.CloudProviders)
continue
}
By("creating a claim with class with allowedTopologies set")
@ -1076,32 +881,32 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
addSingleZoneAllowedTopologyToStorageClass(c, class, zone)
claim := newClaim(test, ns, suffix)
claim.Spec.StorageClassName = &class.Name
pv := testDynamicProvisioning(test, c, claim, class)
pv := testsuites.TestDynamicProvisioning(test, c, claim, class)
checkZoneFromLabelAndAffinity(pv, zone, true)
}
})
})
Describe("DynamicProvisioner delayed binding with allowedTopologies [Slow]", func() {
It("should create persistent volume in the same zone as specified in allowedTopologies after a pod mounting the claim is started", func() {
tests := []storageClassTest{
tests := []testsuites.StorageClassTest{
{
name: "AllowedTopologies and delayed binding EBS storage class test",
cloudProviders: []string{"aws"},
provisioner: "kubernetes.io/aws-ebs",
claimSize: "2Gi",
delayBinding: true,
Name: "AllowedTopologies and delayed binding EBS storage class test",
CloudProviders: []string{"aws"},
Provisioner: "kubernetes.io/aws-ebs",
ClaimSize: "2Gi",
DelayBinding: true,
},
{
name: "AllowedTopologies and delayed binding GCE PD storage class test",
cloudProviders: []string{"gce", "gke"},
provisioner: "kubernetes.io/gce-pd",
claimSize: "2Gi",
delayBinding: true,
Name: "AllowedTopologies and delayed binding GCE PD storage class test",
CloudProviders: []string{"gce", "gke"},
Provisioner: "kubernetes.io/gce-pd",
ClaimSize: "2Gi",
DelayBinding: true,
},
}
for _, test := range tests {
if !framework.ProviderIs(test.cloudProviders...) {
framework.Logf("Skipping %q: cloud providers is not %v", test.name, test.cloudProviders)
if !framework.ProviderIs(test.CloudProviders...) {
framework.Logf("Skipping %q: cloud providers is not %v", test.Name, test.CloudProviders)
continue
}
By("creating a claim with class with WaitForFirstConsumer and allowedTopologies")
@ -1202,59 +1007,8 @@ func getClaim(claimSize string, ns string) *v1.PersistentVolumeClaim {
return &claim
}
func newClaim(t storageClassTest, ns, suffix string) *v1.PersistentVolumeClaim {
return getClaim(t.claimSize, ns)
}
// runInPodWithVolume runs a command in a pod with given claim mounted to /mnt directory.
func runInPodWithVolume(c clientset.Interface, ns, claimName, nodeName, command string) {
pod := &v1.Pod{
TypeMeta: metav1.TypeMeta{
Kind: "Pod",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
GenerateName: "pvc-volume-tester-",
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "volume-tester",
Image: imageutils.GetE2EImage(imageutils.BusyBox),
Command: []string{"/bin/sh"},
Args: []string{"-c", command},
VolumeMounts: []v1.VolumeMount{
{
Name: "my-volume",
MountPath: "/mnt/test",
},
},
},
},
RestartPolicy: v1.RestartPolicyNever,
Volumes: []v1.Volume{
{
Name: "my-volume",
VolumeSource: v1.VolumeSource{
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
ClaimName: claimName,
ReadOnly: false,
},
},
},
},
},
}
if len(nodeName) != 0 {
pod.Spec.NodeName = nodeName
}
pod, err := c.CoreV1().Pods(ns).Create(pod)
framework.ExpectNoError(err, "Failed to create pod: %v", err)
defer func() {
framework.DeletePodOrFail(c, ns, pod.Name)
}()
framework.ExpectNoError(framework.WaitForPodSuccessInNamespaceSlow(c, pod.Name, pod.Namespace))
func newClaim(t testsuites.StorageClassTest, ns, suffix string) *v1.PersistentVolumeClaim {
return getClaim(t.ClaimSize, ns)
}
func getDefaultPluginName() string {
@ -1285,8 +1039,8 @@ func addSingleZoneAllowedTopologyToStorageClass(c clientset.Interface, sc *stora
sc.AllowedTopologies = append(sc.AllowedTopologies, term)
}
func newStorageClass(t storageClassTest, ns string, suffix string) *storage.StorageClass {
pluginName := t.provisioner
func newStorageClass(t testsuites.StorageClassTest, ns string, suffix string) *storage.StorageClass {
pluginName := t.Provisioner
if pluginName == "" {
pluginName = getDefaultPluginName()
}
@ -1294,10 +1048,10 @@ func newStorageClass(t storageClassTest, ns string, suffix string) *storage.Stor
suffix = "sc"
}
bindingMode := storage.VolumeBindingImmediate
if t.delayBinding {
if t.DelayBinding {
bindingMode = storage.VolumeBindingWaitForFirstConsumer
}
return getStorageClass(pluginName, t.parameters, &bindingMode, ns, suffix)
return getStorageClass(pluginName, t.Parameters, &bindingMode, ns, suffix)
}
func getStorageClass(
@ -1326,8 +1080,8 @@ func getStorageClass(
}
// TODO: remove when storage.k8s.io/v1beta1 is removed.
func newBetaStorageClass(t storageClassTest, suffix string) *storagebeta.StorageClass {
pluginName := t.provisioner
func newBetaStorageClass(t testsuites.StorageClassTest, suffix string) *storagebeta.StorageClass {
pluginName := t.Provisioner
if pluginName == "" {
pluginName = getDefaultPluginName()
@ -1344,7 +1098,7 @@ func newBetaStorageClass(t storageClassTest, suffix string) *storagebeta.Storage
GenerateName: suffix + "-",
},
Provisioner: pluginName,
Parameters: t.parameters,
Parameters: t.Parameters,
}
}