github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #69036 from wackxu/snapshottest 

add e2e test for snapshot
pull/564/head
Kubernetes Prow Robot 2019-02-08 13:14:04 -08:00 committed by GitHub
parent 52b3956df4 7280fcef5c
commit 8f7ccf8d4c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 698 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
test/e2e/framework/util.go
View File

@ -186,6 +186,9 @@ const (
// restart before test is considered failed.
RestartPodReadyAgainTimeout = 5 * time.Minute
// How long for snapshot to create snapshotContent
SnapshotCreateTimeout = 5 * time.Minute
// Number of objects that gc can delete in a second.
// GC issues 2 requestes for single delete.
gcThroughput = 10

1
test/e2e/storage/csi_volumes.go
View File

@ -62,6 +62,7 @@ var csiTestSuites = []func() testsuites.TestSuite{
testsuites.InitVolumeModeTestSuite,
testsuites.InitSubPathTestSuite,
testsuites.InitProvisioningTestSuite,
testsuites.InitSnapshottableTestSuite,
}
func csiTunePattern(patterns []testpatterns.TestPattern) []testpatterns.TestPattern {

1
test/e2e/storage/drivers/BUILD
View File

@ -15,6 +15,7 @@ go_library(
"//staging/src/k8s.io/api/storage/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",

25
test/e2e/storage/drivers/csi.go
View File

@ -41,6 +41,7 @@ import (
. "github.com/onsi/ginkgo"
storagev1 "k8s.io/api/storage/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/storage/testpatterns"
@ -60,7 +61,7 @@ type hostpathCSIDriver struct {
manifests []string
}
func initHostPathCSIDriver(name string, config testsuites.TestConfig, manifests ...string) testsuites.TestDriver {
func initHostPathCSIDriver(name string, config testsuites.TestConfig, capabilities map[testsuites.Capability]bool, manifests ...string) testsuites.TestDriver {
return &hostpathCSIDriver{
driverInfo: testsuites.DriverInfo{
Name: name,
@ -69,11 +70,8 @@ func initHostPathCSIDriver(name string, config testsuites.TestConfig, manifests
SupportedFsType: sets.NewString(
"", // Default fsType
),
Capabilities: map[testsuites.Capability]bool{
testsuites.CapPersistence: true,
},
Config: config,
Capabilities: capabilities,
Config: config,
},
manifests: manifests,
}
@ -81,15 +79,19 @@ func initHostPathCSIDriver(name string, config testsuites.TestConfig, manifests
var _ testsuites.TestDriver = &hostpathCSIDriver{}
var _ testsuites.DynamicPVTestDriver = &hostpathCSIDriver{}
var _ testsuites.SnapshottableTestDriver = &hostpathCSIDriver{}
// InitHostPathCSIDriver returns hostpathCSIDriver that implements TestDriver interface
func InitHostPathCSIDriver(config testsuites.TestConfig) testsuites.TestDriver {
return initHostPathCSIDriver("csi-hostpath", config,
map[testsuites.Capability]bool{testsuites.CapPersistence: true, testsuites.CapDataSource: true},
"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
"test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml",
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml",
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml",
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml",
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpathplugin.yaml",
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml",
)
@ -111,6 +113,15 @@ func (h *hostpathCSIDriver) GetDynamicProvisionStorageClass(fsType string) *stor
return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix)
}
func (h *hostpathCSIDriver) GetSnapshotClass() *unstructured.Unstructured {
snapshotter := testsuites.GetUniqueDriverName(h)
parameters := map[string]string{}
ns := h.driverInfo.Config.Framework.Namespace.Name
suffix := fmt.Sprintf("%s-vsc", snapshotter)
return testsuites.GetSnapshotClass(snapshotter, parameters, ns, suffix)
}
func (h *hostpathCSIDriver) GetClaimSize() string {
return "5Gi"
}
@ -133,6 +144,7 @@ func (h *hostpathCSIDriver) CreateDriver() {
DriverContainerName: "hostpath",
DriverContainerArguments: []string{"--drivername=csi-hostpath-" + f.UniqueName},
ProvisionerContainerName: "csi-provisioner",
SnapshotterContainerName: "csi-snapshotter",
NodeName: nodeName,
}
cleanup, err := h.driverInfo.Config.Framework.CreateFromManifests(func(item interface{}) error {
@ -247,6 +259,7 @@ func (m *mockCSIDriver) CleanupDriver() {
// InitHostPathV0CSIDriver returns a variant of hostpathCSIDriver with different manifests.
func InitHostPathV0CSIDriver(config testsuites.TestConfig) testsuites.TestDriver {
return initHostPathCSIDriver("csi-hostpath-v0", config,
map[testsuites.Capability]bool{testsuites.CapPersistence: true},
"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",

29
test/e2e/storage/testpatterns/testpattern.go
View File

@ -17,7 +17,7 @@ limitations under the License.
package testpatterns
import (
v1 "k8s.io/api/core/v1"
"k8s.io/api/core/v1"
"k8s.io/kubernetes/test/e2e/framework"
)
@ -45,13 +45,22 @@ var (
DynamicPV TestVolType = "DynamicPV"
)
// TestSnapshotType represents a snapshot type to be tested in a TestSuite
type TestSnapshotType string
var (
// DynamicCreatedSnapshot represents a snapshot type for dynamic created snapshot
DynamicCreatedSnapshot TestSnapshotType = "DynamicSnapshot"
)
// TestPattern represents a combination of parameters to be tested in a TestSuite
type TestPattern struct {
Name string // Name of TestPattern
FeatureTag string // featureTag for the TestSuite
VolType TestVolType // Volume type of the volume
FsType string // Fstype of the volume
VolMode v1.PersistentVolumeMode // PersistentVolumeMode of the volume
Name string // Name of TestPattern
FeatureTag string // featureTag for the TestSuite
VolType TestVolType // Volume type of the volume
FsType string // Fstype of the volume
VolMode v1.PersistentVolumeMode // PersistentVolumeMode of the volume
SnapshotType TestSnapshotType // Snapshot type of the snapshot
}
var (
@ -165,4 +174,12 @@ var (
VolType: DynamicPV,
VolMode: v1.PersistentVolumeBlock,
}
// Definitions for snapshot case
// DynamicSnapshot is TestPattern for "Dynamic snapshot"
DynamicSnapshot = TestPattern{
Name: "Dynamic Snapshot",
SnapshotType: DynamicCreatedSnapshot,
}
)

4
test/e2e/storage/testsuites/BUILD
View File

@ -6,6 +6,7 @@ go_library(
"base.go",
"driveroperations.go",
"provisioning.go",
"snapshottable.go",
"subpath.go",
"testdriver.go",
"volume_io.go",
@ -20,11 +21,14 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/client-go/dynamic:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//test/e2e/framework:go_default_library",
"//test/e2e/storage/testpatterns:go_default_library",

58
test/e2e/storage/testsuites/base.go
View File

@ -22,11 +22,13 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
@ -79,15 +81,31 @@ func RunTestSuite(f *framework.Framework, driver TestDriver, tsInits []func() Te
// skipUnsupportedTest will skip tests if the combination of driver, testsuite, and testpattern
// is not suitable to be tested.
// Whether it needs to be skipped is checked by following steps:
// 1. Check if Whether volType is supported by driver from its interface
// 2. Check if fsType is supported
// 3. Check with driver specific logic
// 4. Check with testSuite specific logic
// 1. Check if Whether SnapshotType is supported by driver from its interface
// 2. Check if Whether volType is supported by driver from its interface
// 3. Check if fsType is supported
// 4. Check with driver specific logic
// 5. Check with testSuite specific logic
func skipUnsupportedTest(suite TestSuite, driver TestDriver, pattern testpatterns.TestPattern) {
dInfo := driver.GetDriverInfo()
// 1. Check if Whether volType is supported by driver from its interface
var isSupported bool
// 1. Check if Whether SnapshotType is supported by driver from its interface
// if isSupported, so it must be a snapshot test case, we just return.
if len(pattern.SnapshotType) > 0 {
switch pattern.SnapshotType {
case testpatterns.DynamicCreatedSnapshot:
_, isSupported = driver.(SnapshottableTestDriver)
default:
isSupported = false
}
if !isSupported {
framework.Skipf("Driver %s doesn't support snapshot type %v -- skipping", dInfo.Name, pattern.SnapshotType)
}
return
}
// 2. Check if Whether volType is supported by driver from its interface
switch pattern.VolType {
case testpatterns.InlineVolume:
_, isSupported = driver.(InlineVolumeTestDriver)
@ -103,7 +121,7 @@ func skipUnsupportedTest(suite TestSuite, driver TestDriver, pattern testpattern
framework.Skipf("Driver %s doesn't support %v -- skipping", dInfo.Name, pattern.VolType)
}
// 2. Check if fsType is supported
// 3. Check if fsType is supported
if !dInfo.SupportedFsType.Has(pattern.FsType) {
framework.Skipf("Driver %s doesn't support %v -- skipping", dInfo.Name, pattern.FsType)
}
@ -111,10 +129,10 @@ func skipUnsupportedTest(suite TestSuite, driver TestDriver, pattern testpattern
framework.Skipf("Distro doesn't support xfs -- skipping")
}
// 3. Check with driver specific logic
// 4. Check with driver specific logic
driver.SkipUnsupportedTest(pattern)
// 4. Check with testSuite specific logic
// 5. Check with testSuite specific logic
suite.skipUnsupportedTest(pattern, driver)
}
@ -349,3 +367,25 @@ func convertTestConfig(in *TestConfig) framework.VolumeTestConfig {
NodeSelector: in.ClientNodeSelector,
}
}
func getSnapshot(claimName string, ns, snapshotClassName string) *unstructured.Unstructured {
snapshot := &unstructured.Unstructured{
Object: map[string]interface{}{
"kind": "VolumeSnapshot",
"apiVersion": snapshotAPIVersion,
"metadata": map[string]interface{}{
"generateName": "snapshot-",
"namespace": ns,
},
"spec": map[string]interface{}{
"snapshotClassName": snapshotClassName,
"source": map[string]interface{}{
"name": claimName,
"kind": "PersistentVolumeClaim",
},
},
},
}
return snapshot
}

25
test/e2e/storage/testsuites/driveroperations.go
View File

@ -21,6 +21,7 @@ import (
storagev1 "k8s.io/api/storage/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/storage/testpatterns"
)
@ -95,6 +96,30 @@ func GetStorageClass(
}
}
// GetSnapshotClass constructs a new SnapshotClass instance
// with a unique name that is based on namespace + suffix.
func GetSnapshotClass(
snapshotter string,
parameters map[string]string,
ns string,
suffix string,
) *unstructured.Unstructured {
snapshotClass := &unstructured.Unstructured{
Object: map[string]interface{}{
"kind": "VolumeSnapshotClass",
"apiVersion": snapshotAPIVersion,
"metadata": map[string]interface{}{
// Name must be unique, so let's base it on namespace name
"name": ns + "-" + suffix,
},
"snapshotter": snapshotter,
"parameters": parameters,
},
}
return snapshotClass
}
// GetUniqueDriverName returns unique driver name that can be used parallelly in tests
func GetUniqueDriverName(driver TestDriver) string {
return fmt.Sprintf("%s-%s", driver.GetDriverInfo().Name, driver.GetDriverInfo().Config.Framework.UniqueName)

115
test/e2e/storage/testsuites/provisioning.go
View File

@ -23,13 +23,14 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
apierrs "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/api/core/v1"
storage "k8s.io/api/storage/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/dynamic"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/storage/testpatterns"
@ -90,8 +91,10 @@ func createProvisioningTestInput(driver TestDriver, pattern testpatterns.TestPat
ExpectedSize: resource.claimSize,
},
cs: driver.GetDriverInfo().Config.Framework.ClientSet,
dc: driver.GetDriverInfo().Config.Framework.DynamicClient,
pvc: resource.pvc,
sc: resource.sc,
vsc: resource.vsc,
dInfo: driver.GetDriverInfo(),
}
@ -139,6 +142,8 @@ type provisioningTestResource struct {
claimSize string
sc *storage.StorageClass
pvc *v1.PersistentVolumeClaim
// follow parameter is used to test provision volume from snapshot
vsc *unstructured.Unstructured
}
var _ TestResource = &provisioningTestResource{}
@ -157,6 +162,9 @@ func (p *provisioningTestResource) setupResource(driver TestDriver, pattern test
p.pvc = getClaim(p.claimSize, driver.GetDriverInfo().Config.Framework.Namespace.Name)
p.pvc.Spec.StorageClassName = &p.sc.Name
framework.Logf("In creating storage class object and pvc object for driver - sc: %v, pvc: %v", p.sc, p.pvc)
if sDriver, ok := driver.(SnapshottableTestDriver); ok {
p.vsc = sDriver.GetSnapshotClass()
}
}
default:
framework.Failf("Dynamic Provision test doesn't support: %s", pattern.VolType)
@ -169,8 +177,10 @@ func (p *provisioningTestResource) cleanupResource(driver TestDriver, pattern te
type provisioningTestInput struct {
testCase StorageClassTest
cs clientset.Interface
dc dynamic.Interface
pvc *v1.PersistentVolumeClaim
sc *storage.StorageClass
vsc *unstructured.Unstructured
dInfo *DriverInfo
}
@ -198,6 +208,19 @@ func testProvisioning(input *provisioningTestInput) {
input.pvc.Spec.VolumeMode = &block
TestDynamicProvisioning(input.testCase, input.cs, input.pvc, input.sc)
})
It("should provision storage with snapshot data source [Feature:VolumeSnapshotDataSource]", func() {
if !input.dInfo.Capabilities[CapDataSource] {
framework.Skipf("Driver %q does not support populate data from snapshot - skipping", input.dInfo.Name)
}
input.testCase.SkipWriteReadCheck = true
dataSource, cleanupFunc := prepareDataSourceForProvisioning(input.testCase, input.cs, input.dc, input.pvc, input.sc, input.vsc)
defer cleanupFunc()
input.pvc.Spec.DataSource = dataSource
TestDynamicProvisioning(input.testCase, input.cs, input.pvc, input.sc)
})
}
// TestDynamicProvisioning tests dynamic provisioning with specified StorageClassTest and storageClass
@ -205,7 +228,11 @@ func TestDynamicProvisioning(t StorageClassTest, client clientset.Interface, cla
var err error
if class != nil {
By("creating a StorageClass " + class.Name)
class, err = client.StorageV1().StorageClasses().Create(class)
_, err = client.StorageV1().StorageClasses().Create(class)
// The "should provision storage with snapshot data source" test already has created the class.
// TODO: make class creation optional and remove the IsAlreadyExists exception
Expect(err == nil || apierrs.IsAlreadyExists(err)).To(Equal(true))
class, err = client.StorageV1().StorageClasses().Get(class.Name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
defer func() {
framework.Logf("deleting storage class %s", class.Name)
@ -268,6 +295,12 @@ func TestDynamicProvisioning(t StorageClassTest, client clientset.Interface, cla
Expect(err).NotTo(HaveOccurred())
}
if claim.Spec.DataSource != nil {
By("checking the created volume whether has the pre-populated data")
command := fmt.Sprintf("grep '%s' /mnt/test/initialData", claim.Namespace)
runInPodWithVolume(client, claim.Namespace, claim.Name, t.NodeName, command, t.NodeSelector, t.ExpectUnschedulable)
}
if !t.SkipWriteReadCheck {
// We start two pods:
// - The first writes 'hello word' to the /mnt/test (= the volume).
@ -288,6 +321,7 @@ func TestDynamicProvisioning(t StorageClassTest, client clientset.Interface, cla
By("checking the created volume is readable and retains data")
runInPodWithVolume(client, claim.Namespace, claim.Name, t.NodeName, "grep 'hello world' /mnt/test/data", t.NodeSelector, t.ExpectUnschedulable)
}
By(fmt.Sprintf("deleting claim %q/%q", claim.Namespace, claim.Name))
framework.ExpectNoError(client.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, nil))
@ -466,3 +500,76 @@ func verifyPVCsPending(client clientset.Interface, pvcs []*v1.PersistentVolumeCl
Expect(claim.Status.Phase).To(Equal(v1.ClaimPending))
}
}
func prepareDataSourceForProvisioning(
t StorageClassTest,
client clientset.Interface,
dynamicClient dynamic.Interface,
initClaim *v1.PersistentVolumeClaim,
class *storage.StorageClass,
snapshotClass *unstructured.Unstructured,
) (*v1.TypedLocalObjectReference, func()) {
var err error
if class != nil {
By("[Initialize dataSource]creating a StorageClass " + class.Name)
_, err = client.StorageV1().StorageClasses().Create(class)
Expect(err).NotTo(HaveOccurred())
}
By("[Initialize dataSource]creating a initClaim")
updatedClaim, err := client.CoreV1().PersistentVolumeClaims(initClaim.Namespace).Create(initClaim)
Expect(err).NotTo(HaveOccurred())
err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, updatedClaim.Namespace, updatedClaim.Name, framework.Poll, framework.ClaimProvisionTimeout)
Expect(err).NotTo(HaveOccurred())
By("[Initialize dataSource]checking the initClaim")
// Get new copy of the initClaim
_, err = client.CoreV1().PersistentVolumeClaims(updatedClaim.Namespace).Get(updatedClaim.Name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
// write namespace to the /mnt/test (= the volume).
By("[Initialize dataSource]write data to volume")
command := fmt.Sprintf("echo '%s' > /mnt/test/initialData", updatedClaim.GetNamespace())
runInPodWithVolume(client, updatedClaim.Namespace, updatedClaim.Name, t.NodeName, command, t.NodeSelector, t.ExpectUnschedulable)
By("[Initialize dataSource]creating a SnapshotClass")
snapshotClass, err = dynamicClient.Resource(snapshotClassGVR).Create(snapshotClass, metav1.CreateOptions{})
By("[Initialize dataSource]creating a snapshot")
snapshot := getSnapshot(updatedClaim.Name, updatedClaim.Namespace, snapshotClass.GetName())
snapshot, err = dynamicClient.Resource(snapshotGVR).Namespace(updatedClaim.Namespace).Create(snapshot, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
WaitForSnapshotReady(dynamicClient, snapshot.GetNamespace(), snapshot.GetName(), framework.Poll, framework.SnapshotCreateTimeout)
Expect(err).NotTo(HaveOccurred())
By("[Initialize dataSource]checking the snapshot")
// Get new copy of the snapshot
snapshot, err = dynamicClient.Resource(snapshotGVR).Namespace(snapshot.GetNamespace()).Get(snapshot.GetName(), metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
group := "snapshot.storage.k8s.io"
dataSourceRef := &v1.TypedLocalObjectReference{
APIGroup: &group,
Kind: "VolumeSnapshot",
Name: snapshot.GetName(),
}
cleanupFunc := func() {
framework.Logf("deleting snapshot %q/%q", snapshot.GetNamespace(), snapshot.GetName())
err = dynamicClient.Resource(snapshotGVR).Namespace(updatedClaim.Namespace).Delete(snapshot.GetName(), nil)
if err != nil && !apierrs.IsNotFound(err) {
framework.Failf("Error deleting snapshot %q. Error: %v", snapshot.GetName(), err)
}
framework.Logf("deleting initClaim %q/%q", updatedClaim.Namespace, updatedClaim.Name)
err = client.CoreV1().PersistentVolumeClaims(updatedClaim.Namespace).Delete(updatedClaim.Name, nil)
if err != nil && !apierrs.IsNotFound(err) {
framework.Failf("Error deleting initClaim %q. Error: %v", updatedClaim.Name, err)
}
framework.Logf("deleting SnapshotClass %s", snapshotClass.GetName())
framework.ExpectNoError(dynamicClient.Resource(snapshotClassGVR).Delete(snapshotClass.GetName(), nil))
}
return dataSourceRef, cleanupFunc
}

323
test/e2e/storage/testsuites/snapshottable.go Normal file
View File

@ -0,0 +1,323 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package testsuites
import (
"fmt"
"time"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"k8s.io/api/core/v1"
storage "k8s.io/api/storage/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/dynamic"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/storage/testpatterns"
)
// snapshot CRD api group
const snapshotGroup = "snapshot.storage.k8s.io"
// snapshot CRD api version
const snapshotAPIVersion = "snapshot.storage.k8s.io/v1alpha1"
var (
snapshotGVR = schema.GroupVersionResource{Group: snapshotGroup, Version: "v1alpha1", Resource: "volumesnapshots"}
snapshotClassGVR = schema.GroupVersionResource{Group: snapshotGroup, Version: "v1alpha1", Resource: "volumesnapshotclasses"}
snapshotContentGVR = schema.GroupVersionResource{Group: snapshotGroup, Version: "v1alpha1", Resource: "volumesnapshotcontents"}
)
type SnapshotClassTest struct {
Name string
CloudProviders []string
Snapshotter string
Parameters map[string]string
NodeName string
NodeSelector map[string]string // NodeSelector for the pod
SnapshotContentCheck func(snapshotContent *unstructured.Unstructured) error
}
type snapshottableTestSuite struct {
tsInfo TestSuiteInfo
}
var _ TestSuite = &snapshottableTestSuite{}
// InitSnapshottableTestSuite returns snapshottableTestSuite that implements TestSuite interface
func InitSnapshottableTestSuite() TestSuite {
return &snapshottableTestSuite{
tsInfo: TestSuiteInfo{
name: "snapshottable",
testPatterns: []testpatterns.TestPattern{
testpatterns.DynamicSnapshot,
},
},
}
}
func (s *snapshottableTestSuite) getTestSuiteInfo() TestSuiteInfo {
return s.tsInfo
}
func (s *snapshottableTestSuite) skipUnsupportedTest(pattern testpatterns.TestPattern, driver TestDriver) {
}
func createSnapshottableTestInput(driver TestDriver, pattern testpatterns.TestPattern) (snapshottableTestResource, snapshottableTestInput) {
// Setup test resource for driver and testpattern
resource := snapshottableTestResource{}
resource.setupResource(driver, pattern)
input := snapshottableTestInput{
testCase: SnapshotClassTest{},
cs: driver.GetDriverInfo().Config.Framework.ClientSet,
dc: driver.GetDriverInfo().Config.Framework.DynamicClient,
pvc: resource.pvc,
sc: resource.sc,
vsc: resource.vsc,
dInfo: driver.GetDriverInfo(),
}
if driver.GetDriverInfo().Config.ClientNodeName != "" {
input.testCase.NodeName = driver.GetDriverInfo().Config.ClientNodeName
}
return resource, input
}
func (s *snapshottableTestSuite) execTest(driver TestDriver, pattern testpatterns.TestPattern) {
Context(getTestNameStr(s, pattern), func() {
var (
resource snapshottableTestResource
input snapshottableTestInput
needsCleanup bool
)
BeforeEach(func() {
needsCleanup = false
// Skip unsupported tests to avoid unnecessary resource initialization
skipUnsupportedTest(s, driver, pattern)
needsCleanup = true
// Create test input
resource, input = createSnapshottableTestInput(driver, pattern)
})
AfterEach(func() {
if needsCleanup {
resource.cleanupResource(driver, pattern)
}
})
// Ginkgo's "Global Shared Behaviors" require arguments for a shared function
// to be a single struct and to be passed as a pointer.
// Please see https://onsi.github.io/ginkgo/#global-shared-behaviors for details.
testSnapshot(&input)
})
}
type snapshottableTestResource struct {
driver TestDriver
claimSize string
sc *storage.StorageClass
pvc *v1.PersistentVolumeClaim
// volume snapshot class
vsc *unstructured.Unstructured
}
var _ TestResource = &snapshottableTestResource{}
func (s *snapshottableTestResource) setupResource(driver TestDriver, pattern testpatterns.TestPattern) {
// Setup snapshottableTest resource
switch pattern.SnapshotType {
case testpatterns.DynamicCreatedSnapshot:
if dDriver, ok := driver.(DynamicPVTestDriver); ok {
s.sc = dDriver.GetDynamicProvisionStorageClass("")
if s.sc == nil {
framework.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", driver.GetDriverInfo().Name)
}
s.driver = driver
s.claimSize = dDriver.GetClaimSize()
s.pvc = getClaim(s.claimSize, driver.GetDriverInfo().Config.Framework.Namespace.Name)
s.pvc.Spec.StorageClassName = &s.sc.Name
framework.Logf("In creating storage class object and pvc object for driver - sc: %v, pvc: %v", s.sc, s.pvc)
if sDriver, ok := driver.(SnapshottableTestDriver); ok {
s.vsc = sDriver.GetSnapshotClass()
}
}
default:
framework.Failf("Dynamic Snapshot test doesn't support: %s", pattern.SnapshotType)
}
}
func (s *snapshottableTestResource) cleanupResource(driver TestDriver, pattern testpatterns.TestPattern) {
}
type snapshottableTestInput struct {
testCase SnapshotClassTest
cs clientset.Interface
dc dynamic.Interface
pvc *v1.PersistentVolumeClaim
sc *storage.StorageClass
// volume snapshot class
vsc *unstructured.Unstructured
dInfo *DriverInfo
}
func testSnapshot(input *snapshottableTestInput) {
It("should create snapshot with defaults", func() {
if input.dInfo.Name == "csi-hostpath-v0" {
framework.Skipf("skip test when using driver csi-hostpath-v0 - skipping")
}
TestCreateSnapshot(input.testCase, input.cs, input.dc, input.pvc, input.sc, input.vsc)
})
}
// TestCreateSnapshot tests dynamic creating snapshot with specified SnapshotClassTest and snapshotClass
func TestCreateSnapshot(
t SnapshotClassTest,
client clientset.Interface,
dynamicClient dynamic.Interface,
claim *v1.PersistentVolumeClaim,
class *storage.StorageClass,
snapshotClass *unstructured.Unstructured,
) *unstructured.Unstructured {
var err error
if class != nil {
By("creating a StorageClass " + class.Name)
class, err = client.StorageV1().StorageClasses().Create(class)
Expect(err).NotTo(HaveOccurred())
defer func() {
framework.Logf("deleting storage class %s", class.Name)
framework.ExpectNoError(client.StorageV1().StorageClasses().Delete(class.Name, nil))
}()
}
By("creating a claim")
claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(claim)
Expect(err).NotTo(HaveOccurred())
defer func() {
framework.Logf("deleting claim %q/%q", claim.Namespace, claim.Name)
// typically this claim has already been deleted
err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, nil)
if err != nil && !apierrs.IsNotFound(err) {
framework.Failf("Error deleting claim %q. Error: %v", claim.Name, err)
}
}()
err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout)
Expect(err).NotTo(HaveOccurred())
By("checking the claim")
// Get new copy of the claim
claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
// Get the bound PV
pv, err := client.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
By("creating a SnapshotClass")
snapshotClass, err = dynamicClient.Resource(snapshotClassGVR).Create(snapshotClass, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
defer func() {
framework.Logf("deleting SnapshotClass %s", snapshotClass.GetName())
framework.ExpectNoError(dynamicClient.Resource(snapshotClassGVR).Delete(snapshotClass.GetName(), nil))
}()
By("creating a snapshot")
snapshot := getSnapshot(claim.Name, claim.Namespace, snapshotClass.GetName())
snapshot, err = dynamicClient.Resource(snapshotGVR).Namespace(snapshot.GetNamespace()).Create(snapshot, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
defer func() {
framework.Logf("deleting snapshot %q/%q", snapshot.GetNamespace(), snapshot.GetName())
// typically this snapshot has already been deleted
err = dynamicClient.Resource(snapshotGVR).Namespace(snapshot.GetNamespace()).Delete(snapshot.GetName(), nil)
if err != nil && !apierrs.IsNotFound(err) {
framework.Failf("Error deleting snapshot %q. Error: %v", claim.Name, err)
}
}()
err = WaitForSnapshotReady(dynamicClient, snapshot.GetNamespace(), snapshot.GetName(), framework.Poll, framework.SnapshotCreateTimeout)
Expect(err).NotTo(HaveOccurred())
By("checking the snapshot")
// Get new copy of the snapshot
snapshot, err = dynamicClient.Resource(snapshotGVR).Namespace(snapshot.GetNamespace()).Get(snapshot.GetName(), metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
// Get the bound snapshotContent
snapshotSpec := snapshot.Object["spec"].(map[string]interface{})
snapshotContentName := snapshotSpec["snapshotContentName"].(string)
snapshotContent, err := dynamicClient.Resource(snapshotContentGVR).Get(snapshotContentName, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
snapshotContentSpec := snapshotContent.Object["spec"].(map[string]interface{})
volumeSnapshotRef := snapshotContentSpec["volumeSnapshotRef"].(map[string]interface{})
persistentVolumeRef := snapshotContentSpec["persistentVolumeRef"].(map[string]interface{})
// Check SnapshotContent properties
By("checking the SnapshotContent")
Expect(snapshotContentSpec["snapshotClassName"]).To(Equal(snapshotClass.GetName()))
Expect(volumeSnapshotRef["name"]).To(Equal(snapshot.GetName()))
Expect(volumeSnapshotRef["namespace"]).To(Equal(snapshot.GetNamespace()))
Expect(persistentVolumeRef["name"]).To(Equal(pv.Name))
// Run the checker
if t.SnapshotContentCheck != nil {
err = t.SnapshotContentCheck(snapshotContent)
Expect(err).NotTo(HaveOccurred())
}
return snapshotContent
}
// WaitForSnapshotReady waits for a VolumeSnapshot to be ready to use or until timeout occurs, whichever comes first.
func WaitForSnapshotReady(c dynamic.Interface, ns string, snapshotName string, Poll, timeout time.Duration) error {
framework.Logf("Waiting up to %v for VolumeSnapshot %s to become ready", timeout, snapshotName)
for start := time.Now(); time.Since(start) < timeout; time.Sleep(Poll) {
snapshot, err := c.Resource(snapshotGVR).Namespace(ns).Get(snapshotName, metav1.GetOptions{})
if err != nil {
framework.Logf("Failed to get claim %q, retrying in %v. Error: %v", snapshotName, Poll, err)
continue
} else {
status := snapshot.Object["status"]
if status == nil {
framework.Logf("VolumeSnapshot %s found but is not ready.", snapshotName)
continue
}
value := status.(map[string]interface{})
if value["readyToUse"] == true {
framework.Logf("VolumeSnapshot %s found and is ready", snapshotName, time.Since(start))
return nil
} else if value["ready"] == true {
framework.Logf("VolumeSnapshot %s found and is ready", snapshotName, time.Since(start))
return nil
} else {
framework.Logf("VolumeSnapshot %s found but is not ready.", snapshotName)
}
}
}
return fmt.Errorf("VolumeSnapshot %s is not ready within %v", snapshotName, timeout)
}

10
test/e2e/storage/testsuites/testdriver.go
View File

@ -19,6 +19,7 @@ package testsuites
import (
"k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/storage/testpatterns"
@ -79,6 +80,14 @@ type DynamicPVTestDriver interface {
GetClaimSize() string
}
// SnapshottableTestDriver represents an interface for a TestDriver that supports DynamicSnapshot
type SnapshottableTestDriver interface {
TestDriver
// GetSnapshotClass returns a SnapshotClass to create snapshot.
// It will return nil, if the TestDriver doesn't support it.
GetSnapshotClass() *unstructured.Unstructured
}
// Capability represents a feature that a volume plugin supports
type Capability string
@ -87,6 +96,7 @@ const (
CapBlock Capability = "block" // raw block mode
CapFsGroup Capability = "fsGroup" // volume ownership via fsGroup
CapExec Capability = "exec" // exec a file in the volume
CapDataSource Capability = "dataSource" // support populate data from snapshot
)
// DriverInfo represents a combination of parameters to be used in implementation of TestDriver

8
test/e2e/storage/utils/deployment.go
View File

@ -93,6 +93,10 @@ func PatchCSIDeployment(f *framework.Framework, o PatchCSIOptions, object interf
// Driver name is expected to be the same
// as the provisioner here.
container.Args = append(container.Args, "--provisioner="+o.NewDriverName)
case o.SnapshotterContainerName:
// Driver name is expected to be the same
// as the snapshotter here.
container.Args = append(container.Args, "--snapshotter="+o.NewDriverName)
}
}
}
@ -145,6 +149,10 @@ type PatchCSIOptions struct {
// If non-empty, --provisioner with new name will be appended
// to the argument list.
ProvisionerContainerName string
// The name of the container which has the snapshotter binary.
// If non-empty, --snapshotter with new name will be appended
// to the argument list.
SnapshotterContainerName string
// If non-empty, all pods are forced to run on this node.
NodeName string
}

3
test/e2e/testing-manifests/storage-csi/controller-role.yaml
View File

@ -1,3 +1,4 @@
# Replaced by individual roles for external-attacher and external-provisioner:
# Replaced by individual roles for external-attacher, external-provisioner and external-snapshotter:
# - https://github.com/kubernetes-csi/external-attacher/blob/master/deploy/kubernetes/rbac.yaml
# - https://github.com/kubernetes-csi/external-provisioner/blob/master/deploy/kubernetes/rbac.yaml
# - https://github.com/kubernetes-csi/external-snapshotter/blob/master/deploy/kubernetes/rbac.yaml

1
test/e2e/testing-manifests/storage-csi/external-snapshotter/README.md Normal file
View File

@ -0,0 +1 @@
The original file is https://github.com/kubernetes-csi/external-snapshotter/blob/master/deploy/kubernetes/rbac.yaml

65
test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml Normal file
View File

@ -0,0 +1,65 @@
# Together with the RBAC file for external-provisioner, this YAML file
# contains all RBAC objects that are necessary to run external CSI
# snapshotter.
#
# In production, each CSI driver deployment has to be customized:
# - to avoid conflicts, use non-default namespace and different names
# for non-namespaced entities like the ClusterRole
# - optionally rename the non-namespaced ClusterRole if there
# are conflicts with other deployments
apiVersion: v1
kind: ServiceAccount
metadata:
name: csi-snapshotter
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
# rename if there are conflicts
name: external-snapshotter-runner
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["list", "watch", "create", "update", "patch"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotcontents"]
verbs: ["create", "get", "list", "watch", "update", "delete"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshots"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["apiextensions.k8s.io"]
resources: ["customresourcedefinitions"]
verbs: ["create", "list", "watch", "delete"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-snapshotter-role
subjects:
- kind: ServiceAccount
name: csi-snapshotter
# replace with non-default namespace name
namespace: default
roleRef:
kind: ClusterRole
# change the name also here if the ClusterRole gets renamed
name: external-snapshotter-runner
apiGroup: rbac.authorization.k8s.io

4
test/e2e/testing-manifests/storage-csi/hostpath/README.md
View File

@ -1,5 +1,5 @@
A partial copy of https://github.com/kubernetes-csi/docs/tree/master/book/src/example,
with some modifications:
- serviceAccountName is used instead of the deprecated serviceAccount
- the RBAC roles from driver-registrar, external-attacher and external-provisioner
are used
- the RBAC roles from driver-registrar, external-attacher, external-provisioner
and external-snapshotter are used

48
test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml Normal file
View File

@ -0,0 +1,48 @@
kind: Service
apiVersion: v1
metadata:
name: csi-snapshotter
labels:
app: csi-snapshotter
spec:
selector:
app: csi-snapshotter
ports:
- name: dummy
port: 12345
---
kind: StatefulSet
apiVersion: apps/v1
metadata:
name: csi-snapshotter
spec:
serviceName: "csi-snapshotter"
replicas: 1
selector:
matchLabels:
app: csi-snapshotter
template:
metadata:
labels:
app: csi-snapshotter
spec:
serviceAccount: csi-snapshotter
containers:
- name: csi-snapshotter
image: quay.io/k8scsi/csi-snapshotter:v1.0.1
args:
- "--csi-address=$(ADDRESS)"
- "--connection-timeout=15s"
env:
- name: ADDRESS
value: /csi/csi.sock
imagePullPolicy: Always
volumeMounts:
- name: socket-dir
mountPath: /csi
volumes:
- hostPath:
path: /var/lib/kubelet/plugins/csi-hostpath
type: DirectoryOrCreate
name: socket-dir

3
test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml
View File

@ -13,6 +13,9 @@ subjects:
- kind: ServiceAccount
name: csi-provisioner
namespace: default
- kind: ServiceAccount
name: csi-snapshotter
namespace: default
roleRef:
kind: ClusterRole
name: e2e-test-privileged-psp