From a8f318779b511307a53fbaecb5fbb7bb40847ab7 Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Mon, 4 Mar 2019 11:35:15 -0500 Subject: [PATCH] Add CSI volume resizing tests Add some tests for checking node expansion Add new tests for expander --- pkg/volume/csi/BUILD | 3 + pkg/volume/csi/csi_client.go | 23 +- pkg/volume/csi/csi_client_test.go | 105 +++++++ pkg/volume/csi/expander.go | 42 +-- pkg/volume/csi/expander_test.go | 94 ++++++ pkg/volume/csi/fake/fake_client.go | 62 +++- .../operationexecutor/operation_generator.go | 54 +++- test/e2e/storage/csi_mock_volume.go | 273 +++++++++++++++++- test/e2e/storage/drivers/csi.go | 45 ++- .../flexvolume_mounted_volume_resize.go | 2 +- test/e2e/storage/flexvolume_online_resize.go | 2 +- test/e2e/storage/mounted_volume_resize.go | 2 +- test/e2e/storage/volume_expand.go | 26 +- .../storage-csi/external-resizer/README.md | 1 + .../storage-csi/external-resizer/rbac.yaml | 90 ++++++ .../mock/csi-mock-driver-resizer.yaml | 33 +++ .../storage-csi/mock/csi-mock-driver.yaml | 2 +- .../storage-csi/mock/csi-mock-rbac.yaml | 13 + 18 files changed, 791 insertions(+), 81 deletions(-) create mode 100644 pkg/volume/csi/expander_test.go create mode 100644 test/e2e/testing-manifests/storage-csi/external-resizer/README.md create mode 100644 test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml create mode 100644 test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver-resizer.yaml diff --git a/pkg/volume/csi/BUILD b/pkg/volume/csi/BUILD index a51d3cb8a7..71e07796af 100644 --- a/pkg/volume/csi/BUILD +++ b/pkg/volume/csi/BUILD @@ -10,6 +10,7 @@ go_library( "csi_mounter.go", "csi_plugin.go", "csi_util.go", + "expander.go", ], importpath = "k8s.io/kubernetes/pkg/volume/csi", visibility = ["//visibility:public"], @@ -22,6 +23,7 @@ go_library( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", @@ -50,6 +52,7 @@ go_test( "csi_drivers_store_test.go", "csi_mounter_test.go", "csi_plugin_test.go", + "expander_test.go", ], embed = [":go_default_library"], deps = [ diff --git a/pkg/volume/csi/csi_client.go b/pkg/volume/csi/csi_client.go index b373eb950a..075123080b 100644 --- a/pkg/volume/csi/csi_client.go +++ b/pkg/volume/csi/csi_client.go @@ -56,7 +56,7 @@ type csiClient interface { fsType string, mountOptions []string, ) error - NodeExpandVolume(ctx context.Context, volumeid, volumePath string, newSize resource.Quantity) (*resource.Quantity, error) + NodeExpandVolume(ctx context.Context, volumeid, volumePath string, newSize resource.Quantity) (resource.Quantity, error) NodeUnpublishVolume( ctx context.Context, volID string, @@ -307,14 +307,25 @@ func (c *csiDriverClient) NodePublishVolume( } -func (c *csiDriverClient) NodeExpandVolume(ctx context.Context, volumeID, volumePath string, newSize resource.Quantity) (*resource.Quantity, error) { +func (c *csiDriverClient) NodeExpandVolume(ctx context.Context, volumeID, volumePath string, newSize resource.Quantity) (resource.Quantity, error) { if c.nodeV1ClientCreator == nil { - return nil, fmt.Errorf("version of CSI driver does not support volume expansion") + return newSize, fmt.Errorf("version of CSI driver does not support volume expansion") + } + + if volumeID == "" { + return newSize, errors.New("missing volume id") + } + if volumePath == "" { + return newSize, errors.New("missing volume path") + } + + if newSize.Value() < 0 { + return newSize, errors.New("size can not be less than 0") } nodeClient, closer, err := c.nodeV1ClientCreator(c.addr) if err != nil { - return nil, err + return newSize, err } defer closer.Close() @@ -325,10 +336,10 @@ func (c *csiDriverClient) NodeExpandVolume(ctx context.Context, volumeID, volume } resp, err := nodeClient.NodeExpandVolume(ctx, req) if err != nil { - return nil, err + return newSize, err } updatedQuantity := resource.NewQuantity(resp.CapacityBytes, resource.BinarySI) - return updatedQuantity, nil + return *updatedQuantity, nil } func (c *csiDriverClient) nodePublishVolumeV1( diff --git a/pkg/volume/csi/csi_client_test.go b/pkg/volume/csi/csi_client_test.go index 78d7033327..8e5c05ba8f 100644 --- a/pkg/volume/csi/csi_client_test.go +++ b/pkg/volume/csi/csi_client_test.go @@ -25,6 +25,7 @@ import ( csipbv1 "github.com/container-storage-interface/spec/lib/go/csi" api "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/kubernetes/pkg/volume/csi/fake" ) @@ -40,6 +41,13 @@ func newFakeCsiDriverClient(t *testing.T, stagingCapable bool) *fakeCsiDriverCli } } +func newFakeCsiDriverClientWithExpansion(t *testing.T, stagingCapable bool, expansionSet bool) *fakeCsiDriverClient { + return &fakeCsiDriverClient{ + t: t, + nodeClient: fake.NewNodeClientWithExpansion(stagingCapable, expansionSet), + } +} + func (c *fakeCsiDriverClient) NodeGetInfo(ctx context.Context) ( nodeID string, maxVolumePerNode int64, @@ -144,6 +152,28 @@ func (c *fakeCsiDriverClient) NodeUnstageVolume(ctx context.Context, volID, stag return err } +func (c *fakeCsiDriverClient) NodeSupportsNodeExpand(ctx context.Context) (bool, error) { + c.t.Log("calling fake.NodeSupportsNodeExpand...") + req := &csipbv1.NodeGetCapabilitiesRequest{} + + resp, err := c.nodeClient.NodeGetCapabilities(ctx, req) + if err != nil { + return false, err + } + + capabilities := resp.GetCapabilities() + + if capabilities == nil { + return false, nil + } + for _, capability := range capabilities { + if capability.GetRpc().GetType() == csipbv1.NodeServiceCapability_RPC_EXPAND_VOLUME { + return true, nil + } + } + return false, nil +} + func (c *fakeCsiDriverClient) NodeSupportsStageUnstage(ctx context.Context) (bool, error) { c.t.Log("calling fake.NodeGetCapabilities for NodeSupportsStageUnstage...") req := &csipbv1.NodeGetCapabilitiesRequest{} @@ -166,10 +196,29 @@ func (c *fakeCsiDriverClient) NodeSupportsStageUnstage(ctx context.Context) (boo return stageUnstageSet, nil } +func (c *fakeCsiDriverClient) NodeExpandVolume(ctx context.Context, volumeid, volumePath string, newSize resource.Quantity) (resource.Quantity, error) { + c.t.Log("calling fake.NodeExpandVolume") + req := &csipbv1.NodeExpandVolumeRequest{ + VolumeId: volumeid, + VolumePath: volumePath, + CapacityRange: &csipbv1.CapacityRange{RequiredBytes: newSize.Value()}, + } + resp, err := c.nodeClient.NodeExpandVolume(ctx, req) + if err != nil { + return newSize, err + } + updatedQuantity := resource.NewQuantity(resp.CapacityBytes, resource.BinarySI) + return *updatedQuantity, nil +} + func setupClient(t *testing.T, stageUnstageSet bool) csiClient { return newFakeCsiDriverClient(t, stageUnstageSet) } +func setupClientWithExpansion(t *testing.T, stageUnstageSet bool, expansionSet bool) csiClient { + return newFakeCsiDriverClientWithExpansion(t, stageUnstageSet, expansionSet) +} + func checkErr(t *testing.T, expectedAnError bool, actualError error) { t.Helper() @@ -415,3 +464,59 @@ func TestClientNodeUnstageVolume(t *testing.T) { } } } + +func TestNodeExpandVolume(t *testing.T) { + testCases := []struct { + name string + volID string + volumePath string + newSize resource.Quantity + mustFail bool + err error + }{ + { + name: "with all correct values", + volID: "vol-abcde", + volumePath: "/foo/bar", + newSize: resource.MustParse("10Gi"), + mustFail: false, + }, + { + name: "with missing volume-id", + volumePath: "/foo/bar", + newSize: resource.MustParse("10Gi"), + mustFail: true, + }, + { + name: "with missing volume path", + volID: "vol-1234", + newSize: resource.MustParse("10Gi"), + mustFail: true, + }, + { + name: "with invalid quantity", + volID: "vol-1234", + volumePath: "/foo/bar", + newSize: *resource.NewQuantity(-10, resource.DecimalSI), + mustFail: true, + }, + } + + for _, tc := range testCases { + t.Logf("Running test cases : %s", tc.name) + fakeCloser := fake.NewCloser(t) + client := &csiDriverClient{ + driverName: "Fake Driver Name", + nodeV1ClientCreator: func(addr csiAddr) (csipbv1.NodeClient, io.Closer, error) { + nodeClient := fake.NewNodeClient(false /* stagingCapable */) + nodeClient.SetNextError(tc.err) + return nodeClient, fakeCloser, nil + }, + } + _, err := client.NodeExpandVolume(context.Background(), tc.volID, tc.volumePath, tc.newSize) + checkErr(t, tc.mustFail, err) + if !tc.mustFail { + fakeCloser.Check() + } + } +} diff --git a/pkg/volume/csi/expander.go b/pkg/volume/csi/expander.go index 29a507957f..265d5e5453 100644 --- a/pkg/volume/csi/expander.go +++ b/pkg/volume/csi/expander.go @@ -18,9 +18,9 @@ package csi import ( "context" - "errors" "fmt" + api "k8s.io/api/core/v1" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/klog" "k8s.io/kubernetes/pkg/features" @@ -34,7 +34,7 @@ func (c *csiPlugin) RequiresFSResize() bool { // NodeExpand to do the right thing and return early if plugin does not have // node expansion capability. if !utilfeature.DefaultFeatureGate.Enabled(features.ExpandCSIVolumes) { - klog.V(4).Infof("Resizing is not enabled for this CSI volume") + klog.V(4).Infof("Resizing is not enabled for CSI volume") return false } return true @@ -42,40 +42,40 @@ func (c *csiPlugin) RequiresFSResize() bool { func (c *csiPlugin) NodeExpand(resizeOptions volume.NodeResizeOptions) (bool, error) { klog.V(4).Infof(log("Expander.NodeExpand(%s)", resizeOptions.DeviceMountPath)) - pvSource, err := getCSISourceFromSpec(resizeOptions.VolumeSpec) - if err != nil { - return false, err - } - k8s := c.host.GetKubeClient() - if k8s == nil { - klog.Error(log("failed to get a kubernetes client")) - return false, errors.New("failed to get a Kubernetes client") - } - - csiClient, err := newCsiDriverClient(csiDriverName(pvSource.Driver)) - if err != nil { - return false, err - } - csiSource, err := getCSISourceFromSpec(resizeOptions.VolumeSpec) if err != nil { klog.Error(log("Expander.NodeExpand failed to get CSI persistent source: %v", err)) return false, err } + + csClient, err := newCsiDriverClient(csiDriverName(csiSource.Driver)) + if err != nil { + return false, err + } + + return c.nodeExpandWithClient(resizeOptions, csiSource, csClient) +} + +func (c *csiPlugin) nodeExpandWithClient( + resizeOptions volume.NodeResizeOptions, + csiSource *api.CSIPersistentVolumeSource, + csClient csiClient) (bool, error) { + driverName := csiSource.Driver + ctx, cancel := context.WithTimeout(context.Background(), csiTimeout) defer cancel() - nodeExpandSet, err := csiClient.NodeSupportsNodeExpand(ctx) + nodeExpandSet, err := csClient.NodeSupportsNodeExpand(ctx) if err != nil { return false, fmt.Errorf("Expander.NodeExpand failed to check if node supports expansion : %v", err) } if !nodeExpandSet { - return false, fmt.Errorf("Expander.NodeExpand found CSI plugin %s to not support node expansion", c.GetPluginName()) + return false, fmt.Errorf("Expander.NodeExpand found CSI plugin %s/%s to not support node expansion", c.GetPluginName(), driverName) } // Check whether "STAGE_UNSTAGE_VOLUME" is set - stageUnstageSet, err := csiClient.NodeSupportsStageUnstage(ctx) + stageUnstageSet, err := csClient.NodeSupportsStageUnstage(ctx) if err != nil { return false, fmt.Errorf("Expander.NodeExpand failed to check if plugins supports stage_unstage %v", err) } @@ -88,7 +88,7 @@ func (c *csiPlugin) NodeExpand(resizeOptions volume.NodeResizeOptions) (bool, er return false, nil } - _, err = csiClient.NodeExpandVolume(ctx, csiSource.VolumeHandle, resizeOptions.DeviceMountPath, resizeOptions.NewSize) + _, err = csClient.NodeExpandVolume(ctx, csiSource.VolumeHandle, resizeOptions.DeviceMountPath, resizeOptions.NewSize) if err != nil { return false, fmt.Errorf("Expander.NodeExpand failed to expand the volume : %v", err) } diff --git a/pkg/volume/csi/expander_test.go b/pkg/volume/csi/expander_test.go new file mode 100644 index 0000000000..0bd9f128e2 --- /dev/null +++ b/pkg/volume/csi/expander_test.go @@ -0,0 +1,94 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "os" + "testing" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/kubernetes/pkg/volume" +) + +func TestNodeExpand(t *testing.T) { + tests := []struct { + name string + nodeExpansion bool + nodeStageSet bool + volumePhase volume.CSIVolumePhaseType + success bool + }{ + { + name: "when node expansion is not set", + success: false, + }, + { + name: "when nodeExpansion=on, nodeStage=on, volumePhase=staged", + nodeExpansion: true, + nodeStageSet: true, + volumePhase: volume.CSIVolumeStaged, + success: true, + }, + { + name: "when nodeExpansion=on, nodeStage=off, volumePhase=staged", + nodeExpansion: true, + volumePhase: volume.CSIVolumeStaged, + success: false, + }, + { + name: "when nodeExpansion=on, nodeStage=on, volumePhase=published", + nodeExpansion: true, + nodeStageSet: true, + volumePhase: volume.CSIVolumePublished, + success: true, + }, + { + name: "when nodeExpansion=on, nodeStage=off, volumePhase=published", + nodeExpansion: true, + volumePhase: volume.CSIVolumePublished, + success: true, + }, + } + for _, tc := range tests { + plug, tmpDir := newTestPlugin(t, nil) + defer os.RemoveAll(tmpDir) + + spec := volume.NewSpecFromPersistentVolume(makeTestPV("test-pv", 10, "expandable", "test-vol"), false) + + newSize, _ := resource.ParseQuantity("20Gi") + + resizeOptions := volume.NodeResizeOptions{ + VolumeSpec: spec, + NewSize: newSize, + DeviceMountPath: "/foo/bar", + CSIVolumePhase: tc.volumePhase, + } + csiSource, _ := getCSISourceFromSpec(resizeOptions.VolumeSpec) + + csClient := setupClientWithExpansion(t, tc.nodeStageSet, tc.nodeExpansion) + + ok, err := plug.nodeExpandWithClient(resizeOptions, csiSource, csClient) + if ok != tc.success { + if err != nil { + t.Errorf("For %s : expected %v got %v with %v", tc.name, tc.success, ok, err) + } else { + t.Errorf("For %s : expected %v got %v", tc.name, tc.success, ok) + } + + } + } +} diff --git a/pkg/volume/csi/fake/fake_client.go b/pkg/volume/csi/fake/fake_client.go index c9ed5e6b9a..f674fdd440 100644 --- a/pkg/volume/csi/fake/fake_client.go +++ b/pkg/volume/csi/fake/fake_client.go @@ -67,6 +67,7 @@ type NodeClient struct { nodePublishedVolumes map[string]CSIVolume nodeStagedVolumes map[string]CSIVolume stageUnstageSet bool + expansionSet bool nodeGetInfoResp *csipb.NodeGetInfoResponse nextErr error } @@ -80,6 +81,15 @@ func NewNodeClient(stageUnstageSet bool) *NodeClient { } } +func NewNodeClientWithExpansion(stageUnstageSet bool, expansionSet bool) *NodeClient { + return &NodeClient{ + nodePublishedVolumes: make(map[string]CSIVolume), + nodeStagedVolumes: make(map[string]CSIVolume), + stageUnstageSet: stageUnstageSet, + expansionSet: expansionSet, + } +} + // SetNextError injects next expected error func (f *NodeClient) SetNextError(err error) { f.nextErr = err @@ -195,6 +205,29 @@ func (f *NodeClient) NodeUnstageVolume(ctx context.Context, req *csipb.NodeUnsta return &csipb.NodeUnstageVolumeResponse{}, nil } +// NodeExpandVolume implements csi method +func (f *NodeClient) NodeExpandVolume(ctx context.Context, req *csipb.NodeExpandVolumeRequest, opts ...grpc.CallOption) (*csipb.NodeExpandVolumeResponse, error) { + if f.nextErr != nil { + return nil, f.nextErr + } + + if req.GetVolumeId() == "" { + return nil, errors.New("missing volume id") + } + if req.GetVolumePath() == "" { + return nil, errors.New("missing volume path") + } + + if req.GetCapacityRange().RequiredBytes <= 0 { + return nil, errors.New("required bytes should be greater than 0") + } + + resp := &csipb.NodeExpandVolumeResponse{ + CapacityBytes: req.GetCapacityRange().RequiredBytes, + } + return resp, nil +} + // NodeGetId implements csi method func (f *NodeClient) NodeGetInfo(ctx context.Context, in *csipb.NodeGetInfoRequest, opts ...grpc.CallOption) (*csipb.NodeGetInfoResponse, error) { if f.nextErr != nil { @@ -206,20 +239,27 @@ func (f *NodeClient) NodeGetInfo(ctx context.Context, in *csipb.NodeGetInfoReque // NodeGetCapabilities implements csi method func (f *NodeClient) NodeGetCapabilities(ctx context.Context, in *csipb.NodeGetCapabilitiesRequest, opts ...grpc.CallOption) (*csipb.NodeGetCapabilitiesResponse, error) { resp := &csipb.NodeGetCapabilitiesResponse{ - Capabilities: []*csipb.NodeServiceCapability{ - { - Type: &csipb.NodeServiceCapability_Rpc{ - Rpc: &csipb.NodeServiceCapability_RPC{ - Type: csipb.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME, - }, - }, - }, - }, + Capabilities: []*csipb.NodeServiceCapability{}, } if f.stageUnstageSet { - return resp, nil + resp.Capabilities = append(resp.Capabilities, &csipb.NodeServiceCapability{ + Type: &csipb.NodeServiceCapability_Rpc{ + Rpc: &csipb.NodeServiceCapability_RPC{ + Type: csipb.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME, + }, + }, + }) } - return nil, nil + if f.expansionSet { + resp.Capabilities = append(resp.Capabilities, &csipb.NodeServiceCapability{ + Type: &csipb.NodeServiceCapability_Rpc{ + Rpc: &csipb.NodeServiceCapability_RPC{ + Type: csipb.NodeServiceCapability_RPC_EXPAND_VOLUME, + }, + }, + }) + } + return resp, nil } // NodeGetVolumeStats implements csi method diff --git a/pkg/volume/util/operationexecutor/operation_generator.go b/pkg/volume/util/operationexecutor/operation_generator.go index 017ee9fe58..91e4825329 100644 --- a/pkg/volume/util/operationexecutor/operation_generator.go +++ b/pkg/volume/util/operationexecutor/operation_generator.go @@ -640,6 +640,7 @@ func (og *operationGenerator) GenerateMountVolumeFunc( resizeDone, resizeError = og.resizeFileSystem(volumeToMount, resizeOptions, volumePluginName) if resizeError != nil { + klog.Errorf("MountVolume.resizeFileSystem failed with %v", resizeError) return volumeToMount.GenerateError("MountVolume.MountDevice failed while expanding volume", resizeError) } } @@ -669,9 +670,14 @@ func (og *operationGenerator) GenerateMountVolumeFunc( resizeOptions.DeviceMountPath = volumeMounter.GetPath() resizeOptions.CSIVolumePhase = volume.CSIVolumePublished + // We need to call resizing here again in case resizing was not done during device mount. There could be + // two reasons of that: + // - Volume does not support DeviceMounter interface. + // - In case of CSI the volume does not have node stage_unstage capability. if !resizeDone { resizeDone, resizeError = og.resizeFileSystem(volumeToMount, resizeOptions, volumePluginName) if resizeError != nil { + klog.Errorf("MountVolume.resizeFileSystem failed with %v", resizeError) return volumeToMount.GenerateError("MountVolume.Setup failed while expanding volume", resizeError) } } @@ -746,6 +752,9 @@ func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, rsOp if resizeErr != nil { return false, fmt.Errorf("MountVolume.resizeFileSystem failed : %v", resizeErr) } + // Volume resizing is not done but it did not error out. This could happen if a CSI volume + // does not have node stage_unstage capability but was asked to resize the volume before + // node publish. In which case - we must retry resizing after node publish. if !resizeDone { return false, nil } @@ -1473,6 +1482,7 @@ func (og *operationGenerator) GenerateExpandVolumeFSWithoutUnmountingFunc( fsResizeFunc := func() (error, error) { var resizeDone bool + var simpleErr, detailedErr error resizeOptions := volume.NodeResizeOptions{ VolumeSpec: volumeToMount.VolumeSpec, } @@ -1490,16 +1500,11 @@ func (og *operationGenerator) GenerateExpandVolumeFSWithoutUnmountingFunc( return volumeToMount.GenerateError("VolumeFSResize.GetDeviceMountPath failed", err) } resizeOptions.DeviceMountPath = dmp - resizeDone, err = og.resizeFileSystem(volumeToMount, resizeOptions, volumePlugin.GetPluginName()) - if err != nil { - return volumeToMount.GenerateError("VolumeFSResize.resizeFileSystem failed", err) + resizeDone, simpleErr, detailedErr = og.doOnlineExpansion(volumeToMount, actualStateOfWorld, resizeOptions, volumePlugin.GetPluginName()) + if simpleErr != nil || detailedErr != nil { + return simpleErr, detailedErr } if resizeDone { - markFSResizedErr := actualStateOfWorld.MarkVolumeAsResized(volumeToMount.PodName, volumeToMount.VolumeName) - if markFSResizedErr != nil { - // On failure, return error. Caller will log and retry. - return volumeToMount.GenerateError("VolumeFSResize.MarkVolumeAsResized failed", markFSResizedErr) - } return nil, nil } } @@ -1515,16 +1520,11 @@ func (og *operationGenerator) GenerateExpandVolumeFSWithoutUnmountingFunc( resizeOptions.DeviceMountPath = volumeMounter.GetPath() resizeOptions.CSIVolumePhase = volume.CSIVolumePublished - resizeDone, err = og.resizeFileSystem(volumeToMount, resizeOptions, volumePlugin.GetPluginName()) - if err != nil { - return volumeToMount.GenerateError("VolumeFSResize.resizeFileSystem failed", err) + resizeDone, simpleErr, detailedErr = og.doOnlineExpansion(volumeToMount, actualStateOfWorld, resizeOptions, volumePlugin.GetPluginName()) + if simpleErr != nil || detailedErr != nil { + return simpleErr, detailedErr } if resizeDone { - markFSResizedErr := actualStateOfWorld.MarkVolumeAsResized(volumeToMount.PodName, volumeToMount.VolumeName) - if markFSResizedErr != nil { - // On failure, return error. Caller will log and retry. - return volumeToMount.GenerateError("VolumeFSResize.MarkVolumeAsResized failed", markFSResizedErr) - } return nil, nil } // This is a placeholder error - we should NEVER reach here. @@ -1545,6 +1545,28 @@ func (og *operationGenerator) GenerateExpandVolumeFSWithoutUnmountingFunc( }, nil } +func (og *operationGenerator) doOnlineExpansion(volumeToMount VolumeToMount, + actualStateOfWorld ActualStateOfWorldMounterUpdater, + resizeOptions volume.NodeResizeOptions, + pluginName string) (bool, error, error) { + resizeDone, err := og.resizeFileSystem(volumeToMount, resizeOptions, pluginName) + if err != nil { + klog.Errorf("VolumeFSResize.resizeFileSystem failed : %v", err) + e1, e2 := volumeToMount.GenerateError("VolumeFSResize.resizeFileSystem failed", err) + return false, e1, e2 + } + if resizeDone { + markFSResizedErr := actualStateOfWorld.MarkVolumeAsResized(volumeToMount.PodName, volumeToMount.VolumeName) + if markFSResizedErr != nil { + // On failure, return error. Caller will log and retry. + e1, e2 := volumeToMount.GenerateError("VolumeFSResize.MarkVolumeAsResized failed", markFSResizedErr) + return false, e1, e2 + } + return true, nil, nil + } + return false, nil, nil +} + func checkMountOptionSupport(og *operationGenerator, volumeToMount VolumeToMount, plugin volume.VolumePlugin) error { mountOptions := util.MountOptionFromSpec(volumeToMount.VolumeSpec) diff --git a/test/e2e/storage/csi_mock_volume.go b/test/e2e/storage/csi_mock_volume.go index 3331db8604..7a41fa3020 100644 --- a/test/e2e/storage/csi_mock_volume.go +++ b/test/e2e/storage/csi_mock_volume.go @@ -28,6 +28,7 @@ import ( storage "k8s.io/api/storage/v1" storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" @@ -50,16 +51,23 @@ type cleanupFuncs func() const ( csiNodeLimitUpdateTimeout = 5 * time.Minute csiPodUnschedulableTimeout = 5 * time.Minute + csiResizeWaitPeriod = 5 * time.Minute + // how long to wait for Resizing Condition on PVC to appear + csiResizingConditionWait = 2 * time.Minute ) var _ = utils.SIGDescribe("CSI mock volume", func() { type testParameters struct { - disableAttach bool - attachLimit int - registerDriver bool - podInfo *bool - scName string - nodeSelectorKey string + disableAttach bool + attachLimit int + registerDriver bool + podInfo *bool + scName string + nodeSelectorKey string + enableResizing bool // enable resizing for both CSI mock driver and storageClass. + enableNodeExpansion bool // enable node expansion for CSI mock driver + // just disable resizing on driver it overrides enableResizing flag for CSI mock driver + disableResizingOnDriver bool } type mockDriverSetup struct { @@ -87,8 +95,21 @@ var _ = utils.SIGDescribe("CSI mock volume", func() { } cs := f.ClientSet var err error + driverOpts := drivers.CSIMockDriverOpts{ + RegisterDriver: tp.registerDriver, + PodInfo: tp.podInfo, + AttachLimit: tp.attachLimit, + DisableAttach: tp.disableAttach, + EnableResizing: tp.enableResizing, + EnableNodeExpansion: tp.enableNodeExpansion, + } - m.driver = drivers.InitMockCSIDriver(tp.registerDriver, !tp.disableAttach, tp.podInfo, tp.attachLimit) + // this just disable resizing on driver, keeping resizing on SC enabled. + if tp.disableResizingOnDriver { + driverOpts.EnableResizing = false + } + + m.driver = drivers.InitMockCSIDriver(driverOpts) config, testCleanup := m.driver.PrepareTest(f) m.testCleanups = append(m.testCleanups, testCleanup) m.config = config @@ -127,6 +148,11 @@ var _ = utils.SIGDescribe("CSI mock volume", func() { if m.tp.scName != "" { scTest.StorageClassName = m.tp.scName } + + if m.tp.enableResizing { + scTest.AllowVolumeExpansion = true + } + nodeSelection := testsuites.NodeSelection{ // The mock driver only works when everything runs on a single node. Name: nodeName, @@ -149,6 +175,23 @@ var _ = utils.SIGDescribe("CSI mock volume", func() { return class, claim, pod } + createPodWithPVC := func(pvc *v1.PersistentVolumeClaim) (*v1.Pod, error) { + nodeName := m.config.ClientNodeName + nodeSelection := testsuites.NodeSelection{ + Name: nodeName, + } + if len(m.nodeLabel) > 0 { + nodeSelection = testsuites.NodeSelection{ + Selector: m.nodeLabel, + } + } + pod, err := startPausePodWithClaim(m.cs, pvc, nodeSelection, f.Namespace.Name) + if pod != nil { + m.pods = append(m.pods, pod) + } + return pod, err + } + cleanup := func() { cs := f.ClientSet var errs []error @@ -340,6 +383,177 @@ var _ = utils.SIGDescribe("CSI mock volume", func() { }) }) + Context("CSI Volume expansion [Feature:ExpandCSIVolumes]", func() { + tests := []struct { + name string + nodeExpansionRequired bool + disableAttach bool + disableResizingOnDriver bool + expectFailure bool + }{ + { + name: "should expand volume without restarting pod if nodeExpansion=off", + nodeExpansionRequired: false, + }, + { + name: "should expand volume by restarting pod if attach=on, nodeExpansion=on", + nodeExpansionRequired: true, + }, + { + name: "should expand volume by restarting pod if attach=off, nodeExpansion=on", + disableAttach: true, + nodeExpansionRequired: true, + }, + { + name: "should not expand volume if resizingOnDriver=off, resizingOnSC=on", + disableResizingOnDriver: true, + expectFailure: true, + }, + } + for _, t := range tests { + test := t + It(t.name, func() { + var err error + tp := testParameters{ + enableResizing: true, + enableNodeExpansion: test.nodeExpansionRequired, + disableResizingOnDriver: test.disableResizingOnDriver, + } + // disabling attach requires drive registration feature + if test.disableAttach { + tp.disableAttach = true + tp.registerDriver = true + } + + init(tp) + defer cleanup() + + ns := f.Namespace.Name + sc, pvc, pod := createPod() + Expect(pod).NotTo(BeNil(), "while creating pod for resizing") + + Expect(*sc.AllowVolumeExpansion).To(BeTrue(), "failed creating sc with allowed expansion") + + err = framework.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) + framework.ExpectNoError(err, "Failed to start pod1: %v", err) + + By("Expanding current pvc") + newSize := resource.MustParse("6Gi") + pvc, err = expandPVCSize(pvc, newSize, m.cs) + Expect(err).NotTo(HaveOccurred(), "While updating pvc for more size") + Expect(pvc).NotTo(BeNil()) + + pvcSize := pvc.Spec.Resources.Requests[v1.ResourceStorage] + if pvcSize.Cmp(newSize) != 0 { + framework.Failf("error updating pvc size %q", pvc.Name) + } + if test.expectFailure { + err = waitForResizingCondition(pvc, m.cs, csiResizingConditionWait) + Expect(err).To(HaveOccurred(), "unexpected resizing condition on PVC") + return + } + + By("Waiting for persistent volume resize to finish") + err = waitForControllerVolumeResize(pvc, m.cs, csiResizeWaitPeriod) + Expect(err).NotTo(HaveOccurred(), "While waiting for CSI PV resize to finish") + + checkPVCSize := func() { + By("Waiting for PVC resize to finish") + pvc, err = waitForFSResize(pvc, m.cs) + Expect(err).NotTo(HaveOccurred(), "while waiting for PVC resize to finish") + + pvcConditions := pvc.Status.Conditions + Expect(len(pvcConditions)).To(Equal(0), "pvc should not have conditions") + } + + // if node expansion is not required PVC should be resized as well + if !test.nodeExpansionRequired { + checkPVCSize() + } else { + By("Checking for conditions on pvc") + pvc, err = m.cs.CoreV1().PersistentVolumeClaims(ns).Get(pvc.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred(), "While fetching pvc after controller resize") + + inProgressConditions := pvc.Status.Conditions + if len(inProgressConditions) > 0 { + Expect(inProgressConditions[0].Type).To(Equal(v1.PersistentVolumeClaimFileSystemResizePending), "pvc must have fs resizing condition") + } + + By("Deleting the previously created pod") + err = framework.DeletePodWithWait(f, m.cs, pod) + Expect(err).NotTo(HaveOccurred(), "while deleting pod for resizing") + + By("Creating a new pod with same volume") + pod2, err := createPodWithPVC(pvc) + Expect(pod2).NotTo(BeNil(), "while creating pod for csi resizing") + Expect(err).NotTo(HaveOccurred(), "while recreating pod for resizing") + + checkPVCSize() + } + }) + } + }) + Context("CSI online volume expansion [Feature:ExpandCSIVolumes][Feature:ExpandInUseVolumes]", func() { + tests := []struct { + name string + disableAttach bool + }{ + { + name: "should expand volume without restarting pod if attach=on, nodeExpansion=on", + }, + { + name: "should expand volume without restarting pod if attach=off, nodeExpansion=on", + disableAttach: true, + }, + } + for _, t := range tests { + test := t + It(test.name, func() { + var err error + params := testParameters{enableResizing: true, enableNodeExpansion: true} + if test.disableAttach { + params.disableAttach = true + params.registerDriver = true + } + + init(params) + + defer cleanup() + + sc, pvc, pod := createPod() + Expect(pod).NotTo(BeNil(), "while creating pod for resizing") + + Expect(*sc.AllowVolumeExpansion).To(BeTrue(), "failed creating sc with allowed expansion") + + err = framework.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) + framework.ExpectNoError(err, "Failed to start pod1: %v", err) + + By("Expanding current pvc") + newSize := resource.MustParse("6Gi") + pvc, err = expandPVCSize(pvc, newSize, m.cs) + Expect(err).NotTo(HaveOccurred(), "While updating pvc for more size") + Expect(pvc).NotTo(BeNil()) + + pvcSize := pvc.Spec.Resources.Requests[v1.ResourceStorage] + if pvcSize.Cmp(newSize) != 0 { + framework.Failf("error updating pvc size %q", pvc.Name) + } + + By("Waiting for persistent volume resize to finish") + err = waitForControllerVolumeResize(pvc, m.cs, csiResizeWaitPeriod) + Expect(err).NotTo(HaveOccurred(), "While waiting for PV resize to finish") + + By("Waiting for PVC resize to finish") + pvc, err = waitForFSResize(pvc, m.cs) + Expect(err).NotTo(HaveOccurred(), "while waiting for PVC to finish") + + pvcConditions := pvc.Status.Conditions + Expect(len(pvcConditions)).To(Equal(0), "pvc should not have conditions") + + }) + } + }) + }) func waitForMaxVolumeCondition(pod *v1.Pod, cs clientset.Interface) error { @@ -445,6 +659,49 @@ func startPausePod(cs clientset.Interface, t testsuites.StorageClassTest, node t return class, claim, pod } +func startPausePodWithClaim(cs clientset.Interface, pvc *v1.PersistentVolumeClaim, node testsuites.NodeSelection, ns string) (*v1.Pod, error) { + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "pvc-volume-tester-", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "volume-tester", + Image: imageutils.GetE2EImage(imageutils.Pause), + VolumeMounts: []v1.VolumeMount{ + { + Name: "my-volume", + MountPath: "/mnt/test", + }, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + Volumes: []v1.Volume{ + { + Name: "my-volume", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc.Name, + ReadOnly: false, + }, + }, + }, + }, + }, + } + + if node.Name != "" { + pod.Spec.NodeName = node.Name + } + if len(node.Selector) != 0 { + pod.Spec.NodeSelector = node.Selector + } + + return cs.CoreV1().Pods(ns).Create(pod) +} + // checkPodInfo tests that NodePublish was called with expected volume_context func checkPodInfo(cs clientset.Interface, namespace, driverPodName, driverContainerName string, pod *v1.Pod, expectPodInfo bool) error { expectedAttributes := map[string]string{ @@ -508,7 +765,7 @@ func checkPodInfo(cs clientset.Interface, namespace, driverPodName, driverContai } func waitForCSIDriver(cs clientset.Interface, driverName string) error { - timeout := 2 * time.Minute + timeout := 4 * time.Minute framework.Logf("waiting up to %v for CSIDriver %q", timeout, driverName) for start := time.Now(); time.Since(start) < timeout; time.Sleep(framework.Poll) { diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index fd090adce6..e579913cfd 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -169,36 +169,52 @@ func (h *hostpathCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.Per // mockCSI type mockCSIDriver struct { - driverInfo testsuites.DriverInfo - manifests []string - podInfo *bool - attachable bool - attachLimit int + driverInfo testsuites.DriverInfo + manifests []string + podInfo *bool + attachable bool + attachLimit int + enableNodeExpansion bool +} + +// CSIMockDriverOpts defines options used for csi driver +type CSIMockDriverOpts struct { + RegisterDriver bool + DisableAttach bool + PodInfo *bool + AttachLimit int + EnableResizing bool + EnableNodeExpansion bool } var _ testsuites.TestDriver = &mockCSIDriver{} var _ testsuites.DynamicPVTestDriver = &mockCSIDriver{} // InitMockCSIDriver returns a mockCSIDriver that implements TestDriver interface -func InitMockCSIDriver(registerDriver, driverAttachable bool, podInfo *bool, attachLimit int) testsuites.TestDriver { +func InitMockCSIDriver(driverOpts CSIMockDriverOpts) testsuites.TestDriver { driverManifests := []string{ "test/e2e/testing-manifests/storage-csi/cluster-driver-registrar/rbac.yaml", "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml", "test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml", "test/e2e/testing-manifests/storage-csi/mock/csi-storageclass.yaml", "test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml", } - if registerDriver { + if driverOpts.RegisterDriver { driverManifests = append(driverManifests, "test/e2e/testing-manifests/storage-csi/mock/csi-mock-cluster-driver-registrar.yaml") } - if driverAttachable { + if !driverOpts.DisableAttach { driverManifests = append(driverManifests, "test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver-attacher.yaml") } + if driverOpts.EnableResizing { + driverManifests = append(driverManifests, "test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver-resizer.yaml") + } + return &mockCSIDriver{ driverInfo: testsuites.DriverInfo{ Name: "csi-mock", @@ -213,10 +229,11 @@ func InitMockCSIDriver(registerDriver, driverAttachable bool, podInfo *bool, att testsuites.CapExec: false, }, }, - manifests: driverManifests, - podInfo: podInfo, - attachable: driverAttachable, - attachLimit: attachLimit, + manifests: driverManifests, + podInfo: driverOpts.PodInfo, + attachable: !driverOpts.DisableAttach, + attachLimit: driverOpts.AttachLimit, + enableNodeExpansion: driverOpts.EnableNodeExpansion, } } @@ -264,6 +281,10 @@ func (m *mockCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTest containerArgs = append(containerArgs, "--attach-limit", strconv.Itoa(m.attachLimit)) } + if m.enableNodeExpansion { + containerArgs = append(containerArgs, "--node-expand-required=true") + } + // TODO (?): the storage.csi.image.version and storage.csi.image.registry // settings are ignored for this test. We could patch the image definitions. o := utils.PatchCSIOptions{ diff --git a/test/e2e/storage/flexvolume_mounted_volume_resize.go b/test/e2e/storage/flexvolume_mounted_volume_resize.go index 0d95e127fd..92791c1b78 100644 --- a/test/e2e/storage/flexvolume_mounted_volume_resize.go +++ b/test/e2e/storage/flexvolume_mounted_volume_resize.go @@ -159,7 +159,7 @@ var _ = utils.SIGDescribe("Mounted flexvolume expand[Slow]", func() { } By("Waiting for cloudprovider resize to finish") - err = waitForControllerVolumeResize(pvc, c) + err = waitForControllerVolumeResize(pvc, c, totalResizeWaitPeriod) Expect(err).NotTo(HaveOccurred(), "While waiting for pvc resize to finish") By("Getting a pod from deployment") diff --git a/test/e2e/storage/flexvolume_online_resize.go b/test/e2e/storage/flexvolume_online_resize.go index f51e078b97..da88966cf1 100644 --- a/test/e2e/storage/flexvolume_online_resize.go +++ b/test/e2e/storage/flexvolume_online_resize.go @@ -164,7 +164,7 @@ var _ = utils.SIGDescribe("Mounted flexvolume volume expand [Slow] [Feature:Expa } By("Waiting for cloudprovider resize to finish") - err = waitForControllerVolumeResize(pvc, c) + err = waitForControllerVolumeResize(pvc, c, totalResizeWaitPeriod) Expect(err).NotTo(HaveOccurred(), "While waiting for pvc resize to finish") By("Waiting for file system resize to finish") diff --git a/test/e2e/storage/mounted_volume_resize.go b/test/e2e/storage/mounted_volume_resize.go index 15f0e041a6..20ba0a64ed 100644 --- a/test/e2e/storage/mounted_volume_resize.go +++ b/test/e2e/storage/mounted_volume_resize.go @@ -136,7 +136,7 @@ var _ = utils.SIGDescribe("Mounted volume expand", func() { } By("Waiting for cloudprovider resize to finish") - err = waitForControllerVolumeResize(pvc, c) + err = waitForControllerVolumeResize(pvc, c, totalResizeWaitPeriod) Expect(err).NotTo(HaveOccurred(), "While waiting for pvc resize to finish") By("Getting a pod from deployment") diff --git a/test/e2e/storage/volume_expand.go b/test/e2e/storage/volume_expand.go index 5d9486d4a9..b67901b6f9 100644 --- a/test/e2e/storage/volume_expand.go +++ b/test/e2e/storage/volume_expand.go @@ -136,7 +136,7 @@ var _ = utils.SIGDescribe("Volume expand", func() { } By("Waiting for cloudprovider resize to finish") - err = waitForControllerVolumeResize(pvc, c) + err = waitForControllerVolumeResize(pvc, c, totalResizeWaitPeriod) Expect(err).NotTo(HaveOccurred(), "While waiting for pvc resize to finish") By("Checking for conditions on pvc") @@ -198,9 +198,29 @@ func expandPVCSize(origPVC *v1.PersistentVolumeClaim, size resource.Quantity, c return updatedPVC, waitErr } -func waitForControllerVolumeResize(pvc *v1.PersistentVolumeClaim, c clientset.Interface) error { +func waitForResizingCondition(pvc *v1.PersistentVolumeClaim, c clientset.Interface, duration time.Duration) error { + waitErr := wait.PollImmediate(resizePollInterval, duration, func() (bool, error) { + var err error + updatedPVC, err := c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(pvc.Name, metav1.GetOptions{}) + + if err != nil { + return false, fmt.Errorf("error fetching pvc %q for checking for resize status : %v", pvc.Name, err) + } + + pvcConditions := updatedPVC.Status.Conditions + if len(pvcConditions) > 0 { + if pvcConditions[0].Type == v1.PersistentVolumeClaimResizing { + return true, nil + } + } + return false, nil + }) + return waitErr +} + +func waitForControllerVolumeResize(pvc *v1.PersistentVolumeClaim, c clientset.Interface, duration time.Duration) error { pvName := pvc.Spec.VolumeName - return wait.PollImmediate(resizePollInterval, totalResizeWaitPeriod, func() (bool, error) { + return wait.PollImmediate(resizePollInterval, duration, func() (bool, error) { pvcSize := pvc.Spec.Resources.Requests[v1.ResourceStorage] pv, err := c.CoreV1().PersistentVolumes().Get(pvName, metav1.GetOptions{}) diff --git a/test/e2e/testing-manifests/storage-csi/external-resizer/README.md b/test/e2e/testing-manifests/storage-csi/external-resizer/README.md new file mode 100644 index 0000000000..f7029912ef --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/external-resizer/README.md @@ -0,0 +1 @@ +The original file is (or will be) https://github.com/kubernetes-csi/external-resizer/blob/master/deploy/kubernetes/rbac.yaml diff --git a/test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml b/test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml new file mode 100644 index 0000000000..82b5640b5e --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml @@ -0,0 +1,90 @@ +# This YAML file contains all RBAC objects that are necessary to run external +# CSI resizer. +# +# In production, each CSI driver deployment has to be customized: +# - to avoid conflicts, use non-default namespace and different names +# for non-namespaced entities like the ClusterRole +# - decide whether the deployment replicates the external CSI +# resizer, in which case leadership election must be enabled; +# this influences the RBAC setup, see below + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-resizer + # replace with non-default namespace name + namespace: default + +--- +# Resizer must be able to work with PVCs, PVs, SCs. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: external-resizer-runner +rules: + # The following rule should be uncommented for plugins that require secrets + # for provisioning. + # - apiGroups: [""] + # resources: ["secrets"] + # verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-resizer-role +subjects: + - kind: ServiceAccount + name: csi-resizer + # replace with non-default namespace name + namespace: default +roleRef: + kind: ClusterRole + name: external-resizer-runner + apiGroup: rbac.authorization.k8s.io + +--- +# Resizer must be able to work with end point in current namespace +# if (and only if) leadership election is enabled +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + # replace with non-default namespace name + namespace: default + name: external-resizer-cfg +rules: +- apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-resizer-role-cfg + # replace with non-default namespace name + namespace: default +subjects: + - kind: ServiceAccount + name: csi-resizer + # replace with non-default namespace name + namespace: default +roleRef: + kind: Role + name: external-resizer-cfg + apiGroup: rbac.authorization.k8s.io diff --git a/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver-resizer.yaml b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver-resizer.yaml new file mode 100644 index 0000000000..0ea5ce6933 --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver-resizer.yaml @@ -0,0 +1,33 @@ +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-mockplugin-resizer +spec: + selector: + matchLabels: + app: csi-mockplugin-resizer + replicas: 1 + template: + metadata: + labels: + app: csi-mockplugin-resizer + spec: + serviceAccountName: csi-mock + containers: + - name: csi-resizer + image: quay.io/k8scsi/csi-resizer:canary + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: Always + volumeMounts: + - mountPath: /csi + name: socket-dir + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/csi-mock + type: DirectoryOrCreate + name: socket-dir diff --git a/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml index e3fcb9f44a..1f1d649428 100644 --- a/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml +++ b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml @@ -54,7 +54,7 @@ spec: - name: mock - image: quay.io/k8scsi/mock-driver:v1.0.0-1 + image: quay.io/k8scsi/mock-driver:v1.1.1 env: - name: CSI_ENDPOINT value: /csi/csi.sock diff --git a/test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml index dff014b6f2..8936f977d0 100644 --- a/test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml @@ -59,3 +59,16 @@ roleRef: kind: ClusterRole name: e2e-test-privileged-psp apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-controller-resizer-role +subjects: + - kind: ServiceAccount + name: csi-mock + namespace: default +roleRef: + kind: ClusterRole + name: external-resizer-runner + apiGroup: rbac.authorization.k8s.io