From 8b94b9625bb7bbf0a1d9f8c03f715a4aa76ded1a Mon Sep 17 00:00:00 2001 From: Yecheng Fu Date: Tue, 8 Jan 2019 01:18:34 +0800 Subject: [PATCH] Make volume binder resilient to races: unit tests --- .../volume/persistentvolume/framework_test.go | 98 +++- .../persistentvolume/scheduler_binder_test.go | 510 ++++++++++++------ 2 files changed, 433 insertions(+), 175 deletions(-) diff --git a/pkg/controller/volume/persistentvolume/framework_test.go b/pkg/controller/volume/persistentvolume/framework_test.go index ef632cd528..7552a2cb44 100644 --- a/pkg/controller/volume/persistentvolume/framework_test.go +++ b/pkg/controller/volume/persistentvolume/framework_test.go @@ -29,11 +29,13 @@ import ( "k8s.io/klog" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/diff" "k8s.io/apimachinery/pkg/util/wait" @@ -136,6 +138,7 @@ type volumeReactor struct { fakeClaimWatch *watch.FakeWatcher lock sync.Mutex errors []reactorError + watchers map[schema.GroupVersionResource]map[string][]*watch.RaceFreeFakeWatcher } // reactorError is an error that is returned by test reactor (=simulated @@ -189,11 +192,34 @@ func (r *volumeReactor) React(action core.Action) (handled bool, ret runtime.Obj // Store the updated object to appropriate places. r.volumes[volume.Name] = volume + for _, w := range r.getWatches(action.GetResource(), action.GetNamespace()) { + w.Add(volume) + } r.changedObjects = append(r.changedObjects, volume) r.changedSinceLastSync++ klog.V(4).Infof("created volume %s", volume.Name) return true, volume, nil + case action.Matches("create", "persistentvolumeclaims"): + obj := action.(core.UpdateAction).GetObject() + claim := obj.(*v1.PersistentVolumeClaim) + + // check the claim does not exist + _, found := r.claims[claim.Name] + if found { + return true, nil, fmt.Errorf("Cannot create claim %s: claim already exists", claim.Name) + } + + // Store the updated object to appropriate places. + r.claims[claim.Name] = claim + for _, w := range r.getWatches(action.GetResource(), action.GetNamespace()) { + w.Add(claim) + } + r.changedObjects = append(r.changedObjects, claim) + r.changedSinceLastSync++ + klog.V(4).Infof("created claim %s", claim.Name) + return true, claim, nil + case action.Matches("update", "persistentvolumes"): obj := action.(core.UpdateAction).GetObject() volume := obj.(*v1.PersistentVolume) @@ -206,6 +232,10 @@ func (r *volumeReactor) React(action core.Action) (handled bool, ret runtime.Obj if storedVer != requestedVer { return true, obj, versionConflictError } + if reflect.DeepEqual(storedVolume, volume) { + klog.V(4).Infof("nothing updated volume %s", volume.Name) + return true, volume, nil + } // Don't modify the existing object volume = volume.DeepCopy() volume.ResourceVersion = strconv.Itoa(storedVer + 1) @@ -214,6 +244,9 @@ func (r *volumeReactor) React(action core.Action) (handled bool, ret runtime.Obj } // Store the updated object to appropriate places. + for _, w := range r.getWatches(action.GetResource(), action.GetNamespace()) { + w.Modify(volume) + } r.volumes[volume.Name] = volume r.changedObjects = append(r.changedObjects, volume) r.changedSinceLastSync++ @@ -232,6 +265,10 @@ func (r *volumeReactor) React(action core.Action) (handled bool, ret runtime.Obj if storedVer != requestedVer { return true, obj, versionConflictError } + if reflect.DeepEqual(storedClaim, claim) { + klog.V(4).Infof("nothing updated claim %s", claim.Name) + return true, claim, nil + } // Don't modify the existing object claim = claim.DeepCopy() claim.ResourceVersion = strconv.Itoa(storedVer + 1) @@ -240,6 +277,9 @@ func (r *volumeReactor) React(action core.Action) (handled bool, ret runtime.Obj } // Store the updated object to appropriate places. + for _, w := range r.getWatches(action.GetResource(), action.GetNamespace()) { + w.Modify(claim) + } r.claims[claim.Name] = claim r.changedObjects = append(r.changedObjects, claim) r.changedSinceLastSync++ @@ -251,18 +291,32 @@ func (r *volumeReactor) React(action core.Action) (handled bool, ret runtime.Obj volume, found := r.volumes[name] if found { klog.V(4).Infof("GetVolume: found %s", volume.Name) - return true, volume, nil + return true, volume.DeepCopy(), nil } else { klog.V(4).Infof("GetVolume: volume %s not found", name) return true, nil, fmt.Errorf("Cannot find volume %s", name) } + case action.Matches("get", "persistentvolumeclaims"): + name := action.(core.GetAction).GetName() + claim, found := r.claims[name] + if found { + klog.V(4).Infof("GetClaim: found %s", claim.Name) + return true, claim.DeepCopy(), nil + } else { + klog.V(4).Infof("GetClaim: claim %s not found", name) + return true, nil, apierrs.NewNotFound(action.GetResource().GroupResource(), name) + } + case action.Matches("delete", "persistentvolumes"): name := action.(core.DeleteAction).GetName() klog.V(4).Infof("deleted volume %s", name) - _, found := r.volumes[name] + obj, found := r.volumes[name] if found { delete(r.volumes, name) + for _, w := range r.getWatches(action.GetResource(), action.GetNamespace()) { + w.Delete(obj) + } r.changedSinceLastSync++ return true, nil, nil } else { @@ -272,9 +326,12 @@ func (r *volumeReactor) React(action core.Action) (handled bool, ret runtime.Obj case action.Matches("delete", "persistentvolumeclaims"): name := action.(core.DeleteAction).GetName() klog.V(4).Infof("deleted claim %s", name) - _, found := r.volumes[name] + obj, found := r.claims[name] if found { delete(r.claims, name) + for _, w := range r.getWatches(action.GetResource(), action.GetNamespace()) { + w.Delete(obj) + } r.changedSinceLastSync++ return true, nil, nil } else { @@ -285,6 +342,36 @@ func (r *volumeReactor) React(action core.Action) (handled bool, ret runtime.Obj return false, nil, nil } +// Watch watches objects from the volumeReactor. Watch returns a channel which +// will push added / modified / deleted object. +func (r *volumeReactor) Watch(gvr schema.GroupVersionResource, ns string) (watch.Interface, error) { + r.lock.Lock() + defer r.lock.Unlock() + + fakewatcher := watch.NewRaceFreeFake() + + if _, exists := r.watchers[gvr]; !exists { + r.watchers[gvr] = make(map[string][]*watch.RaceFreeFakeWatcher) + } + r.watchers[gvr][ns] = append(r.watchers[gvr][ns], fakewatcher) + return fakewatcher, nil +} + +func (r *volumeReactor) getWatches(gvr schema.GroupVersionResource, ns string) []*watch.RaceFreeFakeWatcher { + watches := []*watch.RaceFreeFakeWatcher{} + if r.watchers[gvr] != nil { + if w := r.watchers[gvr][ns]; w != nil { + watches = append(watches, w...) + } + if ns != metav1.NamespaceAll { + if w := r.watchers[gvr][metav1.NamespaceAll]; w != nil { + watches = append(watches, w...) + } + } + } + return watches +} + // injectReactError returns an error when the test requested given action to // fail. nil is returned otherwise. func (r *volumeReactor) injectReactError(action core.Action) error { @@ -596,11 +683,14 @@ func newVolumeReactor(client *fake.Clientset, ctrl *PersistentVolumeController, fakeVolumeWatch: fakeVolumeWatch, fakeClaimWatch: fakeClaimWatch, errors: errors, + watchers: make(map[schema.GroupVersionResource]map[string][]*watch.RaceFreeFakeWatcher), } client.AddReactor("create", "persistentvolumes", reactor.React) + client.AddReactor("create", "persistentvolumeclaims", reactor.React) client.AddReactor("update", "persistentvolumes", reactor.React) client.AddReactor("update", "persistentvolumeclaims", reactor.React) client.AddReactor("get", "persistentvolumes", reactor.React) + client.AddReactor("get", "persistentvolumeclaims", reactor.React) client.AddReactor("delete", "persistentvolumes", reactor.React) client.AddReactor("delete", "persistentvolumeclaims", reactor.React) diff --git a/pkg/controller/volume/persistentvolume/scheduler_binder_test.go b/pkg/controller/volume/persistentvolume/scheduler_binder_test.go index 789e464fc1..a3792ecec7 100644 --- a/pkg/controller/volume/persistentvolume/scheduler_binder_test.go +++ b/pkg/controller/volume/persistentvolume/scheduler_binder_test.go @@ -17,6 +17,7 @@ limitations under the License. package persistentvolume import ( + "context" "fmt" "reflect" "testing" @@ -28,10 +29,13 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/diff" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/informers" coreinformers "k8s.io/client-go/informers/core/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" "k8s.io/klog" "k8s.io/kubernetes/pkg/api/testapi" "k8s.io/kubernetes/pkg/controller" @@ -75,14 +79,6 @@ var ( pvBoundImmediate = makeTestPV("pv-bound-immediate", "node1", "1G", "1", immediateBoundPVC, immediateClass) pvBoundImmediateNode2 = makeTestPV("pv-bound-immediate", "node2", "1G", "1", immediateBoundPVC, immediateClass) - // PVC/PV bindings for manual binding - binding1a = makeBinding(unboundPVC, pvNode1a) - binding1b = makeBinding(unboundPVC2, pvNode1b) - bindingNoNode = makeBinding(unboundPVC, pvNoNode) - bindingBad = makeBinding(badPVC, pvNode1b) - binding1aBound = makeBinding(unboundPVC, pvNode1aBound) - binding1bBound = makeBinding(unboundPVC2, pvNode1bBound) - // storage class names waitClass = "waitClass" immediateClass = "immediateClass" @@ -109,15 +105,24 @@ type testEnv struct { internalPVCCache *pvcAssumeCache } -func newTestBinder(t *testing.T) *testEnv { +func newTestBinder(t *testing.T, stopCh <-chan struct{}) *testEnv { client := &fake.Clientset{} reactor := newVolumeReactor(client, nil, nil, nil, nil) + // TODO refactor all tests to use real watch mechanism, see #72327 + client.AddWatchReactor("*", func(action k8stesting.Action) (handled bool, ret watch.Interface, err error) { + gvr := action.GetResource() + ns := action.GetNamespace() + watch, err := reactor.Watch(gvr, ns) + if err != nil { + return false, nil, err + } + return true, watch, nil + }) informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc()) nodeInformer := informerFactory.Core().V1().Nodes() pvcInformer := informerFactory.Core().V1().PersistentVolumeClaims() classInformer := informerFactory.Storage().V1().StorageClasses() - binder := NewVolumeBinder( client, nodeInformer, @@ -126,6 +131,14 @@ func newTestBinder(t *testing.T) *testEnv { classInformer, 10*time.Second) + // Wait for informers cache sync + informerFactory.Start(stopCh) + for v, synced := range informerFactory.WaitForCacheSync(stopCh) { + if !synced { + klog.Fatalf("Error syncing informer for %v", v) + } + } + // Add storageclasses waitMode := storagev1.VolumeBindingWaitForFirstConsumer immediateMode := storagev1.VolumeBindingImmediate @@ -247,6 +260,66 @@ func (env *testEnv) initVolumes(cachedPVs []*v1.PersistentVolume, apiPVs []*v1.P } +func (env *testEnv) updateVolumes(t *testing.T, pvs []*v1.PersistentVolume, waitCache bool) { + for _, pv := range pvs { + if _, err := env.client.CoreV1().PersistentVolumes().Update(pv); err != nil { + t.Fatalf("failed to update PV %q", pv.Name) + } + } + if waitCache { + wait.Poll(100*time.Millisecond, 3*time.Second, func() (bool, error) { + for _, pv := range pvs { + obj, err := env.internalPVCache.GetAPIObj(pv.Name) + if obj == nil || err != nil { + return false, nil + } + pvInCache, ok := obj.(*v1.PersistentVolume) + if !ok { + return false, fmt.Errorf("PV %s invalid object", pvInCache.Name) + } + return versioner.CompareResourceVersion(pvInCache, pv) == 0, nil + } + return true, nil + }) + } +} + +func (env *testEnv) updateClaims(t *testing.T, pvcs []*v1.PersistentVolumeClaim, waitCache bool) { + for _, pvc := range pvcs { + if _, err := env.client.CoreV1().PersistentVolumeClaims(pvc.Namespace).Update(pvc); err != nil { + t.Fatalf("failed to update PVC %q", getPVCName(pvc)) + } + } + if waitCache { + wait.Poll(100*time.Millisecond, 3*time.Second, func() (bool, error) { + for _, pvc := range pvcs { + obj, err := env.internalPVCCache.GetAPIObj(getPVCName(pvc)) + if obj == nil || err != nil { + return false, nil + } + pvcInCache, ok := obj.(*v1.PersistentVolumeClaim) + if !ok { + return false, fmt.Errorf("PVC %s invalid object", pvcInCache.Name) + } + return versioner.CompareResourceVersion(pvcInCache, pvc) == 0, nil + } + return true, nil + }) + } +} + +func (env *testEnv) deleteVolumes(pvs []*v1.PersistentVolume) { + for _, pv := range pvs { + env.internalPVCache.delete(pv) + } +} + +func (env *testEnv) deleteClaims(pvcs []*v1.PersistentVolumeClaim) { + for _, pvc := range pvcs { + env.internalPVCCache.delete(pvc) + } +} + func (env *testEnv) assumeVolumes(t *testing.T, name, node string, pod *v1.Pod, bindings []*bindingInfo, provisionings []*v1.PersistentVolumeClaim) { pvCache := env.internalBinder.pvCache for _, binding := range bindings { @@ -540,7 +613,7 @@ func makeTestPV(name, node, capacity, version string, boundToPVC *v1.PersistentV func pvcSetSelectedNode(pvc *v1.PersistentVolumeClaim, node string) *v1.PersistentVolumeClaim { newPVC := pvc.DeepCopy() - metav1.SetMetaDataAnnotation(&pvc.ObjectMeta, annSelectedNode, node) + metav1.SetMetaDataAnnotation(&newPVC.ObjectMeta, annSelectedNode, node) return newPVC } @@ -676,7 +749,7 @@ func TestFindPodVolumesWithoutProvisioning(t *testing.T) { "unbound-pvc,pv-same-node": { podPVCs: []*v1.PersistentVolumeClaim{unboundPVC}, pvs: []*v1.PersistentVolume{pvNode2, pvNode1a, pvNode1b}, - expectedBindings: []*bindingInfo{binding1a}, + expectedBindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1a)}, expectedUnbound: true, expectedBound: true, }, @@ -689,28 +762,28 @@ func TestFindPodVolumesWithoutProvisioning(t *testing.T) { "two-unbound-pvcs": { podPVCs: []*v1.PersistentVolumeClaim{unboundPVC, unboundPVC2}, pvs: []*v1.PersistentVolume{pvNode1a, pvNode1b}, - expectedBindings: []*bindingInfo{binding1a, binding1b}, + expectedBindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1a), makeBinding(unboundPVC2, pvNode1b)}, expectedUnbound: true, expectedBound: true, }, "two-unbound-pvcs,order-by-size": { podPVCs: []*v1.PersistentVolumeClaim{unboundPVC2, unboundPVC}, pvs: []*v1.PersistentVolume{pvNode1a, pvNode1b}, - expectedBindings: []*bindingInfo{binding1a, binding1b}, + expectedBindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1a), makeBinding(unboundPVC2, pvNode1b)}, expectedUnbound: true, expectedBound: true, }, "two-unbound-pvcs,partial-match": { podPVCs: []*v1.PersistentVolumeClaim{unboundPVC, unboundPVC2}, pvs: []*v1.PersistentVolume{pvNode1a}, - expectedBindings: []*bindingInfo{binding1a}, + expectedBindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1a)}, expectedUnbound: false, expectedBound: true, }, "one-bound,one-unbound": { podPVCs: []*v1.PersistentVolumeClaim{unboundPVC, boundPVC}, pvs: []*v1.PersistentVolume{pvBound, pvNode1a}, - expectedBindings: []*bindingInfo{binding1a}, + expectedBindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1a)}, expectedUnbound: true, expectedBound: true, }, @@ -767,11 +840,14 @@ func TestFindPodVolumesWithoutProvisioning(t *testing.T) { }, } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + for name, scenario := range scenarios { klog.V(5).Infof("Running test case %q", name) // Setup - testEnv := newTestBinder(t) + testEnv := newTestBinder(t, ctx.Done()) testEnv.initVolumes(scenario.pvs, scenario.pvs) // a. Init pvc cache @@ -833,7 +909,7 @@ func TestFindPodVolumesWithProvisioning(t *testing.T) { "two-unbound-pvcs,one-matched,one-provisioned": { podPVCs: []*v1.PersistentVolumeClaim{unboundPVC, provisionedPVC}, pvs: []*v1.PersistentVolume{pvNode1a}, - expectedBindings: []*bindingInfo{binding1a}, + expectedBindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1a)}, expectedProvisions: []*v1.PersistentVolumeClaim{provisionedPVC}, expectedUnbound: true, expectedBound: true, @@ -845,6 +921,13 @@ func TestFindPodVolumesWithProvisioning(t *testing.T) { expectedUnbound: true, expectedBound: true, }, + "one-binding,one-selected-node": { + podPVCs: []*v1.PersistentVolumeClaim{boundPVC, selectedNodePVC}, + pvs: []*v1.PersistentVolume{pvBound}, + expectedProvisions: []*v1.PersistentVolumeClaim{selectedNodePVC}, + expectedUnbound: true, + expectedBound: true, + }, "immediate-unbound-pvc": { podPVCs: []*v1.PersistentVolumeClaim{immediateUnboundPVC}, expectedUnbound: false, @@ -879,9 +962,12 @@ func TestFindPodVolumesWithProvisioning(t *testing.T) { }, } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + for name, scenario := range scenarios { // Setup - testEnv := newTestBinder(t) + testEnv := newTestBinder(t, ctx.Done()) testEnv.initVolumes(scenario.pvs, scenario.pvs) // a. Init pvc cache @@ -937,59 +1023,62 @@ func TestAssumePodVolumes(t *testing.T) { }, "one-binding": { podPVCs: []*v1.PersistentVolumeClaim{unboundPVC}, - bindings: []*bindingInfo{binding1a}, + bindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1a)}, pvs: []*v1.PersistentVolume{pvNode1a}, - expectedBindings: []*bindingInfo{binding1aBound}, + expectedBindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)}, expectedProvisionings: []*v1.PersistentVolumeClaim{}, }, "two-bindings": { podPVCs: []*v1.PersistentVolumeClaim{unboundPVC, unboundPVC2}, - bindings: []*bindingInfo{binding1a, binding1b}, + bindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1a), makeBinding(unboundPVC2, pvNode1b)}, pvs: []*v1.PersistentVolume{pvNode1a, pvNode1b}, - expectedBindings: []*bindingInfo{binding1aBound, binding1bBound}, + expectedBindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound), makeBinding(unboundPVC2, pvNode1bBound)}, expectedProvisionings: []*v1.PersistentVolumeClaim{}, }, "pv-already-bound": { podPVCs: []*v1.PersistentVolumeClaim{unboundPVC}, - bindings: []*bindingInfo{binding1aBound}, + bindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)}, pvs: []*v1.PersistentVolume{pvNode1aBound}, - expectedBindings: []*bindingInfo{binding1aBound}, + expectedBindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)}, expectedProvisionings: []*v1.PersistentVolumeClaim{}, }, "claimref-failed": { podPVCs: []*v1.PersistentVolumeClaim{unboundPVC}, - bindings: []*bindingInfo{binding1a, bindingBad}, + bindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1a), makeBinding(badPVC, pvNode1b)}, pvs: []*v1.PersistentVolume{pvNode1a, pvNode1b}, shouldFail: true, }, "tmpupdate-failed": { podPVCs: []*v1.PersistentVolumeClaim{unboundPVC}, - bindings: []*bindingInfo{binding1a, binding1b}, + bindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1a), makeBinding(unboundPVC2, pvNode1b)}, pvs: []*v1.PersistentVolume{pvNode1a}, shouldFail: true, }, "one-binding, one-pvc-provisioned": { podPVCs: []*v1.PersistentVolumeClaim{unboundPVC, provisionedPVC}, - bindings: []*bindingInfo{binding1a}, + bindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1a)}, pvs: []*v1.PersistentVolume{pvNode1a}, provisionedPVCs: []*v1.PersistentVolumeClaim{provisionedPVC}, - expectedBindings: []*bindingInfo{binding1aBound}, + expectedBindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)}, expectedProvisionings: []*v1.PersistentVolumeClaim{selectedNodePVC}, }, "one-binding, one-provision-tmpupdate-failed": { podPVCs: []*v1.PersistentVolumeClaim{unboundPVC, provisionedPVCHigherVersion}, - bindings: []*bindingInfo{binding1a}, + bindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1a)}, pvs: []*v1.PersistentVolume{pvNode1a}, provisionedPVCs: []*v1.PersistentVolumeClaim{provisionedPVC2}, shouldFail: true, }, } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + for name, scenario := range scenarios { klog.V(5).Infof("Running test case %q", name) // Setup - testEnv := newTestBinder(t) + testEnv := newTestBinder(t, ctx.Done()) testEnv.initClaims(scenario.podPVCs, scenario.podPVCs) pod := makePod(scenario.podPVCs) testEnv.initPodCache(pod, "node1", scenario.bindings, scenario.provisionedPVCs) @@ -1062,25 +1151,25 @@ func TestBindAPIUpdate(t *testing.T) { provisionedPVCs: []*v1.PersistentVolumeClaim{}, }, "one-binding": { - bindings: []*bindingInfo{binding1aBound}, + bindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)}, cachedPVs: []*v1.PersistentVolume{pvNode1a}, expectedPVs: []*v1.PersistentVolume{pvNode1aBound}, provisionedPVCs: []*v1.PersistentVolumeClaim{}, }, "two-bindings": { - bindings: []*bindingInfo{binding1aBound, binding1bBound}, + bindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound), makeBinding(unboundPVC2, pvNode1bBound)}, cachedPVs: []*v1.PersistentVolume{pvNode1a, pvNode1b}, expectedPVs: []*v1.PersistentVolume{pvNode1aBound, pvNode1bBound}, provisionedPVCs: []*v1.PersistentVolumeClaim{}, }, "api-already-updated": { - bindings: []*bindingInfo{binding1aBound}, + bindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)}, cachedPVs: []*v1.PersistentVolume{pvNode1aBound}, expectedPVs: []*v1.PersistentVolume{pvNode1aBound}, provisionedPVCs: []*v1.PersistentVolumeClaim{}, }, "api-update-failed": { - bindings: []*bindingInfo{binding1aBound, binding1bBound}, + bindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound), makeBinding(unboundPVC2, pvNode1bBound)}, cachedPVs: []*v1.PersistentVolume{pvNode1a, pvNode1b}, apiPVs: []*v1.PersistentVolume{pvNode1a, pvNode1bBoundHigherVersion}, expectedPVs: []*v1.PersistentVolume{pvNode1aBound, pvNode1b}, @@ -1104,7 +1193,7 @@ func TestBindAPIUpdate(t *testing.T) { shouldFail: true, }, "binding-succeed, provision-api-update-failed": { - bindings: []*bindingInfo{binding1aBound}, + bindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)}, cachedPVs: []*v1.PersistentVolume{pvNode1a}, expectedPVs: []*v1.PersistentVolume{pvNode1aBound}, provisionedPVCs: []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVC), addProvisionAnn(provisionedPVC2)}, @@ -1115,11 +1204,15 @@ func TestBindAPIUpdate(t *testing.T) { shouldFail: true, }, } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + for name, scenario := range scenarios { klog.V(4).Infof("Running test case %q", name) // Setup - testEnv := newTestBinder(t) + testEnv := newTestBinder(t, ctx.Done()) pod := makePod(nil) if scenario.apiPVs == nil { scenario.apiPVs = scenario.cachedPVs @@ -1155,11 +1248,19 @@ func TestBindAPIUpdate(t *testing.T) { func TestCheckBindings(t *testing.T) { scenarios := map[string]struct { // Inputs - bindings []*bindingInfo - cachedPVs []*v1.PersistentVolume + initPVs []*v1.PersistentVolume + initPVCs []*v1.PersistentVolumeClaim + bindings []*bindingInfo provisionedPVCs []*v1.PersistentVolumeClaim - cachedPVCs []*v1.PersistentVolumeClaim + + // api updates before checking + apiPVs []*v1.PersistentVolume + apiPVCs []*v1.PersistentVolumeClaim + + // delete objects before checking + deletePVs bool + deletePVCs bool // Expected return values shouldFail bool @@ -1182,108 +1283,144 @@ func TestCheckBindings(t *testing.T) { expectedBound: true, }, "binding-bound": { - bindings: []*bindingInfo{binding1aBound}, + bindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)}, provisionedPVCs: []*v1.PersistentVolumeClaim{}, - cachedPVs: []*v1.PersistentVolume{pvNode1aBound}, - cachedPVCs: []*v1.PersistentVolumeClaim{boundPVCNode1a}, + initPVs: []*v1.PersistentVolume{pvNode1aBound}, + initPVCs: []*v1.PersistentVolumeClaim{boundPVCNode1a}, expectedBound: true, }, "binding-prebound": { - bindings: []*bindingInfo{binding1aBound}, + bindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)}, provisionedPVCs: []*v1.PersistentVolumeClaim{}, - cachedPVs: []*v1.PersistentVolume{pvNode1aBound}, - cachedPVCs: []*v1.PersistentVolumeClaim{preboundPVCNode1a}, + initPVs: []*v1.PersistentVolume{pvNode1aBound}, + initPVCs: []*v1.PersistentVolumeClaim{preboundPVCNode1a}, }, "binding-unbound": { - bindings: []*bindingInfo{binding1aBound}, + bindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)}, provisionedPVCs: []*v1.PersistentVolumeClaim{}, - cachedPVs: []*v1.PersistentVolume{pvNode1aBound}, - cachedPVCs: []*v1.PersistentVolumeClaim{unboundPVC}, + initPVs: []*v1.PersistentVolume{pvNode1aBound}, + initPVCs: []*v1.PersistentVolumeClaim{unboundPVC}, }, "binding-pvc-not-exists": { - bindings: []*bindingInfo{binding1aBound}, + bindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)}, provisionedPVCs: []*v1.PersistentVolumeClaim{}, - cachedPVs: []*v1.PersistentVolume{pvNode1aBound}, + initPVs: []*v1.PersistentVolume{pvNode1aBound}, shouldFail: true, }, "binding-pv-not-exists": { - bindings: []*bindingInfo{binding1aBound}, + bindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)}, provisionedPVCs: []*v1.PersistentVolumeClaim{}, - cachedPVCs: []*v1.PersistentVolumeClaim{boundPVCNode1a}, + initPVs: []*v1.PersistentVolume{pvNode1aBound}, + initPVCs: []*v1.PersistentVolumeClaim{boundPVCNode1a}, + deletePVs: true, shouldFail: true, }, "binding-claimref-nil": { - bindings: []*bindingInfo{binding1aBound}, + bindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)}, provisionedPVCs: []*v1.PersistentVolumeClaim{}, - cachedPVs: []*v1.PersistentVolume{pvNode1a}, - cachedPVCs: []*v1.PersistentVolumeClaim{boundPVCNode1a}, + initPVs: []*v1.PersistentVolume{pvNode1a}, + initPVCs: []*v1.PersistentVolumeClaim{boundPVCNode1a}, + apiPVs: []*v1.PersistentVolume{pvNode1a}, + apiPVCs: []*v1.PersistentVolumeClaim{boundPVCNode1a}, shouldFail: true, }, "binding-claimref-uid-empty": { - bindings: []*bindingInfo{binding1aBound}, + bindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)}, provisionedPVCs: []*v1.PersistentVolumeClaim{}, - cachedPVs: []*v1.PersistentVolume{pvRemoveClaimUID(pvNode1aBound)}, - cachedPVCs: []*v1.PersistentVolumeClaim{boundPVCNode1a}, + initPVs: []*v1.PersistentVolume{pvNode1aBound}, + initPVCs: []*v1.PersistentVolumeClaim{boundPVCNode1a}, + apiPVs: []*v1.PersistentVolume{pvRemoveClaimUID(pvNode1aBound)}, + apiPVCs: []*v1.PersistentVolumeClaim{boundPVCNode1a}, shouldFail: true, }, "binding-one-bound,one-unbound": { - bindings: []*bindingInfo{binding1aBound, binding1bBound}, + bindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound), makeBinding(unboundPVC2, pvNode1bBound)}, provisionedPVCs: []*v1.PersistentVolumeClaim{}, - cachedPVs: []*v1.PersistentVolume{pvNode1aBound, pvNode1bBound}, - cachedPVCs: []*v1.PersistentVolumeClaim{boundPVCNode1a, unboundPVC2}, + initPVs: []*v1.PersistentVolume{pvNode1aBound, pvNode1bBound}, + initPVCs: []*v1.PersistentVolumeClaim{boundPVCNode1a, unboundPVC2}, }, "provisioning-pvc-bound": { bindings: []*bindingInfo{}, provisionedPVCs: []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVC)}, - cachedPVs: []*v1.PersistentVolume{pvBound}, - cachedPVCs: []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVCBound)}, + initPVs: []*v1.PersistentVolume{pvBound}, + initPVCs: []*v1.PersistentVolumeClaim{provisionedPVCBound}, + apiPVCs: []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVCBound)}, expectedBound: true, }, "provisioning-pvc-unbound": { bindings: []*bindingInfo{}, provisionedPVCs: []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVC)}, - cachedPVCs: []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVC)}, + initPVCs: []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVC)}, }, "provisioning-pvc-not-exists": { bindings: []*bindingInfo{}, provisionedPVCs: []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVC)}, + initPVCs: []*v1.PersistentVolumeClaim{provisionedPVC}, + deletePVCs: true, shouldFail: true, }, "provisioning-pvc-annotations-nil": { bindings: []*bindingInfo{}, provisionedPVCs: []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVC)}, - cachedPVCs: []*v1.PersistentVolumeClaim{provisionedPVC}, + initPVCs: []*v1.PersistentVolumeClaim{provisionedPVC}, + apiPVCs: []*v1.PersistentVolumeClaim{provisionedPVC}, shouldFail: true, }, "provisioning-pvc-selected-node-dropped": { bindings: []*bindingInfo{}, provisionedPVCs: []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVC)}, - cachedPVCs: []*v1.PersistentVolumeClaim{pvcSetEmptyAnnotations(provisionedPVC)}, + initPVCs: []*v1.PersistentVolumeClaim{provisionedPVC}, + apiPVCs: []*v1.PersistentVolumeClaim{pvcSetEmptyAnnotations(provisionedPVC)}, shouldFail: true, }, "provisioning-pvc-selected-node-wrong-node": { + initPVCs: []*v1.PersistentVolumeClaim{provisionedPVC}, bindings: []*bindingInfo{}, provisionedPVCs: []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVC)}, - cachedPVCs: []*v1.PersistentVolumeClaim{pvcSetSelectedNode(provisionedPVC, "wrong-node")}, + apiPVCs: []*v1.PersistentVolumeClaim{pvcSetSelectedNode(provisionedPVC, "wrong-node")}, shouldFail: true, }, "binding-bound-provisioning-unbound": { - bindings: []*bindingInfo{binding1aBound}, + bindings: []*bindingInfo{makeBinding(unboundPVC, pvNode1aBound)}, provisionedPVCs: []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVC)}, - cachedPVs: []*v1.PersistentVolume{pvNode1aBound}, - cachedPVCs: []*v1.PersistentVolumeClaim{boundPVCNode1a, addProvisionAnn(provisionedPVC)}, + initPVs: []*v1.PersistentVolume{pvNode1aBound}, + initPVCs: []*v1.PersistentVolumeClaim{boundPVCNode1a, addProvisionAnn(provisionedPVC)}, + }, + "tolerate-provisioning-pvc-bound-pv-not-found": { + initPVs: []*v1.PersistentVolume{pvNode1a}, + initPVCs: []*v1.PersistentVolumeClaim{provisionedPVC}, + bindings: []*bindingInfo{}, + provisionedPVCs: []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVC)}, + apiPVCs: []*v1.PersistentVolumeClaim{addProvisionAnn(provisionedPVCBound)}, + deletePVs: true, }, } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + for name, scenario := range scenarios { klog.V(4).Infof("Running test case %q", name) // Setup pod := makePod(nil) - testEnv := newTestBinder(t) + testEnv := newTestBinder(t, ctx.Done()) testEnv.initNodes([]*v1.Node{node1}) - testEnv.initVolumes(scenario.cachedPVs, nil) - testEnv.initClaims(scenario.cachedPVCs, nil) + testEnv.initVolumes(scenario.initPVs, nil) + testEnv.initClaims(scenario.initPVCs, nil) + testEnv.assumeVolumes(t, name, "node1", pod, scenario.bindings, scenario.provisionedPVCs) + + // Before execute + if scenario.deletePVs { + testEnv.deleteVolumes(scenario.initPVs) + } else { + testEnv.updateVolumes(t, scenario.apiPVs, true) + } + if scenario.deletePVCs { + testEnv.deleteClaims(scenario.initPVCs) + } else { + testEnv.updateClaims(t, scenario.apiPVCs, true) + } // Execute allBound, err := testEnv.internalBinder.checkBindings(pod, scenario.bindings, scenario.provisionedPVCs) @@ -1302,63 +1439,96 @@ func TestCheckBindings(t *testing.T) { } func TestBindPodVolumes(t *testing.T) { - scenarios := map[string]struct { + type scenarioType struct { // Inputs - // These tests only support a single pv and pvc and static binding - bindingsNil bool // Pass in nil bindings slice - binding *bindingInfo - cachedPVs []*v1.PersistentVolume - cachedPVCs []*v1.PersistentVolumeClaim - provisionedPVCs []*v1.PersistentVolumeClaim - apiPVs []*v1.PersistentVolume - nodes []*v1.Node + bindingsNil bool // Pass in nil bindings slice + + nodes []*v1.Node + + // before assume + initPVs []*v1.PersistentVolume + initPVCs []*v1.PersistentVolumeClaim + + // assume PV & PVC with these binding results + binding *bindingInfo + claimToProvision *v1.PersistentVolumeClaim + + // API updates after assume before bind + apiPV *v1.PersistentVolume + apiPVC *v1.PersistentVolumeClaim // This function runs with a delay of 5 seconds - delayFunc func(*testing.T, *testEnv, *v1.Pod, *bindingInfo, []*v1.PersistentVolumeClaim) + delayFunc func(t *testing.T, testEnv *testEnv, pod *v1.Pod, pvs []*v1.PersistentVolume, pvcs []*v1.PersistentVolumeClaim) // Expected return values shouldFail bool - }{ + } + + scenarios := map[string]scenarioType{ "nothing-to-bind-nil": { bindingsNil: true, shouldFail: true, }, "nothing-to-bind-empty": {}, "already-bound": { - binding: binding1aBound, - cachedPVs: []*v1.PersistentVolume{pvNode1aBound}, - cachedPVCs: []*v1.PersistentVolumeClaim{boundPVCNode1a}, + binding: makeBinding(unboundPVC, pvNode1aBound), + initPVs: []*v1.PersistentVolume{pvNode1aBound}, + initPVCs: []*v1.PersistentVolumeClaim{boundPVCNode1a}, }, - "binding-succeeds-after-time": { - binding: binding1aBound, - cachedPVs: []*v1.PersistentVolume{pvNode1a}, - cachedPVCs: []*v1.PersistentVolumeClaim{unboundPVC}, - delayFunc: func(t *testing.T, testEnv *testEnv, pod *v1.Pod, binding *bindingInfo, pvcs []*v1.PersistentVolumeClaim) { + "binding-static-pv-succeeds-after-time": { + initPVs: []*v1.PersistentVolume{pvNode1a}, + initPVCs: []*v1.PersistentVolumeClaim{unboundPVC}, + binding: makeBinding(unboundPVC, pvNode1aBound), + shouldFail: false, // Will succeed after PVC is fully bound to this PV by pv controller. + delayFunc: func(t *testing.T, testEnv *testEnv, pod *v1.Pod, pvs []*v1.PersistentVolume, pvcs []*v1.PersistentVolumeClaim) { + pvc := pvcs[0] + pv := pvs[0] // Update PVC to be fully bound to PV - newPVC := binding.pvc.DeepCopy() - newPVC.ResourceVersion = "100" - newPVC.Spec.VolumeName = binding.pv.Name + newPVC := pvc.DeepCopy() + newPVC.Spec.VolumeName = pv.Name metav1.SetMetaDataAnnotation(&newPVC.ObjectMeta, annBindCompleted, "yes") - - // Update pvc cache, fake client doesn't invoke informers - internalBinder, ok := testEnv.binder.(*volumeBinder) - if !ok { - t.Fatalf("Failed to convert to internal binder") + if _, err := testEnv.client.CoreV1().PersistentVolumeClaims(newPVC.Namespace).Update(newPVC); err != nil { + t.Errorf("failed to update PVC %q: %v", newPVC.Name, err) } - - pvcCache := internalBinder.pvcCache - internalPVCCache, ok := pvcCache.(*pvcAssumeCache) - if !ok { - t.Fatalf("Failed to convert to internal PVC cache") - } - internalPVCCache.add(newPVC) }, }, + "binding-dynamic-pv-succeeds-after-time": { + claimToProvision: pvcSetSelectedNode(provisionedPVC, "node1"), + initPVCs: []*v1.PersistentVolumeClaim{provisionedPVC}, + delayFunc: func(t *testing.T, testEnv *testEnv, pod *v1.Pod, pvs []*v1.PersistentVolume, pvcs []*v1.PersistentVolumeClaim) { + pvc := pvcs[0] + // Update PVC to be fully bound to PV + newPVC, err := testEnv.client.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(pvc.Name, metav1.GetOptions{}) + if err != nil { + t.Errorf("failed to get PVC %q: %v", pvc.Name, err) + return + } + dynamicPV := makeTestPV("dynamic-pv", "node1", "1G", "1", newPVC, waitClass) + dynamicPV, err = testEnv.client.CoreV1().PersistentVolumes().Create(dynamicPV) + if err != nil { + t.Errorf("failed to create PV %q: %v", dynamicPV.Name, err) + return + } + newPVC.Spec.VolumeName = dynamicPV.Name + metav1.SetMetaDataAnnotation(&newPVC.ObjectMeta, annBindCompleted, "yes") + if _, err := testEnv.client.CoreV1().PersistentVolumeClaims(newPVC.Namespace).Update(newPVC); err != nil { + t.Errorf("failed to update PVC %q: %v", newPVC.Name, err) + } + }, + }, + "bound-by-pv-controller-before-bind": { + initPVs: []*v1.PersistentVolume{pvNode1a}, + initPVCs: []*v1.PersistentVolumeClaim{unboundPVC}, + binding: makeBinding(unboundPVC, pvNode1aBound), + apiPV: pvNode1aBound, + apiPVC: boundPVCNode1a, + shouldFail: true, // bindAPIUpdate will fail because API conflict + }, "pod-deleted-after-time": { - binding: binding1aBound, - cachedPVs: []*v1.PersistentVolume{pvNode1a}, - cachedPVCs: []*v1.PersistentVolumeClaim{unboundPVC}, - delayFunc: func(t *testing.T, testEnv *testEnv, pod *v1.Pod, binding *bindingInfo, pvcs []*v1.PersistentVolumeClaim) { + binding: makeBinding(unboundPVC, pvNode1aBound), + initPVs: []*v1.PersistentVolume{pvNode1a}, + initPVCs: []*v1.PersistentVolumeClaim{unboundPVC}, + delayFunc: func(t *testing.T, testEnv *testEnv, pod *v1.Pod, pvs []*v1.PersistentVolume, pvcs []*v1.PersistentVolumeClaim) { bindingsCache := testEnv.binder.GetBindingsCache() if bindingsCache == nil { t.Fatalf("Failed to get bindings cache") @@ -1376,107 +1546,103 @@ func TestBindPodVolumes(t *testing.T) { shouldFail: true, }, "binding-times-out": { - binding: binding1aBound, - cachedPVs: []*v1.PersistentVolume{pvNode1a}, - cachedPVCs: []*v1.PersistentVolumeClaim{unboundPVC}, + binding: makeBinding(unboundPVC, pvNode1aBound), + initPVs: []*v1.PersistentVolume{pvNode1a}, + initPVCs: []*v1.PersistentVolumeClaim{unboundPVC}, shouldFail: true, }, "binding-fails": { - binding: binding1bBound, - cachedPVs: []*v1.PersistentVolume{pvNode1b}, - apiPVs: []*v1.PersistentVolume{pvNode1bBoundHigherVersion}, - cachedPVCs: []*v1.PersistentVolumeClaim{unboundPVC2}, + binding: makeBinding(unboundPVC2, pvNode1bBound), + initPVs: []*v1.PersistentVolume{pvNode1b}, + initPVCs: []*v1.PersistentVolumeClaim{unboundPVC2}, shouldFail: true, }, "check-fails": { - binding: binding1aBound, - cachedPVs: []*v1.PersistentVolume{pvNode1a}, - cachedPVCs: []*v1.PersistentVolumeClaim{unboundPVC}, - delayFunc: func(t *testing.T, testEnv *testEnv, pod *v1.Pod, binding *bindingInfo, pvcs []*v1.PersistentVolumeClaim) { - // Delete PVC - // Update pvc cache, fake client doesn't invoke informers - internalBinder, ok := testEnv.binder.(*volumeBinder) - if !ok { - t.Fatalf("Failed to convert to internal binder") + binding: makeBinding(unboundPVC, pvNode1aBound), + initPVs: []*v1.PersistentVolume{pvNode1a}, + initPVCs: []*v1.PersistentVolumeClaim{unboundPVC}, + delayFunc: func(t *testing.T, testEnv *testEnv, pod *v1.Pod, pvs []*v1.PersistentVolume, pvcs []*v1.PersistentVolumeClaim) { + pvc := pvcs[0] + // Delete PVC will fail check + if err := testEnv.client.CoreV1().PersistentVolumeClaims(pvc.Namespace).Delete(pvc.Name, &metav1.DeleteOptions{}); err != nil { + t.Errorf("failed to delete PVC %q: %v", pvc.Name, err) } - - pvcCache := internalBinder.pvcCache - internalPVCCache, ok := pvcCache.(*pvcAssumeCache) - if !ok { - t.Fatalf("Failed to convert to internal PVC cache") - } - internalPVCCache.delete(binding.pvc) }, shouldFail: true, }, "node-affinity-fails": { - binding: binding1aBound, - cachedPVs: []*v1.PersistentVolume{pvNode1aBound}, - cachedPVCs: []*v1.PersistentVolumeClaim{boundPVCNode1a}, + binding: makeBinding(unboundPVC, pvNode1aBound), + initPVs: []*v1.PersistentVolume{pvNode1aBound}, + initPVCs: []*v1.PersistentVolumeClaim{boundPVCNode1a}, nodes: []*v1.Node{node1NoLabels}, shouldFail: true, }, "node-affinity-fails-dynamic-provisioning": { - cachedPVs: []*v1.PersistentVolume{pvNode1a, pvNode2}, - cachedPVCs: []*v1.PersistentVolumeClaim{selectedNodePVC}, - provisionedPVCs: []*v1.PersistentVolumeClaim{selectedNodePVC}, - nodes: []*v1.Node{node1, node2}, - delayFunc: func(t *testing.T, testEnv *testEnv, pod *v1.Pod, binding *bindingInfo, pvcs []*v1.PersistentVolumeClaim) { + initPVs: []*v1.PersistentVolume{pvNode1a, pvNode2}, + initPVCs: []*v1.PersistentVolumeClaim{selectedNodePVC}, + claimToProvision: selectedNodePVC, + nodes: []*v1.Node{node1, node2}, + delayFunc: func(t *testing.T, testEnv *testEnv, pod *v1.Pod, pvs []*v1.PersistentVolume, pvcs []*v1.PersistentVolumeClaim) { // Update PVC to be fully bound to a PV with a different node newPVC := pvcs[0].DeepCopy() - newPVC.ResourceVersion = "100" newPVC.Spec.VolumeName = pvNode2.Name metav1.SetMetaDataAnnotation(&newPVC.ObjectMeta, annBindCompleted, "yes") - - // Update PVC cache, fake client doesn't invoke informers - internalBinder, ok := testEnv.binder.(*volumeBinder) - if !ok { - t.Fatalf("Failed to convert to internal binder") + if _, err := testEnv.client.CoreV1().PersistentVolumeClaims(newPVC.Namespace).Update(newPVC); err != nil { + t.Errorf("failed to update PVC %q: %v", newPVC.Name, err) } - - pvcCache := internalBinder.pvcCache - internalPVCCache, ok := pvcCache.(*pvcAssumeCache) - if !ok { - t.Fatalf("Failed to convert to internal PVC cache") - } - internalPVCCache.add(newPVC) }, shouldFail: true, }, } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + for name, scenario := range scenarios { klog.V(4).Infof("Running test case %q", name) // Setup pod := makePod(nil) - if scenario.apiPVs == nil { - scenario.apiPVs = scenario.cachedPVs - } + testEnv := newTestBinder(t, ctx.Done()) if scenario.nodes == nil { scenario.nodes = []*v1.Node{node1} } - if scenario.provisionedPVCs == nil { - scenario.provisionedPVCs = []*v1.PersistentVolumeClaim{} - } - testEnv := newTestBinder(t) if !scenario.bindingsNil { bindings := []*bindingInfo{} if scenario.binding != nil { bindings = []*bindingInfo{scenario.binding} } + claimsToProvision := []*v1.PersistentVolumeClaim{} + if scenario.claimToProvision != nil { + claimsToProvision = []*v1.PersistentVolumeClaim{scenario.claimToProvision} + } testEnv.initNodes(scenario.nodes) - testEnv.initVolumes(scenario.cachedPVs, scenario.apiPVs) - testEnv.initClaims(scenario.cachedPVCs, nil) - testEnv.assumeVolumes(t, name, "node1", pod, bindings, scenario.provisionedPVCs) + testEnv.initVolumes(scenario.initPVs, scenario.initPVs) + testEnv.initClaims(scenario.initPVCs, scenario.initPVCs) + testEnv.assumeVolumes(t, name, "node1", pod, bindings, claimsToProvision) + } + + // Before Execute + if scenario.apiPV != nil { + _, err := testEnv.client.CoreV1().PersistentVolumes().Update(scenario.apiPV) + if err != nil { + t.Fatalf("Test %q failed: failed to update PV %q", name, scenario.apiPV.Name) + } + } + if scenario.apiPVC != nil { + _, err := testEnv.client.CoreV1().PersistentVolumeClaims(scenario.apiPVC.Namespace).Update(scenario.apiPVC) + if err != nil { + t.Fatalf("Test %q failed: failed to update PVC %q", name, getPVCName(scenario.apiPVC)) + } } if scenario.delayFunc != nil { - go func() { + go func(scenario scenarioType) { time.Sleep(5 * time.Second) + // Sleep a while to run after bindAPIUpdate in BindPodVolumes klog.V(5).Infof("Running delay function") - scenario.delayFunc(t, testEnv, pod, scenario.binding, scenario.provisionedPVCs) - }() + scenario.delayFunc(t, testEnv, pod, scenario.initPVs, scenario.initPVCs) + }(scenario) } // Execute @@ -1498,7 +1664,9 @@ func TestFindAssumeVolumes(t *testing.T) { pvs := []*v1.PersistentVolume{pvNode2, pvNode1a, pvNode1c} // Setup - testEnv := newTestBinder(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + testEnv := newTestBinder(t, ctx.Done()) testEnv.initVolumes(pvs, pvs) testEnv.initClaims(podPVCs, podPVCs) pod := makePod(podPVCs) @@ -1548,6 +1716,6 @@ func TestFindAssumeVolumes(t *testing.T) { if !unboundSatisfied { t.Errorf("Test failed: couldn't find PVs for all PVCs") } - testEnv.validatePodCache(t, "after-assume", testNode.Name, pod, expectedBindings, []*v1.PersistentVolumeClaim{}) + testEnv.validatePodCache(t, "after-assume", testNode.Name, pod, expectedBindings, nil) } }