Merge pull request #60592 from xmudrii/crd-tests

Automatic merge from submit-queue (batch tested with PRs 61402, 61143, 61427, 60592). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

apiextensions-apiserver: TestFinaliazationAndDeletion integration test

**What this PR does / why we need it**: This PR adds an integration test for [behavior described in the issue #60538 (comment)](https://github.com/kubernetes/kubernetes/issues/60538#issuecomment-369182483).

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Partially addresses #60538 -- tests

**Special notes for your reviewer**: This is based of the PR #60542, so this shouldn't be merged before that one. Tests are failing on the line 155, due to the timeout/non-nil error. I've tried to debug it furtherly, and found out that the CRD is still terminating. 

Actually, the CRD gets deleted, but it takes some time. For example, if you add `time.Sleep(x * time.Second)` on the line 152, the test would pass. The problem is this isn't predictable. With `x` of 6-10 seconds, I manage to get this test passing, but there's zero guarantees the CRD will get deleted for that 10 seconds.

I have found the `wait.Poll` function in some tests, but I'm not sure can it fix the problem I have.

This is my first PR to the Kubernetes project, so I could be missing something. Any idea is appreciated. :)

**Release note**:

```release-note
NONE
```

/sig api-machinery
/area custom-resources
/cc nikhita sttts liggitt

staging/src/k8s.io/apiextensions-apiserver/test/integration/finalization_test.go

@@ -18,13 +18,16 @@ package integration
 
 import (
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 
 	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+
 	"k8s.io/apiextensions-apiserver/test/integration/testserver"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 func TestFinalization(t *testing.T) {
@@ -89,3 +92,74 @@ func TestFinalization(t *testing.T) {
 	require.Error(t, err)
 	require.True(t, errors.IsNotFound(err), "%#v", err)
 }
+
+func TestFinalizationAndDeletion(t *testing.T) {
+	stopCh, apiExtensionClient, clientPool, err := testserver.StartDefaultServerWithClients()
+	require.NoError(t, err)
+	defer close(stopCh)
+
+	// Create a CRD.
+	noxuDefinition := testserver.NewNoxuCustomResourceDefinition(apiextensionsv1beta1.ClusterScoped)
+	noxuVersionClient, err := testserver.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, clientPool)
+	require.NoError(t, err)
+
+	// Create a CR with a finalizer.
+	ns := "not-the-default"
+	name := "foo123"
+	noxuResourceClient := NewNamespacedCustomResourceClient(ns, noxuVersionClient, noxuDefinition)
+
+	instance := testserver.NewNoxuInstance(ns, name)
+	instance.SetFinalizers([]string{"noxu.example.com/finalizer"})
+	createdNoxuInstance, err := instantiateCustomResource(t, instance, noxuResourceClient, noxuDefinition)
+	require.NoError(t, err)
+
+	// Delete a CR. Because there's a finalizer, it will not get deleted now.
+	uid := createdNoxuInstance.GetUID()
+	err = noxuResourceClient.Delete(name, &metav1.DeleteOptions{
+		Preconditions: &metav1.Preconditions{
+			UID: &uid,
+		},
+	})
+	require.NoError(t, err)
+
+	// Check is the CR scheduled for deletion.
+	gottenNoxuInstance, err := noxuResourceClient.Get(name, metav1.GetOptions{})
+	require.NoError(t, err)
+	require.NotNil(t, gottenNoxuInstance.GetDeletionTimestamp())
+
+	// Delete the CRD.
+	testserver.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient)
+
+	// Check is CR still there after the CRD deletion.
+	gottenNoxuInstance, err = noxuResourceClient.Get(name, metav1.GetOptions{})
+	require.NoError(t, err)
+
+	// Update the CR to remove the finalizer.
+	for {
+		gottenNoxuInstance.SetFinalizers(nil)
+		_, err = noxuResourceClient.Update(gottenNoxuInstance)
+		if err == nil {
+			break
+		}
+		if !errors.IsConflict(err) {
+			require.NoError(t, err) // Fail on unexpected error
+		}
+		gottenNoxuInstance, err = noxuResourceClient.Get(name, metav1.GetOptions{})
+		require.NoError(t, err)
+	}
+
+	// Verify the CR is gone.
+	// It should return the NonFound error.
+	_, err = noxuResourceClient.Get(name, metav1.GetOptions{})
+	if !errors.IsNotFound(err) {
+		t.Fatalf("unable to delete cr: %v", err)
+	}
+
+	err = wait.Poll(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
+		_, err = testserver.GetCustomResourceDefinition(noxuDefinition, apiExtensionClient)
+		return errors.IsNotFound(err), err
+	})
+	if !errors.IsNotFound(err) {
+		t.Fatalf("unable to delete crd: %v", err)
+	}
+}