Merge pull request #60666 from immutableT/kms_mock_flake_issue

Automatic merge from submit-queue (batch tested with PRs 60574, 60666, 60831, 60877, 60357). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Remove potential sources of flakes for kms_transformation_test.go.

**What this PR does / why we need it**:
Remove potential sources for flakes in TestKMSPlugin test.

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #
#60614
**Special notes for your reviewer**:

**Release note**:

```release-note
NONE
```

test/integration/master/BUILD

@@ -188,51 +188,61 @@ go_library(
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
     ] + select({
         "@io_bazel_rules_go//go/platform:android": [
+            "//vendor/github.com/golang/glog:go_default_library",
             "//vendor/golang.org/x/sys/unix:go_default_library",
             "//vendor/google.golang.org/grpc:go_default_library",
             "//vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:darwin": [
+            "//vendor/github.com/golang/glog:go_default_library",
             "//vendor/golang.org/x/sys/unix:go_default_library",
             "//vendor/google.golang.org/grpc:go_default_library",
             "//vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:dragonfly": [
+            "//vendor/github.com/golang/glog:go_default_library",
             "//vendor/golang.org/x/sys/unix:go_default_library",
             "//vendor/google.golang.org/grpc:go_default_library",
             "//vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:freebsd": [
+            "//vendor/github.com/golang/glog:go_default_library",
             "//vendor/golang.org/x/sys/unix:go_default_library",
             "//vendor/google.golang.org/grpc:go_default_library",
             "//vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:linux": [
+            "//vendor/github.com/golang/glog:go_default_library",
             "//vendor/golang.org/x/sys/unix:go_default_library",
             "//vendor/google.golang.org/grpc:go_default_library",
             "//vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:nacl": [
+            "//vendor/github.com/golang/glog:go_default_library",
             "//vendor/golang.org/x/sys/unix:go_default_library",
             "//vendor/google.golang.org/grpc:go_default_library",
             "//vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:netbsd": [
+            "//vendor/github.com/golang/glog:go_default_library",
             "//vendor/golang.org/x/sys/unix:go_default_library",
             "//vendor/google.golang.org/grpc:go_default_library",
             "//vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:openbsd": [
+            "//vendor/github.com/golang/glog:go_default_library",
             "//vendor/golang.org/x/sys/unix:go_default_library",
             "//vendor/google.golang.org/grpc:go_default_library",
             "//vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:plan9": [
+            "//vendor/github.com/golang/glog:go_default_library",
             "//vendor/golang.org/x/sys/unix:go_default_library",
             "//vendor/google.golang.org/grpc:go_default_library",
             "//vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:solaris": [
+            "//vendor/github.com/golang/glog:go_default_library",
             "//vendor/golang.org/x/sys/unix:go_default_library",
             "//vendor/google.golang.org/grpc:go_default_library",
             "//vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",

test/integration/master/kms_plugin_mock.go

@@ -28,6 +28,7 @@ import (
 	"golang.org/x/sys/unix"
 	"google.golang.org/grpc"
 
+	"github.com/golang/glog"
 	kmsapi "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1"
 )
 
@@ -45,7 +46,6 @@ type base64Plugin struct {
 
 	// Allow users of the plugin to sense requests that were passed to KMS.
 	encryptRequest chan *kmsapi.EncryptRequest
-	decryptRequest chan *kmsapi.DecryptRequest
 }
 
 func NewBase64Plugin() (*base64Plugin, error) {
@@ -57,6 +57,7 @@ func NewBase64Plugin() (*base64Plugin, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to listen on the unix socket, error: %v", err)
 	}
+	glog.Infof("Listening on %s", sockFile)
 
 	server := grpc.NewServer()
 
@@ -64,7 +65,6 @@ func NewBase64Plugin() (*base64Plugin, error) {
 		grpcServer:     server,
 		listener:       listener,
 		encryptRequest: make(chan *kmsapi.EncryptRequest, 1),
-		decryptRequest: make(chan *kmsapi.DecryptRequest, 1),
 	}
 
 	kmsapi.RegisterKeyManagementServiceServer(server, result)
@@ -85,7 +85,8 @@ func (s *base64Plugin) Version(ctx context.Context, request *kmsapi.VersionReque
 }
 
 func (s *base64Plugin) Decrypt(ctx context.Context, request *kmsapi.DecryptRequest) (*kmsapi.DecryptResponse, error) {
-	s.decryptRequest <- request
+	glog.Infof("Received Decrypt Request for DEK: %s", string(request.Cipher))
+
 	buf := make([]byte, base64.StdEncoding.DecodedLen(len(request.Cipher)))
 	n, err := base64.StdEncoding.Decode(buf, request.Cipher)
 	if err != nil {
@@ -96,6 +97,7 @@ func (s *base64Plugin) Decrypt(ctx context.Context, request *kmsapi.DecryptReque
 }
 
 func (s *base64Plugin) Encrypt(ctx context.Context, request *kmsapi.EncryptRequest) (*kmsapi.EncryptResponse, error) {
+	glog.Infof("Received Encrypt Request for DEK: %x", request.Plain)
 	s.encryptRequest <- request
 
 	buf := make([]byte, base64.StdEncoding.EncodedLen(len(request.Plain)))

test/integration/master/kms_transformation_test.go

@@ -26,7 +26,6 @@ import (
 	"fmt"
 	"strings"
 	"testing"
-	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apiserver/pkg/storage/value"
@@ -86,7 +85,10 @@ func TestKMSProvider(t *testing.T) {
 		t.Fatalf("failed to create mock of KMS Plugin: %v", err)
 	}
 	defer pluginMock.cleanUp()
-	go pluginMock.grpcServer.Serve(pluginMock.listener)
+	serveErr := make(chan error, 1)
+	go func() {
+		serveErr <- pluginMock.grpcServer.Serve(pluginMock.listener)
+	}()
 
 	test, err := newTransformTest(t, kmsConfigYAML)
 	if err != nil {
@@ -94,6 +96,11 @@ func TestKMSProvider(t *testing.T) {
 	}
 	defer test.cleanUp()
 
+	// As part of newTransformTest a new secret was created, so KMS Mock should have been exercised by this point.
+	if len(serveErr) != 0 {
+		t.Fatalf("KMSPlugin failed while serving requests: %v", <-serveErr)
+	}
+
 	secretETCDPath := test.getETCDPath()
 	var rawSecretAsSeenByETCD rawDEKKEKSecret
 	rawSecretAsSeenByETCD, err = test.getRawSecretFromETCD()
@@ -140,12 +147,14 @@ func TestKMSProvider(t *testing.T) {
 }
 
 func getDEKFromKMSPlugin(pluginMock *base64Plugin) ([]byte, error) {
-	select {
-	case e := <-pluginMock.encryptRequest:
-		return e.Plain, nil
-	case <-time.After(time.Second):
-		return nil, fmt.Errorf("timed-out while getting encryption request from KMS Plugin Mock")
+	// We expect KMS to already have seen an encryptRequest. Hence non-blocking call.
+	e, ok := <-pluginMock.encryptRequest
+
+	if !ok {
+		return nil, fmt.Errorf("failed to sense encryptRequest from KMS Plugin Mock")
 	}
+
+	return e.Plain, nil
 }
 
 func decryptPayload(key []byte, secret rawDEKKEKSecret, secretETCDPath string) ([]byte, error) {