Merge pull request #68253 from davidz627/feature/gceCSISecret

Inject GCP IAM Credentials into GCE PD CSI Driver Test Namespace by Default

test/e2e/storage/csi_objects.go

@@ -21,12 +21,17 @@ package storage
 
 import (
 	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
 	"time"
 
 	"k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/apimachinery/pkg/util/wait"
 
 	clientset "k8s.io/client-go/kubernetes"
@@ -122,6 +127,7 @@ func csiServiceAccount(
 	sa := &v1.ServiceAccount{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      serviceAccountName,
+			Namespace: config.Namespace,
 		},
 	}
 
@@ -158,14 +164,13 @@ func csiClusterRoleBindings(
 	By(fmt.Sprintf("%v cluster roles %v to the CSI service account %v", bindingString, clusterRolesNames, sa.GetName()))
 	clusterRoleBindingClient := client.RbacV1().ClusterRoleBindings()
 	for _, clusterRoleName := range clusterRolesNames {
-
 		binding := &rbacv1.ClusterRoleBinding{
 			ObjectMeta: metav1.ObjectMeta{
-				Name: config.Prefix + "-" + clusterRoleName + "-" + config.Namespace + "-role-binding",
+				Name: clusterRoleName + "-" + config.Namespace + "-" + string(uuid.NewUUID()),
 			},
 			Subjects: []rbacv1.Subject{
 				{
-					Kind:      "ServiceAccount",
+					Kind:      rbacv1.ServiceAccountKind,
 					Name:      sa.GetName(),
 					Namespace: sa.GetNamespace(),
 				},
@@ -454,3 +459,64 @@ func deleteCSICRDs(c apiextensionsclient.Interface) {
 	err = c.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(csiNodeInfoCRDName, &metav1.DeleteOptions{})
 	framework.ExpectNoError(err, "Failed to delete CSI CRD %q: %v", csiNodeInfoCRDName, err)
 }
+
+func shredFile(filePath string) {
+	if _, err := os.Stat(filePath); os.IsNotExist(err) {
+		framework.Logf("File %v was not found, skipping shredding", filePath)
+		return
+	}
+	framework.Logf("Shredding file %v", filePath)
+	_, _, err := framework.RunCmd("shred", "--remove", filePath)
+	if err != nil {
+		framework.Logf("Failed to shred file %v: %v", filePath, err)
+	}
+	if _, err := os.Stat(filePath); os.IsNotExist(err) {
+		framework.Logf("File %v successfully shredded", filePath)
+		return
+	}
+	// Shred failed Try to remove the file for good meausure
+	err = os.Remove(filePath)
+	framework.ExpectNoError(err, "Failed to remove service account file %s", filePath)
+
+}
+
+// createGCESecrets downloads the GCP IAM Key for the default compute service account
+// and puts it in a secret for the GCE PD CSI Driver to consume
+func createGCESecrets(client clientset.Interface, config framework.VolumeTestConfig) {
+	saEnv := "E2E_GOOGLE_APPLICATION_CREDENTIALS"
+	saFile := fmt.Sprintf("/tmp/%s/cloud-sa.json", string(uuid.NewUUID()))
+
+	os.MkdirAll(path.Dir(saFile), 0750)
+	defer os.Remove(path.Dir(saFile))
+
+	premadeSAFile, ok := os.LookupEnv(saEnv)
+	if !ok {
+		framework.Logf("Could not find env var %v, please either create cloud-sa"+
+			" secret manually or rerun test after setting %v to the filepath of"+
+			" the GCP Service Account to give to the GCE Persistent Disk CSI Driver", saEnv, saEnv)
+		return
+	}
+
+	framework.Logf("Found CI service account key at %v", premadeSAFile)
+	// Need to copy it saFile
+	stdout, stderr, err := framework.RunCmd("cp", premadeSAFile, saFile)
+	framework.ExpectNoError(err, "error copying service account key: %s\nstdout: %s\nstderr: %s", err, stdout, stderr)
+	defer shredFile(saFile)
+	// Create Secret with this Service Account
+	fileBytes, err := ioutil.ReadFile(saFile)
+	framework.ExpectNoError(err, "Failed to read file %v", saFile)
+
+	s := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "cloud-sa",
+			Namespace: config.Namespace,
+		},
+		Type: v1.SecretTypeOpaque,
+		Data: map[string][]byte{
+			filepath.Base(saFile): fileBytes,
+		},
+	}
+
+	_, err = client.CoreV1().Secrets(config.Namespace).Create(s)
+	framework.ExpectNoError(err, "Failed to create Secret %v", s.GetName())
+}

test/e2e/storage/csi_volumes.go

@@ -55,8 +55,7 @@ type csiTestDriver interface {
 
 var csiTestDrivers = map[string]func(f *framework.Framework, config framework.VolumeTestConfig) csiTestDriver{
 	"hostPath": initCSIHostpath,
-	// Feature tag to skip test in CI, pending fix of #62237
-	"[Feature: GCE PD CSI Plugin] gcePD": initCSIgcePD,
+	"gcePD":    initCSIgcePD,
 }
 
 var _ = utils.SIGDescribe("[Serial] CSI Volumes", func() {
@@ -361,9 +360,10 @@ type gcePDCSIDriver struct {
 func initCSIgcePD(f *framework.Framework, config framework.VolumeTestConfig) csiTestDriver {
 	cs := f.ClientSet
 	framework.SkipUnlessProviderIs("gce", "gke")
-	// Currently you will need to manually add the required GCP Credentials as a secret "cloud-sa"
-	// kubectl create generic cloud-sa --from-file=PATH/TO/cloud-sa.json --namespace={{config.Namespace}}
-	// TODO(#62561): Inject the necessary credentials automatically to the driver containers in e2e test
+
+	// TODO(#62561): Use credentials through external pod identity when that goes GA instead of downloading keys.
+	createGCESecrets(cs, config)
+
 	framework.SkipUnlessSecretExistsAfterWait(cs, "cloud-sa", config.Namespace, 3*time.Minute)
 
 	return &gcePDCSIDriver{
@@ -402,6 +402,8 @@ func (g *gcePDCSIDriver) createCSIDriver() {
 	g.nodeServiceAccount = csiServiceAccount(cs, config, "gce-node", false /* teardown */)
 	csiClusterRoleBindings(cs, config, false /* teardown */, g.controllerServiceAccount, g.controllerClusterRoles)
 	csiClusterRoleBindings(cs, config, false /* teardown */, g.nodeServiceAccount, g.nodeClusterRoles)
+	utils.PrivilegedTestPSPClusterRoleBinding(cs, config.Namespace,
+		false /* teardown */, []string{g.controllerServiceAccount.Name, g.nodeServiceAccount.Name})
 	deployGCEPDCSIDriver(cs, config, false /* teardown */, f, g.nodeServiceAccount, g.controllerServiceAccount)
 }
 
@@ -413,6 +415,8 @@ func (g *gcePDCSIDriver) cleanupCSIDriver() {
 	deployGCEPDCSIDriver(cs, config, true /* teardown */, f, g.nodeServiceAccount, g.controllerServiceAccount)
 	csiClusterRoleBindings(cs, config, true /* teardown */, g.controllerServiceAccount, g.controllerClusterRoles)
 	csiClusterRoleBindings(cs, config, true /* teardown */, g.nodeServiceAccount, g.nodeClusterRoles)
+	utils.PrivilegedTestPSPClusterRoleBinding(cs, config.Namespace,
+		true /* teardown */, []string{g.controllerServiceAccount.Name, g.nodeServiceAccount.Name})
 	csiServiceAccount(cs, config, "gce-controller", true /* teardown */)
 	csiServiceAccount(cs, config, "gce-node", true /* teardown */)
 }

test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml

@@ -13,7 +13,6 @@ spec:
       labels:
         app: csi-gce-pd-driver
     spec:
-      serviceAccount: csi-gce-pd
       containers:
         - name: csi-external-provisioner
           imagePullPolicy: Always
@@ -42,7 +41,7 @@ spec:
               mountPath: /csi
         - name: gce-driver
           imagePullPolicy: Always
-          image: gcr.io/google-containers/volume-csi/compute-persistent-disk-csi-driver:v0.2.0.alpha
+          image: gcr.io/google-containers/volume-csi/gcp-compute-persistent-disk-csi-driver:v0.1.0.alpha
           args:
             - "--v=5"
             - "--endpoint=$(CSI_ENDPOINT)"

test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml

@@ -12,7 +12,6 @@ spec:
       labels:
         app: csi-gce-driver
     spec:
-      serviceAccount: csi-gce-pd
       containers:
         - name: csi-driver-registrar
           imagePullPolicy: Always
@@ -39,7 +38,7 @@ spec:
           securityContext:
             privileged: true
           imagePullPolicy: Always
-          image: gcr.io/google-containers/volume-csi/compute-persistent-disk-csi-driver:v0.2.0.alpha
+          image: gcr.io/google-containers/volume-csi/gcp-compute-persistent-disk-csi-driver:v0.1.0.alpha
           args:
             - "--v=5"
             - "--endpoint=$(CSI_ENDPOINT)"