Merge pull request #68887 from jsafrane/csi-roles

Add controller roles to CSI e2e tests

test/e2e/manifest/BUILD

@@ -16,6 +16,7 @@ go_library(
         "//staging/src/k8s.io/api/apps/v1:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/api/extensions/v1beta1:go_default_library",
+        "//staging/src/k8s.io/api/rbac/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library",

test/e2e/manifest/manifest.go

@@ -23,6 +23,7 @@ import (
 	apps "k8s.io/api/apps/v1"
 	"k8s.io/api/core/v1"
 	extensions "k8s.io/api/extensions/v1beta1"
+	rbac "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilyaml "k8s.io/apimachinery/pkg/util/yaml"
@@ -142,3 +143,20 @@ func DaemonSetFromManifest(fileName, ns string) (*apps.DaemonSet, error) {
 	ds.Namespace = ns
 	return &ds, nil
 }
+
+// RoleFromManifest returns a Role from a manifest stored in fileName in the Namespace indicated by ns.
+func RoleFromManifest(fileName, ns string) (*rbac.Role, error) {
+	var role rbac.Role
+	data := generated.ReadOrDie(fileName)
+
+	json, err := utilyaml.ToJSON(data)
+	if err != nil {
+		return nil, err
+	}
+	err = runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), json, &role)
+	if err != nil {
+		return nil, err
+	}
+	role.Namespace = ns
+	return &role, nil
+}

test/e2e/storage/csi_objects.go

@@ -200,6 +200,87 @@ func csiClusterRoleBindings(
 	}
 }
 
+func csiControllerRole(
+	client clientset.Interface,
+	config framework.VolumeTestConfig,
+	teardown bool,
+) string {
+	action := "Creating"
+	if teardown {
+		action = "Deleting"
+	}
+
+	By(fmt.Sprintf("%v CSI controller role", action))
+
+	role, err := manifest.RoleFromManifest("test/e2e/testing-manifests/storage-csi/controller-role.yaml", config.Namespace)
+	framework.ExpectNoError(err, "Failed to create Role from manifest")
+
+	client.RbacV1().Roles(role.Namespace).Delete(role.Name, nil)
+	err = wait.Poll(2*time.Second, 10*time.Minute, func() (bool, error) {
+		_, err := client.RbacV1().Roles(role.Namespace).Get(role.Name, metav1.GetOptions{})
+		return apierrs.IsNotFound(err), nil
+	})
+	framework.ExpectNoError(err, "Timed out waiting for deletion: %v", err)
+
+	if teardown {
+		return role.Name
+	}
+
+	_, err = client.RbacV1().Roles(role.Namespace).Create(role)
+	if err != nil {
+		framework.ExpectNoError(err, "Failed to create %s role binding: %v", role.Name, err)
+	}
+	return role.Name
+}
+
+func csiControllerRoleBinding(
+	client clientset.Interface,
+	config framework.VolumeTestConfig,
+	teardown bool,
+	roleName string,
+	sa *v1.ServiceAccount,
+) {
+	bindingString := "Binding"
+	if teardown {
+		bindingString = "Unbinding"
+	}
+	By(fmt.Sprintf("%v roles %v to the CSI service account %v", bindingString, roleName, sa.GetName()))
+	roleBindingClient := client.RbacV1().RoleBindings(config.Namespace)
+	binding := &rbacv1.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: config.Prefix + "-" + roleName + "-" + config.Namespace + "-role-binding",
+		},
+		Subjects: []rbacv1.Subject{
+			{
+				Kind:      "ServiceAccount",
+				Name:      sa.GetName(),
+				Namespace: sa.GetNamespace(),
+			},
+		},
+		RoleRef: rbacv1.RoleRef{
+			Kind:     "Role",
+			Name:     roleName,
+			APIGroup: "rbac.authorization.k8s.io",
+		},
+	}
+
+	roleBindingClient.Delete(binding.GetName(), &metav1.DeleteOptions{})
+	err := wait.Poll(2*time.Second, 10*time.Minute, func() (bool, error) {
+		_, err := roleBindingClient.Get(binding.GetName(), metav1.GetOptions{})
+		return apierrs.IsNotFound(err), nil
+	})
+	framework.ExpectNoError(err, "Timed out waiting for deletion: %v", err)
+
+	if teardown {
+		return
+	}
+
+	_, err = roleBindingClient.Create(binding)
+	if err != nil {
+		framework.ExpectNoError(err, "Failed to create %s role binding: %v", binding.GetName(), err)
+	}
+}
+
 func csiHostPathPod(
 	client clientset.Interface,
 	config framework.VolumeTestConfig,

test/e2e/storage/csi_volumes.go

@@ -334,6 +334,8 @@ func (h *hostpathCSIDriver) createCSIDriver() {
 	config := h.config
 	h.serviceAccount = csiServiceAccount(cs, config, "hostpath", false)
 	csiClusterRoleBindings(cs, config, false, h.serviceAccount, h.combinedClusterRoleNames)
+	role := csiControllerRole(cs, config, false)
+	csiControllerRoleBinding(cs, config, false, role, h.serviceAccount)
 	csiHostPathPod(cs, config, false, f, h.serviceAccount)
 }
 
@@ -344,6 +346,8 @@ func (h *hostpathCSIDriver) cleanupCSIDriver() {
 	config := h.config
 	csiHostPathPod(cs, config, true, f, h.serviceAccount)
 	csiClusterRoleBindings(cs, config, true, h.serviceAccount, h.combinedClusterRoleNames)
+	role := csiControllerRole(cs, config, true)
+	csiControllerRoleBinding(cs, config, true, role, h.serviceAccount)
 	csiServiceAccount(cs, config, "hostpath", true)
 }
 
@@ -402,8 +406,8 @@ func (g *gcePDCSIDriver) createCSIDriver() {
 	g.nodeServiceAccount = csiServiceAccount(cs, config, "gce-node", false /* teardown */)
 	csiClusterRoleBindings(cs, config, false /* teardown */, g.controllerServiceAccount, g.controllerClusterRoles)
 	csiClusterRoleBindings(cs, config, false /* teardown */, g.nodeServiceAccount, g.nodeClusterRoles)
-	utils.PrivilegedTestPSPClusterRoleBinding(cs, config.Namespace,
-		false /* teardown */, []string{g.controllerServiceAccount.Name, g.nodeServiceAccount.Name})
+	role := csiControllerRole(cs, config, false)
+	csiControllerRoleBinding(cs, config, false, role, g.controllerServiceAccount)
 	deployGCEPDCSIDriver(cs, config, false /* teardown */, f, g.nodeServiceAccount, g.controllerServiceAccount)
 }
 
@@ -415,8 +419,8 @@ func (g *gcePDCSIDriver) cleanupCSIDriver() {
 	deployGCEPDCSIDriver(cs, config, true /* teardown */, f, g.nodeServiceAccount, g.controllerServiceAccount)
 	csiClusterRoleBindings(cs, config, true /* teardown */, g.controllerServiceAccount, g.controllerClusterRoles)
 	csiClusterRoleBindings(cs, config, true /* teardown */, g.nodeServiceAccount, g.nodeClusterRoles)
-	utils.PrivilegedTestPSPClusterRoleBinding(cs, config.Namespace,
-		true /* teardown */, []string{g.controllerServiceAccount.Name, g.nodeServiceAccount.Name})
+	role := csiControllerRole(cs, config, true)
+	csiControllerRoleBinding(cs, config, true, role, g.controllerServiceAccount)
 	csiServiceAccount(cs, config, "gce-controller", true /* teardown */)
 	csiServiceAccount(cs, config, "gce-node", true /* teardown */)
 }

test/e2e/testing-manifests/storage-csi/controller-role.yaml

@@ -0,0 +1,11 @@
+# Role for external CSI provisioner and attacher.
+# They need to modify Endpoints and ConfigMap for leader election.
+
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-controller
+rules:
+- apiGroups: [""]
+  resources: ["configmaps", "endpoints"]
+  verbs: ["get", "watch", "list", "delete", "update", "create"]