mirror of https://github.com/k3s-io/k3s
NodeAuthorizer and NodeRestriction integration test for CSINodeInfo permissions
Cheng Xing
parent
02df80d82b
commit
c65d7c9a49
|
|
@ -19,6 +19,7 @@ go_test(
|
|||
],
|
||||
tags = ["integration"],
|
||||
deps = [
|
||||
"//cmd/kube-apiserver/app/testing:go_default_library",
|
||||
"//pkg/api/legacyscheme:go_default_library",
|
||||
"//pkg/api/testapi:go_default_library",
|
||||
"//pkg/apis/authorization:go_default_library",
|
||||
|
|
@ -29,11 +30,9 @@ go_test(
|
|||
"//pkg/apis/policy:go_default_library",
|
||||
"//pkg/apis/rbac:go_default_library",
|
||||
"//pkg/auth/authorizer/abac:go_default_library",
|
||||
"//pkg/auth/nodeidentifier:go_default_library",
|
||||
"//pkg/client/clientset_generated/internalclientset:go_default_library",
|
||||
"//pkg/controller/serviceaccount:go_default_library",
|
||||
"//pkg/features:go_default_library",
|
||||
"//pkg/kubeapiserver/authorizer:go_default_library",
|
||||
"//pkg/master:go_default_library",
|
||||
"//pkg/registry/rbac/clusterrole:go_default_library",
|
||||
"//pkg/registry/rbac/clusterrole/storage:go_default_library",
|
||||
|
|
@ -44,13 +43,14 @@ go_test(
|
|||
"//pkg/registry/rbac/rolebinding:go_default_library",
|
||||
"//pkg/registry/rbac/rolebinding/storage:go_default_library",
|
||||
"//pkg/serviceaccount:go_default_library",
|
||||
"//plugin/pkg/admission/noderestriction:go_default_library",
|
||||
"//plugin/pkg/auth/authenticator/token/bootstrap:go_default_library",
|
||||
"//plugin/pkg/auth/authorizer/rbac:go_default_library",
|
||||
"//staging/src/k8s.io/api/authentication/v1:go_default_library",
|
||||
"//staging/src/k8s.io/api/authentication/v1beta1:go_default_library",
|
||||
"//staging/src/k8s.io/api/core/v1:go_default_library",
|
||||
"//staging/src/k8s.io/api/storage/v1beta1:go_default_library",
|
||||
"//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
|
||||
"//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||
|
|
@ -72,7 +72,6 @@ go_test(
|
|||
"//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library",
|
||||
"//staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/tokentest:go_default_library",
|
||||
"//staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/informers:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/rest:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/tools/clientcmd/api/v1:go_default_library",
|
||||
|
|
@ -80,6 +79,9 @@ go_test(
|
|||
"//staging/src/k8s.io/client-go/transport:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
|
||||
"//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library",
|
||||
"//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library",
|
||||
"//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library",
|
||||
"//staging/src/k8s.io/csi-api/pkg/crd:go_default_library",
|
||||
"//test/e2e/lifecycle/bootstrap:go_default_library",
|
||||
"//test/integration:go_default_library",
|
||||
"//test/integration/framework:go_default_library",
|
||||
|
|
|
|||
|
|
@ -18,46 +18,38 @@ package auth
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
storagev1beta1 "k8s.io/api/storage/v1beta1"
|
||||
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/apiserver/pkg/authentication/request/bearertoken"
|
||||
"k8s.io/apiserver/pkg/authentication/token/tokenfile"
|
||||
"k8s.io/apiserver/pkg/authentication/user"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
|
||||
versionedinformers "k8s.io/client-go/informers"
|
||||
externalclientset "k8s.io/client-go/kubernetes"
|
||||
restclient "k8s.io/client-go/rest"
|
||||
"k8s.io/kubernetes/pkg/api/legacyscheme"
|
||||
csiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1"
|
||||
"k8s.io/kubernetes/pkg/apis/coordination"
|
||||
api "k8s.io/kubernetes/pkg/apis/core"
|
||||
"k8s.io/kubernetes/pkg/apis/policy"
|
||||
"k8s.io/kubernetes/pkg/auth/nodeidentifier"
|
||||
clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
|
||||
"k8s.io/kubernetes/pkg/features"
|
||||
"k8s.io/kubernetes/pkg/kubeapiserver/authorizer"
|
||||
"k8s.io/kubernetes/plugin/pkg/admission/noderestriction"
|
||||
"k8s.io/kubernetes/test/integration/framework"
|
||||
"k8s.io/utils/pointer"
|
||||
|
||||
"io/ioutil"
|
||||
apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
|
||||
csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned"
|
||||
csicrd "k8s.io/csi-api/pkg/crd"
|
||||
kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
|
||||
"k8s.io/kubernetes/pkg/apis/core"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func TestNodeAuthorizer(t *testing.T) {
|
||||
// Start the server so we know the address
|
||||
h := &framework.MasterHolder{Initialized: make(chan struct{})}
|
||||
apiServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
<-h.Initialized
|
||||
h.M.GenericAPIServer.Handler.ServeHTTP(w, req)
|
||||
}))
|
||||
|
||||
const (
|
||||
// Define credentials
|
||||
tokenMaster = "master-token"
|
||||
|
|
@ -66,18 +58,6 @@ func TestNodeAuthorizer(t *testing.T) {
|
|||
tokenNode2 = "node2-token"
|
||||
)
|
||||
|
||||
authenticator := bearertoken.New(tokenfile.New(map[string]*user.DefaultInfo{
|
||||
tokenMaster: {Name: "admin", Groups: []string{"system:masters"}},
|
||||
tokenNodeUnknown: {Name: "unknown", Groups: []string{"system:nodes"}},
|
||||
tokenNode1: {Name: "system:node:node1", Groups: []string{"system:nodes"}},
|
||||
tokenNode2: {Name: "system:node:node2", Groups: []string{"system:nodes"}},
|
||||
}))
|
||||
|
||||
// Build client config, clientset, and informers
|
||||
clientConfig := &restclient.Config{Host: apiServer.URL, ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs}}
|
||||
superuserClient, superuserClientExternal := clientsetForToken(tokenMaster, clientConfig)
|
||||
versionedInformerFactory := versionedinformers.NewSharedInformerFactory(superuserClientExternal, time.Minute)
|
||||
|
||||
// Enabled CSIPersistentVolume feature at startup so volumeattachments get watched
|
||||
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIPersistentVolume, true)()
|
||||
|
||||
|
|
@ -87,36 +67,35 @@ func TestNodeAuthorizer(t *testing.T) {
|
|||
// Enable NodeLease feature so that nodes can create leases
|
||||
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.NodeLease, true)()
|
||||
|
||||
// Set up Node+RBAC authorizer
|
||||
authorizerConfig := &authorizer.AuthorizationConfig{
|
||||
AuthorizationModes: []string{"Node", "RBAC"},
|
||||
VersionedInformerFactory: versionedInformerFactory,
|
||||
}
|
||||
nodeRBACAuthorizer, _, err := authorizerConfig.New()
|
||||
// Enable CSINodeInfo feature so that nodes can create CSINodeInfo objects.
|
||||
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSINodeInfo, true)()
|
||||
|
||||
tokenFile, err := ioutil.TempFile("", "kubeconfig")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tokenFile.WriteString(strings.Join([]string{
|
||||
fmt.Sprintf(`%s,admin,uid1,"system:masters"`, tokenMaster),
|
||||
fmt.Sprintf(`%s,unknown,uid2,"system:nodes"`, tokenNodeUnknown),
|
||||
fmt.Sprintf(`%s,system:node:node1,uid3,"system:nodes"`, tokenNode1),
|
||||
fmt.Sprintf(`%s,system:node:node2,uid4,"system:nodes"`, tokenNode2),
|
||||
}, "\n"))
|
||||
tokenFile.Close()
|
||||
|
||||
// Set up NodeRestriction admission
|
||||
nodeRestrictionAdmission := noderestriction.NewPlugin(nodeidentifier.NewDefaultNodeIdentifier())
|
||||
nodeRestrictionAdmission.SetExternalKubeInformerFactory(versionedInformerFactory)
|
||||
if err := nodeRestrictionAdmission.ValidateInitialization(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{
|
||||
"--authorization-mode", "Node,RBAC",
|
||||
"--token-auth-file", tokenFile.Name(),
|
||||
"--enable-admission-plugins", "NodeRestriction",
|
||||
// The "default" SA is not installed, causing the ServiceAccount plugin to retry for ~1s per
|
||||
// API request.
|
||||
"--disable-admission-plugins", "ServiceAccount",
|
||||
}, framework.SharedEtcd())
|
||||
defer server.TearDownFn()
|
||||
|
||||
// Start the server
|
||||
masterConfig := framework.NewIntegrationTestMasterConfig()
|
||||
masterConfig.GenericConfig.Authentication.Authenticator = authenticator
|
||||
masterConfig.GenericConfig.Authorization.Authorizer = nodeRBACAuthorizer
|
||||
masterConfig.GenericConfig.AdmissionControl = nodeRestrictionAdmission
|
||||
|
||||
_, _, closeFn := framework.RunAMasterUsingServer(masterConfig, apiServer, h)
|
||||
defer closeFn()
|
||||
|
||||
// Start the informers
|
||||
stopCh := make(chan struct{})
|
||||
defer close(stopCh)
|
||||
versionedInformerFactory.Start(stopCh)
|
||||
// Build client config and superuser clientset
|
||||
clientConfig := server.ClientConfig
|
||||
superuserClient, superuserClientExternal := clientsetForToken(tokenMaster, clientConfig)
|
||||
superuserCRDClient := crdClientsetForToken(tokenMaster, clientConfig)
|
||||
|
||||
// Wait for a healthy server
|
||||
for {
|
||||
|
|
@ -130,6 +109,10 @@ func TestNodeAuthorizer(t *testing.T) {
|
|||
}
|
||||
|
||||
// Create objects
|
||||
if _, err := superuserClient.Core().Namespaces().Create(&core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "ns"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err := superuserClient.Core().Secrets("ns").Create(&api.Secret{ObjectMeta: metav1.ObjectMeta{Name: "mysecret"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -175,6 +158,14 @@ func TestNodeAuthorizer(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
|
||||
crd, err := superuserCRDClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(csicrd.CSINodeInfoCRD())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := waitForEstablishedCRD(superuserCRDClient, crd.Name); err != nil {
|
||||
t.Fatalf("Failed to establish CSINodeInfo CRD: %v", err)
|
||||
}
|
||||
|
||||
getSecret := func(client clientset.Interface) func() error {
|
||||
return func() error {
|
||||
_, err := client.Core().Secrets("ns").Get("mysecret", metav1.GetOptions{})
|
||||
|
|
@ -418,9 +409,66 @@ func TestNodeAuthorizer(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
getNode1CSINodeInfo := func(client csiclientset.Interface) func() error {
|
||||
return func() error {
|
||||
_, err := client.CsiV1alpha1().CSINodeInfos().Get("node1", metav1.GetOptions{})
|
||||
return err
|
||||
}
|
||||
}
|
||||
createNode1CSINodeInfo := func(client csiclientset.Interface) func() error {
|
||||
return func() error {
|
||||
nodeInfo := &csiv1alpha1.CSINodeInfo{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "node1",
|
||||
},
|
||||
CSIDrivers: []csiv1alpha1.CSIDriverInfo{
|
||||
{
|
||||
Driver: "com.example.csi/driver1",
|
||||
NodeID: "com.example.csi/node1",
|
||||
TopologyKeys: []string{"com.example.csi/zone"},
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err := client.CsiV1alpha1().CSINodeInfos().Create(nodeInfo)
|
||||
return err
|
||||
}
|
||||
}
|
||||
updateNode1CSINodeInfo := func(client csiclientset.Interface) func() error {
|
||||
return func() error {
|
||||
nodeInfo, err := client.CsiV1alpha1().CSINodeInfos().Get("node1", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
nodeInfo.CSIDrivers = []csiv1alpha1.CSIDriverInfo{
|
||||
{
|
||||
Driver: "com.example.csi/driver1",
|
||||
NodeID: "com.example.csi/node1",
|
||||
TopologyKeys: []string{"com.example.csi/rack"},
|
||||
},
|
||||
}
|
||||
_, err = client.CsiV1alpha1().CSINodeInfos().Update(nodeInfo)
|
||||
return err
|
||||
}
|
||||
}
|
||||
patchNode1CSINodeInfo := func(client csiclientset.Interface) func() error {
|
||||
return func() error {
|
||||
bs := []byte(fmt.Sprintf(`{"csiDrivers": [ { "driver": "net.example.storage/driver2", "nodeID": "net.example.storage/node1", "topologyKeys": [ "net.example.storage/region" ] } ] }`))
|
||||
// StrategicMergePatch is unsupported by CRs. Falling back to MergePatch
|
||||
_, err := client.CsiV1alpha1().CSINodeInfos().Patch("node1", types.MergePatchType, bs)
|
||||
return err
|
||||
}
|
||||
}
|
||||
deleteNode1CSINodeInfo := func(client csiclientset.Interface) func() error {
|
||||
return func() error {
|
||||
return client.CsiV1alpha1().CSINodeInfos().Delete("node1", &metav1.DeleteOptions{})
|
||||
}
|
||||
}
|
||||
|
||||
nodeanonClient, _ := clientsetForToken(tokenNodeUnknown, clientConfig)
|
||||
node1Client, node1ClientExternal := clientsetForToken(tokenNode1, clientConfig)
|
||||
node2Client, node2ClientExternal := clientsetForToken(tokenNode2, clientConfig)
|
||||
csiNode1Client := csiClientsetForToken(tokenNode1, clientConfig)
|
||||
csiNode2Client := csiClientsetForToken(tokenNode2, clientConfig)
|
||||
|
||||
// all node requests from node1 and unknown node fail
|
||||
expectForbidden(t, getSecret(nodeanonClient))
|
||||
|
|
@ -573,7 +621,18 @@ func TestNodeAuthorizer(t *testing.T) {
|
|||
expectForbidden(t, patchNode1Lease(node2Client))
|
||||
expectForbidden(t, deleteNode1Lease(node2Client))
|
||||
|
||||
// TODO (verult) CSINodeInfo tests (issue #68254)
|
||||
// node1 allowed to operate on its own CSINodeInfo
|
||||
expectAllowed(t, createNode1CSINodeInfo(csiNode1Client))
|
||||
expectAllowed(t, getNode1CSINodeInfo(csiNode1Client))
|
||||
expectAllowed(t, updateNode1CSINodeInfo(csiNode1Client))
|
||||
expectAllowed(t, patchNode1CSINodeInfo(csiNode1Client))
|
||||
expectAllowed(t, deleteNode1CSINodeInfo(csiNode1Client))
|
||||
// node2 not allowed to operate on another node's CSINodeInfo
|
||||
expectForbidden(t, createNode1CSINodeInfo(csiNode2Client))
|
||||
expectForbidden(t, getNode1CSINodeInfo(csiNode2Client))
|
||||
expectForbidden(t, updateNode1CSINodeInfo(csiNode2Client))
|
||||
expectForbidden(t, patchNode1CSINodeInfo(csiNode2Client))
|
||||
expectForbidden(t, deleteNode1CSINodeInfo(csiNode2Client))
|
||||
}
|
||||
|
||||
// expect executes a function a set number of times until it either returns the
|
||||
|
|
@ -613,3 +672,25 @@ func expectAllowed(t *testing.T, f func() error) {
|
|||
t.Errorf("Expected no error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func waitForEstablishedCRD(client apiextensionsclientset.Interface, name string) error {
|
||||
return wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
|
||||
crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, cond := range crd.Status.Conditions {
|
||||
switch cond.Type {
|
||||
case apiextensionsv1beta1.Established:
|
||||
if cond.Status == apiextensionsv1beta1.ConditionTrue {
|
||||
return true, err
|
||||
}
|
||||
case apiextensionsv1beta1.NamesAccepted:
|
||||
if cond.Status == apiextensionsv1beta1.ConditionFalse {
|
||||
fmt.Printf("Name conflict: %v\n", cond.Reason)
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import (
|
|||
|
||||
"github.com/golang/glog"
|
||||
|
||||
apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
|
|
@ -41,6 +42,7 @@ import (
|
|||
restclient "k8s.io/client-go/rest"
|
||||
watchtools "k8s.io/client-go/tools/watch"
|
||||
"k8s.io/client-go/transport"
|
||||
csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned"
|
||||
"k8s.io/kubernetes/pkg/api/legacyscheme"
|
||||
"k8s.io/kubernetes/pkg/api/testapi"
|
||||
api "k8s.io/kubernetes/pkg/apis/core"
|
||||
|
|
@ -74,6 +76,19 @@ func clientsetForToken(user string, config *restclient.Config) (clientset.Interf
|
|||
return clientset.NewForConfigOrDie(&configCopy), externalclientset.NewForConfigOrDie(&configCopy)
|
||||
}
|
||||
|
||||
func crdClientsetForToken(user string, config *restclient.Config) apiextensionsclient.Interface {
|
||||
configCopy := *config
|
||||
configCopy.BearerToken = user
|
||||
return apiextensionsclient.NewForConfigOrDie(&configCopy)
|
||||
}
|
||||
|
||||
func csiClientsetForToken(user string, config *restclient.Config) csiclientset.Interface {
|
||||
configCopy := *config
|
||||
configCopy.BearerToken = user
|
||||
configCopy.ContentType = "application/json" // // csi client works with CRDs that support json only
|
||||
return csiclientset.NewForConfigOrDie(&configCopy)
|
||||
}
|
||||
|
||||
type testRESTOptionsGetter struct {
|
||||
config *master.Config
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue