fix(kubernetes): clear user token from kube token cache on logout + update cluster rolebindings for user on change of team/user authorization [EE-6298] (#10598)

* clear user token from kube token cache on logoug + updates cluster role bindings for service accounts on change user/teams authorizations

api/dataservices/endpoint/endpoint.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -144,6 +145,23 @@ func (service *Service) Create(endpoint *portainer.Endpoint) error {
 	})
 }
 
+func (service *Service) EndpointsByTeamID(teamID portainer.TeamID) ([]portainer.Endpoint, error) {
+	var endpoints = make([]portainer.Endpoint, 0)
+
+	return endpoints, service.connection.GetAll(
+		BucketName,
+		&portainer.Endpoint{},
+		dataservices.FilterFn(&endpoints, func(e portainer.Endpoint) bool {
+			for t := range e.TeamAccessPolicies {
+				if t == teamID {
+					return true
+				}
+			}
+			return false
+		}),
+	)
+}
+
 // GetNextIdentifier returns the next identifier for an environment(endpoint).
 func (service *Service) GetNextIdentifier() int {
 	var identifier int

api/dataservices/endpoint/tx.go

@@ -122,6 +122,23 @@ func (service ServiceTx) Create(endpoint *portainer.Endpoint) error {
 	return nil
 }
 
+func (service ServiceTx) EndpointsByTeamID(teamID portainer.TeamID) ([]portainer.Endpoint, error) {
+	var endpoints = make([]portainer.Endpoint, 0)
+
+	return endpoints, service.tx.GetAll(
+		BucketName,
+		&portainer.Endpoint{},
+		dataservices.FilterFn(&endpoints, func(e portainer.Endpoint) bool {
+			for t := range e.TeamAccessPolicies {
+				if t == teamID {
+					return true
+				}
+			}
+			return false
+		}),
+	)
+}
+
 // GetNextIdentifier returns the next identifier for an environment(endpoint).
 func (service ServiceTx) GetNextIdentifier() int {
 	return service.tx.GetNextIdentifier(BucketName)

api/dataservices/interface.go

@@ -95,6 +95,7 @@ type (
 	EndpointService interface {
 		Endpoint(ID portainer.EndpointID) (*portainer.Endpoint, error)
 		EndpointIDByEdgeID(edgeID string) (portainer.EndpointID, bool)
+		EndpointsByTeamID(teamID portainer.TeamID) ([]portainer.Endpoint, error)
 		Heartbeat(endpointID portainer.EndpointID) (int64, bool)
 		UpdateHeartbeat(endpointID portainer.EndpointID)
 		Endpoints() ([]portainer.Endpoint, error)

api/http/handler/auth/handler.go

@@ -24,6 +24,7 @@ type Handler struct {
 	ProxyManager                *proxy.Manager
 	KubernetesTokenCacheManager *kubernetes.TokenCacheManager
 	passwordStrengthChecker     security.PasswordStrengthChecker
+	bouncer                     security.BouncerService
 }
 
 // NewHandler creates a handler to manage authentication operations.
@@ -31,6 +32,7 @@ func NewHandler(bouncer security.BouncerService, rateLimiter *security.RateLimit
 	h := &Handler{
 		Router:                  mux.NewRouter(),
 		passwordStrengthChecker: passwordStrengthChecker,
+		bouncer:                 bouncer,
 	}
 
 	h.Handle("/auth/oauth/validate",
@@ -39,6 +41,5 @@ func NewHandler(bouncer security.BouncerService, rateLimiter *security.RateLimit
 		rateLimiter.LimitAccess(bouncer.PublicAccess(httperror.LoggerHandler(h.authenticate)))).Methods(http.MethodPost)
 	h.Handle("/auth/logout",
 		bouncer.PublicAccess(httperror.LoggerHandler(h.logout))).Methods(http.MethodPost)
-
 	return h
 }

api/http/handler/auth/logout.go

@@ -3,11 +3,9 @@ package auth
 import (
 	"net/http"
 
-	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/logoutcontext"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/response"
-	"github.com/rs/zerolog/log"
 )
 
 // @id Logout
@@ -21,10 +19,7 @@ import (
 // @router /auth/logout [post]
 
 func (handler *Handler) logout(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	tokenData, err := security.RetrieveTokenData(r)
-	if err != nil {
-		log.Warn().Err(err).Msg("unable to retrieve user details from authentication token")
-	}
+	tokenData := handler.bouncer.JWTAuthLookup(r)
 
 	if tokenData != nil {
 		handler.KubernetesTokenCacheManager.RemoveUserFromCache(tokenData.ID)

api/http/handler/teammemberships/handler.go

@@ -3,9 +3,13 @@ package teammemberships
 import (
 	"net/http"
 
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/internal/endpointutils"
+	"github.com/portainer/portainer/api/kubernetes/cli"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
+	"github.com/rs/zerolog/log"
 
 	"github.com/gorilla/mux"
 )
@@ -13,7 +17,8 @@ import (
 // Handler is the HTTP handler used to handle team membership operations.
 type Handler struct {
 	*mux.Router
-	DataStore dataservices.DataStore
+	DataStore        dataservices.DataStore
+	K8sClientFactory *cli.ClientFactory
 }
 
 // NewHandler creates a handler to manage team membership operations.
@@ -31,3 +36,27 @@ func NewHandler(bouncer security.BouncerService) *Handler {
 
 	return h
 }
+
+func (handler *Handler) updateUserServiceAccounts(membership *portainer.TeamMembership) {
+	endpoints, err := handler.DataStore.Endpoint().EndpointsByTeamID(membership.TeamID)
+	if err != nil {
+		log.Error().Err(err).Msgf("failed fetching environments for team %d", membership.TeamID)
+		return
+	}
+	for _, endpoint := range endpoints {
+		restrictDefaultNamespace := endpoint.Kubernetes.Configuration.RestrictDefaultNamespace
+		// update kubernenets service accounts if the team is associated with a kubernetes environment
+		if endpointutils.IsKubernetesEndpoint(&endpoint) {
+			kubecli, err := handler.K8sClientFactory.GetKubeClient(&endpoint)
+			if err != nil {
+				log.Error().Err(err).Msgf("failed getting kube client for environment %d", endpoint.ID)
+				continue
+			}
+			teamIDs := []int{int(membership.TeamID)}
+			err = kubecli.SetupUserServiceAccount(int(membership.UserID), teamIDs, restrictDefaultNamespace)
+			if err != nil {
+				log.Error().Err(err).Msgf("failed setting-up service account for user %d", membership.UserID)
+			}
+		}
+	}
+}

api/http/handler/teammemberships/teammembership_create.go

@@ -91,5 +91,7 @@ func (handler *Handler) teamMembershipCreate(w http.ResponseWriter, r *http.Requ
 		return httperror.InternalServerError("Unable to persist team memberships inside the database", err)
 	}
 
+	defer handler.updateUserServiceAccounts(membership)
+
 	return response.JSON(w, membership)
 }

api/http/handler/teammemberships/teammembership_delete.go

@@ -52,5 +52,7 @@ func (handler *Handler) teamMembershipDelete(w http.ResponseWriter, r *http.Requ
 		return httperror.InternalServerError("Unable to remove the team membership from the database", err)
 	}
 
+	defer handler.updateUserServiceAccounts(membership)
+
 	return response.Empty(w)
 }

api/http/handler/teammemberships/teammembership_update.go

@@ -90,5 +90,7 @@ func (handler *Handler) teamMembershipUpdate(w http.ResponseWriter, r *http.Requ
 		return httperror.InternalServerError("Unable to persist membership changes inside the database", err)
 	}
 
+	defer handler.updateUserServiceAccounts(membership)
+
 	return response.JSON(w, membership)
 }

api/http/proxy/factory/kubernetes/token.go

@@ -1,10 +1,12 @@
 package kubernetes
 
 import (
+	"fmt"
 	"os"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
+	"github.com/rs/zerolog/log"
 )
 
 const defaultServiceAccountTokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token"
@@ -43,28 +45,62 @@ func (manager *tokenManager) GetAdminServiceAccountToken() string {
 	return manager.adminToken
 }
 
-// GetUserServiceAccountToken setup a user's service account if it does not exist, then retrieve its token
-func (manager *tokenManager) GetUserServiceAccountToken(userID int, endpointID portainer.EndpointID) (string, error) {
-	tokenFunc := func() (string, error) {
-		memberships, err := manager.dataStore.TeamMembership().TeamMembershipsByUserID(portainer.UserID(userID))
-		if err != nil {
-			return "", err
-		}
+func (manager *tokenManager) setupUserServiceAccounts(userID portainer.UserID, endpoint *portainer.Endpoint) error {
+	memberships, err := manager.dataStore.TeamMembership().TeamMembershipsByUserID(userID)
+	if err != nil {
+		return err
+	}
+
+	teamIds := make([]int, 0, len(memberships))
+	for _, membership := range memberships {
+		teamIds = append(teamIds, int(membership.TeamID))
+	}
+
+	restrictDefaultNamespace := endpoint.Kubernetes.Configuration.RestrictDefaultNamespace
+	err = manager.kubecli.SetupUserServiceAccount(int(userID), teamIds, restrictDefaultNamespace)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
 
-		teamIds := make([]int, 0, len(memberships))
+func (manager *tokenManager) UpdateUserServiceAccountsForEndpoint(endpointID portainer.EndpointID) {
+	endpoint, err := manager.dataStore.Endpoint().Endpoint(endpointID)
+	if err != nil {
+		log.Error().Err(err).Msgf("failed fetching environments %d", endpointID)
+		return
+	}
+
+	userIDs := make([]portainer.UserID, 0)
+	for u := range endpoint.UserAccessPolicies {
+		userIDs = append(userIDs, u)
+	}
+	for t := range endpoint.TeamAccessPolicies {
+		memberships, _ := manager.dataStore.TeamMembership().TeamMembershipsByTeamID(portainer.TeamID(t))
 		for _, membership := range memberships {
-			teamIds = append(teamIds, int(membership.TeamID))
+			userIDs = append(userIDs, membership.UserID)
 		}
+	}
 
+	for _, userID := range userIDs {
+		if err := manager.setupUserServiceAccounts(userID, endpoint); err != nil {
+			log.Error().Err(err).Msgf("failed setting-up service account for user %d", userID)
+		}
+	}
+}
+
+// GetUserServiceAccountToken setup a user's service account if it does not exist, then retrieve its token
+func (manager *tokenManager) GetUserServiceAccountToken(userID int, endpointID portainer.EndpointID) (string, error) {
+	tokenFunc := func() (string, error) {
 		endpoint, err := manager.dataStore.Endpoint().Endpoint(endpointID)
 		if err != nil {
+			log.Error().Err(err).Msgf("failed fetching environment %d", endpointID)
 			return "", err
 		}
 
-		restrictDefaultNamespace := endpoint.Kubernetes.Configuration.RestrictDefaultNamespace
-		err = manager.kubecli.SetupUserServiceAccount(userID, teamIds, restrictDefaultNamespace)
-		if err != nil {
-			return "", err
+		if err := manager.setupUserServiceAccounts(portainer.UserID(userID), endpoint); err != nil {
+			return "", fmt.Errorf("failed setting-up service account for user %d: %w", userID, err)
 		}
 
 		return manager.kubecli.GetServiceAccountBearerToken(userID)

api/http/proxy/factory/kubernetes/transport.go

@@ -49,7 +49,17 @@ func (transport *baseTransport) proxyKubernetesRequest(request *http.Request) (*
 	apiVersionRe := regexp.MustCompile(`^(/kubernetes)?/(api|apis/apps)/v[0-9](\.[0-9])?`)
 	requestPath := apiVersionRe.ReplaceAllString(request.URL.Path, "")
 
+	endpointRe := regexp.MustCompile(`([0-9]+)`)
+	endpointIDMatch := endpointRe.FindAllString(request.RequestURI, 1)
+	endpointID := 0
+	if len(endpointIDMatch) > 0 {
+		endpointID, _ = strconv.Atoi(endpointIDMatch[0])
+	}
+
 	switch {
+	case strings.EqualFold(requestPath, "/namespaces/portainer/configmaps/portainer-config") && (request.Method == "PUT" || request.Method == "POST"):
+		defer transport.tokenManager.UpdateUserServiceAccountsForEndpoint(portainer.EndpointID(endpointID))
+		return transport.executeKubernetesRequest(request)
 	case strings.EqualFold(requestPath, "/namespaces"):
 		return transport.executeKubernetesRequest(request)
 	case strings.HasPrefix(requestPath, "/namespaces"):

api/http/server.go

@@ -264,6 +264,7 @@ func (server *Server) Start() error {
 
 	var teamMembershipHandler = teammemberships.NewHandler(requestBouncer)
 	teamMembershipHandler.DataStore = server.DataStore
+	teamMembershipHandler.K8sClientFactory = server.KubernetesClientFactory
 
 	var systemHandler = system.NewHandler(requestBouncer,
 		server.Status,

api/internal/testhelpers/datastore.go

@@ -306,6 +306,19 @@ func (s *stubEndpointService) GetNextIdentifier() int {
 	return len(s.endpoints)
 }
 
+func (s *stubEndpointService) EndpointsByTeamID(teamID portainer.TeamID) ([]portainer.Endpoint, error) {
+	var endpoints = make([]portainer.Endpoint, 0)
+
+	for _, e := range s.endpoints {
+		for t := range e.TeamAccessPolicies {
+			if t == teamID {
+				endpoints = append(endpoints, e)
+			}
+		}
+	}
+	return endpoints, nil
+}
+
 // WithEndpoints option will instruct testDatastore to return provided environments(endpoints)
 func WithEndpoints(endpoints []portainer.Endpoint) datastoreOption {
 	return func(d *testDatastore) {