From 75fb26d3c4ba1f779d5dfaf0f0307518fa0a6629 Mon Sep 17 00:00:00 2001 From: stevensbkang Date: Tue, 13 Aug 2024 14:47:55 +1200 Subject: [PATCH] refactor(k8s): performance optimisation --- api/http/handler/kubernetes/namespaces.go | 34 +++++++++++-- api/kubernetes/cli/client.go | 15 +++--- api/kubernetes/cli/namespace.go | 61 +++++++++++++++++++---- api/kubernetes/cli/resource_quota.go | 4 +- api/kubernetes/cli/utils.go | 36 ------------- 5 files changed, 89 insertions(+), 61 deletions(-) delete mode 100644 api/kubernetes/cli/utils.go diff --git a/api/http/handler/kubernetes/namespaces.go b/api/http/handler/kubernetes/namespaces.go index 219104e0e..0fa9fb961 100644 --- a/api/http/handler/kubernetes/namespaces.go +++ b/api/http/handler/kubernetes/namespaces.go @@ -4,6 +4,7 @@ import ( "fmt" "net/http" + "github.com/portainer/portainer/api/http/middlewares" models "github.com/portainer/portainer/api/http/models/kubernetes" httperror "github.com/portainer/portainer/pkg/libhttp/error" "github.com/portainer/portainer/pkg/libhttp/request" @@ -33,17 +34,28 @@ func (handler *Handler) getKubernetesNamespaces(w http.ResponseWriter, r *http.R cli, httpErr := handler.getProxyKubeClient(r) if httpErr != nil { - return httperror.InternalServerError("an error occurred during the getKubernetesNamespaces operation, unable to get a Kubernetes client for the user. Error: ", httpErr) + return httperror.InternalServerError("an error occurred during the getKubernetesNamespacesCount operation, unable to get a Kubernetes client for the user. Error: ", httpErr) } - namespaces, err := cli.GetNamespaces() + endpoint, err := middlewares.FetchEndpoint(r) + if err != nil { + return httperror.NotFound("Unable to find an environment on request context", err) + } + + pcli, err := handler.KubernetesClientFactory.GetPrivilegedKubeClient(endpoint) + if err != nil { + return httperror.InternalServerError("an error occurred during the getKubernetesNamespaces operation, unable to get a privileged Kubernetes client for the user. Error: ", err) + } + pcli.IsKubeAdmin = cli.IsKubeAdmin + pcli.NonAdminNamespaces = cli.NonAdminNamespaces + + namespaces, err := pcli.GetNamespaces() if err != nil { return httperror.InternalServerError("Unable to retrieve namespaces", err) } - // if withResourceQuota is set to true, grab all resource quotas associated with the namespaces if withResourceQuota { - return cli.CombineNamespacesWithResourceQuotas(namespaces, w) + return pcli.CombineNamespacesWithResourceQuotas(namespaces, w) } return response.JSON(w, namespaces) @@ -64,12 +76,24 @@ func (handler *Handler) getKubernetesNamespaces(w http.ResponseWriter, r *http.R // @failure 500 "Server error" // @router /kubernetes/{id}/namespaces/count [get] func (handler *Handler) getKubernetesNamespacesCount(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { + endpoint, err := middlewares.FetchEndpoint(r) + if err != nil { + return httperror.NotFound("Unable to find an environment on request context", err) + } + cli, httpErr := handler.getProxyKubeClient(r) if httpErr != nil { return httperror.InternalServerError("an error occurred during the getKubernetesNamespacesCount operation, unable to get a Kubernetes client for the user. Error: ", httpErr) } - namespaces, err := cli.GetNamespaces() + pcli, err := handler.KubernetesClientFactory.GetPrivilegedKubeClient(endpoint) + if err != nil { + return httperror.InternalServerError("an error occurred during the getKubernetesNamespacesCount operation, unable to get a privileged Kubernetes client for the user. Error: ", err) + } + pcli.IsKubeAdmin = cli.IsKubeAdmin + pcli.NonAdminNamespaces = cli.NonAdminNamespaces + + namespaces, err := pcli.GetNamespaces() if err != nil { return httperror.InternalServerError("an error occurred during the getKubernetesNamespacesCount operation, unable to retrieve namespaces from the Kubernetes cluster to count the total. Error: ", err) } diff --git a/api/kubernetes/cli/client.go b/api/kubernetes/cli/client.go index f0f49ff9b..47735a257 100644 --- a/api/kubernetes/cli/client.go +++ b/api/kubernetes/cli/client.go @@ -44,8 +44,8 @@ type ( cli kubernetes.Interface instanceID string mu sync.Mutex - isKubeAdmin bool - nonAdminNamespaces []string + IsKubeAdmin bool + NonAdminNamespaces []string } ) @@ -127,7 +127,7 @@ func (factory *ClientFactory) SetProxyKubeClient(endpointID, userID string, cli // CreateKubeClientFromKubeConfig creates a KubeClient from a clusterID, and // Kubernetes config. -func (factory *ClientFactory) CreateKubeClientFromKubeConfig(clusterID string, kubeConfig []byte, isKubeAdmin bool, nonAdminNamespaces []string) (*KubeClient, error) { +func (factory *ClientFactory) CreateKubeClientFromKubeConfig(clusterID string, kubeConfig []byte, IsKubeAdmin bool, NonAdminNamespaces []string) (*KubeClient, error) { config, err := clientcmd.NewClientConfigFromBytes(kubeConfig) if err != nil { return nil, fmt.Errorf("failed to create a client config from kubeconfig: %w", err) @@ -149,8 +149,8 @@ func (factory *ClientFactory) CreateKubeClientFromKubeConfig(clusterID string, k return &KubeClient{ cli: cli, instanceID: factory.instanceID, - isKubeAdmin: isKubeAdmin, - nonAdminNamespaces: nonAdminNamespaces, + IsKubeAdmin: IsKubeAdmin, + NonAdminNamespaces: NonAdminNamespaces, }, nil } @@ -161,9 +161,8 @@ func (factory *ClientFactory) createCachedPrivilegedKubeClient(endpoint *portain } return &KubeClient{ - cli: cli, - instanceID: factory.instanceID, - isKubeAdmin: true, + cli: cli, + instanceID: factory.instanceID, }, nil } diff --git a/api/kubernetes/cli/namespace.go b/api/kubernetes/cli/namespace.go index e8bfdb84e..98afe4beb 100644 --- a/api/kubernetes/cli/namespace.go +++ b/api/kubernetes/cli/namespace.go @@ -3,15 +3,20 @@ package cli import ( "context" "fmt" + "net/http" "strconv" "time" "github.com/pkg/errors" portainer "github.com/portainer/portainer/api" + portaineree "github.com/portainer/portainer/api" models "github.com/portainer/portainer/api/http/models/kubernetes" "github.com/portainer/portainer/api/stacks/stackutils" + httperror "github.com/portainer/portainer/pkg/libhttp/error" + "github.com/portainer/portainer/pkg/libhttp/response" "github.com/rs/zerolog/log" v1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -33,7 +38,7 @@ func defaultSystemNamespaces() map[string]struct{} { // GetNamespaces gets the namespaces in the current k8s environment(endpoint). func (kcl *KubeClient) GetNamespaces() (map[string]portainer.K8sNamespaceInfo, error) { - if kcl.isKubeAdmin { + if kcl.IsKubeAdmin { return kcl.fetchNamespacesForAdmin() } return kcl.fetchNamespacesForNonAdmin() @@ -44,7 +49,7 @@ func (kcl *KubeClient) GetNamespaces() (map[string]portainer.K8sNamespaceInfo, e func (kcl *KubeClient) fetchNamespacesForAdmin() (map[string]portainer.K8sNamespaceInfo, error) { namespaces, err := kcl.cli.CoreV1().Namespaces().List(context.TODO(), metav1.ListOptions{}) if err != nil { - return nil, fmt.Errorf("failed to list namespaces for the admin user: %w", err) + return nil, fmt.Errorf("an error occurred during the fetchNamespacesForAdmin operation, unable to list namespaces for the admin user: %w", err) } results := make(map[string]portainer.K8sNamespaceInfo) @@ -56,15 +61,26 @@ func (kcl *KubeClient) fetchNamespacesForAdmin() (map[string]portainer.K8sNamesp // fetchNamespacesForNonAdmin gets the namespaces in the current k8s environment(endpoint) for the non-admin user. func (kcl *KubeClient) fetchNamespacesForNonAdmin() (map[string]portainer.K8sNamespaceInfo, error) { + log.Debug().Msgf("Fetching namespaces for non-admin user: %v", kcl.NonAdminNamespaces) results := make(map[string]portainer.K8sNamespaceInfo) - log.Debug().Msgf("Fetching namespaces for non-admin user: %v", kcl.nonAdminNamespaces) - if len(kcl.nonAdminNamespaces) > 0 { - for _, ns := range kcl.nonAdminNamespaces { - namespace, err := kcl.GetNamespace(ns) - if err != nil { - return nil, fmt.Errorf("failed to get namespace %s for the non-admin user: %w", ns, err) - } - results[ns] = namespace + + if len(kcl.NonAdminNamespaces) == 0 { + return results, nil + } + + namespaces, err := kcl.cli.CoreV1().Namespaces().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("an error occurred during the fetchNamespacesForNonAdmin operation, unable to list namespaces for the non-admin user: %w", err) + } + + nonAdminNamespaceSet := make(map[string]struct{}, len(kcl.NonAdminNamespaces)) + for _, ns := range kcl.NonAdminNamespaces { + nonAdminNamespaceSet[ns] = struct{}{} + } + + for _, namespace := range namespaces.Items { + if _, exists := nonAdminNamespaceSet[namespace.Name]; exists { + results[namespace.Name] = parseNamespace(&namespace) } } @@ -237,3 +253,28 @@ func (kcl *KubeClient) DeleteNamespace(namespace string) error { } return fmt.Errorf("namespace %s not found", namespace) } + +// CombineNamespacesWithResourceQuotas combines namespaces with resource quotas where matching is based on "portainer-rq-"+namespace.Name +func (kcl *KubeClient) CombineNamespacesWithResourceQuotas(namespaces map[string]portaineree.K8sNamespaceInfo, w http.ResponseWriter) *httperror.HandlerError { + resourceQuotas, err := kcl.GetResourceQuotas("") + if err != nil { + return httperror.InternalServerError("an error occurred during the CombineNamespacesWithResourceQuotas operation, unable to retrieve resource quotas from the Kubernetes for an admin user. Error: ", err) + } + + if len(*resourceQuotas) > 0 { + return response.JSON(w, kcl.UpdateNamespacesWithResourceQuotas(namespaces, *resourceQuotas)) + } + + return response.JSON(w, namespaces) +} + +// CombineNamespaceWithResourceQuota combines a namespace with a resource quota prefixed with "portainer-rq-"+namespace.Name +func (kcl *KubeClient) CombineNamespaceWithResourceQuota(namespace portaineree.K8sNamespaceInfo, w http.ResponseWriter) *httperror.HandlerError { + resourceQuota, err := kcl.GetPortainerResourceQuota(namespace.Name) + if err != nil && !k8serrors.IsNotFound(err) { + return httperror.InternalServerError(fmt.Sprintf("an error occurred during the CombineNamespaceWithResourceQuota operation, unable to retrieve the resource quota associated with the namespace: %s for a non-admin user. Error: ", namespace.Name), err) + } + namespace.ResourceQuota = resourceQuota + + return response.JSON(w, namespace) +} diff --git a/api/kubernetes/cli/resource_quota.go b/api/kubernetes/cli/resource_quota.go index 792d9b162..0614be2c0 100644 --- a/api/kubernetes/cli/resource_quota.go +++ b/api/kubernetes/cli/resource_quota.go @@ -13,7 +13,7 @@ import ( // GetResourceQuotas gets all resource quotas in the current k8s environment(endpoint). // The kube client must have cluster scope read access to do this. func (kcl *KubeClient) GetResourceQuotas(namespace string) (*[]corev1.ResourceQuota, error) { - if kcl.isKubeAdmin { + if kcl.IsKubeAdmin { return kcl.fetchResourceQuotasForAdmin() } return kcl.fetchResourceQuotasForNonAdmin() @@ -34,7 +34,7 @@ func (kcl *KubeClient) fetchResourceQuotasForAdmin() (*[]corev1.ResourceQuota, e // the role of the user must have read access to the resource quotas in the defined namespaces. func (kcl *KubeClient) fetchResourceQuotasForNonAdmin() (*[]corev1.ResourceQuota, error) { resourceQuotas := []corev1.ResourceQuota{} - for _, namespace := range kcl.nonAdminNamespaces { + for _, namespace := range kcl.NonAdminNamespaces { resourceQuota, err := kcl.GetResourceQuota(namespace, "portainer-rq-"+namespace) if err != nil && !k8serrors.IsNotFound(err) { return nil, fmt.Errorf("an error occured, failed to get a resource quota %s of the namespace %s for the non-admin user: %w", resourceQuota.Name, namespace, err) diff --git a/api/kubernetes/cli/utils.go b/api/kubernetes/cli/utils.go deleted file mode 100644 index ffda245a0..000000000 --- a/api/kubernetes/cli/utils.go +++ /dev/null @@ -1,36 +0,0 @@ -package cli - -import ( - "fmt" - "net/http" - - portaineree "github.com/portainer/portainer/api" - httperror "github.com/portainer/portainer/pkg/libhttp/error" - "github.com/portainer/portainer/pkg/libhttp/response" - k8serrors "k8s.io/apimachinery/pkg/api/errors" -) - -// CombineNamespacesWithResourceQuotas combines namespaces with resource quotas where matching is based on "portainer-rq-"+namespace.Name -func (kcl *KubeClient) CombineNamespacesWithResourceQuotas(namespaces map[string]portaineree.K8sNamespaceInfo, w http.ResponseWriter) *httperror.HandlerError { - resourceQuotas, err := kcl.GetResourceQuotas("") - if err != nil { - return httperror.InternalServerError("an error occurred during the CombineNamespacesWithResourceQuotas operation, unable to retrieve resource quotas from the Kubernetes for an admin user. Error: ", err) - } - - if len(*resourceQuotas) > 0 { - return response.JSON(w, kcl.UpdateNamespacesWithResourceQuotas(namespaces, *resourceQuotas)) - } - - return response.JSON(w, namespaces) -} - -// CombineNamespaceWithResourceQuota combines a namespace with a resource quota prefixed with "portainer-rq-"+namespace.Name -func (kcl *KubeClient) CombineNamespaceWithResourceQuota(namespace portaineree.K8sNamespaceInfo, w http.ResponseWriter) *httperror.HandlerError { - resourceQuota, err := kcl.GetPortainerResourceQuota(namespace.Name) - if err != nil && !k8serrors.IsNotFound(err) { - return httperror.InternalServerError(fmt.Sprintf("an error occurred during the CombineNamespaceWithResourceQuota operation, unable to retrieve the resource quota associated with the namespace: %s for a non-admin user. Error: ", namespace.Name), err) - } - namespace.ResourceQuota = resourceQuota - - return response.JSON(w, namespace) -}