From 2c15dcd1f2cf43967f6612ed46af2cc168112318 Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Wed, 12 Aug 2020 17:10:28 +1200 Subject: [PATCH] feat(k8s): use instance ID to create unique k8s resources (#4196) --- api/bolt/version/version.go | 39 +++++++++++++++++++++++++-- api/cmd/portainer/main.go | 27 ++++++++++++++++--- api/kubernetes/cli/client.go | 10 ++++--- api/kubernetes/cli/naming.go | 12 ++++----- api/kubernetes/cli/secret.go | 4 +-- api/kubernetes/cli/service_account.go | 8 +++--- api/portainer.go | 2 ++ 7 files changed, 82 insertions(+), 20 deletions(-) diff --git a/api/bolt/version/version.go b/api/bolt/version/version.go index 992224eea..eca755a57 100644 --- a/api/bolt/version/version.go +++ b/api/bolt/version/version.go @@ -10,8 +10,9 @@ import ( const ( // BucketName represents the name of the bucket where this service stores data. - BucketName = "version" - versionKey = "DB_VERSION" + BucketName = "version" + versionKey = "DB_VERSION" + instanceKey = "INSTANCE_ID" ) // Service represents a service to manage stored versions. @@ -64,3 +65,37 @@ func (service *Service) StoreDBVersion(version int) error { return bucket.Put([]byte(versionKey), data) }) } + +// InstanceID retrieves the stored instance ID. +func (service *Service) InstanceID() (string, error) { + var data []byte + + err := service.db.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(BucketName)) + + value := bucket.Get([]byte(instanceKey)) + if value == nil { + return errors.ErrObjectNotFound + } + + data = make([]byte, len(value)) + copy(data, value) + + return nil + }) + if err != nil { + return "", err + } + + return string(data), nil +} + +// StoreInstanceID store the instance ID. +func (service *Service) StoreInstanceID(ID string) error { + return service.db.Update(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(BucketName)) + + data := []byte(ID) + return bucket.Put([]byte(instanceKey), data) + }) +} diff --git a/api/cmd/portainer/main.go b/api/cmd/portainer/main.go index 56abc972c..9d6188597 100644 --- a/api/cmd/portainer/main.go +++ b/api/cmd/portainer/main.go @@ -6,6 +6,10 @@ import ( "strings" "time" + "github.com/gofrs/uuid" + + "github.com/portainer/portainer/api/bolt/errors" + "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/bolt" "github.com/portainer/portainer/api/chisel" @@ -120,8 +124,8 @@ func initDockerClientFactory(signatureService portainer.DigitalSignatureService, return docker.NewClientFactory(signatureService, reverseTunnelService) } -func initKubernetesClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService) *kubecli.ClientFactory { - return kubecli.NewClientFactory(signatureService, reverseTunnelService) +func initKubernetesClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, instanceID string) *kubecli.ClientFactory { + return kubecli.NewClientFactory(signatureService, reverseTunnelService, instanceID) } func initSnapshotService(snapshotInterval string, dataStore portainer.DataStore, dockerClientFactory *docker.ClientFactory, kubernetesClientFactory *kubecli.ClientFactory) (portainer.SnapshotService, error) { @@ -340,6 +344,23 @@ func main() { dataStore := initDataStore(*flags.Data, fileService) defer dataStore.Close() + instanceID, err := dataStore.Version().InstanceID() + if err == errors.ErrObjectNotFound { + uid, err := uuid.NewV4() + if err != nil { + log.Fatal(err) + } + + instanceID = uid.String() + + err = dataStore.Version().StoreInstanceID(instanceID) + if err != nil { + log.Fatal(err) + } + } else if err == nil { + log.Fatal(err) + } + jwtService, err := initJWTService(dataStore) if err != nil { log.Fatal(err) @@ -363,7 +384,7 @@ func main() { reverseTunnelService := chisel.NewService(dataStore) dockerClientFactory := initDockerClientFactory(digitalSignatureService, reverseTunnelService) - kubernetesClientFactory := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService) + kubernetesClientFactory := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService, instanceID) snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory) if err != nil { diff --git a/api/kubernetes/cli/client.go b/api/kubernetes/cli/client.go index b87faac92..707bf3924 100644 --- a/api/kubernetes/cli/client.go +++ b/api/kubernetes/cli/client.go @@ -19,20 +19,23 @@ type ( ClientFactory struct { reverseTunnelService portainer.ReverseTunnelService signatureService portainer.DigitalSignatureService + instanceID string endpointClients cmap.ConcurrentMap } // KubeClient represent a service used to execute Kubernetes operations KubeClient struct { - cli *kubernetes.Clientset + cli *kubernetes.Clientset + instanceID string } ) // NewClientFactory returns a new instance of a ClientFactory -func NewClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService) *ClientFactory { +func NewClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, instanceID string) *ClientFactory { return &ClientFactory{ signatureService: signatureService, reverseTunnelService: reverseTunnelService, + instanceID: instanceID, endpointClients: cmap.New(), } } @@ -62,7 +65,8 @@ func (factory *ClientFactory) createKubeClient(endpoint *portainer.Endpoint) (po } kubecli := &KubeClient{ - cli: cli, + cli: cli, + instanceID: factory.instanceID, } return kubecli, nil diff --git a/api/kubernetes/cli/naming.go b/api/kubernetes/cli/naming.go index cbc0c4675..9297ff6be 100644 --- a/api/kubernetes/cli/naming.go +++ b/api/kubernetes/cli/naming.go @@ -13,14 +13,14 @@ const ( portainerConfigMapAccessPoliciesKey = "NamespaceAccessPolicies" ) -func userServiceAccountName(userID int) string { - return fmt.Sprintf("%s-%d", portainerUserServiceAccountPrefix, userID) +func userServiceAccountName(userID int, instanceID string) string { + return fmt.Sprintf("%s-%s-%d", portainerUserServiceAccountPrefix, instanceID, userID) } -func userServiceAccountTokenSecretName(serviceAccountName string) string { - return fmt.Sprintf("%s-secret", serviceAccountName) +func userServiceAccountTokenSecretName(serviceAccountName string, instanceID string) string { + return fmt.Sprintf("%s-%s-secret", instanceID, serviceAccountName) } -func namespaceClusterRoleBindingName(namespace string) string { - return fmt.Sprintf("%s-%s", portainerRBPrefix, namespace) +func namespaceClusterRoleBindingName(namespace string, instanceID string) string { + return fmt.Sprintf("%s-%s-%s", portainerRBPrefix, instanceID, namespace) } diff --git a/api/kubernetes/cli/secret.go b/api/kubernetes/cli/secret.go index 87ba35f53..3235cb304 100644 --- a/api/kubernetes/cli/secret.go +++ b/api/kubernetes/cli/secret.go @@ -11,7 +11,7 @@ import ( ) func (kcl *KubeClient) createServiceAccountToken(serviceAccountName string) error { - serviceAccountSecretName := userServiceAccountTokenSecretName(serviceAccountName) + serviceAccountSecretName := userServiceAccountTokenSecretName(serviceAccountName, kcl.instanceID) serviceAccountSecret := &v1.Secret{ TypeMeta: metav1.TypeMeta{}, @@ -33,7 +33,7 @@ func (kcl *KubeClient) createServiceAccountToken(serviceAccountName string) erro } func (kcl *KubeClient) getServiceAccountToken(serviceAccountName string) (string, error) { - serviceAccountSecretName := userServiceAccountTokenSecretName(serviceAccountName) + serviceAccountSecretName := userServiceAccountTokenSecretName(serviceAccountName, kcl.instanceID) secret, err := kcl.cli.CoreV1().Secrets(portainerNamespace).Get(serviceAccountSecretName, metav1.GetOptions{}) if err != nil { diff --git a/api/kubernetes/cli/service_account.go b/api/kubernetes/cli/service_account.go index 52e1b1fe5..d8abc6f0f 100644 --- a/api/kubernetes/cli/service_account.go +++ b/api/kubernetes/cli/service_account.go @@ -9,7 +9,7 @@ import ( // GetServiceAccountBearerToken returns the ServiceAccountToken associated to the specified user. func (kcl *KubeClient) GetServiceAccountBearerToken(userID int) (string, error) { - serviceAccountName := userServiceAccountName(userID) + serviceAccountName := userServiceAccountName(userID, kcl.instanceID) return kcl.getServiceAccountToken(serviceAccountName) } @@ -18,7 +18,7 @@ func (kcl *KubeClient) GetServiceAccountBearerToken(userID int) (string, error) // cluster before creating a ServiceAccount and a ServiceAccountToken for the specified Portainer user. //It will also create required default RoleBinding and ClusterRoleBinding rules. func (kcl *KubeClient) SetupUserServiceAccount(userID int, teamIDs []int) error { - serviceAccountName := userServiceAccountName(userID) + serviceAccountName := userServiceAccountName(userID, kcl.instanceID) err := kcl.ensureRequiredResourcesExist() if err != nil { @@ -114,7 +114,7 @@ func (kcl *KubeClient) ensureServiceAccountHasPortainerUserClusterRole(serviceAc } func (kcl *KubeClient) removeNamespaceAccessForServiceAccount(serviceAccountName, namespace string) error { - roleBindingName := namespaceClusterRoleBindingName(namespace) + roleBindingName := namespaceClusterRoleBindingName(namespace, kcl.instanceID) roleBinding, err := kcl.cli.RbacV1().RoleBindings(namespace).Get(roleBindingName, metav1.GetOptions{}) if k8serrors.IsNotFound(err) { @@ -138,7 +138,7 @@ func (kcl *KubeClient) removeNamespaceAccessForServiceAccount(serviceAccountName } func (kcl *KubeClient) ensureNamespaceAccessForServiceAccount(serviceAccountName, namespace string) error { - roleBindingName := namespaceClusterRoleBindingName(namespace) + roleBindingName := namespaceClusterRoleBindingName(namespace, kcl.instanceID) roleBinding, err := kcl.cli.RbacV1().RoleBindings(namespace).Get(roleBindingName, metav1.GetOptions{}) if k8serrors.IsNotFound(err) { diff --git a/api/portainer.go b/api/portainer.go index 58ee3d662..5e87a6a2a 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -1095,6 +1095,8 @@ type ( VersionService interface { DBVersion() (int, error) StoreDBVersion(version int) error + InstanceID() (string, error) + StoreInstanceID(ID string) error } // WebhookService represents a service for managing webhook data.