feat(k8s): use instance ID to create unique k8s resources (#4196)

api/bolt/version/version.go

@@ -10,8 +10,9 @@ import (
 
 const (
 	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "version"
+	BucketName  = "version"
-	versionKey = "DB_VERSION"
+	versionKey  = "DB_VERSION"
+	instanceKey = "INSTANCE_ID"
 )
 
 // Service represents a service to manage stored versions.
@@ -64,3 +65,37 @@ func (service *Service) StoreDBVersion(version int) error {
 		return bucket.Put([]byte(versionKey), data)
 	})
 }
+
+// InstanceID retrieves the stored instance ID.
+func (service *Service) InstanceID() (string, error) {
+	var data []byte
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		value := bucket.Get([]byte(instanceKey))
+		if value == nil {
+			return errors.ErrObjectNotFound
+		}
+
+		data = make([]byte, len(value))
+		copy(data, value)
+
+		return nil
+	})
+	if err != nil {
+		return "", err
+	}
+
+	return string(data), nil
+}
+
+// StoreInstanceID store the instance ID.
+func (service *Service) StoreInstanceID(ID string) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		data := []byte(ID)
+		return bucket.Put([]byte(instanceKey), data)
+	})
+}

api/cmd/portainer/main.go

@@ -6,6 +6,10 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gofrs/uuid"
+
+	"github.com/portainer/portainer/api/bolt/errors"
+
 	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt"
 	"github.com/portainer/portainer/api/chisel"
@@ -120,8 +124,8 @@ func initDockerClientFactory(signatureService portainer.DigitalSignatureService,
 	return docker.NewClientFactory(signatureService, reverseTunnelService)
 }
 
-func initKubernetesClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService) *kubecli.ClientFactory {
+func initKubernetesClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, instanceID string) *kubecli.ClientFactory {
-	return kubecli.NewClientFactory(signatureService, reverseTunnelService)
+	return kubecli.NewClientFactory(signatureService, reverseTunnelService, instanceID)
 }
 
 func initSnapshotService(snapshotInterval string, dataStore portainer.DataStore, dockerClientFactory *docker.ClientFactory, kubernetesClientFactory *kubecli.ClientFactory) (portainer.SnapshotService, error) {
@@ -340,6 +344,23 @@ func main() {
 	dataStore := initDataStore(*flags.Data, fileService)
 	defer dataStore.Close()
 
+	instanceID, err := dataStore.Version().InstanceID()
+	if err == errors.ErrObjectNotFound {
+		uid, err := uuid.NewV4()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		instanceID = uid.String()
+
+		err = dataStore.Version().StoreInstanceID(instanceID)
+		if err != nil {
+			log.Fatal(err)
+		}
+	} else if err == nil {
+		log.Fatal(err)
+	}
+
 	jwtService, err := initJWTService(dataStore)
 	if err != nil {
 		log.Fatal(err)
@@ -363,7 +384,7 @@ func main() {
 	reverseTunnelService := chisel.NewService(dataStore)
 
 	dockerClientFactory := initDockerClientFactory(digitalSignatureService, reverseTunnelService)
-	kubernetesClientFactory := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService)
+	kubernetesClientFactory := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService, instanceID)
 
 	snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory)
 	if err != nil {

api/kubernetes/cli/client.go

@@ -19,20 +19,23 @@ type (
 	ClientFactory struct {
 		reverseTunnelService portainer.ReverseTunnelService
 		signatureService     portainer.DigitalSignatureService
+		instanceID           string
 		endpointClients      cmap.ConcurrentMap
 	}
 
 	// KubeClient represent a service used to execute Kubernetes operations
 	KubeClient struct {
-		cli *kubernetes.Clientset
+		cli        *kubernetes.Clientset
+		instanceID string
 	}
 )
 
 // NewClientFactory returns a new instance of a ClientFactory
-func NewClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService) *ClientFactory {
+func NewClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, instanceID string) *ClientFactory {
 	return &ClientFactory{
 		signatureService:     signatureService,
 		reverseTunnelService: reverseTunnelService,
+		instanceID:           instanceID,
 		endpointClients:      cmap.New(),
 	}
 }
@@ -62,7 +65,8 @@ func (factory *ClientFactory) createKubeClient(endpoint *portainer.Endpoint) (po
 	}
 
 	kubecli := &KubeClient{
-		cli: cli,
+		cli:        cli,
+		instanceID: factory.instanceID,
 	}
 
 	return kubecli, nil

api/kubernetes/cli/naming.go

@@ -13,14 +13,14 @@ const (
 	portainerConfigMapAccessPoliciesKey = "NamespaceAccessPolicies"
 )
 
-func userServiceAccountName(userID int) string {
+func userServiceAccountName(userID int, instanceID string) string {
-	return fmt.Sprintf("%s-%d", portainerUserServiceAccountPrefix, userID)
+	return fmt.Sprintf("%s-%s-%d", portainerUserServiceAccountPrefix, instanceID, userID)
 }
 
-func userServiceAccountTokenSecretName(serviceAccountName string) string {
+func userServiceAccountTokenSecretName(serviceAccountName string, instanceID string) string {
-	return fmt.Sprintf("%s-secret", serviceAccountName)
+	return fmt.Sprintf("%s-%s-secret", instanceID, serviceAccountName)
 }
 
-func namespaceClusterRoleBindingName(namespace string) string {
+func namespaceClusterRoleBindingName(namespace string, instanceID string) string {
-	return fmt.Sprintf("%s-%s", portainerRBPrefix, namespace)
+	return fmt.Sprintf("%s-%s-%s", portainerRBPrefix, instanceID, namespace)
 }

api/kubernetes/cli/secret.go

@@ -11,7 +11,7 @@ import (
 )
 
 func (kcl *KubeClient) createServiceAccountToken(serviceAccountName string) error {
-	serviceAccountSecretName := userServiceAccountTokenSecretName(serviceAccountName)
+	serviceAccountSecretName := userServiceAccountTokenSecretName(serviceAccountName, kcl.instanceID)
 
 	serviceAccountSecret := &v1.Secret{
 		TypeMeta: metav1.TypeMeta{},
@@ -33,7 +33,7 @@ func (kcl *KubeClient) createServiceAccountToken(serviceAccountName string) erro
 }
 
 func (kcl *KubeClient) getServiceAccountToken(serviceAccountName string) (string, error) {
-	serviceAccountSecretName := userServiceAccountTokenSecretName(serviceAccountName)
+	serviceAccountSecretName := userServiceAccountTokenSecretName(serviceAccountName, kcl.instanceID)
 
 	secret, err := kcl.cli.CoreV1().Secrets(portainerNamespace).Get(serviceAccountSecretName, metav1.GetOptions{})
 	if err != nil {

api/kubernetes/cli/service_account.go

@@ -9,7 +9,7 @@ import (
 
 // GetServiceAccountBearerToken returns the ServiceAccountToken associated to the specified user.
 func (kcl *KubeClient) GetServiceAccountBearerToken(userID int) (string, error) {
-	serviceAccountName := userServiceAccountName(userID)
+	serviceAccountName := userServiceAccountName(userID, kcl.instanceID)
 
 	return kcl.getServiceAccountToken(serviceAccountName)
 }
@@ -18,7 +18,7 @@ func (kcl *KubeClient) GetServiceAccountBearerToken(userID int) (string, error)
 // cluster before creating a ServiceAccount and a ServiceAccountToken for the specified Portainer user.
 //It will also create required default RoleBinding and ClusterRoleBinding rules.
 func (kcl *KubeClient) SetupUserServiceAccount(userID int, teamIDs []int) error {
-	serviceAccountName := userServiceAccountName(userID)
+	serviceAccountName := userServiceAccountName(userID, kcl.instanceID)
 
 	err := kcl.ensureRequiredResourcesExist()
 	if err != nil {
@@ -114,7 +114,7 @@ func (kcl *KubeClient) ensureServiceAccountHasPortainerUserClusterRole(serviceAc
 }
 
 func (kcl *KubeClient) removeNamespaceAccessForServiceAccount(serviceAccountName, namespace string) error {
-	roleBindingName := namespaceClusterRoleBindingName(namespace)
+	roleBindingName := namespaceClusterRoleBindingName(namespace, kcl.instanceID)
 
 	roleBinding, err := kcl.cli.RbacV1().RoleBindings(namespace).Get(roleBindingName, metav1.GetOptions{})
 	if k8serrors.IsNotFound(err) {
@@ -138,7 +138,7 @@ func (kcl *KubeClient) removeNamespaceAccessForServiceAccount(serviceAccountName
 }
 
 func (kcl *KubeClient) ensureNamespaceAccessForServiceAccount(serviceAccountName, namespace string) error {
-	roleBindingName := namespaceClusterRoleBindingName(namespace)
+	roleBindingName := namespaceClusterRoleBindingName(namespace, kcl.instanceID)
 
 	roleBinding, err := kcl.cli.RbacV1().RoleBindings(namespace).Get(roleBindingName, metav1.GetOptions{})
 	if k8serrors.IsNotFound(err) {

api/portainer.go

@@ -1095,6 +1095,8 @@ type (
 	VersionService interface {
 		DBVersion() (int, error)
 		StoreDBVersion(version int) error
+		InstanceID() (string, error)
+		StoreInstanceID(ID string) error
 	}
 
 	// WebhookService represents a service for managing webhook data.