fix(ingresses): migrate to new allow/disallow format EE-4465 (#7893)

api/cmd/portainer/main.go

@@ -719,6 +719,23 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatal().Msg("failed to fetch SSL settings from DB")
 	}
 
+	// FIXME: In 2.16 we changed the way ingress controller permissions are
+	// stored. Instead of being stored as annotation on an ingress rule, we keep
+	// them in our database. However, in order to run the migration we need an
+	// admin kube client to run lookup the old ingress rules and compare them
+	// with the current existing ingress classes.
+	//
+	// Unfortunately, our migrations run as part of the database initialization
+	// and our kubeclients require an initialized database. So it is not
+	// possible to do this migration as part of our normal flow. We DO have a
+	// migration which toggles a boolean in kubernetes configuration that
+	// indicated that this "post init" migration should be run. If/when this is
+	// resolved we can remove this function.
+	err = kubernetesClientFactory.PostInitMigrateIngresses()
+	if err != nil {
+		log.Fatal().Err(err).Msg("failure during creation of new database")
+	}
+
 	return &http.Server{
 		AuthorizationService:        authorizationService,
 		ReverseTunnelService:        reverseTunnelService,

api/datastore/migrator/migrate_dbversion70.go

@@ -60,6 +60,7 @@ func (m *Migrator) updateIngressFieldsForEnvDB70() error {
 	for _, endpoint := range endpoints {
 		endpoint.Kubernetes.Configuration.IngressAvailabilityPerNamespace = true
 		endpoint.Kubernetes.Configuration.AllowNoneIngressClass = false
+		endpoint.PostInitMigrations.MigrateIngresses = true
 
 		err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
 		if err != nil {

api/datastore/test_data/output_24_to_latest.json

@@ -66,6 +66,9 @@
       },
       "LastCheckInDate": 0,
       "Name": "local",
+      "PostInitMigrations": {
+        "MigrateIngresses": true
+      },
       "PublicURL": "",
       "QueryDate": 0,
       "SecuritySettings": {

api/http/handler/kubernetes/ingresses.go

@@ -279,6 +279,7 @@ func (handler *Handler) updateKubernetesIngressControllers(w http.ResponseWriter
 			err,
 		)
 	}
+
 	// Add none controller if "AllowNone" is set for endpoint.
 	if endpoint.Kubernetes.Configuration.AllowNoneIngressClass {
 		controllers = append(controllers, models.K8sIngressController{
@@ -287,6 +288,7 @@ func (handler *Handler) updateKubernetesIngressControllers(w http.ResponseWriter
 			Type:      "custom",
 		})
 	}
+
 	var updatedClasses []portainer.KubernetesIngressClassConfig
 	for i := range controllers {
 		controllers[i].Availability = true

api/kubernetes/cli/client.go

@@ -12,6 +12,7 @@ import (
 	"github.com/pkg/errors"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
+	"github.com/rs/zerolog/log"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
@@ -219,3 +220,127 @@ func buildLocalClient() (*kubernetes.Clientset, error) {
 
 	return kubernetes.NewForConfig(config)
 }
+
+func (factory *ClientFactory) PostInitMigrateIngresses() error {
+	endpoints, err := factory.dataStore.Endpoint().Endpoints()
+	if err != nil {
+		return err
+	}
+	for i := range endpoints {
+		// Early exit if we do not need to migrate!
+		if endpoints[i].PostInitMigrations.MigrateIngresses == false {
+			return nil
+		}
+
+		err := factory.migrateEndpointIngresses(&endpoints[i])
+		if err != nil {
+			log.Debug().Err(err).Msg("failure migrating endpoint ingresses")
+		}
+	}
+
+	return nil
+}
+
+func (factory *ClientFactory) migrateEndpointIngresses(e *portainer.Endpoint) error {
+	// classes is a list of controllers which have been manually added to the
+	// cluster setup view. These need to all be allowed globally, but then
+	// blocked in specific namespaces which they were not previously allowed in.
+	classes := e.Kubernetes.Configuration.IngressClasses
+
+	// We need a kube client to gather namespace level permissions. In pre-2.16
+	// versions of portainer, the namespace level permissions were stored by
+	// creating an actual ingress rule in the cluster with a particular
+	// annotation indicating that it's name (the class name) should be allowed.
+	cli, err := factory.GetKubeClient(e)
+	if err != nil {
+		return err
+	}
+
+	detected, err := cli.GetIngressControllers()
+	if err != nil {
+		return err
+	}
+
+	// newControllers is a set of all currently detected controllers.
+	newControllers := make(map[string]struct{})
+	for _, controller := range detected {
+		newControllers[controller.ClassName] = struct{}{}
+	}
+
+	namespaces, err := cli.GetNamespaces()
+	if err != nil {
+		return err
+	}
+
+	// Set of namespaces, if any, in which "allow none" should be true.
+	allow := make(map[string]map[string]struct{})
+	for _, c := range classes {
+		allow[c.Name] = make(map[string]struct{})
+	}
+	allow["none"] = make(map[string]struct{})
+
+	for namespace := range namespaces {
+		// Compare old annotations with currently detected controllers.
+		ingresses, err := cli.GetIngresses(namespace)
+		if err != nil {
+			return fmt.Errorf("failure getting ingresses during migration")
+		}
+		for _, ingress := range ingresses {
+			oldController, ok := ingress.Annotations["ingress.portainer.io/ingress-type"]
+			if !ok {
+				// Skip rules without our old annotation.
+				continue
+			}
+
+			if _, ok := newControllers[oldController]; ok {
+				// Skip rules which match a detected controller.
+				// TODO: Allow this particular controller.
+				allow[oldController][ingress.Namespace] = struct{}{}
+				continue
+			}
+
+			allow["none"][ingress.Namespace] = struct{}{}
+		}
+	}
+
+	// Locally, disable "allow none" for namespaces not inside shouldAllowNone.
+	var newClasses []portainer.KubernetesIngressClassConfig
+	for _, c := range classes {
+		var blocked []string
+		for namespace := range namespaces {
+			if _, ok := allow[c.Name][namespace]; ok {
+				continue
+			}
+			blocked = append(blocked, namespace)
+		}
+
+		newClasses = append(newClasses, portainer.KubernetesIngressClassConfig{
+			Name:              c.Name,
+			Type:              c.Type,
+			GloballyBlocked:   false,
+			BlockedNamespaces: blocked,
+		})
+	}
+
+	// Handle "none".
+	if len(allow["none"]) != 0 {
+		e.Kubernetes.Configuration.AllowNoneIngressClass = true
+		var disallowNone []string
+		for namespace := range namespaces {
+			if _, ok := allow["none"][namespace]; ok {
+				continue
+			}
+			disallowNone = append(disallowNone, namespace)
+		}
+		newClasses = append(newClasses, portainer.KubernetesIngressClassConfig{
+			Name:              "none",
+			Type:              "custom",
+			GloballyBlocked:   false,
+			BlockedNamespaces: disallowNone,
+		})
+	}
+
+	e.Kubernetes.Configuration.IngressClasses = newClasses
+	e.PostInitMigrations.MigrateIngresses = false
+	return factory.dataStore.Endpoint().UpdateEndpoint(e.ID, e)
+}

api/portainer.go

@@ -352,6 +352,9 @@ type (
 		// Whether the device has been trusted or not by the user
 		UserTrusted bool
 
+		// Whether we need to run any "post init migrations".
+		PostInitMigrations EndpointPostInitMigrations `json:"PostInitMigrations"`
+
 		Edge struct {
 			// Whether the device has been started in edge async mode
 			AsyncMode bool
@@ -453,6 +456,11 @@ type (
 		EdgeStacks map[EdgeStackID]bool
 	}
 
+	// EndpointPostInitMigrations
+	EndpointPostInitMigrations struct {
+		MigrateIngresses bool `json:"MigrateIngresses"`
+	}
+
 	// Extension represents a deprecated Portainer extension
 	Extension struct {
 		// Extension Identifier