fix(edge-stack): URI too large error for edge stacks with a large amount of environments [EE-5583] (#9085)

* refactor(edge-stacks): filter endpoints by edgeStack

* feat(api/endpoints): edge stack filter support filtering on status in stack

* refactor(endpoints): use separate query params and not JSON query param when querying for an edge stack

* feat(api/endpoints): handle stack filter on dynamic groups + unique list with multiple groups sharing environments

* fix(app/endpoints): edge stack related query params type definition

* fix(api/endpoints): rebase conflicts on imports

api/http/handler/edgegroups/associated_endpoints.go

@@ -7,7 +7,7 @@ import (
 
 type endpointSetType map[portainer.EndpointID]bool
 
-func getEndpointsByTags(tx dataservices.DataStoreTx, tagIDs []portainer.TagID, partialMatch bool) ([]portainer.EndpointID, error) {
+func GetEndpointsByTags(tx dataservices.DataStoreTx, tagIDs []portainer.TagID, partialMatch bool) ([]portainer.EndpointID, error) {
 	if len(tagIDs) == 0 {
 		return []portainer.EndpointID{}, nil
 	}

api/http/handler/edgegroups/edgegroup_inspect.go

@@ -50,7 +50,7 @@ func getEdgeGroup(tx dataservices.DataStoreTx, ID portainer.EdgeGroupID) (*porta
 	}
 
 	if edgeGroup.Dynamic {
-		endpoints, err := getEndpointsByTags(tx, edgeGroup.TagIDs, edgeGroup.PartialMatch)
+		endpoints, err := GetEndpointsByTags(tx, edgeGroup.TagIDs, edgeGroup.PartialMatch)
 		if err != nil {
 			return nil, httperror.InternalServerError("Unable to retrieve environments and environment groups for Edge group", err)
 		}

api/http/handler/edgegroups/edgegroup_list.go

@@ -84,7 +84,7 @@ func getEdgeGroupList(tx dataservices.DataStoreTx) ([]decoratedEdgeGroup, error)
 			EndpointTypes: []portainer.EndpointType{},
 		}
 		if edgeGroup.Dynamic {
-			endpointIDs, err := getEndpointsByTags(tx, edgeGroup.TagIDs, edgeGroup.PartialMatch)
+			endpointIDs, err := GetEndpointsByTags(tx, edgeGroup.TagIDs, edgeGroup.PartialMatch)
 			if err != nil {
 				return nil, httperror.InternalServerError("Unable to retrieve environments and environment groups for Edge group", err)
 			}

api/http/handler/endpoints/endpoint_list.go

@@ -46,6 +46,8 @@ const (
 // @param edgeCheckInPassedSeconds query number false "if bigger then zero, show only edge agents that checked-in in the last provided seconds (relevant only for edge agents)"
 // @param excludeSnapshots query bool false "if true, the snapshot data won't be retrieved"
 // @param name query string false "will return only environments(endpoints) with this name"
+// @param edgeStackId query portainer.EdgeStackID false "will return the environements of the specified edge stack"
+// @param edgeStackStatus query string false "only applied when edgeStackId exists. Filter the returned environments based on their deployment status in the stack (not the environment status!)" Enum("Pending", "Ok", "Error", "Acknowledged", "Remove", "RemoteUpdateSuccess", "ImagesPulled")
 // @success 200 {array} portainer.Endpoint "Endpoints"
 // @failure 500 "Server error"
 // @router /endpoints [get]

api/http/handler/endpoints/filter.go

@@ -10,8 +10,23 @@ import (
 	"github.com/pkg/errors"
 	"github.com/portainer/libhttp/request"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/http/handler/edgegroups"
 	"github.com/portainer/portainer/api/internal/endpointutils"
 	"github.com/portainer/portainer/api/internal/slices"
+	"github.com/portainer/portainer/api/internal/unique"
+)
+
+type EdgeStackStatusFilter string
+
+const (
+	statusFilterPending             EdgeStackStatusFilter = "Pending"
+	statusFilterOk                  EdgeStackStatusFilter = "Ok"
+	statusFilterError               EdgeStackStatusFilter = "Error"
+	statusFilterAcknowledged        EdgeStackStatusFilter = "Acknowledged"
+	statusFilterRemove              EdgeStackStatusFilter = "Remove"
+	statusFilterRemoteUpdateSuccess EdgeStackStatusFilter = "RemoteUpdateSuccess"
+	statusFilterImagesPulled        EdgeStackStatusFilter = "ImagesPulled"
 )
 
 type EnvironmentsQuery struct {
@@ -29,6 +44,8 @@ type EnvironmentsQuery struct {
 	name                     string
 	agentVersions            []string
 	edgeCheckInPassedSeconds int
+	edgeStackId              portainer.EdgeStackID
+	edgeStackStatus          EdgeStackStatusFilter
 }
 
 func parseQuery(r *http.Request) (EnvironmentsQuery, error) {
@@ -80,6 +97,10 @@ func parseQuery(r *http.Request) (EnvironmentsQuery, error) {
 
 	edgeCheckInPassedSeconds, _ := request.RetrieveNumericQueryParameter(r, "edgeCheckInPassedSeconds", true)
 
+	edgeStackId, _ := request.RetrieveNumericQueryParameter(r, "edgeStackId", true)
+
+	edgeStackStatus, _ := request.RetrieveQueryParameter(r, "edgeStackStatus", true)
+
 	return EnvironmentsQuery{
 		search:                   search,
 		types:                    endpointTypes,
@@ -94,6 +115,8 @@ func parseQuery(r *http.Request) (EnvironmentsQuery, error) {
 		name:                     name,
 		agentVersions:            agentVersions,
 		edgeCheckInPassedSeconds: edgeCheckInPassedSeconds,
+		edgeStackId:              portainer.EdgeStackID(edgeStackId),
+		edgeStackStatus:          EdgeStackStatusFilter(edgeStackStatus),
 	}, nil
 }
 
@@ -179,10 +202,79 @@ func (handler *Handler) filterEndpointsByQuery(filteredEndpoints []portainer.End
 			return !endpointutils.IsAgentEndpoint(&endpoint) || contains(query.agentVersions, endpoint.Agent.Version)
 		})
 	}
+	if query.edgeStackId != 0 {
+		f, err := filterEndpointsByEdgeStack(filteredEndpoints, query.edgeStackId, query.edgeStackStatus, handler.DataStore)
+		if err != nil {
+			return nil, 0, err
+		}
+		filteredEndpoints = f
+	}
 
 	return filteredEndpoints, totalAvailableEndpoints, nil
 }
 
+func endpointStatusInStackMatchesFilter(stackStatus map[portainer.EndpointID]portainer.EdgeStackStatus, envId portainer.EndpointID, statusFilter EdgeStackStatusFilter) bool {
+	status, ok := stackStatus[envId]
+
+	// consider that if the env has no status in the stack it is in Pending state
+	// workaround because Stack.Status[EnvId].Details.Pending is never set to True in the codebase
+	if !ok && statusFilter == statusFilterPending {
+		return true
+	}
+
+	valueMap := map[EdgeStackStatusFilter]bool{
+		statusFilterPending:             status.Details.Pending,
+		statusFilterOk:                  status.Details.Ok,
+		statusFilterError:               status.Details.Error,
+		statusFilterAcknowledged:        status.Details.Acknowledged,
+		statusFilterRemove:              status.Details.Remove,
+		statusFilterRemoteUpdateSuccess: status.Details.RemoteUpdateSuccess,
+		statusFilterImagesPulled:        status.Details.ImagesPulled,
+	}
+
+	currentStatus, ok := valueMap[statusFilter]
+	return ok && currentStatus
+}
+
+func filterEndpointsByEdgeStack(endpoints []portainer.Endpoint, edgeStackId portainer.EdgeStackID, statusFilter EdgeStackStatusFilter, datastore dataservices.DataStore) ([]portainer.Endpoint, error) {
+	stack, err := datastore.EdgeStack().EdgeStack(edgeStackId)
+	if err != nil {
+		return nil, errors.WithMessage(err, "Unable to retrieve edge stack from the database")
+	}
+
+	envIds := make([]portainer.EndpointID, 0)
+	for _, edgeGroupdId := range stack.EdgeGroups {
+		edgeGroup, err := datastore.EdgeGroup().EdgeGroup(edgeGroupdId)
+		if err != nil {
+			return nil, errors.WithMessage(err, "Unable to retrieve edge group from the database")
+		}
+		if edgeGroup.Dynamic {
+			endpointIDs, err := edgegroups.GetEndpointsByTags(datastore, edgeGroup.TagIDs, edgeGroup.PartialMatch)
+			if err != nil {
+				return nil, errors.WithMessage(err, "Unable to retrieve environments and environment groups for Edge group")
+			}
+			edgeGroup.Endpoints = endpointIDs
+		}
+		envIds = append(envIds, edgeGroup.Endpoints...)
+	}
+
+	if statusFilter != "" {
+		n := 0
+		for _, envId := range envIds {
+			if endpointStatusInStackMatchesFilter(stack.Status, envId, statusFilter) {
+				envIds[n] = envId
+				n++
+			}
+		}
+		envIds = envIds[:n]
+	}
+
+	uniqueIds := unique.Unique(envIds)
+	filteredEndpoints := filteredEndpointsByIds(endpoints, uniqueIds)
+
+	return filteredEndpoints, nil
+}
+
 func filterEndpointsByGroupIDs(endpoints []portainer.Endpoint, endpointGroupIDs []portainer.EndpointGroupID) []portainer.Endpoint {
 	n := 0
 	for _, endpoint := range endpoints {
@@ -394,7 +486,6 @@ func filteredEndpointsByIds(endpoints []portainer.Endpoint, ids []portainer.Endp
 	}
 
 	return endpoints[:n]
-
 }
 
 func filterEndpointsByName(endpoints []portainer.Endpoint, name string) []portainer.Endpoint {

api/internal/unique/unique.go

@@ -0,0 +1,41 @@
+package unique
+
+func Unique[T comparable](items []T) []T {
+	return UniqueBy(items, func(item T) T {
+		return item
+	})
+}
+
+func UniqueBy[ItemType any, ComparableType comparable](items []ItemType, accessorFunc func(ItemType) ComparableType) []ItemType {
+	includedItems := make(map[ComparableType]bool)
+	result := []ItemType{}
+
+	for _, item := range items {
+		if _, isIncluded := includedItems[accessorFunc(item)]; !isIncluded {
+			includedItems[accessorFunc(item)] = true
+			result = append(result, item)
+		}
+	}
+
+	return result
+}
+
+/**
+
+type someType struct {
+	id int
+	fn func()
+}
+
+func Test() {
+	ids := []int{1, 2, 3, 3}
+	_ = UniqueBy(ids, func(id int) int { return id })
+	_ = Unique(ids)                                   // shorthand for UniqueBy Identity/self
+
+	as := []someType{{id: 1}, {id: 2}, {id: 3}, {id: 3}}
+	_ = UniqueBy(as, func(item someType) int { return item.id })   // no error
+	_ = UniqueBy(as, func(item someType) someType { return item }) // compile error - someType is not comparable
+	_ = Unique(as)                                                 // compile error - shorthand fails for the same reason
+}
+
+*/

app/edge/views/edge-stacks/editEdgeStackView/editEdgeStackViewController.js

@@ -1,4 +1,3 @@
-import _ from 'lodash-es';
 import { getEnvironments } from '@/react/portainer/environments/environment.service';
 import { confirmWebEditorDiscard } from '@@/modals/confirm';
 import { EnvironmentType } from '@/react/portainer/environments/types';
@@ -30,7 +29,6 @@ export class EditEdgeStackViewController {
     this.deployStack = this.deployStack.bind(this);
     this.deployStackAsync = this.deployStackAsync.bind(this);
     this.getPaginatedEndpoints = this.getPaginatedEndpoints.bind(this);
-    this.getPaginatedEndpointsAsync = this.getPaginatedEndpointsAsync.bind(this);
     this.onEditorChange = this.onEditorChange.bind(this);
     this.isEditorDirty = this.isEditorDirty.bind(this);
   }
@@ -44,7 +42,6 @@ export class EditEdgeStackViewController {
 
         this.edgeGroups = edgeGroups;
         this.stack = model;
-        this.stackEndpointIds = this.filterStackEndpoints(model.EdgeGroups, edgeGroups);
         this.originalFileContent = file;
         this.formValues = {
           content: file,
@@ -88,15 +85,6 @@ export class EditEdgeStackViewController {
     return !this.state.isStackDeployed && this.formValues.content.replace(/(\r\n|\n|\r)/gm, '') !== this.originalFileContent.replace(/(\r\n|\n|\r)/gm, '');
   }
 
-  filterStackEndpoints(groupIds, groups) {
-    return _.flatten(
-      _.map(groupIds, (Id) => {
-        const group = _.find(groups, { Id });
-        return group.Endpoints;
-      })
-    );
-  }
-
   deployStack(values) {
     return this.deployStackAsync(values);
   }
@@ -123,22 +111,19 @@ export class EditEdgeStackViewController {
     }
   }
 
-  getPaginatedEndpoints(...args) {
-    return this.$async(this.getPaginatedEndpointsAsync, ...args);
-  }
+  getPaginatedEndpoints(lastId, limit, search) {
+    return this.$async(async () => {
+      try {
+        const query = {
+          search,
+          edgeStackId: this.stack.Id,
+        };
+        const { value, totalCount } = await getEnvironments({ start: lastId, limit, query });
 
-  async getPaginatedEndpointsAsync(lastId, limit, search) {
-    try {
-      if (this.stackEndpointIds.length === 0) {
-        return { endpoints: [], totalCount: 0 };
+        return { endpoints: value, totalCount };
+      } catch (err) {
+        this.Notifications.error('Failure', err, 'Unable to retrieve environment information');
       }
-
-      const query = { search, endpointIds: this.stackEndpointIds };
-      const { value, totalCount } = await getEnvironments({ start: lastId, limit, query });
-
-      return { endpoints: value, totalCount };
-    } catch (err) {
-      this.Notifications.error('Failure', err, 'Unable to retrieve environment information');
-    }
+    });
   }
 }

app/react/edge/edge-stacks/types.ts

@@ -17,7 +17,7 @@ interface EdgeStackStatusDetails {
   ImagesPulled: boolean;
 }
 
-interface EdgeStackStatus {
+export interface EdgeStackStatus {
   Details: EdgeStackStatusDetails;
   Error: string;
   EndpointID: EnvironmentId;

app/react/portainer/environments/environment.service/index.ts

@@ -3,6 +3,7 @@ import { type EnvironmentGroupId } from '@/react/portainer/environments/environm
 import { type TagId } from '@/portainer/tags/types';
 import { UserId } from '@/portainer/users/types';
 import { TeamId } from '@/react/portainer/users/teams/types';
+import { EdgeStack, EdgeStackStatus } from '@/react/edge/edge-stacks/types';
 
 import type {
   Environment,
@@ -14,7 +15,16 @@ import type {
 
 import { buildUrl } from './utils';
 
-export interface EnvironmentsQueryParams {
+export type EdgeStackEnvironmentsQueryParams =
+  | {
+      edgeStackId?: EdgeStack['Id'];
+    }
+  | {
+      edgeStackId: EdgeStack['Id'];
+      edgeStackStatus?: keyof EdgeStackStatus['Details'];
+    };
+
+export interface BaseEnvironmentsQueryParams {
   search?: string;
   types?: EnvironmentType[] | readonly EnvironmentType[];
   tagIds?: TagId[];
@@ -32,6 +42,9 @@ export interface EnvironmentsQueryParams {
   edgeCheckInPassedSeconds?: number;
 }
 
+export type EnvironmentsQueryParams = BaseEnvironmentsQueryParams &
+  EdgeStackEnvironmentsQueryParams;
+
 export interface GetEnvironmentsOptions {
   start?: number;
   limit?: number;

app/react/portainer/environments/queries/useEnvironmentList.ts

@@ -12,12 +12,12 @@ import { queryKeys } from './query-keys';
 
 export const ENVIRONMENTS_POLLING_INTERVAL = 30000; // in ms
 
-export interface Query extends EnvironmentsQueryParams {
+export type Query = EnvironmentsQueryParams & {
   page?: number;
   pageLimit?: number;
   sort?: string;
   order?: 'asc' | 'desc';
-}
+};
 
 type GetEndpointsResponse = Awaited<ReturnType<typeof getEnvironments>>;