From 2eca5e05d4331232d5415a6ce3b265a23d88303a Mon Sep 17 00:00:00 2001 From: LP B Date: Mon, 19 Jun 2023 11:55:33 +0200 Subject: [PATCH] fix(edge-stack): URI too large error for edge stacks with a large amount of environments [EE-5583] (#9085) * refactor(edge-stacks): filter endpoints by edgeStack * feat(api/endpoints): edge stack filter support filtering on status in stack * refactor(endpoints): use separate query params and not JSON query param when querying for an edge stack * feat(api/endpoints): handle stack filter on dynamic groups + unique list with multiple groups sharing environments * fix(app/endpoints): edge stack related query params type definition * fix(api/endpoints): rebase conflicts on imports --- .../edgegroups/associated_endpoints.go | 2 +- .../handler/edgegroups/edgegroup_inspect.go | 2 +- api/http/handler/edgegroups/edgegroup_list.go | 2 +- api/http/handler/endpoints/endpoint_list.go | 2 + api/http/handler/endpoints/filter.go | 93 ++++++++++++++++++- api/internal/unique/unique.go | 41 ++++++++ .../editEdgeStackViewController.js | 39 +++----- app/react/edge/edge-stacks/types.ts | 2 +- .../environments/environment.service/index.ts | 15 ++- .../queries/useEnvironmentList.ts | 4 +- 10 files changed, 167 insertions(+), 35 deletions(-) create mode 100644 api/internal/unique/unique.go diff --git a/api/http/handler/edgegroups/associated_endpoints.go b/api/http/handler/edgegroups/associated_endpoints.go index ea2ea111b..2f8adda5a 100644 --- a/api/http/handler/edgegroups/associated_endpoints.go +++ b/api/http/handler/edgegroups/associated_endpoints.go @@ -7,7 +7,7 @@ import ( type endpointSetType map[portainer.EndpointID]bool -func getEndpointsByTags(tx dataservices.DataStoreTx, tagIDs []portainer.TagID, partialMatch bool) ([]portainer.EndpointID, error) { +func GetEndpointsByTags(tx dataservices.DataStoreTx, tagIDs []portainer.TagID, partialMatch bool) ([]portainer.EndpointID, error) { if len(tagIDs) == 0 { return []portainer.EndpointID{}, nil } diff --git a/api/http/handler/edgegroups/edgegroup_inspect.go b/api/http/handler/edgegroups/edgegroup_inspect.go index fb611573f..c528f3757 100644 --- a/api/http/handler/edgegroups/edgegroup_inspect.go +++ b/api/http/handler/edgegroups/edgegroup_inspect.go @@ -50,7 +50,7 @@ func getEdgeGroup(tx dataservices.DataStoreTx, ID portainer.EdgeGroupID) (*porta } if edgeGroup.Dynamic { - endpoints, err := getEndpointsByTags(tx, edgeGroup.TagIDs, edgeGroup.PartialMatch) + endpoints, err := GetEndpointsByTags(tx, edgeGroup.TagIDs, edgeGroup.PartialMatch) if err != nil { return nil, httperror.InternalServerError("Unable to retrieve environments and environment groups for Edge group", err) } diff --git a/api/http/handler/edgegroups/edgegroup_list.go b/api/http/handler/edgegroups/edgegroup_list.go index 8fa372968..fa7a53988 100644 --- a/api/http/handler/edgegroups/edgegroup_list.go +++ b/api/http/handler/edgegroups/edgegroup_list.go @@ -84,7 +84,7 @@ func getEdgeGroupList(tx dataservices.DataStoreTx) ([]decoratedEdgeGroup, error) EndpointTypes: []portainer.EndpointType{}, } if edgeGroup.Dynamic { - endpointIDs, err := getEndpointsByTags(tx, edgeGroup.TagIDs, edgeGroup.PartialMatch) + endpointIDs, err := GetEndpointsByTags(tx, edgeGroup.TagIDs, edgeGroup.PartialMatch) if err != nil { return nil, httperror.InternalServerError("Unable to retrieve environments and environment groups for Edge group", err) } diff --git a/api/http/handler/endpoints/endpoint_list.go b/api/http/handler/endpoints/endpoint_list.go index 03c906673..ef3555ca4 100644 --- a/api/http/handler/endpoints/endpoint_list.go +++ b/api/http/handler/endpoints/endpoint_list.go @@ -46,6 +46,8 @@ const ( // @param edgeCheckInPassedSeconds query number false "if bigger then zero, show only edge agents that checked-in in the last provided seconds (relevant only for edge agents)" // @param excludeSnapshots query bool false "if true, the snapshot data won't be retrieved" // @param name query string false "will return only environments(endpoints) with this name" +// @param edgeStackId query portainer.EdgeStackID false "will return the environements of the specified edge stack" +// @param edgeStackStatus query string false "only applied when edgeStackId exists. Filter the returned environments based on their deployment status in the stack (not the environment status!)" Enum("Pending", "Ok", "Error", "Acknowledged", "Remove", "RemoteUpdateSuccess", "ImagesPulled") // @success 200 {array} portainer.Endpoint "Endpoints" // @failure 500 "Server error" // @router /endpoints [get] diff --git a/api/http/handler/endpoints/filter.go b/api/http/handler/endpoints/filter.go index 7a05867e6..17f3807a9 100644 --- a/api/http/handler/endpoints/filter.go +++ b/api/http/handler/endpoints/filter.go @@ -10,8 +10,23 @@ import ( "github.com/pkg/errors" "github.com/portainer/libhttp/request" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" + "github.com/portainer/portainer/api/http/handler/edgegroups" "github.com/portainer/portainer/api/internal/endpointutils" "github.com/portainer/portainer/api/internal/slices" + "github.com/portainer/portainer/api/internal/unique" +) + +type EdgeStackStatusFilter string + +const ( + statusFilterPending EdgeStackStatusFilter = "Pending" + statusFilterOk EdgeStackStatusFilter = "Ok" + statusFilterError EdgeStackStatusFilter = "Error" + statusFilterAcknowledged EdgeStackStatusFilter = "Acknowledged" + statusFilterRemove EdgeStackStatusFilter = "Remove" + statusFilterRemoteUpdateSuccess EdgeStackStatusFilter = "RemoteUpdateSuccess" + statusFilterImagesPulled EdgeStackStatusFilter = "ImagesPulled" ) type EnvironmentsQuery struct { @@ -29,6 +44,8 @@ type EnvironmentsQuery struct { name string agentVersions []string edgeCheckInPassedSeconds int + edgeStackId portainer.EdgeStackID + edgeStackStatus EdgeStackStatusFilter } func parseQuery(r *http.Request) (EnvironmentsQuery, error) { @@ -80,6 +97,10 @@ func parseQuery(r *http.Request) (EnvironmentsQuery, error) { edgeCheckInPassedSeconds, _ := request.RetrieveNumericQueryParameter(r, "edgeCheckInPassedSeconds", true) + edgeStackId, _ := request.RetrieveNumericQueryParameter(r, "edgeStackId", true) + + edgeStackStatus, _ := request.RetrieveQueryParameter(r, "edgeStackStatus", true) + return EnvironmentsQuery{ search: search, types: endpointTypes, @@ -94,6 +115,8 @@ func parseQuery(r *http.Request) (EnvironmentsQuery, error) { name: name, agentVersions: agentVersions, edgeCheckInPassedSeconds: edgeCheckInPassedSeconds, + edgeStackId: portainer.EdgeStackID(edgeStackId), + edgeStackStatus: EdgeStackStatusFilter(edgeStackStatus), }, nil } @@ -179,10 +202,79 @@ func (handler *Handler) filterEndpointsByQuery(filteredEndpoints []portainer.End return !endpointutils.IsAgentEndpoint(&endpoint) || contains(query.agentVersions, endpoint.Agent.Version) }) } + if query.edgeStackId != 0 { + f, err := filterEndpointsByEdgeStack(filteredEndpoints, query.edgeStackId, query.edgeStackStatus, handler.DataStore) + if err != nil { + return nil, 0, err + } + filteredEndpoints = f + } return filteredEndpoints, totalAvailableEndpoints, nil } +func endpointStatusInStackMatchesFilter(stackStatus map[portainer.EndpointID]portainer.EdgeStackStatus, envId portainer.EndpointID, statusFilter EdgeStackStatusFilter) bool { + status, ok := stackStatus[envId] + + // consider that if the env has no status in the stack it is in Pending state + // workaround because Stack.Status[EnvId].Details.Pending is never set to True in the codebase + if !ok && statusFilter == statusFilterPending { + return true + } + + valueMap := map[EdgeStackStatusFilter]bool{ + statusFilterPending: status.Details.Pending, + statusFilterOk: status.Details.Ok, + statusFilterError: status.Details.Error, + statusFilterAcknowledged: status.Details.Acknowledged, + statusFilterRemove: status.Details.Remove, + statusFilterRemoteUpdateSuccess: status.Details.RemoteUpdateSuccess, + statusFilterImagesPulled: status.Details.ImagesPulled, + } + + currentStatus, ok := valueMap[statusFilter] + return ok && currentStatus +} + +func filterEndpointsByEdgeStack(endpoints []portainer.Endpoint, edgeStackId portainer.EdgeStackID, statusFilter EdgeStackStatusFilter, datastore dataservices.DataStore) ([]portainer.Endpoint, error) { + stack, err := datastore.EdgeStack().EdgeStack(edgeStackId) + if err != nil { + return nil, errors.WithMessage(err, "Unable to retrieve edge stack from the database") + } + + envIds := make([]portainer.EndpointID, 0) + for _, edgeGroupdId := range stack.EdgeGroups { + edgeGroup, err := datastore.EdgeGroup().EdgeGroup(edgeGroupdId) + if err != nil { + return nil, errors.WithMessage(err, "Unable to retrieve edge group from the database") + } + if edgeGroup.Dynamic { + endpointIDs, err := edgegroups.GetEndpointsByTags(datastore, edgeGroup.TagIDs, edgeGroup.PartialMatch) + if err != nil { + return nil, errors.WithMessage(err, "Unable to retrieve environments and environment groups for Edge group") + } + edgeGroup.Endpoints = endpointIDs + } + envIds = append(envIds, edgeGroup.Endpoints...) + } + + if statusFilter != "" { + n := 0 + for _, envId := range envIds { + if endpointStatusInStackMatchesFilter(stack.Status, envId, statusFilter) { + envIds[n] = envId + n++ + } + } + envIds = envIds[:n] + } + + uniqueIds := unique.Unique(envIds) + filteredEndpoints := filteredEndpointsByIds(endpoints, uniqueIds) + + return filteredEndpoints, nil +} + func filterEndpointsByGroupIDs(endpoints []portainer.Endpoint, endpointGroupIDs []portainer.EndpointGroupID) []portainer.Endpoint { n := 0 for _, endpoint := range endpoints { @@ -394,7 +486,6 @@ func filteredEndpointsByIds(endpoints []portainer.Endpoint, ids []portainer.Endp } return endpoints[:n] - } func filterEndpointsByName(endpoints []portainer.Endpoint, name string) []portainer.Endpoint { diff --git a/api/internal/unique/unique.go b/api/internal/unique/unique.go new file mode 100644 index 000000000..54c23af01 --- /dev/null +++ b/api/internal/unique/unique.go @@ -0,0 +1,41 @@ +package unique + +func Unique[T comparable](items []T) []T { + return UniqueBy(items, func(item T) T { + return item + }) +} + +func UniqueBy[ItemType any, ComparableType comparable](items []ItemType, accessorFunc func(ItemType) ComparableType) []ItemType { + includedItems := make(map[ComparableType]bool) + result := []ItemType{} + + for _, item := range items { + if _, isIncluded := includedItems[accessorFunc(item)]; !isIncluded { + includedItems[accessorFunc(item)] = true + result = append(result, item) + } + } + + return result +} + +/** + +type someType struct { + id int + fn func() +} + +func Test() { + ids := []int{1, 2, 3, 3} + _ = UniqueBy(ids, func(id int) int { return id }) + _ = Unique(ids) // shorthand for UniqueBy Identity/self + + as := []someType{{id: 1}, {id: 2}, {id: 3}, {id: 3}} + _ = UniqueBy(as, func(item someType) int { return item.id }) // no error + _ = UniqueBy(as, func(item someType) someType { return item }) // compile error - someType is not comparable + _ = Unique(as) // compile error - shorthand fails for the same reason +} + +*/ diff --git a/app/edge/views/edge-stacks/editEdgeStackView/editEdgeStackViewController.js b/app/edge/views/edge-stacks/editEdgeStackView/editEdgeStackViewController.js index 255c51c54..9b6418082 100644 --- a/app/edge/views/edge-stacks/editEdgeStackView/editEdgeStackViewController.js +++ b/app/edge/views/edge-stacks/editEdgeStackView/editEdgeStackViewController.js @@ -1,4 +1,3 @@ -import _ from 'lodash-es'; import { getEnvironments } from '@/react/portainer/environments/environment.service'; import { confirmWebEditorDiscard } from '@@/modals/confirm'; import { EnvironmentType } from '@/react/portainer/environments/types'; @@ -30,7 +29,6 @@ export class EditEdgeStackViewController { this.deployStack = this.deployStack.bind(this); this.deployStackAsync = this.deployStackAsync.bind(this); this.getPaginatedEndpoints = this.getPaginatedEndpoints.bind(this); - this.getPaginatedEndpointsAsync = this.getPaginatedEndpointsAsync.bind(this); this.onEditorChange = this.onEditorChange.bind(this); this.isEditorDirty = this.isEditorDirty.bind(this); } @@ -44,7 +42,6 @@ export class EditEdgeStackViewController { this.edgeGroups = edgeGroups; this.stack = model; - this.stackEndpointIds = this.filterStackEndpoints(model.EdgeGroups, edgeGroups); this.originalFileContent = file; this.formValues = { content: file, @@ -88,15 +85,6 @@ export class EditEdgeStackViewController { return !this.state.isStackDeployed && this.formValues.content.replace(/(\r\n|\n|\r)/gm, '') !== this.originalFileContent.replace(/(\r\n|\n|\r)/gm, ''); } - filterStackEndpoints(groupIds, groups) { - return _.flatten( - _.map(groupIds, (Id) => { - const group = _.find(groups, { Id }); - return group.Endpoints; - }) - ); - } - deployStack(values) { return this.deployStackAsync(values); } @@ -123,22 +111,19 @@ export class EditEdgeStackViewController { } } - getPaginatedEndpoints(...args) { - return this.$async(this.getPaginatedEndpointsAsync, ...args); - } + getPaginatedEndpoints(lastId, limit, search) { + return this.$async(async () => { + try { + const query = { + search, + edgeStackId: this.stack.Id, + }; + const { value, totalCount } = await getEnvironments({ start: lastId, limit, query }); - async getPaginatedEndpointsAsync(lastId, limit, search) { - try { - if (this.stackEndpointIds.length === 0) { - return { endpoints: [], totalCount: 0 }; + return { endpoints: value, totalCount }; + } catch (err) { + this.Notifications.error('Failure', err, 'Unable to retrieve environment information'); } - - const query = { search, endpointIds: this.stackEndpointIds }; - const { value, totalCount } = await getEnvironments({ start: lastId, limit, query }); - - return { endpoints: value, totalCount }; - } catch (err) { - this.Notifications.error('Failure', err, 'Unable to retrieve environment information'); - } + }); } } diff --git a/app/react/edge/edge-stacks/types.ts b/app/react/edge/edge-stacks/types.ts index 0c05567f4..bfdaf27fc 100644 --- a/app/react/edge/edge-stacks/types.ts +++ b/app/react/edge/edge-stacks/types.ts @@ -17,7 +17,7 @@ interface EdgeStackStatusDetails { ImagesPulled: boolean; } -interface EdgeStackStatus { +export interface EdgeStackStatus { Details: EdgeStackStatusDetails; Error: string; EndpointID: EnvironmentId; diff --git a/app/react/portainer/environments/environment.service/index.ts b/app/react/portainer/environments/environment.service/index.ts index 693e7378d..7a3bd448b 100644 --- a/app/react/portainer/environments/environment.service/index.ts +++ b/app/react/portainer/environments/environment.service/index.ts @@ -3,6 +3,7 @@ import { type EnvironmentGroupId } from '@/react/portainer/environments/environm import { type TagId } from '@/portainer/tags/types'; import { UserId } from '@/portainer/users/types'; import { TeamId } from '@/react/portainer/users/teams/types'; +import { EdgeStack, EdgeStackStatus } from '@/react/edge/edge-stacks/types'; import type { Environment, @@ -14,7 +15,16 @@ import type { import { buildUrl } from './utils'; -export interface EnvironmentsQueryParams { +export type EdgeStackEnvironmentsQueryParams = + | { + edgeStackId?: EdgeStack['Id']; + } + | { + edgeStackId: EdgeStack['Id']; + edgeStackStatus?: keyof EdgeStackStatus['Details']; + }; + +export interface BaseEnvironmentsQueryParams { search?: string; types?: EnvironmentType[] | readonly EnvironmentType[]; tagIds?: TagId[]; @@ -32,6 +42,9 @@ export interface EnvironmentsQueryParams { edgeCheckInPassedSeconds?: number; } +export type EnvironmentsQueryParams = BaseEnvironmentsQueryParams & + EdgeStackEnvironmentsQueryParams; + export interface GetEnvironmentsOptions { start?: number; limit?: number; diff --git a/app/react/portainer/environments/queries/useEnvironmentList.ts b/app/react/portainer/environments/queries/useEnvironmentList.ts index 00675b8e4..4b00f0997 100644 --- a/app/react/portainer/environments/queries/useEnvironmentList.ts +++ b/app/react/portainer/environments/queries/useEnvironmentList.ts @@ -12,12 +12,12 @@ import { queryKeys } from './query-keys'; export const ENVIRONMENTS_POLLING_INTERVAL = 30000; // in ms -export interface Query extends EnvironmentsQueryParams { +export type Query = EnvironmentsQueryParams & { page?: number; pageLimit?: number; sort?: string; order?: 'asc' | 'desc'; -} +}; type GetEndpointsResponse = Awaited>;