From 05357ecce50595db1efce34a4450a181f1a9a7e0 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Tue, 19 Jul 2022 18:00:45 +0200 Subject: [PATCH] fix(edge): filtering of edge devices [EE-3210] (#7077) * fix(edge): filtering of edge devices [EE-3210] fixes [EE-3210] changes: - replaces `edgeDeviceFilter` with two filters: - `edgeDevice` - `edgeDeviceUntrusted` these filters will only apply to the edge endpoints in the query (so it's possible to get both regular endpoints and edge devices). if `edgeDevice` is true, will filter out edge agents which are not an edge device. false, will filter out edge devices `edgeDeviceUntrusted` applies only when `edgeDevice` is true. then false (default) will hide the untrusted edge devices, true will show only untrusted edge devices. fix(edge/job-create): retrieve only trusted endpoints + fix endpoint selector pagination limits onChange fix(endpoint-groups): remove listing of untrusted edge envs (aka in waiting room) refactor(endpoints): move filter to another function feat(endpoints): separate edge filters refactor(environments): change getEnv api refactor(endpoints): use single getEnv feat(groups): show error when failed loading envs style(endpoints): remove unused endpointsByGroup * chore(deps): update go to 1.18 * fix(endpoint): filter out untrusted by default * fix(edge): show correct endpoints * style(endpoints): fix typo * fix(endpoints): fix swagger * fix(admin): use new getEnv function Co-authored-by: LP B --- api/go.mod | 5 +- api/go.sum | 6 +- api/http/handler/endpoints/endpoint_list.go | 354 +-------------- .../handler/endpoints/endpoint_list_test.go | 107 +++-- api/http/handler/endpoints/filter.go | 415 ++++++++++++++++++ api/http/handler/endpoints/filter_test.go | 119 +++++ api/http/handler/endpoints/utils.go | 6 + .../EdgeDevicesDatatableContainer.tsx | 5 +- .../WaitingRoomView/WaitingRoomView.tsx | 5 +- .../group-form/groupFormController.js | 52 ++- .../edge-jobs/edgeJob/edgeJobController.js | 8 +- .../editEdgeStackViewController.js | 8 +- .../associatedEndpointsSelector.html | 4 +- .../associatedEndpointsSelectorController.js | 64 +-- .../forms/group-form/groupFormController.js | 45 +- .../environments/environment.service/index.ts | 73 ++- .../queries/useEnvironmentList.ts | 22 +- app/portainer/environments/types.ts | 5 + .../home/EnvironmentList/EnvironmentList.tsx | 5 +- .../KubeconfigButton/KubeconfigButton.tsx | 4 +- .../access-viewer/access-viewer.controller.js | 6 +- app/portainer/services/api/endpointService.js | 12 - app/portainer/services/nameValidator.js | 11 +- app/portainer/views/auth/authController.js | 5 +- .../views/endpoints/endpointsController.js | 7 +- .../views/init/admin/initAdminController.js | 7 +- .../views/stacks/edit/stackController.js | 103 ++--- .../shared/NameField.tsx | 6 +- 28 files changed, 868 insertions(+), 601 deletions(-) create mode 100644 api/http/handler/endpoints/filter.go create mode 100644 api/http/handler/endpoints/filter_test.go create mode 100644 api/http/handler/endpoints/utils.go diff --git a/api/go.mod b/api/go.mod index 03faab64e..0b7c067a0 100644 --- a/api/go.mod +++ b/api/go.mod @@ -1,6 +1,6 @@ module github.com/portainer/portainer/api -go 1.17 +go 1.18 require ( github.com/Microsoft/go-winio v0.5.1 @@ -20,7 +20,7 @@ require ( github.com/go-playground/validator/v10 v10.10.1 github.com/gofrs/uuid v4.0.0+incompatible github.com/golang-jwt/jwt v3.2.2+incompatible - github.com/google/go-cmp v0.5.6 + github.com/google/go-cmp v0.5.8 github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.7.3 github.com/gorilla/securecookie v1.1.1 @@ -43,6 +43,7 @@ require ( github.com/viney-shih/go-lock v1.1.1 go.etcd.io/bbolt v1.3.6 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 + golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f golang.org/x/sync v0.0.0-20210220032951-036812b2e83c gopkg.in/alecthomas/kingpin.v2 v2.2.6 diff --git a/api/go.sum b/api/go.sum index c9e9ad997..5c4f84143 100644 --- a/api/go.sum +++ b/api/go.sum @@ -213,8 +213,9 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= @@ -437,6 +438,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d h1:vtUKgx8dahOomfFzLREU8nSv25YHnTgLBn4rDnWZdU0= +golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -626,7 +629,6 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= diff --git a/api/http/handler/endpoints/endpoint_list.go b/api/http/handler/endpoints/endpoint_list.go index 47aaebcb5..53a982a10 100644 --- a/api/http/handler/endpoints/endpoint_list.go +++ b/api/http/handler/endpoints/endpoint_list.go @@ -4,24 +4,14 @@ import ( "net/http" "sort" "strconv" - "strings" "time" "github.com/portainer/libhttp/request" + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/http/security" httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/response" - portainer "github.com/portainer/portainer/api" - "github.com/portainer/portainer/api/http/security" - "github.com/portainer/portainer/api/internal/endpointutils" - "github.com/portainer/portainer/api/internal/utils" -) - -const ( - EdgeDeviceFilterAll = "all" - EdgeDeviceFilterTrusted = "trusted" - EdgeDeviceFilterUntrusted = "untrusted" - EdgeDeviceFilterNone = "none" ) const ( @@ -42,14 +32,19 @@ var endpointGroupNames map[portainer.EndpointGroupID]string // @security jwt // @produce json // @param start query int false "Start searching from" -// @param search query string false "Search query" -// @param groupId query int false "List environments(endpoints) of this group" // @param limit query int false "Limit results to this value" +// @param sort query int false "Sort results by this value" +// @param order query int false "Order sorted results by desc/asc" Enum("asc", "desc") +// @param search query string false "Search query" +// @param groupIds query []int false "List environments(endpoints) of these groups" +// @param status query []int false "List environments(endpoints) by this status" // @param types query []int false "List environments(endpoints) of this type" // @param tagIds query []int false "search environments(endpoints) with these tags (depends on tagsPartialMatch)" // @param tagsPartialMatch query bool false "If true, will return environment(endpoint) which has one of tagIds, if false (or missing) will return only environments(endpoints) that has all the tags" // @param endpointIds query []int false "will return only these environments(endpoints)" -// @param edgeDeviceFilter query string false "will return only these edge environments, none will return only regular edge environments" Enum("all", "trusted", "untrusted", "none") +// @param provisioned query bool false "If true, will return environment(endpoint) that were provisioned" +// @param edgeDevice query bool false "if exists true show only edge devices, false show only regular edge endpoints. if missing, will show both types (relevant only for edge endpoints)" +// @param edgeDeviceUntrusted query bool false "if true, show only untrusted endpoints, if false show only trusted (relevant only for edge devices, and if edgeDevice is true)" // @param name query string false "will return only environments(endpoints) with this name" // @success 200 {array} portainer.Endpoint "Endpoints" // @failure 500 "Server error" @@ -60,103 +55,42 @@ func (handler *Handler) endpointList(w http.ResponseWriter, r *http.Request) *ht start-- } - search, _ := request.RetrieveQueryParameter(r, "search", true) - if search != "" { - search = strings.ToLower(search) - } - - groupID, _ := request.RetrieveNumericQueryParameter(r, "groupId", true) limit, _ := request.RetrieveNumericQueryParameter(r, "limit", true) sortField, _ := request.RetrieveQueryParameter(r, "sort", true) sortOrder, _ := request.RetrieveQueryParameter(r, "order", true) - var endpointTypes []int - request.RetrieveJSONQueryParameter(r, "types", &endpointTypes, true) - - var tagIDs []portainer.TagID - request.RetrieveJSONQueryParameter(r, "tagIds", &tagIDs, true) - - tagsPartialMatch, _ := request.RetrieveBooleanQueryParameter(r, "tagsPartialMatch", true) - - var endpointIDs []portainer.EndpointID - request.RetrieveJSONQueryParameter(r, "endpointIds", &endpointIDs, true) - - var statuses []int - request.RetrieveJSONQueryParameter(r, "status", &statuses, true) - - var groupIDs []int - request.RetrieveJSONQueryParameter(r, "groupIds", &groupIDs, true) - endpointGroups, err := handler.DataStore.EndpointGroup().EndpointGroups() if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve environment groups from the database", err} + return httperror.InternalServerError("Unable to retrieve environment groups from the database", err) } endpoints, err := handler.DataStore.Endpoint().Endpoints() if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve environments from the database", err} + return httperror.InternalServerError("Unable to retrieve environments from the database", err) } settings, err := handler.DataStore.Settings().Settings() if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve settings from the database", err} + return httperror.InternalServerError("Unable to retrieve settings from the database", err) } securityContext, err := security.RetrieveRestrictedRequestContext(r) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err} + return httperror.InternalServerError("Unable to retrieve info from request context", err) + } + + query, err := parseQuery(r) + if err != nil { + return httperror.BadRequest("Invalid query parameters", err) } filteredEndpoints := security.FilterEndpoints(endpoints, endpointGroups, securityContext) - totalAvailableEndpoints := len(filteredEndpoints) - if groupID != 0 { - filteredEndpoints = filterEndpointsByGroupIDs(filteredEndpoints, []int{groupID}) + filteredEndpoints, totalAvailableEndpoints, err := handler.filterEndpointsByQuery(filteredEndpoints, query, endpointGroups, settings) + if err != nil { + return httperror.InternalServerError("Unable to filter endpoints", err) } - if endpointIDs != nil { - filteredEndpoints = filteredEndpointsByIds(filteredEndpoints, endpointIDs) - } - - if len(groupIDs) > 0 { - filteredEndpoints = filterEndpointsByGroupIDs(filteredEndpoints, groupIDs) - } - - name, _ := request.RetrieveQueryParameter(r, "name", true) - if name != "" { - filteredEndpoints = filterEndpointsByName(filteredEndpoints, name) - } - - edgeDeviceFilter, _ := request.RetrieveQueryParameter(r, "edgeDeviceFilter", false) - if edgeDeviceFilter != "" { - filteredEndpoints = filterEndpointsByEdgeDevice(filteredEndpoints, edgeDeviceFilter) - } - - if len(statuses) > 0 { - filteredEndpoints = filterEndpointsByStatuses(filteredEndpoints, statuses, settings) - } - - if search != "" { - tags, err := handler.DataStore.Tag().Tags() - if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve tags from the database", err} - } - tagsMap := make(map[portainer.TagID]string) - for _, tag := range tags { - tagsMap[tag.ID] = tag.Name - } - filteredEndpoints = filterEndpointsBySearchCriteria(filteredEndpoints, endpointGroups, tagsMap, search) - } - - if endpointTypes != nil { - filteredEndpoints = filterEndpointsByTypes(filteredEndpoints, endpointTypes) - } - - if tagIDs != nil { - filteredEndpoints = filteredEndpointsByTags(filteredEndpoints, tagIDs, endpointGroups, tagsPartialMatch) - } - - // Sort endpoints by field sortEndpointsByField(filteredEndpoints, endpointGroups, sortField, sortOrder == "desc") filteredEndpointCount := len(filteredEndpoints) @@ -196,64 +130,6 @@ func paginateEndpoints(endpoints []portainer.Endpoint, start, limit int) []porta return endpoints[start:end] } -func filterEndpointsByGroupIDs(endpoints []portainer.Endpoint, endpointGroupIDs []int) []portainer.Endpoint { - filteredEndpoints := make([]portainer.Endpoint, 0) - - for _, endpoint := range endpoints { - if utils.Contains(endpointGroupIDs, int(endpoint.GroupID)) { - filteredEndpoints = append(filteredEndpoints, endpoint) - } - } - - return filteredEndpoints -} - -func filterEndpointsBySearchCriteria(endpoints []portainer.Endpoint, endpointGroups []portainer.EndpointGroup, tagsMap map[portainer.TagID]string, searchCriteria string) []portainer.Endpoint { - filteredEndpoints := make([]portainer.Endpoint, 0) - - for _, endpoint := range endpoints { - endpointTags := convertTagIDsToTags(tagsMap, endpoint.TagIDs) - if endpointMatchSearchCriteria(&endpoint, endpointTags, searchCriteria) { - filteredEndpoints = append(filteredEndpoints, endpoint) - continue - } - - if endpointGroupMatchSearchCriteria(&endpoint, endpointGroups, tagsMap, searchCriteria) { - filteredEndpoints = append(filteredEndpoints, endpoint) - } - } - - return filteredEndpoints -} - -func filterEndpointsByStatuses(endpoints []portainer.Endpoint, statuses []int, settings *portainer.Settings) []portainer.Endpoint { - filteredEndpoints := make([]portainer.Endpoint, 0) - - for _, endpoint := range endpoints { - status := endpoint.Status - if endpointutils.IsEdgeEndpoint(&endpoint) { - isCheckValid := false - edgeCheckinInterval := endpoint.EdgeCheckinInterval - if endpoint.EdgeCheckinInterval == 0 { - edgeCheckinInterval = settings.EdgeAgentCheckinInterval - } - if edgeCheckinInterval != 0 && endpoint.LastCheckInDate != 0 { - isCheckValid = time.Now().Unix()-endpoint.LastCheckInDate <= int64(edgeCheckinInterval*EdgeDeviceIntervalMultiplier+EdgeDeviceIntervalAdd) - } - status = portainer.EndpointStatusDown // Offline - if isCheckValid { - status = portainer.EndpointStatusUp // Online - } - } - - if utils.Contains(statuses, int(status)) { - filteredEndpoints = append(filteredEndpoints, endpoint) - } - } - - return filteredEndpoints -} - func sortEndpointsByField(endpoints []portainer.Endpoint, endpointGroups []portainer.EndpointGroup, sortField string, isSortDesc bool) { switch sortField { @@ -294,123 +170,6 @@ func sortEndpointsByField(endpoints []portainer.Endpoint, endpointGroups []porta } } -func endpointMatchSearchCriteria(endpoint *portainer.Endpoint, tags []string, searchCriteria string) bool { - if strings.Contains(strings.ToLower(endpoint.Name), searchCriteria) { - return true - } - - if strings.Contains(strings.ToLower(endpoint.URL), searchCriteria) { - return true - } - - if endpoint.Status == portainer.EndpointStatusUp && searchCriteria == "up" { - return true - } else if endpoint.Status == portainer.EndpointStatusDown && searchCriteria == "down" { - return true - } - for _, tag := range tags { - if strings.Contains(strings.ToLower(tag), searchCriteria) { - return true - } - } - - return false -} - -func endpointGroupMatchSearchCriteria(endpoint *portainer.Endpoint, endpointGroups []portainer.EndpointGroup, tagsMap map[portainer.TagID]string, searchCriteria string) bool { - for _, group := range endpointGroups { - if group.ID == endpoint.GroupID { - if strings.Contains(strings.ToLower(group.Name), searchCriteria) { - return true - } - tags := convertTagIDsToTags(tagsMap, group.TagIDs) - for _, tag := range tags { - if strings.Contains(strings.ToLower(tag), searchCriteria) { - return true - } - } - } - } - - return false -} - -func filterEndpointsByTypes(endpoints []portainer.Endpoint, endpointTypes []int) []portainer.Endpoint { - filteredEndpoints := make([]portainer.Endpoint, 0) - - typeSet := map[portainer.EndpointType]bool{} - for _, endpointType := range endpointTypes { - typeSet[portainer.EndpointType(endpointType)] = true - } - - for _, endpoint := range endpoints { - if typeSet[endpoint.Type] { - filteredEndpoints = append(filteredEndpoints, endpoint) - } - } - return filteredEndpoints -} - -func filterEndpointsByEdgeDevice(endpoints []portainer.Endpoint, edgeDeviceFilter string) []portainer.Endpoint { - filteredEndpoints := make([]portainer.Endpoint, 0) - - for _, endpoint := range endpoints { - if shouldReturnEdgeDevice(endpoint, edgeDeviceFilter) { - filteredEndpoints = append(filteredEndpoints, endpoint) - } - } - return filteredEndpoints -} - -func shouldReturnEdgeDevice(endpoint portainer.Endpoint, edgeDeviceFilter string) bool { - // none - return all endpoints that are not edge devices - if edgeDeviceFilter == EdgeDeviceFilterNone && !endpoint.IsEdgeDevice { - return true - } - - if !endpointutils.IsEdgeEndpoint(&endpoint) { - return false - } - - switch edgeDeviceFilter { - case EdgeDeviceFilterAll: - return true - case EdgeDeviceFilterTrusted: - return endpoint.UserTrusted - case EdgeDeviceFilterUntrusted: - return !endpoint.UserTrusted - } - - return false -} - -func convertTagIDsToTags(tagsMap map[portainer.TagID]string, tagIDs []portainer.TagID) []string { - tags := make([]string, 0) - for _, tagID := range tagIDs { - tags = append(tags, tagsMap[tagID]) - } - return tags -} - -func filteredEndpointsByTags(endpoints []portainer.Endpoint, tagIDs []portainer.TagID, endpointGroups []portainer.EndpointGroup, partialMatch bool) []portainer.Endpoint { - filteredEndpoints := make([]portainer.Endpoint, 0) - - for _, endpoint := range endpoints { - endpointGroup := getEndpointGroup(endpoint.GroupID, endpointGroups) - endpointMatched := false - if partialMatch { - endpointMatched = endpointPartialMatchTags(endpoint, endpointGroup, tagIDs) - } else { - endpointMatched = endpointFullMatchTags(endpoint, endpointGroup, tagIDs) - } - - if endpointMatched { - filteredEndpoints = append(filteredEndpoints, endpoint) - } - } - return filteredEndpoints -} - func getEndpointGroup(groupID portainer.EndpointGroupID, groups []portainer.EndpointGroup) portainer.EndpointGroup { var endpointGroup portainer.EndpointGroup for _, group := range groups { @@ -421,72 +180,3 @@ func getEndpointGroup(groupID portainer.EndpointGroupID, groups []portainer.Endp } return endpointGroup } - -func endpointPartialMatchTags(endpoint portainer.Endpoint, endpointGroup portainer.EndpointGroup, tagIDs []portainer.TagID) bool { - tagSet := make(map[portainer.TagID]bool) - for _, tagID := range tagIDs { - tagSet[tagID] = true - } - for _, tagID := range endpoint.TagIDs { - if tagSet[tagID] { - return true - } - } - for _, tagID := range endpointGroup.TagIDs { - if tagSet[tagID] { - return true - } - } - return false -} - -func endpointFullMatchTags(endpoint portainer.Endpoint, endpointGroup portainer.EndpointGroup, tagIDs []portainer.TagID) bool { - missingTags := make(map[portainer.TagID]bool) - for _, tagID := range tagIDs { - missingTags[tagID] = true - } - for _, tagID := range endpoint.TagIDs { - if missingTags[tagID] { - delete(missingTags, tagID) - } - } - for _, tagID := range endpointGroup.TagIDs { - if missingTags[tagID] { - delete(missingTags, tagID) - } - } - return len(missingTags) == 0 -} - -func filteredEndpointsByIds(endpoints []portainer.Endpoint, ids []portainer.EndpointID) []portainer.Endpoint { - filteredEndpoints := make([]portainer.Endpoint, 0) - - idsSet := make(map[portainer.EndpointID]bool) - for _, id := range ids { - idsSet[id] = true - } - - for _, endpoint := range endpoints { - if idsSet[endpoint.ID] { - filteredEndpoints = append(filteredEndpoints, endpoint) - } - } - - return filteredEndpoints - -} - -func filterEndpointsByName(endpoints []portainer.Endpoint, name string) []portainer.Endpoint { - if name == "" { - return endpoints - } - - filteredEndpoints := make([]portainer.Endpoint, 0) - - for _, endpoint := range endpoints { - if endpoint.Name == name { - filteredEndpoints = append(filteredEndpoints, endpoint) - } - } - return filteredEndpoints -} diff --git a/api/http/handler/endpoints/endpoint_list_test.go b/api/http/handler/endpoints/endpoint_list_test.go index 41d8fc9ed..86dafd692 100644 --- a/api/http/handler/endpoints/endpoint_list_test.go +++ b/api/http/handler/endpoints/endpoint_list_test.go @@ -16,66 +16,64 @@ import ( "github.com/stretchr/testify/assert" ) -type endpointListEdgeDeviceTest struct { +type endpointListTest struct { title string expected []portainer.EndpointID - filter string } -func Test_endpointList(t *testing.T) { - var err error - is := assert.New(t) +func Test_endpointList_edgeDeviceFilter(t *testing.T) { - _, store, teardown := datastore.MustNewTestStore(true, true) - defer teardown() - - trustedEndpoint := portainer.Endpoint{ID: 1, UserTrusted: true, IsEdgeDevice: true, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment} - untrustedEndpoint := portainer.Endpoint{ID: 2, UserTrusted: false, IsEdgeDevice: true, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment} + trustedEdgeDevice := portainer.Endpoint{ID: 1, UserTrusted: true, IsEdgeDevice: true, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment} + untrustedEdgeDevice := portainer.Endpoint{ID: 2, UserTrusted: false, IsEdgeDevice: true, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment} regularUntrustedEdgeEndpoint := portainer.Endpoint{ID: 3, UserTrusted: false, IsEdgeDevice: false, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment} regularTrustedEdgeEndpoint := portainer.Endpoint{ID: 4, UserTrusted: true, IsEdgeDevice: false, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment} regularEndpoint := portainer.Endpoint{ID: 5, UserTrusted: false, IsEdgeDevice: false, GroupID: 1, Type: portainer.DockerEnvironment} - endpoints := []portainer.Endpoint{ - trustedEndpoint, - untrustedEndpoint, + handler, teardown := setup(t, []portainer.Endpoint{ + trustedEdgeDevice, + untrustedEdgeDevice, regularUntrustedEdgeEndpoint, regularTrustedEdgeEndpoint, regularEndpoint, + }) + + defer teardown() + + type endpointListEdgeDeviceTest struct { + endpointListTest + edgeDevice *bool + edgeDeviceUntrusted bool } - for _, endpoint := range endpoints { - err = store.Endpoint().Create(&endpoint) - is.NoError(err, "error creating environment") - } - - err = store.User().Create(&portainer.User{Username: "admin", Role: portainer.AdministratorRole}) - is.NoError(err, "error creating a user") - - bouncer := helper.NewTestRequestBouncer() - h := NewHandler(bouncer, nil) - h.DataStore = store - h.ComposeStackManager = testhelpers.NewComposeStackManager() - tests := []endpointListEdgeDeviceTest{ { - "should show all edge endpoints", - []portainer.EndpointID{trustedEndpoint.ID, untrustedEndpoint.ID, regularUntrustedEdgeEndpoint.ID, regularTrustedEdgeEndpoint.ID}, - EdgeDeviceFilterAll, + endpointListTest: endpointListTest{ + "should show all endpoints expect of the untrusted devices", + []portainer.EndpointID{trustedEdgeDevice.ID, regularUntrustedEdgeEndpoint.ID, regularTrustedEdgeEndpoint.ID, regularEndpoint.ID}, + }, + edgeDevice: nil, }, { - "should show only trusted edge devices", - []portainer.EndpointID{trustedEndpoint.ID, regularTrustedEdgeEndpoint.ID}, - EdgeDeviceFilterTrusted, + endpointListTest: endpointListTest{ + "should show only trusted edge devices and regular endpoints", + []portainer.EndpointID{trustedEdgeDevice.ID, regularEndpoint.ID}, + }, + edgeDevice: BoolAddr(true), }, { - "should show only untrusted edge devices", - []portainer.EndpointID{untrustedEndpoint.ID, regularUntrustedEdgeEndpoint.ID}, - EdgeDeviceFilterUntrusted, + endpointListTest: endpointListTest{ + "should show only untrusted edge devices and regular endpoints", + []portainer.EndpointID{untrustedEdgeDevice.ID, regularEndpoint.ID}, + }, + edgeDevice: BoolAddr(true), + edgeDeviceUntrusted: true, }, { - "should show no edge devices", - []portainer.EndpointID{regularEndpoint.ID, regularUntrustedEdgeEndpoint.ID, regularTrustedEdgeEndpoint.ID}, - EdgeDeviceFilterNone, + endpointListTest: endpointListTest{ + "should show no edge devices", + []portainer.EndpointID{regularEndpoint.ID, regularUntrustedEdgeEndpoint.ID, regularTrustedEdgeEndpoint.ID}, + }, + edgeDevice: BoolAddr(false), }, } @@ -83,8 +81,13 @@ func Test_endpointList(t *testing.T) { t.Run(test.title, func(t *testing.T) { is := assert.New(t) - req := buildEndpointListRequest(test.filter) - resp, err := doEndpointListRequest(req, h, is) + query := fmt.Sprintf("edgeDeviceUntrusted=%v&", test.edgeDeviceUntrusted) + if test.edgeDevice != nil { + query += fmt.Sprintf("edgeDevice=%v&", *test.edgeDevice) + } + + req := buildEndpointListRequest(query) + resp, err := doEndpointListRequest(req, handler, is) is.NoError(err) is.Equal(len(test.expected), len(resp)) @@ -100,8 +103,28 @@ func Test_endpointList(t *testing.T) { } } -func buildEndpointListRequest(filter string) *http.Request { - req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/endpoints?edgeDeviceFilter=%s", filter), nil) +func setup(t *testing.T, endpoints []portainer.Endpoint) (handler *Handler, teardown func()) { + is := assert.New(t) + _, store, teardown := datastore.MustNewTestStore(true, true) + + for _, endpoint := range endpoints { + err := store.Endpoint().Create(&endpoint) + is.NoError(err, "error creating environment") + } + + err := store.User().Create(&portainer.User{Username: "admin", Role: portainer.AdministratorRole}) + is.NoError(err, "error creating a user") + + bouncer := helper.NewTestRequestBouncer() + handler = NewHandler(bouncer, nil) + handler.DataStore = store + handler.ComposeStackManager = testhelpers.NewComposeStackManager() + + return handler, teardown +} + +func buildEndpointListRequest(query string) *http.Request { + req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/endpoints?%s", query), nil) ctx := security.StoreTokenData(req, &portainer.TokenData{ID: 1, Username: "admin", Role: 1}) req = req.WithContext(ctx) diff --git a/api/http/handler/endpoints/filter.go b/api/http/handler/endpoints/filter.go new file mode 100644 index 000000000..5798f512e --- /dev/null +++ b/api/http/handler/endpoints/filter.go @@ -0,0 +1,415 @@ +package endpoints + +import ( + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "github.com/pkg/errors" + "github.com/portainer/libhttp/request" + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/internal/endpointutils" + "golang.org/x/exp/slices" +) + +type EnvironmentsQuery struct { + search string + types []portainer.EndpointType + tagIds []portainer.TagID + endpointIds []portainer.EndpointID + tagsPartialMatch bool + groupIds []portainer.EndpointGroupID + status []portainer.EndpointStatus + edgeDevice *bool + edgeDeviceUntrusted bool + name string +} + +func parseQuery(r *http.Request) (EnvironmentsQuery, error) { + search, _ := request.RetrieveQueryParameter(r, "search", true) + if search != "" { + search = strings.ToLower(search) + } + + status, err := getNumberArrayQueryParameter[portainer.EndpointStatus](r, "status") + if err != nil { + return EnvironmentsQuery{}, err + } + + groupIDs, err := getNumberArrayQueryParameter[portainer.EndpointGroupID](r, "groupIds") + if err != nil { + return EnvironmentsQuery{}, err + } + + endpointTypes, err := getNumberArrayQueryParameter[portainer.EndpointType](r, "types") + if err != nil { + return EnvironmentsQuery{}, err + } + + tagIDs, err := getNumberArrayQueryParameter[portainer.TagID](r, "tagIds") + if err != nil { + return EnvironmentsQuery{}, err + } + + tagsPartialMatch, _ := request.RetrieveBooleanQueryParameter(r, "tagsPartialMatch", true) + + endpointIDs, err := getNumberArrayQueryParameter[portainer.EndpointID](r, "endpointIds") + if err != nil { + return EnvironmentsQuery{}, err + } + + name, _ := request.RetrieveQueryParameter(r, "name", true) + + edgeDeviceParam, _ := request.RetrieveQueryParameter(r, "edgeDevice", true) + + var edgeDevice *bool + if edgeDeviceParam != "" { + edgeDevice = BoolAddr(edgeDeviceParam == "true") + } + + edgeDeviceUntrusted, _ := request.RetrieveBooleanQueryParameter(r, "edgeDeviceUntrusted", true) + + return EnvironmentsQuery{ + search: search, + types: endpointTypes, + tagIds: tagIDs, + endpointIds: endpointIDs, + tagsPartialMatch: tagsPartialMatch, + groupIds: groupIDs, + status: status, + edgeDevice: edgeDevice, + edgeDeviceUntrusted: edgeDeviceUntrusted, + name: name, + }, nil +} + +func (handler *Handler) filterEndpointsByQuery(filteredEndpoints []portainer.Endpoint, query EnvironmentsQuery, groups []portainer.EndpointGroup, settings *portainer.Settings) ([]portainer.Endpoint, int, error) { + totalAvailableEndpoints := len(filteredEndpoints) + + if len(query.endpointIds) > 0 { + filteredEndpoints = filteredEndpointsByIds(filteredEndpoints, query.endpointIds) + } + + if len(query.groupIds) > 0 { + filteredEndpoints = filterEndpointsByGroupIDs(filteredEndpoints, query.groupIds) + } + + if query.name != "" { + filteredEndpoints = filterEndpointsByName(filteredEndpoints, query.name) + } + + if query.edgeDevice != nil { + filteredEndpoints = filterEndpointsByEdgeDevice(filteredEndpoints, *query.edgeDevice, query.edgeDeviceUntrusted) + } else { + // If the edgeDevice parameter is not set, we need to filter out the untrusted edge devices + filteredEndpoints = filter(filteredEndpoints, func(endpoint portainer.Endpoint) bool { + return !endpoint.IsEdgeDevice || endpoint.UserTrusted + }) + } + + if len(query.status) > 0 { + filteredEndpoints = filterEndpointsByStatuses(filteredEndpoints, query.status, settings) + } + + if query.search != "" { + tags, err := handler.DataStore.Tag().Tags() + if err != nil { + return nil, 0, errors.WithMessage(err, "Unable to retrieve tags from the database") + } + + tagsMap := make(map[portainer.TagID]string) + for _, tag := range tags { + tagsMap[tag.ID] = tag.Name + } + + filteredEndpoints = filterEndpointsBySearchCriteria(filteredEndpoints, groups, tagsMap, query.search) + } + + if len(query.types) > 0 { + filteredEndpoints = filterEndpointsByTypes(filteredEndpoints, query.types) + } + + if len(query.tagIds) > 0 { + filteredEndpoints = filteredEndpointsByTags(filteredEndpoints, query.tagIds, groups, query.tagsPartialMatch) + } + + return filteredEndpoints, totalAvailableEndpoints, nil +} + +func filterEndpointsByGroupIDs(endpoints []portainer.Endpoint, endpointGroupIDs []portainer.EndpointGroupID) []portainer.Endpoint { + filteredEndpoints := make([]portainer.Endpoint, 0) + + for _, endpoint := range endpoints { + if slices.Contains(endpointGroupIDs, endpoint.GroupID) { + filteredEndpoints = append(filteredEndpoints, endpoint) + } + } + + return filteredEndpoints +} + +func filterEndpointsBySearchCriteria(endpoints []portainer.Endpoint, endpointGroups []portainer.EndpointGroup, tagsMap map[portainer.TagID]string, searchCriteria string) []portainer.Endpoint { + filteredEndpoints := make([]portainer.Endpoint, 0) + + for _, endpoint := range endpoints { + endpointTags := convertTagIDsToTags(tagsMap, endpoint.TagIDs) + if endpointMatchSearchCriteria(&endpoint, endpointTags, searchCriteria) { + filteredEndpoints = append(filteredEndpoints, endpoint) + continue + } + + if endpointGroupMatchSearchCriteria(&endpoint, endpointGroups, tagsMap, searchCriteria) { + filteredEndpoints = append(filteredEndpoints, endpoint) + } + } + + return filteredEndpoints +} + +func filterEndpointsByStatuses(endpoints []portainer.Endpoint, statuses []portainer.EndpointStatus, settings *portainer.Settings) []portainer.Endpoint { + filteredEndpoints := make([]portainer.Endpoint, 0) + + for _, endpoint := range endpoints { + status := endpoint.Status + if endpointutils.IsEdgeEndpoint(&endpoint) { + isCheckValid := false + edgeCheckinInterval := endpoint.EdgeCheckinInterval + if endpoint.EdgeCheckinInterval == 0 { + edgeCheckinInterval = settings.EdgeAgentCheckinInterval + } + + if edgeCheckinInterval != 0 && endpoint.LastCheckInDate != 0 { + isCheckValid = time.Now().Unix()-endpoint.LastCheckInDate <= int64(edgeCheckinInterval*EdgeDeviceIntervalMultiplier+EdgeDeviceIntervalAdd) + } + + status = portainer.EndpointStatusDown // Offline + if isCheckValid { + status = portainer.EndpointStatusUp // Online + } + } + + if slices.Contains(statuses, status) { + filteredEndpoints = append(filteredEndpoints, endpoint) + } + } + + return filteredEndpoints +} + +func endpointMatchSearchCriteria(endpoint *portainer.Endpoint, tags []string, searchCriteria string) bool { + if strings.Contains(strings.ToLower(endpoint.Name), searchCriteria) { + return true + } + + if strings.Contains(strings.ToLower(endpoint.URL), searchCriteria) { + return true + } + + if endpoint.Status == portainer.EndpointStatusUp && searchCriteria == "up" { + return true + } else if endpoint.Status == portainer.EndpointStatusDown && searchCriteria == "down" { + return true + } + for _, tag := range tags { + if strings.Contains(strings.ToLower(tag), searchCriteria) { + return true + } + } + + return false +} + +func endpointGroupMatchSearchCriteria(endpoint *portainer.Endpoint, endpointGroups []portainer.EndpointGroup, tagsMap map[portainer.TagID]string, searchCriteria string) bool { + for _, group := range endpointGroups { + if group.ID == endpoint.GroupID { + if strings.Contains(strings.ToLower(group.Name), searchCriteria) { + return true + } + tags := convertTagIDsToTags(tagsMap, group.TagIDs) + for _, tag := range tags { + if strings.Contains(strings.ToLower(tag), searchCriteria) { + return true + } + } + } + } + + return false +} + +func filterEndpointsByTypes(endpoints []portainer.Endpoint, endpointTypes []portainer.EndpointType) []portainer.Endpoint { + filteredEndpoints := make([]portainer.Endpoint, 0) + + typeSet := map[portainer.EndpointType]bool{} + for _, endpointType := range endpointTypes { + typeSet[portainer.EndpointType(endpointType)] = true + } + + for _, endpoint := range endpoints { + if typeSet[endpoint.Type] { + filteredEndpoints = append(filteredEndpoints, endpoint) + } + } + return filteredEndpoints +} + +func filterEndpointsByEdgeDevice(endpoints []portainer.Endpoint, edgeDevice bool, untrusted bool) []portainer.Endpoint { + filteredEndpoints := make([]portainer.Endpoint, 0) + + for _, endpoint := range endpoints { + if shouldReturnEdgeDevice(endpoint, edgeDevice, untrusted) { + filteredEndpoints = append(filteredEndpoints, endpoint) + } + } + return filteredEndpoints +} + +func shouldReturnEdgeDevice(endpoint portainer.Endpoint, edgeDeviceParam bool, untrustedParam bool) bool { + if !endpointutils.IsEdgeEndpoint(&endpoint) { + return true + } + + if !edgeDeviceParam { + return !endpoint.IsEdgeDevice + } + + return endpoint.IsEdgeDevice && endpoint.UserTrusted == !untrustedParam +} + +func convertTagIDsToTags(tagsMap map[portainer.TagID]string, tagIDs []portainer.TagID) []string { + tags := make([]string, 0) + for _, tagID := range tagIDs { + tags = append(tags, tagsMap[tagID]) + } + return tags +} + +func filteredEndpointsByTags(endpoints []portainer.Endpoint, tagIDs []portainer.TagID, endpointGroups []portainer.EndpointGroup, partialMatch bool) []portainer.Endpoint { + filteredEndpoints := make([]portainer.Endpoint, 0) + + for _, endpoint := range endpoints { + endpointGroup := getEndpointGroup(endpoint.GroupID, endpointGroups) + endpointMatched := false + if partialMatch { + endpointMatched = endpointPartialMatchTags(endpoint, endpointGroup, tagIDs) + } else { + endpointMatched = endpointFullMatchTags(endpoint, endpointGroup, tagIDs) + } + + if endpointMatched { + filteredEndpoints = append(filteredEndpoints, endpoint) + } + } + return filteredEndpoints +} + +func endpointPartialMatchTags(endpoint portainer.Endpoint, endpointGroup portainer.EndpointGroup, tagIDs []portainer.TagID) bool { + tagSet := make(map[portainer.TagID]bool) + for _, tagID := range tagIDs { + tagSet[tagID] = true + } + for _, tagID := range endpoint.TagIDs { + if tagSet[tagID] { + return true + } + } + for _, tagID := range endpointGroup.TagIDs { + if tagSet[tagID] { + return true + } + } + return false +} + +func endpointFullMatchTags(endpoint portainer.Endpoint, endpointGroup portainer.EndpointGroup, tagIDs []portainer.TagID) bool { + missingTags := make(map[portainer.TagID]bool) + for _, tagID := range tagIDs { + missingTags[tagID] = true + } + for _, tagID := range endpoint.TagIDs { + if missingTags[tagID] { + delete(missingTags, tagID) + } + } + for _, tagID := range endpointGroup.TagIDs { + if missingTags[tagID] { + delete(missingTags, tagID) + } + } + return len(missingTags) == 0 +} + +func filteredEndpointsByIds(endpoints []portainer.Endpoint, ids []portainer.EndpointID) []portainer.Endpoint { + filteredEndpoints := make([]portainer.Endpoint, 0) + + idsSet := make(map[portainer.EndpointID]bool) + for _, id := range ids { + idsSet[id] = true + } + + for _, endpoint := range endpoints { + if idsSet[endpoint.ID] { + filteredEndpoints = append(filteredEndpoints, endpoint) + } + } + + return filteredEndpoints + +} + +func filterEndpointsByName(endpoints []portainer.Endpoint, name string) []portainer.Endpoint { + if name == "" { + return endpoints + } + + filteredEndpoints := make([]portainer.Endpoint, 0) + + for _, endpoint := range endpoints { + if endpoint.Name == name { + filteredEndpoints = append(filteredEndpoints, endpoint) + } + } + return filteredEndpoints +} + +func filter(endpoints []portainer.Endpoint, predicate func(endpoint portainer.Endpoint) bool) []portainer.Endpoint { + filteredEndpoints := make([]portainer.Endpoint, 0) + + for _, endpoint := range endpoints { + if predicate(endpoint) { + filteredEndpoints = append(filteredEndpoints, endpoint) + } + } + return filteredEndpoints +} + +func getArrayQueryParameter(r *http.Request, parameter string) []string { + list, exists := r.Form[fmt.Sprintf("%s[]", parameter)] + if !exists { + list = []string{} + } + + return list +} + +func getNumberArrayQueryParameter[T ~int](r *http.Request, parameter string) ([]T, error) { + list := getArrayQueryParameter(r, parameter) + if list == nil { + return []T{}, nil + } + + var result []T + for _, item := range list { + number, err := strconv.Atoi(item) + if err != nil { + return nil, errors.Wrapf(err, "Unable to parse parameter %s", parameter) + + } + + result = append(result, T(number)) + } + + return result, nil +} diff --git a/api/http/handler/endpoints/filter_test.go b/api/http/handler/endpoints/filter_test.go new file mode 100644 index 000000000..4f96c107a --- /dev/null +++ b/api/http/handler/endpoints/filter_test.go @@ -0,0 +1,119 @@ +package endpoints + +import ( + "testing" + + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/datastore" + "github.com/portainer/portainer/api/internal/testhelpers" + helper "github.com/portainer/portainer/api/internal/testhelpers" + "github.com/stretchr/testify/assert" +) + +type filterTest struct { + title string + expected []portainer.EndpointID + query EnvironmentsQuery +} + +func Test_Filter_edgeDeviceFilter(t *testing.T) { + + trustedEdgeDevice := portainer.Endpoint{ID: 1, UserTrusted: true, IsEdgeDevice: true, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment} + untrustedEdgeDevice := portainer.Endpoint{ID: 2, UserTrusted: false, IsEdgeDevice: true, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment} + regularUntrustedEdgeEndpoint := portainer.Endpoint{ID: 3, UserTrusted: false, IsEdgeDevice: false, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment} + regularTrustedEdgeEndpoint := portainer.Endpoint{ID: 4, UserTrusted: true, IsEdgeDevice: false, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment} + regularEndpoint := portainer.Endpoint{ID: 5, GroupID: 1, Type: portainer.DockerEnvironment} + + endpoints := []portainer.Endpoint{ + trustedEdgeDevice, + untrustedEdgeDevice, + regularUntrustedEdgeEndpoint, + regularTrustedEdgeEndpoint, + regularEndpoint, + } + + handler, teardown := setupFilterTest(t, endpoints) + + defer teardown() + + tests := []filterTest{ + { + "should show all edge endpoints except of the untrusted devices", + []portainer.EndpointID{trustedEdgeDevice.ID, regularUntrustedEdgeEndpoint.ID, regularTrustedEdgeEndpoint.ID}, + EnvironmentsQuery{ + types: []portainer.EndpointType{portainer.EdgeAgentOnDockerEnvironment, portainer.EdgeAgentOnKubernetesEnvironment}, + }, + }, + { + "should show only trusted edge devices and other regular endpoints", + []portainer.EndpointID{trustedEdgeDevice.ID, regularEndpoint.ID}, + EnvironmentsQuery{ + edgeDevice: BoolAddr(true), + }, + }, + { + "should show only untrusted edge devices and other regular endpoints", + []portainer.EndpointID{untrustedEdgeDevice.ID, regularEndpoint.ID}, + EnvironmentsQuery{ + edgeDevice: BoolAddr(true), + edgeDeviceUntrusted: true, + }, + }, + { + "should show no edge devices", + []portainer.EndpointID{regularEndpoint.ID, regularUntrustedEdgeEndpoint.ID, regularTrustedEdgeEndpoint.ID}, + EnvironmentsQuery{ + edgeDevice: BoolAddr(false), + }, + }, + } + + runTests(tests, t, handler, endpoints) +} + +func runTests(tests []filterTest, t *testing.T, handler *Handler, endpoints []portainer.Endpoint) { + for _, test := range tests { + t.Run(test.title, func(t *testing.T) { + runTest(t, test, handler, endpoints) + }) + } +} + +func runTest(t *testing.T, test filterTest, handler *Handler, endpoints []portainer.Endpoint) { + is := assert.New(t) + + filteredEndpoints, _, err := handler.filterEndpointsByQuery(endpoints, test.query, []portainer.EndpointGroup{}, &portainer.Settings{}) + + is.NoError(err) + + is.Equal(len(test.expected), len(filteredEndpoints)) + + respIds := []portainer.EndpointID{} + + for _, endpoint := range filteredEndpoints { + respIds = append(respIds, endpoint.ID) + } + + is.ElementsMatch(test.expected, respIds) + +} + +func setupFilterTest(t *testing.T, endpoints []portainer.Endpoint) (handler *Handler, teardown func()) { + is := assert.New(t) + _, store, teardown := datastore.MustNewTestStore(true, true) + + for _, endpoint := range endpoints { + err := store.Endpoint().Create(&endpoint) + is.NoError(err, "error creating environment") + } + + err := store.User().Create(&portainer.User{Username: "admin", Role: portainer.AdministratorRole}) + is.NoError(err, "error creating a user") + + bouncer := helper.NewTestRequestBouncer() + handler = NewHandler(bouncer, nil) + handler.DataStore = store + handler.ComposeStackManager = testhelpers.NewComposeStackManager() + + return handler, teardown +} diff --git a/api/http/handler/endpoints/utils.go b/api/http/handler/endpoints/utils.go new file mode 100644 index 000000000..a00f36358 --- /dev/null +++ b/api/http/handler/endpoints/utils.go @@ -0,0 +1,6 @@ +package endpoints + +func BoolAddr(b bool) *bool { + boolVar := b + return &boolVar +} diff --git a/app/edge/EdgeDevices/EdgeDevicesView/EdgeDevicesDatatable/EdgeDevicesDatatableContainer.tsx b/app/edge/EdgeDevices/EdgeDevicesView/EdgeDevicesDatatable/EdgeDevicesDatatableContainer.tsx index 06249f082..a1b8384e9 100644 --- a/app/edge/EdgeDevices/EdgeDevicesView/EdgeDevicesDatatable/EdgeDevicesDatatableContainer.tsx +++ b/app/edge/EdgeDevices/EdgeDevicesView/EdgeDevicesDatatable/EdgeDevicesDatatableContainer.tsx @@ -1,7 +1,7 @@ import { useState } from 'react'; import { useEnvironmentList } from '@/portainer/environments/queries/useEnvironmentList'; -import { Environment } from '@/portainer/environments/types'; +import { EdgeTypes, Environment } from '@/portainer/environments/types'; import { useDebounce } from '@/portainer/hooks/useDebounce'; import { useSearchBarState } from '@@/datatables/SearchBar'; @@ -89,8 +89,9 @@ function Loader({ children, storageKey }: LoaderProps) { const { environments, isLoading, totalCount } = useEnvironmentList( { - edgeDeviceFilter: 'trusted', + edgeDevice: true, search: debouncedSearchValue, + types: EdgeTypes, ...pagination, }, settings.autoRefreshRate * 1000 diff --git a/app/edge/EdgeDevices/WaitingRoomView/WaitingRoomView.tsx b/app/edge/EdgeDevices/WaitingRoomView/WaitingRoomView.tsx index 63ed067b4..381b32aa1 100644 --- a/app/edge/EdgeDevices/WaitingRoomView/WaitingRoomView.tsx +++ b/app/edge/EdgeDevices/WaitingRoomView/WaitingRoomView.tsx @@ -2,6 +2,7 @@ import { useRouter } from '@uirouter/react'; import { useEnvironmentList } from '@/portainer/environments/queries/useEnvironmentList'; import { r2a } from '@/react-tools/react2angular'; +import { EdgeTypes } from '@/portainer/environments/types'; import { InformationPanel } from '@@/InformationPanel'; import { TextTip } from '@@/Tip/TextTip'; @@ -15,7 +16,9 @@ export function WaitingRoomView() { const storageKey = 'edge-devices-waiting-room'; const router = useRouter(); const { environments, isLoading, totalCount } = useEnvironmentList({ - edgeDeviceFilter: 'untrusted', + edgeDevice: true, + edgeDeviceUntrusted: true, + types: EdgeTypes, }); if (process.env.PORTAINER_EDITION !== 'BE') { diff --git a/app/edge/components/group-form/groupFormController.js b/app/edge/components/group-form/groupFormController.js index eac5673bc..55547ec61 100644 --- a/app/edge/components/group-form/groupFormController.js +++ b/app/edge/components/group-form/groupFormController.js @@ -1,10 +1,11 @@ import _ from 'lodash-es'; import { confirmAsync } from '@/portainer/services/modal.service/confirm'; +import { EdgeTypes } from '@/portainer/environments/types'; +import { getEnvironments } from '@/portainer/environments/environment.service'; export class EdgeGroupFormController { /* @ngInject */ - constructor(EndpointService, $async, $scope) { - this.EndpointService = EndpointService; + constructor($async, $scope) { this.$async = $async; this.$scope = $scope; @@ -19,7 +20,6 @@ export class EdgeGroupFormController { }; this.associateEndpoint = this.associateEndpoint.bind(this); - this.dissociateEndpointAsync = this.dissociateEndpointAsync.bind(this); this.dissociateEndpoint = this.dissociateEndpoint.bind(this); this.getDynamicEndpointsAsync = this.getDynamicEndpointsAsync.bind(this); this.getDynamicEndpoints = this.getDynamicEndpoints.bind(this); @@ -49,30 +49,28 @@ export class EdgeGroupFormController { } dissociateEndpoint(endpoint) { - return this.$async(this.dissociateEndpointAsync, endpoint); - } + return this.$async(async () => { + const confirmed = await confirmAsync({ + title: 'Confirm action', + message: 'Removing the environment from this group will remove its corresponding edge stacks', + buttons: { + cancel: { + label: 'Cancel', + className: 'btn-default', + }, + confirm: { + label: 'Confirm', + className: 'btn-primary', + }, + }, + }); - async dissociateEndpointAsync(endpoint) { - const confirmed = await confirmAsync({ - title: 'Confirm action', - message: 'Removing the environment from this group will remove its corresponding edge stacks', - buttons: { - cancel: { - label: 'Cancel', - className: 'btn-default', - }, - confirm: { - label: 'Confirm', - className: 'btn-primary', - }, - }, + if (!confirmed) { + return; + } + + this.model.Endpoints = _.filter(this.model.Endpoints, (id) => id !== endpoint.Id); }); - - if (!confirmed) { - return; - } - - this.model.Endpoints = _.filter(this.model.Endpoints, (id) => id !== endpoint.Id); } getDynamicEndpoints() { @@ -82,9 +80,9 @@ export class EdgeGroupFormController { async getDynamicEndpointsAsync() { const { pageNumber, limit, search } = this.endpoints.state; const start = (pageNumber - 1) * limit + 1; - const query = { search, types: [4, 7], tagIds: this.model.TagIds, tagsPartialMatch: this.model.PartialMatch }; + const query = { search, types: EdgeTypes, tagIds: this.model.TagIds, tagsPartialMatch: this.model.PartialMatch }; - const response = await this.EndpointService.endpoints(start, limit, query); + const response = await getEnvironments({ start, limit, query }); const totalCount = parseInt(response.totalCount, 10); this.endpoints.value = response.value; diff --git a/app/edge/views/edge-jobs/edgeJob/edgeJobController.js b/app/edge/views/edge-jobs/edgeJob/edgeJobController.js index 1dd35e2f7..51dbb5a18 100644 --- a/app/edge/views/edge-jobs/edgeJob/edgeJobController.js +++ b/app/edge/views/edge-jobs/edgeJob/edgeJobController.js @@ -1,8 +1,9 @@ import _ from 'lodash-es'; +import { getEnvironments } from '@/portainer/environments/environment.service'; export class EdgeJobController { /* @ngInject */ - constructor($async, $q, $state, $window, ModalService, EdgeJobService, EndpointService, FileSaver, GroupService, HostBrowserService, Notifications, TagService) { + constructor($async, $q, $state, $window, ModalService, EdgeJobService, FileSaver, GroupService, HostBrowserService, Notifications, TagService) { this.state = { actionInProgress: false, showEditorTab: false, @@ -15,7 +16,6 @@ export class EdgeJobController { this.$window = $window; this.ModalService = ModalService; this.EdgeJobService = EdgeJobService; - this.EndpointService = EndpointService; this.FileSaver = FileSaver; this.GroupService = GroupService; this.HostBrowserService = HostBrowserService; @@ -114,7 +114,7 @@ export class EdgeJobController { const results = await this.EdgeJobService.jobResults(id); if (results.length > 0) { const endpointIds = _.map(results, (result) => result.EndpointId); - const endpoints = await this.EndpointService.endpoints(undefined, undefined, { endpointIds }); + const endpoints = await getEnvironments({ query: { endpointIds } }); this.results = this.associateEndpointsToResults(results, endpoints.value); } else { this.results = results; @@ -155,7 +155,7 @@ export class EdgeJobController { if (results.length > 0) { const endpointIds = _.map(results, (result) => result.EndpointId); - const endpoints = await this.EndpointService.endpoints(undefined, undefined, { endpointIds }); + const endpoints = await getEnvironments({ query: { endpointIds } }); this.results = this.associateEndpointsToResults(results, endpoints.value); } else { this.results = results; diff --git a/app/edge/views/edge-stacks/editEdgeStackView/editEdgeStackViewController.js b/app/edge/views/edge-stacks/editEdgeStackView/editEdgeStackViewController.js index 742b31728..cbbb08cc2 100644 --- a/app/edge/views/edge-stacks/editEdgeStackView/editEdgeStackViewController.js +++ b/app/edge/views/edge-stacks/editEdgeStackView/editEdgeStackViewController.js @@ -1,15 +1,15 @@ import _ from 'lodash-es'; +import { getEnvironments } from '@/portainer/environments/environment.service'; export class EditEdgeStackViewController { /* @ngInject */ - constructor($async, $state, $window, ModalService, EdgeGroupService, EdgeStackService, EndpointService, Notifications) { + constructor($async, $state, $window, ModalService, EdgeGroupService, EdgeStackService, Notifications) { this.$async = $async; this.$state = $state; this.$window = $window; this.ModalService = ModalService; this.EdgeGroupService = EdgeGroupService; this.EdgeStackService = EdgeStackService; - this.EndpointService = EndpointService; this.Notifications = Notifications; this.stack = null; @@ -99,8 +99,8 @@ export class EditEdgeStackViewController { async getPaginatedEndpointsAsync(lastId, limit, search) { try { - const query = { search, types: [4, 7], endpointIds: this.stackEndpointIds }; - const { value, totalCount } = await this.EndpointService.endpoints(lastId, limit, query); + const query = { search, endpointIds: this.stackEndpointIds }; + const { value, totalCount } = await getEnvironments({ start: lastId, limit, query }); const endpoints = _.map(value, (endpoint) => { const status = this.stack.Status[endpoint.Id]; endpoint.Status = status; diff --git a/app/portainer/components/associated-endpoints-selector/associatedEndpointsSelector.html b/app/portainer/components/associated-endpoints-selector/associatedEndpointsSelector.html index d1656914e..d9f66635a 100644 --- a/app/portainer/components/associated-endpoints-selector/associatedEndpointsSelector.html +++ b/app/portainer/components/associated-endpoints-selector/associatedEndpointsSelector.html @@ -11,7 +11,7 @@ loaded="$ctrl.loaded" page-type="$ctrl.pageType" table-type="available" - retrieve-page="$ctrl.getPaginatedEndpoints" + retrieve-page="$ctrl.getAvailableEndpoints" dataset="$ctrl.endpoints.available" entry-click="$ctrl.associateEndpoint" pagination-state="$ctrl.state.available" @@ -34,7 +34,7 @@ loaded="$ctrl.loaded" page-type="$ctrl.pageType" table-type="associated" - retrieve-page="$ctrl.getPaginatedEndpoints" + retrieve-page="$ctrl.getAssociatedEndpoints" dataset="$ctrl.endpoints.associated" entry-click="$ctrl.dissociateEndpoint" pagination-state="$ctrl.state.associated" diff --git a/app/portainer/components/associated-endpoints-selector/associatedEndpointsSelectorController.js b/app/portainer/components/associated-endpoints-selector/associatedEndpointsSelectorController.js index c051d46c0..940573b95 100644 --- a/app/portainer/components/associated-endpoints-selector/associatedEndpointsSelectorController.js +++ b/app/portainer/components/associated-endpoints-selector/associatedEndpointsSelectorController.js @@ -1,11 +1,13 @@ import angular from 'angular'; import _ from 'lodash-es'; +import { EdgeTypes } from '@/portainer/environments/types'; +import { getEnvironments } from '@/portainer/environments/environment.service'; + class AssoicatedEndpointsSelectorController { /* @ngInject */ - constructor($async, EndpointService) { + constructor($async) { this.$async = $async; - this.EndpointService = EndpointService; this.state = { available: { @@ -27,12 +29,11 @@ class AssoicatedEndpointsSelectorController { available: null, }; - this.getEndpoints = this.getEndpoints.bind(this); - this.getEndpointsAsync = this.getEndpointsAsync.bind(this); + this.getAvailableEndpoints = this.getAvailableEndpoints.bind(this); this.getAssociatedEndpoints = this.getAssociatedEndpoints.bind(this); - this.getAssociatedEndpointsAsync = this.getAssociatedEndpointsAsync.bind(this); this.associateEndpoint = this.associateEndpoint.bind(this); this.dissociateEndpoint = this.dissociateEndpoint.bind(this); + this.loadData = this.loadData.bind(this); } $onInit() { @@ -46,41 +47,41 @@ class AssoicatedEndpointsSelectorController { } loadData() { + this.getAvailableEndpoints(); this.getAssociatedEndpoints(); - this.getEndpoints(); } - getEndpoints() { - return this.$async(this.getEndpointsAsync); - } + /* #region internal queries to retrieve endpoints per "side" of the selector */ + getAvailableEndpoints() { + return this.$async(async () => { + const { start, search, limit } = this.getPaginationData('available'); + const query = { search, types: EdgeTypes }; - async getEndpointsAsync() { - const { start, search, limit } = this.getPaginationData('available'); - const query = { search, types: [4, 7] }; + const response = await getEnvironments({ start, limit, query }); - const response = await this.EndpointService.endpoints(start, limit, query); - - const endpoints = _.filter(response.value, (endpoint) => !_.includes(this.endpointIds, endpoint.Id)); - this.setTableData('available', endpoints, response.totalCount); - this.noEndpoints = this.state.available.totalCount === 0; + const endpoints = _.filter(response.value, (endpoint) => !_.includes(this.endpointIds, endpoint.Id)); + this.setTableData('available', endpoints, response.totalCount); + this.noEndpoints = this.state.available.totalCount === 0; + }); } getAssociatedEndpoints() { - return this.$async(this.getAssociatedEndpointsAsync); - } - - async getAssociatedEndpointsAsync() { - let response = { value: [], totalCount: 0 }; - if (this.endpointIds.length > 0) { - const { start, search, limit } = this.getPaginationData('associated'); - const query = { search, types: [4, 7], endpointIds: this.endpointIds }; - - response = await this.EndpointService.endpoints(start, limit, query); - } - - this.setTableData('associated', response.value, response.totalCount); + return this.$async(async () => { + let response = { value: [], totalCount: 0 }; + if (this.endpointIds.length > 0) { + // fetch only if already has associated endpoints + const { start, search, limit } = this.getPaginationData('associated'); + const query = { search, types: EdgeTypes, endpointIds: this.endpointIds }; + + response = await getEnvironments({ start, limit, query }); + } + + this.setTableData('associated', response.value, response.totalCount); + }); } + /* #endregion */ + /* #region On endpoint click (either available or associated) */ associateEndpoint(endpoint) { this.onAssociate(endpoint); } @@ -88,7 +89,9 @@ class AssoicatedEndpointsSelectorController { dissociateEndpoint(endpoint) { this.onDissociate(endpoint); } + /* #endregion */ + /* #region Utils funcs */ getPaginationData(tableType) { const { pageNumber, limit, search } = this.state[tableType]; const start = (pageNumber - 1) * limit + 1; @@ -100,6 +103,7 @@ class AssoicatedEndpointsSelectorController { this.endpoints[tableType] = endpoints; this.state[tableType].totalCount = parseInt(totalCount, 10); } + /* #endregion */ } angular.module('portainer.app').controller('AssoicatedEndpointsSelectorController', AssoicatedEndpointsSelectorController); diff --git a/app/portainer/components/forms/group-form/groupFormController.js b/app/portainer/components/forms/group-form/groupFormController.js index 70c390301..2e2ac5524 100644 --- a/app/portainer/components/forms/group-form/groupFormController.js +++ b/app/portainer/components/forms/group-form/groupFormController.js @@ -1,12 +1,13 @@ import _ from 'lodash-es'; import angular from 'angular'; +import { endpointsByGroup } from '@/portainer/environments/environment.service'; +import { notifyError } from '@/portainer/services/notifications'; class GroupFormController { /* @ngInject */ - constructor($q, $scope, EndpointService, GroupService, Notifications, Authentication) { - this.$q = $q; + constructor($async, $scope, GroupService, Notifications, Authentication) { + this.$async = $async; this.$scope = $scope; - this.EndpointService = EndpointService; this.GroupService = GroupService; this.Notifications = Notifications; this.Authentication = Authentication; @@ -75,23 +76,27 @@ class GroupFormController { } getPaginatedEndpointsByGroup(pageType, tableType) { - if (tableType === 'available') { - const context = this.state.available; - const start = (context.pageNumber - 1) * context.limit + 1; - this.EndpointService.endpointsByGroup(start, context.limit, context.filter, 1).then((data) => { - this.availableEndpoints = data.value; - this.state.available.totalCount = data.totalCount; - }); - } else if (tableType === 'associated' && pageType === 'edit') { - const groupId = this.model.Id ? this.model.Id : 1; - const context = this.state.associated; - const start = (context.pageNumber - 1) * context.limit + 1; - this.EndpointService.endpointsByGroup(start, context.limit, context.filter, groupId).then((data) => { - this.associatedEndpoints = data.value; - this.state.associated.totalCount = data.totalCount; - }); - } - // ignore (associated + create) group as there is no backend pagination for this table + this.$async(async () => { + try { + if (tableType === 'available') { + const context = this.state.available; + const start = (context.pageNumber - 1) * context.limit + 1; + const data = await endpointsByGroup(1, start, context.limit, { search: context.filter }); + this.availableEndpoints = data.value; + this.state.available.totalCount = data.totalCount; + } else if (tableType === 'associated' && pageType === 'edit') { + const groupId = this.model.Id ? this.model.Id : 1; + const context = this.state.associated; + const start = (context.pageNumber - 1) * context.limit + 1; + const data = await endpointsByGroup(groupId, start, context.limit, { search: context.filter }); + this.associatedEndpoints = data.value; + this.state.associated.totalCount = data.totalCount; + } + // ignore (associated + create) group as there is no backend pagination for this table + } catch (err) { + notifyError('Failure', err, 'Failed getting endpoints for group'); + } + }); } } diff --git a/app/portainer/environments/environment.service/index.ts b/app/portainer/environments/environment.service/index.ts index 6db67d0d1..0bef44712 100644 --- a/app/portainer/environments/environment.service/index.ts +++ b/app/portainer/environments/environment.service/index.ts @@ -12,61 +12,50 @@ import type { EnvironmentStatus, } from '../types'; -import { arrayToJson, buildUrl } from './utils'; +import { buildUrl } from './utils'; export interface EnvironmentsQueryParams { search?: string; - types?: EnvironmentType[]; + types?: EnvironmentType[] | readonly EnvironmentType[]; tagIds?: TagId[]; endpointIds?: EnvironmentId[]; tagsPartialMatch?: boolean; groupIds?: EnvironmentGroupId[]; status?: EnvironmentStatus[]; - sort?: string; - order?: 'asc' | 'desc'; - edgeDeviceFilter?: 'all' | 'trusted' | 'untrusted' | 'none'; + edgeDevice?: boolean; + edgeDeviceUntrusted?: boolean; + provisioned?: boolean; name?: string; } -export async function getEndpoints( - start: number, - limit: number, +export interface GetEnvironmentsOptions { + start?: number; + limit?: number; + sort?: { by?: string; order?: 'asc' | 'desc' }; + query?: EnvironmentsQueryParams; +} + +export async function getEnvironments( { - types, - tagIds, - endpointIds, - status, - groupIds, - ...query - }: EnvironmentsQueryParams = {} + start, + limit, + sort = { by: '', order: 'asc' }, + query = {}, + }: GetEnvironmentsOptions = { query: {} } ) { - if (tagIds && tagIds.length === 0) { + if (query.tagIds && query.tagIds.length === 0) { return { totalCount: 0, value: [] }; } const url = buildUrl(); - const params: Record = { start, limit, ...query }; - - if (types) { - params.types = arrayToJson(types); - } - - if (tagIds) { - params.tagIds = arrayToJson(tagIds); - } - - if (endpointIds) { - params.endpointIds = arrayToJson(endpointIds); - } - - if (status) { - params.status = arrayToJson(status); - } - - if (groupIds) { - params.groupIds = arrayToJson(groupIds); - } + const params: Record = { + start, + limit, + sort: sort.by, + order: sort.order, + ...query, + }; try { const response = await axios.get(url, { params }); @@ -109,12 +98,16 @@ export async function snapshotEndpoint(id: EnvironmentId) { } export async function endpointsByGroup( + groupId: EnvironmentGroupId, start: number, limit: number, - search: string, - groupId: EnvironmentGroupId + query: Omit ) { - return getEndpoints(start, limit, { search, groupIds: [groupId] }); + return getEnvironments({ + start, + limit, + query: { groupIds: [groupId], ...query }, + }); } export async function disassociateEndpoint(id: EnvironmentId) { diff --git a/app/portainer/environments/queries/useEnvironmentList.ts b/app/portainer/environments/queries/useEnvironmentList.ts index be6ac3abd..1f68324d8 100644 --- a/app/portainer/environments/queries/useEnvironmentList.ts +++ b/app/portainer/environments/queries/useEnvironmentList.ts @@ -3,16 +3,21 @@ import { useQuery } from 'react-query'; import { withError } from '@/react-tools/react-query'; import { EnvironmentStatus } from '../types'; -import { EnvironmentsQueryParams, getEndpoints } from '../environment.service'; +import { + EnvironmentsQueryParams, + getEnvironments, +} from '../environment.service'; export const ENVIRONMENTS_POLLING_INTERVAL = 30000; // in ms -interface Query extends EnvironmentsQueryParams { +export interface Query extends EnvironmentsQueryParams { page?: number; pageLimit?: number; + sort?: string; + order?: 'asc' | 'desc'; } -type GetEndpointsResponse = Awaited>; +type GetEndpointsResponse = Awaited>; export function refetchIfAnyOffline(data?: GetEndpointsResponse) { if (!data) { @@ -31,7 +36,7 @@ export function refetchIfAnyOffline(data?: GetEndpointsResponse) { } export function useEnvironmentList( - { page = 1, pageLimit = 100, ...query }: Query = {}, + { page = 1, pageLimit = 100, sort, order, ...query }: Query = {}, refetchInterval?: | number | false @@ -45,12 +50,19 @@ export function useEnvironmentList( { page, pageLimit, + sort, + order, ...query, }, ], async () => { const start = (page - 1) * pageLimit + 1; - return getEndpoints(start, pageLimit, query); + return getEnvironments({ + start, + limit: pageLimit, + sort: { by: sort, order }, + query, + }); }, { staleTime, diff --git a/app/portainer/environments/types.ts b/app/portainer/environments/types.ts index c3b13f96d..3c6ec897d 100644 --- a/app/portainer/environments/types.ts +++ b/app/portainer/environments/types.ts @@ -20,6 +20,11 @@ export enum EnvironmentType { EdgeAgentOnKubernetes, } +export const EdgeTypes = [ + EnvironmentType.EdgeAgentOnDocker, + EnvironmentType.EdgeAgentOnKubernetes, +] as const; + export enum EnvironmentStatus { Up = 1, Down, diff --git a/app/portainer/home/EnvironmentList/EnvironmentList.tsx b/app/portainer/home/EnvironmentList/EnvironmentList.tsx index 55964fb56..2cf6e9856 100644 --- a/app/portainer/home/EnvironmentList/EnvironmentList.tsx +++ b/app/portainer/home/EnvironmentList/EnvironmentList.tsx @@ -133,7 +133,8 @@ export function EnvironmentList({ onClickItem, onRefresh }: Props) { groupIds: groupFilter, sort: sortByFilter, order: sortByDescending ? 'desc' : 'asc', - edgeDeviceFilter: 'none', + provisioned: true, + edgeDevice: false, tagsPartialMatch: true, }, refetchIfAnyOffline @@ -312,7 +313,7 @@ export function EnvironmentList({ onClickItem, onRefresh }: Props) { groupIds: groupFilter, sort: sortByFilter, order: sortByDescending ? 'desc' : 'asc', - edgeDeviceFilter: 'none', + edgeDevice: false, }} /> diff --git a/app/portainer/home/EnvironmentList/KubeconfigButton/KubeconfigButton.tsx b/app/portainer/home/EnvironmentList/KubeconfigButton/KubeconfigButton.tsx index eaaae6275..2c1332ca3 100644 --- a/app/portainer/home/EnvironmentList/KubeconfigButton/KubeconfigButton.tsx +++ b/app/portainer/home/EnvironmentList/KubeconfigButton/KubeconfigButton.tsx @@ -2,7 +2,7 @@ import { useState } from 'react'; import { Download } from 'react-feather'; import { Environment } from '@/portainer/environments/types'; -import { EnvironmentsQueryParams } from '@/portainer/environments/environment.service/index'; +import { Query } from '@/portainer/environments/queries/useEnvironmentList'; import { isKubernetesEnvironment } from '@/portainer/environments/utils'; import { trackEvent } from '@/angulartics.matomo/analytics-services'; @@ -14,7 +14,7 @@ import '@reach/dialog/styles.css'; export interface Props { environments: Environment[]; - envQueryParams: EnvironmentsQueryParams; + envQueryParams: Query; } export function KubeconfigButton({ environments, envQueryParams }: Props) { const [isOpen, setIsOpen] = useState(false); diff --git a/app/portainer/rbac/components/access-viewer/access-viewer.controller.js b/app/portainer/rbac/components/access-viewer/access-viewer.controller.js index 95b48e7ff..7fc9e9f56 100644 --- a/app/portainer/rbac/components/access-viewer/access-viewer.controller.js +++ b/app/portainer/rbac/components/access-viewer/access-viewer.controller.js @@ -1,15 +1,15 @@ import _ from 'lodash-es'; import { isLimitedToBE } from '@/portainer/feature-flags/feature-flags.service'; +import { getEnvironments } from '@/portainer/environments/environment.service'; import AccessViewerPolicyModel from '../../models/access'; export default class AccessViewerController { /* @ngInject */ - constructor(Notifications, RoleService, UserService, EndpointService, GroupService, TeamService, TeamMembershipService, Authentication) { + constructor(Notifications, RoleService, UserService, GroupService, TeamService, TeamMembershipService, Authentication) { this.Notifications = Notifications; this.RoleService = RoleService; this.UserService = UserService; - this.EndpointService = EndpointService; this.GroupService = GroupService; this.TeamService = TeamService; this.TeamMembershipService = TeamMembershipService; @@ -138,7 +138,7 @@ export default class AccessViewerController { this.isAdmin = this.Authentication.isAdmin(); this.allUsers = await this.UserService.users(); - this.endpoints = _.keyBy((await this.EndpointService.endpoints()).value, 'Id'); + this.endpoints = _.keyBy((await getEnvironments()).value, 'Id'); const groups = await this.GroupService.groups(); this.groupUserAccessPolicies = {}; this.groupTeamAccessPolicies = {}; diff --git a/app/portainer/services/api/endpointService.js b/app/portainer/services/api/endpointService.js index 5f08fdadf..22b1b45a4 100644 --- a/app/portainer/services/api/endpointService.js +++ b/app/portainer/services/api/endpointService.js @@ -16,14 +16,6 @@ angular.module('portainer.app').factory('EndpointService', [ return Endpoints.get({ id: endpointID }).$promise; }; - service.endpoints = function (start, limit, { search, types, tagIds, endpointIds, tagsPartialMatch } = {}) { - if (tagIds && !tagIds.length) { - return Promise.resolve({ value: [], totalCount: 0 }); - } - return Endpoints.query({ start, limit, search, types: JSON.stringify(types), tagIds: JSON.stringify(tagIds), endpointIds: JSON.stringify(endpointIds), tagsPartialMatch }) - .$promise; - }; - service.snapshotEndpoints = function () { return Endpoints.snapshots({}, {}).$promise; }; @@ -32,10 +24,6 @@ angular.module('portainer.app').factory('EndpointService', [ return Endpoints.snapshot({ id: endpointID }, {}).$promise; }; - service.endpointsByGroup = function (start, limit, search, groupId) { - return Endpoints.query({ start, limit, search, groupId }).$promise; - }; - service.updateAccess = function (id, userAccessPolicies, teamAccessPolicies) { return Endpoints.updateAccess({ id: id }, { UserAccessPolicies: userAccessPolicies, TeamAccessPolicies: teamAccessPolicies }).$promise; }; diff --git a/app/portainer/services/nameValidator.js b/app/portainer/services/nameValidator.js index 71dd94be4..fa4f3fe2e 100644 --- a/app/portainer/services/nameValidator.js +++ b/app/portainer/services/nameValidator.js @@ -1,18 +1,17 @@ import angular from 'angular'; +import { getEnvironments } from '../environments/environment.service'; angular.module('portainer.app').factory('NameValidator', NameValidatorFactory); /* @ngInject */ -function NameValidatorFactory(EndpointService, Notifications) { +function NameValidatorFactory(Notifications) { return { validateEnvironmentName, }; - async function validateEnvironmentName(environmentName) { + async function validateEnvironmentName(name) { try { - const endpoints = await EndpointService.endpoints(); - const endpointArray = endpoints.value; - const nameDuplicated = endpointArray.filter((item) => item.Name === environmentName); - return nameDuplicated.length > 0; + const endpoints = await getEnvironments({ limit: 1, name }); + return endpoints.value.length > 0; } catch (err) { Notifications.error('Failure', err, 'Unable to retrieve environment details'); } diff --git a/app/portainer/views/auth/authController.js b/app/portainer/views/auth/authController.js index 03e8e3a02..fd291f4c3 100644 --- a/app/portainer/views/auth/authController.js +++ b/app/portainer/views/auth/authController.js @@ -1,5 +1,6 @@ import angular from 'angular'; import uuidv4 from 'uuid/v4'; +import { getEnvironments } from '@/portainer/environments/environment.service'; class AuthenticationController { /* @ngInject */ @@ -12,7 +13,6 @@ class AuthenticationController { $window, Authentication, UserService, - EndpointService, StateManager, Notifications, SettingsService, @@ -28,7 +28,6 @@ class AuthenticationController { this.$window = $window; this.Authentication = Authentication; this.UserService = UserService; - this.EndpointService = EndpointService; this.StateManager = StateManager; this.Notifications = Notifications; this.SettingsService = SettingsService; @@ -119,8 +118,8 @@ class AuthenticationController { async checkForEndpointsAsync() { try { - const endpoints = await this.EndpointService.endpoints(0, 1); const isAdmin = this.Authentication.isAdmin(); + const endpoints = await getEnvironments({ limit: 1 }); if (this.Authentication.getUserDetails().forceChangePassword) { return this.$state.go('portainer.account'); diff --git a/app/portainer/views/endpoints/endpointsController.js b/app/portainer/views/endpoints/endpointsController.js index d71346954..4f6303667 100644 --- a/app/portainer/views/endpoints/endpointsController.js +++ b/app/portainer/views/endpoints/endpointsController.js @@ -1,5 +1,6 @@ import angular from 'angular'; -import EndpointHelper from 'Portainer/helpers/endpointHelper'; +import EndpointHelper from '@/portainer/helpers/endpointHelper'; +import { getEnvironments } from '@/portainer/environments/environment.service'; angular.module('portainer.app').controller('EndpointsController', EndpointsController); @@ -46,10 +47,10 @@ function EndpointsController($q, $scope, $state, $async, EndpointService, GroupS } $scope.getPaginatedEndpoints = getPaginatedEndpoints; - function getPaginatedEndpoints(lastId, limit, search) { + function getPaginatedEndpoints(start, limit, search) { const deferred = $q.defer(); $q.all({ - endpoints: EndpointService.endpoints(lastId, limit, { search }), + endpoints: getEnvironments({ start, limit, query: { search } }), groups: GroupService.groups(), }) .then(function success(data) { diff --git a/app/portainer/views/init/admin/initAdminController.js b/app/portainer/views/init/admin/initAdminController.js index 833be5714..f7c05c184 100644 --- a/app/portainer/views/init/admin/initAdminController.js +++ b/app/portainer/views/init/admin/initAdminController.js @@ -1,3 +1,5 @@ +import { getEnvironments } from '@/portainer/environments/environment.service'; + angular.module('portainer.app').controller('InitAdminController', [ '$scope', '$state', @@ -6,10 +8,9 @@ angular.module('portainer.app').controller('InitAdminController', [ 'StateManager', 'SettingsService', 'UserService', - 'EndpointService', 'BackupService', 'StatusService', - function ($scope, $state, Notifications, Authentication, StateManager, SettingsService, UserService, EndpointService, BackupService, StatusService) { + function ($scope, $state, Notifications, Authentication, StateManager, SettingsService, UserService, BackupService, StatusService) { $scope.uploadBackup = uploadBackup; $scope.logo = StateManager.getState().application.logo; @@ -50,7 +51,7 @@ angular.module('portainer.app').controller('InitAdminController', [ return StateManager.initialize(); }) .then(function () { - return EndpointService.endpoints(0, 100); + return getEnvironments({ limit: 100 }); }) .then(function success(data) { if (data.value.length === 0) { diff --git a/app/portainer/views/stacks/edit/stackController.js b/app/portainer/views/stacks/edit/stackController.js index 2e86b6b1d..dbceefff7 100644 --- a/app/portainer/views/stacks/edit/stackController.js +++ b/app/portainer/views/stacks/edit/stackController.js @@ -1,6 +1,7 @@ import { ResourceControlType } from '@/portainer/access-control/types'; import { AccessControlFormData } from 'Portainer/components/accessControlForm/porAccessControlFormModel'; import { FeatureId } from 'Portainer/feature-flags/enums'; +import { getEnvironments } from '@/portainer/environments/environment.service'; angular.module('portainer.app').controller('StackController', [ '$async', @@ -20,7 +21,6 @@ angular.module('portainer.app').controller('StackController', [ 'Notifications', 'FormHelper', 'EndpointProvider', - 'EndpointService', 'GroupService', 'ModalService', 'StackHelper', @@ -46,7 +46,6 @@ angular.module('portainer.app').controller('StackController', [ Notifications, FormHelper, EndpointProvider, - EndpointService, GroupService, ModalService, StackHelper, @@ -317,60 +316,62 @@ angular.module('portainer.app').controller('StackController', [ } function loadStack(id) { - var agentProxy = $scope.applicationState.endpoint.mode.agentProxy; + return $async(() => { + var agentProxy = $scope.applicationState.endpoint.mode.agentProxy; - EndpointService.endpoints() - .then(function success(data) { - $scope.endpoints = data.value; - }) - .catch(function error(err) { - Notifications.error('Failure', err, 'Unable to retrieve environments'); - }); - - $q.all({ - stack: StackService.stack(id), - groups: GroupService.groups(), - containers: ContainerService.containers(true), - }) - .then(function success(data) { - var stack = data.stack; - $scope.groups = data.groups; - $scope.stack = stack; - $scope.containerNames = ContainerHelper.getContainerNames(data.containers); - - $scope.formValues.Env = $scope.stack.Env; - - let resourcesPromise = Promise.resolve({}); - if (!stack.Status || stack.Status === 1) { - resourcesPromise = stack.Type === 1 ? retrieveSwarmStackResources(stack.Name, agentProxy) : retrieveComposeStackResources(stack.Name); - } - - return $q.all({ - stackFile: StackService.getStackFile(id), - resources: resourcesPromise, + getEnvironments() + .then(function success(data) { + $scope.endpoints = data.value; + }) + .catch(function error(err) { + Notifications.error('Failure', err, 'Unable to retrieve environments'); }); - }) - .then(function success(data) { - const isSwarm = $scope.stack.Type === 1; - $scope.stackFileContent = data.stackFile; - // workaround for missing status, if stack has resources, set the status to 1 (active), otherwise to 2 (inactive) (https://github.com/portainer/portainer/issues/4422) - if (!$scope.stack.Status) { - $scope.stack.Status = data.resources && ((isSwarm && data.resources.services.length) || data.resources.containers.length) ? 1 : 2; - } - if ($scope.stack.Status === 1) { - if (isSwarm) { - assignSwarmStackResources(data.resources, agentProxy); - } else { - assignComposeStackResources(data.resources); + $q.all({ + stack: StackService.stack(id), + groups: GroupService.groups(), + containers: ContainerService.containers(true), + }) + .then(function success(data) { + var stack = data.stack; + $scope.groups = data.groups; + $scope.stack = stack; + $scope.containerNames = ContainerHelper.getContainerNames(data.containers); + + $scope.formValues.Env = $scope.stack.Env; + + let resourcesPromise = Promise.resolve({}); + if (!stack.Status || stack.Status === 1) { + resourcesPromise = stack.Type === 1 ? retrieveSwarmStackResources(stack.Name, agentProxy) : retrieveComposeStackResources(stack.Name); } - } - $scope.state.yamlError = StackHelper.validateYAML($scope.stackFileContent, $scope.containerNames); - }) - .catch(function error(err) { - Notifications.error('Failure', err, 'Unable to retrieve stack details'); - }); + return $q.all({ + stackFile: StackService.getStackFile(id), + resources: resourcesPromise, + }); + }) + .then(function success(data) { + const isSwarm = $scope.stack.Type === 1; + $scope.stackFileContent = data.stackFile; + // workaround for missing status, if stack has resources, set the status to 1 (active), otherwise to 2 (inactive) (https://github.com/portainer/portainer/issues/4422) + if (!$scope.stack.Status) { + $scope.stack.Status = data.resources && ((isSwarm && data.resources.services.length) || data.resources.containers.length) ? 1 : 2; + } + + if ($scope.stack.Status === 1) { + if (isSwarm) { + assignSwarmStackResources(data.resources, agentProxy); + } else { + assignComposeStackResources(data.resources); + } + } + + $scope.state.yamlError = StackHelper.validateYAML($scope.stackFileContent, $scope.containerNames); + }) + .catch(function error(err) { + Notifications.error('Failure', err, 'Unable to retrieve stack details'); + }); + }); } function retrieveSwarmStackResources(stackName, agentProxy) { diff --git a/app/react/portainer/environments/wizard/EnvironmentsCreationView/shared/NameField.tsx b/app/react/portainer/environments/wizard/EnvironmentsCreationView/shared/NameField.tsx index e02ed78c4..78fa82e36 100644 --- a/app/react/portainer/environments/wizard/EnvironmentsCreationView/shared/NameField.tsx +++ b/app/react/portainer/environments/wizard/EnvironmentsCreationView/shared/NameField.tsx @@ -2,7 +2,7 @@ import { Field, useField } from 'formik'; import { string } from 'yup'; import { debounce } from 'lodash'; -import { getEndpoints } from '@/portainer/environments/environment.service'; +import { getEnvironments } from '@/portainer/environments/environment.service'; import { FormControl } from '@@/form-components/FormControl'; import { Input } from '@@/form-components/Input'; @@ -30,13 +30,13 @@ export function NameField({ readonly }: Props) { ); } -async function isNameUnique(name?: string) { +export async function isNameUnique(name?: string) { if (!name) { return true; } try { - const result = await getEndpoints(0, 1, { name }); + const result = await getEnvironments({ limit: 1, query: { name } }); if (result.totalCount > 0) { return false; }