From 1473cc208b5d8f8f0f471824a0c08948b77b3d80 Mon Sep 17 00:00:00 2001 From: andres-portainer <91705312+andres-portainer@users.noreply.github.com> Date: Tue, 16 May 2023 16:07:03 -0300 Subject: [PATCH] feat(edgegroups): add support for transactions EE-5323 (#8946) --- .../edgegroups/associated_endpoints.go | 14 ++++-- .../handler/edgegroups/edgegroup_delete.go | 39 ++++++++++++---- .../handler/edgegroups/edgegroup_inspect.go | 31 +++++++++---- api/http/handler/edgegroups/edgegroup_list.go | 46 ++++++++++++------- .../handler/edgegroups/edgegroup_list_test.go | 4 +- api/http/handler/edgegroups/handler.go | 1 + 6 files changed, 98 insertions(+), 37 deletions(-) diff --git a/api/http/handler/edgegroups/associated_endpoints.go b/api/http/handler/edgegroups/associated_endpoints.go index 6c240083c..ea2ea111b 100644 --- a/api/http/handler/edgegroups/associated_endpoints.go +++ b/api/http/handler/edgegroups/associated_endpoints.go @@ -2,16 +2,17 @@ package edgegroups import ( portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" ) type endpointSetType map[portainer.EndpointID]bool -func (handler *Handler) getEndpointsByTags(tagIDs []portainer.TagID, partialMatch bool) ([]portainer.EndpointID, error) { +func getEndpointsByTags(tx dataservices.DataStoreTx, tagIDs []portainer.TagID, partialMatch bool) ([]portainer.EndpointID, error) { if len(tagIDs) == 0 { return []portainer.EndpointID{}, nil } - endpoints, err := handler.DataStore.Endpoint().Endpoints() + endpoints, err := tx.Endpoint().Endpoints() if err != nil { return nil, err } @@ -20,10 +21,11 @@ func (handler *Handler) getEndpointsByTags(tagIDs []portainer.TagID, partialMatc tags := []portainer.Tag{} for _, tagID := range tagIDs { - tag, err := handler.DataStore.Tag().Tag(tagID) + tag, err := tx.Tag().Tag(tagID) if err != nil { return nil, err } + tags = append(tags, *tag) } @@ -48,25 +50,31 @@ func (handler *Handler) getEndpointsByTags(tagIDs []portainer.TagID, partialMatc func mapEndpointGroupToEndpoints(endpoints []portainer.Endpoint) map[portainer.EndpointGroupID]endpointSetType { groupEndpoints := map[portainer.EndpointGroupID]endpointSetType{} + for _, endpoint := range endpoints { groupID := endpoint.GroupID if groupEndpoints[groupID] == nil { groupEndpoints[groupID] = endpointSetType{} } + groupEndpoints[groupID][endpoint.ID] = true } + return groupEndpoints } func mapTagsToEndpoints(tags []portainer.Tag, groupEndpoints map[portainer.EndpointGroupID]endpointSetType) []endpointSetType { sets := []endpointSetType{} + for _, tag := range tags { set := tag.Endpoints + for groupID := range tag.EndpointGroups { for endpointID := range groupEndpoints[groupID] { set[endpointID] = true } } + sets = append(sets, set) } diff --git a/api/http/handler/edgegroups/edgegroup_delete.go b/api/http/handler/edgegroups/edgegroup_delete.go index 9ec4b3b26..f0c18d964 100644 --- a/api/http/handler/edgegroups/edgegroup_delete.go +++ b/api/http/handler/edgegroups/edgegroup_delete.go @@ -8,6 +8,8 @@ import ( "github.com/portainer/libhttp/request" "github.com/portainer/libhttp/response" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" + "github.com/portainer/portainer/pkg/featureflags" ) // @id EdgeGroupDelete @@ -27,43 +29,64 @@ func (handler *Handler) edgeGroupDelete(w http.ResponseWriter, r *http.Request) return httperror.BadRequest("Invalid Edge group identifier route variable", err) } - _, err = handler.DataStore.EdgeGroup().EdgeGroup(portainer.EdgeGroupID(edgeGroupID)) - if handler.DataStore.IsErrObjectNotFound(err) { + if featureflags.IsEnabled(portainer.FeatureNoTx) { + err = deleteEdgeGroup(handler.DataStore, portainer.EdgeGroupID(edgeGroupID)) + } else { + err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error { + return deleteEdgeGroup(tx, portainer.EdgeGroupID(edgeGroupID)) + }) + } + + if err != nil { + var httpErr *httperror.HandlerError + if errors.As(err, &httpErr) { + return httpErr + } + + return httperror.InternalServerError("Unexpected error", err) + } + + return response.Empty(w) +} + +func deleteEdgeGroup(tx dataservices.DataStoreTx, ID portainer.EdgeGroupID) error { + _, err := tx.EdgeGroup().EdgeGroup(ID) + if tx.IsErrObjectNotFound(err) { return httperror.NotFound("Unable to find an Edge group with the specified identifier inside the database", err) } else if err != nil { return httperror.InternalServerError("Unable to find an Edge group with the specified identifier inside the database", err) } - edgeStacks, err := handler.DataStore.EdgeStack().EdgeStacks() + edgeStacks, err := tx.EdgeStack().EdgeStacks() if err != nil { return httperror.InternalServerError("Unable to retrieve Edge stacks from the database", err) } for _, edgeStack := range edgeStacks { for _, groupID := range edgeStack.EdgeGroups { - if groupID == portainer.EdgeGroupID(edgeGroupID) { + if groupID == ID { return httperror.NewError(http.StatusConflict, "Edge group is used by an Edge stack", errors.New("edge group is used by an Edge stack")) } } } - edgeJobs, err := handler.DataStore.EdgeJob().EdgeJobs() + edgeJobs, err := tx.EdgeJob().EdgeJobs() if err != nil { return httperror.InternalServerError("Unable to retrieve Edge jobs from the database", err) } for _, edgeJob := range edgeJobs { for _, groupID := range edgeJob.EdgeGroups { - if groupID == portainer.EdgeGroupID(edgeGroupID) { + if groupID == ID { return httperror.NewError(http.StatusConflict, "Edge group is used by an Edge job", errors.New("edge group is used by an Edge job")) } } } - err = handler.DataStore.EdgeGroup().DeleteEdgeGroup(portainer.EdgeGroupID(edgeGroupID)) + err = tx.EdgeGroup().DeleteEdgeGroup(ID) if err != nil { return httperror.InternalServerError("Unable to remove the Edge group from the database", err) } - return response.Empty(w) + return nil } diff --git a/api/http/handler/edgegroups/edgegroup_inspect.go b/api/http/handler/edgegroups/edgegroup_inspect.go index 780313114..fb611573f 100644 --- a/api/http/handler/edgegroups/edgegroup_inspect.go +++ b/api/http/handler/edgegroups/edgegroup_inspect.go @@ -5,8 +5,9 @@ import ( httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" - "github.com/portainer/libhttp/response" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" + "github.com/portainer/portainer/pkg/featureflags" ) // @id EdgeGroupInspect @@ -27,21 +28,35 @@ func (handler *Handler) edgeGroupInspect(w http.ResponseWriter, r *http.Request) return httperror.BadRequest("Invalid Edge group identifier route variable", err) } - edgeGroup, err := handler.DataStore.EdgeGroup().EdgeGroup(portainer.EdgeGroupID(edgeGroupID)) - if handler.DataStore.IsErrObjectNotFound(err) { - return httperror.NotFound("Unable to find an Edge group with the specified identifier inside the database", err) + var edgeGroup *portainer.EdgeGroup + if featureflags.IsEnabled(portainer.FeatureNoTx) { + edgeGroup, err = getEdgeGroup(handler.DataStore, portainer.EdgeGroupID(edgeGroupID)) + } else { + err = handler.DataStore.ViewTx(func(tx dataservices.DataStoreTx) error { + edgeGroup, err = getEdgeGroup(tx, portainer.EdgeGroupID(edgeGroupID)) + return err + }) + } + + return txResponse(w, edgeGroup, err) +} + +func getEdgeGroup(tx dataservices.DataStoreTx, ID portainer.EdgeGroupID) (*portainer.EdgeGroup, error) { + edgeGroup, err := tx.EdgeGroup().EdgeGroup(ID) + if tx.IsErrObjectNotFound(err) { + return nil, httperror.NotFound("Unable to find an Edge group with the specified identifier inside the database", err) } else if err != nil { - return httperror.InternalServerError("Unable to find an Edge group with the specified identifier inside the database", err) + return nil, httperror.InternalServerError("Unable to find an Edge group with the specified identifier inside the database", err) } if edgeGroup.Dynamic { - endpoints, err := handler.getEndpointsByTags(edgeGroup.TagIDs, edgeGroup.PartialMatch) + endpoints, err := getEndpointsByTags(tx, edgeGroup.TagIDs, edgeGroup.PartialMatch) if err != nil { - return httperror.InternalServerError("Unable to retrieve environments and environment groups for Edge group", err) + return nil, httperror.InternalServerError("Unable to retrieve environments and environment groups for Edge group", err) } edgeGroup.Endpoints = endpoints } - return response.JSON(w, edgeGroup) + return edgeGroup, err } diff --git a/api/http/handler/edgegroups/edgegroup_list.go b/api/http/handler/edgegroups/edgegroup_list.go index 33df22ffb..8fa372968 100644 --- a/api/http/handler/edgegroups/edgegroup_list.go +++ b/api/http/handler/edgegroups/edgegroup_list.go @@ -5,10 +5,10 @@ import ( "net/http" httperror "github.com/portainer/libhttp/error" - "github.com/portainer/libhttp/response" portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/dataservices" "github.com/portainer/portainer/api/internal/slices" + "github.com/portainer/portainer/pkg/featureflags" ) type decoratedEdgeGroup struct { @@ -30,14 +30,30 @@ type decoratedEdgeGroup struct { // @failure 503 "Edge compute features are disabled" // @router /edge_groups [get] func (handler *Handler) edgeGroupList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { - edgeGroups, err := handler.DataStore.EdgeGroup().EdgeGroups() + var decoratedEdgeGroups []decoratedEdgeGroup + var err error + + if featureflags.IsEnabled(portainer.FeatureNoTx) { + decoratedEdgeGroups, err = getEdgeGroupList(handler.DataStore) + } else { + err = handler.DataStore.ViewTx(func(tx dataservices.DataStoreTx) error { + decoratedEdgeGroups, err = getEdgeGroupList(tx) + return err + }) + } + + return txResponse(w, decoratedEdgeGroups, err) +} + +func getEdgeGroupList(tx dataservices.DataStoreTx) ([]decoratedEdgeGroup, error) { + edgeGroups, err := tx.EdgeGroup().EdgeGroups() if err != nil { - return httperror.InternalServerError("Unable to retrieve Edge groups from the database", err) + return nil, httperror.InternalServerError("Unable to retrieve Edge groups from the database", err) } - edgeStacks, err := handler.DataStore.EdgeStack().EdgeStacks() + edgeStacks, err := tx.EdgeStack().EdgeStacks() if err != nil { - return httperror.InternalServerError("Unable to retrieve Edge stacks from the database", err) + return nil, httperror.InternalServerError("Unable to retrieve Edge stacks from the database", err) } usedEdgeGroups := make(map[portainer.EdgeGroupID]bool) @@ -48,9 +64,9 @@ func (handler *Handler) edgeGroupList(w http.ResponseWriter, r *http.Request) *h } } - edgeJobs, err := handler.DataStore.EdgeJob().EdgeJobs() + edgeJobs, err := tx.EdgeJob().EdgeJobs() if err != nil { - return httperror.InternalServerError("Unable to retrieve Edge jobs from the database", err) + return nil, httperror.InternalServerError("Unable to retrieve Edge jobs from the database", err) } decoratedEdgeGroups := []decoratedEdgeGroup{} @@ -68,35 +84,33 @@ func (handler *Handler) edgeGroupList(w http.ResponseWriter, r *http.Request) *h EndpointTypes: []portainer.EndpointType{}, } if edgeGroup.Dynamic { - endpointIDs, err := handler.getEndpointsByTags(edgeGroup.TagIDs, edgeGroup.PartialMatch) + endpointIDs, err := getEndpointsByTags(tx, edgeGroup.TagIDs, edgeGroup.PartialMatch) if err != nil { - return httperror.InternalServerError("Unable to retrieve environments and environment groups for Edge group", err) + return nil, httperror.InternalServerError("Unable to retrieve environments and environment groups for Edge group", err) } edgeGroup.Endpoints = endpointIDs } - endpointTypes, err := getEndpointTypes(handler.DataStore.Endpoint(), edgeGroup.Endpoints) + endpointTypes, err := getEndpointTypes(tx, edgeGroup.Endpoints) if err != nil { - return httperror.InternalServerError("Unable to retrieve environment types for Edge group", err) + return nil, httperror.InternalServerError("Unable to retrieve environment types for Edge group", err) } edgeGroup.EndpointTypes = endpointTypes - edgeGroup.HasEdgeStack = usedEdgeGroups[edgeGroup.ID] - edgeGroup.HasEdgeGroup = usedByEdgeJob decoratedEdgeGroups = append(decoratedEdgeGroups, edgeGroup) } - return response.JSON(w, decoratedEdgeGroups) + return decoratedEdgeGroups, nil } -func getEndpointTypes(endpointService dataservices.EndpointService, endpointIds []portainer.EndpointID) ([]portainer.EndpointType, error) { +func getEndpointTypes(tx dataservices.DataStoreTx, endpointIds []portainer.EndpointID) ([]portainer.EndpointType, error) { typeSet := map[portainer.EndpointType]bool{} for _, endpointID := range endpointIds { - endpoint, err := endpointService.Endpoint(endpointID) + endpoint, err := tx.Endpoint().Endpoint(endpointID) if err != nil { return nil, fmt.Errorf("failed fetching environment: %w", err) } diff --git a/api/http/handler/edgegroups/edgegroup_list_test.go b/api/http/handler/edgegroups/edgegroup_list_test.go index 36816fea9..b77b2966e 100644 --- a/api/http/handler/edgegroups/edgegroup_list_test.go +++ b/api/http/handler/edgegroups/edgegroup_list_test.go @@ -38,7 +38,7 @@ func Test_getEndpointTypes(t *testing.T) { } for _, test := range tests { - ans, err := getEndpointTypes(datastore.Endpoint(), test.endpointIds) + ans, err := getEndpointTypes(datastore, test.endpointIds) assert.NoError(t, err, "getEndpointTypes shouldn't fail") assert.ElementsMatch(t, test.expected, ans, "getEndpointTypes expected to return %b for %v, but returned %b", test.expected, test.endpointIds, ans) @@ -48,6 +48,6 @@ func Test_getEndpointTypes(t *testing.T) { func Test_getEndpointTypes_failWhenEndpointDontExist(t *testing.T) { datastore := testhelpers.NewDatastore(testhelpers.WithEndpoints([]portainer.Endpoint{})) - _, err := getEndpointTypes(datastore.Endpoint(), []portainer.EndpointID{1}) + _, err := getEndpointTypes(datastore, []portainer.EndpointID{1}) assert.Error(t, err, "getEndpointTypes should fail") } diff --git a/api/http/handler/edgegroups/handler.go b/api/http/handler/edgegroups/handler.go index 8f9bcc44f..8d3870249 100644 --- a/api/http/handler/edgegroups/handler.go +++ b/api/http/handler/edgegroups/handler.go @@ -35,6 +35,7 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler { bouncer.AdminAccess(bouncer.EdgeComputeOperation(httperror.LoggerHandler(h.edgeGroupUpdate)))).Methods(http.MethodPut) h.Handle("/edge_groups/{id}", bouncer.AdminAccess(bouncer.EdgeComputeOperation(httperror.LoggerHandler(h.edgeGroupDelete)))).Methods(http.MethodDelete) + return h }