From e82c88317e7fd2cb77973d0b9c395bca97130b90 Mon Sep 17 00:00:00 2001 From: andres-portainer <91705312+andres-portainer@users.noreply.github.com> Date: Fri, 5 May 2023 20:39:22 -0300 Subject: [PATCH] feat(edgestacks): add support for transactions EE-5326 (#8908) --- .../handler/edgestacks/edgestack_create.go | 22 +++-- .../edgestacks/edgestack_create_file.go | 12 +-- .../edgestacks/edgestack_create_git.go | 18 ++-- .../edgestacks/edgestack_create_string.go | 19 +++-- .../handler/edgestacks/edgestack_delete.go | 30 ++++++- .../edgestacks/edgestack_status_delete.go | 41 +++++++-- .../edgestacks/edgestack_status_update.go | 84 +++++++++++++++---- .../handler/edgestacks/edgestack_update.go | 84 ++++++++++++------- api/internal/edge/edgestack.go | 10 +-- api/internal/edge/edgestacks/service.go | 36 ++++---- api/internal/edge/edgestacks/service_test.go | 2 +- 11 files changed, 254 insertions(+), 104 deletions(-) diff --git a/api/http/handler/edgestacks/edgestack_create.go b/api/http/handler/edgestacks/edgestack_create.go index 43a2053a2..e64b39a96 100644 --- a/api/http/handler/edgestacks/edgestack_create.go +++ b/api/http/handler/edgestacks/edgestack_create.go @@ -7,8 +7,10 @@ import ( "github.com/portainer/libhttp/request" "github.com/portainer/libhttp/response" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" httperrors "github.com/portainer/portainer/api/http/errors" "github.com/portainer/portainer/api/http/security" + "github.com/portainer/portainer/pkg/featureflags" ) func (handler *Handler) edgeStackCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { @@ -23,7 +25,15 @@ func (handler *Handler) edgeStackCreate(w http.ResponseWriter, r *http.Request) return httperror.InternalServerError("Unable to retrieve user details from authentication token", err) } - edgeStack, err := handler.createSwarmStack(method, dryrun, tokenData.ID, r) + var edgeStack *portainer.EdgeStack + if featureflags.IsEnabled(portainer.FeatureNoTx) { + edgeStack, err = handler.createSwarmStack(handler.DataStore, method, dryrun, tokenData.ID, r) + } else { + err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error { + edgeStack, err = handler.createSwarmStack(tx, method, dryrun, tokenData.ID, r) + return err + }) + } if err != nil { switch { case httperrors.IsInvalidPayloadError(err): @@ -36,15 +46,15 @@ func (handler *Handler) edgeStackCreate(w http.ResponseWriter, r *http.Request) return response.JSON(w, edgeStack) } -func (handler *Handler) createSwarmStack(method string, dryrun bool, userID portainer.UserID, r *http.Request) (*portainer.EdgeStack, error) { - +func (handler *Handler) createSwarmStack(tx dataservices.DataStoreTx, method string, dryrun bool, userID portainer.UserID, r *http.Request) (*portainer.EdgeStack, error) { switch method { case "string": - return handler.createEdgeStackFromFileContent(r, dryrun) + return handler.createEdgeStackFromFileContent(r, tx, dryrun) case "repository": - return handler.createEdgeStackFromGitRepository(r, dryrun, userID) + return handler.createEdgeStackFromGitRepository(r, tx, dryrun, userID) case "file": - return handler.createEdgeStackFromFileUpload(r, dryrun) + return handler.createEdgeStackFromFileUpload(r, tx, dryrun) } + return nil, httperrors.NewInvalidPayloadError("Invalid value for query parameter: method. Value must be one of: string, repository or file") } diff --git a/api/http/handler/edgestacks/edgestack_create_file.go b/api/http/handler/edgestacks/edgestack_create_file.go index 5b669226a..f6853e3c6 100644 --- a/api/http/handler/edgestacks/edgestack_create_file.go +++ b/api/http/handler/edgestacks/edgestack_create_file.go @@ -3,10 +3,12 @@ package edgestacks import ( "net/http" - "github.com/pkg/errors" "github.com/portainer/libhttp/request" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" httperrors "github.com/portainer/portainer/api/http/errors" + + "github.com/pkg/errors" ) type edgeStackFromFileUploadPayload struct { @@ -87,14 +89,14 @@ func (payload *edgeStackFromFileUploadPayload) Validate(r *http.Request) error { // @failure 500 "Internal server error" // @failure 503 "Edge compute features are disabled" // @router /edge_stacks/create/file [post] -func (handler *Handler) createEdgeStackFromFileUpload(r *http.Request, dryrun bool) (*portainer.EdgeStack, error) { +func (handler *Handler) createEdgeStackFromFileUpload(r *http.Request, tx dataservices.DataStoreTx, dryrun bool) (*portainer.EdgeStack, error) { payload := &edgeStackFromFileUploadPayload{} err := payload.Validate(r) if err != nil { return nil, err } - stack, err := handler.edgeStacksService.BuildEdgeStack(payload.Name, payload.DeploymentType, payload.EdgeGroups, payload.Registries, payload.UseManifestNamespaces) + stack, err := handler.edgeStacksService.BuildEdgeStack(tx, payload.Name, payload.DeploymentType, payload.EdgeGroups, payload.Registries, payload.UseManifestNamespaces) if err != nil { return nil, errors.Wrap(err, "failed to create edge stack object") } @@ -103,7 +105,7 @@ func (handler *Handler) createEdgeStackFromFileUpload(r *http.Request, dryrun bo return stack, nil } - return handler.edgeStacksService.PersistEdgeStack(stack, func(stackFolder string, relatedEndpointIds []portainer.EndpointID) (composePath string, manifestPath string, projectPath string, err error) { - return handler.storeFileContent(stackFolder, payload.DeploymentType, relatedEndpointIds, payload.StackFileContent) + return handler.edgeStacksService.PersistEdgeStack(tx, stack, func(stackFolder string, relatedEndpointIds []portainer.EndpointID) (composePath string, manifestPath string, projectPath string, err error) { + return handler.storeFileContent(tx, stackFolder, payload.DeploymentType, relatedEndpointIds, payload.StackFileContent) }) } diff --git a/api/http/handler/edgestacks/edgestack_create_git.go b/api/http/handler/edgestacks/edgestack_create_git.go index a8a4df300..43373a117 100644 --- a/api/http/handler/edgestacks/edgestack_create_git.go +++ b/api/http/handler/edgestacks/edgestack_create_git.go @@ -4,13 +4,15 @@ import ( "fmt" "net/http" - "github.com/asaskevich/govalidator" - "github.com/pkg/errors" "github.com/portainer/libhttp/request" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" "github.com/portainer/portainer/api/filesystem" gittypes "github.com/portainer/portainer/api/git/types" httperrors "github.com/portainer/portainer/api/http/errors" + + "github.com/asaskevich/govalidator" + "github.com/pkg/errors" ) type edgeStackFromGitRepositoryPayload struct { @@ -91,14 +93,14 @@ func (payload *edgeStackFromGitRepositoryPayload) Validate(r *http.Request) erro // @failure 500 "Internal server error" // @failure 503 "Edge compute features are disabled" // @router /edge_stacks/create/repository [post] -func (handler *Handler) createEdgeStackFromGitRepository(r *http.Request, dryrun bool, userID portainer.UserID) (*portainer.EdgeStack, error) { +func (handler *Handler) createEdgeStackFromGitRepository(r *http.Request, tx dataservices.DataStoreTx, dryrun bool, userID portainer.UserID) (*portainer.EdgeStack, error) { var payload edgeStackFromGitRepositoryPayload err := request.DecodeAndValidateJSONPayload(r, &payload) if err != nil { return nil, err } - stack, err := handler.edgeStacksService.BuildEdgeStack(payload.Name, payload.DeploymentType, payload.EdgeGroups, payload.Registries, payload.UseManifestNamespaces) + stack, err := handler.edgeStacksService.BuildEdgeStack(tx, payload.Name, payload.DeploymentType, payload.EdgeGroups, payload.Registries, payload.UseManifestNamespaces) if err != nil { return nil, errors.Wrap(err, "failed to create edge stack object") } @@ -121,13 +123,13 @@ func (handler *Handler) createEdgeStackFromGitRepository(r *http.Request, dryrun } } - return handler.edgeStacksService.PersistEdgeStack(stack, func(stackFolder string, relatedEndpointIds []portainer.EndpointID) (composePath string, manifestPath string, projectPath string, err error) { - return handler.storeManifestFromGitRepository(stackFolder, relatedEndpointIds, payload.DeploymentType, userID, repoConfig) + return handler.edgeStacksService.PersistEdgeStack(tx, stack, func(stackFolder string, relatedEndpointIds []portainer.EndpointID) (composePath string, manifestPath string, projectPath string, err error) { + return handler.storeManifestFromGitRepository(tx, stackFolder, relatedEndpointIds, payload.DeploymentType, userID, repoConfig) }) } -func (handler *Handler) storeManifestFromGitRepository(stackFolder string, relatedEndpointIds []portainer.EndpointID, deploymentType portainer.EdgeStackDeploymentType, currentUserID portainer.UserID, repositoryConfig gittypes.RepoConfig) (composePath, manifestPath, projectPath string, err error) { - hasWrongType, err := hasWrongEnvironmentType(handler.DataStore.Endpoint(), relatedEndpointIds, deploymentType) +func (handler *Handler) storeManifestFromGitRepository(tx dataservices.DataStoreTx, stackFolder string, relatedEndpointIds []portainer.EndpointID, deploymentType portainer.EdgeStackDeploymentType, currentUserID portainer.UserID, repositoryConfig gittypes.RepoConfig) (composePath, manifestPath, projectPath string, err error) { + hasWrongType, err := hasWrongEnvironmentType(tx.Endpoint(), relatedEndpointIds, deploymentType) if err != nil { return "", "", "", fmt.Errorf("unable to check for existence of non fitting environments: %w", err) } diff --git a/api/http/handler/edgestacks/edgestack_create_string.go b/api/http/handler/edgestacks/edgestack_create_string.go index b46c0fb34..a3e2bc918 100644 --- a/api/http/handler/edgestacks/edgestack_create_string.go +++ b/api/http/handler/edgestacks/edgestack_create_string.go @@ -4,12 +4,14 @@ import ( "fmt" "net/http" - "github.com/asaskevich/govalidator" - "github.com/pkg/errors" "github.com/portainer/libhttp/request" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" "github.com/portainer/portainer/api/filesystem" httperrors "github.com/portainer/portainer/api/http/errors" + + "github.com/asaskevich/govalidator" + "github.com/pkg/errors" ) type edgeStackFromStringPayload struct { @@ -64,14 +66,14 @@ func (payload *edgeStackFromStringPayload) Validate(r *http.Request) error { // @failure 500 "Internal server error" // @failure 503 "Edge compute features are disabled" // @router /edge_stacks/create/string [post] -func (handler *Handler) createEdgeStackFromFileContent(r *http.Request, dryrun bool) (*portainer.EdgeStack, error) { +func (handler *Handler) createEdgeStackFromFileContent(r *http.Request, tx dataservices.DataStoreTx, dryrun bool) (*portainer.EdgeStack, error) { var payload edgeStackFromStringPayload err := request.DecodeAndValidateJSONPayload(r, &payload) if err != nil { return nil, err } - stack, err := handler.edgeStacksService.BuildEdgeStack(payload.Name, payload.DeploymentType, payload.EdgeGroups, payload.Registries, payload.UseManifestNamespaces) + stack, err := handler.edgeStacksService.BuildEdgeStack(tx, payload.Name, payload.DeploymentType, payload.EdgeGroups, payload.Registries, payload.UseManifestNamespaces) if err != nil { return nil, errors.Wrap(err, "failed to create Edge stack object") } @@ -80,14 +82,13 @@ func (handler *Handler) createEdgeStackFromFileContent(r *http.Request, dryrun b return stack, nil } - return handler.edgeStacksService.PersistEdgeStack(stack, func(stackFolder string, relatedEndpointIds []portainer.EndpointID) (composePath string, manifestPath string, projectPath string, err error) { - return handler.storeFileContent(stackFolder, payload.DeploymentType, relatedEndpointIds, []byte(payload.StackFileContent)) + return handler.edgeStacksService.PersistEdgeStack(tx, stack, func(stackFolder string, relatedEndpointIds []portainer.EndpointID) (composePath string, manifestPath string, projectPath string, err error) { + return handler.storeFileContent(tx, stackFolder, payload.DeploymentType, relatedEndpointIds, []byte(payload.StackFileContent)) }) - } -func (handler *Handler) storeFileContent(stackFolder string, deploymentType portainer.EdgeStackDeploymentType, relatedEndpointIds []portainer.EndpointID, fileContent []byte) (composePath, manifestPath, projectPath string, err error) { - hasWrongType, err := hasWrongEnvironmentType(handler.DataStore.Endpoint(), relatedEndpointIds, deploymentType) +func (handler *Handler) storeFileContent(tx dataservices.DataStoreTx, stackFolder string, deploymentType portainer.EdgeStackDeploymentType, relatedEndpointIds []portainer.EndpointID, fileContent []byte) (composePath, manifestPath, projectPath string, err error) { + hasWrongType, err := hasWrongEnvironmentType(tx.Endpoint(), relatedEndpointIds, deploymentType) if err != nil { return "", "", "", fmt.Errorf("unable to check for existence of non fitting environments: %w", err) } diff --git a/api/http/handler/edgestacks/edgestack_delete.go b/api/http/handler/edgestacks/edgestack_delete.go index edbd602a8..11ce4216e 100644 --- a/api/http/handler/edgestacks/edgestack_delete.go +++ b/api/http/handler/edgestacks/edgestack_delete.go @@ -1,12 +1,15 @@ package edgestacks import ( + "errors" "net/http" httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" "github.com/portainer/libhttp/response" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" + "github.com/portainer/portainer/pkg/featureflags" ) // @id EdgeStackDelete @@ -27,17 +30,38 @@ func (handler *Handler) edgeStackDelete(w http.ResponseWriter, r *http.Request) return httperror.BadRequest("Invalid edge stack identifier route variable", err) } - edgeStack, err := handler.DataStore.EdgeStack().EdgeStack(portainer.EdgeStackID(edgeStackID)) + if featureflags.IsEnabled(portainer.FeatureNoTx) { + err = handler.deleteEdgeStack(handler.DataStore, portainer.EdgeStackID(edgeStackID)) + } else { + err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error { + return handler.deleteEdgeStack(tx, portainer.EdgeStackID(edgeStackID)) + }) + } + + if err != nil { + var httpErr *httperror.HandlerError + if errors.As(err, &httpErr) { + return httpErr + } + + return httperror.InternalServerError("Unexpected error", err) + } + + return response.Empty(w) +} + +func (handler *Handler) deleteEdgeStack(tx dataservices.DataStoreTx, edgeStackID portainer.EdgeStackID) error { + edgeStack, err := tx.EdgeStack().EdgeStack(portainer.EdgeStackID(edgeStackID)) if handler.DataStore.IsErrObjectNotFound(err) { return httperror.NotFound("Unable to find an edge stack with the specified identifier inside the database", err) } else if err != nil { return httperror.InternalServerError("Unable to find an edge stack with the specified identifier inside the database", err) } - err = handler.edgeStacksService.DeleteEdgeStack(edgeStack.ID, edgeStack.EdgeGroups) + err = handler.edgeStacksService.DeleteEdgeStack(tx, edgeStack.ID, edgeStack.EdgeGroups) if err != nil { return httperror.InternalServerError("Unable to delete edge stack", err) } - return response.Empty(w) + return nil } diff --git a/api/http/handler/edgestacks/edgestack_status_delete.go b/api/http/handler/edgestacks/edgestack_status_delete.go index 2e857d8bf..113e96158 100644 --- a/api/http/handler/edgestacks/edgestack_status_delete.go +++ b/api/http/handler/edgestacks/edgestack_status_delete.go @@ -1,13 +1,16 @@ package edgestacks import ( + "errors" "net/http" httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" "github.com/portainer/libhttp/response" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" "github.com/portainer/portainer/api/http/middlewares" + "github.com/portainer/portainer/pkg/featureflags" ) // @id EdgeStackStatusDelete @@ -39,17 +42,39 @@ func (handler *Handler) edgeStackStatusDelete(w http.ResponseWriter, r *http.Req return httperror.Forbidden("Permission denied to access environment", err) } - stack, err := handler.DataStore.EdgeStack().EdgeStack(portainer.EdgeStackID(stackID)) - if err != nil { - return handler.handlerDBErr(err, "Unable to find a stack with the specified identifier inside the database") + var stack *portainer.EdgeStack + if featureflags.IsEnabled(portainer.FeatureNoTx) { + stack, err = handler.deleteEdgeStackStatus(handler.DataStore, portainer.EdgeStackID(stackID), endpoint) + } else { + err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error { + stack, err = handler.deleteEdgeStackStatus(tx, portainer.EdgeStackID(stackID), endpoint) + return err + }) } - - delete(stack.Status, endpoint.ID) - - err = handler.DataStore.EdgeStack().UpdateEdgeStack(stack.ID, stack) if err != nil { - return httperror.InternalServerError("Unable to persist the stack changes inside the database", err) + var httpErr *httperror.HandlerError + if errors.As(err, &httpErr) { + return httpErr + } + + return httperror.InternalServerError("Unexpected error", err) } return response.JSON(w, stack) } + +func (handler *Handler) deleteEdgeStackStatus(tx dataservices.DataStoreTx, stackID portainer.EdgeStackID, endpoint *portainer.Endpoint) (*portainer.EdgeStack, error) { + stack, err := tx.EdgeStack().EdgeStack(portainer.EdgeStackID(stackID)) + if err != nil { + return nil, handler.handlerDBErr(err, "Unable to find a stack with the specified identifier inside the database") + } + + delete(stack.Status, endpoint.ID) + + err = tx.EdgeStack().UpdateEdgeStack(stack.ID, stack) + if err != nil { + return nil, httperror.InternalServerError("Unable to persist the stack changes inside the database", err) + } + + return stack, nil +} diff --git a/api/http/handler/edgestacks/edgestack_status_update.go b/api/http/handler/edgestacks/edgestack_status_update.go index a6a194c02..3fe80bb07 100644 --- a/api/http/handler/edgestacks/edgestack_status_update.go +++ b/api/http/handler/edgestacks/edgestack_status_update.go @@ -7,6 +7,8 @@ import ( "github.com/portainer/libhttp/request" "github.com/portainer/libhttp/response" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" + "github.com/portainer/portainer/pkg/featureflags" "github.com/asaskevich/govalidator" httperror "github.com/portainer/libhttp/error" @@ -20,15 +22,15 @@ type updateStatusPayload struct { func (payload *updateStatusPayload) Validate(r *http.Request) error { if payload.Status == nil { - return errors.New("Invalid status") + return errors.New("invalid status") } if payload.EndpointID == 0 { - return errors.New("Invalid EnvironmentID") + return errors.New("invalid EnvironmentID") } if *payload.Status == portainer.EdgeStackStatusError && govalidator.IsNull(payload.Error) { - return errors.New("Error message is mandatory when status is error") + return errors.New("error message is mandatory when status is error") } return nil @@ -59,20 +61,74 @@ func (handler *Handler) edgeStackStatusUpdate(w http.ResponseWriter, r *http.Req return httperror.BadRequest("Invalid request payload", err) } - endpoint, err := handler.DataStore.Endpoint().Endpoint(payload.EndpointID) + var stack *portainer.EdgeStack + if featureflags.IsEnabled(portainer.FeatureNoTx) { + stack, err = handler.updateEdgeStackStatus(handler.DataStore, r, portainer.EdgeStackID(stackID), payload) + } else { + err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error { + stack, err = handler.updateEdgeStackStatus(tx, r, portainer.EdgeStackID(stackID), payload) + return err + }) + } + if err != nil { - return handler.handlerDBErr(err, "Unable to find an environment with the specified identifier inside the database") + var httpErr *httperror.HandlerError + if errors.As(err, &httpErr) { + return httpErr + } + + return httperror.InternalServerError("Unexpected error", err) + } + + return response.JSON(w, stack) +} + +func (handler *Handler) updateEdgeStackStatus(tx dataservices.DataStoreTx, r *http.Request, stackID portainer.EdgeStackID, payload updateStatusPayload) (*portainer.EdgeStack, error) { + endpoint, err := tx.Endpoint().Endpoint(payload.EndpointID) + if err != nil { + return nil, handler.handlerDBErr(err, "Unable to find an environment with the specified identifier inside the database") } err = handler.requestBouncer.AuthorizedEdgeEndpointOperation(r, endpoint) if err != nil { - return httperror.Forbidden("Permission denied to access environment", err) + return nil, httperror.Forbidden("Permission denied to access environment", err) } - var stack portainer.EdgeStack + var stack *portainer.EdgeStack - err = handler.DataStore.EdgeStack().UpdateEdgeStackFunc(portainer.EdgeStackID(stackID), func(edgeStack *portainer.EdgeStack) { - details := edgeStack.Status[payload.EndpointID].Details + if featureflags.IsEnabled(portainer.FeatureNoTx) { + err = tx.EdgeStack().UpdateEdgeStackFunc(portainer.EdgeStackID(stackID), func(edgeStack *portainer.EdgeStack) { + details := edgeStack.Status[payload.EndpointID].Details + details.Pending = false + + switch *payload.Status { + case portainer.EdgeStackStatusOk: + details.Ok = true + case portainer.EdgeStackStatusError: + details.Error = true + case portainer.EdgeStackStatusAcknowledged: + details.Acknowledged = true + case portainer.EdgeStackStatusRemove: + details.Remove = true + case portainer.EdgeStackStatusImagesPulled: + details.ImagesPulled = true + } + + edgeStack.Status[payload.EndpointID] = portainer.EdgeStackStatus{ + Details: details, + Error: payload.Error, + EndpointID: payload.EndpointID, + } + + stack = edgeStack + }) + } else { + stack, err = tx.EdgeStack().EdgeStack(stackID) + if err != nil { + return nil, err + } + + details := stack.Status[payload.EndpointID].Details details.Pending = false switch *payload.Status { @@ -88,17 +144,17 @@ func (handler *Handler) edgeStackStatusUpdate(w http.ResponseWriter, r *http.Req details.ImagesPulled = true } - edgeStack.Status[payload.EndpointID] = portainer.EdgeStackStatus{ + stack.Status[payload.EndpointID] = portainer.EdgeStackStatus{ Details: details, Error: payload.Error, EndpointID: payload.EndpointID, } - stack = *edgeStack - }) + err = tx.EdgeStack().UpdateEdgeStack(stackID, stack) + } if err != nil { - return handler.handlerDBErr(err, "Unable to persist the stack changes inside the database") + return nil, handler.handlerDBErr(err, "Unable to persist the stack changes inside the database") } - return response.JSON(w, stack) + return stack, nil } diff --git a/api/http/handler/edgestacks/edgestack_update.go b/api/http/handler/edgestacks/edgestack_update.go index 37486d8b1..cf75fd637 100644 --- a/api/http/handler/edgestacks/edgestack_update.go +++ b/api/http/handler/edgestacks/edgestack_update.go @@ -9,9 +9,12 @@ import ( "github.com/portainer/libhttp/request" "github.com/portainer/libhttp/response" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" "github.com/portainer/portainer/api/filesystem" "github.com/portainer/portainer/api/internal/edge" "github.com/portainer/portainer/api/internal/endpointutils" + "github.com/portainer/portainer/pkg/featureflags" + "github.com/rs/zerolog/log" ) @@ -26,11 +29,13 @@ type updateEdgeStackPayload struct { func (payload *updateEdgeStackPayload) Validate(r *http.Request) error { if payload.StackFileContent == "" { - return errors.New("Invalid stack file content") + return errors.New("invalid stack file content") } + if len(payload.EdgeGroups) == 0 { - return errors.New("Edge Groups are mandatory for an Edge stack") + return errors.New("edge Groups are mandatory for an Edge stack") } + return nil } @@ -55,25 +60,48 @@ func (handler *Handler) edgeStackUpdate(w http.ResponseWriter, r *http.Request) return httperror.BadRequest("Invalid stack identifier route variable", err) } - stack, err := handler.DataStore.EdgeStack().EdgeStack(portainer.EdgeStackID(stackID)) - if err != nil { - return handler.handlerDBErr(err, "Unable to find a stack with the specified identifier inside the database") - } - var payload updateEdgeStackPayload err = request.DecodeAndValidateJSONPayload(r, &payload) if err != nil { return httperror.BadRequest("Invalid request payload", err) } - relationConfig, err := edge.FetchEndpointRelationsConfig(handler.DataStore) + var stack *portainer.EdgeStack + if featureflags.IsEnabled(portainer.FeatureNoTx) { + stack, err = handler.updateEdgeStack(handler.DataStore, portainer.EdgeStackID(stackID), payload) + } else { + err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error { + stack, err = handler.updateEdgeStack(tx, portainer.EdgeStackID(stackID), payload) + return err + }) + } + if err != nil { - return httperror.InternalServerError("Unable to retrieve environments relations config from database", err) + var httpErr *httperror.HandlerError + if errors.As(err, &httpErr) { + return httpErr + } + + return httperror.InternalServerError("Unexpected error", err) + } + + return response.JSON(w, stack) +} + +func (handler *Handler) updateEdgeStack(tx dataservices.DataStoreTx, stackID portainer.EdgeStackID, payload updateEdgeStackPayload) (*portainer.EdgeStack, error) { + stack, err := tx.EdgeStack().EdgeStack(portainer.EdgeStackID(stackID)) + if err != nil { + return nil, handler.handlerDBErr(err, "Unable to find a stack with the specified identifier inside the database") + } + + relationConfig, err := edge.FetchEndpointRelationsConfig(tx) + if err != nil { + return nil, httperror.InternalServerError("Unable to retrieve environments relations config from database", err) } relatedEndpointIds, err := edge.EdgeStackRelatedEndpoints(stack.EdgeGroups, relationConfig.Endpoints, relationConfig.EndpointGroups, relationConfig.EdgeGroups) if err != nil { - return httperror.InternalServerError("Unable to retrieve edge stack related environments from database", err) + return nil, httperror.InternalServerError("Unable to retrieve edge stack related environments from database", err) } endpointsToAdd := map[portainer.EndpointID]bool{} @@ -81,7 +109,7 @@ func (handler *Handler) edgeStackUpdate(w http.ResponseWriter, r *http.Request) if payload.EdgeGroups != nil { newRelated, err := edge.EdgeStackRelatedEndpoints(payload.EdgeGroups, relationConfig.Endpoints, relationConfig.EndpointGroups, relationConfig.EdgeGroups) if err != nil { - return httperror.InternalServerError("Unable to retrieve edge stack related environments from database", err) + return nil, httperror.InternalServerError("Unable to retrieve edge stack related environments from database", err) } oldRelatedSet := endpointutils.EndpointSet(relatedEndpointIds) @@ -95,16 +123,16 @@ func (handler *Handler) edgeStackUpdate(w http.ResponseWriter, r *http.Request) } for endpointID := range endpointsToRemove { - relation, err := handler.DataStore.EndpointRelation().EndpointRelation(endpointID) + relation, err := tx.EndpointRelation().EndpointRelation(endpointID) if err != nil { - return httperror.InternalServerError("Unable to find environment relation in database", err) + return nil, httperror.InternalServerError("Unable to find environment relation in database", err) } delete(relation.EdgeStacks, stack.ID) - err = handler.DataStore.EndpointRelation().UpdateEndpointRelation(endpointID, relation) + err = tx.EndpointRelation().UpdateEndpointRelation(endpointID, relation) if err != nil { - return httperror.InternalServerError("Unable to persist environment relation in database", err) + return nil, httperror.InternalServerError("Unable to persist environment relation in database", err) } } @@ -115,16 +143,16 @@ func (handler *Handler) edgeStackUpdate(w http.ResponseWriter, r *http.Request) } for endpointID := range endpointsToAdd { - relation, err := handler.DataStore.EndpointRelation().EndpointRelation(endpointID) + relation, err := tx.EndpointRelation().EndpointRelation(endpointID) if err != nil { - return httperror.InternalServerError("Unable to find environment relation in database", err) + return nil, httperror.InternalServerError("Unable to find environment relation in database", err) } relation.EdgeStacks[stack.ID] = true - err = handler.DataStore.EndpointRelation().UpdateEndpointRelation(endpointID, relation) + err = tx.EndpointRelation().UpdateEndpointRelation(endpointID, relation) if err != nil { - return httperror.InternalServerError("Unable to persist environment relation in database", err) + return nil, httperror.InternalServerError("Unable to persist environment relation in database", err) } } @@ -146,12 +174,12 @@ func (handler *Handler) edgeStackUpdate(w http.ResponseWriter, r *http.Request) stackFolder := strconv.Itoa(int(stack.ID)) - hasWrongType, err := hasWrongEnvironmentType(handler.DataStore.Endpoint(), relatedEndpointIds, payload.DeploymentType) + hasWrongType, err := hasWrongEnvironmentType(tx.Endpoint(), relatedEndpointIds, payload.DeploymentType) if err != nil { - return httperror.BadRequest("unable to check for existence of non fitting environments: %w", err) + return nil, httperror.BadRequest("unable to check for existence of non fitting environments: %w", err) } if hasWrongType { - return httperror.BadRequest("edge stack with config do not match the environment type", nil) + return nil, httperror.BadRequest("edge stack with config do not match the environment type", nil) } if payload.DeploymentType == portainer.EdgeStackDeploymentCompose { @@ -161,12 +189,12 @@ func (handler *Handler) edgeStackUpdate(w http.ResponseWriter, r *http.Request) _, err := handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, stack.EntryPoint, []byte(payload.StackFileContent)) if err != nil { - return httperror.InternalServerError("Unable to persist updated Compose file on disk", err) + return nil, httperror.InternalServerError("Unable to persist updated Compose file on disk", err) } manifestPath, err := handler.convertAndStoreKubeManifestIfNeeded(stackFolder, stack.ProjectPath, stack.EntryPoint, relatedEndpointIds) if err != nil { - return httperror.InternalServerError("Unable to convert and persist updated Kubernetes manifest file on disk", err) + return nil, httperror.InternalServerError("Unable to convert and persist updated Kubernetes manifest file on disk", err) } stack.ManifestPath = manifestPath @@ -181,7 +209,7 @@ func (handler *Handler) edgeStackUpdate(w http.ResponseWriter, r *http.Request) _, err = handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, stack.ManifestPath, []byte(payload.StackFileContent)) if err != nil { - return httperror.InternalServerError("Unable to persist updated Kubernetes manifest file on disk", err) + return nil, httperror.InternalServerError("Unable to persist updated Kubernetes manifest file on disk", err) } } @@ -197,10 +225,10 @@ func (handler *Handler) edgeStackUpdate(w http.ResponseWriter, r *http.Request) stack.Status = make(map[portainer.EndpointID]portainer.EdgeStackStatus) } - err = handler.DataStore.EdgeStack().UpdateEdgeStack(stack.ID, stack) + err = tx.EdgeStack().UpdateEdgeStack(stack.ID, stack) if err != nil { - return httperror.InternalServerError("Unable to persist the stack changes inside the database", err) + return nil, httperror.InternalServerError("Unable to persist the stack changes inside the database", err) } - return response.JSON(w, stack) + return stack, nil } diff --git a/api/internal/edge/edgestack.go b/api/internal/edge/edgestack.go index 83dd99617..6e6b6901f 100644 --- a/api/internal/edge/edgestack.go +++ b/api/internal/edge/edgestack.go @@ -8,7 +8,7 @@ import ( "github.com/portainer/portainer/api/dataservices" ) -var ErrEdgeGroupNotFound = errors.New("Edge group was not found") +var ErrEdgeGroupNotFound = errors.New("edge group was not found") // EdgeStackRelatedEndpoints returns a list of environments(endpoints) related to this Edge stack func EdgeStackRelatedEndpoints(edgeGroupIDs []portainer.EdgeGroupID, endpoints []portainer.Endpoint, endpointGroups []portainer.EndpointGroup, edgeGroups []portainer.EdgeGroup) ([]portainer.EndpointID, error) { @@ -42,18 +42,18 @@ type EndpointRelationsConfig struct { } // FetchEndpointRelationsConfig fetches config needed for Edge Stack related endpoints -func FetchEndpointRelationsConfig(dataStore dataservices.DataStore) (*EndpointRelationsConfig, error) { - endpoints, err := dataStore.Endpoint().Endpoints() +func FetchEndpointRelationsConfig(tx dataservices.DataStoreTx) (*EndpointRelationsConfig, error) { + endpoints, err := tx.Endpoint().Endpoints() if err != nil { return nil, fmt.Errorf("unable to retrieve environments from database: %w", err) } - endpointGroups, err := dataStore.EndpointGroup().EndpointGroups() + endpointGroups, err := tx.EndpointGroup().EndpointGroups() if err != nil { return nil, fmt.Errorf("unable to retrieve environment groups from database: %w", err) } - edgeGroups, err := dataStore.EdgeGroup().EdgeGroups() + edgeGroups, err := tx.EdgeGroup().EdgeGroups() if err != nil { return nil, fmt.Errorf("unable to retrieve edge groups from database: %w", err) } diff --git a/api/internal/edge/edgestacks/service.go b/api/internal/edge/edgestacks/service.go index 863f186ed..7ad2180fe 100644 --- a/api/internal/edge/edgestacks/service.go +++ b/api/internal/edge/edgestacks/service.go @@ -6,12 +6,13 @@ import ( "strings" "time" - "github.com/pkg/errors" portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/dataservices" httperrors "github.com/portainer/portainer/api/http/errors" "github.com/portainer/portainer/api/internal/edge" edgetypes "github.com/portainer/portainer/api/internal/edge/types" + + "github.com/pkg/errors" ) // Service represents a service for managing edge stacks. @@ -28,20 +29,20 @@ func NewService(dataStore dataservices.DataStore) *Service { // BuildEdgeStack builds the initial edge stack object // PersistEdgeStack is required to be called after this to persist the edge stack -func (service *Service) BuildEdgeStack(name string, +func (service *Service) BuildEdgeStack( + tx dataservices.DataStoreTx, + name string, deploymentType portainer.EdgeStackDeploymentType, edgeGroups []portainer.EdgeGroupID, registries []portainer.RegistryID, useManifestNamespaces bool, ) (*portainer.EdgeStack, error) { - edgeStacksService := service.dataStore.EdgeStack() - - err := validateUniqueName(edgeStacksService.EdgeStacks, name) + err := validateUniqueName(tx.EdgeStack().EdgeStacks, name) if err != nil { return nil, err } - stackID := edgeStacksService.GetNextIdentifier() + stackID := tx.EdgeStack().GetNextIdentifier() return &portainer.EdgeStack{ ID: portainer.EdgeStackID(stackID), Name: name, @@ -65,15 +66,17 @@ func validateUniqueName(edgeStacksGetter func() ([]portainer.EdgeStack, error), return errors.New("Edge stack name must be unique") } } + return nil } // PersistEdgeStack persists the edge stack in the database and its relations func (service *Service) PersistEdgeStack( + tx dataservices.DataStoreTx, stack *portainer.EdgeStack, storeManifest edgetypes.StoreManifestFunc) (*portainer.EdgeStack, error) { - relationConfig, err := edge.FetchEndpointRelationsConfig(service.dataStore) + relationConfig, err := edge.FetchEndpointRelationsConfig(tx) if err != nil { return nil, fmt.Errorf("unable to find environment relations in database: %w", err) @@ -98,12 +101,12 @@ func (service *Service) PersistEdgeStack( stack.EntryPoint = composePath stack.NumDeployments = len(relatedEndpointIds) - err = service.updateEndpointRelations(stack.ID, relatedEndpointIds) + err = service.updateEndpointRelations(tx, stack.ID, relatedEndpointIds) if err != nil { return nil, fmt.Errorf("unable to update endpoint relations: %w", err) } - err = service.dataStore.EdgeStack().Create(stack.ID, stack) + err = tx.EdgeStack().Create(stack.ID, stack) if err != nil { return nil, err } @@ -112,8 +115,8 @@ func (service *Service) PersistEdgeStack( } // updateEndpointRelations adds a relation between the Edge Stack to the related environments(endpoints) -func (service *Service) updateEndpointRelations(edgeStackID portainer.EdgeStackID, relatedEndpointIds []portainer.EndpointID) error { - endpointRelationService := service.dataStore.EndpointRelation() +func (service *Service) updateEndpointRelations(tx dataservices.DataStoreTx, edgeStackID portainer.EdgeStackID, relatedEndpointIds []portainer.EndpointID) error { + endpointRelationService := tx.EndpointRelation() for _, endpointID := range relatedEndpointIds { relation, err := endpointRelationService.EndpointRelation(endpointID) @@ -133,9 +136,8 @@ func (service *Service) updateEndpointRelations(edgeStackID portainer.EdgeStackI } // DeleteEdgeStack deletes the edge stack from the database and its relations -func (service *Service) DeleteEdgeStack(edgeStackID portainer.EdgeStackID, relatedEdgeGroupsIds []portainer.EdgeGroupID) error { - - relationConfig, err := edge.FetchEndpointRelationsConfig(service.dataStore) +func (service *Service) DeleteEdgeStack(tx dataservices.DataStoreTx, edgeStackID portainer.EdgeStackID, relatedEdgeGroupsIds []portainer.EdgeGroupID) error { + relationConfig, err := edge.FetchEndpointRelationsConfig(tx) if err != nil { return errors.WithMessage(err, "Unable to retrieve environments relations config from database") } @@ -146,20 +148,20 @@ func (service *Service) DeleteEdgeStack(edgeStackID portainer.EdgeStackID, relat } for _, endpointID := range relatedEndpointIds { - relation, err := service.dataStore.EndpointRelation().EndpointRelation(endpointID) + relation, err := tx.EndpointRelation().EndpointRelation(endpointID) if err != nil { return errors.WithMessage(err, "Unable to find environment relation in database") } delete(relation.EdgeStacks, edgeStackID) - err = service.dataStore.EndpointRelation().UpdateEndpointRelation(endpointID, relation) + err = tx.EndpointRelation().UpdateEndpointRelation(endpointID, relation) if err != nil { return errors.WithMessage(err, "Unable to persist environment relation in database") } } - err = service.dataStore.EdgeStack().DeleteEdgeStack(portainer.EdgeStackID(edgeStackID)) + err = tx.EdgeStack().DeleteEdgeStack(portainer.EdgeStackID(edgeStackID)) if err != nil { return errors.WithMessage(err, "Unable to remove the edge stack from the database") } diff --git a/api/internal/edge/edgestacks/service_test.go b/api/internal/edge/edgestacks/service_test.go index 47d99f2ac..de211b32e 100644 --- a/api/internal/edge/edgestacks/service_test.go +++ b/api/internal/edge/edgestacks/service_test.go @@ -25,7 +25,7 @@ func Test_updateEndpointRelation_successfulRuns(t *testing.T) { service := NewService(dataStore) - err := service.updateEndpointRelations(edgeStackID, relatedIds) + err := service.updateEndpointRelations(dataStore, edgeStackID, relatedIds) assert.NoError(t, err, "updateEndpointRelations should not fail")