From b4c2820ad702326774fc7c14480f7b143628585a Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Mon, 18 Jun 2018 12:07:56 +0200 Subject: [PATCH] refactor(api): use a standard stack identifier (#1980) --- api/bolt/datastore.go | 4 +- api/bolt/migrate_dbversion11.go | 88 +++++++++++++++++-- api/bolt/migrator.go | 1 + api/bolt/stack_service.go | 27 +++++- api/cmd/portainer/main.go | 6 +- api/filesystem/filesystem.go | 5 ++ .../handler/stacks/create_compose_stack.go | 12 +-- api/http/handler/stacks/create_swarm_stack.go | 12 +-- api/http/handler/stacks/stack_create.go | 11 ++- api/http/handler/stacks/stack_delete.go | 12 ++- api/http/handler/stacks/stack_file.go | 2 +- api/http/handler/stacks/stack_inspect.go | 2 +- api/http/handler/stacks/stack_update.go | 2 +- api/portainer.go | 4 +- 14 files changed, 150 insertions(+), 38 deletions(-) diff --git a/api/bolt/datastore.go b/api/bolt/datastore.go index 37ad96ce7..d5002a413 100644 --- a/api/bolt/datastore.go +++ b/api/bolt/datastore.go @@ -31,6 +31,7 @@ type Store struct { db *bolt.DB checkForDataMigration bool + FileService portainer.FileService } const ( @@ -50,7 +51,7 @@ const ( ) // NewStore initializes a new Store and the associated services -func NewStore(storePath string) (*Store, error) { +func NewStore(storePath string, fileService portainer.FileService) (*Store, error) { store := &Store{ Path: storePath, UserService: &UserService{}, @@ -65,6 +66,7 @@ func NewStore(storePath string) (*Store, error) { DockerHubService: &DockerHubService{}, StackService: &StackService{}, TagService: &TagService{}, + FileService: fileService, } store.UserService.store = store store.TeamService.store = store diff --git a/api/bolt/migrate_dbversion11.go b/api/bolt/migrate_dbversion11.go index dcd0e1100..b53e3fcbf 100644 --- a/api/bolt/migrate_dbversion11.go +++ b/api/bolt/migrate_dbversion11.go @@ -1,6 +1,13 @@ package bolt -import "github.com/portainer/portainer" +import ( + "strconv" + "strings" + + "github.com/boltdb/bolt" + "github.com/portainer/portainer" + "github.com/portainer/portainer/bolt/internal" +) func (m *Migrator) updateEndpointsToVersion12() error { legacyEndpoints, err := m.EndpointService.Endpoints() @@ -38,16 +45,24 @@ func (m *Migrator) updateEndpointGroupsToVersion12() error { return nil } +type legacyStack struct { + ID string `json:"Id"` + Name string `json:"Name"` + EndpointID portainer.EndpointID `json:"EndpointId"` + SwarmID string `json:"SwarmId"` + EntryPoint string `json:"EntryPoint"` + Env []portainer.Pair `json:"Env"` + ProjectPath string +} + func (m *Migrator) updateStacksToVersion12() error { - legacyStacks, err := m.StackService.Stacks() + legacyStacks, err := m.retrieveLegacyStacks() if err != nil { return err } - for _, stack := range legacyStacks { - stack.Type = portainer.DockerSwarmStack - - err = m.StackService.UpdateStack(stack.ID, &stack) + for _, legacyStack := range legacyStacks { + err := m.convertLegacyStack(&legacyStack) if err != nil { return err } @@ -55,3 +70,64 @@ func (m *Migrator) updateStacksToVersion12() error { return nil } + +func (m *Migrator) convertLegacyStack(s *legacyStack) error { + stackID := m.StackService.GetNextIdentifier() + stack := &portainer.Stack{ + ID: portainer.StackID(stackID), + Name: s.Name, + Type: portainer.DockerSwarmStack, + SwarmID: s.SwarmID, + EndpointID: 0, + EntryPoint: s.EntryPoint, + Env: s.Env, + } + + stack.ProjectPath = strings.Replace(s.ProjectPath, s.ID, strconv.Itoa(stackID), 1) + err := m.store.FileService.Rename(s.ProjectPath, stack.ProjectPath) + if err != nil { + return err + } + + err = m.deleteLegacyStack(s.ID) + if err != nil { + return err + } + + return m.StackService.CreateStack(stack) +} + +func (m *Migrator) deleteLegacyStack(legacyID string) error { + return m.store.db.Update(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(stackBucketName)) + err := bucket.Delete([]byte(legacyID)) + if err != nil { + return err + } + return nil + }) +} + +func (m *Migrator) retrieveLegacyStacks() ([]legacyStack, error) { + var legacyStacks = make([]legacyStack, 0) + err := m.store.db.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(stackBucketName)) + cursor := bucket.Cursor() + + for k, v := cursor.First(); k != nil; k, v = cursor.Next() { + var stack legacyStack + err := internal.UnmarshalObject(v, &stack) + if err != nil { + return err + } + legacyStacks = append(legacyStacks, stack) + } + + return nil + }) + if err != nil { + return nil, err + } + + return legacyStacks, nil +} diff --git a/api/bolt/migrator.go b/api/bolt/migrator.go index ef3901d42..84ede8156 100644 --- a/api/bolt/migrator.go +++ b/api/bolt/migrator.go @@ -14,6 +14,7 @@ type Migrator struct { StackService *StackService UserService *UserService VersionService *VersionService + FileService portainer.FileService } // NewMigrator creates a new Migrator. diff --git a/api/bolt/stack_service.go b/api/bolt/stack_service.go index 60230341b..34855e59f 100644 --- a/api/bolt/stack_service.go +++ b/api/bolt/stack_service.go @@ -17,7 +17,7 @@ func (service *StackService) Stack(ID portainer.StackID) (*portainer.Stack, erro var data []byte err := service.store.db.View(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(stackBucketName)) - value := bucket.Get([]byte(ID)) + value := bucket.Get(internal.Itob(int(ID))) if value == nil { return portainer.ErrStackNotFound } @@ -92,17 +92,36 @@ func (service *StackService) Stacks() ([]portainer.Stack, error) { return stacks, nil } +// GetNextIdentifier returns the current bucket identifier incremented by 1. +func (service *StackService) GetNextIdentifier() int { + var identifier int + + service.store.db.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(stackBucketName)) + id := bucket.Sequence() + identifier = int(id) + return nil + }) + + identifier++ + return identifier +} + // CreateStack creates a new stack. func (service *StackService) CreateStack(stack *portainer.Stack) error { return service.store.db.Update(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(stackBucketName)) + err := bucket.SetSequence(uint64(stack.ID)) + if err != nil { + return err + } data, err := internal.MarshalObject(stack) if err != nil { return err } - err = bucket.Put([]byte(stack.ID), data) + err = bucket.Put(internal.Itob(int(stack.ID)), data) if err != nil { return err } @@ -119,7 +138,7 @@ func (service *StackService) UpdateStack(ID portainer.StackID, stack *portainer. return service.store.db.Update(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(stackBucketName)) - err = bucket.Put([]byte(ID), data) + err = bucket.Put(internal.Itob(int(ID)), data) if err != nil { return err } @@ -131,7 +150,7 @@ func (service *StackService) UpdateStack(ID portainer.StackID, stack *portainer. func (service *StackService) DeleteStack(ID portainer.StackID) error { return service.store.db.Update(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(stackBucketName)) - err := bucket.Delete([]byte(ID)) + err := bucket.Delete(internal.Itob(int(ID))) if err != nil { return err } diff --git a/api/cmd/portainer/main.go b/api/cmd/portainer/main.go index 4f37e7846..cd4921590 100644 --- a/api/cmd/portainer/main.go +++ b/api/cmd/portainer/main.go @@ -42,8 +42,8 @@ func initFileService(dataStorePath string) portainer.FileService { return fileService } -func initStore(dataStorePath string) *bolt.Store { - store, err := bolt.NewStore(dataStorePath) +func initStore(dataStorePath string, fileService portainer.FileService) *bolt.Store { + store, err := bolt.NewStore(dataStorePath, fileService) if err != nil { log.Fatal(err) } @@ -307,7 +307,7 @@ func main() { fileService := initFileService(*flags.Data) - store := initStore(*flags.Data) + store := initStore(*flags.Data, fileService) defer store.Close() jwtService := initJWTService(!*flags.NoAuth) diff --git a/api/filesystem/filesystem.go b/api/filesystem/filesystem.go index c3f459ddf..b0efbfe8f 100644 --- a/api/filesystem/filesystem.go +++ b/api/filesystem/filesystem.go @@ -186,6 +186,11 @@ func (service *Service) GetFileContent(filePath string) (string, error) { return string(content), nil } +// Rename renames a file or directory +func (service *Service) Rename(oldPath, newPath string) error { + return os.Rename(oldPath, newPath) +} + // WriteJSONToFile writes JSON to the specified file. func (service *Service) WriteJSONToFile(path string, content interface{}) error { jsonContent, err := json.Marshal(content) diff --git a/api/http/handler/stacks/create_compose_stack.go b/api/http/handler/stacks/create_compose_stack.go index c1321e297..2738e255b 100644 --- a/api/http/handler/stacks/create_compose_stack.go +++ b/api/http/handler/stacks/create_compose_stack.go @@ -46,9 +46,9 @@ func (handler *Handler) createComposeStackFromFileContent(w http.ResponseWriter, } } - stackIdentifier := buildStackIdentifier(payload.Name, endpoint.ID) + stackID := handler.StackService.GetNextIdentifier() stack := &portainer.Stack{ - ID: portainer.StackID(stackIdentifier), + ID: portainer.StackID(stackID), Name: payload.Name, Type: portainer.DockerComposeStack, EndpointID: endpoint.ID, @@ -126,9 +126,9 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite } } - stackIdentifier := buildStackIdentifier(payload.Name, endpoint.ID) + stackID := handler.StackService.GetNextIdentifier() stack := &portainer.Stack{ - ID: portainer.StackID(stackIdentifier), + ID: portainer.StackID(stackID), Name: payload.Name, Type: portainer.DockerComposeStack, EndpointID: endpoint.ID, @@ -211,9 +211,9 @@ func (handler *Handler) createComposeStackFromFileUpload(w http.ResponseWriter, } } - stackIdentifier := buildStackIdentifier(payload.Name, endpoint.ID) + stackID := handler.StackService.GetNextIdentifier() stack := &portainer.Stack{ - ID: portainer.StackID(stackIdentifier), + ID: portainer.StackID(stackID), Name: payload.Name, Type: portainer.DockerComposeStack, EndpointID: endpoint.ID, diff --git a/api/http/handler/stacks/create_swarm_stack.go b/api/http/handler/stacks/create_swarm_stack.go index 0a87fb53f..aeb99e3f5 100644 --- a/api/http/handler/stacks/create_swarm_stack.go +++ b/api/http/handler/stacks/create_swarm_stack.go @@ -51,9 +51,9 @@ func (handler *Handler) createSwarmStackFromFileContent(w http.ResponseWriter, r } } - stackIdentifier := buildStackIdentifier(payload.Name, endpoint.ID) + stackID := handler.StackService.GetNextIdentifier() stack := &portainer.Stack{ - ID: portainer.StackID(stackIdentifier), + ID: portainer.StackID(stackID), Name: payload.Name, Type: portainer.DockerSwarmStack, SwarmID: payload.SwarmID, @@ -138,9 +138,9 @@ func (handler *Handler) createSwarmStackFromGitRepository(w http.ResponseWriter, } } - stackIdentifier := buildStackIdentifier(payload.Name, endpoint.ID) + stackID := handler.StackService.GetNextIdentifier() stack := &portainer.Stack{ - ID: portainer.StackID(stackIdentifier), + ID: portainer.StackID(stackID), Name: payload.Name, Type: portainer.DockerSwarmStack, SwarmID: payload.SwarmID, @@ -239,9 +239,9 @@ func (handler *Handler) createSwarmStackFromFileUpload(w http.ResponseWriter, r } } - stackIdentifier := buildStackIdentifier(payload.Name, endpoint.ID) + stackID := handler.StackService.GetNextIdentifier() stack := &portainer.Stack{ - ID: portainer.StackID(stackIdentifier), + ID: portainer.StackID(stackID), Name: payload.Name, Type: portainer.DockerSwarmStack, SwarmID: payload.SwarmID, diff --git a/api/http/handler/stacks/stack_create.go b/api/http/handler/stacks/stack_create.go index 221ef2c4f..9a4c37c52 100644 --- a/api/http/handler/stacks/stack_create.go +++ b/api/http/handler/stacks/stack_create.go @@ -1,8 +1,8 @@ package stacks import ( + "log" "net/http" - "strconv" "github.com/portainer/portainer" httperror "github.com/portainer/portainer/http/error" @@ -14,14 +14,13 @@ func (handler *Handler) cleanUp(stack *portainer.Stack, doCleanUp *bool) error { return nil } - handler.FileService.RemoveDirectory(stack.ProjectPath) + err := handler.FileService.RemoveDirectory(stack.ProjectPath) + if err != nil { + log.Printf("http error: Unable to cleanup stack creation (err=%s)\n", err) + } return nil } -func buildStackIdentifier(stackName string, endpointID portainer.EndpointID) string { - return stackName + "_" + strconv.Itoa(int(endpointID)) -} - // POST request on /api/stacks?type=&method=&endpointId= func (handler *Handler) stackCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { stackType, err := request.RetrieveNumericQueryParameter(r, "type", false) diff --git a/api/http/handler/stacks/stack_delete.go b/api/http/handler/stacks/stack_delete.go index 3ca5378dd..c46905585 100644 --- a/api/http/handler/stacks/stack_delete.go +++ b/api/http/handler/stacks/stack_delete.go @@ -2,6 +2,7 @@ package stacks import ( "net/http" + "strconv" "github.com/portainer/portainer" httperror "github.com/portainer/portainer/http/error" @@ -12,6 +13,8 @@ import ( ) // DELETE request on /api/stacks/:id?external=&endpointId= +// If the external query parameter is set to true, the id route variable is expected to be +// the name of an external stack as a string. func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { stackID, err := request.RetrieveRouteVariableValue(r, "id") if err != nil { @@ -23,7 +26,12 @@ func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *htt return handler.deleteExternalStack(r, w, stackID) } - stack, err := handler.StackService.Stack(portainer.StackID(stackID)) + id, err := strconv.Atoi(stackID) + if err != nil { + return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err} + } + + stack, err := handler.StackService.Stack(portainer.StackID(id)) if err == portainer.ErrStackNotFound { return &httperror.HandlerError{http.StatusNotFound, "Unable to find a stack with the specified identifier inside the database", err} } else if err != nil { @@ -71,7 +79,7 @@ func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *htt return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err} } - err = handler.StackService.DeleteStack(portainer.StackID(stackID)) + err = handler.StackService.DeleteStack(portainer.StackID(id)) if err != nil { return &httperror.HandlerError{http.StatusInternalServerError, "Unable to remove the stack from the database", err} } diff --git a/api/http/handler/stacks/stack_file.go b/api/http/handler/stacks/stack_file.go index b476a2e47..0875ae8e1 100644 --- a/api/http/handler/stacks/stack_file.go +++ b/api/http/handler/stacks/stack_file.go @@ -18,7 +18,7 @@ type stackFileResponse struct { // GET request on /api/stacks/:id/file func (handler *Handler) stackFile(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { - stackID, err := request.RetrieveRouteVariableValue(r, "id") + stackID, err := request.RetrieveNumericRouteVariableValue(r, "id") if err != nil { return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err} } diff --git a/api/http/handler/stacks/stack_inspect.go b/api/http/handler/stacks/stack_inspect.go index 66a94a601..ec6d99509 100644 --- a/api/http/handler/stacks/stack_inspect.go +++ b/api/http/handler/stacks/stack_inspect.go @@ -13,7 +13,7 @@ import ( // GET request on /api/stacks/:id func (handler *Handler) stackInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { - stackID, err := request.RetrieveRouteVariableValue(r, "id") + stackID, err := request.RetrieveNumericRouteVariableValue(r, "id") if err != nil { return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err} } diff --git a/api/http/handler/stacks/stack_update.go b/api/http/handler/stacks/stack_update.go index 9867c3887..f20f92957 100644 --- a/api/http/handler/stacks/stack_update.go +++ b/api/http/handler/stacks/stack_update.go @@ -38,7 +38,7 @@ func (payload *updateSwarmStackPayload) Validate(r *http.Request) error { // PUT request on /api/stacks/:id?endpointId= func (handler *Handler) stackUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { - stackID, err := request.RetrieveRouteVariableValue(r, "id") + stackID, err := request.RetrieveNumericRouteVariableValue(r, "id") if err != nil { return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err} } diff --git a/api/portainer.go b/api/portainer.go index bfe84a213..37f1e5e19 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -129,7 +129,7 @@ type ( } // StackID represents a stack identifier (it must be composed of Name + "_" + SwarmID to create a unique identifier). - StackID string + StackID int // StackType represents the type of the stack (compose v2, stack deploy v3). StackType int @@ -373,6 +373,7 @@ type ( CreateStack(stack *Stack) error UpdateStack(ID StackID, stack *Stack) error DeleteStack(ID StackID) error + GetNextIdentifier() int } // DockerHubService represents a service for managing the DockerHub object. @@ -434,6 +435,7 @@ type ( // FileService represents a service for managing files. FileService interface { GetFileContent(filePath string) (string, error) + Rename(oldPath, newPath string) error RemoveDirectory(directoryPath string) error StoreTLSFileFromBytes(folder string, fileType TLSFileType, data []byte) (string, error) GetPathForTLSFile(folder string, fileType TLSFileType) (string, error)