From 0bcb57568c637eba82470c85ab83ecae6b32c847 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Thu, 13 Jul 2023 23:55:52 +0300 Subject: [PATCH] feat(edge/stacks): increase status transparency [EE-5554] (#9094) --- .gitignore | 2 +- api/dataservices/interface.go | 1 - .../migrator/migrate_dbversion100.go | 77 +++++- api/datastore/migrator/migrate_dbversion80.go | 4 +- api/datastore/migrator/migrator.go | 1 + .../test_data/output_24_to_latest.json | 2 +- .../edgestacks/edgestack_status_delete.go | 14 +- .../edgestacks/edgestack_status_update.go | 108 +++++---- .../edgestack_status_update_test.go | 26 +- api/http/handler/edgestacks/edgestack_test.go | 27 ++- .../handler/edgestacks/edgestack_update.go | 90 +++---- .../edgestacks/edgestack_update_test.go | 14 +- .../edgestacks/utils_update_stack_version.go | 59 +++++ .../endpointedge_status_inspect_test.go | 2 +- api/http/handler/endpoints/filter.go | 56 ++--- api/http/handler/endpoints/utils.go | 5 +- api/internal/edge/edgestacks/service.go | 2 +- api/internal/edge/edgestacks/status.go | 27 +++ api/internal/slices/slices.go | 21 +- api/portainer.go | 53 ++-- app/assets/ico/icon_up-to-date.svg | 5 + app/assets/ico/icon_updates-available.svg | 11 + app/edge/react/components/index.ts | 2 + .../datatables/genericDatatableController.js | 2 + app/portainer/react/components/index.ts | 2 +- app/react/components/Icon.tsx | 14 +- .../EnvironmentsDatatable.tsx | 110 +++++---- .../EnvironmentsDatatable/columns.tsx | 226 ++++++++++++++++-- .../ItemView/EnvironmentsDatatable/types.ts | 3 + .../edge-stacks/ListView/EdgeStackStatus.tsx | 87 +++++++ .../EdgeStacksDatatable/DeploymentCounter.tsx | 8 +- .../EdgeStacksDatatable.tsx | 21 +- .../EdgeStacksDatatable/EdgeStacksStatus.tsx | 87 +++++++ .../ListView/EdgeStacksDatatable/columns.tsx | 24 +- .../edge/edge-stacks/queries/useEdgeStack.ts | 16 +- .../edge/edge-stacks/queries/useEdgeStacks.ts | 3 +- app/react/edge/edge-stacks/types.ts | 44 +++- .../edge/edge-stacks/utils/uniqueStatus.ts | 16 ++ .../internal/composeplugin/composeplugin.go | 7 +- .../composeplugin/composeplugin_test.go | 13 +- .../compose/internal/composeplugin/status.go | 171 +++++++++++++ .../composeplugin/status_integration_test.go | 116 +++++++++ .../status_test_files/failed.yml | 7 + .../status_test_files/running.yml | 4 + pkg/libstack/libstack.go | 13 + 45 files changed, 1296 insertions(+), 307 deletions(-) create mode 100644 api/http/handler/edgestacks/utils_update_stack_version.go create mode 100644 api/internal/edge/edgestacks/status.go create mode 100644 app/assets/ico/icon_up-to-date.svg create mode 100644 app/assets/ico/icon_updates-available.svg create mode 100644 app/react/edge/edge-stacks/ListView/EdgeStackStatus.tsx create mode 100644 app/react/edge/edge-stacks/ListView/EdgeStacksDatatable/EdgeStacksStatus.tsx create mode 100644 app/react/edge/edge-stacks/utils/uniqueStatus.ts create mode 100644 pkg/libstack/compose/internal/composeplugin/status.go create mode 100644 pkg/libstack/compose/internal/composeplugin/status_integration_test.go create mode 100644 pkg/libstack/compose/internal/composeplugin/status_test_files/failed.yml create mode 100644 pkg/libstack/compose/internal/composeplugin/status_test_files/running.yml diff --git a/.gitignore b/.gitignore index 92778cd18..7239bdcf3 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,7 @@ storybook-static *.DS_Store .eslintcache -__debug_bin +__debug_bin* api/docs .idea diff --git a/api/dataservices/interface.go b/api/dataservices/interface.go index d1212451f..ecb84e15a 100644 --- a/api/dataservices/interface.go +++ b/api/dataservices/interface.go @@ -78,7 +78,6 @@ type ( EdgeStack(ID portainer.EdgeStackID) (*portainer.EdgeStack, error) EdgeStackVersion(ID portainer.EdgeStackID) (int, bool) Create(id portainer.EdgeStackID, edgeStack *portainer.EdgeStack) error - // Deprecated: Use UpdateEdgeStackFunc instead. UpdateEdgeStack(ID portainer.EdgeStackID, edgeStack *portainer.EdgeStack) error UpdateEdgeStackFunc(ID portainer.EdgeStackID, updateFunc func(edgeStack *portainer.EdgeStack)) error DeleteEdgeStack(ID portainer.EdgeStackID) error diff --git a/api/datastore/migrator/migrate_dbversion100.go b/api/datastore/migrator/migrate_dbversion100.go index 1c4b7ab68..cf280b984 100644 --- a/api/datastore/migrator/migrate_dbversion100.go +++ b/api/datastore/migrator/migrate_dbversion100.go @@ -2,8 +2,9 @@ package migrator import ( "os" + "time" - "github.com/portainer/portainer/api" + portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/chisel/crypto" "github.com/portainer/portainer/api/dataservices" "github.com/rs/zerolog/log" @@ -73,3 +74,77 @@ func (m *Migrator) convertSeedToPrivateKeyForDB100() error { } return err } + +func (m *Migrator) updateEdgeStackStatusForDB100() error { + log.Info().Msg("update edge stack status to have deployment steps") + + edgeStacks, err := m.edgeStackService.EdgeStacks() + if err != nil { + return err + } + + for _, edgeStack := range edgeStacks { + + for environmentID, environmentStatus := range edgeStack.Status { + // skip if status is already updated + if len(environmentStatus.Status) > 0 { + continue + } + + statusArray := []portainer.EdgeStackDeploymentStatus{} + if environmentStatus.Details.Pending { + statusArray = append(statusArray, portainer.EdgeStackDeploymentStatus{ + Type: portainer.EdgeStackStatusPending, + Time: time.Now().Unix(), + }) + } + + if environmentStatus.Details.Acknowledged { + statusArray = append(statusArray, portainer.EdgeStackDeploymentStatus{ + Type: portainer.EdgeStackStatusAcknowledged, + Time: time.Now().Unix(), + }) + } + + if environmentStatus.Details.Error { + statusArray = append(statusArray, portainer.EdgeStackDeploymentStatus{ + Type: portainer.EdgeStackStatusError, + Error: environmentStatus.Error, + Time: time.Now().Unix(), + }) + } + + if environmentStatus.Details.Ok { + statusArray = append(statusArray, portainer.EdgeStackDeploymentStatus{ + Type: portainer.EdgeStackStatusRunning, + Time: time.Now().Unix(), + }) + } + + if environmentStatus.Details.ImagesPulled { + statusArray = append(statusArray, portainer.EdgeStackDeploymentStatus{ + Type: portainer.EdgeStackStatusImagesPulled, + Time: time.Now().Unix(), + }) + } + + if environmentStatus.Details.Remove { + statusArray = append(statusArray, portainer.EdgeStackDeploymentStatus{ + Type: portainer.EdgeStackStatusRemoving, + Time: time.Now().Unix(), + }) + } + + environmentStatus.Status = statusArray + + edgeStack.Status[environmentID] = environmentStatus + } + + err = m.edgeStackService.UpdateEdgeStack(edgeStack.ID, &edgeStack) + if err != nil { + return err + } + } + + return nil +} diff --git a/api/datastore/migrator/migrate_dbversion80.go b/api/datastore/migrator/migrate_dbversion80.go index 24cbd4d9a..77671745e 100644 --- a/api/datastore/migrator/migrate_dbversion80.go +++ b/api/datastore/migrator/migrate_dbversion80.go @@ -78,13 +78,13 @@ func (m *Migrator) updateEdgeStackStatusForDB80() error { switch status.Type { case portainer.EdgeStackStatusPending: status.Details.Pending = true - case portainer.EdgeStackStatusOk: + case portainer.EdgeStackStatusDeploymentReceived: status.Details.Ok = true case portainer.EdgeStackStatusError: status.Details.Error = true case portainer.EdgeStackStatusAcknowledged: status.Details.Acknowledged = true - case portainer.EdgeStackStatusRemove: + case portainer.EdgeStackStatusRemoved: status.Details.Remove = true case portainer.EdgeStackStatusRemoteUpdateSuccess: status.Details.RemoteUpdateSuccess = true diff --git a/api/datastore/migrator/migrator.go b/api/datastore/migrator/migrator.go index be634ef68..a1e561175 100644 --- a/api/datastore/migrator/migrator.go +++ b/api/datastore/migrator/migrator.go @@ -215,6 +215,7 @@ func (m *Migrator) initMigrations() { m.addMigrations("2.19", m.convertSeedToPrivateKeyForDB100, m.migrateDockerDesktopExtentionSetting, + m.updateEdgeStackStatusForDB100, ) // Add new migrations below... diff --git a/api/datastore/test_data/output_24_to_latest.json b/api/datastore/test_data/output_24_to_latest.json index 9d0475f7e..5795cf905 100644 --- a/api/datastore/test_data/output_24_to_latest.json +++ b/api/datastore/test_data/output_24_to_latest.json @@ -944,6 +944,6 @@ } ], "version": { - "VERSION": "{\"SchemaVersion\":\"2.19.0\",\"MigratorCount\":2,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}" + "VERSION": "{\"SchemaVersion\":\"2.19.0\",\"MigratorCount\":3,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}" } } \ No newline at end of file diff --git a/api/http/handler/edgestacks/edgestack_status_delete.go b/api/http/handler/edgestacks/edgestack_status_delete.go index 113e96158..d9fb9f952 100644 --- a/api/http/handler/edgestacks/edgestack_status_delete.go +++ b/api/http/handler/edgestacks/edgestack_status_delete.go @@ -3,6 +3,7 @@ package edgestacks import ( "errors" "net/http" + "time" httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" @@ -25,6 +26,7 @@ import ( // @failure 400 // @failure 404 // @failure 403 +// @deprecated // @router /edge_stacks/{id}/status/{environmentId} [delete] func (handler *Handler) edgeStackStatusDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { stackID, err := request.RetrieveNumericRouteVariableValue(r, "id") @@ -69,7 +71,17 @@ func (handler *Handler) deleteEdgeStackStatus(tx dataservices.DataStoreTx, stack return nil, handler.handlerDBErr(err, "Unable to find a stack with the specified identifier inside the database") } - delete(stack.Status, endpoint.ID) + environmentStatus, ok := stack.Status[endpoint.ID] + if !ok { + environmentStatus = portainer.EdgeStackStatus{} + } + + environmentStatus.Status = append(environmentStatus.Status, portainer.EdgeStackDeploymentStatus{ + Time: time.Now().Unix(), + Type: portainer.EdgeStackStatusRemoved, + }) + + stack.Status[endpoint.ID] = environmentStatus err = tx.EdgeStack().UpdateEdgeStack(stack.ID, stack) if err != nil { diff --git a/api/http/handler/edgestacks/edgestack_status_update.go b/api/http/handler/edgestacks/edgestack_status_update.go index 3fe80bb07..98d070c6f 100644 --- a/api/http/handler/edgestacks/edgestack_status_update.go +++ b/api/http/handler/edgestacks/edgestack_status_update.go @@ -3,21 +3,24 @@ package edgestacks import ( "errors" "net/http" + "time" + httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" "github.com/portainer/libhttp/response" portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/dataservices" "github.com/portainer/portainer/pkg/featureflags" + "github.com/rs/zerolog/log" "github.com/asaskevich/govalidator" - httperror "github.com/portainer/libhttp/error" ) type updateStatusPayload struct { Error string Status *portainer.EdgeStackStatusType EndpointID portainer.EndpointID + Time int64 } func (payload *updateStatusPayload) Validate(r *http.Request) error { @@ -33,6 +36,10 @@ func (payload *updateStatusPayload) Validate(r *http.Request) error { return errors.New("error message is mandatory when status is error") } + if payload.Time == 0 { + payload.Time = time.Now().Unix() + } + return nil } @@ -43,6 +50,7 @@ func (payload *updateStatusPayload) Validate(r *http.Request) error { // @accept json // @produce json // @param id path int true "EdgeStack Id" +// @param body body updateStatusPayload true "EdgeStack status payload" // @success 200 {object} portainer.EdgeStack // @failure 500 // @failure 400 @@ -84,6 +92,21 @@ func (handler *Handler) edgeStackStatusUpdate(w http.ResponseWriter, r *http.Req } func (handler *Handler) updateEdgeStackStatus(tx dataservices.DataStoreTx, r *http.Request, stackID portainer.EdgeStackID, payload updateStatusPayload) (*portainer.EdgeStack, error) { + stack, err := tx.EdgeStack().EdgeStack(stackID) + if err != nil { + if dataservices.IsErrObjectNotFound(err) { + // skip error because agent tries to report on deleted stack + log.Warn(). + Err(err). + Int("stackID", int(stackID)). + Int("status", int(*payload.Status)). + Msg("Unable to find a stack inside the database, skipping error") + return nil, nil + } + + return nil, err + } + endpoint, err := tx.Endpoint().Endpoint(payload.EndpointID) if err != nil { return nil, handler.handlerDBErr(err, "Unable to find an environment with the specified identifier inside the database") @@ -94,67 +117,50 @@ func (handler *Handler) updateEdgeStackStatus(tx dataservices.DataStoreTx, r *ht return nil, httperror.Forbidden("Permission denied to access environment", err) } - var stack *portainer.EdgeStack + status := *payload.Status + + log.Debug(). + Int("stackID", int(stackID)). + Int("status", int(status)). + Msg("Updating stack status") + + deploymentStatus := portainer.EdgeStackDeploymentStatus{ + Type: status, + Error: payload.Error, + Time: payload.Time, + } if featureflags.IsEnabled(portainer.FeatureNoTx) { - err = tx.EdgeStack().UpdateEdgeStackFunc(portainer.EdgeStackID(stackID), func(edgeStack *portainer.EdgeStack) { - details := edgeStack.Status[payload.EndpointID].Details - details.Pending = false - - switch *payload.Status { - case portainer.EdgeStackStatusOk: - details.Ok = true - case portainer.EdgeStackStatusError: - details.Error = true - case portainer.EdgeStackStatusAcknowledged: - details.Acknowledged = true - case portainer.EdgeStackStatusRemove: - details.Remove = true - case portainer.EdgeStackStatusImagesPulled: - details.ImagesPulled = true - } - - edgeStack.Status[payload.EndpointID] = portainer.EdgeStackStatus{ - Details: details, - Error: payload.Error, - EndpointID: payload.EndpointID, - } + err = tx.EdgeStack().UpdateEdgeStackFunc(stackID, func(edgeStack *portainer.EdgeStack) { + updateEnvStatus(payload.EndpointID, edgeStack, deploymentStatus) stack = edgeStack }) - } else { - stack, err = tx.EdgeStack().EdgeStack(stackID) if err != nil { - return nil, err + return nil, handler.handlerDBErr(err, "Unable to persist the stack changes inside the database") } + } else { + updateEnvStatus(payload.EndpointID, stack, deploymentStatus) - details := stack.Status[payload.EndpointID].Details - details.Pending = false - - switch *payload.Status { - case portainer.EdgeStackStatusOk: - details.Ok = true - case portainer.EdgeStackStatusError: - details.Error = true - case portainer.EdgeStackStatusAcknowledged: - details.Acknowledged = true - case portainer.EdgeStackStatusRemove: - details.Remove = true - case portainer.EdgeStackStatusImagesPulled: - details.ImagesPulled = true + err = tx.EdgeStack().UpdateEdgeStack(stackID, stack) + if err != nil { + return nil, handler.handlerDBErr(err, "Unable to persist the stack changes inside the database") } + } - stack.Status[payload.EndpointID] = portainer.EdgeStackStatus{ - Details: details, - Error: payload.Error, - EndpointID: payload.EndpointID, - } + return stack, nil +} - err = tx.EdgeStack().UpdateEdgeStack(stackID, stack) - } - if err != nil { - return nil, handler.handlerDBErr(err, "Unable to persist the stack changes inside the database") +func updateEnvStatus(environmentId portainer.EndpointID, stack *portainer.EdgeStack, deploymentStatus portainer.EdgeStackDeploymentStatus) { + environmentStatus, ok := stack.Status[environmentId] + if !ok { + environmentStatus = portainer.EdgeStackStatus{ + EndpointID: environmentId, + Status: []portainer.EdgeStackDeploymentStatus{}, + } } - return stack, nil + environmentStatus.Status = append(environmentStatus.Status, deploymentStatus) + + stack.Status[environmentId] = environmentStatus } diff --git a/api/http/handler/edgestacks/edgestack_status_update_test.go b/api/http/handler/edgestacks/edgestack_status_update_test.go index 0e1f72a99..4ae63aa3f 100644 --- a/api/http/handler/edgestacks/edgestack_status_update_test.go +++ b/api/http/handler/edgestacks/edgestack_status_update_test.go @@ -59,23 +59,31 @@ func TestUpdateStatusAndInspect(t *testing.T) { t.Fatalf("expected a %d response, found: %d", http.StatusOK, rec.Code) } - data := portainer.EdgeStack{} - err = json.NewDecoder(rec.Body).Decode(&data) + updatedStack := portainer.EdgeStack{} + err = json.NewDecoder(rec.Body).Decode(&updatedStack) if err != nil { t.Fatal("error decoding response:", err) } - if !data.Status[endpoint.ID].Details.Error { - t.Fatalf("expected EdgeStackStatusType %d, found %t", payload.Status, data.Status[endpoint.ID].Details.Error) + endpointStatus, ok := updatedStack.Status[payload.EndpointID] + if !ok { + t.Fatal("Missing status") } - if data.Status[endpoint.ID].Error != payload.Error { - t.Fatalf("expected EdgeStackStatusError %s, found %s", payload.Error, data.Status[endpoint.ID].Error) + lastStatus := endpointStatus.Status[len(endpointStatus.Status)-1] + + if len(endpointStatus.Status) == len(edgeStack.Status[payload.EndpointID].Status) { + t.Fatal("expected status array to be updated") + } + + if lastStatus.Type != *payload.Status { + t.Fatalf("expected EdgeStackStatusType %d, found %d", *payload.Status, lastStatus.Type) } - if data.Status[endpoint.ID].EndpointID != payload.EndpointID { - t.Fatalf("expected EndpointID %d, found %d", payload.EndpointID, data.Status[endpoint.ID].EndpointID) + if endpointStatus.EndpointID != portainer.EndpointID(payload.EndpointID) { + t.Fatalf("expected EndpointID %d, found %d", payload.EndpointID, endpointStatus.EndpointID) } + } func TestUpdateStatusWithInvalidPayload(t *testing.T) { handler, _ := setupHandler(t) @@ -85,7 +93,7 @@ func TestUpdateStatusWithInvalidPayload(t *testing.T) { // Update edge stack status statusError := portainer.EdgeStackStatusError - statusOk := portainer.EdgeStackStatusOk + statusOk := portainer.EdgeStackStatusDeploymentReceived cases := []struct { Name string Payload updateStatusPayload diff --git a/api/http/handler/edgestacks/edgestack_test.go b/api/http/handler/edgestacks/edgestack_test.go index 75e25b18b..a2c619a31 100644 --- a/api/http/handler/edgestacks/edgestack_test.go +++ b/api/http/handler/edgestacks/edgestack_test.go @@ -1,6 +1,7 @@ package edgestacks import ( + "os" "strconv" "testing" "time" @@ -41,20 +42,22 @@ func setupHandler(t *testing.T) (*Handler, string) { t.Fatal(err) } - edgeStacksService := edgestacks.NewService(store) + tmpDir, err := os.MkdirTemp(t.TempDir(), "portainer-test") + if err != nil { + t.Fatal(err) + } + + fs, err := filesystem.NewService(tmpDir, "") + if err != nil { + t.Fatal(err) + } handler := NewHandler( security.NewRequestBouncer(store, jwtService, apiKeyService), store, - edgeStacksService, + edgestacks.NewService(store), ) - tmpDir := t.TempDir() - - fs, err := filesystem.NewService(tmpDir, "") - if err != nil { - t.Fatal(err) - } handler.FileService = fs settings, err := handler.DataStore.Settings().Settings() @@ -116,11 +119,9 @@ func createEdgeStack(t *testing.T, store dataservices.DataStore, endpointID port edgeStackID := portainer.EdgeStackID(14) edgeStack := portainer.EdgeStack{ - ID: edgeStackID, - Name: "test-edge-stack-" + strconv.Itoa(int(edgeStackID)), - Status: map[portainer.EndpointID]portainer.EdgeStackStatus{ - endpointID: {Details: portainer.EdgeStackStatusDetails{Ok: true}, Error: "", EndpointID: endpointID}, - }, + ID: edgeStackID, + Name: "test-edge-stack-" + strconv.Itoa(int(edgeStackID)), + Status: map[portainer.EndpointID]portainer.EdgeStackStatus{}, CreationDate: time.Now().Unix(), EdgeGroups: []portainer.EdgeGroupID{edgeGroup.ID}, ProjectPath: "/project/path", diff --git a/api/http/handler/edgestacks/edgestack_update.go b/api/http/handler/edgestacks/edgestack_update.go index 98a5c396b..e9488ac4a 100644 --- a/api/http/handler/edgestacks/edgestack_update.go +++ b/api/http/handler/edgestacks/edgestack_update.go @@ -2,7 +2,7 @@ package edgestacks import ( "net/http" - "strconv" + "time" "github.com/pkg/errors" httperror "github.com/portainer/libhttp/error" @@ -10,12 +10,9 @@ import ( "github.com/portainer/libhttp/response" portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/dataservices" - "github.com/portainer/portainer/api/filesystem" "github.com/portainer/portainer/api/internal/edge" "github.com/portainer/portainer/api/internal/set" "github.com/portainer/portainer/pkg/featureflags" - - "github.com/rs/zerolog/log" ) type updateEdgeStackPayload struct { @@ -116,24 +113,6 @@ func (handler *Handler) updateEdgeStack(tx dataservices.DataStoreTx, stackID por } - entryPoint := stack.EntryPoint - manifestPath := stack.ManifestPath - deploymentType := stack.DeploymentType - - if deploymentType != payload.DeploymentType { - // deployment type was changed - need to delete the old file - err = handler.FileService.RemoveDirectory(stack.ProjectPath) - if err != nil { - log.Warn().Err(err).Msg("Unable to clear old files") - } - - entryPoint = "" - manifestPath = "" - deploymentType = payload.DeploymentType - } - - stackFolder := strconv.Itoa(int(stack.ID)) - hasWrongType, err := hasWrongEnvironmentType(tx.Endpoint(), relatedEndpointIds, payload.DeploymentType) if err != nil { return nil, httperror.BadRequest("unable to check for existence of non fitting environments: %w", err) @@ -142,50 +121,20 @@ func (handler *Handler) updateEdgeStack(tx dataservices.DataStoreTx, stackID por return nil, httperror.BadRequest("edge stack with config do not match the environment type", nil) } - if payload.DeploymentType == portainer.EdgeStackDeploymentCompose { - if entryPoint == "" { - entryPoint = filesystem.ComposeFileDefaultName - } - - _, err := handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, entryPoint, []byte(payload.StackFileContent)) - if err != nil { - return nil, httperror.InternalServerError("Unable to persist updated Compose file on disk", err) - } - - tempManifestPath, err := handler.convertAndStoreKubeManifestIfNeeded(stackFolder, stack.ProjectPath, entryPoint, relatedEndpointIds) - if err != nil { - return nil, httperror.InternalServerError("Unable to convert and persist updated Kubernetes manifest file on disk", err) - } + stack.NumDeployments = len(relatedEndpointIds) - manifestPath = tempManifestPath - } + stack.UseManifestNamespaces = payload.UseManifestNamespaces - if deploymentType == portainer.EdgeStackDeploymentKubernetes { - if manifestPath == "" { - manifestPath = filesystem.ManifestFileDefaultName - } + stack.EdgeGroups = groupsIds - _, err = handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, manifestPath, []byte(payload.StackFileContent)) + if payload.UpdateVersion { + err := handler.updateStackVersion(stack, payload.DeploymentType, []byte(payload.StackFileContent), "", relatedEndpointIds) if err != nil { - return nil, httperror.InternalServerError("Unable to persist updated Kubernetes manifest file on disk", err) + return nil, httperror.InternalServerError("Unable to update stack version", err) } } - err = tx.EdgeStack().UpdateEdgeStackFunc(stack.ID, func(edgeStack *portainer.EdgeStack) { - edgeStack.NumDeployments = len(relatedEndpointIds) - if payload.UpdateVersion { - edgeStack.Status = make(map[portainer.EndpointID]portainer.EdgeStackStatus) - edgeStack.Version++ - } - - edgeStack.UseManifestNamespaces = payload.UseManifestNamespaces - - edgeStack.DeploymentType = deploymentType - edgeStack.EntryPoint = entryPoint - edgeStack.ManifestPath = manifestPath - - edgeStack.EdgeGroups = groupsIds - }) + err = tx.EdgeStack().UpdateEdgeStack(stack.ID, stack) if err != nil { return nil, httperror.InternalServerError("Unable to persist the stack changes inside the database", err) } @@ -246,3 +195,26 @@ func (handler *Handler) handleChangeEdgeGroups(tx dataservices.DataStoreTx, edge return newRelatedEnvironmentIDs, endpointsToAdd, nil } + +func newStatus(oldStatus map[portainer.EndpointID]portainer.EdgeStackStatus, relatedEnvironmentIds []portainer.EndpointID) map[portainer.EndpointID]portainer.EdgeStackStatus { + newStatus := make(map[portainer.EndpointID]portainer.EdgeStackStatus) + for _, endpointID := range relatedEnvironmentIds { + newEnvStatus := portainer.EdgeStackStatus{} + + oldEnvStatus, ok := oldStatus[endpointID] + if ok { + newEnvStatus = oldEnvStatus + } + + newEnvStatus.Status = []portainer.EdgeStackDeploymentStatus{ + { + Time: time.Now().Unix(), + Type: portainer.EdgeStackStatusPending, + }, + } + + newStatus[endpointID] = newEnvStatus + } + + return newStatus +} diff --git a/api/http/handler/edgestacks/edgestack_update_test.go b/api/http/handler/edgestacks/edgestack_update_test.go index 34b7359a1..90542178a 100644 --- a/api/http/handler/edgestacks/edgestack_update_test.go +++ b/api/http/handler/edgestacks/edgestack_update_test.go @@ -94,21 +94,21 @@ func TestUpdateAndInspect(t *testing.T) { t.Fatalf("expected a %d response, found: %d", http.StatusOK, rec.Code) } - data := portainer.EdgeStack{} - err = json.NewDecoder(rec.Body).Decode(&data) + updatedStack := portainer.EdgeStack{} + err = json.NewDecoder(rec.Body).Decode(&updatedStack) if err != nil { t.Fatal("error decoding response:", err) } - if payload.UpdateVersion && data.Version != edgeStack.Version+1 { - t.Fatalf("expected EdgeStackID %d, found %d", edgeStack.Version, data.Version) + if payload.UpdateVersion && updatedStack.Version != edgeStack.Version+1 { + t.Fatalf("expected EdgeStack version %d, found %d", edgeStack.Version+1, updatedStack.Version+1) } - if data.DeploymentType != payload.DeploymentType { - t.Fatalf("expected DeploymentType %d, found %d", edgeStack.DeploymentType, data.DeploymentType) + if updatedStack.DeploymentType != payload.DeploymentType { + t.Fatalf("expected DeploymentType %d, found %d", edgeStack.DeploymentType, updatedStack.DeploymentType) } - if !reflect.DeepEqual(data.EdgeGroups, payload.EdgeGroups) { + if !reflect.DeepEqual(updatedStack.EdgeGroups, payload.EdgeGroups) { t.Fatalf("expected EdgeGroups to be equal") } } diff --git a/api/http/handler/edgestacks/utils_update_stack_version.go b/api/http/handler/edgestacks/utils_update_stack_version.go new file mode 100644 index 000000000..e34b4d449 --- /dev/null +++ b/api/http/handler/edgestacks/utils_update_stack_version.go @@ -0,0 +1,59 @@ +package edgestacks + +import ( + "fmt" + "strconv" + + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/filesystem" + edgestackutils "github.com/portainer/portainer/api/internal/edge/edgestacks" + "github.com/rs/zerolog/log" +) + +func (handler *Handler) updateStackVersion(stack *portainer.EdgeStack, deploymentType portainer.EdgeStackDeploymentType, config []byte, oldGitHash string, relatedEnvironmentsIDs []portainer.EndpointID) error { + + stack.Version = stack.Version + 1 + stack.Status = edgestackutils.NewStatus(stack.Status, relatedEnvironmentsIDs) + + return handler.storeStackFile(stack, deploymentType, config) +} + +func (handler *Handler) storeStackFile(stack *portainer.EdgeStack, deploymentType portainer.EdgeStackDeploymentType, config []byte) error { + + if deploymentType != stack.DeploymentType { + // deployment type was changed - need to delete all old files + err := handler.FileService.RemoveDirectory(stack.ProjectPath) + if err != nil { + log.Warn().Err(err).Msg("Unable to clear old files") + } + + stack.EntryPoint = "" + stack.ManifestPath = "" + stack.DeploymentType = deploymentType + } + + stackFolder := strconv.Itoa(int(stack.ID)) + entryPoint := "" + if deploymentType == portainer.EdgeStackDeploymentCompose { + if stack.EntryPoint == "" { + stack.EntryPoint = filesystem.ComposeFileDefaultName + } + + entryPoint = stack.EntryPoint + } + + if deploymentType == portainer.EdgeStackDeploymentKubernetes { + if stack.ManifestPath == "" { + stack.ManifestPath = filesystem.ManifestFileDefaultName + } + + entryPoint = stack.ManifestPath + } + + _, err := handler.FileService.StoreEdgeStackFileFromBytesByVersion(stackFolder, entryPoint, stack.Version, config) + if err != nil { + return fmt.Errorf("unable to persist updated Compose file with version on disk: %w", err) + } + + return nil +} diff --git a/api/http/handler/endpointedge/endpointedge_status_inspect_test.go b/api/http/handler/endpointedge/endpointedge_status_inspect_test.go index 3c78e80e3..936261783 100644 --- a/api/http/handler/endpointedge/endpointedge_status_inspect_test.go +++ b/api/http/handler/endpointedge/endpointedge_status_inspect_test.go @@ -291,7 +291,7 @@ func TestEdgeStackStatus(t *testing.T) { ID: edgeStackID, Name: "test-edge-stack-17", Status: map[portainer.EndpointID]portainer.EdgeStackStatus{ - endpointID: {Details: portainer.EdgeStackStatusDetails{Ok: true}, Error: "", EndpointID: endpoint.ID}, + endpointID: {}, }, CreationDate: time.Now().Unix(), EdgeGroups: []portainer.EdgeGroupID{1, 2}, diff --git a/api/http/handler/endpoints/filter.go b/api/http/handler/endpoints/filter.go index bf38560f1..75b5360b7 100644 --- a/api/http/handler/endpoints/filter.go +++ b/api/http/handler/endpoints/filter.go @@ -17,18 +17,6 @@ import ( "github.com/portainer/portainer/api/internal/unique" ) -type EdgeStackStatusFilter string - -const ( - statusFilterPending EdgeStackStatusFilter = "Pending" - statusFilterOk EdgeStackStatusFilter = "Ok" - statusFilterError EdgeStackStatusFilter = "Error" - statusFilterAcknowledged EdgeStackStatusFilter = "Acknowledged" - statusFilterRemove EdgeStackStatusFilter = "Remove" - statusFilterRemoteUpdateSuccess EdgeStackStatusFilter = "RemoteUpdateSuccess" - statusFilterImagesPulled EdgeStackStatusFilter = "ImagesPulled" -) - type EnvironmentsQuery struct { search string types []portainer.EndpointType @@ -45,7 +33,7 @@ type EnvironmentsQuery struct { agentVersions []string edgeCheckInPassedSeconds int edgeStackId portainer.EdgeStackID - edgeStackStatus EdgeStackStatusFilter + edgeStackStatus *portainer.EdgeStackStatusType } func parseQuery(r *http.Request) (EnvironmentsQuery, error) { @@ -99,7 +87,18 @@ func parseQuery(r *http.Request) (EnvironmentsQuery, error) { edgeStackId, _ := request.RetrieveNumericQueryParameter(r, "edgeStackId", true) - edgeStackStatus, _ := request.RetrieveQueryParameter(r, "edgeStackStatus", true) + edgeStackStatusQuery, _ := request.RetrieveQueryParameter(r, "edgeStackStatus", true) + var edgeStackStatus *portainer.EdgeStackStatusType + if edgeStackStatusQuery != "" { + edgeStackStatusNumber, err := strconv.Atoi(edgeStackStatusQuery) + if err != nil || + edgeStackStatusNumber < 0 || + edgeStackStatusNumber > int(portainer.EdgeStackStatusRemoving) { + return EnvironmentsQuery{}, errors.New("invalid edgeStackStatus parameter") + } + + edgeStackStatus = ptr(portainer.EdgeStackStatusType(edgeStackStatusNumber)) + } return EnvironmentsQuery{ search: search, @@ -116,7 +115,7 @@ func parseQuery(r *http.Request) (EnvironmentsQuery, error) { agentVersions: agentVersions, edgeCheckInPassedSeconds: edgeCheckInPassedSeconds, edgeStackId: portainer.EdgeStackID(edgeStackId), - edgeStackStatus: EdgeStackStatusFilter(edgeStackStatus), + edgeStackStatus: edgeStackStatus, }, nil } @@ -213,30 +212,21 @@ func (handler *Handler) filterEndpointsByQuery(filteredEndpoints []portainer.End return filteredEndpoints, totalAvailableEndpoints, nil } -func endpointStatusInStackMatchesFilter(stackStatus map[portainer.EndpointID]portainer.EdgeStackStatus, envId portainer.EndpointID, statusFilter EdgeStackStatusFilter) bool { - status, ok := stackStatus[envId] +func endpointStatusInStackMatchesFilter(edgeStackStatus map[portainer.EndpointID]portainer.EdgeStackStatus, envId portainer.EndpointID, statusFilter portainer.EdgeStackStatusType) bool { + status, ok := edgeStackStatus[envId] // consider that if the env has no status in the stack it is in Pending state // workaround because Stack.Status[EnvId].Details.Pending is never set to True in the codebase - if !ok && statusFilter == statusFilterPending { + if !ok && statusFilter == portainer.EdgeStackStatusPending { return true } - valueMap := map[EdgeStackStatusFilter]bool{ - statusFilterPending: status.Details.Pending, - statusFilterOk: status.Details.Ok, - statusFilterError: status.Details.Error, - statusFilterAcknowledged: status.Details.Acknowledged, - statusFilterRemove: status.Details.Remove, - statusFilterRemoteUpdateSuccess: status.Details.RemoteUpdateSuccess, - statusFilterImagesPulled: status.Details.ImagesPulled, - } - - currentStatus, ok := valueMap[statusFilter] - return ok && currentStatus + return slices.ContainsFunc(status.Status, func(s portainer.EdgeStackDeploymentStatus) bool { + return s.Type == statusFilter + }) } -func filterEndpointsByEdgeStack(endpoints []portainer.Endpoint, edgeStackId portainer.EdgeStackID, statusFilter EdgeStackStatusFilter, datastore dataservices.DataStore) ([]portainer.Endpoint, error) { +func filterEndpointsByEdgeStack(endpoints []portainer.Endpoint, edgeStackId portainer.EdgeStackID, statusFilter *portainer.EdgeStackStatusType, datastore dataservices.DataStore) ([]portainer.Endpoint, error) { stack, err := datastore.EdgeStack().EdgeStack(edgeStackId) if err != nil { return nil, errors.WithMessage(err, "Unable to retrieve edge stack from the database") @@ -258,10 +248,10 @@ func filterEndpointsByEdgeStack(endpoints []portainer.Endpoint, edgeStackId port envIds = append(envIds, edgeGroup.Endpoints...) } - if statusFilter != "" { + if statusFilter != nil { n := 0 for _, envId := range envIds { - if endpointStatusInStackMatchesFilter(stack.Status, envId, statusFilter) { + if endpointStatusInStackMatchesFilter(stack.Status, envId, *statusFilter) { envIds[n] = envId n++ } diff --git a/api/http/handler/endpoints/utils.go b/api/http/handler/endpoints/utils.go index a00f36358..5a604713d 100644 --- a/api/http/handler/endpoints/utils.go +++ b/api/http/handler/endpoints/utils.go @@ -1,6 +1,7 @@ package endpoints +func ptr[T any](i T) *T { return &i } + func BoolAddr(b bool) *bool { - boolVar := b - return &boolVar + return ptr(b) } diff --git a/api/internal/edge/edgestacks/service.go b/api/internal/edge/edgestacks/service.go index 38e0b6ddd..0f067f248 100644 --- a/api/internal/edge/edgestacks/service.go +++ b/api/internal/edge/edgestacks/service.go @@ -49,7 +49,7 @@ func (service *Service) BuildEdgeStack( DeploymentType: deploymentType, CreationDate: time.Now().Unix(), EdgeGroups: edgeGroups, - Status: make(map[portainer.EndpointID]portainer.EdgeStackStatus), + Status: make(map[portainer.EndpointID]portainer.EdgeStackStatus, 0), Version: 1, UseManifestNamespaces: useManifestNamespaces, }, nil diff --git a/api/internal/edge/edgestacks/status.go b/api/internal/edge/edgestacks/status.go new file mode 100644 index 000000000..4d909c610 --- /dev/null +++ b/api/internal/edge/edgestacks/status.go @@ -0,0 +1,27 @@ +package edgestacks + +import ( + portainer "github.com/portainer/portainer/api" +) + +// NewStatus returns a new status object for an Edge stack +func NewStatus(oldStatus map[portainer.EndpointID]portainer.EdgeStackStatus, relatedEnvironmentIDs []portainer.EndpointID) map[portainer.EndpointID]portainer.EdgeStackStatus { + status := map[portainer.EndpointID]portainer.EdgeStackStatus{} + + for _, environmentID := range relatedEnvironmentIDs { + + newEnvStatus := portainer.EdgeStackStatus{ + Status: []portainer.EdgeStackDeploymentStatus{}, + EndpointID: portainer.EndpointID(environmentID), + } + + oldEnvStatus, ok := oldStatus[environmentID] + if ok { + newEnvStatus.DeploymentInfo = oldEnvStatus.DeploymentInfo + } + + status[environmentID] = newEnvStatus + } + + return status +} diff --git a/api/internal/slices/slices.go b/api/internal/slices/slices.go index a98505a3b..7eb486398 100644 --- a/api/internal/slices/slices.go +++ b/api/internal/slices/slices.go @@ -2,14 +2,33 @@ package slices // Contains is a generic function that returns true if the element is contained within the slice func Contains[T comparable](elems []T, v T) bool { + return ContainsFunc(elems, func(s T) bool { + return s == v + }) +} + +// Contains is a generic function that returns true if the element is contained within the slice +func ContainsFunc[T any](elems []T, f func(T) bool) bool { for _, s := range elems { - if v == s { + if f(s) { return true } } return false } +func Find[T any](elems []T, f func(T) bool) (T, bool) { + for _, s := range elems { + if f(s) { + return s, true + } + } + + // return default value + var result T + return result, false +} + // IndexFunc returns the first index i satisfying f(s[i]), // or -1 if none do. func IndexFunc[E any](s []E, f func(E) bool) int { diff --git a/api/portainer.go b/api/portainer.go index c342b5a9f..1c672f74d 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -310,15 +310,16 @@ type ( //EdgeStack represents an edge stack EdgeStack struct { // EdgeStack Identifier - ID EdgeStackID `json:"Id" example:"1"` - Name string `json:"Name"` - Status map[EndpointID]EdgeStackStatus `json:"Status"` - CreationDate int64 `json:"CreationDate"` - EdgeGroups []EdgeGroupID `json:"EdgeGroups"` - ProjectPath string `json:"ProjectPath"` - EntryPoint string `json:"EntryPoint"` - Version int `json:"Version"` - NumDeployments int `json:"NumDeployments"` + ID EdgeStackID `json:"Id" example:"1"` + Name string `json:"Name"` + Status map[EndpointID]EdgeStackStatus `json:"Status"` + // StatusArray map[EndpointID][]EdgeStackStatus `json:"StatusArray"` + CreationDate int64 `json:"CreationDate"` + EdgeGroups []EdgeGroupID `json:"EdgeGroups"` + ProjectPath string `json:"ProjectPath"` + EntryPoint string `json:"EntryPoint"` + Version int `json:"Version"` + NumDeployments int `json:"NumDeployments"` ManifestPath string DeploymentType EdgeStackDeploymentType // Uses the manifest's namespaces instead of the default one @@ -345,16 +346,26 @@ type ( //EdgeStackStatus represents an edge stack status EdgeStackStatus struct { - Details EdgeStackStatusDetails `json:"Details"` - Error string `json:"Error"` - EndpointID EndpointID `json:"EndpointID"` + Status []EdgeStackDeploymentStatus + EndpointID EndpointID // EE only feature - DeploymentInfo StackDeploymentInfo `json:"DeploymentInfo"` + DeploymentInfo StackDeploymentInfo + // Deprecated + Details EdgeStackStatusDetails + // Deprecated + Error string // Deprecated Type EdgeStackStatusType `json:"Type"` } + // EdgeStackDeploymentStatus represents an edge stack deployment status + EdgeStackDeploymentStatus struct { + Time int64 + Type EdgeStackStatusType + Error string + } + //EdgeStackStatusType represents an edge stack status type EdgeStackStatusType int @@ -1647,18 +1658,24 @@ const ( const ( // EdgeStackStatusPending represents a pending edge stack EdgeStackStatusPending EdgeStackStatusType = iota - //EdgeStackStatusOk represents a successfully deployed edge stack - EdgeStackStatusOk - //EdgeStackStatusError represents an edge environment(endpoint) which failed to deploy its edge stack + //EdgeStackStatusDeploymentReceived represents an edge environment which received the edge stack deployment + EdgeStackStatusDeploymentReceived + //EdgeStackStatusError represents an edge environment which failed to deploy its edge stack EdgeStackStatusError //EdgeStackStatusAcknowledged represents an acknowledged edge stack EdgeStackStatusAcknowledged - //EdgeStackStatusRemove represents a removed edge stack (status isn't persisted) - EdgeStackStatusRemove + //EdgeStackStatusRemoved represents a removed edge stack + EdgeStackStatusRemoved // StatusRemoteUpdateSuccess represents a successfully updated edge stack EdgeStackStatusRemoteUpdateSuccess // EdgeStackStatusImagesPulled represents a successfully images-pulling EdgeStackStatusImagesPulled + // EdgeStackStatusRunning represents a running Edge stack + EdgeStackStatusRunning + // EdgeStackStatusDeploying represents an Edge stack which is being deployed + EdgeStackStatusDeploying + // EdgeStackStatusRemoving represents an Edge stack which is being removed + EdgeStackStatusRemoving ) const ( diff --git a/app/assets/ico/icon_up-to-date.svg b/app/assets/ico/icon_up-to-date.svg new file mode 100644 index 000000000..82fc74742 --- /dev/null +++ b/app/assets/ico/icon_up-to-date.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/app/assets/ico/icon_updates-available.svg b/app/assets/ico/icon_updates-available.svg new file mode 100644 index 000000000..2b653c2b7 --- /dev/null +++ b/app/assets/ico/icon_updates-available.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/app/edge/react/components/index.ts b/app/edge/react/components/index.ts index a40b04ea6..ff75ae464 100644 --- a/app/edge/react/components/index.ts +++ b/app/edge/react/components/index.ts @@ -13,9 +13,11 @@ import { withUIRouter } from '@/react-tools/withUIRouter'; import { EdgeGroupAssociationTable } from '@/react/edge/components/EdgeGroupAssociationTable'; import { AssociatedEdgeEnvironmentsSelector } from '@/react/edge/components/AssociatedEdgeEnvironmentsSelector'; import { EnvironmentsDatatable } from '@/react/edge/edge-stacks/ItemView/EnvironmentsDatatable'; +import { EdgeStackStatus } from '@/react/edge/edge-stacks/ListView/EdgeStackStatus'; export const componentsModule = angular .module('portainer.edge.react.components', []) + .component('edgeStacksDatatableStatus', r2a(EdgeStackStatus, ['edgeStack'])) .component( 'edgeStackEnvironmentsDatatable', r2a(withUIRouter(withReactQuery(EnvironmentsDatatable)), []) diff --git a/app/portainer/components/datatables/genericDatatableController.js b/app/portainer/components/datatables/genericDatatableController.js index 6c3c1a80c..fd6f6c5b6 100644 --- a/app/portainer/components/datatables/genericDatatableController.js +++ b/app/portainer/components/datatables/genericDatatableController.js @@ -1,6 +1,7 @@ import _ from 'lodash-es'; import './datatable.css'; import { ResourceControlOwnership as RCO } from '@/react/portainer/access-control/types'; +import { isBE } from '@/react/portainer/feature-flags/feature-flags.service'; function isBetween(value, a, b) { return (value >= a && value <= b) || (value >= b && value <= a); @@ -14,6 +15,7 @@ angular.module('portainer.app').controller('GenericDatatableController', [ 'PAGINATION_MAX_ITEMS', function ($interval, PaginationService, DatatableService, PAGINATION_MAX_ITEMS) { this.RCO = RCO; + this.isBE = isBE; this.state = { selectAll: false, diff --git a/app/portainer/react/components/index.ts b/app/portainer/react/components/index.ts index 1ed5e9e86..45644ebce 100644 --- a/app/portainer/react/components/index.ts +++ b/app/portainer/react/components/index.ts @@ -120,7 +120,7 @@ export const ngModule = angular 'fallbackImage', r2a(FallbackImage, ['src', 'fallbackIcon', 'alt', 'size', 'className']) ) - .component('prIcon', r2a(Icon, ['className', 'icon', 'mode', 'size'])) + .component('prIcon', r2a(Icon, ['className', 'icon', 'mode', 'size', 'spin'])) .component('reactQueryDevTools', r2a(ReactQueryDevtoolsWrapper, [])) .component( 'dashboardItem', diff --git a/app/react/components/Icon.tsx b/app/react/components/Icon.tsx index a9794b00c..dd762d3e3 100644 --- a/app/react/components/Icon.tsx +++ b/app/react/components/Icon.tsx @@ -29,15 +29,15 @@ interface Props { className?: string; size?: IconSize; mode?: IconMode; + spin?: boolean; } -export function Icon({ icon, className, mode, size }: Props) { - const classes = clsx( - className, - 'icon inline-flex', - { [`icon-${mode}`]: mode }, - { [`icon-${size}`]: size } - ); +export function Icon({ icon, className, mode, size, spin }: Props) { + const classes = clsx(className, 'icon inline-flex', { + [`icon-${mode}`]: mode, + [`icon-${size}`]: size, + 'animate-spin-slow': spin, + }); if (typeof icon !== 'string') { const Icon = isValidElementType(icon) ? icon : null; diff --git a/app/react/edge/edge-stacks/ItemView/EnvironmentsDatatable/EnvironmentsDatatable.tsx b/app/react/edge/edge-stacks/ItemView/EnvironmentsDatatable/EnvironmentsDatatable.tsx index 475efa1a6..b42f4caa3 100644 --- a/app/react/edge/edge-stacks/ItemView/EnvironmentsDatatable/EnvironmentsDatatable.tsx +++ b/app/react/edge/edge-stacks/ItemView/EnvironmentsDatatable/EnvironmentsDatatable.tsx @@ -6,6 +6,7 @@ import { EdgeStackStatus, StatusType } from '@/react/edge/edge-stacks/types'; import { useEnvironmentList } from '@/react/portainer/environments/queries'; import { isBE } from '@/react/portainer/feature-flags/feature-flags.service'; import { useParamState } from '@/react/hooks/useParamState'; +import { EnvironmentId } from '@/react/portainer/environments/types'; import { Datatable } from '@@/datatables'; import { useTableStateWithoutStorage } from '@@/datatables/useTableState'; @@ -20,17 +21,29 @@ export function EnvironmentsDatatable() { const { params: { stackId }, } = useCurrentStateAndParams(); - const edgeStackQuery = useEdgeStack(stackId); + const edgeStackQuery = useEdgeStack(stackId, { + refetchInterval(data) { + if (!data) { + return 0; + } + + return Object.values(data.Status).some((status) => + status.Status.every((s) => s.Type === StatusType.Running) + ) + ? 0 + : 10000; + }, + }); const [page, setPage] = useState(0); const [statusFilter, setStatusFilter] = useParamState( 'status', - parseStatusFilter + (value) => (value ? parseInt(value, 10) : undefined) ); const tableState = useTableStateWithoutStorage('name'); const endpointsQuery = useEnvironmentList({ pageLimit: tableState.pageSize, - page, + page: page + 1, search: tableState.search, sort: tableState.sortBy.id as 'Group' | 'Name', order: tableState.sortBy.desc ? 'desc' : 'asc', @@ -38,27 +51,32 @@ export function EnvironmentsDatatable() { edgeStackStatus: statusFilter, }); + const currentFileVersion = + edgeStackQuery.data?.StackFileVersion?.toString() || ''; + const gitConfigURL = edgeStackQuery.data?.GitConfig?.URL || ''; + const gitConfigCommitHash = edgeStackQuery.data?.GitConfig?.ConfigHash || ''; const environments: Array = useMemo( () => - endpointsQuery.environments.map((env) => ({ - ...env, - StackStatus: - edgeStackQuery.data?.Status[env.Id] || + endpointsQuery.environments.map( + (env) => ({ - Details: { - Pending: true, - Acknowledged: false, - ImagesPulled: false, - Error: false, - Ok: false, - RemoteUpdateSuccess: false, - Remove: false, - }, - EndpointID: env.Id, - Error: '', - } satisfies EdgeStackStatus), - })), - [edgeStackQuery.data?.Status, endpointsQuery.environments] + ...env, + TargetFileVersion: currentFileVersion, + GitConfigURL: gitConfigURL, + TargetCommitHash: gitConfigCommitHash, + StackStatus: getEnvStackStatus( + env.Id, + edgeStackQuery.data?.Status[env.Id] + ), + } satisfies EdgeStackEnvironment) + ), + [ + currentFileVersion, + edgeStackQuery.data?.Status, + endpointsQuery.environments, + gitConfigCommitHash, + gitConfigURL, + ] ); return ( @@ -81,11 +99,11 @@ export function EnvironmentsDatatable() { value={statusFilter} onChange={(e) => setStatusFilter(e || undefined)} options={[ - { value: 'Pending', label: 'Pending' }, - { value: 'Acknowledged', label: 'Acknowledged' }, - { value: 'ImagesPulled', label: 'Images pre-pulled' }, - { value: 'Ok', label: 'Deployed' }, - { value: 'Error', label: 'Failed' }, + { value: StatusType.Pending, label: 'Pending' }, + { value: StatusType.Acknowledged, label: 'Acknowledged' }, + { value: StatusType.ImagesPulled, label: 'Images pre-pulled' }, + { value: StatusType.Running, label: 'Deployed' }, + { value: StatusType.Error, label: 'Failed' }, ]} /> @@ -95,19 +113,31 @@ export function EnvironmentsDatatable() { ); } -function parseStatusFilter(status: string | undefined): StatusType | undefined { - switch (status) { - case 'Pending': - return 'Pending'; - case 'Acknowledged': - return 'Acknowledged'; - case 'ImagesPulled': - return 'ImagesPulled'; - case 'Ok': - return 'Ok'; - case 'Error': - return 'Error'; - default: - return undefined; +function getEnvStackStatus( + envId: EnvironmentId, + envStatus: EdgeStackStatus | undefined +) { + const pendingStatus = { + Type: StatusType.Pending, + Error: '', + Time: new Date().valueOf() / 1000, + }; + + let status = envStatus; + if (!status) { + status = { + EndpointID: envId, + DeploymentInfo: { + ConfigHash: '', + FileVersion: 0, + }, + Status: [], + }; } + + if (status.Status.length === 0) { + status.Status.push(pendingStatus); + } + + return status; } diff --git a/app/react/edge/edge-stacks/ItemView/EnvironmentsDatatable/columns.tsx b/app/react/edge/edge-stacks/ItemView/EnvironmentsDatatable/columns.tsx index ffb168adc..0177c2e94 100644 --- a/app/react/edge/edge-stacks/ItemView/EnvironmentsDatatable/columns.tsx +++ b/app/react/edge/edge-stacks/ItemView/EnvironmentsDatatable/columns.tsx @@ -2,13 +2,19 @@ import { CellContext, createColumnHelper } from '@tanstack/react-table'; import { ChevronDown, ChevronRight } from 'lucide-react'; import clsx from 'clsx'; import { useState } from 'react'; +import _ from 'lodash'; +import UpdatesAvailable from '@/assets/ico/icon_updates-available.svg?c'; +import UpToDate from '@/assets/ico/icon_up-to-date.svg?c'; +import { isoDateFromTimestamp } from '@/portainer/filters/filters'; import { isBE } from '@/react/portainer/feature-flags/feature-flags.service'; +import { getDashboardRoute } from '@/react/portainer/environments/utils'; import { Button } from '@@/buttons'; import { Icon } from '@@/Icon'; +import { Link } from '@@/Link'; -import { EdgeStackStatus } from '../../types'; +import { DeploymentStatus, EdgeStackStatus, StatusType } from '../../types'; import { EnvironmentActions } from './EnvironmentActions'; import { ActionStatus } from './ActionStatus'; @@ -16,20 +22,75 @@ import { EdgeStackEnvironment } from './types'; const columnHelper = createColumnHelper(); -export const columns = [ +export const columns = _.compact([ columnHelper.accessor('Name', { id: 'name', header: 'Name', + cell({ row: { original: env } }) { + const { to, params } = getDashboardRoute(env); + return ( + + {env.Name} + + ); + }, }), - columnHelper.accessor((env) => endpointStatusLabel(env.StackStatus), { + columnHelper.accessor((env) => endpointStatusLabel(env.StackStatus.Status), { id: 'status', header: 'Status', + cell({ row: { original: env } }) { + return ( +
    + {env.StackStatus.Status.map((s) => ( +
  • + +
    • + ))} +
+ ); + }, }), - columnHelper.accessor((env) => env.StackStatus.Error, { - id: 'error', - header: 'Error', - cell: ErrorCell, + columnHelper.accessor((env) => _.last(env.StackStatus.Status)?.Time, { + id: 'statusDate', + header: 'Time', + cell({ row: { original: env } }) { + return ( +
    + {env.StackStatus.Status.map((s) => ( +
  • + {isoDateFromTimestamp(s.Time)} +
    • + ))} +
+ ); + }, }), + ...(isBE + ? [ + columnHelper.accessor((env) => endpointTargetVersionLabel(env), { + id: 'targetVersion', + header: 'Target version', + cell: TargetVersionCell, + }), + columnHelper.accessor( + (env) => endpointDeployedVersionLabel(env.StackStatus), + { + id: 'deployedVersion', + header: 'Deployed version', + cell: DeployedVersionCell, + } + ), + ] + : []), + columnHelper.accessor( + (env) => + env.StackStatus.Status.find((s) => s.Type === StatusType.Error)?.Error, + { + id: 'error', + header: 'Error', + cell: ErrorCell, + } + ), ...(isBE ? [ columnHelper.display({ @@ -48,7 +109,7 @@ export const columns = [ }), ] : []), -]; +]); function ErrorCell({ getValue }: CellContext) { const [isExpanded, setIsExpanded] = useState(false); @@ -77,30 +138,151 @@ function ErrorCell({ getValue }: CellContext) { ); } -function endpointStatusLabel(status: EdgeStackStatus) { - const details = (status && status.Details) || {}; - +function endpointStatusLabel(statusArray: Array) { const labels = []; - if (details.Acknowledged) { - labels.push('Acknowledged'); + statusArray.forEach((status) => { + if (status.Type === StatusType.Acknowledged) { + labels.push('Acknowledged'); + } + if (status.Type === StatusType.ImagesPulled) { + labels.push('Images pre-pulled'); + } + if (status.Type === StatusType.Running) { + labels.push('Deployed'); + } + if (status.Type === StatusType.Error) { + labels.push('Failed'); + } + }); + + if (!labels.length) { + labels.push('Pending'); } - if (details.ImagesPulled) { - labels.push('Images pre-pulled'); + return _.uniq(labels).join(', '); +} + +function TargetVersionCell({ + row, + getValue, +}: CellContext) { + const value = getValue(); + if (!value) { + return ''; } - if (details.Ok) { - labels.push('Deployed'); + return ( + <> + {row.original.TargetCommitHash ? ( + + ) : ( +
{value}
+ )} + + ); +} + +function endpointTargetVersionLabel(env: EdgeStackEnvironment) { + if (env.TargetCommitHash) { + return env.TargetCommitHash.slice(0, 7).toString(); } + return env.TargetFileVersion.toString() || ''; +} - if (details.Error) { - labels.push('Failed'); +function DeployedVersionCell({ + row, + getValue, +}: CellContext) { + const value = getValue(); + if (!value || value === '0') { + return ( +
+ +
+ ); } - if (!labels.length) { - labels.push('Pending'); + let statusIcon = ; + if ( + (row.original.TargetCommitHash && + row.original.TargetCommitHash.slice(0, 7) !== value) || + (!row.original.TargetCommitHash && row.original.TargetFileVersion !== value) + ) { + statusIcon = ; + } + + return ( + <> + {row.original.TargetCommitHash ? ( +
+ {statusIcon} + + {value} + +
+ ) : ( +
+ {statusIcon} + {value} +
+ )} + + ); +} + +function endpointDeployedVersionLabel(status: EdgeStackStatus) { + if (status.DeploymentInfo?.ConfigHash) { + return status.DeploymentInfo?.ConfigHash.slice(0, 7).toString(); } + return status.DeploymentInfo?.FileVersion.toString() || ''; +} + +function Status({ value }: { value: StatusType }) { + const color = getStateColor(value); + + return ( +
+ - return labels.join(', '); + {_.startCase(StatusType[value])} +
+ ); +} + +function getStateColor(type: StatusType): 'orange' | 'green' | 'red' { + switch (type) { + case StatusType.Acknowledged: + case StatusType.ImagesPulled: + case StatusType.DeploymentReceived: + case StatusType.Running: + case StatusType.RemoteUpdateSuccess: + case StatusType.Removed: + return 'green'; + case StatusType.Error: + return 'red'; + case StatusType.Pending: + case StatusType.Deploying: + case StatusType.Removing: + default: + return 'orange'; + } } diff --git a/app/react/edge/edge-stacks/ItemView/EnvironmentsDatatable/types.ts b/app/react/edge/edge-stacks/ItemView/EnvironmentsDatatable/types.ts index 65597781a..b56a2b116 100644 --- a/app/react/edge/edge-stacks/ItemView/EnvironmentsDatatable/types.ts +++ b/app/react/edge/edge-stacks/ItemView/EnvironmentsDatatable/types.ts @@ -4,4 +4,7 @@ import { EdgeStackStatus } from '../../types'; export type EdgeStackEnvironment = Environment & { StackStatus: EdgeStackStatus; + TargetFileVersion: string; + GitConfigURL: string; + TargetCommitHash: string; }; diff --git a/app/react/edge/edge-stacks/ListView/EdgeStackStatus.tsx b/app/react/edge/edge-stacks/ListView/EdgeStackStatus.tsx new file mode 100644 index 000000000..803584405 --- /dev/null +++ b/app/react/edge/edge-stacks/ListView/EdgeStackStatus.tsx @@ -0,0 +1,87 @@ +import _ from 'lodash'; +import { + AlertTriangle, + CheckCircle, + type Icon as IconType, + Loader2, + XCircle, +} from 'lucide-react'; + +import { Icon, IconMode } from '@@/Icon'; + +import { DeploymentStatus, EdgeStack, StatusType } from '../types'; + +export function EdgeStackStatus({ edgeStack }: { edgeStack: EdgeStack }) { + const status = Object.values(edgeStack.Status); + const lastStatus = _.compact(status.map((s) => _.last(s.Status))); + + const { icon, label, mode, spin } = getStatus( + edgeStack.NumDeployments, + lastStatus + ); + + return ( +
+ {icon && } + {label} +
+ ); +} + +function getStatus( + numDeployments: number, + envStatus: Array +): { + label: string; + icon?: IconType; + spin?: boolean; + mode?: IconMode; +} { + if (envStatus.length < numDeployments) { + return { + label: 'Deploying', + icon: Loader2, + spin: true, + mode: 'primary', + }; + } + + const allFailed = envStatus.every((s) => s.Type === StatusType.Error); + + if (allFailed) { + return { + label: 'Failed', + icon: XCircle, + mode: 'danger', + }; + } + + const allRunning = envStatus.every((s) => s.Type === StatusType.Running); + + if (allRunning) { + return { + label: 'Running', + icon: CheckCircle, + mode: 'success', + }; + } + + const hasDeploying = envStatus.some((s) => s.Type === StatusType.Deploying); + const hasRunning = envStatus.some((s) => s.Type === StatusType.Running); + const hasFailed = envStatus.some((s) => s.Type === StatusType.Error); + + if (hasRunning && hasFailed && !hasDeploying) { + return { + label: 'Partially Running', + icon: AlertTriangle, + mode: 'warning', + }; + } + + return { + label: 'Deploying', + icon: Loader2, + spin: true, + mode: 'primary', + }; +} diff --git a/app/react/edge/edge-stacks/ListView/EdgeStacksDatatable/DeploymentCounter.tsx b/app/react/edge/edge-stacks/ListView/EdgeStacksDatatable/DeploymentCounter.tsx index 1618fd673..ec25cb777 100644 --- a/app/react/edge/edge-stacks/ListView/EdgeStacksDatatable/DeploymentCounter.tsx +++ b/app/react/edge/edge-stacks/ListView/EdgeStacksDatatable/DeploymentCounter.tsx @@ -38,10 +38,10 @@ export function DeploymentCounter({ return ( diff --git a/app/react/edge/edge-stacks/ListView/EdgeStacksDatatable/EdgeStacksDatatable.tsx b/app/react/edge/edge-stacks/ListView/EdgeStacksDatatable/EdgeStacksDatatable.tsx index 1bca53d4f..2e908a45f 100644 --- a/app/react/edge/edge-stacks/ListView/EdgeStacksDatatable/EdgeStacksDatatable.tsx +++ b/app/react/edge/edge-stacks/ListView/EdgeStacksDatatable/EdgeStacksDatatable.tsx @@ -4,7 +4,7 @@ import { Datatable } from '@@/datatables'; import { useTableState } from '@@/datatables/useTableState'; import { useEdgeStacks } from '../../queries/useEdgeStacks'; -import { EdgeStack } from '../../types'; +import { EdgeStack, StatusType } from '../../types'; import { createStore } from './store'; import { columns } from './columns'; @@ -51,11 +51,16 @@ export function EdgeStacksDatatable() { function aggregateStackStatus(stackStatus: EdgeStack['Status']) { const aggregateStatus = { ok: 0, error: 0, acknowledged: 0, imagesPulled: 0 }; - return Object.values(stackStatus).reduce((acc, envStatus) => { - acc.ok += Number(envStatus.Details.Ok); - acc.error += Number(envStatus.Details.Error); - acc.acknowledged += Number(envStatus.Details.Acknowledged); - acc.imagesPulled += Number(envStatus.Details.ImagesPulled); - return acc; - }, aggregateStatus); + return Object.values(stackStatus).reduce( + (acc, envStatus) => + envStatus.Status.reduce((acc, status) => { + const { Type } = status; + acc.ok += Number(Type === StatusType.Running); + acc.error += Number(Type === StatusType.Error); + acc.acknowledged += Number(Type === StatusType.Acknowledged); + acc.imagesPulled += Number(Type === StatusType.ImagesPulled); + return acc; + }, acc), + aggregateStatus + ); } diff --git a/app/react/edge/edge-stacks/ListView/EdgeStacksDatatable/EdgeStacksStatus.tsx b/app/react/edge/edge-stacks/ListView/EdgeStacksDatatable/EdgeStacksStatus.tsx new file mode 100644 index 000000000..99ab8ee23 --- /dev/null +++ b/app/react/edge/edge-stacks/ListView/EdgeStacksDatatable/EdgeStacksStatus.tsx @@ -0,0 +1,87 @@ +import _ from 'lodash'; +import { + AlertTriangle, + CheckCircle, + type Icon as IconType, + Loader2, + XCircle, +} from 'lucide-react'; + +import { Icon, IconMode } from '@@/Icon'; + +import { DeploymentStatus, EdgeStack, StatusType } from '../../types'; + +export function EdgeStackStatus({ edgeStack }: { edgeStack: EdgeStack }) { + const status = Object.values(edgeStack.Status); + const lastStatus = _.compact(status.map((s) => _.last(s.Status))); + + const { icon, label, mode, spin } = getStatus( + edgeStack.NumDeployments, + lastStatus + ); + + return ( +
+ {icon && } + {label} +
+ ); +} + +function getStatus( + numDeployments: number, + envStatus: Array +): { + label: string; + icon?: IconType; + spin?: boolean; + mode?: IconMode; +} { + if (envStatus.length < numDeployments) { + return { + label: 'Deploying', + icon: Loader2, + spin: true, + mode: 'primary', + }; + } + + const allFailed = envStatus.every((s) => s.Type === StatusType.Error); + + if (allFailed) { + return { + label: 'Failed', + icon: XCircle, + mode: 'danger', + }; + } + + const allRunning = envStatus.every((s) => s.Type === StatusType.Running); + + if (allRunning) { + return { + label: 'Running', + icon: CheckCircle, + mode: 'success', + }; + } + + const hasDeploying = envStatus.some((s) => s.Type === StatusType.Deploying); + const hasRunning = envStatus.some((s) => s.Type === StatusType.Running); + const hasFailed = envStatus.some((s) => s.Type === StatusType.Error); + + if (hasRunning && hasFailed && !hasDeploying) { + return { + label: 'Partially Running', + icon: AlertTriangle, + mode: 'warning', + }; + } + + return { + label: 'Deploying', + icon: Loader2, + spin: true, + mode: 'primary', + }; +} diff --git a/app/react/edge/edge-stacks/ListView/EdgeStacksDatatable/columns.tsx b/app/react/edge/edge-stacks/ListView/EdgeStacksDatatable/columns.tsx index 859c515f4..041076dbe 100644 --- a/app/react/edge/edge-stacks/ListView/EdgeStacksDatatable/columns.tsx +++ b/app/react/edge/edge-stacks/ListView/EdgeStacksDatatable/columns.tsx @@ -6,6 +6,9 @@ import { isBE } from '@/react/portainer/feature-flags/feature-flags.service'; import { buildNameColumn } from '@@/datatables/NameCell'; +import { StatusType } from '../../types'; +import { EdgeStackStatus } from '../EdgeStackStatus'; + import { DecoratedEdgeStack } from './types'; import { DeploymentCounter, DeploymentCounterLink } from './DeploymentCounter'; @@ -25,7 +28,7 @@ export const columns = _.compact([ cell: ({ getValue, row }) => ( ), @@ -39,7 +42,7 @@ export const columns = _.compact([ cell: ({ getValue, row }) => ( ), @@ -54,7 +57,7 @@ export const columns = _.compact([ cell: ({ getValue, row }) => ( ), @@ -69,7 +72,7 @@ export const columns = _.compact([ cell: ({ getValue, row }) => ( ), @@ -79,6 +82,19 @@ export const columns = _.compact([ className: '[&>*]:justify-center', }, }), + columnHelper.accessor('Status', { + header: 'Status', + cell: ({ row }) => ( +
+ +
+ ), + enableSorting: false, + enableHiding: false, + meta: { + className: '[&>*]:justify-center', + }, + }), columnHelper.accessor('NumDeployments', { header: 'Deployments', cell: ({ getValue }) => ( diff --git a/app/react/edge/edge-stacks/queries/useEdgeStack.ts b/app/react/edge/edge-stacks/queries/useEdgeStack.ts index 6aabfb26e..53ef6660c 100644 --- a/app/react/edge/edge-stacks/queries/useEdgeStack.ts +++ b/app/react/edge/edge-stacks/queries/useEdgeStack.ts @@ -8,10 +8,24 @@ import { EdgeStack } from '../types'; import { buildUrl } from './buildUrl'; import { queryKeys } from './query-keys'; -export function useEdgeStack(id?: EdgeStack['Id']) { +export function useEdgeStack( + id?: EdgeStack['Id'], + { + refetchInterval, + }: { + /** + * If set to a number, the query will continuously refetch at this frequency in milliseconds. If set to a function, the function will be executed with the latest data and query to compute a frequency Defaults to false. + */ + refetchInterval?: + | number + | false + | ((data?: Awaited>) => false | number); + } = {} +) { return useQuery(id ? queryKeys.item(id) : [], () => getEdgeStack(id), { ...withError('Failed loading Edge stack'), enabled: !!id, + refetchInterval, }); } diff --git a/app/react/edge/edge-stacks/queries/useEdgeStacks.ts b/app/react/edge/edge-stacks/queries/useEdgeStacks.ts index f420febd1..e03da2b15 100644 --- a/app/react/edge/edge-stacks/queries/useEdgeStacks.ts +++ b/app/react/edge/edge-stacks/queries/useEdgeStacks.ts @@ -6,6 +6,7 @@ import axios, { parseAxiosError } from '@/portainer/services/axios'; import { EdgeStack } from '../types'; import { buildUrl } from './buildUrl'; +import { queryKeys } from './query-keys'; export function useEdgeStacks>({ select, @@ -19,7 +20,7 @@ export function useEdgeStacks>({ select?: (stacks: EdgeStack[]) => T; refetchInterval?: number | false | ((data?: T) => false | number); } = {}) { - return useQuery(['edge_stacks'], () => getEdgeStacks(), { + return useQuery(queryKeys.base(), () => getEdgeStacks(), { ...withError('Failed loading Edge stack'), select, refetchInterval, diff --git a/app/react/edge/edge-stacks/types.ts b/app/react/edge/edge-stacks/types.ts index 9ffe137e4..42319d341 100644 --- a/app/react/edge/edge-stacks/types.ts +++ b/app/react/edge/edge-stacks/types.ts @@ -9,22 +9,44 @@ import { EnvVar } from '@@/form-components/EnvironmentVariablesFieldset/types'; import { EdgeGroup } from '../edge-groups/types'; -interface EdgeStackStatusDetails { - Pending: boolean; - Ok: boolean; - Error: boolean; - Acknowledged: boolean; - Remove: boolean; - RemoteUpdateSuccess: boolean; - ImagesPulled: boolean; +export enum StatusType { + /** Pending represents a pending edge stack */ + Pending, + /** DeploymentReceived represents an edge environment which received the edge stack deployment */ + DeploymentReceived, + /** Error represents an edge environment which failed to deploy its edge stack */ + Error, + /** Acknowledged represents an acknowledged edge stack */ + Acknowledged, + /** Removed represents a removed edge stack */ + Removed, + /** StatusRemoteUpdateSuccess represents a successfully updated edge stack */ + RemoteUpdateSuccess, + /** ImagesPulled represents a successfully images-pulling */ + ImagesPulled, + /** Running represents a running Edge stack */ + Running, + /** Deploying represents an Edge stack which is being deployed */ + Deploying, + /** Removing represents an Edge stack which is being removed */ + Removing, } -export type StatusType = keyof EdgeStackStatusDetails; +export interface DeploymentStatus { + Type: StatusType; + Error: string; + Time: number; +} + +interface EdgeStackDeploymentInfo { + FileVersion: number; + ConfigHash: string; +} export interface EdgeStackStatus { - Details: EdgeStackStatusDetails; - Error: string; + Status: Array; EndpointID: EnvironmentId; + DeploymentInfo?: EdgeStackDeploymentInfo; } export enum DeploymentType { diff --git a/app/react/edge/edge-stacks/utils/uniqueStatus.ts b/app/react/edge/edge-stacks/utils/uniqueStatus.ts new file mode 100644 index 000000000..b6d8fc2ae --- /dev/null +++ b/app/react/edge/edge-stacks/utils/uniqueStatus.ts @@ -0,0 +1,16 @@ +import { DeploymentStatus } from '../types'; + +/** + * returns the latest status object of each type + */ +export function uniqueStatus(statusArray: Array = []) { + // keep only the last status object of each type, assume that the last status is the most recent + return statusArray.reduce((acc, status) => { + const index = acc.findIndex((s) => s.Type === status.Type); + if (index === -1) { + return [...acc, status]; + } + + return [...acc.slice(0, index), ...acc.slice(index + 1), status]; + }, [] as Array); +} diff --git a/pkg/libstack/compose/internal/composeplugin/composeplugin.go b/pkg/libstack/compose/internal/composeplugin/composeplugin.go index 73d230f16..ca98426c0 100644 --- a/pkg/libstack/compose/internal/composeplugin/composeplugin.go +++ b/pkg/libstack/compose/internal/composeplugin/composeplugin.go @@ -3,6 +3,7 @@ package composeplugin import ( "bytes" "context" + "fmt" "os" "os/exec" "strings" @@ -160,7 +161,11 @@ func (wrapper *PluginWrapper) command(command composeCommand, options libstack.O Err(err). Msg("docker compose command failed") - return nil, errors.New(errOutput) + if errOutput != "" { + return nil, errors.New(errOutput) + } + + return nil, fmt.Errorf("docker compose command failed: %w", err) } return output, nil diff --git a/pkg/libstack/compose/internal/composeplugin/composeplugin_test.go b/pkg/libstack/compose/internal/composeplugin/composeplugin_test.go index b979b6cb6..450d1e81a 100644 --- a/pkg/libstack/compose/internal/composeplugin/composeplugin_test.go +++ b/pkg/libstack/compose/internal/composeplugin/composeplugin_test.go @@ -2,7 +2,6 @@ package composeplugin import ( "context" - "errors" "fmt" "log" "os" @@ -16,9 +15,9 @@ import ( ) func checkPrerequisites(t *testing.T) { - if _, err := os.Stat("docker-compose"); errors.Is(err, os.ErrNotExist) { - t.Fatal("docker-compose binary not found, please run download.sh and re-run this test suite") - } + // if _, err := os.Stat("docker-compose"); errors.Is(err, os.ErrNotExist) { + // t.Fatal("docker-compose binary not found, please run download.sh and re-run this test suite") + // } } func setup(t *testing.T) libstack.Deployer { @@ -118,7 +117,11 @@ func createFile(dir, fileName, content string) (string, error) { return "", err } - f.WriteString(content) + _, err = f.WriteString(content) + if err != nil { + return "", err + } + f.Close() return filePath, nil diff --git a/pkg/libstack/compose/internal/composeplugin/status.go b/pkg/libstack/compose/internal/composeplugin/status.go new file mode 100644 index 000000000..4b6d9299d --- /dev/null +++ b/pkg/libstack/compose/internal/composeplugin/status.go @@ -0,0 +1,171 @@ +package composeplugin + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/portainer/portainer/pkg/libstack" + "github.com/rs/zerolog/log" +) + +type publisher struct { + URL string + TargetPort int + PublishedPort int + Protocol string +} + +type service struct { + ID string + Name string + Image string + Command string + Project string + Service string + Created int64 + State string + Status string + Health string + ExitCode int + Publishers []publisher +} + +// docker container state can be one of "created", "running", "paused", "restarting", "removing", "exited", or "dead" +func getServiceStatus(service service) (libstack.Status, string) { + log.Debug(). + Str("service", service.Name). + Str("state", service.State). + Int("exitCode", service.ExitCode). + Msg("getServiceStatus") + + switch service.State { + case "created", "restarting", "paused": + return libstack.StatusStarting, "" + case "running": + return libstack.StatusRunning, "" + case "removing": + return libstack.StatusRemoving, "" + case "exited", "dead": + if service.ExitCode != 0 { + return libstack.StatusError, fmt.Sprintf("service %s exited with code %d", service.Name, service.ExitCode) + } + + return libstack.StatusRemoved, "" + default: + return libstack.StatusUnknown, "" + } +} + +func aggregateStatuses(services []service) (libstack.Status, string) { + servicesCount := len(services) + + if servicesCount == 0 { + log.Debug(). + Msg("no services found") + + return libstack.StatusRemoved, "" + } + + statusCounts := make(map[libstack.Status]int) + errorMessage := "" + for _, service := range services { + status, serviceError := getServiceStatus(service) + if serviceError != "" { + errorMessage = serviceError + } + statusCounts[status]++ + } + + log.Debug(). + Interface("statusCounts", statusCounts). + Str("errorMessage", errorMessage). + Msg("check_status") + + switch { + case errorMessage != "": + return libstack.StatusError, errorMessage + case statusCounts[libstack.StatusStarting] > 0: + return libstack.StatusStarting, "" + case statusCounts[libstack.StatusRemoving] > 0: + return libstack.StatusRemoving, "" + case statusCounts[libstack.StatusRunning] == servicesCount: + return libstack.StatusRunning, "" + case statusCounts[libstack.StatusStopped] == servicesCount: + return libstack.StatusStopped, "" + case statusCounts[libstack.StatusRemoved] == servicesCount: + return libstack.StatusRemoved, "" + default: + return libstack.StatusUnknown, "" + } + +} + +func (wrapper *PluginWrapper) WaitForStatus(ctx context.Context, name string, status libstack.Status) <-chan string { + errorMessageCh := make(chan string) + + go func() { + for { + select { + case <-ctx.Done(): + errorMessageCh <- fmt.Sprintf("failed to wait for status: %s", ctx.Err().Error()) + default: + } + + time.Sleep(1 * time.Second) + + output, err := wrapper.command(newCommand([]string{"ps", "-a", "--format", "json"}, nil), libstack.Options{ + ProjectName: name, + }) + if len(output) == 0 { + log.Debug(). + Str("project_name", name). + Msg("no output from docker compose ps") + continue + } + + if err != nil { + log.Debug(). + Str("project_name", name). + Err(err). + Msg("error from docker compose ps") + continue + } + + var services []service + err = json.Unmarshal(output, &services) + if err != nil { + log.Debug(). + Str("project_name", name). + Err(err). + Msg("failed to parse docker compose output") + continue + } + + if len(services) == 0 && status == libstack.StatusRemoved { + errorMessageCh <- "" + return + } + + aggregateStatus, errorMessage := aggregateStatuses(services) + if aggregateStatus == status { + errorMessageCh <- "" + return + } + + if errorMessage != "" { + errorMessageCh <- errorMessage + return + } + + log.Debug(). + Str("project_name", name). + Str("status", string(aggregateStatus)). + Msg("waiting for status") + + } + }() + + return errorMessageCh +} diff --git a/pkg/libstack/compose/internal/composeplugin/status_integration_test.go b/pkg/libstack/compose/internal/composeplugin/status_integration_test.go new file mode 100644 index 000000000..29cce2eec --- /dev/null +++ b/pkg/libstack/compose/internal/composeplugin/status_integration_test.go @@ -0,0 +1,116 @@ +package composeplugin + +import ( + "context" + "os" + "testing" + "time" + + "github.com/portainer/portainer/pkg/libstack" +) + +/* + +1. starting = docker compose file that runs several services, one of them should be with status starting +2. running = docker compose file that runs successfully and returns status running +3. removing = run docker compose config, remove the stack, and return removing status +4. failed = run a valid docker compose file, but one of the services should fail to start (so "docker compose up" should run successfully, but one of the services should do something like `CMD ["exit", "1"] +5. removed = remove a compose stack and return status removed + +*/ + +func ensureIntegrationTest(t *testing.T) { + if _, ok := os.LookupEnv("INTEGRATION_TEST"); !ok { + t.Skip("skip an integration test") + } +} + +func TestComposeProjectStatus(t *testing.T) { + ensureIntegrationTest(t) + + testCases := []struct { + TestName string + ComposeFile string + ExpectedStatus libstack.Status + ExpectedStatusMessage bool + }{ + + { + TestName: "running", + ComposeFile: "status_test_files/running.yml", + ExpectedStatus: libstack.StatusRunning, + }, + + { + TestName: "failed", + ComposeFile: "status_test_files/failed.yml", + ExpectedStatus: libstack.StatusError, + ExpectedStatusMessage: true, + }, + } + + w := setup(t) + ctx := context.Background() + + for _, testCase := range testCases { + t.Run(testCase.TestName, func(t *testing.T) { + projectName := testCase.TestName + err := w.Deploy(ctx, []string{testCase.ComposeFile}, libstack.DeployOptions{ + Options: libstack.Options{ + ProjectName: projectName, + }, + }) + if err != nil { + t.Fatalf("[test: %s] Failed to deploy compose file: %v", testCase.TestName, err) + } + + time.Sleep(5 * time.Second) + + status, statusMessage, err := waitForStatus(w, ctx, projectName, libstack.StatusRunning) + if err != nil { + t.Fatalf("[test: %s] Failed to get compose project status: %v", testCase.TestName, err) + } + + if status != testCase.ExpectedStatus { + t.Fatalf("[test: %s] Expected status: %s, got: %s", testCase.TestName, testCase.ExpectedStatus, status) + } + + if testCase.ExpectedStatusMessage && statusMessage == "" { + t.Fatalf("[test: %s] Expected status message but got empty", testCase.TestName) + } + + err = w.Remove(ctx, projectName, nil, libstack.Options{}) + if err != nil { + t.Fatalf("[test: %s] Failed to remove compose project: %v", testCase.TestName, err) + } + + time.Sleep(20 * time.Second) + + status, statusMessage, err = waitForStatus(w, ctx, projectName, libstack.StatusRemoved) + if err != nil { + t.Fatalf("[test: %s] Failed to get compose project status: %v", testCase.TestName, err) + } + + if status != libstack.StatusRemoved { + t.Fatalf("[test: %s] Expected stack to be removed, got %s", testCase.TestName, status) + } + + if statusMessage != "" { + t.Fatalf("[test: %s] Expected empty status message: %s, got: %s", "", testCase.TestName, statusMessage) + } + }) + } +} + +func waitForStatus(deployer libstack.Deployer, ctx context.Context, stackName string, requiredStatus libstack.Status) (libstack.Status, string, error) { + ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) + defer cancel() + + statusCh := deployer.WaitForStatus(ctx, stackName, requiredStatus) + result := <-statusCh + if result == "" { + return requiredStatus, "", nil + } + + return libstack.StatusError, result, nil +} diff --git a/pkg/libstack/compose/internal/composeplugin/status_test_files/failed.yml b/pkg/libstack/compose/internal/composeplugin/status_test_files/failed.yml new file mode 100644 index 000000000..a57699b35 --- /dev/null +++ b/pkg/libstack/compose/internal/composeplugin/status_test_files/failed.yml @@ -0,0 +1,7 @@ +version: '3' +services: + web: + image: nginx:latest + failing-service: + image: busybox + command: ["false"] \ No newline at end of file diff --git a/pkg/libstack/compose/internal/composeplugin/status_test_files/running.yml b/pkg/libstack/compose/internal/composeplugin/status_test_files/running.yml new file mode 100644 index 000000000..1950aaa58 --- /dev/null +++ b/pkg/libstack/compose/internal/composeplugin/status_test_files/running.yml @@ -0,0 +1,4 @@ +version: '3' +services: + web: + image: nginx:latest diff --git a/pkg/libstack/libstack.go b/pkg/libstack/libstack.go index d51dae8be..294eb1c94 100644 --- a/pkg/libstack/libstack.go +++ b/pkg/libstack/libstack.go @@ -13,8 +13,21 @@ type Deployer interface { Remove(ctx context.Context, projectName string, filePaths []string, options Options) error Pull(ctx context.Context, filePaths []string, options Options) error Validate(ctx context.Context, filePaths []string, options Options) error + WaitForStatus(ctx context.Context, name string, status Status) <-chan string } +type Status string + +const ( + StatusUnknown Status = "unknown" + StatusStarting Status = "starting" + StatusRunning Status = "running" + StatusStopped Status = "stopped" + StatusError Status = "error" + StatusRemoving Status = "removing" + StatusRemoved Status = "removed" +) + type Options struct { WorkingDir string Host string