mirror of https://github.com/portainer/portainer
feat(edge/stacks): increase status transparency [EE-5554] (#9094)
Chaim Lev-Ari
1 year ago
committed by
GitHub
parent
db61fb149b
commit
0bcb57568c
@ -0,0 +1,59 @@
|
||||
package edgestacks
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
portainer "github.com/portainer/portainer/api"
|
||||
"github.com/portainer/portainer/api/filesystem"
|
||||
edgestackutils "github.com/portainer/portainer/api/internal/edge/edgestacks"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
func (handler *Handler) updateStackVersion(stack *portainer.EdgeStack, deploymentType portainer.EdgeStackDeploymentType, config []byte, oldGitHash string, relatedEnvironmentsIDs []portainer.EndpointID) error {
|
||||
|
||||
stack.Version = stack.Version + 1
|
||||
stack.Status = edgestackutils.NewStatus(stack.Status, relatedEnvironmentsIDs)
|
||||
|
||||
return handler.storeStackFile(stack, deploymentType, config)
|
||||
}
|
||||
|
||||
func (handler *Handler) storeStackFile(stack *portainer.EdgeStack, deploymentType portainer.EdgeStackDeploymentType, config []byte) error {
|
||||
|
||||
if deploymentType != stack.DeploymentType {
|
||||
// deployment type was changed - need to delete all old files
|
||||
err := handler.FileService.RemoveDirectory(stack.ProjectPath)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("Unable to clear old files")
|
||||
}
|
||||
|
||||
stack.EntryPoint = ""
|
||||
stack.ManifestPath = ""
|
||||
stack.DeploymentType = deploymentType
|
||||
}
|
||||
|
||||
stackFolder := strconv.Itoa(int(stack.ID))
|
||||
entryPoint := ""
|
||||
if deploymentType == portainer.EdgeStackDeploymentCompose {
|
||||
if stack.EntryPoint == "" {
|
||||
stack.EntryPoint = filesystem.ComposeFileDefaultName
|
||||
}
|
||||
|
||||
entryPoint = stack.EntryPoint
|
||||
}
|
||||
|
||||
if deploymentType == portainer.EdgeStackDeploymentKubernetes {
|
||||
if stack.ManifestPath == "" {
|
||||
stack.ManifestPath = filesystem.ManifestFileDefaultName
|
||||
}
|
||||
|
||||
entryPoint = stack.ManifestPath
|
||||
}
|
||||
|
||||
_, err := handler.FileService.StoreEdgeStackFileFromBytesByVersion(stackFolder, entryPoint, stack.Version, config)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to persist updated Compose file with version on disk: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@ -1,6 +1,7 @@
|
||||
package endpoints
|
||||
|
||||
func ptr[T any](i T) *T { return &i }
|
||||
|
||||
func BoolAddr(b bool) *bool {
|
||||
boolVar := b
|
||||
return &boolVar
|
||||
return ptr(b)
|
||||
}
|
||||
|
||||
@ -0,0 +1,27 @@
|
||||
package edgestacks
|
||||
|
||||
import (
|
||||
portainer "github.com/portainer/portainer/api"
|
||||
)
|
||||
|
||||
// NewStatus returns a new status object for an Edge stack
|
||||
func NewStatus(oldStatus map[portainer.EndpointID]portainer.EdgeStackStatus, relatedEnvironmentIDs []portainer.EndpointID) map[portainer.EndpointID]portainer.EdgeStackStatus {
|
||||
status := map[portainer.EndpointID]portainer.EdgeStackStatus{}
|
||||
|
||||
for _, environmentID := range relatedEnvironmentIDs {
|
||||
|
||||
newEnvStatus := portainer.EdgeStackStatus{
|
||||
Status: []portainer.EdgeStackDeploymentStatus{},
|
||||
EndpointID: portainer.EndpointID(environmentID),
|
||||
}
|
||||
|
||||
oldEnvStatus, ok := oldStatus[environmentID]
|
||||
if ok {
|
||||
newEnvStatus.DeploymentInfo = oldEnvStatus.DeploymentInfo
|
||||
}
|
||||
|
||||
status[environmentID] = newEnvStatus
|
||||
}
|
||||
|
||||
return status
|
||||
}
|
||||
|
After Width: | Height: | Size: 1.4 KiB |
|
After Width: | Height: | Size: 908 B |
@ -0,0 +1,87 @@
|
||||
import _ from 'lodash';
|
||||
import {
|
||||
AlertTriangle,
|
||||
CheckCircle,
|
||||
type Icon as IconType,
|
||||
Loader2,
|
||||
XCircle,
|
||||
} from 'lucide-react';
|
||||
|
||||
import { Icon, IconMode } from '@@/Icon';
|
||||
|
||||
import { DeploymentStatus, EdgeStack, StatusType } from '../types';
|
||||
|
||||
export function EdgeStackStatus({ edgeStack }: { edgeStack: EdgeStack }) {
|
||||
const status = Object.values(edgeStack.Status);
|
||||
const lastStatus = _.compact(status.map((s) => _.last(s.Status)));
|
||||
|
||||
const { icon, label, mode, spin } = getStatus(
|
||||
edgeStack.NumDeployments,
|
||||
lastStatus
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="mx-auto inline-flex items-center gap-2">
|
||||
{icon && <Icon icon={icon} spin={spin} mode={mode} />}
|
||||
{label}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function getStatus(
|
||||
numDeployments: number,
|
||||
envStatus: Array<DeploymentStatus>
|
||||
): {
|
||||
label: string;
|
||||
icon?: IconType;
|
||||
spin?: boolean;
|
||||
mode?: IconMode;
|
||||
} {
|
||||
if (envStatus.length < numDeployments) {
|
||||
return {
|
||||
label: 'Deploying',
|
||||
icon: Loader2,
|
||||
spin: true,
|
||||
mode: 'primary',
|
||||
};
|
||||
}
|
||||
|
||||
const allFailed = envStatus.every((s) => s.Type === StatusType.Error);
|
||||
|
||||
if (allFailed) {
|
||||
return {
|
||||
label: 'Failed',
|
||||
icon: XCircle,
|
||||
mode: 'danger',
|
||||
};
|
||||
}
|
||||
|
||||
const allRunning = envStatus.every((s) => s.Type === StatusType.Running);
|
||||
|
||||
if (allRunning) {
|
||||
return {
|
||||
label: 'Running',
|
||||
icon: CheckCircle,
|
||||
mode: 'success',
|
||||
};
|
||||
}
|
||||
|
||||
const hasDeploying = envStatus.some((s) => s.Type === StatusType.Deploying);
|
||||
const hasRunning = envStatus.some((s) => s.Type === StatusType.Running);
|
||||
const hasFailed = envStatus.some((s) => s.Type === StatusType.Error);
|
||||
|
||||
if (hasRunning && hasFailed && !hasDeploying) {
|
||||
return {
|
||||
label: 'Partially Running',
|
||||
icon: AlertTriangle,
|
||||
mode: 'warning',
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
label: 'Deploying',
|
||||
icon: Loader2,
|
||||
spin: true,
|
||||
mode: 'primary',
|
||||
};
|
||||
}
|
||||
@ -0,0 +1,87 @@
|
||||
import _ from 'lodash';
|
||||
import {
|
||||
AlertTriangle,
|
||||
CheckCircle,
|
||||
type Icon as IconType,
|
||||
Loader2,
|
||||
XCircle,
|
||||
} from 'lucide-react';
|
||||
|
||||
import { Icon, IconMode } from '@@/Icon';
|
||||
|
||||
import { DeploymentStatus, EdgeStack, StatusType } from '../../types';
|
||||
|
||||
export function EdgeStackStatus({ edgeStack }: { edgeStack: EdgeStack }) {
|
||||
const status = Object.values(edgeStack.Status);
|
||||
const lastStatus = _.compact(status.map((s) => _.last(s.Status)));
|
||||
|
||||
const { icon, label, mode, spin } = getStatus(
|
||||
edgeStack.NumDeployments,
|
||||
lastStatus
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="mx-auto inline-flex items-center gap-2">
|
||||
{icon && <Icon icon={icon} spin={spin} mode={mode} />}
|
||||
{label}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function getStatus(
|
||||
numDeployments: number,
|
||||
envStatus: Array<DeploymentStatus>
|
||||
): {
|
||||
label: string;
|
||||
icon?: IconType;
|
||||
spin?: boolean;
|
||||
mode?: IconMode;
|
||||
} {
|
||||
if (envStatus.length < numDeployments) {
|
||||
return {
|
||||
label: 'Deploying',
|
||||
icon: Loader2,
|
||||
spin: true,
|
||||
mode: 'primary',
|
||||
};
|
||||
}
|
||||
|
||||
const allFailed = envStatus.every((s) => s.Type === StatusType.Error);
|
||||
|
||||
if (allFailed) {
|
||||
return {
|
||||
label: 'Failed',
|
||||
icon: XCircle,
|
||||
mode: 'danger',
|
||||
};
|
||||
}
|
||||
|
||||
const allRunning = envStatus.every((s) => s.Type === StatusType.Running);
|
||||
|
||||
if (allRunning) {
|
||||
return {
|
||||
label: 'Running',
|
||||
icon: CheckCircle,
|
||||
mode: 'success',
|
||||
};
|
||||
}
|
||||
|
||||
const hasDeploying = envStatus.some((s) => s.Type === StatusType.Deploying);
|
||||
const hasRunning = envStatus.some((s) => s.Type === StatusType.Running);
|
||||
const hasFailed = envStatus.some((s) => s.Type === StatusType.Error);
|
||||
|
||||
if (hasRunning && hasFailed && !hasDeploying) {
|
||||
return {
|
||||
label: 'Partially Running',
|
||||
icon: AlertTriangle,
|
||||
mode: 'warning',
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
label: 'Deploying',
|
||||
icon: Loader2,
|
||||
spin: true,
|
||||
mode: 'primary',
|
||||
};
|
||||
}
|
||||
@ -0,0 +1,16 @@
|
||||
import { DeploymentStatus } from '../types';
|
||||
|
||||
/**
|
||||
* returns the latest status object of each type
|
||||
*/
|
||||
export function uniqueStatus(statusArray: Array<DeploymentStatus> = []) {
|
||||
// keep only the last status object of each type, assume that the last status is the most recent
|
||||
return statusArray.reduce((acc, status) => {
|
||||
const index = acc.findIndex((s) => s.Type === status.Type);
|
||||
if (index === -1) {
|
||||
return [...acc, status];
|
||||
}
|
||||
|
||||
return [...acc.slice(0, index), ...acc.slice(index + 1), status];
|
||||
}, [] as Array<DeploymentStatus>);
|
||||
}
|
||||
@ -0,0 +1,171 @@
|
||||
package composeplugin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/portainer/portainer/pkg/libstack"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
type publisher struct {
|
||||
URL string
|
||||
TargetPort int
|
||||
PublishedPort int
|
||||
Protocol string
|
||||
}
|
||||
|
||||
type service struct {
|
||||
ID string
|
||||
Name string
|
||||
Image string
|
||||
Command string
|
||||
Project string
|
||||
Service string
|
||||
Created int64
|
||||
State string
|
||||
Status string
|
||||
Health string
|
||||
ExitCode int
|
||||
Publishers []publisher
|
||||
}
|
||||
|
||||
// docker container state can be one of "created", "running", "paused", "restarting", "removing", "exited", or "dead"
|
||||
func getServiceStatus(service service) (libstack.Status, string) {
|
||||
log.Debug().
|
||||
Str("service", service.Name).
|
||||
Str("state", service.State).
|
||||
Int("exitCode", service.ExitCode).
|
||||
Msg("getServiceStatus")
|
||||
|
||||
switch service.State {
|
||||
case "created", "restarting", "paused":
|
||||
return libstack.StatusStarting, ""
|
||||
case "running":
|
||||
return libstack.StatusRunning, ""
|
||||
case "removing":
|
||||
return libstack.StatusRemoving, ""
|
||||
case "exited", "dead":
|
||||
if service.ExitCode != 0 {
|
||||
return libstack.StatusError, fmt.Sprintf("service %s exited with code %d", service.Name, service.ExitCode)
|
||||
}
|
||||
|
||||
return libstack.StatusRemoved, ""
|
||||
default:
|
||||
return libstack.StatusUnknown, ""
|
||||
}
|
||||
}
|
||||
|
||||
func aggregateStatuses(services []service) (libstack.Status, string) {
|
||||
servicesCount := len(services)
|
||||
|
||||
if servicesCount == 0 {
|
||||
log.Debug().
|
||||
Msg("no services found")
|
||||
|
||||
return libstack.StatusRemoved, ""
|
||||
}
|
||||
|
||||
statusCounts := make(map[libstack.Status]int)
|
||||
errorMessage := ""
|
||||
for _, service := range services {
|
||||
status, serviceError := getServiceStatus(service)
|
||||
if serviceError != "" {
|
||||
errorMessage = serviceError
|
||||
}
|
||||
statusCounts[status]++
|
||||
}
|
||||
|
||||
log.Debug().
|
||||
Interface("statusCounts", statusCounts).
|
||||
Str("errorMessage", errorMessage).
|
||||
Msg("check_status")
|
||||
|
||||
switch {
|
||||
case errorMessage != "":
|
||||
return libstack.StatusError, errorMessage
|
||||
case statusCounts[libstack.StatusStarting] > 0:
|
||||
return libstack.StatusStarting, ""
|
||||
case statusCounts[libstack.StatusRemoving] > 0:
|
||||
return libstack.StatusRemoving, ""
|
||||
case statusCounts[libstack.StatusRunning] == servicesCount:
|
||||
return libstack.StatusRunning, ""
|
||||
case statusCounts[libstack.StatusStopped] == servicesCount:
|
||||
return libstack.StatusStopped, ""
|
||||
case statusCounts[libstack.StatusRemoved] == servicesCount:
|
||||
return libstack.StatusRemoved, ""
|
||||
default:
|
||||
return libstack.StatusUnknown, ""
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (wrapper *PluginWrapper) WaitForStatus(ctx context.Context, name string, status libstack.Status) <-chan string {
|
||||
errorMessageCh := make(chan string)
|
||||
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
errorMessageCh <- fmt.Sprintf("failed to wait for status: %s", ctx.Err().Error())
|
||||
default:
|
||||
}
|
||||
|
||||
time.Sleep(1 * time.Second)
|
||||
|
||||
output, err := wrapper.command(newCommand([]string{"ps", "-a", "--format", "json"}, nil), libstack.Options{
|
||||
ProjectName: name,
|
||||
})
|
||||
if len(output) == 0 {
|
||||
log.Debug().
|
||||
Str("project_name", name).
|
||||
Msg("no output from docker compose ps")
|
||||
continue
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
log.Debug().
|
||||
Str("project_name", name).
|
||||
Err(err).
|
||||
Msg("error from docker compose ps")
|
||||
continue
|
||||
}
|
||||
|
||||
var services []service
|
||||
err = json.Unmarshal(output, &services)
|
||||
if err != nil {
|
||||
log.Debug().
|
||||
Str("project_name", name).
|
||||
Err(err).
|
||||
Msg("failed to parse docker compose output")
|
||||
continue
|
||||
}
|
||||
|
||||
if len(services) == 0 && status == libstack.StatusRemoved {
|
||||
errorMessageCh <- ""
|
||||
return
|
||||
}
|
||||
|
||||
aggregateStatus, errorMessage := aggregateStatuses(services)
|
||||
if aggregateStatus == status {
|
||||
errorMessageCh <- ""
|
||||
return
|
||||
}
|
||||
|
||||
if errorMessage != "" {
|
||||
errorMessageCh <- errorMessage
|
||||
return
|
||||
}
|
||||
|
||||
log.Debug().
|
||||
Str("project_name", name).
|
||||
Str("status", string(aggregateStatus)).
|
||||
Msg("waiting for status")
|
||||
|
||||
}
|
||||
}()
|
||||
|
||||
return errorMessageCh
|
||||
}
|
||||
@ -0,0 +1,116 @@
|
||||
package composeplugin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/portainer/portainer/pkg/libstack"
|
||||
)
|
||||
|
||||
/*
|
||||
|
||||
1. starting = docker compose file that runs several services, one of them should be with status starting
|
||||
2. running = docker compose file that runs successfully and returns status running
|
||||
3. removing = run docker compose config, remove the stack, and return removing status
|
||||
4. failed = run a valid docker compose file, but one of the services should fail to start (so "docker compose up" should run successfully, but one of the services should do something like `CMD ["exit", "1"]
|
||||
5. removed = remove a compose stack and return status removed
|
||||
|
||||
*/
|
||||
|
||||
func ensureIntegrationTest(t *testing.T) {
|
||||
if _, ok := os.LookupEnv("INTEGRATION_TEST"); !ok {
|
||||
t.Skip("skip an integration test")
|
||||
}
|
||||
}
|
||||
|
||||
func TestComposeProjectStatus(t *testing.T) {
|
||||
ensureIntegrationTest(t)
|
||||
|
||||
testCases := []struct {
|
||||
TestName string
|
||||
ComposeFile string
|
||||
ExpectedStatus libstack.Status
|
||||
ExpectedStatusMessage bool
|
||||
}{
|
||||
|
||||
{
|
||||
TestName: "running",
|
||||
ComposeFile: "status_test_files/running.yml",
|
||||
ExpectedStatus: libstack.StatusRunning,
|
||||
},
|
||||
|
||||
{
|
||||
TestName: "failed",
|
||||
ComposeFile: "status_test_files/failed.yml",
|
||||
ExpectedStatus: libstack.StatusError,
|
||||
ExpectedStatusMessage: true,
|
||||
},
|
||||
}
|
||||
|
||||
w := setup(t)
|
||||
ctx := context.Background()
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.TestName, func(t *testing.T) {
|
||||
projectName := testCase.TestName
|
||||
err := w.Deploy(ctx, []string{testCase.ComposeFile}, libstack.DeployOptions{
|
||||
Options: libstack.Options{
|
||||
ProjectName: projectName,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("[test: %s] Failed to deploy compose file: %v", testCase.TestName, err)
|
||||
}
|
||||
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
status, statusMessage, err := waitForStatus(w, ctx, projectName, libstack.StatusRunning)
|
||||
if err != nil {
|
||||
t.Fatalf("[test: %s] Failed to get compose project status: %v", testCase.TestName, err)
|
||||
}
|
||||
|
||||
if status != testCase.ExpectedStatus {
|
||||
t.Fatalf("[test: %s] Expected status: %s, got: %s", testCase.TestName, testCase.ExpectedStatus, status)
|
||||
}
|
||||
|
||||
if testCase.ExpectedStatusMessage && statusMessage == "" {
|
||||
t.Fatalf("[test: %s] Expected status message but got empty", testCase.TestName)
|
||||
}
|
||||
|
||||
err = w.Remove(ctx, projectName, nil, libstack.Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("[test: %s] Failed to remove compose project: %v", testCase.TestName, err)
|
||||
}
|
||||
|
||||
time.Sleep(20 * time.Second)
|
||||
|
||||
status, statusMessage, err = waitForStatus(w, ctx, projectName, libstack.StatusRemoved)
|
||||
if err != nil {
|
||||
t.Fatalf("[test: %s] Failed to get compose project status: %v", testCase.TestName, err)
|
||||
}
|
||||
|
||||
if status != libstack.StatusRemoved {
|
||||
t.Fatalf("[test: %s] Expected stack to be removed, got %s", testCase.TestName, status)
|
||||
}
|
||||
|
||||
if statusMessage != "" {
|
||||
t.Fatalf("[test: %s] Expected empty status message: %s, got: %s", "", testCase.TestName, statusMessage)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func waitForStatus(deployer libstack.Deployer, ctx context.Context, stackName string, requiredStatus libstack.Status) (libstack.Status, string, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 1*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
statusCh := deployer.WaitForStatus(ctx, stackName, requiredStatus)
|
||||
result := <-statusCh
|
||||
if result == "" {
|
||||
return requiredStatus, "", nil
|
||||
}
|
||||
|
||||
return libstack.StatusError, result, nil
|
||||
}
|
||||
@ -0,0 +1,7 @@
|
||||
version: '3'
|
||||
services:
|
||||
web:
|
||||
image: nginx:latest
|
||||
failing-service:
|
||||
image: busybox
|
||||
command: ["false"]
|
||||
@ -0,0 +1,4 @@
|
||||
version: '3'
|
||||
services:
|
||||
web:
|
||||
image: nginx:latest
|
||||
Loading…
Reference in new issue