From 2ecc8ab5c9a1fe7c4a220981c441471c94e507e4 Mon Sep 17 00:00:00 2001 From: Dmitry Salakhov Date: Thu, 30 Sep 2021 12:58:10 +1300 Subject: [PATCH] feat(k8s): support git automated sync for k8s applications [EE-577] (#5548) * feat(stack): backport changes to CE EE-1189 * feat(stack): front end backport changes to CE EE-1199 (#5455) * feat(stack): front end backport changes to CE EE-1199 * fix k8s deploy logic * fixed web editor confirmation message typo. EE-1501 * fix(stack): fixed issue auth detail not remembered EE-1502 (#5459) * show status in buttons * removed onChangeRef function. * moved buttons in git form to its own component * removed unused variable. Co-authored-by: ArrisLee * moved formvalue to kube app component * fix(stack): failed to pull and redeploy compose format k8s stack * fixed form value * fix(k8s): file content overridden when deployment failed with compose format EE-1548 * updated API response to get IsComposeFormat and show appropriate text. * feat(k8s): front end backport to CE * feat(kube): kube app auto update backend (#5547) * error message updates for different file type * not display creation source for external application * added confirmation modal to advanced app created by web editor * stop showing confirmation modal when updating application * disable rollback button when application type is not applicatiom form * only update file after deployment succeded * Revert "only update file after deployment succeded" This reverts commit b94bd2e96f27ca337e9d2fa698da2ab264d9ac86. * fix(k8s): file content overridden when deployment failed with compose format EE-1556 * added analytics-on directive to pull and redeploy button * fix(kube): don't valide resource control access for kube (#5568) * added missing question mark to k8s confirmation modal * fixed webhook format issue * added question marks to k8s app confirmation modal * added space in additional file list. * ignoring error on deletion * fix(k8s): Git authentication info not persisted * added RepositoryMechanismTypes constant * updated analytics functions * covert RepositoryMechanism to constant * fixed typo * removed unused function. * post tech review updates * fixed save settings n redeploy button * refact kub deploy logic * Revert "refact kub deploy logic" This reverts commit cbfdd58eceed438b148c836ed09004dd270d0aa6. * feat(k8s): utilize user token for k8s auto update EE-1594 * feat(k8s): persist kub stack name EE-1630 * feat(k8s): support delete kub stack * fix(app): updated logic to delete stack for different kind apps. (#5648) * fix(app): updated logic to delete stack for different kind apps. * renamed variable * fix import * added StackName field. * fixed stack id not found issue. * fix(k8s): fixed qusetion mark alignment issue in PAT field. (#5611) * fix(k8s): fixed qusetion mark alignment issue in PAT field. * moved inline css to file. * fix(git-form: made auth input text full width * add ignore deleted arg * tech review updates * typo fix * fix(k8s): added console error when deleting k8s service. * fix(console): added no-console config * fix(deploy): added missing service. * fix: use stack editor as an owner when exists (#5678) * fix: tempalte/content based stacks edit/delete * fix(stack): remove stack when no app. (#5769) * fix(stack): remove stack when no app. * support compose format in delete Co-authored-by: ArrisLee Co-authored-by: Hui Co-authored-by: fhanportainer <79428273+fhanportainer@users.noreply.github.com> Co-authored-by: Felix Han --- api/cmd/portainer/main.go | 2 +- api/exec/compose_stack.go | 4 +- api/exec/exectest/kubernetes_mocks.go | 8 +- api/exec/kubernetes_deploy.go | 55 +++--- api/filesystem/write.go | 23 +++ api/filesystem/write_test.go | 48 +++++ api/http/handler/helm/helm_install.go | 22 ++- api/http/handler/stacks/autoupdate.go | 6 +- .../handler/stacks/create_compose_stack.go | 10 +- .../handler/stacks/create_kubernetes_stack.go | 164 +++++++++++------- .../stacks/create_kubernetes_stack_test.go | 68 -------- api/http/handler/stacks/create_swarm_stack.go | 12 +- api/http/handler/stacks/handler.go | 13 +- api/http/handler/stacks/stack_delete.go | 104 +++++++---- api/http/handler/stacks/stack_migrate.go | 72 ++++---- api/http/handler/stacks/stack_start.go | 52 +++--- api/http/handler/stacks/stack_stop.go | 26 +-- api/http/handler/stacks/stack_update_git.go | 20 ++- .../stacks/stack_update_git_redeploy.go | 16 +- .../handler/stacks/update_kubernetes_stack.go | 54 +++++- api/http/handler/stacks/webhook_invoke.go | 7 +- api/http/proxy/factory/agent.go | 1 + api/internal/endpointutils/endpointutils.go | 3 +- api/internal/stackutils/stackutils.go | 40 +++++ api/kubernetes/yaml.go | 19 +- api/kubernetes/yaml_test.go | 102 ++--------- api/portainer.go | 4 +- api/scheduler/scheduler.go | 46 +++-- api/scheduler/scheduler_test.go | 107 ++++++++---- api/stacks/deploy.go | 47 +++-- api/stacks/deploy_test.go | 42 +++-- api/stacks/deployer.go | 41 ++++- api/stacks/scheduled.go | 8 +- app/config.js | 2 +- app/edge/components/group-form/groupForm.html | 4 +- .../containersDatatable.html | 10 +- .../kube-config-download-button.html | 28 +-- app/kubernetes/models/deploy.js | 5 + app/kubernetes/services/serviceService.js | 3 +- .../applications/applicationsController.js | 25 ++- .../create/createApplication.html | 6 +- app/kubernetes/views/deploy/deploy.html | 43 ++--- .../views/deploy/deployController.js | 76 +++++--- .../associatedEndpointsSelector.html | 4 +- .../git-form-auth-fieldset.css | 11 ++ .../git-form-auth-fieldset.html | 10 +- .../git-form/git-form-auth-fieldset/index.js | 1 + .../git-form-compose-path-field/index.js | 1 - .../git-form-info-panel.html | 6 +- .../git-form/git-form-info-panel/index.js | 1 + .../forms/group-form/groupForm.html | 4 +- .../kubernetes-app-git-form.controller.js | 1 - ...rnetes-redeploy-app-git-form.controller.js | 147 ++++++++++++++++ .../kubernetes-redeploy-app-git-form.html | 64 +++++++ .../kubernetes-redeploy-app-git-form.js | 13 ++ .../stack-redeploy-git-form.controller.js | 12 +- app/portainer/services/api/stackService.js | 15 +- .../createCustomTemplateViewController.js | 2 +- app/portainer/views/groups/groups.html | 9 +- .../stacks/create/createStackController.js | 12 +- .../views/stacks/create/createstack.html | 2 + 61 files changed, 1193 insertions(+), 570 deletions(-) create mode 100644 api/filesystem/write.go create mode 100644 api/filesystem/write_test.go delete mode 100644 api/http/handler/stacks/create_kubernetes_stack_test.go create mode 100644 app/portainer/components/forms/git-form/git-form-auth-fieldset/git-form-auth-fieldset.css create mode 100644 app/portainer/components/forms/kubernetes-redeploy-app-git-form/kubernetes-redeploy-app-git-form.controller.js create mode 100644 app/portainer/components/forms/kubernetes-redeploy-app-git-form/kubernetes-redeploy-app-git-form.html create mode 100644 app/portainer/components/forms/kubernetes-redeploy-app-git-form/kubernetes-redeploy-app-git-form.js diff --git a/api/cmd/portainer/main.go b/api/cmd/portainer/main.go index 78ca69e17..5a9139dae 100644 --- a/api/cmd/portainer/main.go +++ b/api/cmd/portainer/main.go @@ -553,7 +553,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server { } scheduler := scheduler.NewScheduler(shutdownCtx) - stackDeployer := stacks.NewStackDeployer(swarmStackManager, composeStackManager) + stackDeployer := stacks.NewStackDeployer(swarmStackManager, composeStackManager, kubernetesDeployer) stacks.StartStackSchedules(scheduler, stackDeployer, dataStore, gitService) return &http.Server{ diff --git a/api/exec/compose_stack.go b/api/exec/compose_stack.go index 4fb4d163e..28c4d8278 100644 --- a/api/exec/compose_stack.go +++ b/api/exec/compose_stack.go @@ -47,7 +47,7 @@ func (manager *ComposeStackManager) ComposeSyntaxMaxVersion() string { func (manager *ComposeStackManager) Up(ctx context.Context, stack *portainer.Stack, endpoint *portainer.Endpoint) error { url, proxy, err := manager.fetchEndpointProxy(endpoint) if err != nil { - return errors.Wrap(err, "failed to fetch endpoint proxy") + return errors.Wrap(err, "failed to fetch environment proxy") } if proxy != nil { @@ -80,7 +80,7 @@ func (manager *ComposeStackManager) Down(ctx context.Context, stack *portainer.S } // NormalizeStackName returns a new stack name with unsupported characters replaced -func (w *ComposeStackManager) NormalizeStackName(name string) string { +func (manager *ComposeStackManager) NormalizeStackName(name string) string { r := regexp.MustCompile("[^a-z0-9]+") return r.ReplaceAllString(strings.ToLower(name), "") } diff --git a/api/exec/exectest/kubernetes_mocks.go b/api/exec/exectest/kubernetes_mocks.go index 2809df2c5..cabc4a716 100644 --- a/api/exec/exectest/kubernetes_mocks.go +++ b/api/exec/exectest/kubernetes_mocks.go @@ -1,8 +1,6 @@ package exectest import ( - "net/http" - portainer "github.com/portainer/portainer/api" ) @@ -12,7 +10,11 @@ func NewKubernetesDeployer() portainer.KubernetesDeployer { return &kubernetesMockDeployer{} } -func (deployer *kubernetesMockDeployer) Deploy(request *http.Request, endpoint *portainer.Endpoint, data string, namespace string) (string, error) { +func (deployer *kubernetesMockDeployer) Deploy(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) { + return "", nil +} + +func (deployer *kubernetesMockDeployer) Remove(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) { return "", nil } diff --git a/api/exec/kubernetes_deploy.go b/api/exec/kubernetes_deploy.go index af97af174..9759ef70c 100644 --- a/api/exec/kubernetes_deploy.go +++ b/api/exec/kubernetes_deploy.go @@ -3,7 +3,6 @@ package exec import ( "bytes" "fmt" - "net/http" "os/exec" "path" "runtime" @@ -13,7 +12,6 @@ import ( "github.com/portainer/portainer/api/http/proxy" "github.com/portainer/portainer/api/http/proxy/factory" "github.com/portainer/portainer/api/http/proxy/factory/kubernetes" - "github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/kubernetes/cli" portainer "github.com/portainer/portainer/api" @@ -43,12 +41,7 @@ func NewKubernetesDeployer(kubernetesTokenCacheManager *kubernetes.TokenCacheMan } } -func (deployer *KubernetesDeployer) getToken(request *http.Request, endpoint *portainer.Endpoint, setLocalAdminToken bool) (string, error) { - tokenData, err := security.RetrieveTokenData(request) - if err != nil { - return "", err - } - +func (deployer *KubernetesDeployer) getToken(userID portainer.UserID, endpoint *portainer.Endpoint, setLocalAdminToken bool) (string, error) { kubeCLI, err := deployer.kubernetesClientFactory.GetKubeClient(endpoint) if err != nil { return "", err @@ -61,11 +54,16 @@ func (deployer *KubernetesDeployer) getToken(request *http.Request, endpoint *po return "", err } - if tokenData.Role == portainer.AdministratorRole { + user, err := deployer.dataStore.User().User(userID) + if err != nil { + return "", errors.Wrap(err, "failed to fetch the user") + } + + if user.Role == portainer.AdministratorRole { return tokenManager.GetAdminServiceAccountToken(), nil } - token, err := tokenManager.GetUserServiceAccountToken(int(tokenData.ID), endpoint.ID) + token, err := tokenManager.GetUserServiceAccountToken(int(user.ID), endpoint.ID) if err != nil { return "", err } @@ -76,15 +74,31 @@ func (deployer *KubernetesDeployer) getToken(request *http.Request, endpoint *po return token, nil } -// Deploy will deploy a Kubernetes manifest inside an optional namespace in a Kubernetes environment(endpoint). -// Otherwise it will use kubectl to deploy the manifest. -func (deployer *KubernetesDeployer) Deploy(request *http.Request, endpoint *portainer.Endpoint, stackConfig string, namespace string) (string, error) { +// Deploy upserts Kubernetes resources defined in manifest(s) +func (deployer *KubernetesDeployer) Deploy(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) { + return deployer.command("apply", userID, endpoint, manifestFiles, namespace) +} + +// Remove deletes Kubernetes resources defined in manifest(s) +func (deployer *KubernetesDeployer) Remove(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) { + return deployer.command("delete", userID, endpoint, manifestFiles, namespace) +} + +func (deployer *KubernetesDeployer) command(operation string, userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) { + token, err := deployer.getToken(userID, endpoint, endpoint.Type == portainer.KubernetesLocalEnvironment) + if err != nil { + return "", errors.Wrap(err, "failed generating a user token") + } + command := path.Join(deployer.binaryPath, "kubectl") if runtime.GOOS == "windows" { command = path.Join(deployer.binaryPath, "kubectl.exe") } - args := make([]string, 0) + args := []string{"--token", token} + if namespace != "" { + args = append(args, "--namespace", namespace) + } if endpoint.Type == portainer.AgentOnKubernetesEnvironment || endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment { url, proxy, err := deployer.getAgentURL(endpoint) @@ -97,21 +111,18 @@ func (deployer *KubernetesDeployer) Deploy(request *http.Request, endpoint *port args = append(args, "--insecure-skip-tls-verify") } - token, err := deployer.getToken(request, endpoint, endpoint.Type == portainer.KubernetesLocalEnvironment) - if err != nil { - return "", err + if operation == "delete" { + args = append(args, "--ignore-not-found=true") } - args = append(args, "--token", token) - if namespace != "" { - args = append(args, "--namespace", namespace) + args = append(args, operation) + for _, path := range manifestFiles { + args = append(args, "-f", strings.TrimSpace(path)) } - args = append(args, "apply", "-f", "-") var stderr bytes.Buffer cmd := exec.Command(command, args...) cmd.Stderr = &stderr - cmd.Stdin = strings.NewReader(stackConfig) output, err := cmd.Output() if err != nil { diff --git a/api/filesystem/write.go b/api/filesystem/write.go new file mode 100644 index 000000000..235511933 --- /dev/null +++ b/api/filesystem/write.go @@ -0,0 +1,23 @@ +package filesystem + +import ( + "os" + "path/filepath" + + "github.com/pkg/errors" +) + +func WriteToFile(dst string, content []byte) error { + if err := os.MkdirAll(filepath.Dir(dst), 0744); err != nil { + return errors.Wrapf(err, "failed to create filestructure for the path %q", dst) + } + + file, err := os.Create(dst) + if err != nil { + return errors.Wrapf(err, "failed to open a file %q", dst) + } + defer file.Close() + + _, err = file.Write(content) + return errors.Wrapf(err, "failed to write a file %q", dst) +} diff --git a/api/filesystem/write_test.go b/api/filesystem/write_test.go new file mode 100644 index 000000000..89223a20e --- /dev/null +++ b/api/filesystem/write_test.go @@ -0,0 +1,48 @@ +package filesystem + +import ( + "io/ioutil" + "path" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_WriteFile_CanStoreContentInANewFile(t *testing.T) { + tmpDir := t.TempDir() + tmpFilePath := path.Join(tmpDir, "dummy") + + content := []byte("content") + err := WriteToFile(tmpFilePath, content) + assert.NoError(t, err) + + fileContent, _ := ioutil.ReadFile(tmpFilePath) + assert.Equal(t, content, fileContent) +} + +func Test_WriteFile_CanOverwriteExistingFile(t *testing.T) { + tmpDir := t.TempDir() + tmpFilePath := path.Join(tmpDir, "dummy") + + err := WriteToFile(tmpFilePath, []byte("content")) + assert.NoError(t, err) + + content := []byte("new content") + err = WriteToFile(tmpFilePath, content) + assert.NoError(t, err) + + fileContent, _ := ioutil.ReadFile(tmpFilePath) + assert.Equal(t, content, fileContent) +} + +func Test_WriteFile_CanWriteANestedPath(t *testing.T) { + tmpDir := t.TempDir() + tmpFilePath := path.Join(tmpDir, "dir", "sub-dir", "dummy") + + content := []byte("content") + err := WriteToFile(tmpFilePath, content) + assert.NoError(t, err) + + fileContent, _ := ioutil.ReadFile(tmpFilePath) + assert.Equal(t, content, fileContent) +} diff --git a/api/http/handler/helm/helm_install.go b/api/http/handler/helm/helm_install.go index 57e17825f..ab196d8c5 100644 --- a/api/http/handler/helm/helm_install.go +++ b/api/http/handler/helm/helm_install.go @@ -2,6 +2,7 @@ package helm import ( "fmt" + "io/ioutil" "net/http" "os" "strings" @@ -182,6 +183,11 @@ func (handler *Handler) updateHelmAppManifest(r *http.Request, manifest []byte, return errors.Wrap(err, "unable to find an endpoint on request context") } + tokenData, err := security.RetrieveTokenData(r) + if err != nil { + return errors.Wrap(err, "unable to retrieve user details from authentication token") + } + // extract list of yaml resources from helm manifest yamlResources, err := kubernetes.ExtractDocuments(manifest, nil) if err != nil { @@ -193,6 +199,19 @@ func (handler *Handler) updateHelmAppManifest(r *http.Request, manifest []byte, for _, resource := range yamlResources { resource := resource // https://golang.org/doc/faq#closures_and_goroutines g.Go(func() error { + tmpfile, err := ioutil.TempFile("", "helm-manifest-*") + if err != nil { + return errors.Wrap(err, "failed to create a tmp helm manifest file") + } + defer func() { + tmpfile.Close() + os.Remove(tmpfile.Name()) + }() + + if _, err := tmpfile.Write(resource); err != nil { + return errors.Wrap(err, "failed to write a tmp helm manifest file") + } + // get resource namespace, fallback to provided namespace if not explicit on resource resourceNamespace, err := kubernetes.GetNamespace(resource) if err != nil { @@ -201,7 +220,8 @@ func (handler *Handler) updateHelmAppManifest(r *http.Request, manifest []byte, if resourceNamespace == "" { resourceNamespace = namespace } - _, err = handler.kubernetesDeployer.Deploy(r, endpoint, string(resource), resourceNamespace) + + _, err = handler.kubernetesDeployer.Deploy(tokenData.ID, endpoint, []string{tmpfile.Name()}, resourceNamespace) return err }) } diff --git a/api/http/handler/stacks/autoupdate.go b/api/http/handler/stacks/autoupdate.go index 867e71fc2..237b0b58a 100644 --- a/api/http/handler/stacks/autoupdate.go +++ b/api/http/handler/stacks/autoupdate.go @@ -17,10 +17,8 @@ func startAutoupdate(stackID portainer.StackID, interval string, scheduler *sche return "", &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Unable to parse stack's auto update interval", Err: err} } - jobID = scheduler.StartJobEvery(d, func() { - if err := stacks.RedeployWhenChanged(stackID, stackDeployer, datastore, gitService); err != nil { - log.Printf("[ERROR] [http,stacks] [message: failed redeploying] [err: %s]\n", err) - } + jobID = scheduler.StartJobEvery(d, func() error { + return stacks.RedeployWhenChanged(stackID, stackDeployer, datastore, gitService) }) return jobID, nil diff --git a/api/http/handler/stacks/create_compose_stack.go b/api/http/handler/stacks/create_compose_stack.go index fe6d30c97..7f3615500 100644 --- a/api/http/handler/stacks/create_compose_stack.go +++ b/api/http/handler/stacks/create_compose_stack.go @@ -46,7 +46,7 @@ func (handler *Handler) createComposeStackFromFileContent(w http.ResponseWriter, payload.Name = handler.ComposeStackManager.NormalizeStackName(payload.Name) - isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, false) + isUnique, err := handler.checkUniqueStackNameInDocker(endpoint, payload.Name, 0, false) if err != nil { return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err} } @@ -152,7 +152,7 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite payload.ComposeFile = filesystem.ComposeFileDefaultName } - isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, false) + isUnique, err := handler.checkUniqueStackNameInDocker(endpoint, payload.Name, 0, false) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err} } @@ -208,11 +208,11 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err} } - commitId, err := handler.latestCommitID(payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) + commitID, err := handler.latestCommitID(payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to fetch git repository id", Err: err} } - stack.GitConfig.ConfigHash = commitId + stack.GitConfig.ConfigHash = commitID config, configErr := handler.createComposeDeployConfig(r, stack, endpoint) if configErr != nil { @@ -281,7 +281,7 @@ func (handler *Handler) createComposeStackFromFileUpload(w http.ResponseWriter, payload.Name = handler.ComposeStackManager.NormalizeStackName(payload.Name) - isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, false) + isUnique, err := handler.checkUniqueStackNameInDocker(endpoint, payload.Name, 0, false) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err} } diff --git a/api/http/handler/stacks/create_kubernetes_stack.go b/api/http/handler/stacks/create_kubernetes_stack.go index f4133ddae..654bda92f 100644 --- a/api/http/handler/stacks/create_kubernetes_stack.go +++ b/api/http/handler/stacks/create_kubernetes_stack.go @@ -2,9 +2,8 @@ package stacks import ( "fmt" - "io/ioutil" "net/http" - "path/filepath" + "os" "strconv" "time" @@ -19,16 +18,19 @@ import ( "github.com/portainer/portainer/api/filesystem" gittypes "github.com/portainer/portainer/api/git/types" "github.com/portainer/portainer/api/http/client" + "github.com/portainer/portainer/api/internal/stackutils" k "github.com/portainer/portainer/api/kubernetes" ) type kubernetesStringDeploymentPayload struct { + StackName string ComposeFormat bool Namespace string StackFileContent string } type kubernetesGitDeploymentPayload struct { + StackName string ComposeFormat bool Namespace string RepositoryURL string @@ -36,10 +38,13 @@ type kubernetesGitDeploymentPayload struct { RepositoryAuthentication bool RepositoryUsername string RepositoryPassword string - FilePathInRepository string + ManifestFile string + AdditionalFiles []string + AutoUpdate *portainer.StackAutoUpdate } type kubernetesManifestURLDeploymentPayload struct { + StackName string Namespace string ComposeFormat bool ManifestURL string @@ -52,6 +57,9 @@ func (payload *kubernetesStringDeploymentPayload) Validate(r *http.Request) erro if govalidator.IsNull(payload.Namespace) { return errors.New("Invalid namespace") } + if govalidator.IsNull(payload.StackName) { + return errors.New("Invalid stack name") + } return nil } @@ -65,12 +73,18 @@ func (payload *kubernetesGitDeploymentPayload) Validate(r *http.Request) error { if payload.RepositoryAuthentication && govalidator.IsNull(payload.RepositoryPassword) { return errors.New("Invalid repository credentials. Password must be specified when authentication is enabled") } - if govalidator.IsNull(payload.FilePathInRepository) { - return errors.New("Invalid file path in repository") + if govalidator.IsNull(payload.ManifestFile) { + return errors.New("Invalid manifest file in repository") } if govalidator.IsNull(payload.RepositoryReferenceName) { payload.RepositoryReferenceName = defaultGitReferenceName } + if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil { + return err + } + if govalidator.IsNull(payload.StackName) { + return errors.New("Invalid stack name") + } return nil } @@ -78,6 +92,9 @@ func (payload *kubernetesManifestURLDeploymentPayload) Validate(r *http.Request) if govalidator.IsNull(payload.ManifestURL) || !govalidator.IsURL(payload.ManifestURL) { return errors.New("Invalid manifest URL") } + if govalidator.IsNull(payload.StackName) { + return errors.New("Invalid stack name") + } return nil } @@ -95,6 +112,13 @@ func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWrit if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to load user information from the database", Err: err} } + isUnique, err := handler.checkUniqueStackName(endpoint, payload.StackName, 0) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err} + } + if !isUnique { + return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.StackName), Err: errStackAlreadyExists} + } stackID := handler.DataStore.Stack().GetNextIdentifier() stack := &portainer.Stack{ @@ -102,6 +126,7 @@ func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWrit Type: portainer.KubernetesStack, EndpointID: endpoint.ID, EntryPoint: filesystem.ManifestFileDefaultName, + Name: payload.StackName, Namespace: payload.Namespace, Status: portainer.StackStatusActive, CreationDate: time.Now().Unix(), @@ -124,11 +149,11 @@ func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWrit doCleanUp := true defer handler.cleanUp(stack, &doCleanUp) - output, err := handler.deployKubernetesStack(r, endpoint, payload.StackFileContent, payload.ComposeFormat, payload.Namespace, k.KubeAppLabels{ - StackID: stackID, - Name: stack.Name, - Owner: stack.CreatedBy, - Kind: "content", + output, err := handler.deployKubernetesStack(user.ID, endpoint, stack, k.KubeAppLabels{ + StackID: stackID, + StackName: stack.Name, + Owner: stack.CreatedBy, + Kind: "content", }) if err != nil { @@ -140,12 +165,11 @@ func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWrit return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the Kubernetes stack inside the database", Err: err} } - doCleanUp = false - resp := &createKubernetesStackResponse{ Output: output, } + doCleanUp = false return response.JSON(w, resp) } @@ -159,23 +183,44 @@ func (handler *Handler) createKubernetesStackFromGitRepository(w http.ResponseWr if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to load user information from the database", Err: err} } + isUnique, err := handler.checkUniqueStackName(endpoint, payload.StackName, 0) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err} + } + if !isUnique { + return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.StackName), Err: errStackAlreadyExists} + } + + //make sure the webhook ID is unique + if payload.AutoUpdate != nil && payload.AutoUpdate.Webhook != "" { + isUnique, err := handler.checkUniqueWebhookID(payload.AutoUpdate.Webhook) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for webhook ID collision", Err: err} + } + if !isUnique { + return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("Webhook ID: %s already exists", payload.AutoUpdate.Webhook), Err: errWebhookIDAlreadyExists} + } + } stackID := handler.DataStore.Stack().GetNextIdentifier() stack := &portainer.Stack{ ID: portainer.StackID(stackID), Type: portainer.KubernetesStack, EndpointID: endpoint.ID, - EntryPoint: payload.FilePathInRepository, + EntryPoint: payload.ManifestFile, GitConfig: &gittypes.RepoConfig{ URL: payload.RepositoryURL, ReferenceName: payload.RepositoryReferenceName, - ConfigFilePath: payload.FilePathInRepository, + ConfigFilePath: payload.ManifestFile, }, Namespace: payload.Namespace, + Name: payload.StackName, Status: portainer.StackStatusActive, CreationDate: time.Now().Unix(), CreatedBy: user.Username, IsComposeFormat: payload.ComposeFormat, + AutoUpdate: payload.AutoUpdate, + AdditionalFiles: payload.AdditionalFiles, } if payload.RepositoryAuthentication { @@ -197,33 +242,48 @@ func (handler *Handler) createKubernetesStackFromGitRepository(w http.ResponseWr } stack.GitConfig.ConfigHash = commitID - stackFileContent, err := handler.cloneManifestContentFromGitRepo(&payload, stack.ProjectPath) - if err != nil { - return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Failed to process manifest from Git repository", Err: err} + repositoryUsername := payload.RepositoryUsername + repositoryPassword := payload.RepositoryPassword + if !payload.RepositoryAuthentication { + repositoryUsername = "" + repositoryPassword = "" } - output, err := handler.deployKubernetesStack(r, endpoint, stackFileContent, payload.ComposeFormat, payload.Namespace, k.KubeAppLabels{ - StackID: stackID, - Name: stack.Name, - Owner: stack.CreatedBy, - Kind: "git", + err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Failed to clone git repository", Err: err} + } + + output, err := handler.deployKubernetesStack(user.ID, endpoint, stack, k.KubeAppLabels{ + StackID: stackID, + StackName: stack.Name, + Owner: stack.CreatedBy, + Kind: "git", }) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to deploy Kubernetes stack", Err: err} } + if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" { + jobID, e := startAutoupdate(stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService) + if e != nil { + return e + } + + stack.AutoUpdate.JobID = jobID + } + err = handler.DataStore.Stack().CreateStack(stack) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack inside the database", Err: err} } - doCleanUp = false - resp := &createKubernetesStackResponse{ Output: output, } + doCleanUp = false return response.JSON(w, resp) } @@ -237,6 +297,13 @@ func (handler *Handler) createKubernetesStackFromManifestURL(w http.ResponseWrit if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to load user information from the database", Err: err} } + isUnique, err := handler.checkUniqueStackName(endpoint, payload.StackName, 0) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err} + } + if !isUnique { + return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.StackName), Err: errStackAlreadyExists} + } stackID := handler.DataStore.Stack().GetNextIdentifier() stack := &portainer.Stack{ @@ -245,6 +312,7 @@ func (handler *Handler) createKubernetesStackFromManifestURL(w http.ResponseWrit EndpointID: endpoint.ID, EntryPoint: filesystem.ManifestFileDefaultName, Namespace: payload.Namespace, + Name: payload.StackName, Status: portainer.StackStatusActive, CreationDate: time.Now().Unix(), CreatedBy: user.Username, @@ -267,11 +335,11 @@ func (handler *Handler) createKubernetesStackFromManifestURL(w http.ResponseWrit doCleanUp := true defer handler.cleanUp(stack, &doCleanUp) - output, err := handler.deployKubernetesStack(r, endpoint, string(manifestContent), payload.ComposeFormat, payload.Namespace, k.KubeAppLabels{ - StackID: stackID, - Name: stack.Name, - Owner: stack.CreatedBy, - Kind: "url", + output, err := handler.deployKubernetesStack(user.ID, endpoint, stack, k.KubeAppLabels{ + StackID: stackID, + StackName: stack.Name, + Owner: stack.CreatedBy, + Kind: "url", }) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to deploy Kubernetes stack", Err: err} @@ -291,42 +359,14 @@ func (handler *Handler) createKubernetesStackFromManifestURL(w http.ResponseWrit return response.JSON(w, resp) } -func (handler *Handler) deployKubernetesStack(request *http.Request, endpoint *portainer.Endpoint, stackConfig string, composeFormat bool, namespace string, appLabels k.KubeAppLabels) (string, error) { +func (handler *Handler) deployKubernetesStack(userID portainer.UserID, endpoint *portainer.Endpoint, stack *portainer.Stack, appLabels k.KubeAppLabels) (string, error) { handler.stackCreationMutex.Lock() defer handler.stackCreationMutex.Unlock() - manifest := []byte(stackConfig) - if composeFormat { - convertedConfig, err := handler.KubernetesDeployer.ConvertCompose(manifest) - if err != nil { - return "", errors.Wrap(err, "failed to convert docker compose file to a kube manifest") - } - manifest = convertedConfig - } - - manifest, err := k.AddAppLabels(manifest, appLabels.ToMap()) + manifestFilePaths, tempDir, err := stackutils.CreateTempK8SDeploymentFiles(stack, handler.KubernetesDeployer, appLabels) if err != nil { - return "", errors.Wrap(err, "failed to add application labels") + return "", errors.Wrap(err, "failed to create temp kub deployment files") } - - return handler.KubernetesDeployer.Deploy(request, endpoint, string(manifest), namespace) -} - -func (handler *Handler) cloneManifestContentFromGitRepo(gitInfo *kubernetesGitDeploymentPayload, projectPath string) (string, error) { - repositoryUsername := gitInfo.RepositoryUsername - repositoryPassword := gitInfo.RepositoryPassword - if !gitInfo.RepositoryAuthentication { - repositoryUsername = "" - repositoryPassword = "" - } - - err := handler.GitService.CloneRepository(projectPath, gitInfo.RepositoryURL, gitInfo.RepositoryReferenceName, repositoryUsername, repositoryPassword) - if err != nil { - return "", err - } - content, err := ioutil.ReadFile(filepath.Join(projectPath, gitInfo.FilePathInRepository)) - if err != nil { - return "", err - } - return string(content), nil + defer os.RemoveAll(tempDir) + return handler.KubernetesDeployer.Deploy(userID, endpoint, manifestFilePaths, stack.Namespace) } diff --git a/api/http/handler/stacks/create_kubernetes_stack_test.go b/api/http/handler/stacks/create_kubernetes_stack_test.go deleted file mode 100644 index 2bcd35ab5..000000000 --- a/api/http/handler/stacks/create_kubernetes_stack_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package stacks - -import ( - "io/ioutil" - "os" - "path" - "testing" - - "github.com/stretchr/testify/assert" -) - -type git struct { - content string -} - -func (g *git) CloneRepository(destination string, repositoryURL, referenceName, username, password string) error { - return g.ClonePublicRepository(repositoryURL, referenceName, destination) -} -func (g *git) ClonePublicRepository(repositoryURL string, referenceName string, destination string) error { - return ioutil.WriteFile(path.Join(destination, "deployment.yml"), []byte(g.content), 0755) -} -func (g *git) ClonePrivateRepositoryWithBasicAuth(repositoryURL, referenceName string, destination, username, password string) error { - return g.ClonePublicRepository(repositoryURL, referenceName, destination) -} - -func (g *git) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) { - return "", nil -} - -func TestCloneAndConvertGitRepoFile(t *testing.T) { - dir, err := os.MkdirTemp("", "kube-create-stack") - assert.NoError(t, err, "failed to create a tmp dir") - defer os.RemoveAll(dir) - - content := `apiVersion: apps/v1 - kind: Deployment - metadata: - name: nginx-deployment - labels: - app: nginx - spec: - replicas: 3 - selector: - matchLabels: - app: nginx - template: - metadata: - labels: - app: nginx - spec: - containers: - - name: nginx - image: nginx:1.14.2 - ports: - - containerPort: 80` - - h := &Handler{ - GitService: &git{ - content: content, - }, - } - gitInfo := &kubernetesGitDeploymentPayload{ - FilePathInRepository: "deployment.yml", - } - fileContent, err := h.cloneManifestContentFromGitRepo(gitInfo, dir) - assert.NoError(t, err, "failed to clone or convert the file from Git repo") - assert.Equal(t, content, fileContent) -} diff --git a/api/http/handler/stacks/create_swarm_stack.go b/api/http/handler/stacks/create_swarm_stack.go index 7e615acc3..e898b5502 100644 --- a/api/http/handler/stacks/create_swarm_stack.go +++ b/api/http/handler/stacks/create_swarm_stack.go @@ -51,7 +51,8 @@ func (handler *Handler) createSwarmStackFromFileContent(w http.ResponseWriter, r payload.Name = handler.SwarmStackManager.NormalizeStackName(payload.Name) - isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, true) + isUnique, err := handler.checkUniqueStackNameInDocker(endpoint, payload.Name, 0, true) + if err != nil { return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err} } @@ -161,7 +162,7 @@ func (handler *Handler) createSwarmStackFromGitRepository(w http.ResponseWriter, payload.Name = handler.SwarmStackManager.NormalizeStackName(payload.Name) - isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, true) + isUnique, err := handler.checkUniqueStackNameInDocker(endpoint, payload.Name, 0, true) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err} } @@ -218,11 +219,11 @@ func (handler *Handler) createSwarmStackFromGitRepository(w http.ResponseWriter, return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err} } - commitId, err := handler.latestCommitID(payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) + commitID, err := handler.latestCommitID(payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to fetch git repository id", Err: err} } - stack.GitConfig.ConfigHash = commitId + stack.GitConfig.ConfigHash = commitID config, configErr := handler.createSwarmDeployConfig(r, stack, endpoint, false) if configErr != nil { @@ -298,7 +299,8 @@ func (handler *Handler) createSwarmStackFromFileUpload(w http.ResponseWriter, r payload.Name = handler.SwarmStackManager.NormalizeStackName(payload.Name) - isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, true) + isUnique, err := handler.checkUniqueStackNameInDocker(endpoint, payload.Name, 0, true) + if err != nil { return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err} } diff --git a/api/http/handler/stacks/handler.go b/api/http/handler/stacks/handler.go index f5ba2c983..8795e0fe3 100644 --- a/api/http/handler/stacks/handler.go +++ b/api/http/handler/stacks/handler.go @@ -127,7 +127,7 @@ func (handler *Handler) userCanCreateStack(securityContext *security.RestrictedR return handler.userIsAdminOrEndpointAdmin(user, endpointID) } -func (handler *Handler) checkUniqueName(endpoint *portainer.Endpoint, name string, stackID portainer.StackID, swarmMode bool) (bool, error) { +func (handler *Handler) checkUniqueStackName(endpoint *portainer.Endpoint, name string, stackID portainer.StackID) (bool, error) { stacks, err := handler.DataStore.Stack().Stacks() if err != nil { return false, err @@ -139,6 +139,15 @@ func (handler *Handler) checkUniqueName(endpoint *portainer.Endpoint, name strin } } + return true, nil +} + +func (handler *Handler) checkUniqueStackNameInDocker(endpoint *portainer.Endpoint, name string, stackID portainer.StackID, swarmMode bool) (bool, error) { + isUniqueStackName, err := handler.checkUniqueStackName(endpoint, name, stackID) + if err != nil { + return false, err + } + dockerClient, err := handler.DockerClientFactory.CreateClient(endpoint, "") if err != nil { return false, err @@ -171,7 +180,7 @@ func (handler *Handler) checkUniqueName(endpoint *portainer.Endpoint, name strin } } - return true, nil + return isUniqueStackName, nil } func (handler *Handler) checkUniqueWebhookID(webhookID string) (bool, error) { diff --git a/api/http/handler/stacks/stack_delete.go b/api/http/handler/stacks/stack_delete.go index cd5e2cd26..799f6f612 100644 --- a/api/http/handler/stacks/stack_delete.go +++ b/api/http/handler/stacks/stack_delete.go @@ -2,15 +2,20 @@ package stacks import ( "context" - "errors" + "fmt" + "io/ioutil" "net/http" + "os" + "path" "strconv" + "github.com/pkg/errors" httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" "github.com/portainer/libhttp/response" portainer "github.com/portainer/portainer/api" bolterrors "github.com/portainer/portainer/api/bolt/errors" + "github.com/portainer/portainer/api/filesystem" httperrors "github.com/portainer/portainer/api/http/errors" "github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/internal/stackutils" @@ -34,12 +39,12 @@ import ( func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { stackID, err := request.RetrieveRouteVariableValue(r, "id") if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid stack identifier route variable", Err: err} } securityContext, err := security.RetrieveRestrictedRequestContext(r) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err} } externalStack, _ := request.RetrieveBooleanQueryParameter(r, "external", true) @@ -49,52 +54,52 @@ func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *htt id, err := strconv.Atoi(stackID) if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid stack identifier route variable", Err: err} } stack, err := handler.DataStore.Stack().Stack(portainer.StackID(id)) if err == bolterrors.ErrObjectNotFound { - return &httperror.HandlerError{http.StatusNotFound, "Unable to find a stack with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find a stack with the specified identifier inside the database", Err: err} } else if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find a stack with the specified identifier inside the database", Err: err} } endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", true) if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: endpointId", err} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid query parameter: endpointId", Err: err} } isOrphaned := portainer.EndpointID(endpointID) != stack.EndpointID if isOrphaned && !securityContext.IsAdmin { - return &httperror.HandlerError{http.StatusForbidden, "Permission denied to remove orphaned stack", errors.New("Permission denied to remove orphaned stack")} + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to remove orphaned stack", Err: errors.New("Permission denied to remove orphaned stack")} } endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID)) if err == bolterrors.ErrObjectNotFound { - return &httperror.HandlerError{http.StatusNotFound, "Unable to find the environment associated to the stack inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err} } else if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find the environment associated to the stack inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err} } resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve a resource control associated to the stack", Err: err} } if !isOrphaned { err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint) if err != nil { - return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access environment", err} + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to access endpoint", Err: err} } if stack.Type == portainer.DockerSwarmStack || stack.Type == portainer.DockerComposeStack { access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to verify user authorizations to validate stack access", Err: err} } if !access { - return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", httperrors.ErrResourceAccessDenied} + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Access denied to resource", Err: httperrors.ErrResourceAccessDenied} } } } @@ -104,26 +109,26 @@ func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *htt stopAutoupdate(stack.ID, stack.AutoUpdate.JobID, *handler.Scheduler) } - err = handler.deleteStack(stack, endpoint) + err = handler.deleteStack(securityContext.UserID, stack, endpoint) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err} } err = handler.DataStore.Stack().DeleteStack(portainer.StackID(id)) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to remove the stack from the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to remove the stack from the database", Err: err} } if resourceControl != nil { err = handler.DataStore.ResourceControl().DeleteResourceControl(resourceControl.ID) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to remove the associated resource control from the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to remove the associated resource control from the database", Err: err} } } err = handler.FileService.RemoveDirectory(stack.ProjectPath) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to remove stack files from disk", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to remove stack files from disk", Err: err} } return response.Empty(w) @@ -132,31 +137,31 @@ func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *htt func (handler *Handler) deleteExternalStack(r *http.Request, w http.ResponseWriter, stackName string, securityContext *security.RestrictedRequestContext) *httperror.HandlerError { endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", false) if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: endpointId", err} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid query parameter: endpointId", Err: err} } if !securityContext.IsAdmin { - return &httperror.HandlerError{http.StatusUnauthorized, "Permission denied to delete the stack", httperrors.ErrUnauthorized} + return &httperror.HandlerError{StatusCode: http.StatusUnauthorized, Message: "Permission denied to delete the stack", Err: httperrors.ErrUnauthorized} } stack, err := handler.DataStore.Stack().StackByName(stackName) if err != nil && err != bolterrors.ErrObjectNotFound { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for stack existence inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for stack existence inside the database", Err: err} } if stack != nil { - return &httperror.HandlerError{http.StatusBadRequest, "A stack with this name exists inside the database. Cannot use external delete method", errors.New("A tag already exists with this name")} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "A stack with this name exists inside the database. Cannot use external delete method", Err: errors.New("A tag already exists with this name")} } endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID)) if err == bolterrors.ErrObjectNotFound { - return &httperror.HandlerError{http.StatusNotFound, "Unable to find the environment associated to the stack inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err} } else if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find the environment associated to the stack inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err} } err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint) if err != nil { - return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access environment", err} + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to access endpoint", Err: err} } stack = &portainer.Stack{ @@ -164,18 +169,57 @@ func (handler *Handler) deleteExternalStack(r *http.Request, w http.ResponseWrit Type: portainer.DockerSwarmStack, } - err = handler.deleteStack(stack, endpoint) + err = handler.deleteStack(securityContext.UserID, stack, endpoint) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to delete stack", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to delete stack", Err: err} } return response.Empty(w) } -func (handler *Handler) deleteStack(stack *portainer.Stack, endpoint *portainer.Endpoint) error { +func (handler *Handler) deleteStack(userID portainer.UserID, stack *portainer.Stack, endpoint *portainer.Endpoint) error { if stack.Type == portainer.DockerSwarmStack { return handler.SwarmStackManager.Remove(stack, endpoint) } + if stack.Type == portainer.DockerComposeStack { + return handler.ComposeStackManager.Down(context.TODO(), stack, endpoint) + } + if stack.Type == portainer.KubernetesStack { + var manifestFiles []string - return handler.ComposeStackManager.Down(context.TODO(), stack, endpoint) + //if it is a compose format kub stack, create a temp dir and convert the manifest files into it + //then process the remove operation + if stack.IsComposeFormat { + fileNames := append([]string{stack.EntryPoint}, stack.AdditionalFiles...) + tmpDir, err := ioutil.TempDir("", "kub_delete") + if err != nil { + return errors.Wrap(err, "failed to create temp directory for deleting kub stack") + } + defer os.RemoveAll(tmpDir) + + for _, fileName := range fileNames { + manifestFilePath := path.Join(tmpDir, fileName) + manifestContent, err := ioutil.ReadFile(path.Join(stack.ProjectPath, fileName)) + if err != nil { + return errors.Wrap(err, "failed to read manifest file") + } + + manifestContent, err = handler.KubernetesDeployer.ConvertCompose(manifestContent) + if err != nil { + return errors.Wrap(err, "failed to convert docker compose file to a kube manifest") + } + + err = filesystem.WriteToFile(manifestFilePath, []byte(manifestContent)) + if err != nil { + return errors.Wrap(err, "failed to create temp manifest file") + } + manifestFiles = append(manifestFiles, manifestFilePath) + } + } else { + manifestFiles = stackutils.GetStackFilePaths(stack) + } + out, err := handler.KubernetesDeployer.Remove(userID, endpoint, manifestFiles, stack.Namespace) + return errors.WithMessagef(err, "failed to remove kubernetes resources: %q", out) + } + return fmt.Errorf("unsupported stack type: %v", stack.Type) } diff --git a/api/http/handler/stacks/stack_migrate.go b/api/http/handler/stacks/stack_migrate.go index c631f8fe3..d69666d90 100644 --- a/api/http/handler/stacks/stack_migrate.go +++ b/api/http/handler/stacks/stack_migrate.go @@ -50,52 +50,54 @@ func (payload *stackMigratePayload) Validate(r *http.Request) error { func (handler *Handler) stackMigrate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { stackID, err := request.RetrieveNumericRouteVariableValue(r, "id") if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid stack identifier route variable", Err: err} } var payload stackMigratePayload err = request.DecodeAndValidateJSONPayload(r, &payload) if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err} } stack, err := handler.DataStore.Stack().Stack(portainer.StackID(stackID)) if err == bolterrors.ErrObjectNotFound { - return &httperror.HandlerError{http.StatusNotFound, "Unable to find a stack with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find a stack with the specified identifier inside the database", Err: err} } else if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find a stack with the specified identifier inside the database", Err: err} + } + + if stack.Type == portainer.KubernetesStack { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Migrating a kubernetes stack is not supported", Err: err} } endpoint, err := handler.DataStore.Endpoint().Endpoint(stack.EndpointID) if err == bolterrors.ErrObjectNotFound { - return &httperror.HandlerError{http.StatusNotFound, "Unable to find an environment with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find an endpoint with the specified identifier inside the database", Err: err} } else if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an environment with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find an endpoint with the specified identifier inside the database", Err: err} } err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint) if err != nil { - return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access environment", err} + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to access endpoint", Err: err} } - if stack.Type == portainer.DockerSwarmStack || stack.Type == portainer.DockerComposeStack { - resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl) - if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err} - } + securityContext, err := security.RetrieveRestrictedRequestContext(r) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err} + } - securityContext, err := security.RetrieveRestrictedRequestContext(r) - if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err} - } + resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve a resource control associated to the stack", Err: err} + } - access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl) - if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err} - } - if !access { - return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", httperrors.ErrResourceAccessDenied} - } + access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to verify user authorizations to validate stack access", Err: err} + } + if !access { + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Access denied to resource", Err: httperrors.ErrResourceAccessDenied} } // TODO: this is a work-around for stacks created with Portainer version >= 1.17.1 @@ -103,7 +105,7 @@ func (handler *Handler) stackMigrate(w http.ResponseWriter, r *http.Request) *ht // can use the optional EndpointID query parameter to associate a valid environment(endpoint) identifier to the stack. endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", true) if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: endpointId", err} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid query parameter: endpointId", Err: err} } if endpointID != int(stack.EndpointID) { stack.EndpointID = portainer.EndpointID(endpointID) @@ -111,9 +113,9 @@ func (handler *Handler) stackMigrate(w http.ResponseWriter, r *http.Request) *ht targetEndpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(payload.EndpointID)) if err == bolterrors.ErrObjectNotFound { - return &httperror.HandlerError{http.StatusNotFound, "Unable to find an environment with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find an endpoint with the specified identifier inside the database", Err: err} } else if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an environment with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find an endpoint with the specified identifier inside the database", Err: err} } stack.EndpointID = portainer.EndpointID(payload.EndpointID) @@ -126,14 +128,14 @@ func (handler *Handler) stackMigrate(w http.ResponseWriter, r *http.Request) *ht stack.Name = payload.Name } - isUnique, err := handler.checkUniqueName(targetEndpoint, stack.Name, stack.ID, stack.SwarmID != "") + isUnique, err := handler.checkUniqueStackNameInDocker(targetEndpoint, stack.Name, stack.ID, stack.SwarmID != "") if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err} } if !isUnique { - errorMessage := fmt.Sprintf("A stack with the name '%s' is already running on environment '%s'", stack.Name, targetEndpoint.Name) - return &httperror.HandlerError{http.StatusConflict, errorMessage, errors.New(errorMessage)} + errorMessage := fmt.Sprintf("A stack with the name '%s' is already running on endpoint '%s'", stack.Name, targetEndpoint.Name) + return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: errorMessage, Err: errors.New(errorMessage)} } migrationError := handler.migrateStack(r, stack, targetEndpoint) @@ -142,14 +144,14 @@ func (handler *Handler) stackMigrate(w http.ResponseWriter, r *http.Request) *ht } stack.Name = oldName - err = handler.deleteStack(stack, endpoint) + err = handler.deleteStack(securityContext.UserID, stack, endpoint) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err} } err = handler.DataStore.Stack().UpdateStack(stack.ID, stack) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack changes inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack changes inside the database", Err: err} } if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { @@ -175,7 +177,7 @@ func (handler *Handler) migrateComposeStack(r *http.Request, stack *portainer.St err := handler.deployComposeStack(config) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err} } return nil @@ -189,7 +191,7 @@ func (handler *Handler) migrateSwarmStack(r *http.Request, stack *portainer.Stac err := handler.deploySwarmStack(config) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err} } return nil diff --git a/api/http/handler/stacks/stack_start.go b/api/http/handler/stacks/stack_start.go index 3a6ca0285..7c1f20f97 100644 --- a/api/http/handler/stacks/stack_start.go +++ b/api/http/handler/stacks/stack_start.go @@ -33,59 +33,61 @@ import ( func (handler *Handler) stackStart(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { stackID, err := request.RetrieveNumericRouteVariableValue(r, "id") if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid stack identifier route variable", Err: err} } securityContext, err := security.RetrieveRestrictedRequestContext(r) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err} } stack, err := handler.DataStore.Stack().Stack(portainer.StackID(stackID)) if err == bolterrors.ErrObjectNotFound { - return &httperror.HandlerError{http.StatusNotFound, "Unable to find a stack with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find a stack with the specified identifier inside the database", Err: err} } else if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find a stack with the specified identifier inside the database", Err: err} + } + + if stack.Type == portainer.KubernetesStack { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Starting a kubernetes stack is not supported", Err: err} } endpoint, err := handler.DataStore.Endpoint().Endpoint(stack.EndpointID) if err == bolterrors.ErrObjectNotFound { - return &httperror.HandlerError{http.StatusNotFound, "Unable to find an environment with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find an endpoint with the specified identifier inside the database", Err: err} } else if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an environment with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find an endpoint with the specified identifier inside the database", Err: err} } err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint) if err != nil { - return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access environment", err} + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to access endpoint", Err: err} } - isUnique, err := handler.checkUniqueName(endpoint, stack.Name, stack.ID, stack.SwarmID != "") + isUnique, err := handler.checkUniqueStackNameInDocker(endpoint, stack.Name, stack.ID, stack.SwarmID != "") if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err} } if !isUnique { errorMessage := fmt.Sprintf("A stack with the name '%s' is already running", stack.Name) - return &httperror.HandlerError{http.StatusConflict, errorMessage, errors.New(errorMessage)} + return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: errorMessage, Err: errors.New(errorMessage)} } - if stack.Type == portainer.DockerSwarmStack || stack.Type == portainer.DockerComposeStack { - resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl) - if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err} - } + resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve a resource control associated to the stack", Err: err} + } - access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl) - if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err} - } - if !access { - return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", httperrors.ErrResourceAccessDenied} - } + access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to verify user authorizations to validate stack access", Err: err} + } + if !access { + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Access denied to resource", Err: httperrors.ErrResourceAccessDenied} } if stack.Status == portainer.StackStatusActive { - return &httperror.HandlerError{http.StatusBadRequest, "Stack is already active", errors.New("Stack is already active")} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Stack is already active", Err: errors.New("Stack is already active")} } if stack.AutoUpdate != nil && stack.AutoUpdate.Interval != "" { @@ -101,13 +103,13 @@ func (handler *Handler) stackStart(w http.ResponseWriter, r *http.Request) *http err = handler.startStack(stack, endpoint) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to start stack", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to start stack", Err: err} } stack.Status = portainer.StackStatusActive err = handler.DataStore.Stack().UpdateStack(stack.ID, stack) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update stack status", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to update stack status", Err: err} } if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { diff --git a/api/http/handler/stacks/stack_stop.go b/api/http/handler/stacks/stack_stop.go index 6aea8e375..0b9816ef4 100644 --- a/api/http/handler/stacks/stack_stop.go +++ b/api/http/handler/stacks/stack_stop.go @@ -46,6 +46,10 @@ func (handler *Handler) stackStop(w http.ResponseWriter, r *http.Request) *httpe return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err} } + if stack.Type == portainer.KubernetesStack { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Stopping a kubernetes stack is not supported", Err: err} + } + endpoint, err := handler.DataStore.Endpoint().Endpoint(stack.EndpointID) if err == bolterrors.ErrObjectNotFound { return &httperror.HandlerError{http.StatusNotFound, "Unable to find an environment with the specified identifier inside the database", err} @@ -58,19 +62,17 @@ func (handler *Handler) stackStop(w http.ResponseWriter, r *http.Request) *httpe return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access environment", err} } - if stack.Type == portainer.DockerSwarmStack || stack.Type == portainer.DockerComposeStack { - resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl) - if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err} - } + resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl) + if err != nil { + return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err} + } - access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl) - if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err} - } - if !access { - return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", httperrors.ErrResourceAccessDenied} - } + access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl) + if err != nil { + return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err} + } + if !access { + return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", httperrors.ErrResourceAccessDenied} } if stack.Status == portainer.StackStatusInactive { diff --git a/api/http/handler/stacks/stack_update_git.go b/api/http/handler/stacks/stack_update_git.go index 45576fe61..a37ba54f5 100644 --- a/api/http/handler/stacks/stack_update_git.go +++ b/api/http/handler/stacks/stack_update_git.go @@ -1,10 +1,11 @@ package stacks import ( - "errors" "net/http" + "time" "github.com/asaskevich/govalidator" + "github.com/pkg/errors" httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" "github.com/portainer/libhttp/response" @@ -98,17 +99,22 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) * return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to access environment", Err: err} } + securityContext, err := security.RetrieveRestrictedRequestContext(r) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err} + } + + user, err := handler.DataStore.User().User(securityContext.UserID) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Cannot find context user", Err: errors.Wrap(err, "failed to fetch the user")} + } + if stack.Type == portainer.DockerSwarmStack || stack.Type == portainer.DockerComposeStack { resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve a resource control associated to the stack", Err: err} } - securityContext, err := security.RetrieveRestrictedRequestContext(r) - if err != nil { - return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err} - } - access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to verify user authorizations to validate stack access", Err: err} @@ -127,6 +133,8 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) * stack.GitConfig.ReferenceName = payload.RepositoryReferenceName stack.AutoUpdate = payload.AutoUpdate stack.Env = payload.Env + stack.UpdatedBy = user.Username + stack.UpdateDate = time.Now().Unix() stack.GitConfig.Authentication = nil if payload.RepositoryAuthentication { diff --git a/api/http/handler/stacks/stack_update_git_redeploy.go b/api/http/handler/stacks/stack_update_git_redeploy.go index d5d87c4f0..300a33d07 100644 --- a/api/http/handler/stacks/stack_update_git_redeploy.go +++ b/api/http/handler/stacks/stack_update_git_redeploy.go @@ -2,10 +2,8 @@ package stacks import ( "fmt" - "io/ioutil" "log" "net/http" - "path/filepath" "time" "github.com/asaskevich/govalidator" @@ -216,15 +214,15 @@ func (handler *Handler) deployStack(r *http.Request, stack *portainer.Stack, end if stack.Namespace == "" { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Invalid namespace", Err: errors.New("Namespace must not be empty when redeploying kubernetes stacks")} } - content, err := ioutil.ReadFile(filepath.Join(stack.ProjectPath, stack.GitConfig.ConfigFilePath)) + tokenData, err := security.RetrieveTokenData(r) if err != nil { - return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to read deployment.yml manifest file", Err: errors.Wrap(err, "failed to read manifest file")} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Failed to retrieve user token data", Err: err} } - _, err = handler.deployKubernetesStack(r, endpoint, string(content), stack.IsComposeFormat, stack.Namespace, k.KubeAppLabels{ - StackID: int(stack.ID), - Name: stack.Name, - Owner: stack.CreatedBy, - Kind: "git", + _, err = handler.deployKubernetesStack(tokenData.ID, endpoint, stack, k.KubeAppLabels{ + StackID: int(stack.ID), + StackName: stack.Name, + Owner: tokenData.Username, + Kind: "git", }) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to redeploy Kubernetes stack", Err: errors.WithMessage(err, "failed to deploy kube application")} diff --git a/api/http/handler/stacks/update_kubernetes_stack.go b/api/http/handler/stacks/update_kubernetes_stack.go index 114552b68..e5d659a52 100644 --- a/api/http/handler/stacks/update_kubernetes_stack.go +++ b/api/http/handler/stacks/update_kubernetes_stack.go @@ -2,7 +2,10 @@ package stacks import ( "fmt" + "io/ioutil" "net/http" + "os" + "path" "strconv" "github.com/asaskevich/govalidator" @@ -10,7 +13,9 @@ import ( httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/filesystem" gittypes "github.com/portainer/portainer/api/git/types" + "github.com/portainer/portainer/api/http/security" k "github.com/portainer/portainer/api/kubernetes" ) @@ -23,6 +28,7 @@ type kubernetesGitStackUpdatePayload struct { RepositoryAuthentication bool RepositoryUsername string RepositoryPassword string + AutoUpdate *portainer.StackAutoUpdate } func (payload *kubernetesFileStackUpdatePayload) Validate(r *http.Request) error { @@ -36,12 +42,20 @@ func (payload *kubernetesGitStackUpdatePayload) Validate(r *http.Request) error if govalidator.IsNull(payload.RepositoryReferenceName) { payload.RepositoryReferenceName = defaultGitReferenceName } + if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil { + return err + } return nil } func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer.Stack, endpoint *portainer.Endpoint) *httperror.HandlerError { if stack.GitConfig != nil { + //stop the autoupdate job if there is any + if stack.AutoUpdate != nil { + stopAutoupdate(stack.ID, stack.AutoUpdate.JobID, *handler.Scheduler) + } + var payload kubernetesGitStackUpdatePayload if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil { @@ -49,6 +63,8 @@ func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer. } stack.GitConfig.ReferenceName = payload.RepositoryReferenceName + stack.AutoUpdate = payload.AutoUpdate + if payload.RepositoryAuthentication { password := payload.RepositoryPassword if password == "" && stack.GitConfig != nil && stack.GitConfig.Authentication != nil { @@ -61,6 +77,15 @@ func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer. } else { stack.GitConfig.Authentication = nil } + + if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" { + jobID, e := startAutoupdate(stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService) + if e != nil { + return e + } + stack.AutoUpdate.JobID = jobID + } + return nil } @@ -71,11 +96,27 @@ func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer. return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err} } - _, err = handler.deployKubernetesStack(r, endpoint, payload.StackFileContent, stack.IsComposeFormat, stack.Namespace, k.KubeAppLabels{ - StackID: int(stack.ID), - Name: stack.Name, - Owner: stack.CreatedBy, - Kind: "content", + tokenData, err := security.RetrieveTokenData(r) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Failed to retrieve user token data", Err: err} + } + + tempFileDir, _ := ioutil.TempDir("", "kub_file_content") + defer os.RemoveAll(tempFileDir) + + if err := filesystem.WriteToFile(path.Join(tempFileDir, stack.EntryPoint), []byte(payload.StackFileContent)); err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Failed to persist deployment file in a temp directory", Err: err} + } + + //use temp dir as the stack project path for deployment + //so if the deployment failed, the original file won't be over-written + stack.ProjectPath = tempFileDir + + _, err = handler.deployKubernetesStack(tokenData.ID, endpoint, stack, k.KubeAppLabels{ + StackID: int(stack.ID), + StackName: stack.Name, + Owner: stack.CreatedBy, + Kind: "content", }) if err != nil { @@ -83,7 +124,7 @@ func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer. } stackFolder := strconv.Itoa(int(stack.ID)) - _, err = handler.FileService.StoreStackFileFromBytes(stackFolder, stack.EntryPoint, []byte(payload.StackFileContent)) + projectPath, err := handler.FileService.StoreStackFileFromBytes(stackFolder, stack.EntryPoint, []byte(payload.StackFileContent)) if err != nil { fileType := "Manifest" if stack.IsComposeFormat { @@ -92,6 +133,7 @@ func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer. errMsg := fmt.Sprintf("Unable to persist Kubernetes %s file on disk", fileType) return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: errMsg, Err: err} } + stack.ProjectPath = projectPath return nil } diff --git a/api/http/handler/stacks/webhook_invoke.go b/api/http/handler/stacks/webhook_invoke.go index 01c9d701b..5868dc79e 100644 --- a/api/http/handler/stacks/webhook_invoke.go +++ b/api/http/handler/stacks/webhook_invoke.go @@ -1,10 +1,10 @@ package stacks import ( - "log" "net/http" "github.com/gofrs/uuid" + "github.com/sirupsen/logrus" "github.com/portainer/libhttp/response" @@ -31,7 +31,10 @@ func (handler *Handler) webhookInvoke(w http.ResponseWriter, r *http.Request) *h } if err = stacks.RedeployWhenChanged(stack.ID, handler.StackDeployer, handler.DataStore, handler.GitService); err != nil { - log.Printf("[ERROR] %s\n", err) + if _, ok := err.(*stacks.StackAuthorMissingErr); ok { + return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: "Autoupdate for the stack isn't available", Err: err} + } + logrus.WithError(err).Error("failed to update the stack") return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Failed to update the stack", Err: err} } diff --git a/api/http/proxy/factory/agent.go b/api/http/proxy/factory/agent.go index 159934421..74b01d084 100644 --- a/api/http/proxy/factory/agent.go +++ b/api/http/proxy/factory/agent.go @@ -24,6 +24,7 @@ type ProxyServer struct { // NewAgentProxy creates a new instance of ProxyServer that wrap http requests with agent headers func (factory *ProxyFactory) NewAgentProxy(endpoint *portainer.Endpoint) (*ProxyServer, error) { urlString := endpoint.URL + if endpointutils.IsEdgeEndpoint((endpoint)) { tunnel, err := factory.reverseTunnelService.GetActiveTunnel(endpoint) if err != nil { diff --git a/api/internal/endpointutils/endpointutils.go b/api/internal/endpointutils/endpointutils.go index 67cde11ed..4c54c4196 100644 --- a/api/internal/endpointutils/endpointutils.go +++ b/api/internal/endpointutils/endpointutils.go @@ -11,7 +11,7 @@ func IsLocalEndpoint(endpoint *portainer.Endpoint) bool { return strings.HasPrefix(endpoint.URL, "unix://") || strings.HasPrefix(endpoint.URL, "npipe://") || endpoint.Type == 5 } -// IsKubernetesEndpoint returns true if this is a kubernetes environment(endpoint) +// IsKubernetesEndpoint returns true if this is a kubernetes endpoint func IsKubernetesEndpoint(endpoint *portainer.Endpoint) bool { return endpoint.Type == portainer.KubernetesLocalEnvironment || endpoint.Type == portainer.AgentOnKubernetesEnvironment || @@ -25,6 +25,7 @@ func IsDockerEndpoint(endpoint *portainer.Endpoint) bool { endpoint.Type == portainer.EdgeAgentOnDockerEnvironment } +// IsEdgeEndpoint returns true if this is an Edge endpoint func IsEdgeEndpoint(endpoint *portainer.Endpoint) bool { return endpoint.Type == portainer.EdgeAgentOnDockerEnvironment || endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment } diff --git a/api/internal/stackutils/stackutils.go b/api/internal/stackutils/stackutils.go index 7e94bff17..4945eeaa1 100644 --- a/api/internal/stackutils/stackutils.go +++ b/api/internal/stackutils/stackutils.go @@ -2,9 +2,13 @@ package stackutils import ( "fmt" + "io/ioutil" "path" + "github.com/pkg/errors" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/filesystem" + k "github.com/portainer/portainer/api/kubernetes" ) // ResourceControlID returns the stack resource control id @@ -20,3 +24,39 @@ func GetStackFilePaths(stack *portainer.Stack) []string { } return filePaths } + +// CreateTempK8SDeploymentFiles reads manifest files from original stack project path +// then add app labels into the file contents and create temp files for deployment +// return temp file paths and temp dir +func CreateTempK8SDeploymentFiles(stack *portainer.Stack, kubeDeployer portainer.KubernetesDeployer, appLabels k.KubeAppLabels) ([]string, string, error) { + fileNames := append([]string{stack.EntryPoint}, stack.AdditionalFiles...) + var manifestFilePaths []string + tmpDir, err := ioutil.TempDir("", "kub_deployment") + if err != nil { + return nil, "", errors.Wrap(err, "failed to create temp kub deployment directory") + } + + for _, fileName := range fileNames { + manifestFilePath := path.Join(tmpDir, fileName) + manifestContent, err := ioutil.ReadFile(path.Join(stack.ProjectPath, fileName)) + if err != nil { + return nil, "", errors.Wrap(err, "failed to read manifest file") + } + if stack.IsComposeFormat { + manifestContent, err = kubeDeployer.ConvertCompose(manifestContent) + if err != nil { + return nil, "", errors.Wrap(err, "failed to convert docker compose file to a kube manifest") + } + } + manifestContent, err = k.AddAppLabels(manifestContent, appLabels.ToMap()) + if err != nil { + return nil, "", errors.Wrap(err, "failed to add application labels") + } + err = filesystem.WriteToFile(manifestFilePath, []byte(manifestContent)) + if err != nil { + return nil, "", errors.Wrap(err, "failed to create temp manifest file") + } + manifestFilePaths = append(manifestFilePaths, manifestFilePath) + } + return manifestFilePaths, tmpDir, nil +} diff --git a/api/kubernetes/yaml.go b/api/kubernetes/yaml.go index b90020641..89dbc8009 100644 --- a/api/kubernetes/yaml.go +++ b/api/kubernetes/yaml.go @@ -12,6 +12,7 @@ import ( ) const ( + labelPortainerAppStack = "io.portainer.kubernetes.application.stack" labelPortainerAppStackID = "io.portainer.kubernetes.application.stackid" labelPortainerAppName = "io.portainer.kubernetes.application.name" labelPortainerAppOwner = "io.portainer.kubernetes.application.owner" @@ -20,17 +21,18 @@ const ( // KubeAppLabels are labels applied to all resources deployed in a kubernetes stack type KubeAppLabels struct { - StackID int - Name string - Owner string - Kind string + StackID int + StackName string + Owner string + Kind string } // ToMap converts KubeAppLabels to a map[string]string func (kal *KubeAppLabels) ToMap() map[string]string { return map[string]string{ labelPortainerAppStackID: strconv.Itoa(kal.StackID), - labelPortainerAppName: kal.Name, + labelPortainerAppStack: kal.StackName, + labelPortainerAppName: kal.StackName, labelPortainerAppOwner: kal.Owner, labelPortainerAppKind: kal.Kind, } @@ -167,13 +169,6 @@ func addLabels(obj map[string]interface{}, appLabels map[string]string) { labels[k] = v } - // fallback to metadata name if name label not explicitly provided - if name, ok := labels[labelPortainerAppName]; !ok || name == "" { - if n, ok := metadata["name"]; ok { - labels[labelPortainerAppName] = n.(string) - } - } - metadata["labels"] = labels obj["metadata"] = metadata } diff --git a/api/kubernetes/yaml_test.go b/api/kubernetes/yaml_test.go index 167b97038..48ca95373 100644 --- a/api/kubernetes/yaml_test.go +++ b/api/kubernetes/yaml_test.go @@ -39,6 +39,7 @@ metadata: io.portainer.kubernetes.application.kind: git io.portainer.kubernetes.application.name: best-name io.portainer.kubernetes.application.owner: best-owner + io.portainer.kubernetes.application.stack: best-name io.portainer.kubernetes.application.stackid: "123" name: busybox spec: @@ -86,6 +87,7 @@ metadata: io.portainer.kubernetes.application.kind: git io.portainer.kubernetes.application.name: best-name io.portainer.kubernetes.application.owner: best-owner + io.portainer.kubernetes.application.stack: best-name io.portainer.kubernetes.application.stackid: "123" name: busybox spec: @@ -174,6 +176,7 @@ items: io.portainer.kubernetes.application.kind: git io.portainer.kubernetes.application.name: best-name io.portainer.kubernetes.application.owner: best-owner + io.portainer.kubernetes.application.stack: best-name io.portainer.kubernetes.application.stackid: "123" name: web spec: @@ -194,6 +197,7 @@ items: io.portainer.kubernetes.application.kind: git io.portainer.kubernetes.application.name: best-name io.portainer.kubernetes.application.owner: best-owner + io.portainer.kubernetes.application.stack: best-name io.portainer.kubernetes.application.stackid: "123" name: redis spec: @@ -216,6 +220,7 @@ items: io.portainer.kubernetes.application.kind: git io.portainer.kubernetes.application.name: best-name io.portainer.kubernetes.application.owner: best-owner + io.portainer.kubernetes.application.stack: best-name io.portainer.kubernetes.application.stackid: "123" name: web spec: @@ -297,6 +302,7 @@ metadata: io.portainer.kubernetes.application.kind: git io.portainer.kubernetes.application.name: best-name io.portainer.kubernetes.application.owner: best-owner + io.portainer.kubernetes.application.stack: best-name io.portainer.kubernetes.application.stackid: "123" name: busybox spec: @@ -322,6 +328,7 @@ metadata: io.portainer.kubernetes.application.kind: git io.portainer.kubernetes.application.name: best-name io.portainer.kubernetes.application.owner: best-owner + io.portainer.kubernetes.application.stack: best-name io.portainer.kubernetes.application.stackid: "123" name: web spec: @@ -340,6 +347,7 @@ metadata: io.portainer.kubernetes.application.kind: git io.portainer.kubernetes.application.name: best-name io.portainer.kubernetes.application.owner: best-owner + io.portainer.kubernetes.application.stack: best-name io.portainer.kubernetes.application.stackid: "123" name: busybox spec: @@ -388,6 +396,7 @@ metadata: io.portainer.kubernetes.application.kind: git io.portainer.kubernetes.application.name: best-name io.portainer.kubernetes.application.owner: best-owner + io.portainer.kubernetes.application.stack: best-name io.portainer.kubernetes.application.stackid: "123" name: web spec: @@ -402,10 +411,10 @@ spec: } labels := KubeAppLabels{ - StackID: 123, - Name: "best-name", - Owner: "best-owner", - Kind: "git", + StackID: 123, + StackName: "best-name", + Owner: "best-owner", + Kind: "git", } for _, tt := range tests { @@ -417,81 +426,6 @@ spec: } } -func Test_AddAppLabels_PickingName_WhenLabelNameIsEmpty(t *testing.T) { - labels := KubeAppLabels{ - StackID: 123, - Owner: "best-owner", - Kind: "git", - } - - input := `apiVersion: v1 -kind: Service -metadata: - name: web -spec: - ports: - - name: "5000" - port: 5000 - targetPort: 5000 -` - - expected := `apiVersion: v1 -kind: Service -metadata: - labels: - io.portainer.kubernetes.application.kind: git - io.portainer.kubernetes.application.name: web - io.portainer.kubernetes.application.owner: best-owner - io.portainer.kubernetes.application.stackid: "123" - name: web -spec: - ports: - - name: "5000" - port: 5000 - targetPort: 5000 -` - - result, err := AddAppLabels([]byte(input), labels.ToMap()) - assert.NoError(t, err) - assert.Equal(t, expected, string(result)) -} - -func Test_AddAppLabels_PickingName_WhenLabelAndMetadataNameAreEmpty(t *testing.T) { - labels := KubeAppLabels{ - StackID: 123, - Owner: "best-owner", - Kind: "git", - } - - input := `apiVersion: v1 -kind: Service -spec: - ports: - - name: "5000" - port: 5000 - targetPort: 5000 -` - - expected := `apiVersion: v1 -kind: Service -metadata: - labels: - io.portainer.kubernetes.application.kind: git - io.portainer.kubernetes.application.name: "" - io.portainer.kubernetes.application.owner: best-owner - io.portainer.kubernetes.application.stackid: "123" -spec: - ports: - - name: "5000" - port: 5000 - targetPort: 5000 -` - - result, err := AddAppLabels([]byte(input), labels.ToMap()) - assert.NoError(t, err) - assert.Equal(t, expected, string(result)) -} - func Test_AddAppLabels_HelmApp(t *testing.T) { labels := GetHelmAppLabels("best-name", "best-owner") @@ -658,10 +592,10 @@ spec: func Test_DocumentSeperator(t *testing.T) { labels := KubeAppLabels{ - StackID: 123, - Name: "best-name", - Owner: "best-owner", - Kind: "git", + StackID: 123, + StackName: "best-name", + Owner: "best-owner", + Kind: "git", } input := `apiVersion: v1 @@ -684,6 +618,7 @@ metadata: io.portainer.kubernetes.application.kind: git io.portainer.kubernetes.application.name: best-name io.portainer.kubernetes.application.owner: best-owner + io.portainer.kubernetes.application.stack: best-name io.portainer.kubernetes.application.stackid: "123" --- apiVersion: v1 @@ -694,6 +629,7 @@ metadata: io.portainer.kubernetes.application.kind: git io.portainer.kubernetes.application.name: best-name io.portainer.kubernetes.application.owner: best-owner + io.portainer.kubernetes.application.stack: best-name io.portainer.kubernetes.application.stackid: "123" ` result, err := AddAppLabels([]byte(input), labels.ToMap()) diff --git a/api/portainer.go b/api/portainer.go index b91413c30..d9642c5e9 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -3,7 +3,6 @@ package portainer import ( "context" "io" - "net/http" "time" gittypes "github.com/portainer/portainer/api/git/types" @@ -1281,7 +1280,8 @@ type ( // KubernetesDeployer represents a service to deploy a manifest inside a Kubernetes environment(endpoint) KubernetesDeployer interface { - Deploy(request *http.Request, endpoint *Endpoint, data string, namespace string) (string, error) + Deploy(userID UserID, endpoint *Endpoint, manifestFiles []string, namespace string) (string, error) + Remove(userID UserID, endpoint *Endpoint, manifestFiles []string, namespace string) (string, error) ConvertCompose(data []byte) ([]byte, error) } diff --git a/api/scheduler/scheduler.go b/api/scheduler/scheduler.go index 6568f753f..529574f0c 100644 --- a/api/scheduler/scheduler.go +++ b/api/scheduler/scheduler.go @@ -8,11 +8,12 @@ import ( "github.com/pkg/errors" "github.com/robfig/cron/v3" + "github.com/sirupsen/logrus" ) type Scheduler struct { - crontab *cron.Cron - shutdownCtx context.Context + crontab *cron.Cron + activeJobs map[cron.EntryID]context.CancelFunc } func NewScheduler(ctx context.Context) *Scheduler { @@ -20,7 +21,8 @@ func NewScheduler(ctx context.Context) *Scheduler { crontab.Start() s := &Scheduler{ - crontab: crontab, + crontab: crontab, + activeJobs: make(map[cron.EntryID]context.CancelFunc), } if ctx != nil { @@ -43,8 +45,10 @@ func (s *Scheduler) Shutdown() error { ctx := s.crontab.Stop() <-ctx.Done() - for _, j := range s.crontab.Entries() { - s.crontab.Remove(j.ID) + for _, job := range s.crontab.Entries() { + if cancel, ok := s.activeJobs[job.ID]; ok { + cancel() + } } err := ctx.Err() @@ -60,14 +64,36 @@ func (s *Scheduler) StopJob(jobID string) error { if err != nil { return errors.Wrapf(err, "failed convert jobID %q to int", jobID) } - s.crontab.Remove(cron.EntryID(id)) + entryID := cron.EntryID(id) + if cancel, ok := s.activeJobs[entryID]; ok { + cancel() + } return nil } // StartJobEvery schedules a new periodic job with a given duration. -// Returns job id that could be used to stop the given job -func (s *Scheduler) StartJobEvery(duration time.Duration, job func()) string { - entryId := s.crontab.Schedule(cron.Every(duration), cron.FuncJob(job)) - return strconv.Itoa(int(entryId)) +// Returns job id that could be used to stop the given job. +// When job run returns an error, that job won't be run again. +func (s *Scheduler) StartJobEvery(duration time.Duration, job func() error) string { + ctx, cancel := context.WithCancel(context.Background()) + + j := cron.FuncJob(func() { + if err := job(); err != nil { + logrus.Debug("job returned an error") + cancel() + } + }) + + entryID := s.crontab.Schedule(cron.Every(duration), j) + + s.activeJobs[entryID] = cancel + + go func(entryID cron.EntryID) { + <-ctx.Done() + logrus.Debug("job cancelled, stopping") + s.crontab.Remove(entryID) + }(entryID) + + return strconv.Itoa(int(entryID)) } diff --git a/api/scheduler/scheduler_test.go b/api/scheduler/scheduler_test.go index 6d21e49ec..2d66f2e6b 100644 --- a/api/scheduler/scheduler_test.go +++ b/api/scheduler/scheduler_test.go @@ -9,49 +9,92 @@ import ( "github.com/stretchr/testify/assert" ) -func Test_CanStartAndTerminate(t *testing.T) { - s := NewScheduler(context.Background()) - s.StartJobEvery(1*time.Minute, func() { fmt.Println("boop") }) +var jobInterval = time.Second - err := s.Shutdown() - assert.NoError(t, err, "Shutdown should return no errors") - assert.Empty(t, s.crontab.Entries(), "all jobs should have been removed") -} - -func Test_CanTerminateByCancellingContext(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - s := NewScheduler(ctx) - s.StartJobEvery(1*time.Minute, func() { fmt.Println("boop") }) - - cancel() - - for i := 0; i < 100; i++ { - if len(s.crontab.Entries()) == 0 { - return - } - time.Sleep(10 * time.Millisecond) - } - t.Fatal("all jobs are expected to be cleaned by now; it might be a timing issue, otherwise implementation defect") -} - -func Test_StartAndStopJob(t *testing.T) { +func Test_ScheduledJobRuns(t *testing.T) { s := NewScheduler(context.Background()) defer s.Shutdown() - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 2*jobInterval) - var jobOne string var workDone bool - jobOne = s.StartJobEvery(time.Second, func() { - assert.Equal(t, 1, len(s.crontab.Entries()), "scheduler should have one active job") + s.StartJobEvery(jobInterval, func() error { workDone = true - s.StopJob(jobOne) cancel() + return nil }) <-ctx.Done() assert.True(t, workDone, "value should been set in the job") - assert.Equal(t, 0, len(s.crontab.Entries()), "scheduler should have no active jobs") - +} + +func Test_JobCanBeStopped(t *testing.T) { + s := NewScheduler(context.Background()) + defer s.Shutdown() + + ctx, cancel := context.WithTimeout(context.Background(), 2*jobInterval) + + var workDone bool + jobID := s.StartJobEvery(jobInterval, func() error { + workDone = true + + cancel() + return nil + }) + s.StopJob(jobID) + + <-ctx.Done() + assert.False(t, workDone, "job shouldn't had a chance to run") +} + +func Test_JobShouldStop_UponError(t *testing.T) { + s := NewScheduler(context.Background()) + defer s.Shutdown() + + var acc int + s.StartJobEvery(jobInterval, func() error { + acc++ + return fmt.Errorf("failed") + }) + + <-time.After(3 * jobInterval) + assert.Equal(t, 1, acc, "job stop after the first run because it returns an error") +} + +func Test_CanTerminateAllJobs_ByShuttingDownScheduler(t *testing.T) { + s := NewScheduler(context.Background()) + + ctx, cancel := context.WithTimeout(context.Background(), 2*jobInterval) + + var workDone bool + s.StartJobEvery(jobInterval, func() error { + workDone = true + + cancel() + return nil + }) + + s.Shutdown() + + <-ctx.Done() + assert.False(t, workDone, "job shouldn't had a chance to run") +} + +func Test_CanTerminateAllJobs_ByCancellingParentContext(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 2*jobInterval) + s := NewScheduler(ctx) + + var workDone bool + s.StartJobEvery(jobInterval, func() error { + workDone = true + + cancel() + return nil + }) + + cancel() + + <-ctx.Done() + assert.False(t, workDone, "job shouldn't had a chance to run") } diff --git a/api/stacks/deploy.go b/api/stacks/deploy.go index 797a415e3..2aa48868c 100644 --- a/api/stacks/deploy.go +++ b/api/stacks/deploy.go @@ -1,15 +1,29 @@ package stacks import ( + "fmt" "strings" "time" "github.com/pkg/errors" portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/http/security" + log "github.com/sirupsen/logrus" ) +type StackAuthorMissingErr struct { + stackID int + authorName string +} + +func (e *StackAuthorMissingErr) Error() string { + return fmt.Sprintf("stack's %v author %s is missing", e.stackID, e.authorName) +} + func RedeployWhenChanged(stackID portainer.StackID, deployer StackDeployer, datastore portainer.DataStore, gitService portainer.GitService) error { + logger := log.WithFields(log.Fields{"stackID": stackID}) + logger.Debug("redeploying stack") + stack, err := datastore.Stack().Stack(stackID) if err != nil { return errors.WithMessagef(err, "failed to get the stack %v", stackID) @@ -19,6 +33,17 @@ func RedeployWhenChanged(stackID portainer.StackID, deployer StackDeployer, data return nil // do nothing if it isn't a git-based stack } + author := stack.UpdatedBy + if author == "" { + author = stack.CreatedBy + } + + user, err := datastore.User().UserByUsername(author) + if err != nil { + logger.WithFields(log.Fields{"author": author, "stack": stack.Name, "endpointID": stack.EndpointID}).Warn("cannot autoupdate a stack, stack author user is missing") + return &StackAuthorMissingErr{int(stack.ID), author} + } + username, password := "", "" if stack.GitConfig.Authentication != nil { username, password = stack.GitConfig.Authentication.Username, stack.GitConfig.Authentication.Password @@ -54,12 +79,7 @@ func RedeployWhenChanged(stackID portainer.StackID, deployer StackDeployer, data return errors.WithMessagef(err, "failed to find the environment %v associated to the stack %v", stack.EndpointID, stack.ID) } - author := stack.UpdatedBy - if author == "" { - author = stack.CreatedBy - } - - registries, err := getUserRegistries(datastore, author, endpoint.ID) + registries, err := getUserRegistries(datastore, user, endpoint.ID) if err != nil { return err } @@ -75,6 +95,12 @@ func RedeployWhenChanged(stackID portainer.StackID, deployer StackDeployer, data if err != nil { return errors.WithMessagef(err, "failed to deploy a docker compose stack %v", stackID) } + case portainer.KubernetesStack: + logger.Debugf("deploying a kube app") + err := deployer.DeployKubernetesStack(stack, endpoint, user) + if err != nil { + return errors.WithMessagef(err, "failed to deploy a kubternetes app stack %v", stackID) + } default: return errors.Errorf("cannot update stack, type %v is unsupported", stack.Type) } @@ -88,24 +114,19 @@ func RedeployWhenChanged(stackID portainer.StackID, deployer StackDeployer, data return nil } -func getUserRegistries(datastore portainer.DataStore, authorUsername string, endpointID portainer.EndpointID) ([]portainer.Registry, error) { +func getUserRegistries(datastore portainer.DataStore, user *portainer.User, endpointID portainer.EndpointID) ([]portainer.Registry, error) { registries, err := datastore.Registry().Registries() if err != nil { return nil, errors.WithMessage(err, "unable to retrieve registries from the database") } - user, err := datastore.User().UserByUsername(authorUsername) - if err != nil { - return nil, errors.WithMessagef(err, "failed to fetch a stack's author [%s]", authorUsername) - } - if user.Role == portainer.AdministratorRole { return registries, nil } userMemberships, err := datastore.TeamMembership().TeamMembershipsByUserID(user.ID) if err != nil { - return nil, errors.WithMessagef(err, "failed to fetch memberships of the stack author [%s]", authorUsername) + return nil, errors.WithMessagef(err, "failed to fetch memberships of the stack author [%s]", user.Username) } filteredRegistries := make([]portainer.Registry, 0, len(registries)) diff --git a/api/stacks/deploy_test.go b/api/stacks/deploy_test.go index dd0b3ff69..9067f9232 100644 --- a/api/stacks/deploy_test.go +++ b/api/stacks/deploy_test.go @@ -35,6 +35,10 @@ func (s *noopDeployer) DeployComposeStack(stack *portainer.Stack, endpoint *port return nil } +func (s *noopDeployer) DeployKubernetesStack(stack *portainer.Stack, endpoint *portainer.Endpoint, user *portainer.User) error { + return nil +} + func Test_redeployWhenChanged_FailsWhenCannotFindStack(t *testing.T) { store, teardown := bolt.MustNewTestStore(true) defer teardown() @@ -48,7 +52,11 @@ func Test_redeployWhenChanged_DoesNothingWhenNotAGitBasedStack(t *testing.T) { store, teardown := bolt.MustNewTestStore(true) defer teardown() - err := store.Stack().CreateStack(&portainer.Stack{ID: 1}) + admin := &portainer.User{ID: 1, Username: "admin"} + err := store.User().CreateUser(admin) + assert.NoError(t, err, "error creating an admin") + + err = store.Stack().CreateStack(&portainer.Stack{ID: 1, CreatedBy: "admin"}) assert.NoError(t, err, "failed to create a test stack") err = RedeployWhenChanged(1, nil, store, &gitService{nil, ""}) @@ -61,8 +69,13 @@ func Test_redeployWhenChanged_DoesNothingWhenNoGitChanges(t *testing.T) { tmpDir, _ := ioutil.TempDir("", "stack") - err := store.Stack().CreateStack(&portainer.Stack{ + admin := &portainer.User{ID: 1, Username: "admin"} + err := store.User().CreateUser(admin) + assert.NoError(t, err, "error creating an admin") + + err = store.Stack().CreateStack(&portainer.Stack{ ID: 1, + CreatedBy: "admin", ProjectPath: tmpDir, GitConfig: &gittypes.RepoConfig{ URL: "url", @@ -80,8 +93,13 @@ func Test_redeployWhenChanged_FailsWhenCannotClone(t *testing.T) { store, teardown := bolt.MustNewTestStore(true) defer teardown() - err := store.Stack().CreateStack(&portainer.Stack{ - ID: 1, + admin := &portainer.User{ID: 1, Username: "admin"} + err := store.User().CreateUser(admin) + assert.NoError(t, err, "error creating an admin") + + err = store.Stack().CreateStack(&portainer.Stack{ + ID: 1, + CreatedBy: "admin", GitConfig: &gittypes.RepoConfig{ URL: "url", ReferenceName: "ref", @@ -136,12 +154,12 @@ func Test_redeployWhenChanged(t *testing.T) { assert.NoError(t, err) }) - t.Run("can NOT deploy kube stack", func(t *testing.T) { + t.Run("can deploy kube app", func(t *testing.T) { stack.Type = portainer.KubernetesStack store.Stack().UpdateStack(stack.ID, &stack) err = RedeployWhenChanged(1, &noopDeployer{}, store, &gitService{nil, "newHash"}) - assert.EqualError(t, err, "cannot update stack, type 3 is unsupported") + assert.NoError(t, err) }) } @@ -151,12 +169,12 @@ func Test_getUserRegistries(t *testing.T) { endpointID := 123 - admin := portainer.User{ID: 1, Username: "admin", Role: portainer.AdministratorRole} - err := store.User().CreateUser(&admin) + admin := &portainer.User{ID: 1, Username: "admin", Role: portainer.AdministratorRole} + err := store.User().CreateUser(admin) assert.NoError(t, err, "error creating an admin") - user := portainer.User{ID: 2, Username: "user", Role: portainer.StandardUserRole} - err = store.User().CreateUser(&user) + user := &portainer.User{ID: 2, Username: "user", Role: portainer.StandardUserRole} + err = store.User().CreateUser(user) assert.NoError(t, err, "error creating a user") team := portainer.Team{ID: 1, Name: "team"} @@ -208,13 +226,13 @@ func Test_getUserRegistries(t *testing.T) { assert.NoError(t, err, "couldn't create a registry") t.Run("admin should has access to all registries", func(t *testing.T) { - registries, err := getUserRegistries(store, admin.Username, portainer.EndpointID(endpointID)) + registries, err := getUserRegistries(store, admin, portainer.EndpointID(endpointID)) assert.NoError(t, err) assert.ElementsMatch(t, []portainer.Registry{registryReachableByUser, registryReachableByTeam, registryRestricted}, registries) }) t.Run("regular user has access to registries allowed to him and/or his team", func(t *testing.T) { - registries, err := getUserRegistries(store, user.Username, portainer.EndpointID(endpointID)) + registries, err := getUserRegistries(store, user, portainer.EndpointID(endpointID)) assert.NoError(t, err) assert.ElementsMatch(t, []portainer.Registry{registryReachableByUser, registryReachableByTeam}, registries) }) diff --git a/api/stacks/deployer.go b/api/stacks/deployer.go index c594e48ec..714bae066 100644 --- a/api/stacks/deployer.go +++ b/api/stacks/deployer.go @@ -2,27 +2,36 @@ package stacks import ( "context" + "os" "sync" + "github.com/pkg/errors" + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/internal/stackutils" + k "github.com/portainer/portainer/api/kubernetes" ) type StackDeployer interface { DeploySwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, prune bool) error DeployComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry) error + DeployKubernetesStack(stack *portainer.Stack, endpoint *portainer.Endpoint, user *portainer.User) error } type stackDeployer struct { lock *sync.Mutex swarmStackManager portainer.SwarmStackManager composeStackManager portainer.ComposeStackManager + kubernetesDeployer portainer.KubernetesDeployer } -func NewStackDeployer(swarmStackManager portainer.SwarmStackManager, composeStackManager portainer.ComposeStackManager) *stackDeployer { +// NewStackDeployer inits a stackDeployer struct with a SwarmStackManager, a ComposeStackManager and a KubernetesDeployer +func NewStackDeployer(swarmStackManager portainer.SwarmStackManager, composeStackManager portainer.ComposeStackManager, kubernetesDeployer portainer.KubernetesDeployer) *stackDeployer { return &stackDeployer{ lock: &sync.Mutex{}, swarmStackManager: swarmStackManager, composeStackManager: composeStackManager, + kubernetesDeployer: kubernetesDeployer, } } @@ -45,3 +54,33 @@ func (d *stackDeployer) DeployComposeStack(stack *portainer.Stack, endpoint *por return d.composeStackManager.Up(context.TODO(), stack, endpoint) } + +func (d *stackDeployer) DeployKubernetesStack(stack *portainer.Stack, endpoint *portainer.Endpoint, user *portainer.User) error { + d.lock.Lock() + defer d.lock.Unlock() + + appLabels := k.KubeAppLabels{ + StackID: int(stack.ID), + StackName: stack.Name, + Owner: user.Username, + } + + if stack.GitConfig == nil { + appLabels.Kind = "content" + } else { + appLabels.Kind = "git" + } + + manifestFilePaths, tempDir, err := stackutils.CreateTempK8SDeploymentFiles(stack, d.kubernetesDeployer, appLabels) + if err != nil { + return errors.Wrap(err, "failed to create temp kub deployment files") + } + defer os.RemoveAll(tempDir) + + _, err = d.kubernetesDeployer.Deploy(user.ID, endpoint, manifestFilePaths, stack.Namespace) + if err != nil { + return errors.Wrap(err, "failed to deploy kubernetes application") + } + + return nil +} diff --git a/api/stacks/scheduled.go b/api/stacks/scheduled.go index fb90ca22c..238939fee 100644 --- a/api/stacks/scheduled.go +++ b/api/stacks/scheduled.go @@ -1,7 +1,6 @@ package stacks import ( - "log" "time" "github.com/pkg/errors" @@ -19,10 +18,9 @@ func StartStackSchedules(scheduler *scheduler.Scheduler, stackdeployer StackDepl if err != nil { return errors.Wrap(err, "Unable to parse auto update interval") } - jobID := scheduler.StartJobEvery(d, func() { - if err := RedeployWhenChanged(stack.ID, stackdeployer, datastore, gitService); err != nil { - log.Printf("[ERROR] %s\n", err) - } + stackID := stack.ID // to be captured by the scheduled function + jobID := scheduler.StartJobEvery(d, func() error { + return RedeployWhenChanged(stackID, stackdeployer, datastore, gitService) }) stack.AutoUpdate.JobID = jobID diff --git a/app/config.js b/app/config.js index 0ff080aaf..1793e5006 100644 --- a/app/config.js +++ b/app/config.js @@ -56,7 +56,7 @@ angular.module('portainer').config([ closeButton: true, progressBar: true, tapToDismiss: false, - } + }; Terminal.applyAddon(fit); diff --git a/app/edge/components/group-form/groupForm.html b/app/edge/components/group-form/groupForm.html index 2d9c79dc4..0bbcfc1b8 100644 --- a/app/edge/components/group-form/groupForm.html +++ b/app/edge/components/group-form/groupForm.html @@ -65,7 +65,9 @@
-
No Edge environments are available. Head over to the Environments view to add environments.
+
+ No Edge environments are available. Head over to the Environments view to add environments. +
diff --git a/app/kubernetes/components/datatables/application/containers-datatable/containersDatatable.html b/app/kubernetes/components/datatables/application/containers-datatable/containersDatatable.html index 23f932074..1da166857 100644 --- a/app/kubernetes/components/datatables/application/containers-datatable/containersDatatable.html +++ b/app/kubernetes/components/datatables/application/containers-datatable/containersDatatable.html @@ -100,11 +100,11 @@ - - Pod IP - - - + + Pod IP + + + diff --git a/app/kubernetes/components/kube-config-download-button/kube-config-download-button.html b/app/kubernetes/components/kube-config-download-button/kube-config-download-button.html index 343073af9..d2c2c4cf4 100644 --- a/app/kubernetes/components/kube-config-download-button/kube-config-download-button.html +++ b/app/kubernetes/components/kube-config-download-button/kube-config-download-button.html @@ -1,13 +1,19 @@ - - + diff --git a/app/kubernetes/models/deploy.js b/app/kubernetes/models/deploy.js index e3405be63..ab60cd046 100644 --- a/app/kubernetes/models/deploy.js +++ b/app/kubernetes/models/deploy.js @@ -15,3 +15,8 @@ export const KubernetesDeployRequestMethods = Object.freeze({ STRING: 'string', URL: 'url', }); + +export const RepositoryMechanismTypes = Object.freeze({ + WEBHOOK: 'Webhook', + INTERVAL: 'Interval', +}); diff --git a/app/kubernetes/services/serviceService.js b/app/kubernetes/services/serviceService.js index a85292d76..64dcaa3cb 100644 --- a/app/kubernetes/services/serviceService.js +++ b/app/kubernetes/services/serviceService.js @@ -102,7 +102,8 @@ class KubernetesServiceService { const namespace = service.Namespace; await this.KubernetesServices(namespace).delete(params).$promise; } catch (err) { - throw new PortainerError('Unable to remove service', err); + // eslint-disable-next-line no-console + console.error('unable to remove service', err); } } diff --git a/app/kubernetes/views/applications/applicationsController.js b/app/kubernetes/views/applications/applicationsController.js index dd5e089b0..3825fb1ba 100644 --- a/app/kubernetes/views/applications/applicationsController.js +++ b/app/kubernetes/views/applications/applicationsController.js @@ -7,7 +7,7 @@ import { KubernetesApplicationTypes } from 'Kubernetes/models/application/models class KubernetesApplicationsController { /* @ngInject */ - constructor($async, $state, Notifications, KubernetesApplicationService, HelmService, KubernetesConfigurationService, Authentication, ModalService, LocalStorage) { + constructor($async, $state, Notifications, KubernetesApplicationService, HelmService, KubernetesConfigurationService, Authentication, ModalService, LocalStorage, StackService) { this.$async = $async; this.$state = $state; this.Notifications = Notifications; @@ -17,6 +17,7 @@ class KubernetesApplicationsController { this.Authentication = Authentication; this.ModalService = ModalService; this.LocalStorage = LocalStorage; + this.StackService = StackService; this.onInit = this.onInit.bind(this); this.getApplications = this.getApplications.bind(this); @@ -36,8 +37,18 @@ class KubernetesApplicationsController { let actionCount = selectedItems.length; for (const stack of selectedItems) { try { - const promises = _.map(stack.Applications, (app) => this.KubernetesApplicationService.delete(app)); - await Promise.all(promises); + const isAppFormCreated = stack.Applications.some((x) => !x.ApplicationKind); + + if (isAppFormCreated) { + const promises = _.map(stack.Applications, (app) => this.KubernetesApplicationService.delete(app)); + await Promise.all(promises); + } else { + const application = stack.Applications.find((x) => x.StackId !== null); + if (application && application.StackId) { + await this.StackService.remove({ Id: application.StackId }, false, this.endpoint.Id); + } + } + this.Notifications.success('Stack successfully removed', stack.Name); _.remove(this.state.stacks, { Name: stack.Name }); } catch (err) { @@ -70,6 +81,14 @@ class KubernetesApplicationsController { await this.HelmService.uninstall(this.endpoint.Id, application); } else { await this.KubernetesApplicationService.delete(application); + // Update applications in stack + const stack = this.state.stacks.find((x) => x.Name === application.StackName); + const index = stack.Applications.indexOf(application); + stack.Applications.splice(index, 1); + // remove stack if no app left in the stack + if (stack.Applications.length === 0 && application.StackId) { + await this.StackService.remove({ Id: application.StackId }, false, this.endpoint.Id); + } } this.Notifications.success('Application successfully removed', application.Name); const index = this.state.applications.indexOf(application); diff --git a/app/kubernetes/views/applications/create/createApplication.html b/app/kubernetes/views/applications/create/createApplication.html index a776053be..d590c6c38 100644 --- a/app/kubernetes/views/applications/create/createApplication.html +++ b/app/kubernetes/views/applications/create/createApplication.html @@ -21,6 +21,8 @@ class-name="text-muted" url="ctrl.stack.GitConfig.URL" config-file-path="ctrl.stack.GitConfig.ConfigFilePath" + additional-files="ctrl.stack.AdditionalFiles" + type="application" >
Namespace @@ -56,11 +58,11 @@ - + > diff --git a/app/kubernetes/views/deploy/deploy.html b/app/kubernetes/views/deploy/deploy.html index 65b2e9740..26bd22e5e 100644 --- a/app/kubernetes/views/deploy/deploy.html +++ b/app/kubernetes/views/deploy/deploy.html @@ -29,6 +29,13 @@
+
+ +
+ +
+
+
Build method
@@ -42,32 +49,16 @@ -
-
- Git repository -
- - -
- - Indicate the path to the yaml file from the root of your repository. - -
-
- -
- -
-
- -
+ { if (this.state.templateId === templateId) { @@ -184,6 +210,7 @@ class KubernetesDeployController { const payload = { ComposeFormat: composeFormat, Namespace: this.formValues.Namespace, + StackName: this.formValues.StackName, }; if (method === KubernetesDeployRequestMethods.REPOSITORY) { @@ -194,7 +221,16 @@ class KubernetesDeployController { payload.RepositoryUsername = this.formValues.RepositoryUsername; payload.RepositoryPassword = this.formValues.RepositoryPassword; } - payload.FilePathInRepository = this.formValues.FilePathInRepository; + payload.ManifestFile = this.formValues.ComposeFilePathInRepository; + payload.AdditionalFiles = this.formValues.AdditionalFiles; + if (this.formValues.RepositoryAutomaticUpdates) { + payload.AutoUpdate = {}; + if (this.formValues.RepositoryMechanism === RepositoryMechanismTypes.INTERVAL) { + payload.AutoUpdate.Interval = this.formValues.RepositoryFetchInterval; + } else if (this.formValues.RepositoryMechanism === RepositoryMechanismTypes.WEBHOOK) { + payload.AutoUpdate.Webhook = this.formValues.RepositoryWebhookURL.split('/').reverse()[0]; + } + } } else if (method === KubernetesDeployRequestMethods.STRING) { payload.StackFileContent = this.formValues.EditorContent; } else { diff --git a/app/portainer/components/associated-endpoints-selector/associatedEndpointsSelector.html b/app/portainer/components/associated-endpoints-selector/associatedEndpointsSelector.html index 5cd20d771..0c66ff2ae 100644 --- a/app/portainer/components/associated-endpoints-selector/associatedEndpointsSelector.html +++ b/app/portainer/components/associated-endpoints-selector/associatedEndpointsSelector.html @@ -1,6 +1,6 @@
- You can select which environment should be part of this group by moving them to the associated environments table. Simply click on any environment entry to move it from one table to the - other. + You can select which environment should be part of this group by moving them to the associated environments table. Simply click on any environment entry to move it from one table + to the other.
diff --git a/app/portainer/components/forms/git-form/git-form-auth-fieldset/git-form-auth-fieldset.css b/app/portainer/components/forms/git-form/git-form-auth-fieldset/git-form-auth-fieldset.css new file mode 100644 index 000000000..dcd209054 --- /dev/null +++ b/app/portainer/components/forms/git-form/git-form-auth-fieldset/git-form-auth-fieldset.css @@ -0,0 +1,11 @@ +.inline-label { + display: inline-block; + padding: 0 15px; + min-width: 200px; +} + +.inline-input { + display: inline-block; + margin-left: 15px; + width: calc(100% - 235px); +} diff --git a/app/portainer/components/forms/git-form/git-form-auth-fieldset/git-form-auth-fieldset.html b/app/portainer/components/forms/git-form/git-form-auth-fieldset/git-form-auth-fieldset.html index a197325dc..da5761f04 100644 --- a/app/portainer/components/forms/git-form/git-form-auth-fieldset/git-form-auth-fieldset.html +++ b/app/portainer/components/forms/git-form/git-form-auth-fieldset/git-form-auth-fieldset.html @@ -15,8 +15,10 @@
- -
+ +
-
- You can select which environment should be part of this group by moving them to the associated environments table. Simply click on any environment entry to move it from one table to - the other. + You can select which environment should be part of this group by moving them to the associated environments table. Simply click on any environment entry to move it from one + table to the other.
diff --git a/app/portainer/components/forms/kubernetes-app-git-form/kubernetes-app-git-form.controller.js b/app/portainer/components/forms/kubernetes-app-git-form/kubernetes-app-git-form.controller.js index af7a8c691..b31f914bc 100644 --- a/app/portainer/components/forms/kubernetes-app-git-form/kubernetes-app-git-form.controller.js +++ b/app/portainer/components/forms/kubernetes-app-git-form/kubernetes-app-git-form.controller.js @@ -83,7 +83,6 @@ class KubernetesAppGitFormController { } $onInit() { - console.log(this); this.formValues.RefName = this.stack.GitConfig.ReferenceName; if (this.stack.GitConfig && this.stack.GitConfig.Authentication) { this.formValues.RepositoryUsername = this.stack.GitConfig.Authentication.Username; diff --git a/app/portainer/components/forms/kubernetes-redeploy-app-git-form/kubernetes-redeploy-app-git-form.controller.js b/app/portainer/components/forms/kubernetes-redeploy-app-git-form/kubernetes-redeploy-app-git-form.controller.js new file mode 100644 index 000000000..9fd397d72 --- /dev/null +++ b/app/portainer/components/forms/kubernetes-redeploy-app-git-form/kubernetes-redeploy-app-git-form.controller.js @@ -0,0 +1,147 @@ +import uuidv4 from 'uuid/v4'; +import { RepositoryMechanismTypes } from 'Kubernetes/models/deploy'; +class KubernetesRedeployAppGitFormController { + /* @ngInject */ + constructor($async, $state, StackService, ModalService, Notifications, WebhookHelper) { + this.$async = $async; + this.$state = $state; + this.StackService = StackService; + this.ModalService = ModalService; + this.Notifications = Notifications; + this.WebhookHelper = WebhookHelper; + + this.state = { + saveGitSettingsInProgress: false, + redeployInProgress: false, + showConfig: false, + isEdit: false, + hasUnsavedChanges: false, + }; + + this.formValues = { + RefName: '', + RepositoryAuthentication: false, + RepositoryUsername: '', + RepositoryPassword: '', + // auto update + AutoUpdate: { + RepositoryAutomaticUpdates: false, + RepositoryMechanism: RepositoryMechanismTypes.INTERVAL, + RepositoryFetchInterval: '5m', + RepositoryWebhookURL: '', + }, + }; + + this.onChange = this.onChange.bind(this); + this.onChangeRef = this.onChangeRef.bind(this); + } + + onChangeRef(value) { + this.onChange({ RefName: value }); + } + + onChange(values) { + this.formValues = { + ...this.formValues, + ...values, + }; + this.state.hasUnsavedChanges = angular.toJson(this.savedFormValues) !== angular.toJson(this.formValues); + } + + buildAnalyticsProperties() { + const metadata = { + 'automatic-updates': automaticUpdatesLabel(this.formValues.AutoUpdate.RepositoryAutomaticUpdates, this.formValues.AutoUpdate.RepositoryMechanism), + }; + + return { metadata }; + + function automaticUpdatesLabel(repositoryAutomaticUpdates, repositoryMechanism) { + switch (repositoryAutomaticUpdates && repositoryMechanism) { + case RepositoryMechanismTypes.INTERVAL: + return 'polling'; + case RepositoryMechanismTypes.WEBHOOK: + return 'webhook'; + default: + return 'off'; + } + } + } + + async pullAndRedeployApplication() { + return this.$async(async () => { + try { + const confirmed = await this.ModalService.confirmAsync({ + title: 'Are you sure?', + message: 'Any changes to this application will be overriden by the definition in git and may cause a service interruption. Do you wish to continue?', + buttons: { + confirm: { + label: 'Update', + className: 'btn-warning', + }, + }, + }); + if (!confirmed) { + return; + } + this.state.redeployInProgress = true; + await this.StackService.updateKubeGit(this.stack.Id, this.stack.EndpointId, this.namespace, this.formValues); + this.Notifications.success('Pulled and redeployed stack successfully'); + await this.$state.reload(); + } catch (err) { + this.Notifications.error('Failure', err, 'Failed redeploying application'); + } finally { + this.state.redeployInProgress = false; + } + }); + } + + async saveGitSettings() { + return this.$async(async () => { + try { + this.state.saveGitSettingsInProgress = true; + await this.StackService.updateKubeStack({ EndpointId: this.stack.EndpointId, Id: this.stack.Id }, null, this.formValues); + this.savedFormValues = angular.copy(this.formValues); + this.state.hasUnsavedChanges = false; + this.Notifications.success('Save stack settings successfully'); + } catch (err) { + this.Notifications.error('Failure', err, 'Unable to save application settings'); + } finally { + this.state.saveGitSettingsInProgress = false; + } + }); + } + + isSubmitButtonDisabled() { + return this.state.saveGitSettingsInProgress || this.state.redeployInProgress; + } + + $onInit() { + this.formValues.RefName = this.stack.GitConfig.ReferenceName; + // Init auto update + if (this.stack.AutoUpdate && (this.stack.AutoUpdate.Interval || this.stack.AutoUpdate.Webhook)) { + this.formValues.AutoUpdate.RepositoryAutomaticUpdates = true; + + if (this.stack.AutoUpdate.Interval) { + this.formValues.AutoUpdate.RepositoryMechanism = RepositoryMechanismTypes.INTERVAL; + this.formValues.AutoUpdate.RepositoryFetchInterval = this.stack.AutoUpdate.Interval; + } else if (this.stack.AutoUpdate.Webhook) { + this.formValues.AutoUpdate.RepositoryMechanism = RepositoryMechanismTypes.WEBHOOK; + this.formValues.AutoUpdate.RepositoryWebhookURL = this.WebhookHelper.returnStackWebhookUrl(this.stack.AutoUpdate.Webhook); + } + } + + if (!this.formValues.AutoUpdate.RepositoryWebhookURL) { + this.formValues.AutoUpdate.RepositoryWebhookURL = this.WebhookHelper.returnStackWebhookUrl(uuidv4()); + } + + if (this.stack.GitConfig && this.stack.GitConfig.Authentication) { + this.formValues.RepositoryUsername = this.stack.GitConfig.Authentication.Username; + this.formValues.RepositoryAuthentication = true; + this.state.isEdit = true; + } + + this.savedFormValues = angular.copy(this.formValues); + } +} + +export default KubernetesRedeployAppGitFormController; diff --git a/app/portainer/components/forms/kubernetes-redeploy-app-git-form/kubernetes-redeploy-app-git-form.html b/app/portainer/components/forms/kubernetes-redeploy-app-git-form/kubernetes-redeploy-app-git-form.html new file mode 100644 index 000000000..f2fca4db1 --- /dev/null +++ b/app/portainer/components/forms/kubernetes-redeploy-app-git-form/kubernetes-redeploy-app-git-form.html @@ -0,0 +1,64 @@ +
+
+ Redeploy from git repository +
+
+
+

+ Pull the latest manifest from git and redeploy the application. +

+
+
+ +
+
+

+ + + {{ $ctrl.state.showConfig ? 'Hide' : 'Advanced' }} configuration + +

+
+
+ + + +
+ Actions +
+ + + + diff --git a/app/portainer/components/forms/kubernetes-redeploy-app-git-form/kubernetes-redeploy-app-git-form.js b/app/portainer/components/forms/kubernetes-redeploy-app-git-form/kubernetes-redeploy-app-git-form.js new file mode 100644 index 000000000..1590621bb --- /dev/null +++ b/app/portainer/components/forms/kubernetes-redeploy-app-git-form/kubernetes-redeploy-app-git-form.js @@ -0,0 +1,13 @@ +import angular from 'angular'; +import controller from './kubernetes-redeploy-app-git-form.controller'; + +const kubernetesRedeployAppGitForm = { + templateUrl: './kubernetes-redeploy-app-git-form.html', + controller, + bindings: { + stack: '<', + namespace: '<', + }, +}; + +angular.module('portainer.app').component('kubernetesRedeployAppGitForm', kubernetesRedeployAppGitForm); diff --git a/app/portainer/components/forms/stack-redeploy-git-form/stack-redeploy-git-form.controller.js b/app/portainer/components/forms/stack-redeploy-git-form/stack-redeploy-git-form.controller.js index 5012b7241..06e1389f3 100644 --- a/app/portainer/components/forms/stack-redeploy-git-form/stack-redeploy-git-form.controller.js +++ b/app/portainer/components/forms/stack-redeploy-git-form/stack-redeploy-git-form.controller.js @@ -1,5 +1,5 @@ import uuidv4 from 'uuid/v4'; - +import { RepositoryMechanismTypes } from 'Kubernetes/models/deploy'; class StackRedeployGitFormController { /* @ngInject */ constructor($async, $state, StackService, ModalService, Notifications, WebhookHelper, FormHelper) { @@ -28,7 +28,7 @@ class StackRedeployGitFormController { // auto update AutoUpdate: { RepositoryAutomaticUpdates: false, - RepositoryMechanism: 'Interval', + RepositoryMechanism: RepositoryMechanismTypes.INTERVAL, RepositoryFetchInterval: '5m', RepositoryWebhookURL: '', }, @@ -50,9 +50,9 @@ class StackRedeployGitFormController { function autoSyncLabel(type) { switch (type) { - case 'Interval': + case RepositoryMechanismTypes.INTERVAL: return 'polling'; - case 'Webhook': + case RepositoryMechanismTypes.WEBHOOK: return 'webhook'; } return 'off'; @@ -156,10 +156,10 @@ class StackRedeployGitFormController { this.formValues.AutoUpdate.RepositoryAutomaticUpdates = true; if (this.stack.AutoUpdate.Interval) { - this.formValues.AutoUpdate.RepositoryMechanism = `Interval`; + this.formValues.AutoUpdate.RepositoryMechanism = RepositoryMechanismTypes.INTERVAL; this.formValues.AutoUpdate.RepositoryFetchInterval = this.stack.AutoUpdate.Interval; } else if (this.stack.AutoUpdate.Webhook) { - this.formValues.AutoUpdate.RepositoryMechanism = `Webhook`; + this.formValues.AutoUpdate.RepositoryMechanism = RepositoryMechanismTypes.WEBHOOK; this.formValues.AutoUpdate.RepositoryWebhookURL = this.WebhookHelper.returnStackWebhookUrl(this.stack.AutoUpdate.Webhook); } } diff --git a/app/portainer/services/api/stackService.js b/app/portainer/services/api/stackService.js index 33ee75401..08f94065d 100644 --- a/app/portainer/services/api/stackService.js +++ b/app/portainer/services/api/stackService.js @@ -1,4 +1,5 @@ import _ from 'lodash-es'; +import { RepositoryMechanismTypes } from 'Kubernetes/models/deploy'; import { StackViewModel, OrphanedStackViewModel } from '../../models/stack'; angular.module('portainer.app').factory('StackService', [ @@ -277,7 +278,17 @@ angular.module('portainer.app').factory('StackService', [ StackFileContent: stackFile, }; } else { + const autoUpdate = {}; + if (gitConfig.AutoUpdate && gitConfig.AutoUpdate.RepositoryAutomaticUpdates) { + if (gitConfig.AutoUpdate.RepositoryMechanism === RepositoryMechanismTypes.INTERVAL) { + autoUpdate.Interval = gitConfig.AutoUpdate.RepositoryFetchInterval; + } else if (gitConfig.AutoUpdate.RepositoryMechanism === RepositoryMechanismTypes.WEBHOOK) { + autoUpdate.Webhook = gitConfig.AutoUpdate.RepositoryWebhookURL.split('/').reverse()[0]; + } + } + payload = { + AutoUpdate: autoUpdate, RepositoryReferenceName: gitConfig.RefName, RepositoryAuthentication: gitConfig.RepositoryAuthentication, RepositoryUsername: gitConfig.RepositoryUsername, @@ -455,9 +466,9 @@ angular.module('portainer.app').factory('StackService', [ const autoUpdate = {}; if (gitConfig.AutoUpdate.RepositoryAutomaticUpdates) { - if (gitConfig.AutoUpdate.RepositoryMechanism === 'Interval') { + if (gitConfig.AutoUpdate.RepositoryMechanism === RepositoryMechanismTypes.INTERVAL) { autoUpdate.Interval = gitConfig.AutoUpdate.RepositoryFetchInterval; - } else if (gitConfig.AutoUpdate.RepositoryMechanism === 'Webhook') { + } else if (gitConfig.AutoUpdate.RepositoryMechanism === RepositoryMechanismTypes.WEBHOOK) { autoUpdate.Webhook = gitConfig.AutoUpdate.RepositoryWebhookURL.split('/').reverse()[0]; } } diff --git a/app/portainer/views/custom-templates/create-custom-template-view/createCustomTemplateViewController.js b/app/portainer/views/custom-templates/create-custom-template-view/createCustomTemplateViewController.js index 1a0857e1b..7e4153659 100644 --- a/app/portainer/views/custom-templates/create-custom-template-view/createCustomTemplateViewController.js +++ b/app/portainer/views/custom-templates/create-custom-template-view/createCustomTemplateViewController.js @@ -30,7 +30,7 @@ class CreateCustomTemplateViewController { ComposeFilePathInRepository: 'docker-compose.yml', Description: '', Note: '', - Logo:'', + Logo: '', Platform: 1, Type: 1, AccessControlData: new AccessControlFormData(), diff --git a/app/portainer/views/groups/groups.html b/app/portainer/views/groups/groups.html index 714af8c68..d6b7baa29 100644 --- a/app/portainer/views/groups/groups.html +++ b/app/portainer/views/groups/groups.html @@ -9,6 +9,13 @@
- +
diff --git a/app/portainer/views/stacks/create/createStackController.js b/app/portainer/views/stacks/create/createStackController.js index d01646ef8..beccf278d 100644 --- a/app/portainer/views/stacks/create/createStackController.js +++ b/app/portainer/views/stacks/create/createStackController.js @@ -1,6 +1,6 @@ import angular from 'angular'; import uuidv4 from 'uuid/v4'; - +import { RepositoryMechanismTypes } from 'Kubernetes/models/deploy'; import { AccessControlFormData } from '../../../components/accessControlForm/porAccessControlFormModel'; angular @@ -42,7 +42,7 @@ angular ComposeFilePathInRepository: 'docker-compose.yml', AccessControlData: new AccessControlFormData(), RepositoryAutomaticUpdates: true, - RepositoryMechanism: 'Interval', + RepositoryMechanism: RepositoryMechanismTypes.INTERVAL, RepositoryFetchInterval: '5m', RepositoryWebhookURL: WebhookHelper.returnStackWebhookUrl(uuidv4()), }; @@ -111,9 +111,9 @@ angular function autoSyncLabel(type) { switch (type) { - case 'Interval': + case RepositoryMechanismTypes.INTERVAL: return 'polling'; - case 'Webhook': + case RepositoryMechanismTypes.WEBHOOK: return 'webhook'; } return 'off'; @@ -166,9 +166,9 @@ angular function getAutoUpdatesProperty(repositoryOptions) { if ($scope.formValues.RepositoryAutomaticUpdates) { repositoryOptions.AutoUpdate = {}; - if ($scope.formValues.RepositoryMechanism === 'Interval') { + if ($scope.formValues.RepositoryMechanism === RepositoryMechanismTypes.INTERVAL) { repositoryOptions.AutoUpdate.Interval = $scope.formValues.RepositoryFetchInterval; - } else if ($scope.formValues.RepositoryMechanism === 'Webhook') { + } else if ($scope.formValues.RepositoryMechanism === RepositoryMechanismTypes.WEBHOOK) { repositoryOptions.AutoUpdate.Webhook = $scope.formValues.RepositoryWebhookURL.split('/').reverse()[0]; } } diff --git a/app/portainer/views/stacks/create/createstack.html b/app/portainer/views/stacks/create/createstack.html index 9d32ca5df..b6d6d1f21 100644 --- a/app/portainer/views/stacks/create/createstack.html +++ b/app/portainer/views/stacks/create/createstack.html @@ -113,6 +113,8 @@ additional-file="true" auto-update="true" show-auth-explanation="true" + path-text-title="Compose path" + path-placeholder="docker-compose.yml" >