From bcccdfb6691a97f106553a1e80a495bc05ccdfcf Mon Sep 17 00:00:00 2001 From: Dmitry Salakhov Date: Tue, 17 Aug 2021 13:12:07 +1200 Subject: [PATCH] feat(stacks): support automated sync for stacks [EE-248] (#5340) --- api/backup/backup.go | 3 +- api/backup/copy_test.go | 105 --------- api/backup/restore.go | 3 +- api/bolt/migrator/migrate_dbversion33.go | 32 +++ api/bolt/migrator/migrate_dbversion33_test.go | 51 ++++ api/bolt/migrator/migrator.go | 6 + api/bolt/stack/stack.go | 76 ++++++ api/bolt/stack/tests/stack_test.go | 111 +++++++++ api/cmd/portainer/main.go | 17 +- api/exec/compose_stack.go | 31 ++- api/exec/compose_stack_integration_test.go | 2 - api/exec/compose_stack_test.go | 48 +--- api/exec/swarm_stack.go | 21 +- api/exec/swarm_stack_test.go | 15 ++ api/{backup => filesystem}/copy.go | 24 +- api/filesystem/copy_test.go | 92 ++++++++ .../testdata}/copy_test/dir/.dotfile | 0 .../testdata}/copy_test/dir/inner | 0 .../testdata}/copy_test/outer | 0 api/git/azure.go | 80 ++++++- api/git/azure_integration_test.go | 12 + api/git/azure_test.go | 127 +++++++++- api/git/git.go | 73 +++++- api/git/git_integration_test.go | 17 +- api/git/git_test.go | 18 +- api/git/testdata/azure-repo copy.zip | Bin 0 -> 163 bytes api/git/testdata/test-clone-git-repo.tar.gz | Bin 11003 -> 14141 bytes api/git/types/types.go | 12 +- api/go.mod | 3 +- api/go.sum | 6 +- api/http/handler/stacks/autoupdate.go | 38 +++ .../handler/stacks/create_compose_stack.go | 108 ++++++--- .../stacks/create_kubernetes_stack_test.go | 4 + api/http/handler/stacks/create_swarm_stack.go | 102 +++++--- api/http/handler/stacks/handler.go | 53 ++++- api/http/handler/stacks/helper.go | 24 ++ api/http/handler/stacks/helper_test.go | 42 ++++ api/http/handler/stacks/stack_associate.go | 10 +- api/http/handler/stacks/stack_create.go | 33 +-- api/http/handler/stacks/stack_create_test.go | 29 --- api/http/handler/stacks/stack_delete.go | 5 + api/http/handler/stacks/stack_inspect.go | 5 + api/http/handler/stacks/stack_list.go | 10 +- api/http/handler/stacks/stack_migrate.go | 5 + api/http/handler/stacks/stack_start.go | 16 ++ api/http/handler/stacks/stack_stop.go | 11 + api/http/handler/stacks/stack_update.go | 5 + api/http/handler/stacks/stack_update_git.go | 152 +++++------- .../stacks/stack_update_git_redeploy.go | 190 +++++++++++++++ api/http/handler/stacks/webhook_invoke.go | 54 +++++ .../handler/stacks/webhook_invoke_test.go | 59 +++++ api/http/server.go | 10 +- api/internal/stackutils/stackutils.go | 10 + api/internal/stackutils/stackutils_test.go | 26 +++ api/libcompose/compose_stack.go | 20 +- api/portainer.go | 17 ++ api/scheduler/scheduler.go | 73 ++++++ api/scheduler/scheduler_test.go | 57 +++++ api/stacks/deploy.go | 138 +++++++++++ api/stacks/deploy_test.go | 221 ++++++++++++++++++ api/stacks/deployer.go | 46 ++++ api/stacks/scheduled.go | 34 +++ .../containersDatatable.html | 2 +- .../containersDatatable.js | 1 + .../services-datatable/servicesDatatable.html | 2 +- .../services-datatable/servicesDatatable.js | 1 + app/portainer/components/focusIf.js | 22 ++ ...it-form-additional-file-item.controller.js | 22 ++ .../git-form-additional-file-item.html | 20 ++ .../git-form-additional-file-item/index.js | 14 ++ ...-form-additional-files-panel.controller.js | 26 +++ .../git-form-additional-files-panel.html | 14 ++ .../git-form-additional-files-panel/index.js | 10 + .../git-form-auth-fieldset.html | 28 +-- .../git-form/git-form-auth-fieldset/index.js | 1 + ...it-form-auto-update-fieldset.controller.js | 26 +++ .../git-form-auto-update-fieldset.html | 69 ++++++ .../git-form-auto-update-fieldset/index.js | 10 + .../git-form-ref-field.html | 4 +- .../components/forms/git-form/git-form.html | 4 +- .../components/forms/git-form/git-form.js | 3 + .../components/forms/git-form/index.js | 6 + .../stack-redeploy-git-form.controller.js | 64 ++++- .../stack-redeploy-git-form.html | 50 +++- app/portainer/components/intervalFormat.js | 26 +++ app/portainer/helpers/webhookHelper.js | 17 +- app/portainer/models/stack.js | 2 + app/portainer/rest/stack.js | 5 +- app/portainer/services/api/stackService.js | 41 +++- .../stacks/create/createStackController.js | 44 +++- .../views/stacks/create/createstack.html | 11 +- app/portainer/views/stacks/edit/stack.html | 4 +- package.json | 3 +- yarn.lock | 5 + 94 files changed, 2680 insertions(+), 469 deletions(-) delete mode 100644 api/backup/copy_test.go create mode 100644 api/bolt/migrator/migrate_dbversion33.go create mode 100644 api/bolt/migrator/migrate_dbversion33_test.go create mode 100644 api/bolt/stack/tests/stack_test.go create mode 100644 api/exec/swarm_stack_test.go rename api/{backup => filesystem}/copy.go (63%) create mode 100644 api/filesystem/copy_test.go rename api/{backup/test_assets => filesystem/testdata}/copy_test/dir/.dotfile (100%) rename api/{backup/test_assets => filesystem/testdata}/copy_test/dir/inner (100%) rename api/{backup/test_assets => filesystem/testdata}/copy_test/outer (100%) create mode 100644 api/git/testdata/azure-repo copy.zip create mode 100644 api/http/handler/stacks/autoupdate.go create mode 100644 api/http/handler/stacks/helper.go create mode 100644 api/http/handler/stacks/helper_test.go delete mode 100644 api/http/handler/stacks/stack_create_test.go create mode 100644 api/http/handler/stacks/stack_update_git_redeploy.go create mode 100644 api/http/handler/stacks/webhook_invoke.go create mode 100644 api/http/handler/stacks/webhook_invoke_test.go create mode 100644 api/internal/stackutils/stackutils_test.go create mode 100644 api/scheduler/scheduler.go create mode 100644 api/scheduler/scheduler_test.go create mode 100644 api/stacks/deploy.go create mode 100644 api/stacks/deploy_test.go create mode 100644 api/stacks/deployer.go create mode 100644 api/stacks/scheduled.go create mode 100644 app/portainer/components/focusIf.js create mode 100644 app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-file-item/git-form-additional-file-item.controller.js create mode 100644 app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-file-item/git-form-additional-file-item.html create mode 100644 app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-file-item/index.js create mode 100644 app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-files-panel.controller.js create mode 100644 app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-files-panel.html create mode 100644 app/portainer/components/forms/git-form/git-form-additional-files-panel/index.js create mode 100644 app/portainer/components/forms/git-form/git-form-auto-update-fieldset/git-form-auto-update-fieldset.controller.js create mode 100644 app/portainer/components/forms/git-form/git-form-auto-update-fieldset/git-form-auto-update-fieldset.html create mode 100644 app/portainer/components/forms/git-form/git-form-auto-update-fieldset/index.js create mode 100644 app/portainer/components/intervalFormat.js diff --git a/api/backup/backup.go b/api/backup/backup.go index 3da032c2c..0a62786f5 100644 --- a/api/backup/backup.go +++ b/api/backup/backup.go @@ -10,6 +10,7 @@ import ( portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/archive" "github.com/portainer/portainer/api/crypto" + "github.com/portainer/portainer/api/filesystem" "github.com/portainer/portainer/api/http/offlinegate" ) @@ -32,7 +33,7 @@ func CreateBackupArchive(password string, gate *offlinegate.OfflineGate, datasto } for _, filename := range filesToBackup { - err := copyPath(filepath.Join(filestorePath, filename), backupDirPath) + err := filesystem.CopyPath(filepath.Join(filestorePath, filename), backupDirPath) if err != nil { return "", errors.Wrap(err, "Failed to create backup file") } diff --git a/api/backup/copy_test.go b/api/backup/copy_test.go deleted file mode 100644 index b9ceaeaab..000000000 --- a/api/backup/copy_test.go +++ /dev/null @@ -1,105 +0,0 @@ -package backup - -import ( - "io/ioutil" - "os" - "path" - "path/filepath" - "testing" - - "github.com/docker/docker/pkg/ioutils" - "github.com/stretchr/testify/assert" -) - -func listFiles(dir string) []string { - items := make([]string, 0) - filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if path == dir { - return nil - } - items = append(items, path) - return nil - }) - - return items -} - -func contains(t *testing.T, list []string, path string) { - assert.Contains(t, list, path) - copyContent, _ := ioutil.ReadFile(path) - assert.Equal(t, "content\n", string(copyContent)) -} - -func Test_copyFile_returnsError_whenSourceDoesNotExist(t *testing.T) { - tmpdir, _ := ioutils.TempDir("", "backup") - defer os.RemoveAll(tmpdir) - - err := copyFile("does-not-exist", tmpdir) - assert.NotNil(t, err) -} - -func Test_copyFile_shouldMakeAbackup(t *testing.T) { - tmpdir, _ := ioutils.TempDir("", "backup") - defer os.RemoveAll(tmpdir) - - content := []byte("content") - ioutil.WriteFile(path.Join(tmpdir, "origin"), content, 0600) - - err := copyFile(path.Join(tmpdir, "origin"), path.Join(tmpdir, "copy")) - assert.Nil(t, err) - - copyContent, _ := ioutil.ReadFile(path.Join(tmpdir, "copy")) - assert.Equal(t, content, copyContent) -} - -func Test_copyDir_shouldCopyAllFilesAndDirectories(t *testing.T) { - destination, _ := ioutils.TempDir("", "destination") - defer os.RemoveAll(destination) - err := copyDir("./test_assets/copy_test", destination) - assert.Nil(t, err) - - createdFiles := listFiles(destination) - - contains(t, createdFiles, filepath.Join(destination, "copy_test", "outer")) - contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", ".dotfile")) - contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", "inner")) -} - -func Test_backupPath_shouldSkipWhenNotExist(t *testing.T) { - tmpdir, _ := ioutils.TempDir("", "backup") - defer os.RemoveAll(tmpdir) - - err := copyPath("does-not-exists", tmpdir) - assert.Nil(t, err) - - assert.Empty(t, listFiles(tmpdir)) -} - -func Test_backupPath_shouldCopyFile(t *testing.T) { - tmpdir, _ := ioutils.TempDir("", "backup") - defer os.RemoveAll(tmpdir) - - content := []byte("content") - ioutil.WriteFile(path.Join(tmpdir, "file"), content, 0600) - - os.MkdirAll(path.Join(tmpdir, "backup"), 0700) - err := copyPath(path.Join(tmpdir, "file"), path.Join(tmpdir, "backup")) - assert.Nil(t, err) - - copyContent, err := ioutil.ReadFile(path.Join(tmpdir, "backup", "file")) - assert.Nil(t, err) - assert.Equal(t, content, copyContent) -} - -func Test_backupPath_shouldCopyDir(t *testing.T) { - destination, _ := ioutils.TempDir("", "destination") - defer os.RemoveAll(destination) - err := copyPath("./test_assets/copy_test", destination) - assert.Nil(t, err) - - createdFiles := listFiles(destination) - - contains(t, createdFiles, filepath.Join(destination, "copy_test", "outer")) - contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", ".dotfile")) - contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", "inner")) -} diff --git a/api/backup/restore.go b/api/backup/restore.go index b0d7acee2..e5329e913 100644 --- a/api/backup/restore.go +++ b/api/backup/restore.go @@ -11,6 +11,7 @@ import ( portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/archive" "github.com/portainer/portainer/api/crypto" + "github.com/portainer/portainer/api/filesystem" "github.com/portainer/portainer/api/http/offlinegate" ) @@ -59,7 +60,7 @@ func extractArchive(r io.Reader, destinationDirPath string) error { func restoreFiles(srcDir string, destinationDir string) error { for _, filename := range filesToRestore { - err := copyPath(filepath.Join(srcDir, filename), destinationDir) + err := filesystem.CopyPath(filepath.Join(srcDir, filename), destinationDir) if err != nil { return err } diff --git a/api/bolt/migrator/migrate_dbversion33.go b/api/bolt/migrator/migrate_dbversion33.go new file mode 100644 index 000000000..d7277ada7 --- /dev/null +++ b/api/bolt/migrator/migrate_dbversion33.go @@ -0,0 +1,32 @@ +package migrator + +import ( + portainer "github.com/portainer/portainer/api" +) + +func (m *Migrator) migrateDBVersionTo33() error { + err := migrateStackEntryPoint(m.stackService) + if err != nil { + return err + } + + return nil +} + +func migrateStackEntryPoint(stackService portainer.StackService) error { + stacks, err := stackService.Stacks() + if err != nil { + return err + } + for i := range stacks { + stack := &stacks[i] + if stack.GitConfig == nil { + continue + } + stack.GitConfig.ConfigFilePath = stack.EntryPoint + if err := stackService.UpdateStack(stack.ID, stack); err != nil { + return err + } + } + return nil +} diff --git a/api/bolt/migrator/migrate_dbversion33_test.go b/api/bolt/migrator/migrate_dbversion33_test.go new file mode 100644 index 000000000..256cc121e --- /dev/null +++ b/api/bolt/migrator/migrate_dbversion33_test.go @@ -0,0 +1,51 @@ +package migrator + +import ( + "path" + "testing" + "time" + + "github.com/boltdb/bolt" + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/bolt/internal" + "github.com/portainer/portainer/api/bolt/stack" + gittypes "github.com/portainer/portainer/api/git/types" + "github.com/stretchr/testify/assert" +) + +func TestMigrateStackEntryPoint(t *testing.T) { + dbConn, err := bolt.Open(path.Join(t.TempDir(), "portainer-ee-mig-33.db"), 0600, &bolt.Options{Timeout: 1 * time.Second}) + assert.NoError(t, err, "failed to init testing DB connection") + defer dbConn.Close() + + stackService, err := stack.NewService(&internal.DbConnection{DB: dbConn}) + assert.NoError(t, err, "failed to init testing Stack service") + + stacks := []*portainer.Stack{ + { + ID: 1, + EntryPoint: "dir/sub/compose.yml", + }, + { + ID: 2, + EntryPoint: "dir/sub/compose.yml", + GitConfig: &gittypes.RepoConfig{}, + }, + } + + for _, s := range stacks { + err := stackService.CreateStack(s) + assert.NoError(t, err, "failed to create stack") + } + + err = migrateStackEntryPoint(stackService) + assert.NoError(t, err, "failed to migrate entry point to Git ConfigFilePath") + + s, err := stackService.Stack(1) + assert.NoError(t, err) + assert.Nil(t, s.GitConfig, "first stack should not have git config") + + s, err = stackService.Stack(2) + assert.NoError(t, err) + assert.Equal(t, "dir/sub/compose.yml", s.GitConfig.ConfigFilePath, "second stack should have config file path migrated") +} diff --git a/api/bolt/migrator/migrator.go b/api/bolt/migrator/migrator.go index fc00578f0..df3ad0436 100644 --- a/api/bolt/migrator/migrator.go +++ b/api/bolt/migrator/migrator.go @@ -381,5 +381,11 @@ func (m *Migrator) Migrate() error { } } + if m.currentDBVersion < 33 { + if err := m.migrateDBVersionTo33(); err != nil { + return err + } + } + return m.versionService.StoreDBVersion(portainer.DBVersion) } diff --git a/api/bolt/stack/stack.go b/api/bolt/stack/stack.go index f9cfafad7..094a52b2e 100644 --- a/api/bolt/stack/stack.go +++ b/api/bolt/stack/stack.go @@ -1,11 +1,14 @@ package stack import ( + "strings" + portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/bolt/errors" "github.com/portainer/portainer/api/bolt/internal" "github.com/boltdb/bolt" + pkgerrors "github.com/pkg/errors" ) const ( @@ -133,3 +136,76 @@ func (service *Service) DeleteStack(ID portainer.StackID) error { identifier := internal.Itob(int(ID)) return internal.DeleteObject(service.connection, BucketName, identifier) } + +// StackByWebhookID returns a pointer to a stack object by webhook ID. +// It returns nil, errors.ErrObjectNotFound if there's no stack associated with the webhook ID. +func (service *Service) StackByWebhookID(id string) (*portainer.Stack, error) { + if id == "" { + return nil, pkgerrors.New("webhook ID can't be empty string") + } + var stack portainer.Stack + found := false + + err := service.connection.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(BucketName)) + cursor := bucket.Cursor() + + for k, v := cursor.First(); k != nil; k, v = cursor.Next() { + var t struct { + AutoUpdate *struct { + WebhookID string `json:"Webhook"` + } `json:"AutoUpdate"` + } + + err := internal.UnmarshalObject(v, &t) + if err != nil { + return err + } + + if t.AutoUpdate != nil && strings.EqualFold(t.AutoUpdate.WebhookID, id) { + found = true + err := internal.UnmarshalObject(v, &stack) + if err != nil { + return err + } + break + } + } + + return nil + }) + + if err != nil { + return nil, err + } + if !found { + return nil, errors.ErrObjectNotFound + } + + return &stack, nil +} + +// RefreshableStacks returns stacks that are configured for a periodic update +func (service *Service) RefreshableStacks() ([]portainer.Stack, error) { + stacks := make([]portainer.Stack, 0) + err := service.connection.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(BucketName)) + cursor := bucket.Cursor() + + var stack portainer.Stack + for k, v := cursor.First(); k != nil; k, v = cursor.Next() { + err := internal.UnmarshalObject(v, &stack) + if err != nil { + return err + } + + if stack.AutoUpdate != nil && stack.AutoUpdate.Interval != "" { + stacks = append(stacks, stack) + } + } + + return nil + }) + + return stacks, err +} diff --git a/api/bolt/stack/tests/stack_test.go b/api/bolt/stack/tests/stack_test.go new file mode 100644 index 000000000..d0c66dadf --- /dev/null +++ b/api/bolt/stack/tests/stack_test.go @@ -0,0 +1,111 @@ +package tests + +import ( + "testing" + "time" + + "github.com/portainer/portainer/api/bolt" + + bolterrors "github.com/portainer/portainer/api/bolt/errors" + + "github.com/portainer/portainer/api/bolt/bolttest" + + "github.com/gofrs/uuid" + + "github.com/stretchr/testify/assert" + + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/filesystem" +) + +func newGuidString(t *testing.T) string { + uuid, err := uuid.NewV4() + assert.NoError(t, err) + + return uuid.String() +} + +type stackBuilder struct { + t *testing.T + count int + store *bolt.Store +} + +func TestService_StackByWebhookID(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode. Normally takes ~1s to run.") + } + store, teardown := bolttest.MustNewTestStore(true) + defer teardown() + + b := stackBuilder{t: t, store: store} + b.createNewStack(newGuidString(t)) + for i := 0; i < 10; i++ { + b.createNewStack("") + } + webhookID := newGuidString(t) + stack := b.createNewStack(webhookID) + + // can find a stack by webhook ID + got, err := store.StackService.StackByWebhookID(webhookID) + assert.NoError(t, err) + assert.Equal(t, stack, *got) + + // returns nil and object not found error if there's no stack associated with the webhook + got, err = store.StackService.StackByWebhookID(newGuidString(t)) + assert.Nil(t, got) + assert.ErrorIs(t, err, bolterrors.ErrObjectNotFound) +} + +func (b *stackBuilder) createNewStack(webhookID string) portainer.Stack { + b.count++ + stack := portainer.Stack{ + ID: portainer.StackID(b.count), + Name: "Name", + Type: portainer.DockerComposeStack, + EndpointID: 2, + EntryPoint: filesystem.ComposeFileDefaultName, + Env: []portainer.Pair{{"Name1", "Value1"}}, + Status: portainer.StackStatusActive, + CreationDate: time.Now().Unix(), + ProjectPath: "/tmp/project", + CreatedBy: "test", + } + + if webhookID == "" { + if b.count%2 == 0 { + stack.AutoUpdate = &portainer.StackAutoUpdate{ + Interval: "", + Webhook: "", + } + } // else keep AutoUpdate nil + } else { + stack.AutoUpdate = &portainer.StackAutoUpdate{Webhook: webhookID} + } + + err := b.store.StackService.CreateStack(&stack) + assert.NoError(b.t, err) + + return stack +} + +func Test_RefreshableStacks(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode. Normally takes ~1s to run.") + } + store, teardown := bolttest.MustNewTestStore(true) + defer teardown() + + staticStack := portainer.Stack{ID: 1} + stackWithWebhook := portainer.Stack{ID: 2, AutoUpdate: &portainer.StackAutoUpdate{Webhook: "webhook"}} + refreshableStack := portainer.Stack{ID: 3, AutoUpdate: &portainer.StackAutoUpdate{Interval: "1m"}} + + for _, stack := range []*portainer.Stack{&staticStack, &stackWithWebhook, &refreshableStack} { + err := store.Stack().CreateStack(stack) + assert.NoError(t, err) + } + + stacks, err := store.Stack().RefreshableStacks() + assert.NoError(t, err) + assert.ElementsMatch(t, []portainer.Stack{refreshableStack}, stacks) +} diff --git a/api/cmd/portainer/main.go b/api/cmd/portainer/main.go index e9fd9e98c..717c3fbc3 100644 --- a/api/cmd/portainer/main.go +++ b/api/cmd/portainer/main.go @@ -6,7 +6,6 @@ import ( "os" "strings" - wrapper "github.com/portainer/docker-compose-wrapper" portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/bolt" "github.com/portainer/portainer/api/chisel" @@ -31,6 +30,8 @@ import ( "github.com/portainer/portainer/api/ldap" "github.com/portainer/portainer/api/libcompose" "github.com/portainer/portainer/api/oauth" + "github.com/portainer/portainer/api/scheduler" + "github.com/portainer/portainer/api/stacks" ) func initCLI() *portainer.CLIFlags { @@ -88,12 +89,8 @@ func shutdownDatastore(shutdownCtx context.Context, datastore portainer.DataStor func initComposeStackManager(assetsPath string, dataStorePath string, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager { composeWrapper, err := exec.NewComposeStackManager(assetsPath, dataStorePath, proxyManager) if err != nil { - if err == wrapper.ErrBinaryNotFound { - log.Printf("[INFO] [message: docker-compose binary not found, falling back to libcompose]") - return libcompose.NewComposeStackManager(dataStorePath, reverseTunnelService) - } - - log.Fatalf("failed initalizing compose stack manager; err=%s", err) + log.Printf("[INFO] [main,compose] [message: falling-back to libcompose] [error: %s]", err) + return libcompose.NewComposeStackManager(dataStorePath, reverseTunnelService) } return composeWrapper @@ -525,6 +522,10 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server { log.Fatalf("failed to fetch ssl settings from DB") } + scheduler := scheduler.NewScheduler(shutdownCtx) + stackDeployer := stacks.NewStackDeployer(swarmStackManager, composeStackManager) + stacks.StartStackSchedules(scheduler, stackDeployer, dataStore, gitService) + return &http.Server{ AuthorizationService: authorizationService, ReverseTunnelService: reverseTunnelService, @@ -550,8 +551,10 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server { SSLService: sslService, DockerClientFactory: dockerClientFactory, KubernetesClientFactory: kubernetesClientFactory, + Scheduler: scheduler, ShutdownCtx: shutdownCtx, ShutdownTrigger: shutdownTrigger, + StackDeployer: stackDeployer, } } diff --git a/api/exec/compose_stack.go b/api/exec/compose_stack.go index bb6e9f055..36283c6d9 100644 --- a/api/exec/compose_stack.go +++ b/api/exec/compose_stack.go @@ -7,8 +7,8 @@ import ( "regexp" "strings" + "github.com/pkg/errors" wrapper "github.com/portainer/docker-compose-wrapper" - portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/http/proxy" "github.com/portainer/portainer/api/http/proxy/factory" @@ -35,12 +35,6 @@ func NewComposeStackManager(binaryPath string, configPath string, proxyManager * }, nil } -// NormalizeStackName returns a new stack name with unsupported characters replaced -func (w *ComposeStackManager) NormalizeStackName(name string) string { - r := regexp.MustCompile("[^a-z0-9]+") - return r.ReplaceAllString(strings.ToLower(name), "") -} - // ComposeSyntaxMaxVersion returns the maximum supported version of the docker compose syntax func (w *ComposeStackManager) ComposeSyntaxMaxVersion() string { return portainer.ComposeSyntaxMaxVersion @@ -50,7 +44,7 @@ func (w *ComposeStackManager) ComposeSyntaxMaxVersion() string { func (w *ComposeStackManager) Up(stack *portainer.Stack, endpoint *portainer.Endpoint) error { url, proxy, err := w.fetchEndpointProxy(endpoint) if err != nil { - return err + return errors.Wrap(err, "failed to featch endpoint proxy") } if proxy != nil { @@ -59,13 +53,12 @@ func (w *ComposeStackManager) Up(stack *portainer.Stack, endpoint *portainer.End envFilePath, err := createEnvFile(stack) if err != nil { - return err + return errors.Wrap(err, "failed to create env file") } - filePath := stackFilePath(stack) - - _, err = w.wrapper.Up([]string{filePath}, url, stack.Name, envFilePath, w.configPath) - return err + filePaths := append([]string{stack.EntryPoint}, stack.AdditionalFiles...) + _, err = w.wrapper.Up(filePaths, stack.ProjectPath, url, stack.Name, envFilePath, w.configPath) + return errors.Wrap(err, "failed to deploy a stack") } // Down stops and removes containers, networks, images, and volumes. Wraps `docker-compose down --remove-orphans` command @@ -78,14 +71,16 @@ func (w *ComposeStackManager) Down(stack *portainer.Stack, endpoint *portainer.E defer proxy.Close() } - filePath := stackFilePath(stack) + filePaths := append([]string{stack.EntryPoint}, stack.AdditionalFiles...) - _, err = w.wrapper.Down([]string{filePath}, url, stack.Name) + _, err = w.wrapper.Down(filePaths, stack.ProjectPath, url, stack.Name) return err } -func stackFilePath(stack *portainer.Stack) string { - return path.Join(stack.ProjectPath, stack.EntryPoint) +// NormalizeStackName returns a new stack name with unsupported characters replaced +func (w *ComposeStackManager) NormalizeStackName(name string) string { + r := regexp.MustCompile("[^a-z0-9]+") + return r.ReplaceAllString(strings.ToLower(name), "") } func (w *ComposeStackManager) fetchEndpointProxy(endpoint *portainer.Endpoint) (string, *factory.ProxyServer, error) { @@ -118,5 +113,5 @@ func createEnvFile(stack *portainer.Stack) (string, error) { } envfile.Close() - return envFilePath, nil + return "stack.env", nil } diff --git a/api/exec/compose_stack_integration_test.go b/api/exec/compose_stack_integration_test.go index 8214d3461..f0b279659 100644 --- a/api/exec/compose_stack_integration_test.go +++ b/api/exec/compose_stack_integration_test.go @@ -1,5 +1,3 @@ -// +build integration - package exec import ( diff --git a/api/exec/compose_stack_test.go b/api/exec/compose_stack_test.go index 80e2818df..c61285ebd 100644 --- a/api/exec/compose_stack_test.go +++ b/api/exec/compose_stack_test.go @@ -10,47 +10,6 @@ import ( "github.com/stretchr/testify/assert" ) -func Test_stackFilePath(t *testing.T) { - tests := []struct { - name string - stack *portainer.Stack - expected string - }{ - // { - // name: "should return empty result if stack is missing", - // stack: nil, - // expected: "", - // }, - // { - // name: "should return empty result if stack don't have entrypoint", - // stack: &portainer.Stack{}, - // expected: "", - // }, - { - name: "should allow file name and dir", - stack: &portainer.Stack{ - ProjectPath: "dir", - EntryPoint: "file", - }, - expected: path.Join("dir", "file"), - }, - { - name: "should allow file name only", - stack: &portainer.Stack{ - EntryPoint: "file", - }, - expected: "file", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := stackFilePath(tt.stack) - assert.Equal(t, tt.expected, result) - }) - } -} - func Test_createEnvFile(t *testing.T) { dir := t.TempDir() @@ -60,11 +19,6 @@ func Test_createEnvFile(t *testing.T) { expected string expectedFile bool }{ - // { - // name: "should not add env file option if stack is missing", - // stack: nil, - // expected: "", - // }, { name: "should not add env file option if stack doesn't have env variables", stack: &portainer.Stack{ @@ -98,7 +52,7 @@ func Test_createEnvFile(t *testing.T) { result, _ := createEnvFile(tt.stack) if tt.expected != "" { - assert.Equal(t, path.Join(tt.stack.ProjectPath, "stack.env"), result) + assert.Equal(t, "stack.env", result) f, _ := os.Open(path.Join(dir, "stack.env")) content, _ := ioutil.ReadAll(f) diff --git a/api/exec/swarm_stack.go b/api/exec/swarm_stack.go index 69f91581c..eecb1047a 100644 --- a/api/exec/swarm_stack.go +++ b/api/exec/swarm_stack.go @@ -13,6 +13,7 @@ import ( "strings" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/internal/stackutils" ) // SwarmStackManager represents a service for managing stacks. @@ -63,22 +64,23 @@ func (manager *SwarmStackManager) Logout(endpoint *portainer.Endpoint) error { // Deploy executes the docker stack deploy command. func (manager *SwarmStackManager) Deploy(stack *portainer.Stack, prune bool, endpoint *portainer.Endpoint) error { - stackFilePath := path.Join(stack.ProjectPath, stack.EntryPoint) + filePaths := stackutils.GetStackFilePaths(stack) command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint) if prune { - args = append(args, "stack", "deploy", "--prune", "--with-registry-auth", "--compose-file", stackFilePath, stack.Name) + args = append(args, "stack", "deploy", "--prune", "--with-registry-auth") } else { - args = append(args, "stack", "deploy", "--with-registry-auth", "--compose-file", stackFilePath, stack.Name) + args = append(args, "stack", "deploy", "--with-registry-auth") } + args = configureFilePaths(args, filePaths) + args = append(args, stack.Name) + env := make([]string, 0) for _, envvar := range stack.Env { env = append(env, envvar.Name+"="+envvar.Value) } - - stackFolder := path.Dir(stackFilePath) - return runCommandAndCaptureStdErr(command, args, env, stackFolder) + return runCommandAndCaptureStdErr(command, args, env, stack.ProjectPath) } // Remove executes the docker stack rm command. @@ -191,3 +193,10 @@ func (manager *SwarmStackManager) NormalizeStackName(name string) string { r := regexp.MustCompile("[^a-z0-9]+") return r.ReplaceAllString(strings.ToLower(name), "") } + +func configureFilePaths(args []string, filePaths []string) []string { + for _, path := range filePaths { + args = append(args, "--compose-file", path) + } + return args +} diff --git a/api/exec/swarm_stack_test.go b/api/exec/swarm_stack_test.go new file mode 100644 index 000000000..47d28ce2c --- /dev/null +++ b/api/exec/swarm_stack_test.go @@ -0,0 +1,15 @@ +package exec + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConfigFilePaths(t *testing.T) { + args := []string{"stack", "deploy", "--with-registry-auth"} + filePaths := []string{"dir/file", "dir/file-two", "dir/file-three"} + expected := []string{"stack", "deploy", "--with-registry-auth", "--compose-file", "dir/file", "--compose-file", "dir/file-two", "--compose-file", "dir/file-three"} + output := configureFilePaths(args, filePaths) + assert.ElementsMatch(t, expected, output, "wrong output file paths") +} diff --git a/api/backup/copy.go b/api/filesystem/copy.go similarity index 63% rename from api/backup/copy.go rename to api/filesystem/copy.go index 6aaefd54c..abf4d33aa 100644 --- a/api/backup/copy.go +++ b/api/filesystem/copy.go @@ -1,4 +1,4 @@ -package backup +package filesystem import ( "errors" @@ -8,7 +8,8 @@ import ( "strings" ) -func copyPath(path string, toDir string) error { +// CopyPath copies file or directory defined by the path to the toDir path +func CopyPath(path string, toDir string) error { info, err := os.Stat(path) if err != nil && errors.Is(err, os.ErrNotExist) { // skip copy if file does not exist @@ -20,17 +21,30 @@ func copyPath(path string, toDir string) error { return copyFile(path, destination) } - return copyDir(path, toDir) + return CopyDir(path, toDir, true) } -func copyDir(fromDir, toDir string) error { +// CopyDir copies contents of fromDir to toDir. +// When keepParent is true, contents will be copied with their immediate parent dir, +// i.e. given /from/dirA and /to/dirB with keepParent == true, result will be /to/dirB/dirA/ +func CopyDir(fromDir, toDir string, keepParent bool) error { cleanedSourcePath := filepath.Clean(fromDir) parentDirectory := filepath.Dir(cleanedSourcePath) err := filepath.Walk(cleanedSourcePath, func(path string, info os.FileInfo, err error) error { if err != nil { return err } - destination := filepath.Join(toDir, strings.TrimPrefix(path, parentDirectory)) + var destination string + if keepParent { + destination = filepath.Join(toDir, strings.TrimPrefix(path, parentDirectory)) + } else { + destination = filepath.Join(toDir, strings.TrimPrefix(path, cleanedSourcePath)) + } + + if destination == "" { + return nil + } + if info.IsDir() { return nil // skip directory creations } diff --git a/api/filesystem/copy_test.go b/api/filesystem/copy_test.go new file mode 100644 index 000000000..2fcef9e6b --- /dev/null +++ b/api/filesystem/copy_test.go @@ -0,0 +1,92 @@ +package filesystem + +import ( + "io/ioutil" + "os" + "path" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_copyFile_returnsError_whenSourceDoesNotExist(t *testing.T) { + tmpdir, _ := ioutil.TempDir("", "backup") + defer os.RemoveAll(tmpdir) + + err := copyFile("does-not-exist", tmpdir) + assert.Error(t, err) +} + +func Test_copyFile_shouldMakeAbackup(t *testing.T) { + tmpdir, _ := ioutil.TempDir("", "backup") + defer os.RemoveAll(tmpdir) + + content := []byte("content") + ioutil.WriteFile(path.Join(tmpdir, "origin"), content, 0600) + + err := copyFile(path.Join(tmpdir, "origin"), path.Join(tmpdir, "copy")) + assert.NoError(t, err) + + copyContent, _ := ioutil.ReadFile(path.Join(tmpdir, "copy")) + assert.Equal(t, content, copyContent) +} + +func Test_CopyDir_shouldCopyAllFilesAndDirectories(t *testing.T) { + destination, _ := ioutil.TempDir("", "destination") + defer os.RemoveAll(destination) + err := CopyDir("./testdata/copy_test", destination, true) + assert.NoError(t, err) + + assert.FileExists(t, filepath.Join(destination, "copy_test", "outer")) + assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", ".dotfile")) + assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", "inner")) +} + +func Test_CopyDir_shouldCopyOnlyDirContents(t *testing.T) { + destination, _ := ioutil.TempDir("", "destination") + defer os.RemoveAll(destination) + err := CopyDir("./testdata/copy_test", destination, false) + assert.NoError(t, err) + + assert.FileExists(t, filepath.Join(destination, "outer")) + assert.FileExists(t, filepath.Join(destination, "dir", ".dotfile")) + assert.FileExists(t, filepath.Join(destination, "dir", "inner")) +} + +func Test_CopyPath_shouldSkipWhenNotExist(t *testing.T) { + tmpdir, _ := ioutil.TempDir("", "backup") + defer os.RemoveAll(tmpdir) + + err := CopyPath("does-not-exists", tmpdir) + assert.NoError(t, err) + + assert.NoFileExists(t, tmpdir) +} + +func Test_CopyPath_shouldCopyFile(t *testing.T) { + tmpdir, _ := ioutil.TempDir("", "backup") + defer os.RemoveAll(tmpdir) + + content := []byte("content") + ioutil.WriteFile(path.Join(tmpdir, "file"), content, 0600) + + os.MkdirAll(path.Join(tmpdir, "backup"), 0700) + err := CopyPath(path.Join(tmpdir, "file"), path.Join(tmpdir, "backup")) + assert.NoError(t, err) + + copyContent, err := ioutil.ReadFile(path.Join(tmpdir, "backup", "file")) + assert.NoError(t, err) + assert.Equal(t, content, copyContent) +} + +func Test_CopyPath_shouldCopyDir(t *testing.T) { + destination, _ := ioutil.TempDir("", "destination") + defer os.RemoveAll(destination) + err := CopyPath("./testdata/copy_test", destination) + assert.NoError(t, err) + + assert.FileExists(t, filepath.Join(destination, "copy_test", "outer")) + assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", ".dotfile")) + assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", "inner")) +} diff --git a/api/backup/test_assets/copy_test/dir/.dotfile b/api/filesystem/testdata/copy_test/dir/.dotfile similarity index 100% rename from api/backup/test_assets/copy_test/dir/.dotfile rename to api/filesystem/testdata/copy_test/dir/.dotfile diff --git a/api/backup/test_assets/copy_test/dir/inner b/api/filesystem/testdata/copy_test/dir/inner similarity index 100% rename from api/backup/test_assets/copy_test/dir/inner rename to api/filesystem/testdata/copy_test/dir/inner diff --git a/api/backup/test_assets/copy_test/outer b/api/filesystem/testdata/copy_test/outer similarity index 100% rename from api/backup/test_assets/copy_test/outer rename to api/filesystem/testdata/copy_test/outer diff --git a/api/git/azure.go b/api/git/azure.go index 78f10e52d..417d5db4b 100644 --- a/api/git/azure.go +++ b/api/git/azure.go @@ -2,15 +2,17 @@ package git import ( "context" + "encoding/json" "fmt" - "github.com/pkg/errors" - "github.com/portainer/portainer/api/archive" "io" "io/ioutil" "net/http" "net/url" "os" "strings" + + "github.com/pkg/errors" + "github.com/portainer/portainer/api/archive" ) const ( @@ -37,7 +39,7 @@ type azureDownloader struct { func NewAzureDownloader(client *http.Client) *azureDownloader { return &azureDownloader{ - client: client, + client: client, baseUrl: "https://dev.azure.com", } } @@ -100,6 +102,57 @@ func (a *azureDownloader) downloadZipFromAzureDevOps(ctx context.Context, option return zipFile.Name(), nil } +func (a *azureDownloader) latestCommitID(ctx context.Context, options fetchOptions) (string, error) { + config, err := parseUrl(options.repositoryUrl) + if err != nil { + return "", errors.WithMessage(err, "failed to parse url") + } + + refsUrl, err := a.buildRefsUrl(config, options.referenceName) + if err != nil { + return "", errors.WithMessage(err, "failed to build azure refs url") + } + + req, err := http.NewRequestWithContext(ctx, "GET", refsUrl, nil) + if options.username != "" || options.password != "" { + req.SetBasicAuth(options.username, options.password) + } else if config.username != "" || config.password != "" { + req.SetBasicAuth(config.username, config.password) + } + + if err != nil { + return "", errors.WithMessage(err, "failed to create a new HTTP request") + } + + resp, err := a.client.Do(req) + if err != nil { + return "", errors.WithMessage(err, "failed to make an HTTP request") + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("failed to get repository refs with a status \"%v\"", resp.Status) + } + + var refs struct { + Value []struct { + Name string `json:"name"` + ObjectId string `json:"objectId"` + } + } + if err := json.NewDecoder(resp.Body).Decode(&refs); err != nil { + return "", errors.Wrap(err, "could not parse Azure Refs response") + } + + for _, ref := range refs.Value { + if strings.EqualFold(ref.Name, options.referenceName) { + return ref.ObjectId, nil + } + } + + return "", errors.Errorf("could not find ref %q in the repository", options.referenceName) +} + func parseUrl(rawUrl string) (*azureOptions, error) { if strings.HasPrefix(rawUrl, "https://") || strings.HasPrefix(rawUrl, "http://") { return parseHttpUrl(rawUrl) @@ -193,6 +246,27 @@ func (a *azureDownloader) buildDownloadUrl(config *azureOptions, referenceName s return u.String(), nil } +func (a *azureDownloader) buildRefsUrl(config *azureOptions, referenceName string) (string, error) { + rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/refs", + a.baseUrl, + url.PathEscape(config.organisation), + url.PathEscape(config.project), + url.PathEscape(config.repository)) + u, err := url.Parse(rawUrl) + + if err != nil { + return "", errors.Wrapf(err, "failed to parse refs url path %s", rawUrl) + } + + // filterContains=main&api-version=6.0 + q := u.Query() + q.Set("filterContains", formatReferenceName(referenceName)) + q.Set("api-version", "6.0") + u.RawQuery = q.Encode() + + return u.String(), nil +} + const ( branchPrefix = "refs/heads/" tagPrefix = "refs/tags/" diff --git a/api/git/azure_integration_test.go b/api/git/azure_integration_test.go index 6d684d877..200747daa 100644 --- a/api/git/azure_integration_test.go +++ b/api/git/azure_integration_test.go @@ -78,6 +78,18 @@ func TestService_ClonePrivateRepository_Azure(t *testing.T) { assert.FileExists(t, filepath.Join(dst, "README.md")) } +func TestService_LatestCommitID_Azure(t *testing.T) { + ensureIntegrationTest(t) + + pat := getRequiredValue(t, "AZURE_DEVOPS_PAT") + service := NewService() + + repositoryUrl := "https://portainer.visualstudio.com/Playground/_git/dev_integration" + id, err := service.LatestCommitID(repositoryUrl, "refs/heads/main", "", pat) + assert.NoError(t, err) + assert.NotEmpty(t, id, "cannot guarantee commit id, but it should be not empty") +} + func getRequiredValue(t *testing.T, name string) string { value, ok := os.LookupEnv(name) if !ok { diff --git a/api/git/azure_test.go b/api/git/azure_test.go index 18417e9e6..b95dc981e 100644 --- a/api/git/azure_test.go +++ b/api/git/azure_test.go @@ -2,11 +2,12 @@ package git import ( "context" - "github.com/stretchr/testify/assert" "net/http" "net/http/httptest" "net/url" "testing" + + "github.com/stretchr/testify/assert" ) func Test_buildDownloadUrl(t *testing.T) { @@ -27,6 +28,23 @@ func Test_buildDownloadUrl(t *testing.T) { } } +func Test_buildRefsUrl(t *testing.T) { + a := NewAzureDownloader(nil) + u, err := a.buildRefsUrl(&azureOptions{ + organisation: "organisation", + project: "project", + repository: "repository", + }, "refs/heads/main") + + expectedUrl, _ := url.Parse("https://dev.azure.com/organisation/project/_apis/git/repositories/repository/refs?filterContains=main&api-version=6.0") + actualUrl, _ := url.Parse(u) + assert.NoError(t, err) + assert.Equal(t, expectedUrl.Host, actualUrl.Host) + assert.Equal(t, expectedUrl.Scheme, actualUrl.Scheme) + assert.Equal(t, expectedUrl.Path, actualUrl.Path) + assert.Equal(t, expectedUrl.Query(), actualUrl.Query()) +} + func Test_parseAzureUrl(t *testing.T) { type args struct { url string @@ -248,3 +266,110 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) { }) } } + +func Test_azureDownloader_latestCommitID(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + response := `{ + "value": [ + { + "name": "refs/heads/feature/calcApp", + "objectId": "ffe9cba521f00d7f60e322845072238635edb451", + "creator": { + "displayName": "Normal Paulk", + "url": "https://vssps.dev.azure.com/fabrikam/_apis/Identities/ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "_links": { + "avatar": { + "href": "https://dev.azure.com/fabrikam/_apis/GraphProfile/MemberAvatars/aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5" + } + }, + "id": "ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "uniqueName": "dev@mailserver.com", + "imageUrl": "https://dev.azure.com/fabrikam/_api/_common/identityImage?id=ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "descriptor": "aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5" + }, + "url": "https://dev.azure.com/fabrikam/7484f783-66a3-4f27-b7cd-6b08b0b077ed/_apis/git/repositories/d3d1760b-311c-4175-a726-20dfc6a7f885/refs?filter=heads%2Ffeature%2FcalcApp" + }, + { + "name": "refs/heads/feature/replacer", + "objectId": "917131a709996c5cfe188c3b57e9a6ad90e8b85c", + "creator": { + "displayName": "Normal Paulk", + "url": "https://vssps.dev.azure.com/fabrikam/_apis/Identities/ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "_links": { + "avatar": { + "href": "https://dev.azure.com/fabrikam/_apis/GraphProfile/MemberAvatars/aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5" + } + }, + "id": "ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "uniqueName": "dev@mailserver.com", + "imageUrl": "https://dev.azure.com/fabrikam/_api/_common/identityImage?id=ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "descriptor": "aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5" + }, + "url": "https://dev.azure.com/fabrikam/7484f783-66a3-4f27-b7cd-6b08b0b077ed/_apis/git/repositories/d3d1760b-311c-4175-a726-20dfc6a7f885/refs?filter=heads%2Ffeature%2Freplacer" + }, + { + "name": "refs/heads/master", + "objectId": "ffe9cba521f00d7f60e322845072238635edb451", + "creator": { + "displayName": "Normal Paulk", + "url": "https://vssps.dev.azure.com/fabrikam/_apis/Identities/ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "_links": { + "avatar": { + "href": "https://dev.azure.com/fabrikam/_apis/GraphProfile/MemberAvatars/aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5" + } + }, + "id": "ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "uniqueName": "dev@mailserver.com", + "imageUrl": "https://dev.azure.com/fabrikam/_api/_common/identityImage?id=ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "descriptor": "aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5" + }, + "url": "https://dev.azure.com/fabrikam/7484f783-66a3-4f27-b7cd-6b08b0b077ed/_apis/git/repositories/d3d1760b-311c-4175-a726-20dfc6a7f885/refs?filter=heads%2Fmaster" + } + ], + "count": 3 + }` + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(response)) + })) + defer server.Close() + + a := &azureDownloader{ + client: server.Client(), + baseUrl: server.URL, + } + + tests := []struct { + name string + args fetchOptions + want string + wantErr bool + }{ + { + name: "should be able to parse response", + args: fetchOptions{ + referenceName: "refs/heads/master", + repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository"}, + want: "ffe9cba521f00d7f60e322845072238635edb451", + wantErr: false, + }, + { + name: "should be able to parse response", + args: fetchOptions{ + referenceName: "refs/heads/unknown", + repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository"}, + want: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + id, err := a.latestCommitID(context.Background(), tt.args) + if (err != nil) != tt.wantErr { + t.Errorf("azureDownloader.latestCommitID() error = %v, wantErr %v", err, tt.wantErr) + return + } + assert.Equal(t, tt.want, id) + }) + } +} diff --git a/api/git/git.go b/api/git/git.go index 519d699a8..57c8eb106 100644 --- a/api/git/git.go +++ b/api/git/git.go @@ -6,16 +6,26 @@ import ( "net/http" "os" "path/filepath" + "strings" "time" "github.com/pkg/errors" "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/transport/client" githttp "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/go-git/go-git/v5/storage/memory" ) +type fetchOptions struct { + repositoryUrl string + username string + password string + referenceName string +} + type cloneOptions struct { repositoryUrl string username string @@ -26,6 +36,7 @@ type cloneOptions struct { type downloader interface { download(ctx context.Context, dst string, opt cloneOptions) error + latestCommitID(ctx context.Context, opt fetchOptions) (string, error) } type gitClient struct { @@ -36,13 +47,7 @@ func (c gitClient) download(ctx context.Context, dst string, opt cloneOptions) e gitOptions := git.CloneOptions{ URL: opt.repositoryUrl, Depth: opt.depth, - } - - if opt.password != "" || opt.username != "" { - gitOptions.Auth = &githttp.BasicAuth{ - Username: opt.username, - Password: opt.password, - } + Auth: getAuth(opt.username, opt.password), } if opt.referenceName != "" { @@ -62,6 +67,44 @@ func (c gitClient) download(ctx context.Context, dst string, opt cloneOptions) e return nil } +func (c gitClient) latestCommitID(ctx context.Context, opt fetchOptions) (string, error) { + remote := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{ + Name: "origin", + URLs: []string{opt.repositoryUrl}, + }) + + listOptions := &git.ListOptions{ + Auth: getAuth(opt.username, opt.password), + } + + refs, err := remote.List(listOptions) + if err != nil { + return "", errors.Wrap(err, "failed to list repository refs") + } + + for _, ref := range refs { + if strings.EqualFold(ref.Name().String(), opt.referenceName) { + return ref.Hash().String(), nil + } + } + + return "", errors.Errorf("could not find ref %q in the repository", opt.referenceName) +} + +func getAuth(username, password string) *githttp.BasicAuth { + if password != "" { + if username == "" { + username = "token" + } + + return &githttp.BasicAuth{ + Username: username, + Password: password, + } + } + return nil +} + // Service represents a service for managing Git. type Service struct { httpsCli *http.Client @@ -109,3 +152,19 @@ func (service *Service) cloneRepository(destination string, options cloneOptions return service.git.download(context.TODO(), destination, options) } + +// LatestCommitID returns SHA1 of the latest commit of the specified reference +func (service *Service) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) { + options := fetchOptions{ + repositoryUrl: repositoryURL, + username: username, + password: password, + referenceName: referenceName, + } + + if isAzureUrl(options.repositoryUrl) { + return service.azure.latestCommitID(context.TODO(), options) + } + + return service.git.latestCommitID(context.TODO(), options) +} diff --git a/api/git/git_integration_test.go b/api/git/git_integration_test.go index d35ba8d52..6f123c130 100644 --- a/api/git/git_integration_test.go +++ b/api/git/git_integration_test.go @@ -12,7 +12,7 @@ import ( func TestService_ClonePrivateRepository_GitHub(t *testing.T) { ensureIntegrationTest(t) - pat := getRequiredValue(t, "GITHUB_PAT") + accessToken := getRequiredValue(t, "GITHUB_PAT") username := getRequiredValue(t, "GITHUB_USERNAME") service := NewService() @@ -21,7 +21,20 @@ func TestService_ClonePrivateRepository_GitHub(t *testing.T) { defer os.RemoveAll(dst) repositoryUrl := "https://github.com/portainer/private-test-repository.git" - err = service.CloneRepository(dst, repositoryUrl, "refs/heads/main", username, pat) + err = service.CloneRepository(dst, repositoryUrl, "refs/heads/main", username, accessToken) assert.NoError(t, err) assert.FileExists(t, filepath.Join(dst, "README.md")) } + +func TestService_LatestCommitID_GitHub(t *testing.T) { + ensureIntegrationTest(t) + + accessToken := getRequiredValue(t, "GITHUB_PAT") + username := getRequiredValue(t, "GITHUB_USERNAME") + service := NewService() + + repositoryUrl := "https://github.com/portainer/private-test-repository.git" + id, err := service.LatestCommitID(repositoryUrl, "refs/heads/main", username, accessToken) + assert.NoError(t, err) + assert.NotEmpty(t, id, "cannot guarantee commit id, but it should be not empty") +} diff --git a/api/git/git_test.go b/api/git/git_test.go index 14878b304..11ff5e5c8 100644 --- a/api/git/git_test.go +++ b/api/git/git_test.go @@ -105,7 +105,19 @@ func Test_cloneRepository(t *testing.T) { }) assert.NoError(t, err) - assert.Equal(t, 3, getCommitHistoryLength(t, err, dir), "cloned repo has incorrect depth") + assert.Equal(t, 4, getCommitHistoryLength(t, err, dir), "cloned repo has incorrect depth") +} + +func Test_latestCommitID(t *testing.T) { + service := Service{git: gitClient{preserveGitDirectory: true}} // no need for http client since the test access the repo via file system. + + repositoryURL := bareRepoDir + referenceName := "refs/heads/main" + + id, err := service.LatestCommitID(repositoryURL, referenceName, "", "") + + assert.NoError(t, err) + assert.Equal(t, "68dcaa7bd452494043c64252ab90db0f98ecf8d2", id) } func getCommitHistoryLength(t *testing.T, err error, dir string) int { @@ -137,6 +149,10 @@ func (t *testDownloader) download(_ context.Context, _ string, _ cloneOptions) e return nil } +func (t *testDownloader) latestCommitID(_ context.Context, _ fetchOptions) (string, error) { + return "", nil +} + func Test_cloneRepository_azure(t *testing.T) { tests := []struct { name string diff --git a/api/git/testdata/azure-repo copy.zip b/api/git/testdata/azure-repo copy.zip new file mode 100644 index 0000000000000000000000000000000000000000..d4b53d0b83a53ff87773844da7a48545815a39bf GIT binary patch literal 163 zcmWIWW@Zs#-~hr1?Ug|cNI(e4PRUQsPA$?+&d)8#FHY5~%+2BU*3~+9=KSU$gC|T3 x0p9E!%{N60c!5f}fjGdMkx7IBVH~m?$T(DhX?TD)D;r1+BM>?OX~ds4!6tZ;H);M%W7@Mxqb&&T9lX)mp!=?P& z9nAjE?XWp=>z|F1e*FW2;+Mrp6m|fIVT?@r-_~OP=W(O|C#U~^|1vxr<{N|nm)QT6 zaza3ia?yHG;c(-}6>6xVkc&zjwt(LSO(S#8V))N2>G{82lExoC1#l4h-|5cj|E!es z>rXZWhi#05Sbs;(|DT=G%KFm{;b9n~jq9)R|9Bkkoc))T(#rZr`NOmcIB@>=*d6ZN z`e&nLx8f%^C2VWi_xJXvJl~A!IS_9`cm6ANV32IHlq{) zfE+T-p@Vvq6%^rWH}W#qr)pSj#qeQ$2MwvM>e;J$7&nDepuQ>06)KjoHAUvmIBNmQL0jQ7-tjpsRNshuiVqtLtXY#8v z61YOk*wI|xn9er*-x4S^5I866@(8Hqo$OrC*5Sw_^lQ^iJ>qJ$HTe%jVUh(OuL!@V z{I}VhPJ0XaZ@1;-e>O_8{I8TE&9YcmALWYt2F?z4yAl5{=K28-NL(e?mv7+15|wfw zHuMP+(Z|Jw91C;f0IeXuf+#tNLQpFqwn5|uP)tE5B77bv`HUc@1Hi;#6hw`x{6k@c z&x5kmfW3xMv;h+#^WaC6|Iml30hs(Z$DvaJ@>AVJ)CJKv5=wLI0c0KsWl$GN329Ye z<4h(}Cl*nnvcNZxFNz?GAY7V2JHd>`3lt<2w*_{!flubc{(6`*_Dg3Q;EE)f6B;7X zX0EAT2-C+@KEv!{vWzpO4Gk*T054@WRCP_LJ%|d3<3);ye%3}%sMf?_EItI#2AZLZ z20^Z)g1Rs)OpF$)kP$4_3DKwqKu`dJpN}a5RdD!=7==EXh?12BB^HXJzK8?PhT$SH zUr1EyVH^OM9suarvoai2huzbG%ZR`ReM@C;K|V@fyADtg*5#4ul<775%>uh#Cy=l* z`r8#qJxX(fPXd7ov=-zrIOPfuxJ-&#xl#aF0FnYqKxkn(1@#Dm6q!8PwPWSFbf&wIn)*u-j8~z8WyZcAd3RpHlz|N zXkhKyB@+rXlfw8r8bve&0}PNV1#Of;{c$a6Mfl!LN3^^^4JFzotVt+D_(Fl-3BrJE z2(nBQ67x*aQTl3NnHYjXA!5}u$_YlQQrH;57!HvjE(&Eg9U!Du4A;`HbhUs~O+X1z zU=1MKXtcQulnb#}#?oqWkgkZ+g=rEzOzKO|NF)I-ZG;@19+*-z*<4CVPG?mOJLq0jcDc$tS4(mLjvyx%M%0xfoqmxs!@ux;S&;c zhlY|A4mHguG7FltshO#DyA9w zHGCHzEdeAQ_6l8Ix|LHYi1wFu=~M|4iy$g7UX-$jy6AZawP1NvWH5Hch*c&;7+^Tj zjKrxyV1hvNEHOnUIx-^2A!7Z(I1SB=T>zH_6+`-5HrhI-lsyOI88O^ht$~{wYF8&_ zpCBeOq$(9kYQlQfP0=M9&<)UG10O;5&C9clEgEe#d3jTCOcw)Z!TNbcpt;J*Ihz5$ zP307@KBGmJny|&lz_X1ZKY{w<5Jn*+G;l@nE>rphW8F|dJI)(TdoHk(`K9gKKE`06 z{QylAbtGa);R-bHPbLGJ$^lDLSyfUOaixCGgB={aw4(^c+-K+pD^#PN!AMbQ8)4g}N9o z2b8q>xSnss%^R>lLQN9wtIB*(ui=0hI|kwhgCdC3&BKNjPvRMIm*{j{XKql~FQjS? zG(i;Fp`L+mjDI3*f$~ZR5`j?}APRSF9AC$SE-w;9WFRge%7Q-%ViZV146EP34%y$h zak=XU96+~dnrTOf1A{}ua8N%0s{*@5GZB_3DZ>J#dS6Tgwod3gkvbV98KJ=T0xyR| zl)6Am{CGhD3lSPOwSdSE{1l~*2gD1pQAh_wJBkt2%b)^B_Vu_qR*1L;BTpnq2n&AP zM9a-Y#IPAO;K=wv0D%s`0EpuS*fYXRWYzHD6cGu7qHP4oJ)$hMb0ty|kj#jIw)WTa z;W~l%9>|n%M3st+6s-t%14AvPSXzM*4)x6#8By_Z&y5l2PYz&cJv9;3VHwWQ3!%dm z5LmBNrqcBSY&42|h^o}_s6jb;6(lATe+K-hE5Mz8&#EEpk_fC`;u(+(Hoc zQ=+km7>JXP1`3%+@+Xb`Apz=xCSp4khAO}(-H=4_qn4?uh6l#xQn-s4g2`Y2f{|>1 zw$$AK>N$xL7YCb3pmbb0+B70aNJSji0~SL$efP;y>p)0bt zhE;;lRO>0u8>Hi5xKqKga1xY(IpPl+h$sSL2g?G+4K*86Y#-7(4P(OwNjpcNtme2L zWHX?vL0CdXx-5iv=5R_Fx-f$X^Yjp528@N=M4BgLAy{RiM%)BrG8vw+oRMLc*@|p% z6dsy^ia^=d^ARPnX;jdAhs+U zIF+PVlmMx+5lIbIUsD;)nh=V<(VEdZh5MAfM3T>?9? zU@&gapdn8@#!*=iIC>E?(GNXu1L0V2qCat_D6JVE-k^G~s0oxT{t>SgLo2JQ$SiTr zu(K7IdD6%OK7DlZcqiAaBaNU9ulPlg4bIGdlPTI95z2g$6e2iO%t@6E2*MD7ZYwz; zmI$R6B^nT9*{l;L3VzO3-pP*jDrHFl%MJ`aynM{T%987+uyYC%smwP@VjvytrK(?? z93aDPNwdx71A&D3sEJhPKr;#xBO!?$;|6(ul$!YT{FgWu2pVK!kFpaB`a7pYcrXOO zl$4&vh*0^oUkHH-_uf2??QEiVE?eIaI@7LV_9M zA1~mDGP()SikBHEutvZx=pZn1MIxA&*a^-b<~Jb*s0MwG9v)(%`^1nKCFa9WDFiF4 za5gI#q;YpvR5lo?U4y-lxlM?nHN*e_9>%-eU8}qU;)uvT!GHR0+sUd#FZORSz9q+Kv^YD+7#dl zM{}lWV`{90+!U_K&zbxNM5C2)mme_KwQEhULA`aoWLc6+xC)R%4OC*$L-S;!AqKpe zd311pLk~ZI;9$KQ6m{F6@(gZb>B6$D3+!vyR)*24LQN5nf)wW=b>GGZ8bw8Arnq!b zqXcw}IdLaUWh#CnM+W5lKEC}@`ylX=5R4eQmeD!HX_zq$1JYadh#IY1i--m4KgH3S z>P1*KLec^v5H?5DuZEyXutg0A5S%#b6hiMoUJNk~R#)2u`%hsat~miRzV`9OHJSd) z4XOXo(whAToF8f?`|U%E0m~Hs>v6bK>_5BPkvsp-N;w?yUmUvF!k{mphXl<~U5w_8 zsR0CrCP+zpJip&~uKihlbd0vfbES>u!Y+QFc&=oQL95Y>BGLrr8dzDeh>nf{zm@5! zs??g3SQ8r8bE)B%20aa2GED@lBHo1_tO$-s;iQKJlWkq}1>N;ibtP>OxvlYkP-&3D zXf&Cca7qn)_~2wfruZMLD<%Jf-Qlw3;=i*}fd94Un>roZL-bUnlzeXDWsJK~=(noD zl_e#Ez#z1j>ROc-7$Z?9BZrP)v;)

Z8$!Qev@y%`L#rVLTFBeavSTC5uX876X+7 zb@WI`i3rqP77(J;VU7_$LK8iGlEC&5(Zvf4m?rQi!Yk-^=Q%Z-Q4tHe0ZY_wAYFub z;47N92!;bO23^n$NNC&g7&wX8AfhM0_AyMt1aJ#XPo=aSU_~lnC|D_Hf(f9hPmu;2 zC7PSL!5{(@t^o)Wx*@JQFgC@;c(kW1Nm2D!tBIX?ku4xXs9+qYdLqW-7607=;|?>fz9?;A1P>CSN*OwG8m;yd7LYt9 z>+%Ak!0CsFl~@GLdq66kRbs-VH?>)z+CiZi#^ub^ldbO!q(CCqcmx3`2P}JJDYMy3 zz?-T=)D!^fYH>Mh$y%0Ss8%go-dfm37a2-vJz0p#&G5U8eIKVt;o8Xfx>B9c$cGSW zwi)F54INZnQZjOAsV*;G3nLTM=Xg#Iy-dZ!$$-?d<>0~TTMizKP{Z&cJ{I=Z*Q$$5 zKvZHrF6BA)jF3vuh5D?PU=mx9Kmbum%$>Fq0!0N9iz}dPS4-p3;IXNEGAJzn_^C#& z`(*N!aN5+V*xgh_C_<6ciosA+KpKj$QSMIK5TdIExbyZ)a&eehk62a0J#i6EDp+si z^yPR0Pl#WwYoc|{@JEkO0Vj)Ll0%KQD2YXb#d`7srDu4IdYnR66Q)fxgCyzrs_Xj{+kuBz;1ErbCsgXp659=|){Uz+yxSlSFP4 zCdOyU(K`BM;?-m52w|PxVE9oh3#ge@7Rm8oVRDl=$K+*T7c}$=x)CP;;sTfX?>tk(~MC`HFgoNklhB#s&l#K+VTaKw0p9JY^yA1E?O1 z41icXid9cW(6bu_C=y{_XeppG4f>D;zYz8}bJRb8a~4AA5rmlsM56V=BvOH#fPq>J z1r$A23yBTXEGOI+cOPmz5$I1c)6?n}Kqc@7oe6RSp7KIJ@snVnhN#RlUltV*7{!T_ z2*?9|D{6@{BWE>oHUlS2;Piey6-;Rmi_E1@qlol+BAW=I(Zwy}x^~s)^{Vbk;*~~I zjf;XLg`12tK#F0yB+Y6X$?pt%CERE|Vn{!RKGxuZ{3Kz>$7nsM6jF-`F8!D>7%v6? zk&vfWq#-doR4yPew@o0Ds%B(VCJ+;Hk0PX{q@y($^CfE0I_z<23;BeqjLAAnDc4oy zdkEv9z-t2k$^O+ zFU>rP7zUF;RPEZ5L=SBoG!_P-qiC@|jkAocnQS=(*__T4h-+^OsH(Kd$`i*WhGEj* zz!)G;rZUTd*-Pb|mB<)YUGot@Zm`6{316)ug&LWaZSKs%Pn|9Q?(@tzDjAgUya!YBQr^B0=!J1mbHCxpiwJ`P9-#2_keUdX@O*O zAcok%SYoN12;WDFGY`HDTSh9v6^XQGRq1IGMoVMxS~Ce` zB2kc{9BtNf!4O|ZOd$-#1o{g|RKIBfpxu(&Gt6j$$`XdcG3QtlMmII##xGnLg$-^q zX;0l(3c0JnHqIpQ2=>hakAae)Y#YTuj1MulR$fbkoLFZ?nU?x6uXQVyeUaXxH&AfW zX<1EO(i7eMq;5UH3?uZ(R>(?pa0|JZJcR?A0qfWn7hfscHr2M&JW>VI*g+_J3LBJW zr4a9Cs1N+jbxmTpkck#;UuE0~G3>CHl=Kmzl}!P*SV=XeaZ%Y)Q)8*Ipj?^C;h8h( zNZr~&yOyo+X-SEnbvacdyp!NQ^4Aq}ner|J$ve-2MNo zl(wG#5Q~IGC`4kg9h68uKy^GtC;ia7Rid6Vg`)d!mlu#!1*M1oVnSI>i>lBG)M57$NhNYG zk=~&q2*TTl>`IlM{2$-CYo=nt8G}NbOuESLTsBpY7qGBpDb<6BK-v?H3V;uIaWO9s z<9mukt_$pO4G3~rz=(k(O`o~)`<4vyKP4%|?{WTTwR_-8%K4wu>dDFfY?QXje?|)j znE@tU&X=T&FY&x9YL7;COP}`0LeA7Iv}!lB3Ayu58Wk$AJBrDEZN`{Fyh9`73rENW zrxxe^iPm&$#IXn|iU9QCD(e01l+XmDE?XpS9NzRE!Z2ZD|tIJ8e~@2Ua4$_i}iqoIR_52=I)cHPjv z6}GYhhfbGr$5=G``0-$h2wWj`CQ?6JQG;cyIn$b2%x0*TUtlrd&DRttNa7r3WF$dj zEI?R@$&t@-#wi#@iTAIJn3)H!+oT0Bf|sH5*>w)}tO+;K+!gx`1C3M470@ydH`4&E z1w&a12qAar!j+YA)^xygV8Xy?oelJZnsA|I9wORsSg(UlMI^E$4|up^!jpoKAOp-M zF>C^~_%T?AJyjDEr9pK*mvluw3Ez4uI|#O0L9Pa;&{s!_p<1NWE?v?tZ$5W`He0Lu zUxHt#+vWG%|A7JEO5Ok2a{1q~Qw~)B^IQa@!?g+HiIbLW4e{D}J{O2J_&8vXu!2Do z$^mP`1l6G$jE5Q1Mp(M9CIrNgr|3j|E9pWvpO#JU|;s0@1bNrv3@;l%E zBIJ{$>t9it0}}5S?Kd2lCRRB3>tI@TNE6O0D=KrrZ01krL6EjJ}>hwjX&~qCi`4cRi z6v-d&LL14id3a!v{0CPu@jrUn5AF7MX8)h7#r+?fGpGNvQxg1tu#tokuf z8-CHN>`PsJS>Z0l7JEqQ`BUk!-ovWT$f(d#9!3> zk%({_;mC99E4%2p_2}D-TUPP83KL>{h)%UCOs_Pc>y#~BTresO5_0pGKtPd%5MkVV zNTl~5NakW{W0AYUQ9c~tW%LlDPmqOD;xH6V9M_e`;K@R;T>UJG0r%1c7n3d`u+B97wLX=3Xn9ubE^oP{!Ebycxo&qt^MWXMdKGaU|0=m$|(%u*e=c5Z0 z4_+B^u@br7U8aQxj|}Nh7^9IO@|5S%;x0Y@uD*JO)qVrcLCN< zb=T+RF}0h}x-^|*m_yEFIlZ9)`p3vvdV>ZlvM1m=ZFQSn| z|1rMQCmo{#)cB9^ua4!RJkruVb7@Wg9|qSSDH-&?$7XGj|J&xu+5cH7ZMFY>A^T8AjoaR=FC9wI66t)2V=*tFjCu& z@>9B5PrB#mjHV&avN zosG=bMU41Z*p#G5;UVnnWQ_Tzf~{aO`;?!b=wm81+2dKXH0+d=NK2h$o@RZ~vzR)n zkt;|V)u>+NNzx%C=VqLA)WFF^&%*%=2grO0{)OFQD8Rz}fyk{v% z^i-PS%DI#TdMTSTy0kEjLbXX@Jc}^RIB!m~Vgm^n2Y`c&OGRH2l1$^wc!*;_RZIzP z1X9r#q>5xxp_TQkRILnNlkoh|mQaO04W*?{qJ<7R5PCaY@KTE_72i?JP?s>j(l``(1VFJlEpcNE)HCZ;_J=Q8~^9{IC5qGrPcGF7}z6GiRSc^V!v1Xhs)t;k^jkN z&G~<_QrdX_!{i6A9^jbBAp*%zi*K1kolZ^)Tb`_l;YRYd5MMJnKh-*f@C6X$ZV#~t zct1V3O`IK(`#^ML6rVwmfMLH-q!>`9qVGQ;Qfc<`_(dUXjRzIvwQULTqlx#lMcGMK zA|aG|G)1yHk(WQ9OF_N=hPu0w*6l%ih`$(vY#`#fb48X=7Y|D4uvK9t3Tl_)CgA`u z(wA@X@$(;=i9NuH`$_GD4@nvgd|O!3(UT??4^PZMOOV)GiZ9KEUY)DD8)LJjsFruGqtPH@9Z^pv=bxswBTV9ojkkF9%W};pl3qO>IqX+j3lPQ-ffahJoJQ- zUcveK-8$#U|0reHCvOwO^LeIxEjT22Vrhw-9 z;$@=N$$O=USr6;nh)x~&O8s7b>g-C6VqkQV^!i%j?o6EXY2HS|kfzJm*bH^_&8>9C z6n7;$^{m52Qn!*&bWNm0%L7lU;933^N?AoJi(v#kMo38{`90Mu1Vtg`7%+a7W!o0&W z=Cz5E7l*0*o&mKQQe;FJCs&fcS%c6PFOuwyuW(e~tP{T%f>UHy9npVA_cVOKL?&a3 zai#{E6`XkOa7KXD0DXgy&=AAOdeSB+L^ZMqdx|>^F_oCWvQMCwBtMZB50}-*u~#KW znKe0YB&{*&EP!$`{t`tLQHeu$jm+{1(I$Z{uI9ZQl;J2-lb0IftBFmL zY$9n$V3{zLH%1aU0f^OV zqf%`^h=inO%=;7z`{~WpFnJvg$PB2cx3NiEnklG~D22n@+vNU~}3ktln06$(~N3?*Z7XnYS4 zSdL~E{1qSxqxJp(hj)ueU^so}8OvdaQBexLZ`)BD*3I%qPryqSbzxjpCFTRs6vNCL zCVZ78zH^6N!KG>WBDgq2uQS&oNuQ!uy~Z6A$*F1n4wGp@OcbK~8A03i2!@M9?iW<6 zG0@{YCe>m@u4k@d^%@Y*$x^JYK8XOOpc)Q=JMUWPrk1@o3<#wgr$-^7=SsL<0|yVg zymr`-UcGAZB2I1JUL(~P4kVm%0+lKD#H$7v(Ac;36$9k^{$CZ)(zj?xgETdxw~Dm* zHJ+B(E5HHUgu`?gj1QYCwGs#Qntm(eui9irU9yakh;%1t&0Sw^LrK5?3rTh8N(m~K z!!bst_+Q*FvHhRN<#cgYo5$|5x8q!gWAIGne|-O!Y5l3g@X*FM;Ptm#9S(PH{j*Wp zw*C#g7(Vzh$Q2*w{?{KC9@pRHaNArqyNksCcyjmuvr?>WESxuBwfdY+!EW>Le6hzK zusfXryRF#ibGQN?pSRfI@Cky;%Jn3Ob1+ z+;*!C=)&dTy4dViYaVghw0RhJalp^>9$&!evOB#XJsf_w6VwLp^I8KwYtUOP_=Cj( z`~Kh|U%QjHwtzh^n?;F$WVaR<+dUpFNJn~!e%F$o|M7OEMVLtL{vY}nnfTw~PPzYM z^SGTk{?AIGm)%23e2+xlZ-gJ%@|$pXS=NFoU{5oEMuewkbjmLz?mw2qfh34R9}2I& z)(iIR3i?}N15#;B3AU$iti;eRno9vasZx+!4J4!D_Ax5Rux~Aj#dAaw|4_!f4X5*EbijNu>okK{g=@HVE@~5_Fq;?>+A0gWO4nq z`#*=xnOpyCl-AeZYCl5jZ@1fW>z|F%`uh7lM~we->z|d<`ugK>#i3Kc0j$4j|9foK z-1=vuw7&ix;fV3SBe(w9D6OwQ+3_9n7ze}uc2{owvr#hGfBcc+e``+uXQi~h{=RJP z|8w@=5h$&%zYsh^{vU@sr~k83GOYiRZHonB|L)9H3O9bS*4)iJCX;?c_auhrp5 ziT`ot-v5)OGO>OCH&zcCVs!?J`kOkhdvpCWL#$hlYF8FM_T?iDTCQZFWYqsYFYol& z+(Dn;VFl**`E9(*<96Hpwt&YaIPKPe-Cpc%b&L%B@3cE?De+%UM=t&&J7r?~M@EXX zFL?g6wRL-SzN2@Yy5gd>WB+vg$cm?p$F%=L=~cH{Zg{5sf_aYDI=r}1T)K1SgL}W3 zwsLZ??%4C&N55|D@YTdA!;SLdwR6Xq*IV}e_|=K4s#kp77%aK`={c{=Tb<{6{qak; zZ|ku7xVQS>c5~I~<=+ml^m+QHhwfFjj(hs`^P0a|e9t2L(WY(<>*fg~OIE)%^7A); zY}bDGjI(!ICk-9_`AeslubuPgb-Vt3@|8_L{8b$I${W?4o1Q%(SiXDW`#-#Q){AeQ z(5wD5pKaKmMszcbuM=-FEgt0^Ipy-FuiRZ+`RwqQ_PsOi!lTU{3#B{iT6xNqllOi0_xrmIKJ)dweZPJ5h44|;El>7qDC_cZ^RA9KP|rO_0J+xcTIYs ze8;Mv%3pr%x)VGN&&PZV|1hI=>!^Kec6d)*`R%Q}K76*v#lZ`|e)plWJ~MW-|FH6p z({Fw3#pi$9dG+nL{rjp_KW%Qe;n%J!n&h$@f6kv_o^6|OvfS{_&kHZ_DeigYhDGDT zp`-q~Yve6+nwLMY^v)~aJZFUMjUVPc| zQU46>p7@71zgv6m{B93b4DRy%Xu>MB`?P)DExk{9_>zXd&%N%PcOUxZ#Uama z+qt*)%0266T)gk!AD?pZ%dbpY`{(PM&$#)-Rew2ebI5pFWOr1Tc+k3H> z=l}fLrfPf9V^wzBtgoMN-DQ{&e){ey&wurDaNHHQUK6F+ zpFeZuwi#1nUvB&A*$-apx&70jGZy7{dU)fiFHgSkY4hg2BY%rbdg`2st9-iGKV5k8 zm)0HoeybOJU*1*M@#RnQ-v8G*H!is2J^QD({dn~BxlZagdLe9oX~IQ(rRNw6$s0s0-ikJ>a*Ae;Mn(^O74M?s5C} z%T`P{rew`_#iK~>oa$hT~Sl^$$kDg|86XNH}9Cc7rf^Re-ip5cW^0F7=Mht)~780^gnjJ?VN5Wt$J|w;F|V(mmalo z@!Dl~*1R+A=MBB@9oF&L(tnKJGiLha+XwqD&b#2GiuRAM={No4e>@@Tnil{4rtOOs zzIfK+YiA5^KC%6Vc8@hKofn%qub*oKKO@wsps35JdmlViE;^sTFtq&jzF*wDe*M{{ z>)pSuKHanL`$g;WtCln_|J(MnEnnU8%IA&AN45Lq&u3jW_P#z>b~x(P zvYF*S+IxYE*&%bdeTb~amj-`qog>kWS$*n8uwaWnJvZ;sgY;iX+_`u}>}ZRUqwxb{hFU6HNW(bc%L z_-t8g z^`GrnG@`cR=JMsW!>6peX!lU(3+G)w+Oc%`?p?oj?78j7?Mu3>>|gTWKlq2Y$}g<# zeD~E8M_H$QP_l9Md7;HSuQ+AghS$8ao4xj1E)aTOcjJ>kpW^NCkDqsxeOL78tmy+^ z-*fBLXBg`T&0Vl&;FqV5y|?(*ttap3^X!ts5AF!P`(eo?B@f-0H~x6rInw!4j@#1w z?$JY@d%2=!bA8ue*L{1fe8HWQI_QLr?QVYdvWG6Yen|I~!^gk2bK+^+{(02qduulY zFFxn7`JIjnmiH=sdh){i!bP9x-dL!=>#4tvt32T)`$fCvy*G65-nzLbJ@nW^6VBW{ zK6LwKgM)^C<4!vIoNfLA>$`k5ch@}+t{4`1?9_r?54_ar+?i8uELpSUSlieqFE9`I z#Cp<%=C7uoGidavYv-I**W=hpr+#$rio5IjOgMXa;lAU~`fkWE(e*j_S4S`haF}5to;Rrp}X$?mA`HGIqR>>f2;l23;H~6zH084 z56&}xYkTzBbDsNk_r*;MKm6j;w{DKywaa__f{Nk%gm1^6x%0R$md%~N{ZGo0`L|#A z?ITMZ>vrWo{=jcm!F<8 zwC2K_N8MET+4lL)cdOnUUw89@*tC+~<}Wu)z4()dcYOHWDbM}5<@|NqM%~U&I`{ZB zgTI)#UOaYH<*&2nI9?s{Noo1ZBGM@iAT|uiO=yLeng45=81cF`_ug-3ddxzb9>82@2+ zxopMGVqkP<(CzRQyKEl3br&r530~gowF~Z`GjNFdzoq}*>9OYcKN}^T|9{(2?aD&O zE<4hoBC=R;ZuAt9mb$9~~zu;+gjLi1GJC*-E z_MHDeE2S;=|KM?p-km%2%J1fdN4{$C-f1v}JN{{9yU*&*C_mEB=gOg$jPbu7E6_df z^aXg{@A6sQzF@%X@j8M*zug*i1>IJ`EAZ}C$2f5P@3Q62|FTok>HqDZ|0Bn~a->1a zl`NEu_kVnLuQgy50#2vfEm%E1tB(%`oV>^7ce#SR*XeQ`9~W3Tsg#&@&2#VY6txvu-gQy;1^tezbojn2@bEFw+3x4s|^eQXR)o- zF*5PL-EMEe|894V|Fcmhwtr;pK;3fdsonYay(_<7`Nyg&kJjtQ7Ti5xN&TGLI(6u} zY(tk9rp){H(z#zee${1L9}EOHKYiiH4@9lkb}WAT0rxw{cZ+uBZvAJEYc3quC1jd? z`)BKH)m=6%eCnlFH-2mQ_>QsfE^7!@Rjpco?2`9&zx>&DLZ5f~<$d_~>-KTi{&dll zhYG)Vd-muJZ!DQGcEOqLAK7)3r=sVOBaM;&t&%bSgU#zI_FFwJuZPFXT*Y>q&kK4# z;PX4}0jJyM@_|WkX#0P=+m_1zc6ToRH!B7Be_aLg|FMG!|KB!X-ox!rJI!)ib*GWf zR#m;NyYG`N9X9QKW!LOM&2v}j9$Wk1S;2EAoxeE$BgI)hvhtH}uKuum-2AN%1itFg zKeY1e6Bp ze8IHm&VBO7Q8RW{yKkTO<6o|_zcA>439@Rx7*O#6) z@RL=K#EQkmYjmH^3+=gT{!80e)P7#K_0xOr={Il3r-dJE{>O3plE1weoB8NVvrgW7 zJiqjWzFkIqb8%^t^u+SYU;g^6ZpP}a?Z5x>(pg6uW&c$rWB#|cW!Y?XjEwy6^rW8u zyKK4h|Ev_`|AE!VbVU3Ap6Sb~j=yn>#n$_lvki~+_-J+a84s4dQdRrv=g&7c$2NWY z%Z96Vivv$Hoix?)Wb=sW53HRvde$+0KPWop(esvc>a^hGxf||W7aCA;=8{{BdR@^y zuq5xMA1>%$y{^M)(wX;uHF>~Z!?JHr**oFihFA7nX9<74{I=J8*%BkZ|-?u(tOLtX&23ZqT8Z-2S0bD(UdE>k}J9LUswJQ-d-V< H00sd7No-yj literal 11003 zcmVzUGaSM^p^&+M>T8K2;o|;b62*JcA#w4~mc7Ql=0tv=Q zIbdvzF<%GrL2*9#7(+PZe%`C9{>=2u?rcv${d88-UHv}y-FJWQzWZ(k(hoBArt8S; zsvYF+xE&>*&sQrIL4RlXZ@yUOzp98rIbSZ9N;8#0N#qOVQn_@8sN5E4dP>I+ObQ3kDl-E|uF>fVh3 z_?Rh|w_g8Zq0qzsGo_iCJ4Al(K-juG%h&%x-Sy<9#Dpi?u5Smfx7l#LmKm%|&$nGi zEQ)+$qG2~>%e5qY3%rg@Ow>#dzckFI58s;Zs@ZIMve9W$&x6a}RAynz4txix88ey(Nuf%L}G_}Nkh@xSdf+`Zfc?0)|@Qz%u(@_!U% z(E5Wtw|C0_j+@f|paI6~KZ-JF{VnO&J-Z!%{l6z;4AcLGzWradjP~DH|KI*{-f_&9 zv_!Q3TM*K+0}-s*zQB#2NXFK_;3f=r)%opj-*W9 zovYbS&RYNsWgAWeOc2f=O7XY?^Z2f>;dh&9uXkD>b);@p~sXwB_3 zEr1e!rp_JL$-F{(E_4A)6m)!HHUjAh+xI)RvnotSnC-UbwmsVnr0!DK4S_wFPT=G8 z{J?GF$j}Rq9Us$%KATN<1Lg-+m3O2UNSa5Muy_tCIcBXX*@=REq=hGy+eEunperNd zF(aFRmggOO0%)QEY7z#;dZuLwQ=GL|9cg9UMk51gZrTp@ffZxxaMZ;~(MmoH9d4=i0wWe-f{x~4HNE*5g_WD2@l z1rS8$sxOSoQ_!^`jH!Yl9)DcO`Wm2$pDiuXW5m@toTH}Jx)}h4`oik8n|2)+x*uCU zpcuuSFbuH>-AvPn$;&+eCDp)*3E{h50Q2ZL_EnLo>lsOr%w4`9CN3Q);Qw=0GDUD+ zPfQS~OZODT(e2@wAo?vNZtS~S-CF(wNn|`(Gktk`<$s}2E*E>{e`TgTmj9zD-SU6g zZEt$^>RKSC>nTwLyFHEnpA`=R54d7kJYcp=$7Lx8V#A1t#0bTO*Kx#SfL4%S4U`0TUtf;764I(1+Fl zO8#d<=rlle)=i`?NaMID&9MiNc_5TQUHJ3VYN{YI8LHE1`++CT7Jc!hXM=Fr0PO@b zx*{1Q6t`1F-Js@~PJIpLjQ#R#EivtSLblq$rr21M4nJo3472Na9?p~-rZm_9FS8q} zx((DGqyj>`xNWY0SF2}*3FJD zS;65ib^v{BkdpPJ-)RP@FG9f8FrwY5HEn+l#sPr&03e~BE#jyN?4Ac+B!Lb3mSykM zBud|60#M*2CTKc~Mr_A9RWuTk!piu!N0EB`%~s6?feN(N5E!2FCf zKnVyfET`9L2NBSg49q&hiY*fjJCUG}G-HZJFrY~y2Ee`d<_@G16AKFo{ImTKF3iG1 zc@DLuZ`M&SorDG3khCbEZJU}1{fg6eY83v%+6_wSZJJKnVe`29RwKY%YR&A$#RAR|`RUG^C5nE8t;QUxp%)2wrZ4 z96cDA1wd+m-Fvd-t|PLt;cV0j)`_r**x~2U{(-fEUL9aFkV|0RAdST44y@ z7nY|127=ghJK88k+NjBh?$A(jo#y5WHdrb2AScAdWxYOA*A)xhKvM&3k4Iz8i(530 z%7Eq(*Tvvq>nGN;#Ui8un`AZAx zIT+7~;RCt`Zfe-Bj%FVb6Ah`Q!u&80)Di?s(z? z_dHdk`3>y5cqx_Qet;&X?Vm=*J;4a5@6|us!DYcSNi(|?9f2LojrH-*qJkg zMFwD^<5(cbq6aq;eYFjt2Pj-pD5PreFB!={BMy?$FK@KqL+(Fx=;F~+%PYrD9!(_T zN|`sIW<#!aOwaNM>f<$Y9XD^l0+E^|?W<)zs8`3zV#h%IU{ExWx=q-y;z_(B?)gMQ z91y3Ry6n{)XoC3Mp`k!eho6WoP+lcKA}A^YMB%PYn5!n}@@Z)!0}0FaWIX^e3M64W z`Wx6GTa6pa-8$d^x<%70K0*i#4h_RWeG66vc8ysQmMAG5$y8tK*ud72&a>Ic;CcuJ zwwI>Yv{C8;EtyxO4;CUc-spp<4*V3b;{owPHVSn>+R<)nFM|e-_Vu_q_DNiWk!KSU z99hRrw6{qjM$MpwBjX1N0v&(>5U)tsGr~+fZTJY^M#5mUJq&V>ltp~5BqagKiWshRdLa_H z0#fzLGL_d0u(58NO;)K_P=gA-4ASw)p8-E6rf{c!^!OQdO$1=Yx!69~!-dM&8$jOEg_Ou*YV(LJ<4; zL8on7A^F6h@Ju9sYFuwhs1KS*?Nk)10H3@e3Gic_saeAVV~ZZ{B15nn3_viF4bYa} z4WOQLnYaXOCV|p%<+y2Ex=2Mr9D&7fk^6W>LJ1N>Bz1tGXoP}90S+d6BQagv4y%OF zUh6634eIzD?o@CroCGtlAb;3Hh$0|%uqf zSVAM+lTA}OoFYRPX3$oi9uj82SjbJ(Jd%ZAmDw?I)983Kykfbu!mP3t)!-;RVgpTs zvR^aXeza-y(R+u?lY+H_osrs5q0f-$kVQ>|%ESp7Bz@6DePOC_D#f8d`x6t;#?53h zL;oqFrxrP7>FK|HpRpD`Frg*>tBJz{E9H8g4X6`)ivQ}+Tlh~L>93iK{fo-QzKZdB zH3D(DtEQ@#6We#0BdqTPfe|9Lo=+qWr(p`n;mfp7>lPv`$`2F2pzFF}UjrepSQ32z zigVb4RWga%9HBN1+aZoctru*;b*ima+6ao0#yBthijEXw5th|jlhQcL)GJDWUfGDG z2F5aqV%Z~g*@j+L{e=zyytV}!%fl@3qyq-GXRgY`k-mY?p@MF;J0K50bR!WD+>l&m zY?2pXa1B+c2Ja&P>nIn1v(q9nnqw0Ly9KL9(~ARB><32ep|h-8)gY|)$sePIz}BKL zq0PVj_R!tlvc3OTorT=q^S@G|QtCVZD~#iRM^bir{s-2W-9R^{Nql40+)kh*1QQ9# zi0#h-VY{x60sa0nJ!ss2s<+ECJ@P2`pX%u|yTWWd6mg1q@Y_- zEafb@o^wE+h~m;c1?5svR>~rI9@!%?DULgKfL1p5Jr6svMkBOm(2ysOao|ZQ_$p@9 z51+Sza9qpqPa+d=&2adX_Fl0G)GhuIuZPbrA3sjB#5u#xRx0yk-2{Brc=C8BH=CeF z(8l!Zwyg$NW`8CVY_{cM&2^iELqFS9IfWpck-V)GKrC&h7eBD1=VcS}nyib$(mh42 zx9qt-Sax9W;pJliD@zwpVdvzKR5sUL+ZqJ>g7%Bk0W$2CV%u!fvLf>_7O5^kGsujCl>s>@Y^OB0$@sd&{?!TA&&R6B5jj`W1;IdgvxVE8bI}z#0L&po1VSrfo1U zu@js>%qL2rJG<0VLUMkIY4faBFyV^l(NC5ym zjCZ-g3T{g@=kd086BRL5h0UeC>X3m>pk)&iw9RD_A%X@wk-hGj46aoM9ggrBf6KrE zqPO@kIuEBb<>QH~II+BP_{53DWt?=z63Gi9^Tef#`J{MUY}7@jo@5e{S6;}0Kvg}w|u>A&^&{iSYBAQb%A}2 z+R7+e_1P2wDabf)>V2DOt=qn4rck=*D1n4xPTUEzOvP_>WI*Tl;r2`SLGV%tMh9KX z=o}I;W*oqPe5)Q&6L)KoSYZDtj<&I8!?F=cOGF@Sj@Yk;pt`U{bu0uYM4eCc-Z1T^ z;$Xen8Q6cyYq;hFWVrU>;$|}6u7On%c42@OAxH{$J)F8; zIA~1WXWU54??B{k&HoL*hZ zw1@aqWPv_!m>$O6`26ko>E-$PQ(zDl7ZTgb3yhIi^^ij!R@4L4r`Lj@?a$|OU~@~@ zIb1<3_$7cVg2fWV)&>7(D> z6gr#Hv`uu;@JD^7z5@I?y<6S^T+9T=PTdKm5Lxo)73wKmk57i|F%LLcKe z^@*5?W30mBoIHPGx1#K7M|ztf>l>k z`4yh|3e6l3x$naR<%RB0SlrQAvesE$<<;*#`JKuO-RXcn=`a{~n4$V&+Q0}NBtlIY z2|A6|{WucI<9cGkvZXN2JAS8)<~<;lXBADDdehC4x`SjE#uZuiWE=MbDNy8^i68*w zfMt&?mCa@eyc-Eb%@m++1($P$)^ZU;wN}*fuD~{WI<>&{JQ;YK@Owdhf6RBCmG+g@ zg@jx;n+Ubq42lQOo;o={|H#>e#6(yNBNOya}o|g6kOF#A{&a2(ixnVED0>1=Or6+jKmb>~0cr%)}6OK|^oqUUC8;E`S+2o)a-a zW(mY`W(g6JS+XRaf=`FAr~2neRH)~O2gZxEw8iO*2bLc=add?ctUezLRvCUNWTlh8*h zwgGkp9!O&T!)-nBZkLhjo%EGMUDGS99pn6Cv06Jk5 zt3e|e>P7*IM3@&^3g}FOK0H~MPJL6be*ot!o9GdQnOin#J%=I{=mZSZV$<>stk$$! zY?c$Zh3>BCcM=4(E0&83pBt6`e<^U;$ z>G=VxnWoR&u&`&Ogd|WU!S**wS;2(>o@s+BgaJUK2g43mEYV}PFAP?m+VmzG4HWQ?jF z`v@Snavdk~)%tF8U0K=L11d}o1HCXRQpb%cxN=Gi&={GFjsV?4Su%AEQkK2il7g># z+HI*ZcVJ|8X-$Av3Dm`%FAg-?2GPBQ#&vh4)2RiL&29{_o3X@tIT5};9AesrU7Ok5 zFgJb0B(Z&*>nIPt2U|uK;l4}lRaHJs!f0s>UfZNVCW?Zzy$f-n+%4UnQAQI~b}{>*ba0ZTQ zkIS(8pLJ=k-|7X>VfX*@<-Ys>)zbL>|47PC&wt1wQ4tF54r~Wq$_I!}S8}=PW)uy- zNq4p-U7PQ!&Aw2Is9=Sb#EK|q7nA5SCEJu^KXg;PcZqvE+Sl=qM!Zi{umG2JGr>KI zyj64$AF()Kpkajj>GNkFSUGk2-0@QWA-E*@nn8gRR?5 zSgux32K=uh7ccfzg-)Q3x|c|m=w2eb+8Do-ohsMJfj?e|CzVrU5HE)fCijZOipbyvA_qTgO6Vi#vI662g)(dV* z74>i$XM;JX+kqyiH#>Zx3lE3X8u{e@R2x#fslfN1HA0O^ak?o{qezEty7VX3$z~0) zCY$a4;0#~ZUA%=54J@@A7O%O)qWmbhPX%jaJOe{;o$kCe>8^ATq{!)xI<+yhpNnow zP;iFY{NvKA>;yO(*b}(rt;2?&7bKH`k?~MgZH_T~5$|1uv=loL284T7NZY1m`8!Fd zlV%#N)8VxrpmP9kby+swY19`9Jy^UZp0=*KF6Y>R_l;-DV(wZM+OUsjhSAvnJd`i$}K% z0}UzVQD`}Vn`wYHhoLM}M92pZip52d9|XJr69z^*km3(I;X;2NB5ru@*iEM*vQ?4? zJlx6PNkLP30CUE6GJuvk2J5J&Ix*1zR8NYoEAn0VHduBLwzolU45!f7-Hf3Xq|}24 z2QKfV*rm<3jsADx7k0Yb-u)jK0F~bOzx+7=+i1#e^}i|F7#$u@7*9@G+8W}u^GRWK zS~UUe;rJLd;qO`-Ms$a2Fbp&1M;TvwFiZyDQ8M0~+#h=4|P{VU)cka)jn%Wz;$tZ?(!!Qu=tAiOt<($z^4dhb$V&~8HCfK|b} zTh`rC1n;et?fE~7XTF{B|1*_7{x8pr-~Tp}vX}gi;i{Yezuo}231*LGGJw_jMW)d6 z4kGy@mhOt=54(s*^2Z+TE|UM|%J%$^e)e0v{XN|OSLwU|Qz(t~|7c3Y|2ND2bV!5l zkZ#qFf!gqkq1l&ReRPcoV_eqQTNOy{OrwZICdih!Dj_G|De9$FFQ+s z%13{V81zy(72`M3_|i6JGQ|}&9zK5B5Yu#>*Tx*P?ka{vVy-hDpL3LUye51o8i@qb z(4f2a889!QgG13>|Htv7a#H8cONwMZnMx%1co6fq2%7l{rsCvJc$1w6rHASxJWb#! zVU(~jtj*~f5uzxu;-q*0(BIVAS>VZhm;y2CBI>)V58Vk~K$jaT?M<3EA6=k$@T!oD zmFRl+VjLbkGUTB!R7+o7;q0RnHq+A~UXU5kVFX<^+ub`}RK~vf(gu1LVEweaequtY z-ALGKV7iHZ z-eaJ$Bg7^^N{ZAC<}krTfWBmXLU#oErmFR-zB;T=xu~Cbei3nG6XbHk#9K%3`=*N3 ztkEfH3>KE%(Dx8Xs2)YA4+i6wfS}umo5O)Hadb-E2V;j<7}>U?9GqZVWMhL;ceo$h zZtkopb~^nqyW-xV0GNK0Gl=d86joW70hMMv2qx&!RkK;mVh1q$Xc zeKffeibvQ&0UiN`$3_aU34bmby5U)~A%jgevN;J2ojfW3k|8cFh*(Z-<0#<+)Ah#< z>M*RR9s?t*!icz`n2se%;WUt(r7MWrbgN&nQn;Foxkh94!JJbk z*s+Fma&7d#%-A>p7P^5JAk)Hu0|&%uZq^l0LaGEzeHIXs_`aJg;wk=#IQT{}%-iRa zUPXS$(r5u>vswIui3XDxY@n=3%z4f^RZUcX*<0Oes8my$^g}a0VgmkYg;q#y7gTEW z*bizs%^*XV$OL*isrdU6GzX^jfYg`+6{D9%9!M)+7a8##C)1@!;UVnnJdF8g!8UbW zeVUw%`nbU+dzeKl#!g*{wDcq=V(W{Z#okfVVybJ@w7$sGr9&v^W=J|Za5C!oJYa!t zzoRY$sbs>MWPAZ@fn-hUIi5ZWeU#=Z8qrZ-@!bN2WXAEv%fon6RKggGqax(Vp@=xD zH76_YUCek}D2bsF#NKs{LmBdZZ{D+>Bzl^r#FFSqpx3iGJioOi)X)#0wBAYd z>7d=v+g`!T7FRF61I$ntnO}n&3}U9fc6%|`o?dTh(gzsc!`cB#*LS9EU*~L#KfgDX z=n_usiQxe7j)SS?9MVtdEekV@pwfw$4n3Gyqg$LabRlp(itm1v-uS!6$1yHjmu=2} zI$(_iE@$*}JGU?Xqf#pM=6@7yyi>O?PpK$n8P0f)MKnd=_m9>N!6&;}yRoolOv zx_D5+!yb410MstyCUO9z`Q=+?c>cqg*ezjik=h9#QW_0>TUgi8Q!Ew_Ps~6|Q0y(^ zE4HE6bJe>sHCtA#3yi*+tB%079-lL{>*#m&8awVJ&7KxKOybF_>+FF#>jFK~+|(yb zf!&U#qTX%NO+0+UNLay_Y^+Pqvn{Ev186@}O|F^iw%hR*@T`d58^Z1jOTR+Zlk|=j zJ|BxUj+$aj0q6QMJyPrRUMVu`VV&2}sRLix?^S1KS9cVHqLZ%I*OI$4I_HbMjYc6Y zF&VQNR{71XJY&Y)d_tdf6ex8og`&qICHfCMS;4FPD@<9_n#C}J9wVe&N`8;|#W=%0 znaHu#b-ljzPh}H$WqLZ*{Q|kE13hnxJ4FY*2qfJXL_!Q-tkd<%n}PJD-Kk-&n3|^& zbF_3L%scF1UYmek9H#kw2x?=bsE9BmSIXZULulWNB=5&pIO;d+g!e*lit4H(`pANN}cUC2EszuDqtA^F*dy_Fwk!_fnMk?26PUPz+A}>pj5uPu1UWU?L_WQ zI^e{YA?eg9e8Q{OSWTmAjj1yW%=i%A?HCgWj`n6%*UgRhvNm`*Y ztQ1Fg{53eCLjo{PRMM8LwlJxMnnrh9YK6d1-a^WzZMFmT>M9hhm>9~#? zD)`HygwaOb5_q?W0>k;8XDWvwMn(C2-}df!Shva_eHAZRtU9==ey0Yase_p}GWaS> zeCLk3g3D?7+PFBR*OhCL(x({OYkYTQ@F3KvoC7gVbq(Bmd1 z)lx)mC|9w*24o7)?X0eK5un@9;Sjj_}TDaGhj0}5#DJN}9R`o8t60{VT6sR2mS8ND^q!fQPJ*qg!u zcZ9=17z~H)m0Hn3eXQT*@Ku}2sOy$7E=hMn>-hTe4$7eYU(;PhM@qxb?adg&;(u|! zr1pQ+O1UEPg=(QtxkFU;X87&PvwZ)z{ra=J@K(pz_4*g#zruL^M^SdN{w>pXZe`2PP$N`8k$%vt$-tz4GHLe(^9t3|6=E?dRIY`IpdSk>CxY^hX} zvXU1^36bJR$Lh#+aftDnt=(5|c4`}@C$n`o>%3w~6lQ?Qfi5Z~aj;O#=O@T%v)}l5 zwv|Erk9R9`@*3Uz--|JZ@qcNi=l)NjS}BbAetU@eN-o&P^yD)pTImB#OX7*V-)=Zk;uqo>Z~%hvRZGY6jf{ilE9O#VyX zcgLc0*T3Cw(8gs1Wr+SS&(^B>Vq*@tzfqej6^e89nTA!ZH8<&^Q+%{;x9k;rZ@i5jo-QJj;|g0vAO*Qe_U>{4B7t`E0w}* zdA2@NE;nXMwb@FcYF2AjW40#e%(=OuoN1J;Tju{h{STi?WBwmS8KnRJ4D^5Vt`F=t zXyY=1GKBxDjVc)bv$8Q`TGg4lIR_u=rP)ShrqZYt@})VeRF~BuV+_~-Grjy@sgC1+ zMpAa7|4%>mFF*C>vp@MipL8DiU~29cQW@v&pa1Y3UtIl>rTvC}TyC`t@&8xzK=o$1 zW|?NaQp?ZO8rEEOuGDDMi}^;SF_V{b(wrGG#_sihB|nb;8%-Ic|GxtIzkS#5?l)-T zGJ-PX{J&P5%UgMAmCG|TGGDFbYi7eLo7GCaQfZiT<#!c=2$NWE%vJ?IP zg#(X#=VRac!#A(J_uJKj&wOY4@o#^m>b~HP|8w8--@f1AkIOBVA@P6Zd=d1&RV>K7 ztjkKhUTIVcvNTsT^Nj*%X)plFvxOmJ4CDV|vDm-=FO2zr6y@5T@A&vbi4Wy}_&)QG z=05zj5C81(pL~vCT%P*H6Yp7j;~VaI{-F%Wtz9U-(xq z{L{Av`B&XN`^mS?{K0eY4GxIc|JNgb|AmhoY-V2fb6GG#O&}tk%{*kBede0vvzV+V1_rK^59-R2hfB5O|h*$l^-+la_CBOQ~*IoF;@4e^h z<#+z*o$vU@_f-!cJ+t2!d3Kc{@!y5H%4|Jft;|(Ty#6&?EY#*e?_0HcxoDMV3Y8j| z1h=~XFU}Nt`M*#cpZ|@d0RKOA82SIM)5QO8IPv5^zw>#|%e~>`J&*k6@#CLN{L<&X z^!(3$`**+bx>K8P{z&3qeEe-c*7#em_}jlW`6qsP>5=8nJ@ZqaS$gd4Uw*6g-;cbw z`Qfkqz&jJx!app0^OgVTCw}CL@xtHw(jUI<7hYoj%4`1Y-4B20*Z<=6Kk(VPcU=9c zKl$;Oe(no@=ZW8X!Mnfy@}K$3lQTc}$ z)i1wk{e79|yyb`7x4-Mix%G*=&cFJ|>DRpGEl(er`{tLvJ(v97PyF)ilh^;p%b)w! z-}va$cfI{oVY2w~r@YTBeDTQ1=|`Skc-}*w`^Y;wv-YojEb;j#o8Nrp+u#3{f4}md zSHJxEfAW?GpZss1Pk#Cf|LLzA^Z)LBomc9OLwEkK|MHU8?l;QDWn9K(T)rpD{{zE1Q|JKf007i0*Tw(< diff --git a/api/git/types/types.go b/api/git/types/types.go index 055222700..d24283b52 100644 --- a/api/git/types/types.go +++ b/api/git/types/types.go @@ -1,10 +1,20 @@ package gittypes +// RepoConfig represents a configuration for a repo type RepoConfig struct { // The repo url - URL string `example:"https://github.com/portainer/portainer-ee.git"` + URL string `example:"https://github.com/portainer/portainer.git"` // The reference name ReferenceName string `example:"refs/heads/branch_name"` // Path to where the config file is in this url/refName ConfigFilePath string `example:"docker-compose.yml"` + // Git credentials + Authentication *GitAuthentication + // Repository hash + ConfigHash string `example:"bc4c183d756879ea4d173315338110b31004b8e0"` +} + +type GitAuthentication struct { + Username string + Password string } diff --git a/api/go.mod b/api/go.mod index 4859e6120..f0e916dcb 100644 --- a/api/go.mod +++ b/api/go.mod @@ -27,10 +27,11 @@ require ( github.com/mitchellh/mapstructure v1.1.2 // indirect github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6 github.com/pkg/errors v0.9.1 - github.com/portainer/docker-compose-wrapper v0.0.0-20210527221011-0a1418224b92 + github.com/portainer/docker-compose-wrapper v0.0.0-20210810234209-d01bc85eb481 github.com/portainer/libcompose v0.5.3 github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a github.com/portainer/libhttp v0.0.0-20190806161843-ba068f58be33 + github.com/robfig/cron/v3 v3.0.1 github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.0 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 diff --git a/api/go.sum b/api/go.sum index e966dc90a..8906e08cf 100644 --- a/api/go.sum +++ b/api/go.sum @@ -241,8 +241,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/portainer/docker-compose-wrapper v0.0.0-20210527221011-0a1418224b92 h1:Hh7SHCf3SJblVywU0TTn5lpTKsH5W23LAKH5sqWggig= -github.com/portainer/docker-compose-wrapper v0.0.0-20210527221011-0a1418224b92/go.mod h1:PF2O2O4UNYWdtPcp6n/mIKpKk+f1jhFTezS8txbf+XM= +github.com/portainer/docker-compose-wrapper v0.0.0-20210810234209-d01bc85eb481 h1:5c8N9Gh21Ja/9EIpfyHFmQvTCKgOjnRhosmo0ZshkFk= +github.com/portainer/docker-compose-wrapper v0.0.0-20210810234209-d01bc85eb481/go.mod h1:WxDlJWZxCnicdLCPnLNEv7/gRhjeIVuCGmsv+iOPH3c= github.com/portainer/libcompose v0.5.3 h1:tE4WcPuGvo+NKeDkDWpwNavNLZ5GHIJ4RvuZXsI9uI8= github.com/portainer/libcompose v0.5.3/go.mod h1:7SKd/ho69rRKHDFSDUwkbMcol2TMKU5OslDsajr8Ro8= github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a h1:qY8TbocN75n5PDl16o0uVr5MevtM5IhdwSelXEd4nFM= @@ -263,6 +263,8 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= diff --git a/api/http/handler/stacks/autoupdate.go b/api/http/handler/stacks/autoupdate.go new file mode 100644 index 000000000..867e71fc2 --- /dev/null +++ b/api/http/handler/stacks/autoupdate.go @@ -0,0 +1,38 @@ +package stacks + +import ( + "log" + "net/http" + "time" + + httperror "github.com/portainer/libhttp/error" + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/scheduler" + "github.com/portainer/portainer/api/stacks" +) + +func startAutoupdate(stackID portainer.StackID, interval string, scheduler *scheduler.Scheduler, stackDeployer stacks.StackDeployer, datastore portainer.DataStore, gitService portainer.GitService) (jobID string, e *httperror.HandlerError) { + d, err := time.ParseDuration(interval) + if err != nil { + return "", &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Unable to parse stack's auto update interval", Err: err} + } + + jobID = scheduler.StartJobEvery(d, func() { + if err := stacks.RedeployWhenChanged(stackID, stackDeployer, datastore, gitService); err != nil { + log.Printf("[ERROR] [http,stacks] [message: failed redeploying] [err: %s]\n", err) + } + }) + + return jobID, nil +} + +func stopAutoupdate(stackID portainer.StackID, jobID string, scheduler scheduler.Scheduler) { + if jobID == "" { + return + } + + if err := scheduler.StopJob(jobID); err != nil { + log.Printf("[WARN] could not stop the job for the stack %v", stackID) + } + +} diff --git a/api/http/handler/stacks/create_compose_stack.go b/api/http/handler/stacks/create_compose_stack.go index f66c2572d..4ca76967f 100644 --- a/api/http/handler/stacks/create_compose_stack.go +++ b/api/http/handler/stacks/create_compose_stack.go @@ -1,7 +1,6 @@ package stacks import ( - "errors" "fmt" "net/http" "path" @@ -9,10 +8,12 @@ import ( "time" "github.com/asaskevich/govalidator" + "github.com/pkg/errors" httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/filesystem" + gittypes "github.com/portainer/portainer/api/git/types" "github.com/portainer/portainer/api/http/security" ) @@ -100,7 +101,6 @@ func (handler *Handler) createComposeStackFromFileContent(w http.ResponseWriter, type composeStackFromGitRepositoryPayload struct { // Name of the stack Name string `example:"myStack" validate:"required"` - // URL of a Git repository hosting the Stack file RepositoryURL string `example:"https://github.com/openfaas/faas" validate:"required"` // Reference name of a Git repository hosting the Stack file @@ -112,8 +112,11 @@ type composeStackFromGitRepositoryPayload struct { // Password used in basic authentication. Required when RepositoryAuthentication is true. RepositoryPassword string `example:"myGitPassword"` // Path to the Stack file inside the Git repository - ComposeFilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"` - + ComposeFile string `example:"docker-compose.yml" default:"docker-compose.yml"` + // Applicable when deploying with multiple stack files + AdditionalFiles []string `example:"[nz.compose.yml, uat.compose.yml]"` + // Optional auto update configuration + AutoUpdate *portainer.StackAutoUpdate // A list of environment variables used during stack deployment Env []portainer.Pair } @@ -122,14 +125,18 @@ func (payload *composeStackFromGitRepositoryPayload) Validate(r *http.Request) e if govalidator.IsNull(payload.Name) { return errors.New("Invalid stack name") } - if govalidator.IsNull(payload.RepositoryURL) || !govalidator.IsURL(payload.RepositoryURL) { return errors.New("Invalid repository URL. Must correspond to a valid URL format") } - if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) { - return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled") + if govalidator.IsNull(payload.RepositoryReferenceName) { + payload.RepositoryReferenceName = defaultGitReferenceName + } + if payload.RepositoryAuthentication && govalidator.IsNull(payload.RepositoryPassword) { + return errors.New("Invalid repository credentials. Password must be specified when authentication is enabled") + } + if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil { + return err } - return nil } @@ -141,42 +148,72 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite } payload.Name = handler.ComposeStackManager.NormalizeStackName(payload.Name) - if payload.ComposeFilePathInRepository == "" { - payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName + if payload.ComposeFile == "" { + payload.ComposeFile = filesystem.ComposeFileDefaultName } isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, false) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err} } if !isUnique { - errorMessage := fmt.Sprintf("A stack with the name '%s' already exists", payload.Name) - return &httperror.HandlerError{http.StatusConflict, errorMessage, errors.New(errorMessage)} + return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.Name), Err: errStackAlreadyExists} + } + + //make sure the webhook ID is unique + if payload.AutoUpdate != nil && payload.AutoUpdate.Webhook != "" { + isUnique, err := handler.checkUniqueWebhookID(payload.AutoUpdate.Webhook) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for webhook ID collision", Err: err} + } + if !isUnique { + return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("Webhook ID: %s already exists", payload.AutoUpdate.Webhook), Err: errWebhookIDAlreadyExists} + } } stackID := handler.DataStore.Stack().GetNextIdentifier() stack := &portainer.Stack{ - ID: portainer.StackID(stackID), - Name: payload.Name, - Type: portainer.DockerComposeStack, - EndpointID: endpoint.ID, - EntryPoint: payload.ComposeFilePathInRepository, - Env: payload.Env, + ID: portainer.StackID(stackID), + Name: payload.Name, + Type: portainer.DockerComposeStack, + EndpointID: endpoint.ID, + EntryPoint: payload.ComposeFile, + AdditionalFiles: payload.AdditionalFiles, + AutoUpdate: payload.AutoUpdate, + Env: payload.Env, + GitConfig: &gittypes.RepoConfig{ + URL: payload.RepositoryURL, + ReferenceName: payload.RepositoryReferenceName, + ConfigFilePath: payload.ComposeFile, + }, Status: portainer.StackStatusActive, CreationDate: time.Now().Unix(), } + if payload.RepositoryAuthentication { + stack.GitConfig.Authentication = &gittypes.GitAuthentication{ + Username: payload.RepositoryUsername, + Password: payload.RepositoryPassword, + } + } + projectPath := handler.FileService.GetStackProjectPath(strconv.Itoa(int(stack.ID))) stack.ProjectPath = projectPath doCleanUp := true defer handler.cleanUp(stack, &doCleanUp) - err = handler.cloneAndSaveConfig(stack, projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, payload.ComposeFilePathInRepository, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) + err = handler.clone(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err} } + commitId, err := handler.latestCommitID(payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to fetch git repository id", Err: err} + } + stack.GitConfig.ConfigHash = commitId + config, configErr := handler.createComposeDeployConfig(r, stack, endpoint) if configErr != nil { return configErr @@ -187,6 +224,15 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err} } + if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" { + jobID, e := startAutoupdate(stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService) + if e != nil { + return e + } + + stack.AutoUpdate.JobID = jobID + } + stack.CreatedBy = config.user.Username err = handler.DataStore.Stack().CreateStack(stack) @@ -331,7 +377,7 @@ func (handler *Handler) createComposeDeployConfig(r *http.Request, stack *portai func (handler *Handler) deployComposeStack(config *composeStackDeploymentConfig) error { isAdminOrEndpointAdmin, err := handler.userIsAdminOrEndpointAdmin(config.user, config.endpoint.ID) if err != nil { - return err + return errors.Wrap(err, "failed to check user priviliges deploying a stack") } securitySettings := &config.endpoint.SecuritySettings @@ -344,15 +390,17 @@ func (handler *Handler) deployComposeStack(config *composeStackDeploymentConfig) !securitySettings.AllowContainerCapabilitiesForRegularUsers) && !isAdminOrEndpointAdmin { - composeFilePath := path.Join(config.stack.ProjectPath, config.stack.EntryPoint) - stackContent, err := handler.FileService.GetFileContent(composeFilePath) - if err != nil { - return err - } + for _, file := range append([]string{config.stack.EntryPoint}, config.stack.AdditionalFiles...) { + path := path.Join(config.stack.ProjectPath, file) + stackContent, err := handler.FileService.GetFileContent(path) + if err != nil { + return errors.Wrapf(err, "failed to get stack file content `%q`", path) + } - err = handler.isValidStackFile(stackContent, securitySettings) - if err != nil { - return err + err = handler.isValidStackFile(stackContent, securitySettings) + if err != nil { + return errors.Wrap(err, "compose file is invalid") + } } } @@ -363,7 +411,7 @@ func (handler *Handler) deployComposeStack(config *composeStackDeploymentConfig) err = handler.ComposeStackManager.Up(config.stack, config.endpoint) if err != nil { - return err + return errors.Wrap(err, "failed to start up the stack") } return handler.SwarmStackManager.Logout(config.endpoint) diff --git a/api/http/handler/stacks/create_kubernetes_stack_test.go b/api/http/handler/stacks/create_kubernetes_stack_test.go index f1b47286e..2bcd35ab5 100644 --- a/api/http/handler/stacks/create_kubernetes_stack_test.go +++ b/api/http/handler/stacks/create_kubernetes_stack_test.go @@ -23,6 +23,10 @@ func (g *git) ClonePrivateRepositoryWithBasicAuth(repositoryURL, referenceName s return g.ClonePublicRepository(repositoryURL, referenceName, destination) } +func (g *git) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) { + return "", nil +} + func TestCloneAndConvertGitRepoFile(t *testing.T) { dir, err := os.MkdirTemp("", "kube-create-stack") assert.NoError(t, err, "failed to create a tmp dir") diff --git a/api/http/handler/stacks/create_swarm_stack.go b/api/http/handler/stacks/create_swarm_stack.go index 99558b817..e76338384 100644 --- a/api/http/handler/stacks/create_swarm_stack.go +++ b/api/http/handler/stacks/create_swarm_stack.go @@ -13,6 +13,7 @@ import ( "github.com/portainer/libhttp/request" portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/filesystem" + gittypes "github.com/portainer/portainer/api/git/types" "github.com/portainer/portainer/api/http/security" ) @@ -121,7 +122,11 @@ type swarmStackFromGitRepositoryPayload struct { // Password used in basic authentication. Required when RepositoryAuthentication is true. RepositoryPassword string `example:"myGitPassword"` // Path to the Stack file inside the Git repository - ComposeFilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"` + ComposeFile string `example:"docker-compose.yml" default:"docker-compose.yml"` + // Applicable when deploying with multiple stack files + AdditionalFiles []string `example:"[nz.compose.yml, uat.compose.yml]"` + // Optional auto update configuration + AutoUpdate *portainer.StackAutoUpdate } func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) error { @@ -134,11 +139,14 @@ func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) err if govalidator.IsNull(payload.RepositoryURL) || !govalidator.IsURL(payload.RepositoryURL) { return errors.New("Invalid repository URL. Must correspond to a valid URL format") } - if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) { - return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled") + if govalidator.IsNull(payload.RepositoryReferenceName) { + payload.RepositoryReferenceName = defaultGitReferenceName } - if govalidator.IsNull(payload.ComposeFilePathInRepository) { - payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName + if payload.RepositoryAuthentication && govalidator.IsNull(payload.RepositoryPassword) { + return errors.New("Invalid repository credentials. Password must be specified when authentication is enabled") + } + if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil { + return err } return nil } @@ -147,44 +155,74 @@ func (handler *Handler) createSwarmStackFromGitRepository(w http.ResponseWriter, var payload swarmStackFromGitRepositoryPayload err := request.DecodeAndValidateJSONPayload(r, &payload) if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err} } payload.Name = handler.SwarmStackManager.NormalizeStackName(payload.Name) isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, true) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err} } if !isUnique { - errorMessage := fmt.Sprintf("A stack with the name '%s' is already running", payload.Name) - return &httperror.HandlerError{http.StatusConflict, errorMessage, errors.New(errorMessage)} + return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.Name), Err: errStackAlreadyExists} + } + + //make sure the webhook ID is unique + if payload.AutoUpdate != nil && payload.AutoUpdate.Webhook != "" { + isUnique, err := handler.checkUniqueWebhookID(payload.AutoUpdate.Webhook) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for webhook ID collision", Err: err} + } + if !isUnique { + return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("Webhook ID: %s already exists", payload.AutoUpdate.Webhook), Err: errWebhookIDAlreadyExists} + } } stackID := handler.DataStore.Stack().GetNextIdentifier() stack := &portainer.Stack{ - ID: portainer.StackID(stackID), - Name: payload.Name, - Type: portainer.DockerSwarmStack, - SwarmID: payload.SwarmID, - EndpointID: endpoint.ID, - EntryPoint: payload.ComposeFilePathInRepository, + ID: portainer.StackID(stackID), + Name: payload.Name, + Type: portainer.DockerSwarmStack, + SwarmID: payload.SwarmID, + EndpointID: endpoint.ID, + EntryPoint: payload.ComposeFile, + AdditionalFiles: payload.AdditionalFiles, + AutoUpdate: payload.AutoUpdate, + GitConfig: &gittypes.RepoConfig{ + URL: payload.RepositoryURL, + ReferenceName: payload.RepositoryReferenceName, + ConfigFilePath: payload.ComposeFile, + }, Env: payload.Env, Status: portainer.StackStatusActive, CreationDate: time.Now().Unix(), } + if payload.RepositoryAuthentication { + stack.GitConfig.Authentication = &gittypes.GitAuthentication{ + Username: payload.RepositoryUsername, + Password: payload.RepositoryPassword, + } + } + projectPath := handler.FileService.GetStackProjectPath(strconv.Itoa(int(stack.ID))) stack.ProjectPath = projectPath doCleanUp := true defer handler.cleanUp(stack, &doCleanUp) - err = handler.cloneAndSaveConfig(stack, projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, payload.ComposeFilePathInRepository, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) + err = handler.clone(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err} } + commitId, err := handler.latestCommitID(payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to fetch git repository id", Err: err} + } + stack.GitConfig.ConfigHash = commitId + config, configErr := handler.createSwarmDeployConfig(r, stack, endpoint, false) if configErr != nil { return configErr @@ -192,14 +230,23 @@ func (handler *Handler) createSwarmStackFromGitRepository(w http.ResponseWriter, err = handler.deploySwarmStack(config) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err} + } + + if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" { + jobID, e := startAutoupdate(stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService) + if e != nil { + return e + } + + stack.AutoUpdate.JobID = jobID } stack.CreatedBy = config.user.Username err = handler.DataStore.Stack().CreateStack(stack) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack inside the database", Err: err} } doCleanUp = false @@ -350,16 +397,17 @@ func (handler *Handler) deploySwarmStack(config *swarmStackDeploymentConfig) err settings := &config.endpoint.SecuritySettings if !settings.AllowBindMountsForRegularUsers && !isAdminOrEndpointAdmin { - composeFilePath := path.Join(config.stack.ProjectPath, config.stack.EntryPoint) + for _, file := range append([]string{config.stack.EntryPoint}, config.stack.AdditionalFiles...) { + path := path.Join(config.stack.ProjectPath, file) + stackContent, err := handler.FileService.GetFileContent(path) + if err != nil { + return err + } - stackContent, err := handler.FileService.GetFileContent(composeFilePath) - if err != nil { - return err - } - - err = handler.isValidStackFile(stackContent, settings) - if err != nil { - return err + err = handler.isValidStackFile(stackContent, settings) + if err != nil { + return err + } } } diff --git a/api/http/handler/stacks/handler.go b/api/http/handler/stacks/handler.go index dcdfce41b..f5ba2c983 100644 --- a/api/http/handler/stacks/handler.go +++ b/api/http/handler/stacks/handler.go @@ -2,23 +2,30 @@ package stacks import ( "context" - "errors" + "fmt" "net/http" "strings" "sync" "github.com/docker/docker/api/types" "github.com/gorilla/mux" + "github.com/pkg/errors" httperror "github.com/portainer/libhttp/error" portainer "github.com/portainer/portainer/api" + bolterrors "github.com/portainer/portainer/api/bolt/errors" "github.com/portainer/portainer/api/docker" "github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/internal/authorization" + "github.com/portainer/portainer/api/scheduler" + "github.com/portainer/portainer/api/stacks" ) +const defaultGitReferenceName = "refs/heads/master" + var ( - errStackAlreadyExists = errors.New("A stack already exists with this name") - errStackNotExternal = errors.New("Not an external stack") + errStackAlreadyExists = errors.New("A stack already exists with this name") + errWebhookIDAlreadyExists = errors.New("A webhook ID already exists") + errStackNotExternal = errors.New("Not an external stack") ) // Handler is the HTTP handler used to handle stack operations. @@ -34,6 +41,8 @@ type Handler struct { SwarmStackManager portainer.SwarmStackManager ComposeStackManager portainer.ComposeStackManager KubernetesDeployer portainer.KubernetesDeployer + Scheduler *scheduler.Scheduler + StackDeployer stacks.StackDeployer } // NewHandler creates a handler to manage stack operations. @@ -57,7 +66,9 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler { h.Handle("/stacks/{id}", bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackUpdate))).Methods(http.MethodPut) h.Handle("/stacks/{id}/git", - bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackUpdateGit))).Methods(http.MethodPut) + bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackUpdateGit))).Methods(http.MethodPost) + h.Handle("/stacks/{id}/git/redeploy", + bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackGitRedeploy))).Methods(http.MethodPut) h.Handle("/stacks/{id}/file", bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackFile))).Methods(http.MethodGet) h.Handle("/stacks/{id}/migrate", @@ -66,6 +77,9 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler { bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackStart))).Methods(http.MethodPost) h.Handle("/stacks/{id}/stop", bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackStop))).Methods(http.MethodPost) + h.Handle("/stacks/webhooks/{webhookID}", + httperror.LoggerHandler(h.webhookInvoke)).Methods(http.MethodPost) + return h } @@ -159,3 +173,34 @@ func (handler *Handler) checkUniqueName(endpoint *portainer.Endpoint, name strin return true, nil } + +func (handler *Handler) checkUniqueWebhookID(webhookID string) (bool, error) { + _, err := handler.DataStore.Stack().StackByWebhookID(webhookID) + if err == bolterrors.ErrObjectNotFound { + return true, nil + } + return false, err +} + +func (handler *Handler) clone(projectPath, repositoryURL, refName string, auth bool, username, password string) error { + if !auth { + username = "" + password = "" + } + + err := handler.GitService.CloneRepository(projectPath, repositoryURL, refName, username, password) + if err != nil { + return fmt.Errorf("unable to clone git repository: %w", err) + } + + return nil +} + +func (handler *Handler) latestCommitID(repositoryURL, refName string, auth bool, username, password string) (string, error) { + if !auth { + username = "" + password = "" + } + + return handler.GitService.LatestCommitID(repositoryURL, refName, username, password) +} diff --git a/api/http/handler/stacks/helper.go b/api/http/handler/stacks/helper.go new file mode 100644 index 000000000..dd42330c4 --- /dev/null +++ b/api/http/handler/stacks/helper.go @@ -0,0 +1,24 @@ +package stacks + +import ( + "time" + + "github.com/asaskevich/govalidator" + "github.com/pkg/errors" + portainer "github.com/portainer/portainer/api" +) + +func validateStackAutoUpdate(autoUpdate *portainer.StackAutoUpdate) error { + if autoUpdate == nil { + return nil + } + if autoUpdate.Webhook != "" && !govalidator.IsUUID(autoUpdate.Webhook) { + return errors.New("invalid Webhook format") + } + if autoUpdate.Interval != "" { + if _, err := time.ParseDuration(autoUpdate.Interval); err != nil { + return errors.New("invalid Interval format") + } + } + return nil +} diff --git a/api/http/handler/stacks/helper_test.go b/api/http/handler/stacks/helper_test.go new file mode 100644 index 000000000..c3e564349 --- /dev/null +++ b/api/http/handler/stacks/helper_test.go @@ -0,0 +1,42 @@ +package stacks + +import ( + "testing" + + portainer "github.com/portainer/portainer/api" + "github.com/stretchr/testify/assert" +) + +func Test_ValidateStackAutoUpdate(t *testing.T) { + tests := []struct { + name string + value *portainer.StackAutoUpdate + wantErr bool + }{ + { + name: "webhook is not a valid UUID", + value: &portainer.StackAutoUpdate{Webhook: "fake-webhook"}, + wantErr: true, + }, + { + name: "incorrect interval value", + value: &portainer.StackAutoUpdate{Interval: "1dd2hh3mm"}, + wantErr: true, + }, + { + name: "valid auto update", + value: &portainer.StackAutoUpdate{ + Webhook: "8dce8c2f-9ca1-482b-ad20-271e86536ada", + Interval: "5h30m40s10ms", + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validateStackAutoUpdate(tt.value) + assert.Equalf(t, tt.wantErr, err != nil, "received %+v", err) + }) + } +} diff --git a/api/http/handler/stacks/stack_associate.go b/api/http/handler/stacks/stack_associate.go index 55397a323..14a04ed89 100644 --- a/api/http/handler/stacks/stack_associate.go +++ b/api/http/handler/stacks/stack_associate.go @@ -2,6 +2,9 @@ package stacks import ( "fmt" + "net/http" + "time" + httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" "github.com/portainer/libhttp/response" @@ -9,8 +12,6 @@ import ( bolterrors "github.com/portainer/portainer/api/bolt/errors" "github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/internal/stackutils" - "net/http" - "time" ) // PUT request on /api/stacks/:id/associate?endpointId=&swarmId=&orphanedRunning= @@ -87,5 +88,10 @@ func (handler *Handler) stackAssociate(w http.ResponseWriter, r *http.Request) * stack.ResourceControl = resourceControl + if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { + // sanitize password in the http response to minimise possible security leaks + stack.GitConfig.Authentication.Password = "" + } + return response.JSON(w, stack) } diff --git a/api/http/handler/stacks/stack_create.go b/api/http/handler/stacks/stack_create.go index 53533ca33..d21bf3a1c 100644 --- a/api/http/handler/stacks/stack_create.go +++ b/api/http/handler/stacks/stack_create.go @@ -1,19 +1,17 @@ package stacks import ( - "errors" - "fmt" "log" "net/http" "github.com/docker/cli/cli/compose/loader" "github.com/docker/cli/cli/compose/types" + "github.com/pkg/errors" httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" "github.com/portainer/libhttp/response" portainer "github.com/portainer/portainer/api" bolterrors "github.com/portainer/portainer/api/bolt/errors" - gittypes "github.com/portainer/portainer/api/git/types" "github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/internal/authorization" "github.com/portainer/portainer/api/internal/endpointutils" @@ -129,7 +127,7 @@ func (handler *Handler) createComposeStack(w http.ResponseWriter, r *http.Reques return handler.createComposeStackFromFileUpload(w, r, endpoint, userID) } - return &httperror.HandlerError{http.StatusBadRequest, "Invalid value for query parameter: method. Value must be one of: string, repository or file", errors.New(request.ErrInvalidQueryParameter)} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid value for query parameter: method. Value must be one of: string, repository or file", Err: errors.New(request.ErrInvalidQueryParameter)} } func (handler *Handler) createSwarmStack(w http.ResponseWriter, r *http.Request, method string, endpoint *portainer.Endpoint, userID portainer.UserID) *httperror.HandlerError { @@ -142,7 +140,7 @@ func (handler *Handler) createSwarmStack(w http.ResponseWriter, r *http.Request, return handler.createSwarmStackFromFileUpload(w, r, endpoint, userID) } - return &httperror.HandlerError{http.StatusBadRequest, "Invalid value for query parameter: method. Value must be one of: string, repository or file", errors.New(request.ErrInvalidQueryParameter)} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid value for query parameter: method. Value must be one of: string, repository or file", Err: errors.New(request.ErrInvalidQueryParameter)} } func (handler *Handler) createKubernetesStack(w http.ResponseWriter, r *http.Request, method string, endpoint *portainer.Endpoint) *httperror.HandlerError { @@ -232,24 +230,11 @@ func (handler *Handler) decorateStackResponse(w http.ResponseWriter, stack *port } stack.ResourceControl = resourceControl + + if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { + // sanitize password in the http response to minimise possible security leaks + stack.GitConfig.Authentication.Password = "" + } + return response.JSON(w, stack) } - -func (handler *Handler) cloneAndSaveConfig(stack *portainer.Stack, projectPath, repositoryURL, refName, configFilePath string, auth bool, username, password string) error { - if !auth { - username = "" - password = "" - } - - err := handler.GitService.CloneRepository(projectPath, repositoryURL, refName, username, password) - if err != nil { - return fmt.Errorf("unable to clone git repository: %w", err) - } - - stack.GitConfig = &gittypes.RepoConfig{ - URL: repositoryURL, - ReferenceName: refName, - ConfigFilePath: configFilePath, - } - return nil -} diff --git a/api/http/handler/stacks/stack_create_test.go b/api/http/handler/stacks/stack_create_test.go deleted file mode 100644 index 414948378..000000000 --- a/api/http/handler/stacks/stack_create_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package stacks - -import ( - "testing" - - portainer "github.com/portainer/portainer/api" - gittypes "github.com/portainer/portainer/api/git/types" - "github.com/portainer/portainer/api/http/security" - "github.com/portainer/portainer/api/internal/testhelpers" - "github.com/stretchr/testify/assert" -) - -func Test_stackHandler_cloneAndSaveConfig_shouldCallGitCloneAndSaveConfigOnStack(t *testing.T) { - handler := NewHandler(&security.RequestBouncer{}) - handler.GitService = testhelpers.NewGitService() - - url := "url" - refName := "ref" - configPath := "path" - stack := &portainer.Stack{} - err := handler.cloneAndSaveConfig(stack, "", url, refName, configPath, false, "", "") - assert.NoError(t, err, "clone and save should not fail") - - assert.Equal(t, gittypes.RepoConfig{ - URL: url, - ReferenceName: refName, - ConfigFilePath: configPath, - }, *stack.GitConfig) -} diff --git a/api/http/handler/stacks/stack_delete.go b/api/http/handler/stacks/stack_delete.go index 11d2c31b5..f7ce69bf1 100644 --- a/api/http/handler/stacks/stack_delete.go +++ b/api/http/handler/stacks/stack_delete.go @@ -96,6 +96,11 @@ func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *htt } } + // stop scheduler updates of the stack before removal + if stack.AutoUpdate != nil { + stopAutoupdate(stack.ID, stack.AutoUpdate.JobID, *handler.Scheduler) + } + err = handler.deleteStack(stack, endpoint) if err != nil { return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err} diff --git a/api/http/handler/stacks/stack_inspect.go b/api/http/handler/stacks/stack_inspect.go index 7f445397f..d0797700e 100644 --- a/api/http/handler/stacks/stack_inspect.go +++ b/api/http/handler/stacks/stack_inspect.go @@ -78,5 +78,10 @@ func (handler *Handler) stackInspect(w http.ResponseWriter, r *http.Request) *ht } } + if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { + // sanitize password in the http response to minimise possible security leaks + stack.GitConfig.Authentication.Password = "" + } + return response.JSON(w, stack) } diff --git a/api/http/handler/stacks/stack_list.go b/api/http/handler/stacks/stack_list.go index 81255d032..f4b4fb673 100644 --- a/api/http/handler/stacks/stack_list.go +++ b/api/http/handler/stacks/stack_list.go @@ -1,9 +1,10 @@ package stacks import ( - httperrors "github.com/portainer/portainer/api/http/errors" "net/http" + httperrors "github.com/portainer/portainer/api/http/errors" + httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" "github.com/portainer/libhttp/response" @@ -80,6 +81,13 @@ func (handler *Handler) stackList(w http.ResponseWriter, r *http.Request) *httpe stacks = authorization.FilterAuthorizedStacks(stacks, user, userTeamIDs) } + for _, stack := range stacks { + if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { + // sanitize password in the http response to minimise possible security leaks + stack.GitConfig.Authentication.Password = "" + } + } + return response.JSON(w, stacks) } diff --git a/api/http/handler/stacks/stack_migrate.go b/api/http/handler/stacks/stack_migrate.go index 9f13ef1f5..2323a50ab 100644 --- a/api/http/handler/stacks/stack_migrate.go +++ b/api/http/handler/stacks/stack_migrate.go @@ -150,6 +150,11 @@ func (handler *Handler) stackMigrate(w http.ResponseWriter, r *http.Request) *ht return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack changes inside the database", err} } + if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { + // sanitize password in the http response to minimise possible security leaks + stack.GitConfig.Authentication.Password = "" + } + return response.JSON(w, stack) } diff --git a/api/http/handler/stacks/stack_start.go b/api/http/handler/stacks/stack_start.go index dc4163e92..ff3f5f268 100644 --- a/api/http/handler/stacks/stack_start.go +++ b/api/http/handler/stacks/stack_start.go @@ -85,6 +85,17 @@ func (handler *Handler) stackStart(w http.ResponseWriter, r *http.Request) *http return &httperror.HandlerError{http.StatusBadRequest, "Stack is already active", errors.New("Stack is already active")} } + if stack.AutoUpdate != nil && stack.AutoUpdate.Interval != "" { + stopAutoupdate(stack.ID, stack.AutoUpdate.JobID, *handler.Scheduler) + + jobID, e := startAutoupdate(stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService) + if e != nil { + return e + } + + stack.AutoUpdate.JobID = jobID + } + err = handler.startStack(stack, endpoint) if err != nil { return &httperror.HandlerError{http.StatusInternalServerError, "Unable to start stack", err} @@ -96,6 +107,11 @@ func (handler *Handler) stackStart(w http.ResponseWriter, r *http.Request) *http return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update stack status", err} } + if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { + // sanitize password in the http response to minimise possible security leaks + stack.GitConfig.Authentication.Password = "" + } + return response.JSON(w, stack) } diff --git a/api/http/handler/stacks/stack_stop.go b/api/http/handler/stacks/stack_stop.go index 700cec976..1e24d9607 100644 --- a/api/http/handler/stacks/stack_stop.go +++ b/api/http/handler/stacks/stack_stop.go @@ -74,6 +74,12 @@ func (handler *Handler) stackStop(w http.ResponseWriter, r *http.Request) *httpe return &httperror.HandlerError{http.StatusBadRequest, "Stack is already inactive", errors.New("Stack is already inactive")} } + // stop scheduler updates of the stack before stopping + if stack.AutoUpdate != nil && stack.AutoUpdate.JobID != "" { + stopAutoupdate(stack.ID, stack.AutoUpdate.JobID, *handler.Scheduler) + stack.AutoUpdate.JobID = "" + } + err = handler.stopStack(stack, endpoint) if err != nil { return &httperror.HandlerError{http.StatusInternalServerError, "Unable to stop stack", err} @@ -85,6 +91,11 @@ func (handler *Handler) stackStop(w http.ResponseWriter, r *http.Request) *httpe return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update stack status", err} } + if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { + // sanitize password in the http response to minimise possible security leaks + stack.GitConfig.Authentication.Password = "" + } + return response.JSON(w, stack) } diff --git a/api/http/handler/stacks/stack_update.go b/api/http/handler/stacks/stack_update.go index ee02fb2b5..24f85f817 100644 --- a/api/http/handler/stacks/stack_update.go +++ b/api/http/handler/stacks/stack_update.go @@ -128,6 +128,11 @@ func (handler *Handler) stackUpdate(w http.ResponseWriter, r *http.Request) *htt return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack changes inside the database", err} } + if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { + // sanitize password in the http response to minimise possible security leaks + stack.GitConfig.Authentication.Password = "" + } + return response.JSON(w, stack) } diff --git a/api/http/handler/stacks/stack_update_git.go b/api/http/handler/stacks/stack_update_git.go index dd46e409e..48b7d2af4 100644 --- a/api/http/handler/stacks/stack_update_git.go +++ b/api/http/handler/stacks/stack_update_git.go @@ -2,10 +2,7 @@ package stacks import ( "errors" - "fmt" - "log" "net/http" - "time" "github.com/asaskevich/govalidator" httperror "github.com/portainer/libhttp/error" @@ -13,22 +10,28 @@ import ( "github.com/portainer/libhttp/response" portainer "github.com/portainer/portainer/api" bolterrors "github.com/portainer/portainer/api/bolt/errors" - "github.com/portainer/portainer/api/filesystem" + gittypes "github.com/portainer/portainer/api/git/types" httperrors "github.com/portainer/portainer/api/http/errors" "github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/internal/stackutils" ) -type updateStackGitPayload struct { +type stackGitUpdatePayload struct { + AutoUpdate *portainer.StackAutoUpdate + Env []portainer.Pair RepositoryReferenceName string RepositoryAuthentication bool RepositoryUsername string RepositoryPassword string } -func (payload *updateStackGitPayload) Validate(r *http.Request) error { - if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) { - return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled") +func (payload *stackGitUpdatePayload) Validate(r *http.Request) error { + if govalidator.IsNull(payload.RepositoryReferenceName) { + payload.RepositoryReferenceName = defaultGitReferenceName + } + + if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil { + return err } return nil } @@ -53,18 +56,23 @@ func (payload *updateStackGitPayload) Validate(r *http.Request) error { func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { stackID, err := request.RetrieveNumericRouteVariableValue(r, "id") if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid stack identifier route variable", Err: err} + } + + var payload stackGitUpdatePayload + err = request.DecodeAndValidateJSONPayload(r, &payload) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err} } stack, err := handler.DataStore.Stack().Stack(portainer.StackID(stackID)) if err == bolterrors.ErrObjectNotFound { - return &httperror.HandlerError{http.StatusNotFound, "Unable to find a stack with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find a stack with the specified identifier inside the database", Err: err} } else if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err} - } - - if stack.GitConfig == nil { - return &httperror.HandlerError{http.StatusBadRequest, "Stack is not created from git", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find a stack with the specified identifier inside the database", Err: err} + } else if stack.GitConfig == nil { + msg := "No Git config in the found stack" + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: msg, Err: errors.New(msg)} } // TODO: this is a work-around for stacks created with Portainer version >= 1.17.1 @@ -72,7 +80,7 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) * // can use the optional EndpointID query parameter to associate a valid endpoint identifier to the stack. endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", true) if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: endpointId", err} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid query parameter: endpointId", Err: err} } if endpointID != int(stack.EndpointID) { stack.EndpointID = portainer.EndpointID(endpointID) @@ -80,117 +88,75 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) * endpoint, err := handler.DataStore.Endpoint().Endpoint(stack.EndpointID) if err == bolterrors.ErrObjectNotFound { - return &httperror.HandlerError{http.StatusNotFound, "Unable to find the endpoint associated to the stack inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err} } else if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find the endpoint associated to the stack inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err} } err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint) if err != nil { - return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err} + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to access endpoint", Err: err} } resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve a resource control associated to the stack", Err: err} } securityContext, err := security.RetrieveRestrictedRequestContext(r) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err} } access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to verify user authorizations to validate stack access", Err: err} } if !access { - return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", httperrors.ErrResourceAccessDenied} + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Access denied to resource", Err: httperrors.ErrResourceAccessDenied} } - var payload updateStackGitPayload - err = request.DecodeAndValidateJSONPayload(r, &payload) - if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err} + //stop the autoupdate job if there is any + if stack.AutoUpdate != nil { + stopAutoupdate(stack.ID, stack.AutoUpdate.JobID, *handler.Scheduler) } + //update retrieved stack data based on the payload stack.GitConfig.ReferenceName = payload.RepositoryReferenceName + stack.AutoUpdate = payload.AutoUpdate + stack.Env = payload.Env - backupProjectPath := fmt.Sprintf("%s-old", stack.ProjectPath) - err = filesystem.MoveDirectory(stack.ProjectPath, backupProjectPath) - if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to move git repository directory", err} + stack.GitConfig.Authentication = nil + if payload.RepositoryAuthentication { + password := payload.RepositoryPassword + if password == "" && stack.GitConfig != nil && stack.GitConfig.Authentication != nil { + password = stack.GitConfig.Authentication.Password + } + stack.GitConfig.Authentication = &gittypes.GitAuthentication{ + Username: payload.RepositoryUsername, + Password: password, + } } - repositoryUsername := payload.RepositoryUsername - repositoryPassword := payload.RepositoryPassword - if !payload.RepositoryAuthentication { - repositoryUsername = "" - repositoryPassword = "" - } - - err = handler.GitService.CloneRepository(stack.ProjectPath, stack.GitConfig.URL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword) - if err != nil { - restoreError := filesystem.MoveDirectory(backupProjectPath, stack.ProjectPath) - if restoreError != nil { - log.Printf("[WARN] [http,stacks,git] [error: %s] [message: failed restoring backup folder]", restoreError) + if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" { + jobID, e := startAutoupdate(stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService) + if e != nil { + return e } - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to clone git repository", err} - } - - defer func() { - err = handler.FileService.RemoveDirectory(backupProjectPath) - if err != nil { - log.Printf("[WARN] [http,stacks,git] [error: %s] [message: unable to remove git repository directory]", err) - } - }() - - httpErr := handler.deployStack(r, stack, endpoint) - if httpErr != nil { - return httpErr + stack.AutoUpdate.JobID = jobID } + //save the updated stack to DB err = handler.DataStore.Stack().UpdateStack(stack.ID, stack) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack changes inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack changes inside the database", Err: err} + } + + if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { + // sanitize password in the http response to minimise possible security leaks + stack.GitConfig.Authentication.Password = "" } return response.JSON(w, stack) } - -func (handler *Handler) deployStack(r *http.Request, stack *portainer.Stack, endpoint *portainer.Endpoint) *httperror.HandlerError { - if stack.Type == portainer.DockerSwarmStack { - config, httpErr := handler.createSwarmDeployConfig(r, stack, endpoint, false) - if httpErr != nil { - return httpErr - } - - err := handler.deploySwarmStack(config) - if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err} - } - - stack.UpdateDate = time.Now().Unix() - stack.UpdatedBy = config.user.Username - stack.Status = portainer.StackStatusActive - - return nil - } - - config, httpErr := handler.createComposeDeployConfig(r, stack, endpoint) - if httpErr != nil { - return httpErr - } - - err := handler.deployComposeStack(config) - if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err} - } - - stack.UpdateDate = time.Now().Unix() - stack.UpdatedBy = config.user.Username - stack.Status = portainer.StackStatusActive - - return nil -} diff --git a/api/http/handler/stacks/stack_update_git_redeploy.go b/api/http/handler/stacks/stack_update_git_redeploy.go new file mode 100644 index 000000000..c338757e0 --- /dev/null +++ b/api/http/handler/stacks/stack_update_git_redeploy.go @@ -0,0 +1,190 @@ +package stacks + +import ( + "fmt" + "log" + "net/http" + "time" + + "github.com/asaskevich/govalidator" + httperror "github.com/portainer/libhttp/error" + "github.com/portainer/libhttp/request" + "github.com/portainer/libhttp/response" + portainer "github.com/portainer/portainer/api" + bolterrors "github.com/portainer/portainer/api/bolt/errors" + "github.com/portainer/portainer/api/filesystem" + httperrors "github.com/portainer/portainer/api/http/errors" + "github.com/portainer/portainer/api/http/security" + "github.com/portainer/portainer/api/internal/stackutils" +) + +type stackGitRedployPayload struct { + RepositoryReferenceName string + RepositoryAuthentication bool + RepositoryUsername string + RepositoryPassword string + Env []portainer.Pair +} + +func (payload *stackGitRedployPayload) Validate(r *http.Request) error { + if govalidator.IsNull(payload.RepositoryReferenceName) { + payload.RepositoryReferenceName = defaultGitReferenceName + } + + return nil +} + +// PUT request on /api/stacks/:id/git?endpointId= +func (handler *Handler) stackGitRedeploy(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { + stackID, err := request.RetrieveNumericRouteVariableValue(r, "id") + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid stack identifier route variable", Err: err} + } + + stack, err := handler.DataStore.Stack().Stack(portainer.StackID(stackID)) + if err == bolterrors.ErrObjectNotFound { + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find a stack with the specified identifier inside the database", Err: err} + } else if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find a stack with the specified identifier inside the database", Err: err} + } + + if stack.GitConfig == nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Stack is not created from git", Err: err} + } + + // TODO: this is a work-around for stacks created with Portainer version >= 1.17.1 + // The EndpointID property is not available for these stacks, this API endpoint + // can use the optional EndpointID query parameter to associate a valid endpoint identifier to the stack. + endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", true) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid query parameter: endpointId", Err: err} + } + if endpointID != int(stack.EndpointID) { + stack.EndpointID = portainer.EndpointID(endpointID) + } + + endpoint, err := handler.DataStore.Endpoint().Endpoint(stack.EndpointID) + if err == bolterrors.ErrObjectNotFound { + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err} + } else if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err} + } + + err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to access endpoint", Err: err} + } + + resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve a resource control associated to the stack", Err: err} + } + + securityContext, err := security.RetrieveRestrictedRequestContext(r) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err} + } + + access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to verify user authorizations to validate stack access", Err: err} + } + if !access { + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Access denied to resource", Err: httperrors.ErrResourceAccessDenied} + } + + var payload stackGitRedployPayload + err = request.DecodeAndValidateJSONPayload(r, &payload) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err} + } + + stack.GitConfig.ReferenceName = payload.RepositoryReferenceName + stack.Env = payload.Env + + backupProjectPath := fmt.Sprintf("%s-old", stack.ProjectPath) + err = filesystem.MoveDirectory(stack.ProjectPath, backupProjectPath) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to move git repository directory", Err: err} + } + + repositoryUsername := "" + repositoryPassword := "" + if payload.RepositoryAuthentication { + repositoryPassword = payload.RepositoryPassword + if repositoryPassword == "" && stack.GitConfig != nil && stack.GitConfig.Authentication != nil { + repositoryPassword = stack.GitConfig.Authentication.Password + } + repositoryUsername = payload.RepositoryUsername + } + + err = handler.GitService.CloneRepository(stack.ProjectPath, stack.GitConfig.URL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword) + if err != nil { + restoreError := filesystem.MoveDirectory(backupProjectPath, stack.ProjectPath) + if restoreError != nil { + log.Printf("[WARN] [http,stacks,git] [error: %s] [message: failed restoring backup folder]", restoreError) + } + + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err} + } + + defer func() { + err = handler.FileService.RemoveDirectory(backupProjectPath) + if err != nil { + log.Printf("[WARN] [http,stacks,git] [error: %s] [message: unable to remove git repository directory]", err) + } + }() + + httpErr := handler.deployStack(r, stack, endpoint) + if httpErr != nil { + return httpErr + } + + err = handler.DataStore.Stack().UpdateStack(stack.ID, stack) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack changes inside the database", Err: err} + } + + if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { + // sanitize password in the http response to minimise possible security leaks + stack.GitConfig.Authentication.Password = "" + } + + return response.JSON(w, stack) +} + +func (handler *Handler) deployStack(r *http.Request, stack *portainer.Stack, endpoint *portainer.Endpoint) *httperror.HandlerError { + if stack.Type == portainer.DockerSwarmStack { + config, httpErr := handler.createSwarmDeployConfig(r, stack, endpoint, false) + if httpErr != nil { + return httpErr + } + + err := handler.deploySwarmStack(config) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err} + } + + stack.UpdateDate = time.Now().Unix() + stack.UpdatedBy = config.user.Username + stack.Status = portainer.StackStatusActive + + return nil + } + + config, httpErr := handler.createComposeDeployConfig(r, stack, endpoint) + if httpErr != nil { + return httpErr + } + + err := handler.deployComposeStack(config) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err} + } + + stack.UpdateDate = time.Now().Unix() + stack.UpdatedBy = config.user.Username + stack.Status = portainer.StackStatusActive + + return nil +} diff --git a/api/http/handler/stacks/webhook_invoke.go b/api/http/handler/stacks/webhook_invoke.go new file mode 100644 index 000000000..01c9d701b --- /dev/null +++ b/api/http/handler/stacks/webhook_invoke.go @@ -0,0 +1,54 @@ +package stacks + +import ( + "log" + "net/http" + + "github.com/gofrs/uuid" + + "github.com/portainer/libhttp/response" + + bolterrors "github.com/portainer/portainer/api/bolt/errors" + "github.com/portainer/portainer/api/stacks" + + httperror "github.com/portainer/libhttp/error" + "github.com/portainer/libhttp/request" +) + +func (handler *Handler) webhookInvoke(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { + webhookID, err := retrieveUUIDRouteVariableValue(r, "webhookID") + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid webhook identifier route variable", Err: err} + } + + stack, err := handler.DataStore.Stack().StackByWebhookID(webhookID.String()) + if err != nil { + statusCode := http.StatusInternalServerError + if err == bolterrors.ErrObjectNotFound { + statusCode = http.StatusNotFound + } + return &httperror.HandlerError{StatusCode: statusCode, Message: "Unable to find the stack by webhook ID", Err: err} + } + + if err = stacks.RedeployWhenChanged(stack.ID, handler.StackDeployer, handler.DataStore, handler.GitService); err != nil { + log.Printf("[ERROR] %s\n", err) + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Failed to update the stack", Err: err} + } + + return response.Empty(w) +} + +func retrieveUUIDRouteVariableValue(r *http.Request, name string) (uuid.UUID, error) { + webhookID, err := request.RetrieveRouteVariableValue(r, name) + if err != nil { + return uuid.Nil, err + } + + uid, err := uuid.FromString(webhookID) + + if err != nil { + return uuid.Nil, err + } + + return uid, nil +} diff --git a/api/http/handler/stacks/webhook_invoke_test.go b/api/http/handler/stacks/webhook_invoke_test.go new file mode 100644 index 000000000..cc6656519 --- /dev/null +++ b/api/http/handler/stacks/webhook_invoke_test.go @@ -0,0 +1,59 @@ +package stacks + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/gofrs/uuid" + "github.com/stretchr/testify/assert" + + portainer "github.com/portainer/portainer/api" + + "github.com/portainer/portainer/api/bolt/bolttest" +) + +func TestHandler_webhookInvoke(t *testing.T) { + store, teardown := bolttest.MustNewTestStore(true) + defer teardown() + + webhookID := newGuidString(t) + store.StackService.CreateStack(&portainer.Stack{ + AutoUpdate: &portainer.StackAutoUpdate{ + Webhook: webhookID, + }, + }) + + h := NewHandler(nil) + h.DataStore = store + + t.Run("invalid uuid results in http.StatusBadRequest", func(t *testing.T) { + w := httptest.NewRecorder() + req := newRequest("notuuid") + h.Router.ServeHTTP(w, req) + assert.Equal(t, http.StatusBadRequest, w.Code) + }) + t.Run("registered webhook ID in http.StatusNoContent", func(t *testing.T) { + w := httptest.NewRecorder() + req := newRequest(webhookID) + h.Router.ServeHTTP(w, req) + assert.Equal(t, http.StatusNoContent, w.Code) + }) + t.Run("unregistered webhook ID in http.StatusNotFound", func(t *testing.T) { + w := httptest.NewRecorder() + req := newRequest(newGuidString(t)) + h.Router.ServeHTTP(w, req) + assert.Equal(t, http.StatusNotFound, w.Code) + }) +} + +func newGuidString(t *testing.T) string { + uuid, err := uuid.NewV4() + assert.NoError(t, err) + + return uuid.String() +} + +func newRequest(webhookID string) *http.Request { + return httptest.NewRequest(http.MethodPost, "/stacks/webhooks/"+webhookID, nil) +} diff --git a/api/http/server.go b/api/http/server.go index 13ab4dcda..a38c1922a 100644 --- a/api/http/server.go +++ b/api/http/server.go @@ -50,6 +50,8 @@ import ( "github.com/portainer/portainer/api/internal/authorization" "github.com/portainer/portainer/api/internal/ssl" "github.com/portainer/portainer/api/kubernetes/cli" + "github.com/portainer/portainer/api/scheduler" + stackdeployer "github.com/portainer/portainer/api/stacks" ) // Server implements the portainer.Server interface @@ -79,8 +81,10 @@ type Server struct { DockerClientFactory *docker.ClientFactory KubernetesClientFactory *cli.ClientFactory KubernetesDeployer portainer.KubernetesDeployer + Scheduler *scheduler.Scheduler ShutdownCtx context.Context ShutdownTrigger context.CancelFunc + StackDeployer stackdeployer.StackDeployer } // Start starts the HTTP server @@ -185,10 +189,12 @@ func (server *Server) Start() error { stackHandler.DataStore = server.DataStore stackHandler.DockerClientFactory = server.DockerClientFactory stackHandler.FileService = server.FileService - stackHandler.SwarmStackManager = server.SwarmStackManager - stackHandler.ComposeStackManager = server.ComposeStackManager stackHandler.KubernetesDeployer = server.KubernetesDeployer stackHandler.GitService = server.GitService + stackHandler.Scheduler = server.Scheduler + stackHandler.SwarmStackManager = server.SwarmStackManager + stackHandler.ComposeStackManager = server.ComposeStackManager + stackHandler.StackDeployer = server.StackDeployer var tagHandler = tags.NewHandler(requestBouncer) tagHandler.DataStore = server.DataStore diff --git a/api/internal/stackutils/stackutils.go b/api/internal/stackutils/stackutils.go index 5b1e9bf43..7e94bff17 100644 --- a/api/internal/stackutils/stackutils.go +++ b/api/internal/stackutils/stackutils.go @@ -2,6 +2,7 @@ package stackutils import ( "fmt" + "path" portainer "github.com/portainer/portainer/api" ) @@ -10,3 +11,12 @@ import ( func ResourceControlID(endpointID portainer.EndpointID, name string) string { return fmt.Sprintf("%d_%s", endpointID, name) } + +// GetStackFilePaths returns a list of file paths based on stack project path +func GetStackFilePaths(stack *portainer.Stack) []string { + var filePaths []string + for _, file := range append([]string{stack.EntryPoint}, stack.AdditionalFiles...) { + filePaths = append(filePaths, path.Join(stack.ProjectPath, file)) + } + return filePaths +} diff --git a/api/internal/stackutils/stackutils_test.go b/api/internal/stackutils/stackutils_test.go new file mode 100644 index 000000000..6af19d8af --- /dev/null +++ b/api/internal/stackutils/stackutils_test.go @@ -0,0 +1,26 @@ +package stackutils + +import ( + "testing" + + portainer "github.com/portainer/portainer/api" + "github.com/stretchr/testify/assert" +) + +func Test_GetStackFilePaths(t *testing.T) { + stack := &portainer.Stack{ + ProjectPath: "/tmp/stack/1", + EntryPoint: "file-one.yml", + } + + t.Run("stack doesn't have additional files", func(t *testing.T) { + expected := []string{"/tmp/stack/1/file-one.yml"} + assert.ElementsMatch(t, expected, GetStackFilePaths(stack)) + }) + + t.Run("stack has additional files", func(t *testing.T) { + stack.AdditionalFiles = []string{"file-two.yml", "file-three.yml"} + expected := []string{"/tmp/stack/1/file-one.yml", "/tmp/stack/1/file-two.yml", "/tmp/stack/1/file-three.yml"} + assert.ElementsMatch(t, expected, GetStackFilePaths(stack)) + }) +} diff --git a/api/libcompose/compose_stack.go b/api/libcompose/compose_stack.go index 41da7239d..f2dfff230 100644 --- a/api/libcompose/compose_stack.go +++ b/api/libcompose/compose_stack.go @@ -16,6 +16,7 @@ import ( "github.com/portainer/libcompose/project" "github.com/portainer/libcompose/project/options" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/internal/stackutils" ) const ( @@ -86,12 +87,12 @@ func (manager *ComposeStackManager) Up(stack *portainer.Stack, endpoint *portain for _, envvar := range stack.Env { env[envvar.Name] = envvar.Value } + filePaths := stackutils.GetStackFilePaths(stack) - composeFilePath := path.Join(stack.ProjectPath, stack.EntryPoint) proj, err := docker.NewProject(&ctx.Context{ ConfigDir: manager.dataPath, Context: project.Context{ - ComposeFiles: []string{composeFilePath}, + ComposeFiles: filePaths, EnvironmentLookup: &lookup.ComposableEnvLookup{ Lookups: []config.EnvironmentLookup{ &lookup.EnvfileLookup{ @@ -120,10 +121,13 @@ func (manager *ComposeStackManager) Down(stack *portainer.Stack, endpoint *porta return err } - composeFilePath := path.Join(stack.ProjectPath, stack.EntryPoint) + var composeFiles []string + for _, file := range append([]string{stack.EntryPoint}, stack.AdditionalFiles...) { + composeFiles = append(composeFiles, path.Join(stack.ProjectPath, file)) + } proj, err := docker.NewProject(&ctx.Context{ Context: project.Context{ - ComposeFiles: []string{composeFilePath}, + ComposeFiles: composeFiles, ProjectName: stack.Name, }, ClientFactory: clientFactory, @@ -134,3 +138,11 @@ func (manager *ComposeStackManager) Down(stack *portainer.Stack, endpoint *porta return proj.Down(context.Background(), options.Down{RemoveVolume: false, RemoveOrphans: true}) } + +func stackFilePaths(stack *portainer.Stack) []string { + var filePaths []string + for _, file := range append([]string{stack.EntryPoint}, stack.AdditionalFiles...) { + filePaths = append(filePaths, path.Join(stack.ProjectPath, file)) + } + return filePaths +} diff --git a/api/portainer.go b/api/portainer.go index 7b7ff45f3..73e286d6d 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -744,10 +744,24 @@ type ( UpdateDate int64 `example:"1587399600"` // The username which last updated this stack UpdatedBy string `example:"bob"` + // Only applies when deploying stack with multiple files + AdditionalFiles []string `json:"AdditionalFiles"` + // The auto update settings of a git stack + AutoUpdate *StackAutoUpdate `json:"AutoUpdate"` // The git config of this stack GitConfig *gittypes.RepoConfig } + //StackAutoUpdate represents the git auto sync config for stack deployment + StackAutoUpdate struct { + // Auto update interval + Interval string `example:"1m30s"` + // A UUID generated from client + Webhook string `example:"05de31a2-79fa-4644-9c12-faa67e5c49f0"` + // Autoupdate job id + JobID string `example:"15"` + } + // StackID represents a stack identifier (it must be composed of Name + "_" + SwarmID to create a unique identifier) StackID int @@ -1187,6 +1201,7 @@ type ( // GitService represents a service for managing Git GitService interface { CloneRepository(destination string, repositoryURL, referenceName, username, password string) error + LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) } // JWTService represents a service for managing JWT tokens @@ -1302,6 +1317,8 @@ type ( UpdateStack(ID StackID, stack *Stack) error DeleteStack(ID StackID) error GetNextIdentifier() int + StackByWebhookID(ID string) (*Stack, error) + RefreshableStacks() ([]Stack, error) } // SnapshotService represents a service for managing endpoint snapshots diff --git a/api/scheduler/scheduler.go b/api/scheduler/scheduler.go new file mode 100644 index 000000000..6568f753f --- /dev/null +++ b/api/scheduler/scheduler.go @@ -0,0 +1,73 @@ +package scheduler + +import ( + "context" + "log" + "strconv" + "time" + + "github.com/pkg/errors" + "github.com/robfig/cron/v3" +) + +type Scheduler struct { + crontab *cron.Cron + shutdownCtx context.Context +} + +func NewScheduler(ctx context.Context) *Scheduler { + crontab := cron.New(cron.WithChain(cron.Recover(cron.DefaultLogger))) + crontab.Start() + + s := &Scheduler{ + crontab: crontab, + } + + if ctx != nil { + go func() { + <-ctx.Done() + s.Shutdown() + }() + } + + return s +} + +// Shutdown stops the scheduler and waits for it to stop if it is running; otherwise does nothing. +func (s *Scheduler) Shutdown() error { + if s.crontab == nil { + return nil + } + + log.Println("[DEBUG] Stopping scheduler") + ctx := s.crontab.Stop() + <-ctx.Done() + + for _, j := range s.crontab.Entries() { + s.crontab.Remove(j.ID) + } + + err := ctx.Err() + if err == context.Canceled { + return nil + } + return err +} + +// StopJob stops the job from being run in the future +func (s *Scheduler) StopJob(jobID string) error { + id, err := strconv.Atoi(jobID) + if err != nil { + return errors.Wrapf(err, "failed convert jobID %q to int", jobID) + } + s.crontab.Remove(cron.EntryID(id)) + + return nil +} + +// StartJobEvery schedules a new periodic job with a given duration. +// Returns job id that could be used to stop the given job +func (s *Scheduler) StartJobEvery(duration time.Duration, job func()) string { + entryId := s.crontab.Schedule(cron.Every(duration), cron.FuncJob(job)) + return strconv.Itoa(int(entryId)) +} diff --git a/api/scheduler/scheduler_test.go b/api/scheduler/scheduler_test.go new file mode 100644 index 000000000..6d21e49ec --- /dev/null +++ b/api/scheduler/scheduler_test.go @@ -0,0 +1,57 @@ +package scheduler + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func Test_CanStartAndTerminate(t *testing.T) { + s := NewScheduler(context.Background()) + s.StartJobEvery(1*time.Minute, func() { fmt.Println("boop") }) + + err := s.Shutdown() + assert.NoError(t, err, "Shutdown should return no errors") + assert.Empty(t, s.crontab.Entries(), "all jobs should have been removed") +} + +func Test_CanTerminateByCancellingContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + s := NewScheduler(ctx) + s.StartJobEvery(1*time.Minute, func() { fmt.Println("boop") }) + + cancel() + + for i := 0; i < 100; i++ { + if len(s.crontab.Entries()) == 0 { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatal("all jobs are expected to be cleaned by now; it might be a timing issue, otherwise implementation defect") +} + +func Test_StartAndStopJob(t *testing.T) { + s := NewScheduler(context.Background()) + defer s.Shutdown() + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + + var jobOne string + var workDone bool + jobOne = s.StartJobEvery(time.Second, func() { + assert.Equal(t, 1, len(s.crontab.Entries()), "scheduler should have one active job") + workDone = true + + s.StopJob(jobOne) + cancel() + }) + + <-ctx.Done() + assert.True(t, workDone, "value should been set in the job") + assert.Equal(t, 0, len(s.crontab.Entries()), "scheduler should have no active jobs") + +} diff --git a/api/stacks/deploy.go b/api/stacks/deploy.go new file mode 100644 index 000000000..ccf5eb441 --- /dev/null +++ b/api/stacks/deploy.go @@ -0,0 +1,138 @@ +package stacks + +import ( + "strings" + "time" + + "github.com/pkg/errors" + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/http/security" +) + +func RedeployWhenChanged(stackID portainer.StackID, deployer StackDeployer, datastore portainer.DataStore, gitService portainer.GitService) error { + stack, err := datastore.Stack().Stack(stackID) + if err != nil { + return errors.WithMessagef(err, "failed to get the stack %v", stackID) + } + + if stack.GitConfig == nil { + return nil // do nothing if it isn't a git-based stack + } + + username, password := "", "" + if stack.GitConfig.Authentication != nil { + username, password = stack.GitConfig.Authentication.Username, stack.GitConfig.Authentication.Password + } + + newHash, err := gitService.LatestCommitID(stack.GitConfig.URL, stack.GitConfig.ReferenceName, username, password) + if err != nil { + return errors.WithMessagef(err, "failed to fetch latest commit id of the stack %v", stack.ID) + } + + if strings.EqualFold(newHash, string(stack.GitConfig.ConfigHash)) { + return nil + } + + cloneParams := &cloneRepositoryParameters{ + url: stack.GitConfig.URL, + ref: stack.GitConfig.ReferenceName, + toDir: stack.ProjectPath, + } + if stack.GitConfig.Authentication != nil { + cloneParams.auth = &gitAuth{ + username: username, + password: password, + } + } + + if err := cloneGitRepository(gitService, cloneParams); err != nil { + return errors.WithMessagef(err, "failed to do a fresh clone of the stack %v", stack.ID) + } + + endpoint, err := datastore.Endpoint().Endpoint(stack.EndpointID) + if err != nil { + return errors.WithMessagef(err, "failed to find the endpoint %v associated to the stack %v", stack.EndpointID, stack.ID) + } + + author := stack.UpdatedBy + if author == "" { + author = stack.CreatedBy + } + + registries, err := getUserRegistries(datastore, author, endpoint.ID) + if err != nil { + return err + } + + switch stack.Type { + case portainer.DockerComposeStack: + err := deployer.DeployComposeStack(stack, endpoint, registries) + if err != nil { + return errors.WithMessagef(err, "failed to deploy a docker compose stack %v", stackID) + } + case portainer.DockerSwarmStack: + err := deployer.DeploySwarmStack(stack, endpoint, registries, true) + if err != nil { + return errors.WithMessagef(err, "failed to deploy a docker compose stack %v", stackID) + } + default: + return errors.Errorf("cannot update stack, type %v is unsupported", stack.Type) + } + + stack.UpdateDate = time.Now().Unix() + stack.GitConfig.ConfigHash = newHash + if err := datastore.Stack().UpdateStack(stack.ID, stack); err != nil { + return errors.WithMessagef(err, "failed to update the stack %v", stack.ID) + } + + return nil +} + +func getUserRegistries(datastore portainer.DataStore, authorUsername string, endpointID portainer.EndpointID) ([]portainer.Registry, error) { + registries, err := datastore.Registry().Registries() + if err != nil { + return nil, errors.WithMessage(err, "unable to retrieve registries from the database") + } + + user, err := datastore.User().UserByUsername(authorUsername) + if err != nil { + return nil, errors.WithMessagef(err, "failed to fetch a stack's author [%s]", authorUsername) + } + + if user.Role == portainer.AdministratorRole { + return registries, nil + } + + userMemberships, err := datastore.TeamMembership().TeamMembershipsByUserID(user.ID) + if err != nil { + return nil, errors.WithMessagef(err, "failed to fetch memberships of the stack author [%s]", authorUsername) + } + + filteredRegistries := make([]portainer.Registry, 0, len(registries)) + for _, registry := range registries { + if security.AuthorizedRegistryAccess(®istry, user, userMemberships, endpointID) { + filteredRegistries = append(filteredRegistries, registry) + } + } + + return filteredRegistries, nil +} + +type cloneRepositoryParameters struct { + url string + ref string + toDir string + auth *gitAuth +} + +type gitAuth struct { + username string + password string +} + +func cloneGitRepository(gitService portainer.GitService, cloneParams *cloneRepositoryParameters) error { + if cloneParams.auth != nil { + return gitService.CloneRepository(cloneParams.toDir, cloneParams.url, cloneParams.ref, cloneParams.auth.username, cloneParams.auth.password) + } + return gitService.CloneRepository(cloneParams.toDir, cloneParams.url, cloneParams.ref, "", "") +} diff --git a/api/stacks/deploy_test.go b/api/stacks/deploy_test.go new file mode 100644 index 000000000..58b7de913 --- /dev/null +++ b/api/stacks/deploy_test.go @@ -0,0 +1,221 @@ +package stacks + +import ( + "errors" + "io/ioutil" + "strings" + "testing" + + portainer "github.com/portainer/portainer/api" + bolt "github.com/portainer/portainer/api/bolt/bolttest" + gittypes "github.com/portainer/portainer/api/git/types" + "github.com/stretchr/testify/assert" +) + +type gitService struct { + cloneErr error + id string +} + +func (g *gitService) CloneRepository(destination, repositoryURL, referenceName, username, password string) error { + return g.cloneErr +} + +func (g *gitService) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) { + return g.id, nil +} + +type noopDeployer struct{} + +func (s *noopDeployer) DeploySwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, prune bool) error { + return nil +} + +func (s *noopDeployer) DeployComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry) error { + return nil +} + +func Test_redeployWhenChanged_FailsWhenCannotFindStack(t *testing.T) { + store, teardown := bolt.MustNewTestStore(true) + defer teardown() + + err := RedeployWhenChanged(1, nil, store, nil) + assert.Error(t, err) + assert.Truef(t, strings.HasPrefix(err.Error(), "failed to get the stack"), "it isn't an error we expected: %v", err.Error()) +} + +func Test_redeployWhenChanged_DoesNothingWhenNotAGitBasedStack(t *testing.T) { + store, teardown := bolt.MustNewTestStore(true) + defer teardown() + + err := store.Stack().CreateStack(&portainer.Stack{ID: 1}) + assert.NoError(t, err, "failed to create a test stack") + + err = RedeployWhenChanged(1, nil, store, &gitService{nil, ""}) + assert.NoError(t, err) +} + +func Test_redeployWhenChanged_DoesNothingWhenNoGitChanges(t *testing.T) { + store, teardown := bolt.MustNewTestStore(true) + defer teardown() + + tmpDir, _ := ioutil.TempDir("", "stack") + + err := store.Stack().CreateStack(&portainer.Stack{ + ID: 1, + ProjectPath: tmpDir, + GitConfig: &gittypes.RepoConfig{ + URL: "url", + ReferenceName: "ref", + ConfigHash: "oldHash", + }}) + assert.NoError(t, err, "failed to create a test stack") + + err = RedeployWhenChanged(1, nil, store, &gitService{nil, "oldHash"}) + assert.NoError(t, err) +} + +func Test_redeployWhenChanged_FailsWhenCannotClone(t *testing.T) { + cloneErr := errors.New("failed to clone") + store, teardown := bolt.MustNewTestStore(true) + defer teardown() + + err := store.Stack().CreateStack(&portainer.Stack{ + ID: 1, + GitConfig: &gittypes.RepoConfig{ + URL: "url", + ReferenceName: "ref", + ConfigHash: "oldHash", + }}) + assert.NoError(t, err, "failed to create a test stack") + + err = RedeployWhenChanged(1, nil, store, &gitService{cloneErr, "newHash"}) + assert.Error(t, err) + assert.ErrorIs(t, err, cloneErr, "should failed to clone but didn't, check test setup") +} + +func Test_redeployWhenChanged(t *testing.T) { + store, teardown := bolt.MustNewTestStore(true) + defer teardown() + + tmpDir, _ := ioutil.TempDir("", "stack") + + err := store.Endpoint().CreateEndpoint(&portainer.Endpoint{ID: 1}) + assert.NoError(t, err, "error creating endpoint") + + username := "user" + err = store.User().CreateUser(&portainer.User{Username: username, Role: portainer.AdministratorRole}) + assert.NoError(t, err, "error creating a user") + + stack := portainer.Stack{ + ID: 1, + EndpointID: 1, + ProjectPath: tmpDir, + UpdatedBy: username, + GitConfig: &gittypes.RepoConfig{ + URL: "url", + ReferenceName: "ref", + ConfigHash: "oldHash", + }} + err = store.Stack().CreateStack(&stack) + assert.NoError(t, err, "failed to create a test stack") + + t.Run("can deploy docker compose stack", func(t *testing.T) { + stack.Type = portainer.DockerComposeStack + store.Stack().UpdateStack(stack.ID, &stack) + + err = RedeployWhenChanged(1, &noopDeployer{}, store, &gitService{nil, "newHash"}) + assert.NoError(t, err) + }) + + t.Run("can deploy docker swarm stack", func(t *testing.T) { + stack.Type = portainer.DockerSwarmStack + store.Stack().UpdateStack(stack.ID, &stack) + + err = RedeployWhenChanged(1, &noopDeployer{}, store, &gitService{nil, "newHash"}) + assert.NoError(t, err) + }) + + t.Run("can NOT deploy kube stack", func(t *testing.T) { + stack.Type = portainer.KubernetesStack + store.Stack().UpdateStack(stack.ID, &stack) + + err = RedeployWhenChanged(1, &noopDeployer{}, store, &gitService{nil, "newHash"}) + assert.EqualError(t, err, "cannot update stack, type 3 is unsupported") + }) +} + +func Test_getUserRegistries(t *testing.T) { + store, teardown := bolt.MustNewTestStore(true) + defer teardown() + + endpointID := 123 + + admin := portainer.User{ID: 1, Username: "admin", Role: portainer.AdministratorRole} + err := store.User().CreateUser(&admin) + assert.NoError(t, err, "error creating an admin") + + user := portainer.User{ID: 2, Username: "user", Role: portainer.StandardUserRole} + err = store.User().CreateUser(&user) + assert.NoError(t, err, "error creating a user") + + team := portainer.Team{ID: 1, Name: "team"} + + store.TeamMembership().CreateTeamMembership(&portainer.TeamMembership{ + ID: 1, + UserID: user.ID, + TeamID: team.ID, + Role: portainer.TeamMember, + }) + + registryReachableByUser := portainer.Registry{ + ID: 1, + RegistryAccesses: portainer.RegistryAccesses{ + portainer.EndpointID(endpointID): { + UserAccessPolicies: map[portainer.UserID]portainer.AccessPolicy{ + user.ID: {RoleID: portainer.RoleID(portainer.StandardUserRole)}, + }, + }, + }, + } + err = store.Registry().CreateRegistry(®istryReachableByUser) + assert.NoError(t, err, "couldn't create a registry") + + registryReachableByTeam := portainer.Registry{ + ID: 2, + RegistryAccesses: portainer.RegistryAccesses{ + portainer.EndpointID(endpointID): { + TeamAccessPolicies: map[portainer.TeamID]portainer.AccessPolicy{ + team.ID: {RoleID: portainer.RoleID(portainer.StandardUserRole)}, + }, + }, + }, + } + err = store.Registry().CreateRegistry(®istryReachableByTeam) + assert.NoError(t, err, "couldn't create a registry") + + registryRestricted := portainer.Registry{ + ID: 3, + RegistryAccesses: portainer.RegistryAccesses{ + portainer.EndpointID(endpointID): { + UserAccessPolicies: map[portainer.UserID]portainer.AccessPolicy{ + user.ID + 100: {RoleID: portainer.RoleID(portainer.StandardUserRole)}, + }, + }, + }, + } + err = store.Registry().CreateRegistry(®istryRestricted) + assert.NoError(t, err, "couldn't create a registry") + + t.Run("admin should has access to all registries", func(t *testing.T) { + registries, err := getUserRegistries(store, admin.Username, portainer.EndpointID(endpointID)) + assert.NoError(t, err) + assert.ElementsMatch(t, []portainer.Registry{registryReachableByUser, registryReachableByTeam, registryRestricted}, registries) + }) + + t.Run("regular user has access to registries allowed to him and/or his team", func(t *testing.T) { + registries, err := getUserRegistries(store, user.Username, portainer.EndpointID(endpointID)) + assert.NoError(t, err) + assert.ElementsMatch(t, []portainer.Registry{registryReachableByUser, registryReachableByTeam}, registries) + }) +} diff --git a/api/stacks/deployer.go b/api/stacks/deployer.go new file mode 100644 index 000000000..d38c50cbc --- /dev/null +++ b/api/stacks/deployer.go @@ -0,0 +1,46 @@ +package stacks + +import ( + "sync" + + portainer "github.com/portainer/portainer/api" +) + +type StackDeployer interface { + DeploySwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, prune bool) error + DeployComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry) error +} + +type stackDeployer struct { + lock *sync.Mutex + swarmStackManager portainer.SwarmStackManager + composeStackManager portainer.ComposeStackManager +} + +func NewStackDeployer(swarmStackManager portainer.SwarmStackManager, composeStackManager portainer.ComposeStackManager) *stackDeployer { + return &stackDeployer{ + lock: &sync.Mutex{}, + swarmStackManager: swarmStackManager, + composeStackManager: composeStackManager, + } +} + +func (d *stackDeployer) DeploySwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, prune bool) error { + d.lock.Lock() + defer d.lock.Unlock() + + d.swarmStackManager.Login(registries, endpoint) + defer d.swarmStackManager.Logout(endpoint) + + return d.swarmStackManager.Deploy(stack, prune, endpoint) +} + +func (d *stackDeployer) DeployComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry) error { + d.lock.Lock() + defer d.lock.Unlock() + + d.swarmStackManager.Login(registries, endpoint) + defer d.swarmStackManager.Logout(endpoint) + + return d.composeStackManager.Up(stack, endpoint) +} diff --git a/api/stacks/scheduled.go b/api/stacks/scheduled.go new file mode 100644 index 000000000..fb90ca22c --- /dev/null +++ b/api/stacks/scheduled.go @@ -0,0 +1,34 @@ +package stacks + +import ( + "log" + "time" + + "github.com/pkg/errors" + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/scheduler" +) + +func StartStackSchedules(scheduler *scheduler.Scheduler, stackdeployer StackDeployer, datastore portainer.DataStore, gitService portainer.GitService) error { + stacks, err := datastore.Stack().RefreshableStacks() + if err != nil { + return errors.Wrap(err, "failed to fetch refreshable stacks") + } + for _, stack := range stacks { + d, err := time.ParseDuration(stack.AutoUpdate.Interval) + if err != nil { + return errors.Wrap(err, "Unable to parse auto update interval") + } + jobID := scheduler.StartJobEvery(d, func() { + if err := RedeployWhenChanged(stack.ID, stackdeployer, datastore, gitService); err != nil { + log.Printf("[ERROR] %s\n", err) + } + }) + + stack.AutoUpdate.JobID = jobID + if err := datastore.Stack().UpdateStack(stack.ID, &stack); err != nil { + return errors.Wrap(err, "failed to update stack job id") + } + } + return nil +} diff --git a/app/docker/components/datatables/containers-datatable/containersDatatable.html b/app/docker/components/datatables/containers-datatable/containersDatatable.html index 3299d8c87..5cd726759 100644 --- a/app/docker/components/datatables/containers-datatable/containersDatatable.html +++ b/app/docker/components/datatables/containers-datatable/containersDatatable.html @@ -92,7 +92,7 @@ ng-model="$ctrl.state.textFilter" ng-change="$ctrl.onTextFilterChange()" placeholder="Search..." - auto-focus + focus-if="!$ctrl.notAutoFocus" ng-model-options="{ debounce: 300 }" /> diff --git a/app/docker/components/datatables/containers-datatable/containersDatatable.js b/app/docker/components/datatables/containers-datatable/containersDatatable.js index 0edc97de0..d748c27aa 100644 --- a/app/docker/components/datatables/containers-datatable/containersDatatable.js +++ b/app/docker/components/datatables/containers-datatable/containersDatatable.js @@ -12,5 +12,6 @@ angular.module('portainer.docker').component('containersDatatable', { showAddAction: '<', offlineMode: '<', refreshCallback: '<', + notAutoFocus: '<', }, }); diff --git a/app/docker/components/datatables/services-datatable/servicesDatatable.html b/app/docker/components/datatables/services-datatable/servicesDatatable.html index 3e72d09f4..cd7bb808e 100644 --- a/app/docker/components/datatables/services-datatable/servicesDatatable.html +++ b/app/docker/components/datatables/services-datatable/servicesDatatable.html @@ -56,7 +56,7 @@ ng-model="$ctrl.state.textFilter" ng-change="$ctrl.onTextFilterChange()" placeholder="Search..." - auto-focus + focus-if="!$ctrl.notAutoFocus" ng-model-options="{ debounce: 300 }" /> diff --git a/app/docker/components/datatables/services-datatable/servicesDatatable.js b/app/docker/components/datatables/services-datatable/servicesDatatable.js index 6e5bbfb77..0354eab2f 100644 --- a/app/docker/components/datatables/services-datatable/servicesDatatable.js +++ b/app/docker/components/datatables/services-datatable/servicesDatatable.js @@ -15,5 +15,6 @@ angular.module('portainer.docker').component('servicesDatatable', { showStackColumn: '<', showTaskLogsButton: '<', refreshCallback: '<', + notAutoFocus: '<', }, }); diff --git a/app/portainer/components/focusIf.js b/app/portainer/components/focusIf.js new file mode 100644 index 000000000..29c76b373 --- /dev/null +++ b/app/portainer/components/focusIf.js @@ -0,0 +1,22 @@ +import angular from 'angular'; +// ng-focus-if pkg from: https://github.com/hiebj/ng-focus-if +angular.module('portainer.app').directive('focusIf', function ($timeout) { + return { + restrict: 'A', + link: function ($scope, $element, $attrs) { + var dom = $element[0]; + if ($attrs.focusIf) { + $scope.$watch($attrs.focusIf, focus); + } else { + focus(true); + } + function focus(condition) { + if (condition) { + $timeout(function () { + dom.focus(); + }, $scope.$eval($attrs.focusDelay) || 0); + } + } + }, + }; +}); diff --git a/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-file-item/git-form-additional-file-item.controller.js b/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-file-item/git-form-additional-file-item.controller.js new file mode 100644 index 000000000..f712c9b61 --- /dev/null +++ b/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-file-item/git-form-additional-file-item.controller.js @@ -0,0 +1,22 @@ +class GitFormAdditionalFileItemController { + /* @ngInject */ + constructor() {} + + onChangePath(value) { + const fieldIsInvalid = typeof value === 'undefined'; + if (fieldIsInvalid) { + return; + } + this.onChange(this.index, { value }); + } + + removeValue() { + this.onChange(this.index); + } + + $onInit() { + this.formName = `variableForm${this.index}`; + } +} + +export default GitFormAdditionalFileItemController; diff --git a/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-file-item/git-form-additional-file-item.html b/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-file-item/git-form-additional-file-item.html new file mode 100644 index 000000000..3796ac4eb --- /dev/null +++ b/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-file-item/git-form-additional-file-item.html @@ -0,0 +1,20 @@ + +

+
+
+ path + +
+ +
+
+
+
+

Path is required.

+
+
+
+
+ diff --git a/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-file-item/index.js b/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-file-item/index.js new file mode 100644 index 000000000..f50efd346 --- /dev/null +++ b/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-file-item/index.js @@ -0,0 +1,14 @@ +import controller from './git-form-additional-file-item.controller.js'; + +export const gitFormAdditionalFileItem = { + templateUrl: './git-form-additional-file-item.html', + controller, + + bindings: { + variable: '<', + index: '<', + + onChange: '<', + onRemove: '<', + }, +}; diff --git a/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-files-panel.controller.js b/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-files-panel.controller.js new file mode 100644 index 000000000..8f3d4c22b --- /dev/null +++ b/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-files-panel.controller.js @@ -0,0 +1,26 @@ +class GitFormAutoUpdateFieldsetController { + /* @ngInject */ + constructor() { + this.add = this.add.bind(this); + this.onChangeVariable = this.onChangeVariable.bind(this); + } + + add() { + this.model.AdditionalFiles.push(''); + } + + onChangeVariable(index, variable) { + if (!variable) { + this.model.AdditionalFiles.splice(index, 1); + } else { + this.model.AdditionalFiles[index] = variable.value; + } + + this.onChange({ + ...this.model, + AdditionalFiles: this.model.AdditionalFiles, + }); + } +} + +export default GitFormAutoUpdateFieldsetController; diff --git a/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-files-panel.html b/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-files-panel.html new file mode 100644 index 000000000..b012c4a5d --- /dev/null +++ b/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-files-panel.html @@ -0,0 +1,14 @@ +
+
+ + add file +
+
+ +
+
diff --git a/app/portainer/components/forms/git-form/git-form-additional-files-panel/index.js b/app/portainer/components/forms/git-form/git-form-additional-files-panel/index.js new file mode 100644 index 000000000..d6d525438 --- /dev/null +++ b/app/portainer/components/forms/git-form/git-form-additional-files-panel/index.js @@ -0,0 +1,10 @@ +import controller from './git-form-additional-files-panel.controller.js'; + +export const gitFormAdditionalFilesPanel = { + templateUrl: './git-form-additional-files-panel.html', + controller, + bindings: { + model: '<', + onChange: '<', + }, +}; diff --git a/app/portainer/components/forms/git-form/git-form-auth-fieldset/git-form-auth-fieldset.html b/app/portainer/components/forms/git-form/git-form-auth-fieldset/git-form-auth-fieldset.html index b884c248d..7f820783c 100644 --- a/app/portainer/components/forms/git-form/git-form-auth-fieldset/git-form-auth-fieldset.html +++ b/app/portainer/components/forms/git-form/git-form-auth-fieldset/git-form-auth-fieldset.html @@ -3,36 +3,38 @@ +
+ + Enabling authentication will store the credentials and it is advisable to use a git service account +
- - If your git account has 2FA enabled, you may receive an authentication required error when deploying your stack. In this case, you will need to provide a - personal-access token instead of your password. - -
-
- -
+ +
-
+
+ -
+
diff --git a/app/portainer/components/forms/git-form/git-form-auth-fieldset/index.js b/app/portainer/components/forms/git-form/git-form-auth-fieldset/index.js index 5c2373387..3000869c3 100644 --- a/app/portainer/components/forms/git-form/git-form-auth-fieldset/index.js +++ b/app/portainer/components/forms/git-form/git-form-auth-fieldset/index.js @@ -6,5 +6,6 @@ export const gitFormAuthFieldset = { bindings: { model: '<', onChange: '<', + isEdit: '<', }, }; diff --git a/app/portainer/components/forms/git-form/git-form-auto-update-fieldset/git-form-auto-update-fieldset.controller.js b/app/portainer/components/forms/git-form/git-form-auto-update-fieldset/git-form-auto-update-fieldset.controller.js new file mode 100644 index 000000000..9025ab0df --- /dev/null +++ b/app/portainer/components/forms/git-form/git-form-auto-update-fieldset/git-form-auto-update-fieldset.controller.js @@ -0,0 +1,26 @@ +class GitFormAutoUpdateFieldsetController { + /* @ngInject */ + constructor(clipboard) { + this.onChangeAutoUpdate = this.onChangeField('RepositoryAutomaticUpdates'); + this.onChangeMechanism = this.onChangeField('RepositoryMechanism'); + this.onChangeInterval = this.onChangeField('RepositoryFetchInterval'); + this.clipboard = clipboard; + } + + copyWebhook() { + this.clipboard.copyText(this.model.RepositoryWebhookURL); + $('#copyNotification').show(); + $('#copyNotification').fadeOut(2000); + } + + onChangeField(field) { + return (value) => { + this.onChange({ + ...this.model, + [field]: value, + }); + }; + } +} + +export default GitFormAutoUpdateFieldsetController; diff --git a/app/portainer/components/forms/git-form/git-form-auto-update-fieldset/git-form-auto-update-fieldset.html b/app/portainer/components/forms/git-form/git-form-auto-update-fieldset/git-form-auto-update-fieldset.html new file mode 100644 index 000000000..000b68e0f --- /dev/null +++ b/app/portainer/components/forms/git-form/git-form-auto-update-fieldset/git-form-auto-update-fieldset.html @@ -0,0 +1,69 @@ + +
+
+ +
+
+
+ + Any changes to this stack made locally in Portainer will be overriden by the definition in git and may cause service interruption. +
+
+ +
+
+
+ + +
+
+
+
+ +
+ +
+ {{ $ctrl.model.RepositoryWebhookURL | truncatelr }} + + + + +
+
+
+ +
+ +
+
+
+
+
+

This field is required.

+

Please enter a valid time interval.

+

Minimum interval is 1m

+
+
+
+ diff --git a/app/portainer/components/forms/git-form/git-form-auto-update-fieldset/index.js b/app/portainer/components/forms/git-form/git-form-auto-update-fieldset/index.js new file mode 100644 index 000000000..237cb6006 --- /dev/null +++ b/app/portainer/components/forms/git-form/git-form-auto-update-fieldset/index.js @@ -0,0 +1,10 @@ +import controller from './git-form-auto-update-fieldset.controller.js'; + +export const gitFormAutoUpdateFieldset = { + templateUrl: './git-form-auto-update-fieldset.html', + controller, + bindings: { + model: '<', + onChange: '<', + }, +}; diff --git a/app/portainer/components/forms/git-form/git-form-ref-field/git-form-ref-field.html b/app/portainer/components/forms/git-form/git-form-ref-field/git-form-ref-field.html index a12639d24..86164f650 100644 --- a/app/portainer/components/forms/git-form/git-form-ref-field/git-form-ref-field.html +++ b/app/portainer/components/forms/git-form/git-form-ref-field/git-form-ref-field.html @@ -5,8 +5,8 @@
- -
+ +
diff --git a/app/portainer/components/forms/git-form/git-form.html b/app/portainer/components/forms/git-form/git-form.html index 16b131008..ba35fcf67 100644 --- a/app/portainer/components/forms/git-form/git-form.html +++ b/app/portainer/components/forms/git-form/git-form.html @@ -5,5 +5,7 @@ - + + +
diff --git a/app/portainer/components/forms/git-form/git-form.js b/app/portainer/components/forms/git-form/git-form.js index 4a1c5733e..affa081dc 100644 --- a/app/portainer/components/forms/git-form/git-form.js +++ b/app/portainer/components/forms/git-form/git-form.js @@ -6,5 +6,8 @@ export const gitForm = { bindings: { model: '<', onChange: '<', + additionalFile: '<', + autoUpdate: '<', + showAuthExplanation: '<', }, }; diff --git a/app/portainer/components/forms/git-form/index.js b/app/portainer/components/forms/git-form/index.js index 323c1d271..60eff71e6 100644 --- a/app/portainer/components/forms/git-form/index.js +++ b/app/portainer/components/forms/git-form/index.js @@ -2,6 +2,9 @@ import angular from 'angular'; import { gitForm } from './git-form'; import { gitFormAuthFieldset } from './git-form-auth-fieldset'; +import { gitFormAdditionalFilesPanel } from './git-form-additional-files-panel'; +import { gitFormAdditionalFileItem } from './/git-form-additional-files-panel/git-form-additional-file-item'; +import { gitFormAutoUpdateFieldset } from './git-form-auto-update-fieldset'; import { gitFormComposePathField } from './git-form-compose-path-field'; import { gitFormRefField } from './git-form-ref-field'; import { gitFormUrlField } from './git-form-url-field'; @@ -12,4 +15,7 @@ export default angular .component('gitFormRefField', gitFormRefField) .component('gitForm', gitForm) .component('gitFormUrlField', gitFormUrlField) + .component('gitFormAdditionalFilesPanel', gitFormAdditionalFilesPanel) + .component('gitFormAdditionalFileItem', gitFormAdditionalFileItem) + .component('gitFormAutoUpdateFieldset', gitFormAutoUpdateFieldset) .component('gitFormAuthFieldset', gitFormAuthFieldset).name; diff --git a/app/portainer/components/forms/stack-redeploy-git-form/stack-redeploy-git-form.controller.js b/app/portainer/components/forms/stack-redeploy-git-form/stack-redeploy-git-form.controller.js index f489cc2d9..f4179817d 100644 --- a/app/portainer/components/forms/stack-redeploy-git-form/stack-redeploy-git-form.controller.js +++ b/app/portainer/components/forms/stack-redeploy-git-form/stack-redeploy-git-form.controller.js @@ -1,14 +1,20 @@ +import uuidv4 from 'uuid/v4'; class StackRedeployGitFormController { /* @ngInject */ - constructor($async, $state, StackService, ModalService, Notifications) { + constructor($async, $state, StackService, ModalService, Notifications, WebhookHelper, FormHelper) { this.$async = $async; this.$state = $state; this.StackService = StackService; this.ModalService = ModalService; this.Notifications = Notifications; + this.WebhookHelper = WebhookHelper; + this.FormHelper = FormHelper; this.state = { inProgress: false, + redeployInProgress: false, + showConfig: false, + isEdit: false, }; this.formValues = { @@ -16,10 +22,19 @@ class StackRedeployGitFormController { RepositoryAuthentication: false, RepositoryUsername: '', RepositoryPassword: '', + Env: [], + // auto upadte + AutoUpdate: { + RepositoryAutomaticUpdates: false, + RepositoryMechanism: 'Interval', + RepositoryFetchInterval: '5m', + RepositoryWebhookURL: '', + }, }; this.onChange = this.onChange.bind(this); this.onChangeRef = this.onChangeRef.bind(this); + this.handleEnvVarChange = this.handleEnvVarChange.bind(this); } onChangeRef(value) { @@ -50,13 +65,27 @@ class StackRedeployGitFormController { return; } - this.state.inProgress = true; + this.state.redeployInProgress = true; - await this.StackService.updateGit(this.stack.Id, this.stack.EndpointId, [], false, this.formValues); + await this.StackService.updateGit(this.stack.Id, this.stack.EndpointId, this.FormHelper.removeInvalidEnvVars(this.formValues.Env), false, this.formValues); await this.$state.reload(); } catch (err) { this.Notifications.error('Failure', err, 'Failed redeploying stack'); + } finally { + this.state.redeployInProgress = false; + } + }); + } + + async saveGitSettings() { + return this.$async(async () => { + try { + this.state.inProgress = true; + await this.StackService.updateGitStackSettings(this.stack.Id, this.stack.EndpointId, this.FormHelper.removeInvalidEnvVars(this.formValues.Env), this.formValues); + this.Notifications.success('Save stack settings successfully'); + } catch (err) { + this.Notifications.error('Failure', err, 'Unable to save stack settings'); } finally { this.state.inProgress = false; } @@ -64,11 +93,38 @@ class StackRedeployGitFormController { } isSubmitButtonDisabled() { - return this.state.inProgress; + return this.state.inProgress || this.state.redeployInProgress; + } + + handleEnvVarChange(value) { + this.formValues.Env = value; } $onInit() { this.formValues.RefName = this.model.ReferenceName; + this.formValues.Env = this.stack.Env; + // Init auto update + if (this.stack.AutoUpdate && (this.stack.AutoUpdate.Interval || this.stack.AutoUpdate.Webhook)) { + this.formValues.AutoUpdate.RepositoryAutomaticUpdates = true; + + if (this.stack.AutoUpdate.Interval) { + this.formValues.AutoUpdate.RepositoryMechanism = `Interval`; + this.formValues.AutoUpdate.RepositoryFetchInterval = this.stack.AutoUpdate.Interval; + } else if (this.stack.AutoUpdate.Webhook) { + this.formValues.AutoUpdate.RepositoryMechanism = `Webhook`; + this.formValues.AutoUpdate.RepositoryWebhookURL = this.WebhookHelper.returnStackWebhookUrl(this.stack.AutoUpdate.Webhook); + } + } + + if (!this.formValues.AutoUpdate.RepositoryWebhookURL) { + this.formValues.AutoUpdate.RepositoryWebhookURL = this.WebhookHelper.returnStackWebhookUrl(uuidv4()); + } + + if (this.stack.GitConfig && this.stack.GitConfig.Authentication) { + this.formValues.RepositoryUsername = this.stack.GitConfig.Authentication.Username; + this.formValues.RepositoryAuthentication = true; + this.state.isEdit = true; + } } } diff --git a/app/portainer/components/forms/stack-redeploy-git-form/stack-redeploy-git-form.html b/app/portainer/components/forms/stack-redeploy-git-form/stack-redeploy-git-form.html index 5b5efc42b..340400573 100644 --- a/app/portainer/components/forms/stack-redeploy-git-form/stack-redeploy-git-form.html +++ b/app/portainer/components/forms/stack-redeploy-git-form/stack-redeploy-git-form.html @@ -9,22 +9,62 @@ .

- Update {{ $ctrl.model.ConfigFilePath }} in git and pull from here to update the stack. + Update + {{ $ctrl.model.ConfigFilePath }},{{ $ctrl.stack.AdditionalFiles.join(',') }} + in git and pull from here to update the stack.

- - + +
+
+

+ + + {{ $ctrl.state.showConfig ? 'Hide' : 'Advanced' }} configuration + +

+
+
+ + + + + diff --git a/app/portainer/components/intervalFormat.js b/app/portainer/components/intervalFormat.js new file mode 100644 index 000000000..73126928e --- /dev/null +++ b/app/portainer/components/intervalFormat.js @@ -0,0 +1,26 @@ +import angular from 'angular'; +import parse from 'parse-duration'; + +angular.module('portainer.app').directive('intervalFormat', function () { + return { + restrict: 'A', + require: 'ngModel', + link: function ($scope, $element, $attrs, ngModel) { + ngModel.$validators.invalidIntervalFormat = function (modelValue) { + try { + return modelValue && modelValue.toUpperCase().match(/^P?(?!$)(\d+Y)?(\d+M)?(\d+W)?(\d+D)?(T?(?=\d+[HMS])(\d+H)?(\d+M)?(\d+S)?)?$/gm) !== null; + } catch (error) { + return false; + } + }; + + ngModel.$validators.minimumInterval = function (modelValue) { + try { + return modelValue && parse(modelValue, 'minute') >= 1; + } catch (error) { + return false; + } + }; + }, + }; +}); diff --git a/app/portainer/helpers/webhookHelper.js b/app/portainer/helpers/webhookHelper.js index 8411544fd..69a487c0e 100644 --- a/app/portainer/helpers/webhookHelper.js +++ b/app/portainer/helpers/webhookHelper.js @@ -1,16 +1,21 @@ angular.module('portainer.app').factory('WebhookHelper', [ '$location', 'API_ENDPOINT_WEBHOOKS', - function WebhookHelperFactory($location, API_ENDPOINT_WEBHOOKS) { + 'API_ENDPOINT_STACKS', + function WebhookHelperFactory($location, API_ENDPOINT_WEBHOOKS, API_ENDPOINT_STACKS) { 'use strict'; + var helper = {}; + const protocol = $location.protocol().toLowerCase(); + const port = $location.port(); + const displayPort = (protocol === 'http' && port === 80) || (protocol === 'https' && port === 443) ? '' : ':' + port; helper.returnWebhookUrl = function (token) { - var displayPort = - ($location.protocol().toLowerCase() === 'http' && $location.port() === 80) || ($location.protocol().toLowerCase() === 'https' && $location.port() === 443) - ? '' - : ':' + $location.port(); - return $location.protocol() + '://' + $location.host() + displayPort + '/' + API_ENDPOINT_WEBHOOKS + '/' + token; + return `${protocol}://${$location.host()}${displayPort}/${API_ENDPOINT_WEBHOOKS}/${token}`; + }; + + helper.returnStackWebhookUrl = function (token) { + return `${protocol}://${$location.host()}${displayPort}/${API_ENDPOINT_STACKS}/webhooks/${token}`; }; return helper; diff --git a/app/portainer/models/stack.js b/app/portainer/models/stack.js index 1b11fa432..41c32f70f 100644 --- a/app/portainer/models/stack.js +++ b/app/portainer/models/stack.js @@ -20,6 +20,8 @@ export function StackViewModel(data) { this.Orphaned = false; this.Checked = false; this.GitConfig = data.GitConfig; + this.AdditionalFiles = data.AdditionalFiles; + this.AutoUpdate = data.AutoUpdate; } export function ExternalStackViewModel(name, type, creationDate) { diff --git a/app/portainer/rest/stack.js b/app/portainer/rest/stack.js index 6327fc647..d41bdf04a 100644 --- a/app/portainer/rest/stack.js +++ b/app/portainer/rest/stack.js @@ -5,7 +5,7 @@ angular.module('portainer.app').factory('Stack', [ function StackFactory($resource, EndpointProvider, API_ENDPOINT_STACKS) { 'use strict'; return $resource( - API_ENDPOINT_STACKS + '/:id/:action', + API_ENDPOINT_STACKS + '/:id/:action/:subaction', {}, { get: { method: 'GET', params: { id: '@id' } }, @@ -18,7 +18,8 @@ angular.module('portainer.app').factory('Stack', [ migrate: { method: 'POST', params: { id: '@id', action: 'migrate', endpointId: '@endpointId' }, ignoreLoadingBar: true }, start: { method: 'POST', params: { id: '@id', action: 'start' } }, stop: { method: 'POST', params: { id: '@id', action: 'stop' } }, - updateGit: { method: 'PUT', params: { action: 'git' } }, + updateGit: { method: 'PUT', params: { id: '@id', action: 'git', subaction: 'redeploy' } }, + updateGitStackSettings: { method: 'POST', params: { id: '@id', action: 'git' }, ignoreLoadingBar: true }, } ); }, diff --git a/app/portainer/services/api/stackService.js b/app/portainer/services/api/stackService.js index a26c6c98b..86ad1905a 100644 --- a/app/portainer/services/api/stackService.js +++ b/app/portainer/services/api/stackService.js @@ -326,12 +326,18 @@ angular.module('portainer.app').factory('StackService', [ Name: name, RepositoryURL: repositoryOptions.RepositoryURL, RepositoryReferenceName: repositoryOptions.RepositoryReferenceName, - ComposeFilePathInRepository: repositoryOptions.ComposeFilePathInRepository, + ComposeFile: repositoryOptions.ComposeFilePathInRepository, + AdditionalFiles: repositoryOptions.AdditionalFiles, RepositoryAuthentication: repositoryOptions.RepositoryAuthentication, RepositoryUsername: repositoryOptions.RepositoryUsername, RepositoryPassword: repositoryOptions.RepositoryPassword, Env: env, }; + + if (repositoryOptions.AutoUpdate) { + payload.AutoUpdate = repositoryOptions.AutoUpdate; + } + return Stack.create({ method: 'repository', type: 2, endpointId: endpointId }, payload).$promise; }; @@ -346,12 +352,18 @@ angular.module('portainer.app').factory('StackService', [ SwarmID: swarm.Id, RepositoryURL: repositoryOptions.RepositoryURL, RepositoryReferenceName: repositoryOptions.RepositoryReferenceName, - ComposeFilePathInRepository: repositoryOptions.ComposeFilePathInRepository, + ComposeFile: repositoryOptions.ComposeFilePathInRepository, + AdditionalFiles: repositoryOptions.AdditionalFiles, RepositoryAuthentication: repositoryOptions.RepositoryAuthentication, RepositoryUsername: repositoryOptions.RepositoryUsername, RepositoryPassword: repositoryOptions.RepositoryPassword, Env: env, }; + + if (repositoryOptions.AutoUpdate) { + payload.AutoUpdate = repositoryOptions.AutoUpdate; + } + return Stack.create({ method: 'repository', type: 1, endpointId: endpointId }, payload).$promise; }) .then(function success(data) { @@ -405,6 +417,31 @@ angular.module('portainer.app').factory('StackService', [ ).$promise; } + service.updateGitStackSettings = function (id, endpointId, env, gitConfig) { + // prepare auto update + const autoUpdate = {}; + + if (gitConfig.AutoUpdate.RepositoryAutomaticUpdates) { + if (gitConfig.AutoUpdate.RepositoryMechanism === 'Interval') { + autoUpdate.Interval = gitConfig.AutoUpdate.RepositoryFetchInterval; + } else if (gitConfig.AutoUpdate.RepositoryMechanism === 'Webhook') { + autoUpdate.Webhook = gitConfig.AutoUpdate.RepositoryWebhookURL.split('/').reverse()[0]; + } + } + + return Stack.updateGitStackSettings( + { endpointId, id }, + { + AutoUpdate: autoUpdate, + Env: env, + RepositoryReferenceName: gitConfig.RefName, + RepositoryAuthentication: gitConfig.RepositoryAuthentication, + RepositoryUsername: gitConfig.RepositoryUsername, + RepositoryPassword: gitConfig.RepositoryPassword, + } + ).$promise; + }; + return service; }, ]); diff --git a/app/portainer/views/stacks/create/createStackController.js b/app/portainer/views/stacks/create/createStackController.js index 2b42600b2..168179142 100644 --- a/app/portainer/views/stacks/create/createStackController.js +++ b/app/portainer/views/stacks/create/createStackController.js @@ -1,6 +1,6 @@ import angular from 'angular'; import _ from 'lodash-es'; - +import uuidv4 from 'uuid/v4'; import { AccessControlFormData } from '../../../components/accessControlForm/porAccessControlFormModel'; angular @@ -21,7 +21,9 @@ angular StackHelper, ContainerHelper, CustomTemplateService, - ContainerService + ContainerService, + WebhookHelper, + clipboard ) { $scope.formValues = { Name: '', @@ -33,8 +35,13 @@ angular RepositoryUsername: '', RepositoryPassword: '', Env: [], + AdditionalFiles: [], ComposeFilePathInRepository: 'docker-compose.yml', AccessControlData: new AccessControlFormData(), + RepositoryAutomaticUpdates: true, + RepositoryMechanism: 'Interval', + RepositoryFetchInterval: '5m', + RepositoryWebhookURL: WebhookHelper.returnStackWebhookUrl(uuidv4()), }; $scope.state = { @@ -67,6 +74,14 @@ angular $scope.formValues.Env.splice(index, 1); }; + $scope.addAdditionalFiles = function () { + $scope.formValues.AdditionalFiles.push(''); + }; + + $scope.removeAdditionalFiles = function (index) { + $scope.formValues.AdditionalFiles.splice(index, 1); + }; + function validateForm(accessControlData, isAdmin) { $scope.state.formValidationError = ''; var error = ''; @@ -95,6 +110,7 @@ angular if (method === 'repository') { var repositoryOptions = { + AdditionalFiles: $scope.formValues.AdditionalFiles, RepositoryURL: $scope.formValues.RepositoryURL, RepositoryReferenceName: $scope.formValues.RepositoryReferenceName, ComposeFilePathInRepository: $scope.formValues.ComposeFilePathInRepository, @@ -102,10 +118,24 @@ angular RepositoryUsername: $scope.formValues.RepositoryUsername, RepositoryPassword: $scope.formValues.RepositoryPassword, }; + + getAutoUpdatesProperty(repositoryOptions); + return StackService.createSwarmStackFromGitRepository(name, repositoryOptions, env, endpointId); } } + function getAutoUpdatesProperty(repositoryOptions) { + if ($scope.formValues.RepositoryAutomaticUpdates) { + repositoryOptions.AutoUpdate = {}; + if ($scope.formValues.RepositoryMechanism === 'Interval') { + repositoryOptions.AutoUpdate.Interval = $scope.formValues.RepositoryFetchInterval; + } else if ($scope.formValues.RepositoryMechanism === 'Webhook') { + repositoryOptions.AutoUpdate.Webhook = $scope.formValues.RepositoryWebhookURL.split('/').reverse()[0]; + } + } + } + function createComposeStack(name, method) { var env = FormHelper.removeInvalidEnvVars($scope.formValues.Env); const endpointId = +$state.params.endpointId; @@ -118,6 +148,7 @@ angular return StackService.createComposeStackFromFileUpload(name, stackFile, env, endpointId); } else if (method === 'repository') { var repositoryOptions = { + AdditionalFiles: $scope.formValues.AdditionalFiles, RepositoryURL: $scope.formValues.RepositoryURL, RepositoryReferenceName: $scope.formValues.RepositoryReferenceName, ComposeFilePathInRepository: $scope.formValues.ComposeFilePathInRepository, @@ -125,10 +156,19 @@ angular RepositoryUsername: $scope.formValues.RepositoryUsername, RepositoryPassword: $scope.formValues.RepositoryPassword, }; + + getAutoUpdatesProperty(repositoryOptions); + return StackService.createComposeStackFromGitRepository(name, repositoryOptions, env, endpointId); } } + $scope.copyWebhook = function () { + clipboard.copyText($scope.formValues.RepositoryWebhookURL); + $('#copyNotification').show(); + $('#copyNotification').fadeOut(2000); + }; + $scope.handleEnvVarChange = handleEnvVarChange; function handleEnvVarChange(value) { $scope.formValues.Env = value; diff --git a/app/portainer/views/stacks/create/createstack.html b/app/portainer/views/stacks/create/createstack.html index 8e545b7e3..abb4c935b 100644 --- a/app/portainer/views/stacks/create/createstack.html +++ b/app/portainer/views/stacks/create/createstack.html @@ -130,7 +130,14 @@ - +
@@ -207,7 +214,7 @@ || (state.Method === 'editor' && (!formValues.StackFileContent || state.editorYamlValidationError)) || (state.Method === 'upload' && (!formValues.StackFile || state.uploadYamlValidationError)) || (state.Method === 'template' && (!formValues.StackFileContent || !selectedTemplate || state.editorYamlValidationError)) - || (state.Method === 'repository' && ((!formValues.RepositoryURL || !formValues.ComposeFilePathInRepository) || (formValues.RepositoryAuthentication && (!formValues.RepositoryUsername || !formValues.RepositoryPassword)))) + || (state.Method === 'repository' && ((!formValues.RepositoryURL || !formValues.ComposeFilePathInRepository) || (formValues.RepositoryAuthentication && !formValues.RepositoryPassword))) || !formValues.Name" ng-click="deployStack()" button-spinner="state.actionInProgress" diff --git a/app/portainer/views/stacks/edit/stack.html b/app/portainer/views/stacks/edit/stack.html index 88ae401b8..57fdc69f6 100644 --- a/app/portainer/views/stacks/edit/stack.html +++ b/app/portainer/views/stacks/edit/stack.html @@ -125,7 +125,7 @@ - + Editor
@@ -214,6 +214,7 @@ order-by="Status" show-host-column="false" show-add-action="false" + not-auto-focus="true" >
@@ -233,6 +234,7 @@ show-task-logs-button="applicationState.endpoint.apiVersion >= 1.30" show-add-action="false" show-stack-column="false" + not-auto-focus="true" >
diff --git a/package.json b/package.json index 56c9858c6..56f9cda6e 100644 --- a/package.json +++ b/package.json @@ -91,9 +91,10 @@ "jquery": "^3.5.1", "js-base64": "^3.6.0", "js-yaml": "^3.14.0", - "lodash-es": "^4.17.15", + "lodash-es": "^4.17.15", "moment": "^2.21.0", "ng-file-upload": "~12.2.13", + "parse-duration": "^1.0.0", "source-map-loader": "^1.1.2", "spinkit": "^2.0.1", "splitargs": "github:deviantony/splitargs#semver:~0.2.0", diff --git a/yarn.lock b/yarn.lock index 2f6daba4d..62b8a406f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7967,6 +7967,11 @@ parse-asn1@^5.0.0, parse-asn1@^5.1.5: pbkdf2 "^3.0.3" safe-buffer "^5.1.1" +parse-duration@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/parse-duration/-/parse-duration-1.0.0.tgz#8605651745f61088f6fb14045c887526c291858c" + integrity sha512-X4kUkCTHU1N/kEbwK9FpUJ0UZQa90VzeczfS704frR30gljxDG0pSziws06XlK+CGRSo/1wtG1mFIdBFQTMQNw== + parse-filepath@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/parse-filepath/-/parse-filepath-1.0.2.tgz#a632127f53aaf3d15876f5872f3ffac763d6c891"