diff --git a/api/backup/backup.go b/api/backup/backup.go index 3da032c2c..0a62786f5 100644 --- a/api/backup/backup.go +++ b/api/backup/backup.go @@ -10,6 +10,7 @@ import ( portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/archive" "github.com/portainer/portainer/api/crypto" + "github.com/portainer/portainer/api/filesystem" "github.com/portainer/portainer/api/http/offlinegate" ) @@ -32,7 +33,7 @@ func CreateBackupArchive(password string, gate *offlinegate.OfflineGate, datasto } for _, filename := range filesToBackup { - err := copyPath(filepath.Join(filestorePath, filename), backupDirPath) + err := filesystem.CopyPath(filepath.Join(filestorePath, filename), backupDirPath) if err != nil { return "", errors.Wrap(err, "Failed to create backup file") } diff --git a/api/backup/copy_test.go b/api/backup/copy_test.go deleted file mode 100644 index b9ceaeaab..000000000 --- a/api/backup/copy_test.go +++ /dev/null @@ -1,105 +0,0 @@ -package backup - -import ( - "io/ioutil" - "os" - "path" - "path/filepath" - "testing" - - "github.com/docker/docker/pkg/ioutils" - "github.com/stretchr/testify/assert" -) - -func listFiles(dir string) []string { - items := make([]string, 0) - filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if path == dir { - return nil - } - items = append(items, path) - return nil - }) - - return items -} - -func contains(t *testing.T, list []string, path string) { - assert.Contains(t, list, path) - copyContent, _ := ioutil.ReadFile(path) - assert.Equal(t, "content\n", string(copyContent)) -} - -func Test_copyFile_returnsError_whenSourceDoesNotExist(t *testing.T) { - tmpdir, _ := ioutils.TempDir("", "backup") - defer os.RemoveAll(tmpdir) - - err := copyFile("does-not-exist", tmpdir) - assert.NotNil(t, err) -} - -func Test_copyFile_shouldMakeAbackup(t *testing.T) { - tmpdir, _ := ioutils.TempDir("", "backup") - defer os.RemoveAll(tmpdir) - - content := []byte("content") - ioutil.WriteFile(path.Join(tmpdir, "origin"), content, 0600) - - err := copyFile(path.Join(tmpdir, "origin"), path.Join(tmpdir, "copy")) - assert.Nil(t, err) - - copyContent, _ := ioutil.ReadFile(path.Join(tmpdir, "copy")) - assert.Equal(t, content, copyContent) -} - -func Test_copyDir_shouldCopyAllFilesAndDirectories(t *testing.T) { - destination, _ := ioutils.TempDir("", "destination") - defer os.RemoveAll(destination) - err := copyDir("./test_assets/copy_test", destination) - assert.Nil(t, err) - - createdFiles := listFiles(destination) - - contains(t, createdFiles, filepath.Join(destination, "copy_test", "outer")) - contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", ".dotfile")) - contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", "inner")) -} - -func Test_backupPath_shouldSkipWhenNotExist(t *testing.T) { - tmpdir, _ := ioutils.TempDir("", "backup") - defer os.RemoveAll(tmpdir) - - err := copyPath("does-not-exists", tmpdir) - assert.Nil(t, err) - - assert.Empty(t, listFiles(tmpdir)) -} - -func Test_backupPath_shouldCopyFile(t *testing.T) { - tmpdir, _ := ioutils.TempDir("", "backup") - defer os.RemoveAll(tmpdir) - - content := []byte("content") - ioutil.WriteFile(path.Join(tmpdir, "file"), content, 0600) - - os.MkdirAll(path.Join(tmpdir, "backup"), 0700) - err := copyPath(path.Join(tmpdir, "file"), path.Join(tmpdir, "backup")) - assert.Nil(t, err) - - copyContent, err := ioutil.ReadFile(path.Join(tmpdir, "backup", "file")) - assert.Nil(t, err) - assert.Equal(t, content, copyContent) -} - -func Test_backupPath_shouldCopyDir(t *testing.T) { - destination, _ := ioutils.TempDir("", "destination") - defer os.RemoveAll(destination) - err := copyPath("./test_assets/copy_test", destination) - assert.Nil(t, err) - - createdFiles := listFiles(destination) - - contains(t, createdFiles, filepath.Join(destination, "copy_test", "outer")) - contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", ".dotfile")) - contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", "inner")) -} diff --git a/api/backup/restore.go b/api/backup/restore.go index b0d7acee2..e5329e913 100644 --- a/api/backup/restore.go +++ b/api/backup/restore.go @@ -11,6 +11,7 @@ import ( portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/archive" "github.com/portainer/portainer/api/crypto" + "github.com/portainer/portainer/api/filesystem" "github.com/portainer/portainer/api/http/offlinegate" ) @@ -59,7 +60,7 @@ func extractArchive(r io.Reader, destinationDirPath string) error { func restoreFiles(srcDir string, destinationDir string) error { for _, filename := range filesToRestore { - err := copyPath(filepath.Join(srcDir, filename), destinationDir) + err := filesystem.CopyPath(filepath.Join(srcDir, filename), destinationDir) if err != nil { return err } diff --git a/api/bolt/migrator/migrate_dbversion33.go b/api/bolt/migrator/migrate_dbversion33.go new file mode 100644 index 000000000..d7277ada7 --- /dev/null +++ b/api/bolt/migrator/migrate_dbversion33.go @@ -0,0 +1,32 @@ +package migrator + +import ( + portainer "github.com/portainer/portainer/api" +) + +func (m *Migrator) migrateDBVersionTo33() error { + err := migrateStackEntryPoint(m.stackService) + if err != nil { + return err + } + + return nil +} + +func migrateStackEntryPoint(stackService portainer.StackService) error { + stacks, err := stackService.Stacks() + if err != nil { + return err + } + for i := range stacks { + stack := &stacks[i] + if stack.GitConfig == nil { + continue + } + stack.GitConfig.ConfigFilePath = stack.EntryPoint + if err := stackService.UpdateStack(stack.ID, stack); err != nil { + return err + } + } + return nil +} diff --git a/api/bolt/migrator/migrate_dbversion33_test.go b/api/bolt/migrator/migrate_dbversion33_test.go new file mode 100644 index 000000000..256cc121e --- /dev/null +++ b/api/bolt/migrator/migrate_dbversion33_test.go @@ -0,0 +1,51 @@ +package migrator + +import ( + "path" + "testing" + "time" + + "github.com/boltdb/bolt" + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/bolt/internal" + "github.com/portainer/portainer/api/bolt/stack" + gittypes "github.com/portainer/portainer/api/git/types" + "github.com/stretchr/testify/assert" +) + +func TestMigrateStackEntryPoint(t *testing.T) { + dbConn, err := bolt.Open(path.Join(t.TempDir(), "portainer-ee-mig-33.db"), 0600, &bolt.Options{Timeout: 1 * time.Second}) + assert.NoError(t, err, "failed to init testing DB connection") + defer dbConn.Close() + + stackService, err := stack.NewService(&internal.DbConnection{DB: dbConn}) + assert.NoError(t, err, "failed to init testing Stack service") + + stacks := []*portainer.Stack{ + { + ID: 1, + EntryPoint: "dir/sub/compose.yml", + }, + { + ID: 2, + EntryPoint: "dir/sub/compose.yml", + GitConfig: &gittypes.RepoConfig{}, + }, + } + + for _, s := range stacks { + err := stackService.CreateStack(s) + assert.NoError(t, err, "failed to create stack") + } + + err = migrateStackEntryPoint(stackService) + assert.NoError(t, err, "failed to migrate entry point to Git ConfigFilePath") + + s, err := stackService.Stack(1) + assert.NoError(t, err) + assert.Nil(t, s.GitConfig, "first stack should not have git config") + + s, err = stackService.Stack(2) + assert.NoError(t, err) + assert.Equal(t, "dir/sub/compose.yml", s.GitConfig.ConfigFilePath, "second stack should have config file path migrated") +} diff --git a/api/bolt/migrator/migrator.go b/api/bolt/migrator/migrator.go index fc00578f0..df3ad0436 100644 --- a/api/bolt/migrator/migrator.go +++ b/api/bolt/migrator/migrator.go @@ -381,5 +381,11 @@ func (m *Migrator) Migrate() error { } } + if m.currentDBVersion < 33 { + if err := m.migrateDBVersionTo33(); err != nil { + return err + } + } + return m.versionService.StoreDBVersion(portainer.DBVersion) } diff --git a/api/bolt/stack/stack.go b/api/bolt/stack/stack.go index f9cfafad7..094a52b2e 100644 --- a/api/bolt/stack/stack.go +++ b/api/bolt/stack/stack.go @@ -1,11 +1,14 @@ package stack import ( + "strings" + portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/bolt/errors" "github.com/portainer/portainer/api/bolt/internal" "github.com/boltdb/bolt" + pkgerrors "github.com/pkg/errors" ) const ( @@ -133,3 +136,76 @@ func (service *Service) DeleteStack(ID portainer.StackID) error { identifier := internal.Itob(int(ID)) return internal.DeleteObject(service.connection, BucketName, identifier) } + +// StackByWebhookID returns a pointer to a stack object by webhook ID. +// It returns nil, errors.ErrObjectNotFound if there's no stack associated with the webhook ID. +func (service *Service) StackByWebhookID(id string) (*portainer.Stack, error) { + if id == "" { + return nil, pkgerrors.New("webhook ID can't be empty string") + } + var stack portainer.Stack + found := false + + err := service.connection.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(BucketName)) + cursor := bucket.Cursor() + + for k, v := cursor.First(); k != nil; k, v = cursor.Next() { + var t struct { + AutoUpdate *struct { + WebhookID string `json:"Webhook"` + } `json:"AutoUpdate"` + } + + err := internal.UnmarshalObject(v, &t) + if err != nil { + return err + } + + if t.AutoUpdate != nil && strings.EqualFold(t.AutoUpdate.WebhookID, id) { + found = true + err := internal.UnmarshalObject(v, &stack) + if err != nil { + return err + } + break + } + } + + return nil + }) + + if err != nil { + return nil, err + } + if !found { + return nil, errors.ErrObjectNotFound + } + + return &stack, nil +} + +// RefreshableStacks returns stacks that are configured for a periodic update +func (service *Service) RefreshableStacks() ([]portainer.Stack, error) { + stacks := make([]portainer.Stack, 0) + err := service.connection.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(BucketName)) + cursor := bucket.Cursor() + + var stack portainer.Stack + for k, v := cursor.First(); k != nil; k, v = cursor.Next() { + err := internal.UnmarshalObject(v, &stack) + if err != nil { + return err + } + + if stack.AutoUpdate != nil && stack.AutoUpdate.Interval != "" { + stacks = append(stacks, stack) + } + } + + return nil + }) + + return stacks, err +} diff --git a/api/bolt/stack/tests/stack_test.go b/api/bolt/stack/tests/stack_test.go new file mode 100644 index 000000000..d0c66dadf --- /dev/null +++ b/api/bolt/stack/tests/stack_test.go @@ -0,0 +1,111 @@ +package tests + +import ( + "testing" + "time" + + "github.com/portainer/portainer/api/bolt" + + bolterrors "github.com/portainer/portainer/api/bolt/errors" + + "github.com/portainer/portainer/api/bolt/bolttest" + + "github.com/gofrs/uuid" + + "github.com/stretchr/testify/assert" + + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/filesystem" +) + +func newGuidString(t *testing.T) string { + uuid, err := uuid.NewV4() + assert.NoError(t, err) + + return uuid.String() +} + +type stackBuilder struct { + t *testing.T + count int + store *bolt.Store +} + +func TestService_StackByWebhookID(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode. Normally takes ~1s to run.") + } + store, teardown := bolttest.MustNewTestStore(true) + defer teardown() + + b := stackBuilder{t: t, store: store} + b.createNewStack(newGuidString(t)) + for i := 0; i < 10; i++ { + b.createNewStack("") + } + webhookID := newGuidString(t) + stack := b.createNewStack(webhookID) + + // can find a stack by webhook ID + got, err := store.StackService.StackByWebhookID(webhookID) + assert.NoError(t, err) + assert.Equal(t, stack, *got) + + // returns nil and object not found error if there's no stack associated with the webhook + got, err = store.StackService.StackByWebhookID(newGuidString(t)) + assert.Nil(t, got) + assert.ErrorIs(t, err, bolterrors.ErrObjectNotFound) +} + +func (b *stackBuilder) createNewStack(webhookID string) portainer.Stack { + b.count++ + stack := portainer.Stack{ + ID: portainer.StackID(b.count), + Name: "Name", + Type: portainer.DockerComposeStack, + EndpointID: 2, + EntryPoint: filesystem.ComposeFileDefaultName, + Env: []portainer.Pair{{"Name1", "Value1"}}, + Status: portainer.StackStatusActive, + CreationDate: time.Now().Unix(), + ProjectPath: "/tmp/project", + CreatedBy: "test", + } + + if webhookID == "" { + if b.count%2 == 0 { + stack.AutoUpdate = &portainer.StackAutoUpdate{ + Interval: "", + Webhook: "", + } + } // else keep AutoUpdate nil + } else { + stack.AutoUpdate = &portainer.StackAutoUpdate{Webhook: webhookID} + } + + err := b.store.StackService.CreateStack(&stack) + assert.NoError(b.t, err) + + return stack +} + +func Test_RefreshableStacks(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode. Normally takes ~1s to run.") + } + store, teardown := bolttest.MustNewTestStore(true) + defer teardown() + + staticStack := portainer.Stack{ID: 1} + stackWithWebhook := portainer.Stack{ID: 2, AutoUpdate: &portainer.StackAutoUpdate{Webhook: "webhook"}} + refreshableStack := portainer.Stack{ID: 3, AutoUpdate: &portainer.StackAutoUpdate{Interval: "1m"}} + + for _, stack := range []*portainer.Stack{&staticStack, &stackWithWebhook, &refreshableStack} { + err := store.Stack().CreateStack(stack) + assert.NoError(t, err) + } + + stacks, err := store.Stack().RefreshableStacks() + assert.NoError(t, err) + assert.ElementsMatch(t, []portainer.Stack{refreshableStack}, stacks) +} diff --git a/api/cmd/portainer/main.go b/api/cmd/portainer/main.go index e9fd9e98c..717c3fbc3 100644 --- a/api/cmd/portainer/main.go +++ b/api/cmd/portainer/main.go @@ -6,7 +6,6 @@ import ( "os" "strings" - wrapper "github.com/portainer/docker-compose-wrapper" portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/bolt" "github.com/portainer/portainer/api/chisel" @@ -31,6 +30,8 @@ import ( "github.com/portainer/portainer/api/ldap" "github.com/portainer/portainer/api/libcompose" "github.com/portainer/portainer/api/oauth" + "github.com/portainer/portainer/api/scheduler" + "github.com/portainer/portainer/api/stacks" ) func initCLI() *portainer.CLIFlags { @@ -88,12 +89,8 @@ func shutdownDatastore(shutdownCtx context.Context, datastore portainer.DataStor func initComposeStackManager(assetsPath string, dataStorePath string, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager { composeWrapper, err := exec.NewComposeStackManager(assetsPath, dataStorePath, proxyManager) if err != nil { - if err == wrapper.ErrBinaryNotFound { - log.Printf("[INFO] [message: docker-compose binary not found, falling back to libcompose]") - return libcompose.NewComposeStackManager(dataStorePath, reverseTunnelService) - } - - log.Fatalf("failed initalizing compose stack manager; err=%s", err) + log.Printf("[INFO] [main,compose] [message: falling-back to libcompose] [error: %s]", err) + return libcompose.NewComposeStackManager(dataStorePath, reverseTunnelService) } return composeWrapper @@ -525,6 +522,10 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server { log.Fatalf("failed to fetch ssl settings from DB") } + scheduler := scheduler.NewScheduler(shutdownCtx) + stackDeployer := stacks.NewStackDeployer(swarmStackManager, composeStackManager) + stacks.StartStackSchedules(scheduler, stackDeployer, dataStore, gitService) + return &http.Server{ AuthorizationService: authorizationService, ReverseTunnelService: reverseTunnelService, @@ -550,8 +551,10 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server { SSLService: sslService, DockerClientFactory: dockerClientFactory, KubernetesClientFactory: kubernetesClientFactory, + Scheduler: scheduler, ShutdownCtx: shutdownCtx, ShutdownTrigger: shutdownTrigger, + StackDeployer: stackDeployer, } } diff --git a/api/exec/compose_stack.go b/api/exec/compose_stack.go index bb6e9f055..36283c6d9 100644 --- a/api/exec/compose_stack.go +++ b/api/exec/compose_stack.go @@ -7,8 +7,8 @@ import ( "regexp" "strings" + "github.com/pkg/errors" wrapper "github.com/portainer/docker-compose-wrapper" - portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/http/proxy" "github.com/portainer/portainer/api/http/proxy/factory" @@ -35,12 +35,6 @@ func NewComposeStackManager(binaryPath string, configPath string, proxyManager * }, nil } -// NormalizeStackName returns a new stack name with unsupported characters replaced -func (w *ComposeStackManager) NormalizeStackName(name string) string { - r := regexp.MustCompile("[^a-z0-9]+") - return r.ReplaceAllString(strings.ToLower(name), "") -} - // ComposeSyntaxMaxVersion returns the maximum supported version of the docker compose syntax func (w *ComposeStackManager) ComposeSyntaxMaxVersion() string { return portainer.ComposeSyntaxMaxVersion @@ -50,7 +44,7 @@ func (w *ComposeStackManager) ComposeSyntaxMaxVersion() string { func (w *ComposeStackManager) Up(stack *portainer.Stack, endpoint *portainer.Endpoint) error { url, proxy, err := w.fetchEndpointProxy(endpoint) if err != nil { - return err + return errors.Wrap(err, "failed to featch endpoint proxy") } if proxy != nil { @@ -59,13 +53,12 @@ func (w *ComposeStackManager) Up(stack *portainer.Stack, endpoint *portainer.End envFilePath, err := createEnvFile(stack) if err != nil { - return err + return errors.Wrap(err, "failed to create env file") } - filePath := stackFilePath(stack) - - _, err = w.wrapper.Up([]string{filePath}, url, stack.Name, envFilePath, w.configPath) - return err + filePaths := append([]string{stack.EntryPoint}, stack.AdditionalFiles...) + _, err = w.wrapper.Up(filePaths, stack.ProjectPath, url, stack.Name, envFilePath, w.configPath) + return errors.Wrap(err, "failed to deploy a stack") } // Down stops and removes containers, networks, images, and volumes. Wraps `docker-compose down --remove-orphans` command @@ -78,14 +71,16 @@ func (w *ComposeStackManager) Down(stack *portainer.Stack, endpoint *portainer.E defer proxy.Close() } - filePath := stackFilePath(stack) + filePaths := append([]string{stack.EntryPoint}, stack.AdditionalFiles...) - _, err = w.wrapper.Down([]string{filePath}, url, stack.Name) + _, err = w.wrapper.Down(filePaths, stack.ProjectPath, url, stack.Name) return err } -func stackFilePath(stack *portainer.Stack) string { - return path.Join(stack.ProjectPath, stack.EntryPoint) +// NormalizeStackName returns a new stack name with unsupported characters replaced +func (w *ComposeStackManager) NormalizeStackName(name string) string { + r := regexp.MustCompile("[^a-z0-9]+") + return r.ReplaceAllString(strings.ToLower(name), "") } func (w *ComposeStackManager) fetchEndpointProxy(endpoint *portainer.Endpoint) (string, *factory.ProxyServer, error) { @@ -118,5 +113,5 @@ func createEnvFile(stack *portainer.Stack) (string, error) { } envfile.Close() - return envFilePath, nil + return "stack.env", nil } diff --git a/api/exec/compose_stack_integration_test.go b/api/exec/compose_stack_integration_test.go index 8214d3461..f0b279659 100644 --- a/api/exec/compose_stack_integration_test.go +++ b/api/exec/compose_stack_integration_test.go @@ -1,5 +1,3 @@ -// +build integration - package exec import ( diff --git a/api/exec/compose_stack_test.go b/api/exec/compose_stack_test.go index 80e2818df..c61285ebd 100644 --- a/api/exec/compose_stack_test.go +++ b/api/exec/compose_stack_test.go @@ -10,47 +10,6 @@ import ( "github.com/stretchr/testify/assert" ) -func Test_stackFilePath(t *testing.T) { - tests := []struct { - name string - stack *portainer.Stack - expected string - }{ - // { - // name: "should return empty result if stack is missing", - // stack: nil, - // expected: "", - // }, - // { - // name: "should return empty result if stack don't have entrypoint", - // stack: &portainer.Stack{}, - // expected: "", - // }, - { - name: "should allow file name and dir", - stack: &portainer.Stack{ - ProjectPath: "dir", - EntryPoint: "file", - }, - expected: path.Join("dir", "file"), - }, - { - name: "should allow file name only", - stack: &portainer.Stack{ - EntryPoint: "file", - }, - expected: "file", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := stackFilePath(tt.stack) - assert.Equal(t, tt.expected, result) - }) - } -} - func Test_createEnvFile(t *testing.T) { dir := t.TempDir() @@ -60,11 +19,6 @@ func Test_createEnvFile(t *testing.T) { expected string expectedFile bool }{ - // { - // name: "should not add env file option if stack is missing", - // stack: nil, - // expected: "", - // }, { name: "should not add env file option if stack doesn't have env variables", stack: &portainer.Stack{ @@ -98,7 +52,7 @@ func Test_createEnvFile(t *testing.T) { result, _ := createEnvFile(tt.stack) if tt.expected != "" { - assert.Equal(t, path.Join(tt.stack.ProjectPath, "stack.env"), result) + assert.Equal(t, "stack.env", result) f, _ := os.Open(path.Join(dir, "stack.env")) content, _ := ioutil.ReadAll(f) diff --git a/api/exec/swarm_stack.go b/api/exec/swarm_stack.go index 69f91581c..eecb1047a 100644 --- a/api/exec/swarm_stack.go +++ b/api/exec/swarm_stack.go @@ -13,6 +13,7 @@ import ( "strings" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/internal/stackutils" ) // SwarmStackManager represents a service for managing stacks. @@ -63,22 +64,23 @@ func (manager *SwarmStackManager) Logout(endpoint *portainer.Endpoint) error { // Deploy executes the docker stack deploy command. func (manager *SwarmStackManager) Deploy(stack *portainer.Stack, prune bool, endpoint *portainer.Endpoint) error { - stackFilePath := path.Join(stack.ProjectPath, stack.EntryPoint) + filePaths := stackutils.GetStackFilePaths(stack) command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint) if prune { - args = append(args, "stack", "deploy", "--prune", "--with-registry-auth", "--compose-file", stackFilePath, stack.Name) + args = append(args, "stack", "deploy", "--prune", "--with-registry-auth") } else { - args = append(args, "stack", "deploy", "--with-registry-auth", "--compose-file", stackFilePath, stack.Name) + args = append(args, "stack", "deploy", "--with-registry-auth") } + args = configureFilePaths(args, filePaths) + args = append(args, stack.Name) + env := make([]string, 0) for _, envvar := range stack.Env { env = append(env, envvar.Name+"="+envvar.Value) } - - stackFolder := path.Dir(stackFilePath) - return runCommandAndCaptureStdErr(command, args, env, stackFolder) + return runCommandAndCaptureStdErr(command, args, env, stack.ProjectPath) } // Remove executes the docker stack rm command. @@ -191,3 +193,10 @@ func (manager *SwarmStackManager) NormalizeStackName(name string) string { r := regexp.MustCompile("[^a-z0-9]+") return r.ReplaceAllString(strings.ToLower(name), "") } + +func configureFilePaths(args []string, filePaths []string) []string { + for _, path := range filePaths { + args = append(args, "--compose-file", path) + } + return args +} diff --git a/api/exec/swarm_stack_test.go b/api/exec/swarm_stack_test.go new file mode 100644 index 000000000..47d28ce2c --- /dev/null +++ b/api/exec/swarm_stack_test.go @@ -0,0 +1,15 @@ +package exec + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConfigFilePaths(t *testing.T) { + args := []string{"stack", "deploy", "--with-registry-auth"} + filePaths := []string{"dir/file", "dir/file-two", "dir/file-three"} + expected := []string{"stack", "deploy", "--with-registry-auth", "--compose-file", "dir/file", "--compose-file", "dir/file-two", "--compose-file", "dir/file-three"} + output := configureFilePaths(args, filePaths) + assert.ElementsMatch(t, expected, output, "wrong output file paths") +} diff --git a/api/backup/copy.go b/api/filesystem/copy.go similarity index 63% rename from api/backup/copy.go rename to api/filesystem/copy.go index 6aaefd54c..abf4d33aa 100644 --- a/api/backup/copy.go +++ b/api/filesystem/copy.go @@ -1,4 +1,4 @@ -package backup +package filesystem import ( "errors" @@ -8,7 +8,8 @@ import ( "strings" ) -func copyPath(path string, toDir string) error { +// CopyPath copies file or directory defined by the path to the toDir path +func CopyPath(path string, toDir string) error { info, err := os.Stat(path) if err != nil && errors.Is(err, os.ErrNotExist) { // skip copy if file does not exist @@ -20,17 +21,30 @@ func copyPath(path string, toDir string) error { return copyFile(path, destination) } - return copyDir(path, toDir) + return CopyDir(path, toDir, true) } -func copyDir(fromDir, toDir string) error { +// CopyDir copies contents of fromDir to toDir. +// When keepParent is true, contents will be copied with their immediate parent dir, +// i.e. given /from/dirA and /to/dirB with keepParent == true, result will be /to/dirB/dirA/ +func CopyDir(fromDir, toDir string, keepParent bool) error { cleanedSourcePath := filepath.Clean(fromDir) parentDirectory := filepath.Dir(cleanedSourcePath) err := filepath.Walk(cleanedSourcePath, func(path string, info os.FileInfo, err error) error { if err != nil { return err } - destination := filepath.Join(toDir, strings.TrimPrefix(path, parentDirectory)) + var destination string + if keepParent { + destination = filepath.Join(toDir, strings.TrimPrefix(path, parentDirectory)) + } else { + destination = filepath.Join(toDir, strings.TrimPrefix(path, cleanedSourcePath)) + } + + if destination == "" { + return nil + } + if info.IsDir() { return nil // skip directory creations } diff --git a/api/filesystem/copy_test.go b/api/filesystem/copy_test.go new file mode 100644 index 000000000..2fcef9e6b --- /dev/null +++ b/api/filesystem/copy_test.go @@ -0,0 +1,92 @@ +package filesystem + +import ( + "io/ioutil" + "os" + "path" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_copyFile_returnsError_whenSourceDoesNotExist(t *testing.T) { + tmpdir, _ := ioutil.TempDir("", "backup") + defer os.RemoveAll(tmpdir) + + err := copyFile("does-not-exist", tmpdir) + assert.Error(t, err) +} + +func Test_copyFile_shouldMakeAbackup(t *testing.T) { + tmpdir, _ := ioutil.TempDir("", "backup") + defer os.RemoveAll(tmpdir) + + content := []byte("content") + ioutil.WriteFile(path.Join(tmpdir, "origin"), content, 0600) + + err := copyFile(path.Join(tmpdir, "origin"), path.Join(tmpdir, "copy")) + assert.NoError(t, err) + + copyContent, _ := ioutil.ReadFile(path.Join(tmpdir, "copy")) + assert.Equal(t, content, copyContent) +} + +func Test_CopyDir_shouldCopyAllFilesAndDirectories(t *testing.T) { + destination, _ := ioutil.TempDir("", "destination") + defer os.RemoveAll(destination) + err := CopyDir("./testdata/copy_test", destination, true) + assert.NoError(t, err) + + assert.FileExists(t, filepath.Join(destination, "copy_test", "outer")) + assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", ".dotfile")) + assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", "inner")) +} + +func Test_CopyDir_shouldCopyOnlyDirContents(t *testing.T) { + destination, _ := ioutil.TempDir("", "destination") + defer os.RemoveAll(destination) + err := CopyDir("./testdata/copy_test", destination, false) + assert.NoError(t, err) + + assert.FileExists(t, filepath.Join(destination, "outer")) + assert.FileExists(t, filepath.Join(destination, "dir", ".dotfile")) + assert.FileExists(t, filepath.Join(destination, "dir", "inner")) +} + +func Test_CopyPath_shouldSkipWhenNotExist(t *testing.T) { + tmpdir, _ := ioutil.TempDir("", "backup") + defer os.RemoveAll(tmpdir) + + err := CopyPath("does-not-exists", tmpdir) + assert.NoError(t, err) + + assert.NoFileExists(t, tmpdir) +} + +func Test_CopyPath_shouldCopyFile(t *testing.T) { + tmpdir, _ := ioutil.TempDir("", "backup") + defer os.RemoveAll(tmpdir) + + content := []byte("content") + ioutil.WriteFile(path.Join(tmpdir, "file"), content, 0600) + + os.MkdirAll(path.Join(tmpdir, "backup"), 0700) + err := CopyPath(path.Join(tmpdir, "file"), path.Join(tmpdir, "backup")) + assert.NoError(t, err) + + copyContent, err := ioutil.ReadFile(path.Join(tmpdir, "backup", "file")) + assert.NoError(t, err) + assert.Equal(t, content, copyContent) +} + +func Test_CopyPath_shouldCopyDir(t *testing.T) { + destination, _ := ioutil.TempDir("", "destination") + defer os.RemoveAll(destination) + err := CopyPath("./testdata/copy_test", destination) + assert.NoError(t, err) + + assert.FileExists(t, filepath.Join(destination, "copy_test", "outer")) + assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", ".dotfile")) + assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", "inner")) +} diff --git a/api/backup/test_assets/copy_test/dir/.dotfile b/api/filesystem/testdata/copy_test/dir/.dotfile similarity index 100% rename from api/backup/test_assets/copy_test/dir/.dotfile rename to api/filesystem/testdata/copy_test/dir/.dotfile diff --git a/api/backup/test_assets/copy_test/dir/inner b/api/filesystem/testdata/copy_test/dir/inner similarity index 100% rename from api/backup/test_assets/copy_test/dir/inner rename to api/filesystem/testdata/copy_test/dir/inner diff --git a/api/backup/test_assets/copy_test/outer b/api/filesystem/testdata/copy_test/outer similarity index 100% rename from api/backup/test_assets/copy_test/outer rename to api/filesystem/testdata/copy_test/outer diff --git a/api/git/azure.go b/api/git/azure.go index 78f10e52d..417d5db4b 100644 --- a/api/git/azure.go +++ b/api/git/azure.go @@ -2,15 +2,17 @@ package git import ( "context" + "encoding/json" "fmt" - "github.com/pkg/errors" - "github.com/portainer/portainer/api/archive" "io" "io/ioutil" "net/http" "net/url" "os" "strings" + + "github.com/pkg/errors" + "github.com/portainer/portainer/api/archive" ) const ( @@ -37,7 +39,7 @@ type azureDownloader struct { func NewAzureDownloader(client *http.Client) *azureDownloader { return &azureDownloader{ - client: client, + client: client, baseUrl: "https://dev.azure.com", } } @@ -100,6 +102,57 @@ func (a *azureDownloader) downloadZipFromAzureDevOps(ctx context.Context, option return zipFile.Name(), nil } +func (a *azureDownloader) latestCommitID(ctx context.Context, options fetchOptions) (string, error) { + config, err := parseUrl(options.repositoryUrl) + if err != nil { + return "", errors.WithMessage(err, "failed to parse url") + } + + refsUrl, err := a.buildRefsUrl(config, options.referenceName) + if err != nil { + return "", errors.WithMessage(err, "failed to build azure refs url") + } + + req, err := http.NewRequestWithContext(ctx, "GET", refsUrl, nil) + if options.username != "" || options.password != "" { + req.SetBasicAuth(options.username, options.password) + } else if config.username != "" || config.password != "" { + req.SetBasicAuth(config.username, config.password) + } + + if err != nil { + return "", errors.WithMessage(err, "failed to create a new HTTP request") + } + + resp, err := a.client.Do(req) + if err != nil { + return "", errors.WithMessage(err, "failed to make an HTTP request") + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("failed to get repository refs with a status \"%v\"", resp.Status) + } + + var refs struct { + Value []struct { + Name string `json:"name"` + ObjectId string `json:"objectId"` + } + } + if err := json.NewDecoder(resp.Body).Decode(&refs); err != nil { + return "", errors.Wrap(err, "could not parse Azure Refs response") + } + + for _, ref := range refs.Value { + if strings.EqualFold(ref.Name, options.referenceName) { + return ref.ObjectId, nil + } + } + + return "", errors.Errorf("could not find ref %q in the repository", options.referenceName) +} + func parseUrl(rawUrl string) (*azureOptions, error) { if strings.HasPrefix(rawUrl, "https://") || strings.HasPrefix(rawUrl, "http://") { return parseHttpUrl(rawUrl) @@ -193,6 +246,27 @@ func (a *azureDownloader) buildDownloadUrl(config *azureOptions, referenceName s return u.String(), nil } +func (a *azureDownloader) buildRefsUrl(config *azureOptions, referenceName string) (string, error) { + rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/refs", + a.baseUrl, + url.PathEscape(config.organisation), + url.PathEscape(config.project), + url.PathEscape(config.repository)) + u, err := url.Parse(rawUrl) + + if err != nil { + return "", errors.Wrapf(err, "failed to parse refs url path %s", rawUrl) + } + + // filterContains=main&api-version=6.0 + q := u.Query() + q.Set("filterContains", formatReferenceName(referenceName)) + q.Set("api-version", "6.0") + u.RawQuery = q.Encode() + + return u.String(), nil +} + const ( branchPrefix = "refs/heads/" tagPrefix = "refs/tags/" diff --git a/api/git/azure_integration_test.go b/api/git/azure_integration_test.go index 6d684d877..200747daa 100644 --- a/api/git/azure_integration_test.go +++ b/api/git/azure_integration_test.go @@ -78,6 +78,18 @@ func TestService_ClonePrivateRepository_Azure(t *testing.T) { assert.FileExists(t, filepath.Join(dst, "README.md")) } +func TestService_LatestCommitID_Azure(t *testing.T) { + ensureIntegrationTest(t) + + pat := getRequiredValue(t, "AZURE_DEVOPS_PAT") + service := NewService() + + repositoryUrl := "https://portainer.visualstudio.com/Playground/_git/dev_integration" + id, err := service.LatestCommitID(repositoryUrl, "refs/heads/main", "", pat) + assert.NoError(t, err) + assert.NotEmpty(t, id, "cannot guarantee commit id, but it should be not empty") +} + func getRequiredValue(t *testing.T, name string) string { value, ok := os.LookupEnv(name) if !ok { diff --git a/api/git/azure_test.go b/api/git/azure_test.go index 18417e9e6..b95dc981e 100644 --- a/api/git/azure_test.go +++ b/api/git/azure_test.go @@ -2,11 +2,12 @@ package git import ( "context" - "github.com/stretchr/testify/assert" "net/http" "net/http/httptest" "net/url" "testing" + + "github.com/stretchr/testify/assert" ) func Test_buildDownloadUrl(t *testing.T) { @@ -27,6 +28,23 @@ func Test_buildDownloadUrl(t *testing.T) { } } +func Test_buildRefsUrl(t *testing.T) { + a := NewAzureDownloader(nil) + u, err := a.buildRefsUrl(&azureOptions{ + organisation: "organisation", + project: "project", + repository: "repository", + }, "refs/heads/main") + + expectedUrl, _ := url.Parse("https://dev.azure.com/organisation/project/_apis/git/repositories/repository/refs?filterContains=main&api-version=6.0") + actualUrl, _ := url.Parse(u) + assert.NoError(t, err) + assert.Equal(t, expectedUrl.Host, actualUrl.Host) + assert.Equal(t, expectedUrl.Scheme, actualUrl.Scheme) + assert.Equal(t, expectedUrl.Path, actualUrl.Path) + assert.Equal(t, expectedUrl.Query(), actualUrl.Query()) +} + func Test_parseAzureUrl(t *testing.T) { type args struct { url string @@ -248,3 +266,110 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) { }) } } + +func Test_azureDownloader_latestCommitID(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + response := `{ + "value": [ + { + "name": "refs/heads/feature/calcApp", + "objectId": "ffe9cba521f00d7f60e322845072238635edb451", + "creator": { + "displayName": "Normal Paulk", + "url": "https://vssps.dev.azure.com/fabrikam/_apis/Identities/ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "_links": { + "avatar": { + "href": "https://dev.azure.com/fabrikam/_apis/GraphProfile/MemberAvatars/aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5" + } + }, + "id": "ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "uniqueName": "dev@mailserver.com", + "imageUrl": "https://dev.azure.com/fabrikam/_api/_common/identityImage?id=ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "descriptor": "aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5" + }, + "url": "https://dev.azure.com/fabrikam/7484f783-66a3-4f27-b7cd-6b08b0b077ed/_apis/git/repositories/d3d1760b-311c-4175-a726-20dfc6a7f885/refs?filter=heads%2Ffeature%2FcalcApp" + }, + { + "name": "refs/heads/feature/replacer", + "objectId": "917131a709996c5cfe188c3b57e9a6ad90e8b85c", + "creator": { + "displayName": "Normal Paulk", + "url": "https://vssps.dev.azure.com/fabrikam/_apis/Identities/ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "_links": { + "avatar": { + "href": "https://dev.azure.com/fabrikam/_apis/GraphProfile/MemberAvatars/aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5" + } + }, + "id": "ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "uniqueName": "dev@mailserver.com", + "imageUrl": "https://dev.azure.com/fabrikam/_api/_common/identityImage?id=ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "descriptor": "aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5" + }, + "url": "https://dev.azure.com/fabrikam/7484f783-66a3-4f27-b7cd-6b08b0b077ed/_apis/git/repositories/d3d1760b-311c-4175-a726-20dfc6a7f885/refs?filter=heads%2Ffeature%2Freplacer" + }, + { + "name": "refs/heads/master", + "objectId": "ffe9cba521f00d7f60e322845072238635edb451", + "creator": { + "displayName": "Normal Paulk", + "url": "https://vssps.dev.azure.com/fabrikam/_apis/Identities/ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "_links": { + "avatar": { + "href": "https://dev.azure.com/fabrikam/_apis/GraphProfile/MemberAvatars/aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5" + } + }, + "id": "ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "uniqueName": "dev@mailserver.com", + "imageUrl": "https://dev.azure.com/fabrikam/_api/_common/identityImage?id=ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "descriptor": "aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5" + }, + "url": "https://dev.azure.com/fabrikam/7484f783-66a3-4f27-b7cd-6b08b0b077ed/_apis/git/repositories/d3d1760b-311c-4175-a726-20dfc6a7f885/refs?filter=heads%2Fmaster" + } + ], + "count": 3 + }` + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(response)) + })) + defer server.Close() + + a := &azureDownloader{ + client: server.Client(), + baseUrl: server.URL, + } + + tests := []struct { + name string + args fetchOptions + want string + wantErr bool + }{ + { + name: "should be able to parse response", + args: fetchOptions{ + referenceName: "refs/heads/master", + repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository"}, + want: "ffe9cba521f00d7f60e322845072238635edb451", + wantErr: false, + }, + { + name: "should be able to parse response", + args: fetchOptions{ + referenceName: "refs/heads/unknown", + repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository"}, + want: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + id, err := a.latestCommitID(context.Background(), tt.args) + if (err != nil) != tt.wantErr { + t.Errorf("azureDownloader.latestCommitID() error = %v, wantErr %v", err, tt.wantErr) + return + } + assert.Equal(t, tt.want, id) + }) + } +} diff --git a/api/git/git.go b/api/git/git.go index 519d699a8..57c8eb106 100644 --- a/api/git/git.go +++ b/api/git/git.go @@ -6,16 +6,26 @@ import ( "net/http" "os" "path/filepath" + "strings" "time" "github.com/pkg/errors" "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/transport/client" githttp "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/go-git/go-git/v5/storage/memory" ) +type fetchOptions struct { + repositoryUrl string + username string + password string + referenceName string +} + type cloneOptions struct { repositoryUrl string username string @@ -26,6 +36,7 @@ type cloneOptions struct { type downloader interface { download(ctx context.Context, dst string, opt cloneOptions) error + latestCommitID(ctx context.Context, opt fetchOptions) (string, error) } type gitClient struct { @@ -36,13 +47,7 @@ func (c gitClient) download(ctx context.Context, dst string, opt cloneOptions) e gitOptions := git.CloneOptions{ URL: opt.repositoryUrl, Depth: opt.depth, - } - - if opt.password != "" || opt.username != "" { - gitOptions.Auth = &githttp.BasicAuth{ - Username: opt.username, - Password: opt.password, - } + Auth: getAuth(opt.username, opt.password), } if opt.referenceName != "" { @@ -62,6 +67,44 @@ func (c gitClient) download(ctx context.Context, dst string, opt cloneOptions) e return nil } +func (c gitClient) latestCommitID(ctx context.Context, opt fetchOptions) (string, error) { + remote := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{ + Name: "origin", + URLs: []string{opt.repositoryUrl}, + }) + + listOptions := &git.ListOptions{ + Auth: getAuth(opt.username, opt.password), + } + + refs, err := remote.List(listOptions) + if err != nil { + return "", errors.Wrap(err, "failed to list repository refs") + } + + for _, ref := range refs { + if strings.EqualFold(ref.Name().String(), opt.referenceName) { + return ref.Hash().String(), nil + } + } + + return "", errors.Errorf("could not find ref %q in the repository", opt.referenceName) +} + +func getAuth(username, password string) *githttp.BasicAuth { + if password != "" { + if username == "" { + username = "token" + } + + return &githttp.BasicAuth{ + Username: username, + Password: password, + } + } + return nil +} + // Service represents a service for managing Git. type Service struct { httpsCli *http.Client @@ -109,3 +152,19 @@ func (service *Service) cloneRepository(destination string, options cloneOptions return service.git.download(context.TODO(), destination, options) } + +// LatestCommitID returns SHA1 of the latest commit of the specified reference +func (service *Service) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) { + options := fetchOptions{ + repositoryUrl: repositoryURL, + username: username, + password: password, + referenceName: referenceName, + } + + if isAzureUrl(options.repositoryUrl) { + return service.azure.latestCommitID(context.TODO(), options) + } + + return service.git.latestCommitID(context.TODO(), options) +} diff --git a/api/git/git_integration_test.go b/api/git/git_integration_test.go index d35ba8d52..6f123c130 100644 --- a/api/git/git_integration_test.go +++ b/api/git/git_integration_test.go @@ -12,7 +12,7 @@ import ( func TestService_ClonePrivateRepository_GitHub(t *testing.T) { ensureIntegrationTest(t) - pat := getRequiredValue(t, "GITHUB_PAT") + accessToken := getRequiredValue(t, "GITHUB_PAT") username := getRequiredValue(t, "GITHUB_USERNAME") service := NewService() @@ -21,7 +21,20 @@ func TestService_ClonePrivateRepository_GitHub(t *testing.T) { defer os.RemoveAll(dst) repositoryUrl := "https://github.com/portainer/private-test-repository.git" - err = service.CloneRepository(dst, repositoryUrl, "refs/heads/main", username, pat) + err = service.CloneRepository(dst, repositoryUrl, "refs/heads/main", username, accessToken) assert.NoError(t, err) assert.FileExists(t, filepath.Join(dst, "README.md")) } + +func TestService_LatestCommitID_GitHub(t *testing.T) { + ensureIntegrationTest(t) + + accessToken := getRequiredValue(t, "GITHUB_PAT") + username := getRequiredValue(t, "GITHUB_USERNAME") + service := NewService() + + repositoryUrl := "https://github.com/portainer/private-test-repository.git" + id, err := service.LatestCommitID(repositoryUrl, "refs/heads/main", username, accessToken) + assert.NoError(t, err) + assert.NotEmpty(t, id, "cannot guarantee commit id, but it should be not empty") +} diff --git a/api/git/git_test.go b/api/git/git_test.go index 14878b304..11ff5e5c8 100644 --- a/api/git/git_test.go +++ b/api/git/git_test.go @@ -105,7 +105,19 @@ func Test_cloneRepository(t *testing.T) { }) assert.NoError(t, err) - assert.Equal(t, 3, getCommitHistoryLength(t, err, dir), "cloned repo has incorrect depth") + assert.Equal(t, 4, getCommitHistoryLength(t, err, dir), "cloned repo has incorrect depth") +} + +func Test_latestCommitID(t *testing.T) { + service := Service{git: gitClient{preserveGitDirectory: true}} // no need for http client since the test access the repo via file system. + + repositoryURL := bareRepoDir + referenceName := "refs/heads/main" + + id, err := service.LatestCommitID(repositoryURL, referenceName, "", "") + + assert.NoError(t, err) + assert.Equal(t, "68dcaa7bd452494043c64252ab90db0f98ecf8d2", id) } func getCommitHistoryLength(t *testing.T, err error, dir string) int { @@ -137,6 +149,10 @@ func (t *testDownloader) download(_ context.Context, _ string, _ cloneOptions) e return nil } +func (t *testDownloader) latestCommitID(_ context.Context, _ fetchOptions) (string, error) { + return "", nil +} + func Test_cloneRepository_azure(t *testing.T) { tests := []struct { name string diff --git a/api/git/testdata/azure-repo copy.zip b/api/git/testdata/azure-repo copy.zip new file mode 100644 index 000000000..d4b53d0b8 Binary files /dev/null and b/api/git/testdata/azure-repo copy.zip differ diff --git a/api/git/testdata/test-clone-git-repo.tar.gz b/api/git/testdata/test-clone-git-repo.tar.gz index ca63d337c..cba76a0d8 100644 Binary files a/api/git/testdata/test-clone-git-repo.tar.gz and b/api/git/testdata/test-clone-git-repo.tar.gz differ diff --git a/api/git/types/types.go b/api/git/types/types.go index 055222700..d24283b52 100644 --- a/api/git/types/types.go +++ b/api/git/types/types.go @@ -1,10 +1,20 @@ package gittypes +// RepoConfig represents a configuration for a repo type RepoConfig struct { // The repo url - URL string `example:"https://github.com/portainer/portainer-ee.git"` + URL string `example:"https://github.com/portainer/portainer.git"` // The reference name ReferenceName string `example:"refs/heads/branch_name"` // Path to where the config file is in this url/refName ConfigFilePath string `example:"docker-compose.yml"` + // Git credentials + Authentication *GitAuthentication + // Repository hash + ConfigHash string `example:"bc4c183d756879ea4d173315338110b31004b8e0"` +} + +type GitAuthentication struct { + Username string + Password string } diff --git a/api/go.mod b/api/go.mod index 4859e6120..f0e916dcb 100644 --- a/api/go.mod +++ b/api/go.mod @@ -27,10 +27,11 @@ require ( github.com/mitchellh/mapstructure v1.1.2 // indirect github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6 github.com/pkg/errors v0.9.1 - github.com/portainer/docker-compose-wrapper v0.0.0-20210527221011-0a1418224b92 + github.com/portainer/docker-compose-wrapper v0.0.0-20210810234209-d01bc85eb481 github.com/portainer/libcompose v0.5.3 github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a github.com/portainer/libhttp v0.0.0-20190806161843-ba068f58be33 + github.com/robfig/cron/v3 v3.0.1 github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.0 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 diff --git a/api/go.sum b/api/go.sum index e966dc90a..8906e08cf 100644 --- a/api/go.sum +++ b/api/go.sum @@ -241,8 +241,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/portainer/docker-compose-wrapper v0.0.0-20210527221011-0a1418224b92 h1:Hh7SHCf3SJblVywU0TTn5lpTKsH5W23LAKH5sqWggig= -github.com/portainer/docker-compose-wrapper v0.0.0-20210527221011-0a1418224b92/go.mod h1:PF2O2O4UNYWdtPcp6n/mIKpKk+f1jhFTezS8txbf+XM= +github.com/portainer/docker-compose-wrapper v0.0.0-20210810234209-d01bc85eb481 h1:5c8N9Gh21Ja/9EIpfyHFmQvTCKgOjnRhosmo0ZshkFk= +github.com/portainer/docker-compose-wrapper v0.0.0-20210810234209-d01bc85eb481/go.mod h1:WxDlJWZxCnicdLCPnLNEv7/gRhjeIVuCGmsv+iOPH3c= github.com/portainer/libcompose v0.5.3 h1:tE4WcPuGvo+NKeDkDWpwNavNLZ5GHIJ4RvuZXsI9uI8= github.com/portainer/libcompose v0.5.3/go.mod h1:7SKd/ho69rRKHDFSDUwkbMcol2TMKU5OslDsajr8Ro8= github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a h1:qY8TbocN75n5PDl16o0uVr5MevtM5IhdwSelXEd4nFM= @@ -263,6 +263,8 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= diff --git a/api/http/handler/stacks/autoupdate.go b/api/http/handler/stacks/autoupdate.go new file mode 100644 index 000000000..867e71fc2 --- /dev/null +++ b/api/http/handler/stacks/autoupdate.go @@ -0,0 +1,38 @@ +package stacks + +import ( + "log" + "net/http" + "time" + + httperror "github.com/portainer/libhttp/error" + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/scheduler" + "github.com/portainer/portainer/api/stacks" +) + +func startAutoupdate(stackID portainer.StackID, interval string, scheduler *scheduler.Scheduler, stackDeployer stacks.StackDeployer, datastore portainer.DataStore, gitService portainer.GitService) (jobID string, e *httperror.HandlerError) { + d, err := time.ParseDuration(interval) + if err != nil { + return "", &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Unable to parse stack's auto update interval", Err: err} + } + + jobID = scheduler.StartJobEvery(d, func() { + if err := stacks.RedeployWhenChanged(stackID, stackDeployer, datastore, gitService); err != nil { + log.Printf("[ERROR] [http,stacks] [message: failed redeploying] [err: %s]\n", err) + } + }) + + return jobID, nil +} + +func stopAutoupdate(stackID portainer.StackID, jobID string, scheduler scheduler.Scheduler) { + if jobID == "" { + return + } + + if err := scheduler.StopJob(jobID); err != nil { + log.Printf("[WARN] could not stop the job for the stack %v", stackID) + } + +} diff --git a/api/http/handler/stacks/create_compose_stack.go b/api/http/handler/stacks/create_compose_stack.go index f66c2572d..4ca76967f 100644 --- a/api/http/handler/stacks/create_compose_stack.go +++ b/api/http/handler/stacks/create_compose_stack.go @@ -1,7 +1,6 @@ package stacks import ( - "errors" "fmt" "net/http" "path" @@ -9,10 +8,12 @@ import ( "time" "github.com/asaskevich/govalidator" + "github.com/pkg/errors" httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/filesystem" + gittypes "github.com/portainer/portainer/api/git/types" "github.com/portainer/portainer/api/http/security" ) @@ -100,7 +101,6 @@ func (handler *Handler) createComposeStackFromFileContent(w http.ResponseWriter, type composeStackFromGitRepositoryPayload struct { // Name of the stack Name string `example:"myStack" validate:"required"` - // URL of a Git repository hosting the Stack file RepositoryURL string `example:"https://github.com/openfaas/faas" validate:"required"` // Reference name of a Git repository hosting the Stack file @@ -112,8 +112,11 @@ type composeStackFromGitRepositoryPayload struct { // Password used in basic authentication. Required when RepositoryAuthentication is true. RepositoryPassword string `example:"myGitPassword"` // Path to the Stack file inside the Git repository - ComposeFilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"` - + ComposeFile string `example:"docker-compose.yml" default:"docker-compose.yml"` + // Applicable when deploying with multiple stack files + AdditionalFiles []string `example:"[nz.compose.yml, uat.compose.yml]"` + // Optional auto update configuration + AutoUpdate *portainer.StackAutoUpdate // A list of environment variables used during stack deployment Env []portainer.Pair } @@ -122,14 +125,18 @@ func (payload *composeStackFromGitRepositoryPayload) Validate(r *http.Request) e if govalidator.IsNull(payload.Name) { return errors.New("Invalid stack name") } - if govalidator.IsNull(payload.RepositoryURL) || !govalidator.IsURL(payload.RepositoryURL) { return errors.New("Invalid repository URL. Must correspond to a valid URL format") } - if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) { - return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled") + if govalidator.IsNull(payload.RepositoryReferenceName) { + payload.RepositoryReferenceName = defaultGitReferenceName + } + if payload.RepositoryAuthentication && govalidator.IsNull(payload.RepositoryPassword) { + return errors.New("Invalid repository credentials. Password must be specified when authentication is enabled") + } + if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil { + return err } - return nil } @@ -141,42 +148,72 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite } payload.Name = handler.ComposeStackManager.NormalizeStackName(payload.Name) - if payload.ComposeFilePathInRepository == "" { - payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName + if payload.ComposeFile == "" { + payload.ComposeFile = filesystem.ComposeFileDefaultName } isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, false) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err} } if !isUnique { - errorMessage := fmt.Sprintf("A stack with the name '%s' already exists", payload.Name) - return &httperror.HandlerError{http.StatusConflict, errorMessage, errors.New(errorMessage)} + return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.Name), Err: errStackAlreadyExists} + } + + //make sure the webhook ID is unique + if payload.AutoUpdate != nil && payload.AutoUpdate.Webhook != "" { + isUnique, err := handler.checkUniqueWebhookID(payload.AutoUpdate.Webhook) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for webhook ID collision", Err: err} + } + if !isUnique { + return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("Webhook ID: %s already exists", payload.AutoUpdate.Webhook), Err: errWebhookIDAlreadyExists} + } } stackID := handler.DataStore.Stack().GetNextIdentifier() stack := &portainer.Stack{ - ID: portainer.StackID(stackID), - Name: payload.Name, - Type: portainer.DockerComposeStack, - EndpointID: endpoint.ID, - EntryPoint: payload.ComposeFilePathInRepository, - Env: payload.Env, + ID: portainer.StackID(stackID), + Name: payload.Name, + Type: portainer.DockerComposeStack, + EndpointID: endpoint.ID, + EntryPoint: payload.ComposeFile, + AdditionalFiles: payload.AdditionalFiles, + AutoUpdate: payload.AutoUpdate, + Env: payload.Env, + GitConfig: &gittypes.RepoConfig{ + URL: payload.RepositoryURL, + ReferenceName: payload.RepositoryReferenceName, + ConfigFilePath: payload.ComposeFile, + }, Status: portainer.StackStatusActive, CreationDate: time.Now().Unix(), } + if payload.RepositoryAuthentication { + stack.GitConfig.Authentication = &gittypes.GitAuthentication{ + Username: payload.RepositoryUsername, + Password: payload.RepositoryPassword, + } + } + projectPath := handler.FileService.GetStackProjectPath(strconv.Itoa(int(stack.ID))) stack.ProjectPath = projectPath doCleanUp := true defer handler.cleanUp(stack, &doCleanUp) - err = handler.cloneAndSaveConfig(stack, projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, payload.ComposeFilePathInRepository, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) + err = handler.clone(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err} } + commitId, err := handler.latestCommitID(payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to fetch git repository id", Err: err} + } + stack.GitConfig.ConfigHash = commitId + config, configErr := handler.createComposeDeployConfig(r, stack, endpoint) if configErr != nil { return configErr @@ -187,6 +224,15 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err} } + if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" { + jobID, e := startAutoupdate(stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService) + if e != nil { + return e + } + + stack.AutoUpdate.JobID = jobID + } + stack.CreatedBy = config.user.Username err = handler.DataStore.Stack().CreateStack(stack) @@ -331,7 +377,7 @@ func (handler *Handler) createComposeDeployConfig(r *http.Request, stack *portai func (handler *Handler) deployComposeStack(config *composeStackDeploymentConfig) error { isAdminOrEndpointAdmin, err := handler.userIsAdminOrEndpointAdmin(config.user, config.endpoint.ID) if err != nil { - return err + return errors.Wrap(err, "failed to check user priviliges deploying a stack") } securitySettings := &config.endpoint.SecuritySettings @@ -344,15 +390,17 @@ func (handler *Handler) deployComposeStack(config *composeStackDeploymentConfig) !securitySettings.AllowContainerCapabilitiesForRegularUsers) && !isAdminOrEndpointAdmin { - composeFilePath := path.Join(config.stack.ProjectPath, config.stack.EntryPoint) - stackContent, err := handler.FileService.GetFileContent(composeFilePath) - if err != nil { - return err - } + for _, file := range append([]string{config.stack.EntryPoint}, config.stack.AdditionalFiles...) { + path := path.Join(config.stack.ProjectPath, file) + stackContent, err := handler.FileService.GetFileContent(path) + if err != nil { + return errors.Wrapf(err, "failed to get stack file content `%q`", path) + } - err = handler.isValidStackFile(stackContent, securitySettings) - if err != nil { - return err + err = handler.isValidStackFile(stackContent, securitySettings) + if err != nil { + return errors.Wrap(err, "compose file is invalid") + } } } @@ -363,7 +411,7 @@ func (handler *Handler) deployComposeStack(config *composeStackDeploymentConfig) err = handler.ComposeStackManager.Up(config.stack, config.endpoint) if err != nil { - return err + return errors.Wrap(err, "failed to start up the stack") } return handler.SwarmStackManager.Logout(config.endpoint) diff --git a/api/http/handler/stacks/create_kubernetes_stack_test.go b/api/http/handler/stacks/create_kubernetes_stack_test.go index f1b47286e..2bcd35ab5 100644 --- a/api/http/handler/stacks/create_kubernetes_stack_test.go +++ b/api/http/handler/stacks/create_kubernetes_stack_test.go @@ -23,6 +23,10 @@ func (g *git) ClonePrivateRepositoryWithBasicAuth(repositoryURL, referenceName s return g.ClonePublicRepository(repositoryURL, referenceName, destination) } +func (g *git) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) { + return "", nil +} + func TestCloneAndConvertGitRepoFile(t *testing.T) { dir, err := os.MkdirTemp("", "kube-create-stack") assert.NoError(t, err, "failed to create a tmp dir") diff --git a/api/http/handler/stacks/create_swarm_stack.go b/api/http/handler/stacks/create_swarm_stack.go index 99558b817..e76338384 100644 --- a/api/http/handler/stacks/create_swarm_stack.go +++ b/api/http/handler/stacks/create_swarm_stack.go @@ -13,6 +13,7 @@ import ( "github.com/portainer/libhttp/request" portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/filesystem" + gittypes "github.com/portainer/portainer/api/git/types" "github.com/portainer/portainer/api/http/security" ) @@ -121,7 +122,11 @@ type swarmStackFromGitRepositoryPayload struct { // Password used in basic authentication. Required when RepositoryAuthentication is true. RepositoryPassword string `example:"myGitPassword"` // Path to the Stack file inside the Git repository - ComposeFilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"` + ComposeFile string `example:"docker-compose.yml" default:"docker-compose.yml"` + // Applicable when deploying with multiple stack files + AdditionalFiles []string `example:"[nz.compose.yml, uat.compose.yml]"` + // Optional auto update configuration + AutoUpdate *portainer.StackAutoUpdate } func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) error { @@ -134,11 +139,14 @@ func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) err if govalidator.IsNull(payload.RepositoryURL) || !govalidator.IsURL(payload.RepositoryURL) { return errors.New("Invalid repository URL. Must correspond to a valid URL format") } - if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) { - return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled") + if govalidator.IsNull(payload.RepositoryReferenceName) { + payload.RepositoryReferenceName = defaultGitReferenceName } - if govalidator.IsNull(payload.ComposeFilePathInRepository) { - payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName + if payload.RepositoryAuthentication && govalidator.IsNull(payload.RepositoryPassword) { + return errors.New("Invalid repository credentials. Password must be specified when authentication is enabled") + } + if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil { + return err } return nil } @@ -147,44 +155,74 @@ func (handler *Handler) createSwarmStackFromGitRepository(w http.ResponseWriter, var payload swarmStackFromGitRepositoryPayload err := request.DecodeAndValidateJSONPayload(r, &payload) if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err} } payload.Name = handler.SwarmStackManager.NormalizeStackName(payload.Name) isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, true) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err} } if !isUnique { - errorMessage := fmt.Sprintf("A stack with the name '%s' is already running", payload.Name) - return &httperror.HandlerError{http.StatusConflict, errorMessage, errors.New(errorMessage)} + return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.Name), Err: errStackAlreadyExists} + } + + //make sure the webhook ID is unique + if payload.AutoUpdate != nil && payload.AutoUpdate.Webhook != "" { + isUnique, err := handler.checkUniqueWebhookID(payload.AutoUpdate.Webhook) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for webhook ID collision", Err: err} + } + if !isUnique { + return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("Webhook ID: %s already exists", payload.AutoUpdate.Webhook), Err: errWebhookIDAlreadyExists} + } } stackID := handler.DataStore.Stack().GetNextIdentifier() stack := &portainer.Stack{ - ID: portainer.StackID(stackID), - Name: payload.Name, - Type: portainer.DockerSwarmStack, - SwarmID: payload.SwarmID, - EndpointID: endpoint.ID, - EntryPoint: payload.ComposeFilePathInRepository, + ID: portainer.StackID(stackID), + Name: payload.Name, + Type: portainer.DockerSwarmStack, + SwarmID: payload.SwarmID, + EndpointID: endpoint.ID, + EntryPoint: payload.ComposeFile, + AdditionalFiles: payload.AdditionalFiles, + AutoUpdate: payload.AutoUpdate, + GitConfig: &gittypes.RepoConfig{ + URL: payload.RepositoryURL, + ReferenceName: payload.RepositoryReferenceName, + ConfigFilePath: payload.ComposeFile, + }, Env: payload.Env, Status: portainer.StackStatusActive, CreationDate: time.Now().Unix(), } + if payload.RepositoryAuthentication { + stack.GitConfig.Authentication = &gittypes.GitAuthentication{ + Username: payload.RepositoryUsername, + Password: payload.RepositoryPassword, + } + } + projectPath := handler.FileService.GetStackProjectPath(strconv.Itoa(int(stack.ID))) stack.ProjectPath = projectPath doCleanUp := true defer handler.cleanUp(stack, &doCleanUp) - err = handler.cloneAndSaveConfig(stack, projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, payload.ComposeFilePathInRepository, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) + err = handler.clone(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err} } + commitId, err := handler.latestCommitID(payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to fetch git repository id", Err: err} + } + stack.GitConfig.ConfigHash = commitId + config, configErr := handler.createSwarmDeployConfig(r, stack, endpoint, false) if configErr != nil { return configErr @@ -192,14 +230,23 @@ func (handler *Handler) createSwarmStackFromGitRepository(w http.ResponseWriter, err = handler.deploySwarmStack(config) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err} + } + + if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" { + jobID, e := startAutoupdate(stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService) + if e != nil { + return e + } + + stack.AutoUpdate.JobID = jobID } stack.CreatedBy = config.user.Username err = handler.DataStore.Stack().CreateStack(stack) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack inside the database", Err: err} } doCleanUp = false @@ -350,16 +397,17 @@ func (handler *Handler) deploySwarmStack(config *swarmStackDeploymentConfig) err settings := &config.endpoint.SecuritySettings if !settings.AllowBindMountsForRegularUsers && !isAdminOrEndpointAdmin { - composeFilePath := path.Join(config.stack.ProjectPath, config.stack.EntryPoint) + for _, file := range append([]string{config.stack.EntryPoint}, config.stack.AdditionalFiles...) { + path := path.Join(config.stack.ProjectPath, file) + stackContent, err := handler.FileService.GetFileContent(path) + if err != nil { + return err + } - stackContent, err := handler.FileService.GetFileContent(composeFilePath) - if err != nil { - return err - } - - err = handler.isValidStackFile(stackContent, settings) - if err != nil { - return err + err = handler.isValidStackFile(stackContent, settings) + if err != nil { + return err + } } } diff --git a/api/http/handler/stacks/handler.go b/api/http/handler/stacks/handler.go index dcdfce41b..f5ba2c983 100644 --- a/api/http/handler/stacks/handler.go +++ b/api/http/handler/stacks/handler.go @@ -2,23 +2,30 @@ package stacks import ( "context" - "errors" + "fmt" "net/http" "strings" "sync" "github.com/docker/docker/api/types" "github.com/gorilla/mux" + "github.com/pkg/errors" httperror "github.com/portainer/libhttp/error" portainer "github.com/portainer/portainer/api" + bolterrors "github.com/portainer/portainer/api/bolt/errors" "github.com/portainer/portainer/api/docker" "github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/internal/authorization" + "github.com/portainer/portainer/api/scheduler" + "github.com/portainer/portainer/api/stacks" ) +const defaultGitReferenceName = "refs/heads/master" + var ( - errStackAlreadyExists = errors.New("A stack already exists with this name") - errStackNotExternal = errors.New("Not an external stack") + errStackAlreadyExists = errors.New("A stack already exists with this name") + errWebhookIDAlreadyExists = errors.New("A webhook ID already exists") + errStackNotExternal = errors.New("Not an external stack") ) // Handler is the HTTP handler used to handle stack operations. @@ -34,6 +41,8 @@ type Handler struct { SwarmStackManager portainer.SwarmStackManager ComposeStackManager portainer.ComposeStackManager KubernetesDeployer portainer.KubernetesDeployer + Scheduler *scheduler.Scheduler + StackDeployer stacks.StackDeployer } // NewHandler creates a handler to manage stack operations. @@ -57,7 +66,9 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler { h.Handle("/stacks/{id}", bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackUpdate))).Methods(http.MethodPut) h.Handle("/stacks/{id}/git", - bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackUpdateGit))).Methods(http.MethodPut) + bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackUpdateGit))).Methods(http.MethodPost) + h.Handle("/stacks/{id}/git/redeploy", + bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackGitRedeploy))).Methods(http.MethodPut) h.Handle("/stacks/{id}/file", bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackFile))).Methods(http.MethodGet) h.Handle("/stacks/{id}/migrate", @@ -66,6 +77,9 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler { bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackStart))).Methods(http.MethodPost) h.Handle("/stacks/{id}/stop", bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackStop))).Methods(http.MethodPost) + h.Handle("/stacks/webhooks/{webhookID}", + httperror.LoggerHandler(h.webhookInvoke)).Methods(http.MethodPost) + return h } @@ -159,3 +173,34 @@ func (handler *Handler) checkUniqueName(endpoint *portainer.Endpoint, name strin return true, nil } + +func (handler *Handler) checkUniqueWebhookID(webhookID string) (bool, error) { + _, err := handler.DataStore.Stack().StackByWebhookID(webhookID) + if err == bolterrors.ErrObjectNotFound { + return true, nil + } + return false, err +} + +func (handler *Handler) clone(projectPath, repositoryURL, refName string, auth bool, username, password string) error { + if !auth { + username = "" + password = "" + } + + err := handler.GitService.CloneRepository(projectPath, repositoryURL, refName, username, password) + if err != nil { + return fmt.Errorf("unable to clone git repository: %w", err) + } + + return nil +} + +func (handler *Handler) latestCommitID(repositoryURL, refName string, auth bool, username, password string) (string, error) { + if !auth { + username = "" + password = "" + } + + return handler.GitService.LatestCommitID(repositoryURL, refName, username, password) +} diff --git a/api/http/handler/stacks/helper.go b/api/http/handler/stacks/helper.go new file mode 100644 index 000000000..dd42330c4 --- /dev/null +++ b/api/http/handler/stacks/helper.go @@ -0,0 +1,24 @@ +package stacks + +import ( + "time" + + "github.com/asaskevich/govalidator" + "github.com/pkg/errors" + portainer "github.com/portainer/portainer/api" +) + +func validateStackAutoUpdate(autoUpdate *portainer.StackAutoUpdate) error { + if autoUpdate == nil { + return nil + } + if autoUpdate.Webhook != "" && !govalidator.IsUUID(autoUpdate.Webhook) { + return errors.New("invalid Webhook format") + } + if autoUpdate.Interval != "" { + if _, err := time.ParseDuration(autoUpdate.Interval); err != nil { + return errors.New("invalid Interval format") + } + } + return nil +} diff --git a/api/http/handler/stacks/helper_test.go b/api/http/handler/stacks/helper_test.go new file mode 100644 index 000000000..c3e564349 --- /dev/null +++ b/api/http/handler/stacks/helper_test.go @@ -0,0 +1,42 @@ +package stacks + +import ( + "testing" + + portainer "github.com/portainer/portainer/api" + "github.com/stretchr/testify/assert" +) + +func Test_ValidateStackAutoUpdate(t *testing.T) { + tests := []struct { + name string + value *portainer.StackAutoUpdate + wantErr bool + }{ + { + name: "webhook is not a valid UUID", + value: &portainer.StackAutoUpdate{Webhook: "fake-webhook"}, + wantErr: true, + }, + { + name: "incorrect interval value", + value: &portainer.StackAutoUpdate{Interval: "1dd2hh3mm"}, + wantErr: true, + }, + { + name: "valid auto update", + value: &portainer.StackAutoUpdate{ + Webhook: "8dce8c2f-9ca1-482b-ad20-271e86536ada", + Interval: "5h30m40s10ms", + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validateStackAutoUpdate(tt.value) + assert.Equalf(t, tt.wantErr, err != nil, "received %+v", err) + }) + } +} diff --git a/api/http/handler/stacks/stack_associate.go b/api/http/handler/stacks/stack_associate.go index 55397a323..14a04ed89 100644 --- a/api/http/handler/stacks/stack_associate.go +++ b/api/http/handler/stacks/stack_associate.go @@ -2,6 +2,9 @@ package stacks import ( "fmt" + "net/http" + "time" + httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" "github.com/portainer/libhttp/response" @@ -9,8 +12,6 @@ import ( bolterrors "github.com/portainer/portainer/api/bolt/errors" "github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/internal/stackutils" - "net/http" - "time" ) // PUT request on /api/stacks/:id/associate?endpointId=&swarmId=&orphanedRunning= @@ -87,5 +88,10 @@ func (handler *Handler) stackAssociate(w http.ResponseWriter, r *http.Request) * stack.ResourceControl = resourceControl + if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { + // sanitize password in the http response to minimise possible security leaks + stack.GitConfig.Authentication.Password = "" + } + return response.JSON(w, stack) } diff --git a/api/http/handler/stacks/stack_create.go b/api/http/handler/stacks/stack_create.go index 53533ca33..d21bf3a1c 100644 --- a/api/http/handler/stacks/stack_create.go +++ b/api/http/handler/stacks/stack_create.go @@ -1,19 +1,17 @@ package stacks import ( - "errors" - "fmt" "log" "net/http" "github.com/docker/cli/cli/compose/loader" "github.com/docker/cli/cli/compose/types" + "github.com/pkg/errors" httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" "github.com/portainer/libhttp/response" portainer "github.com/portainer/portainer/api" bolterrors "github.com/portainer/portainer/api/bolt/errors" - gittypes "github.com/portainer/portainer/api/git/types" "github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/internal/authorization" "github.com/portainer/portainer/api/internal/endpointutils" @@ -129,7 +127,7 @@ func (handler *Handler) createComposeStack(w http.ResponseWriter, r *http.Reques return handler.createComposeStackFromFileUpload(w, r, endpoint, userID) } - return &httperror.HandlerError{http.StatusBadRequest, "Invalid value for query parameter: method. Value must be one of: string, repository or file", errors.New(request.ErrInvalidQueryParameter)} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid value for query parameter: method. Value must be one of: string, repository or file", Err: errors.New(request.ErrInvalidQueryParameter)} } func (handler *Handler) createSwarmStack(w http.ResponseWriter, r *http.Request, method string, endpoint *portainer.Endpoint, userID portainer.UserID) *httperror.HandlerError { @@ -142,7 +140,7 @@ func (handler *Handler) createSwarmStack(w http.ResponseWriter, r *http.Request, return handler.createSwarmStackFromFileUpload(w, r, endpoint, userID) } - return &httperror.HandlerError{http.StatusBadRequest, "Invalid value for query parameter: method. Value must be one of: string, repository or file", errors.New(request.ErrInvalidQueryParameter)} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid value for query parameter: method. Value must be one of: string, repository or file", Err: errors.New(request.ErrInvalidQueryParameter)} } func (handler *Handler) createKubernetesStack(w http.ResponseWriter, r *http.Request, method string, endpoint *portainer.Endpoint) *httperror.HandlerError { @@ -232,24 +230,11 @@ func (handler *Handler) decorateStackResponse(w http.ResponseWriter, stack *port } stack.ResourceControl = resourceControl + + if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { + // sanitize password in the http response to minimise possible security leaks + stack.GitConfig.Authentication.Password = "" + } + return response.JSON(w, stack) } - -func (handler *Handler) cloneAndSaveConfig(stack *portainer.Stack, projectPath, repositoryURL, refName, configFilePath string, auth bool, username, password string) error { - if !auth { - username = "" - password = "" - } - - err := handler.GitService.CloneRepository(projectPath, repositoryURL, refName, username, password) - if err != nil { - return fmt.Errorf("unable to clone git repository: %w", err) - } - - stack.GitConfig = &gittypes.RepoConfig{ - URL: repositoryURL, - ReferenceName: refName, - ConfigFilePath: configFilePath, - } - return nil -} diff --git a/api/http/handler/stacks/stack_create_test.go b/api/http/handler/stacks/stack_create_test.go deleted file mode 100644 index 414948378..000000000 --- a/api/http/handler/stacks/stack_create_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package stacks - -import ( - "testing" - - portainer "github.com/portainer/portainer/api" - gittypes "github.com/portainer/portainer/api/git/types" - "github.com/portainer/portainer/api/http/security" - "github.com/portainer/portainer/api/internal/testhelpers" - "github.com/stretchr/testify/assert" -) - -func Test_stackHandler_cloneAndSaveConfig_shouldCallGitCloneAndSaveConfigOnStack(t *testing.T) { - handler := NewHandler(&security.RequestBouncer{}) - handler.GitService = testhelpers.NewGitService() - - url := "url" - refName := "ref" - configPath := "path" - stack := &portainer.Stack{} - err := handler.cloneAndSaveConfig(stack, "", url, refName, configPath, false, "", "") - assert.NoError(t, err, "clone and save should not fail") - - assert.Equal(t, gittypes.RepoConfig{ - URL: url, - ReferenceName: refName, - ConfigFilePath: configPath, - }, *stack.GitConfig) -} diff --git a/api/http/handler/stacks/stack_delete.go b/api/http/handler/stacks/stack_delete.go index 11d2c31b5..f7ce69bf1 100644 --- a/api/http/handler/stacks/stack_delete.go +++ b/api/http/handler/stacks/stack_delete.go @@ -96,6 +96,11 @@ func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *htt } } + // stop scheduler updates of the stack before removal + if stack.AutoUpdate != nil { + stopAutoupdate(stack.ID, stack.AutoUpdate.JobID, *handler.Scheduler) + } + err = handler.deleteStack(stack, endpoint) if err != nil { return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err} diff --git a/api/http/handler/stacks/stack_inspect.go b/api/http/handler/stacks/stack_inspect.go index 7f445397f..d0797700e 100644 --- a/api/http/handler/stacks/stack_inspect.go +++ b/api/http/handler/stacks/stack_inspect.go @@ -78,5 +78,10 @@ func (handler *Handler) stackInspect(w http.ResponseWriter, r *http.Request) *ht } } + if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { + // sanitize password in the http response to minimise possible security leaks + stack.GitConfig.Authentication.Password = "" + } + return response.JSON(w, stack) } diff --git a/api/http/handler/stacks/stack_list.go b/api/http/handler/stacks/stack_list.go index 81255d032..f4b4fb673 100644 --- a/api/http/handler/stacks/stack_list.go +++ b/api/http/handler/stacks/stack_list.go @@ -1,9 +1,10 @@ package stacks import ( - httperrors "github.com/portainer/portainer/api/http/errors" "net/http" + httperrors "github.com/portainer/portainer/api/http/errors" + httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" "github.com/portainer/libhttp/response" @@ -80,6 +81,13 @@ func (handler *Handler) stackList(w http.ResponseWriter, r *http.Request) *httpe stacks = authorization.FilterAuthorizedStacks(stacks, user, userTeamIDs) } + for _, stack := range stacks { + if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { + // sanitize password in the http response to minimise possible security leaks + stack.GitConfig.Authentication.Password = "" + } + } + return response.JSON(w, stacks) } diff --git a/api/http/handler/stacks/stack_migrate.go b/api/http/handler/stacks/stack_migrate.go index 9f13ef1f5..2323a50ab 100644 --- a/api/http/handler/stacks/stack_migrate.go +++ b/api/http/handler/stacks/stack_migrate.go @@ -150,6 +150,11 @@ func (handler *Handler) stackMigrate(w http.ResponseWriter, r *http.Request) *ht return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack changes inside the database", err} } + if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { + // sanitize password in the http response to minimise possible security leaks + stack.GitConfig.Authentication.Password = "" + } + return response.JSON(w, stack) } diff --git a/api/http/handler/stacks/stack_start.go b/api/http/handler/stacks/stack_start.go index dc4163e92..ff3f5f268 100644 --- a/api/http/handler/stacks/stack_start.go +++ b/api/http/handler/stacks/stack_start.go @@ -85,6 +85,17 @@ func (handler *Handler) stackStart(w http.ResponseWriter, r *http.Request) *http return &httperror.HandlerError{http.StatusBadRequest, "Stack is already active", errors.New("Stack is already active")} } + if stack.AutoUpdate != nil && stack.AutoUpdate.Interval != "" { + stopAutoupdate(stack.ID, stack.AutoUpdate.JobID, *handler.Scheduler) + + jobID, e := startAutoupdate(stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService) + if e != nil { + return e + } + + stack.AutoUpdate.JobID = jobID + } + err = handler.startStack(stack, endpoint) if err != nil { return &httperror.HandlerError{http.StatusInternalServerError, "Unable to start stack", err} @@ -96,6 +107,11 @@ func (handler *Handler) stackStart(w http.ResponseWriter, r *http.Request) *http return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update stack status", err} } + if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { + // sanitize password in the http response to minimise possible security leaks + stack.GitConfig.Authentication.Password = "" + } + return response.JSON(w, stack) } diff --git a/api/http/handler/stacks/stack_stop.go b/api/http/handler/stacks/stack_stop.go index 700cec976..1e24d9607 100644 --- a/api/http/handler/stacks/stack_stop.go +++ b/api/http/handler/stacks/stack_stop.go @@ -74,6 +74,12 @@ func (handler *Handler) stackStop(w http.ResponseWriter, r *http.Request) *httpe return &httperror.HandlerError{http.StatusBadRequest, "Stack is already inactive", errors.New("Stack is already inactive")} } + // stop scheduler updates of the stack before stopping + if stack.AutoUpdate != nil && stack.AutoUpdate.JobID != "" { + stopAutoupdate(stack.ID, stack.AutoUpdate.JobID, *handler.Scheduler) + stack.AutoUpdate.JobID = "" + } + err = handler.stopStack(stack, endpoint) if err != nil { return &httperror.HandlerError{http.StatusInternalServerError, "Unable to stop stack", err} @@ -85,6 +91,11 @@ func (handler *Handler) stackStop(w http.ResponseWriter, r *http.Request) *httpe return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update stack status", err} } + if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { + // sanitize password in the http response to minimise possible security leaks + stack.GitConfig.Authentication.Password = "" + } + return response.JSON(w, stack) } diff --git a/api/http/handler/stacks/stack_update.go b/api/http/handler/stacks/stack_update.go index ee02fb2b5..24f85f817 100644 --- a/api/http/handler/stacks/stack_update.go +++ b/api/http/handler/stacks/stack_update.go @@ -128,6 +128,11 @@ func (handler *Handler) stackUpdate(w http.ResponseWriter, r *http.Request) *htt return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack changes inside the database", err} } + if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { + // sanitize password in the http response to minimise possible security leaks + stack.GitConfig.Authentication.Password = "" + } + return response.JSON(w, stack) } diff --git a/api/http/handler/stacks/stack_update_git.go b/api/http/handler/stacks/stack_update_git.go index dd46e409e..48b7d2af4 100644 --- a/api/http/handler/stacks/stack_update_git.go +++ b/api/http/handler/stacks/stack_update_git.go @@ -2,10 +2,7 @@ package stacks import ( "errors" - "fmt" - "log" "net/http" - "time" "github.com/asaskevich/govalidator" httperror "github.com/portainer/libhttp/error" @@ -13,22 +10,28 @@ import ( "github.com/portainer/libhttp/response" portainer "github.com/portainer/portainer/api" bolterrors "github.com/portainer/portainer/api/bolt/errors" - "github.com/portainer/portainer/api/filesystem" + gittypes "github.com/portainer/portainer/api/git/types" httperrors "github.com/portainer/portainer/api/http/errors" "github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/internal/stackutils" ) -type updateStackGitPayload struct { +type stackGitUpdatePayload struct { + AutoUpdate *portainer.StackAutoUpdate + Env []portainer.Pair RepositoryReferenceName string RepositoryAuthentication bool RepositoryUsername string RepositoryPassword string } -func (payload *updateStackGitPayload) Validate(r *http.Request) error { - if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) { - return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled") +func (payload *stackGitUpdatePayload) Validate(r *http.Request) error { + if govalidator.IsNull(payload.RepositoryReferenceName) { + payload.RepositoryReferenceName = defaultGitReferenceName + } + + if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil { + return err } return nil } @@ -53,18 +56,23 @@ func (payload *updateStackGitPayload) Validate(r *http.Request) error { func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { stackID, err := request.RetrieveNumericRouteVariableValue(r, "id") if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid stack identifier route variable", Err: err} + } + + var payload stackGitUpdatePayload + err = request.DecodeAndValidateJSONPayload(r, &payload) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err} } stack, err := handler.DataStore.Stack().Stack(portainer.StackID(stackID)) if err == bolterrors.ErrObjectNotFound { - return &httperror.HandlerError{http.StatusNotFound, "Unable to find a stack with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find a stack with the specified identifier inside the database", Err: err} } else if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err} - } - - if stack.GitConfig == nil { - return &httperror.HandlerError{http.StatusBadRequest, "Stack is not created from git", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find a stack with the specified identifier inside the database", Err: err} + } else if stack.GitConfig == nil { + msg := "No Git config in the found stack" + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: msg, Err: errors.New(msg)} } // TODO: this is a work-around for stacks created with Portainer version >= 1.17.1 @@ -72,7 +80,7 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) * // can use the optional EndpointID query parameter to associate a valid endpoint identifier to the stack. endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", true) if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: endpointId", err} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid query parameter: endpointId", Err: err} } if endpointID != int(stack.EndpointID) { stack.EndpointID = portainer.EndpointID(endpointID) @@ -80,117 +88,75 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) * endpoint, err := handler.DataStore.Endpoint().Endpoint(stack.EndpointID) if err == bolterrors.ErrObjectNotFound { - return &httperror.HandlerError{http.StatusNotFound, "Unable to find the endpoint associated to the stack inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err} } else if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find the endpoint associated to the stack inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err} } err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint) if err != nil { - return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err} + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to access endpoint", Err: err} } resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve a resource control associated to the stack", Err: err} } securityContext, err := security.RetrieveRestrictedRequestContext(r) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err} } access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to verify user authorizations to validate stack access", Err: err} } if !access { - return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", httperrors.ErrResourceAccessDenied} + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Access denied to resource", Err: httperrors.ErrResourceAccessDenied} } - var payload updateStackGitPayload - err = request.DecodeAndValidateJSONPayload(r, &payload) - if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err} + //stop the autoupdate job if there is any + if stack.AutoUpdate != nil { + stopAutoupdate(stack.ID, stack.AutoUpdate.JobID, *handler.Scheduler) } + //update retrieved stack data based on the payload stack.GitConfig.ReferenceName = payload.RepositoryReferenceName + stack.AutoUpdate = payload.AutoUpdate + stack.Env = payload.Env - backupProjectPath := fmt.Sprintf("%s-old", stack.ProjectPath) - err = filesystem.MoveDirectory(stack.ProjectPath, backupProjectPath) - if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to move git repository directory", err} + stack.GitConfig.Authentication = nil + if payload.RepositoryAuthentication { + password := payload.RepositoryPassword + if password == "" && stack.GitConfig != nil && stack.GitConfig.Authentication != nil { + password = stack.GitConfig.Authentication.Password + } + stack.GitConfig.Authentication = &gittypes.GitAuthentication{ + Username: payload.RepositoryUsername, + Password: password, + } } - repositoryUsername := payload.RepositoryUsername - repositoryPassword := payload.RepositoryPassword - if !payload.RepositoryAuthentication { - repositoryUsername = "" - repositoryPassword = "" - } - - err = handler.GitService.CloneRepository(stack.ProjectPath, stack.GitConfig.URL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword) - if err != nil { - restoreError := filesystem.MoveDirectory(backupProjectPath, stack.ProjectPath) - if restoreError != nil { - log.Printf("[WARN] [http,stacks,git] [error: %s] [message: failed restoring backup folder]", restoreError) + if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" { + jobID, e := startAutoupdate(stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService) + if e != nil { + return e } - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to clone git repository", err} - } - - defer func() { - err = handler.FileService.RemoveDirectory(backupProjectPath) - if err != nil { - log.Printf("[WARN] [http,stacks,git] [error: %s] [message: unable to remove git repository directory]", err) - } - }() - - httpErr := handler.deployStack(r, stack, endpoint) - if httpErr != nil { - return httpErr + stack.AutoUpdate.JobID = jobID } + //save the updated stack to DB err = handler.DataStore.Stack().UpdateStack(stack.ID, stack) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack changes inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack changes inside the database", Err: err} + } + + if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { + // sanitize password in the http response to minimise possible security leaks + stack.GitConfig.Authentication.Password = "" } return response.JSON(w, stack) } - -func (handler *Handler) deployStack(r *http.Request, stack *portainer.Stack, endpoint *portainer.Endpoint) *httperror.HandlerError { - if stack.Type == portainer.DockerSwarmStack { - config, httpErr := handler.createSwarmDeployConfig(r, stack, endpoint, false) - if httpErr != nil { - return httpErr - } - - err := handler.deploySwarmStack(config) - if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err} - } - - stack.UpdateDate = time.Now().Unix() - stack.UpdatedBy = config.user.Username - stack.Status = portainer.StackStatusActive - - return nil - } - - config, httpErr := handler.createComposeDeployConfig(r, stack, endpoint) - if httpErr != nil { - return httpErr - } - - err := handler.deployComposeStack(config) - if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err} - } - - stack.UpdateDate = time.Now().Unix() - stack.UpdatedBy = config.user.Username - stack.Status = portainer.StackStatusActive - - return nil -} diff --git a/api/http/handler/stacks/stack_update_git_redeploy.go b/api/http/handler/stacks/stack_update_git_redeploy.go new file mode 100644 index 000000000..c338757e0 --- /dev/null +++ b/api/http/handler/stacks/stack_update_git_redeploy.go @@ -0,0 +1,190 @@ +package stacks + +import ( + "fmt" + "log" + "net/http" + "time" + + "github.com/asaskevich/govalidator" + httperror "github.com/portainer/libhttp/error" + "github.com/portainer/libhttp/request" + "github.com/portainer/libhttp/response" + portainer "github.com/portainer/portainer/api" + bolterrors "github.com/portainer/portainer/api/bolt/errors" + "github.com/portainer/portainer/api/filesystem" + httperrors "github.com/portainer/portainer/api/http/errors" + "github.com/portainer/portainer/api/http/security" + "github.com/portainer/portainer/api/internal/stackutils" +) + +type stackGitRedployPayload struct { + RepositoryReferenceName string + RepositoryAuthentication bool + RepositoryUsername string + RepositoryPassword string + Env []portainer.Pair +} + +func (payload *stackGitRedployPayload) Validate(r *http.Request) error { + if govalidator.IsNull(payload.RepositoryReferenceName) { + payload.RepositoryReferenceName = defaultGitReferenceName + } + + return nil +} + +// PUT request on /api/stacks/:id/git?endpointId= +func (handler *Handler) stackGitRedeploy(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { + stackID, err := request.RetrieveNumericRouteVariableValue(r, "id") + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid stack identifier route variable", Err: err} + } + + stack, err := handler.DataStore.Stack().Stack(portainer.StackID(stackID)) + if err == bolterrors.ErrObjectNotFound { + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find a stack with the specified identifier inside the database", Err: err} + } else if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find a stack with the specified identifier inside the database", Err: err} + } + + if stack.GitConfig == nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Stack is not created from git", Err: err} + } + + // TODO: this is a work-around for stacks created with Portainer version >= 1.17.1 + // The EndpointID property is not available for these stacks, this API endpoint + // can use the optional EndpointID query parameter to associate a valid endpoint identifier to the stack. + endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", true) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid query parameter: endpointId", Err: err} + } + if endpointID != int(stack.EndpointID) { + stack.EndpointID = portainer.EndpointID(endpointID) + } + + endpoint, err := handler.DataStore.Endpoint().Endpoint(stack.EndpointID) + if err == bolterrors.ErrObjectNotFound { + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err} + } else if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err} + } + + err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to access endpoint", Err: err} + } + + resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve a resource control associated to the stack", Err: err} + } + + securityContext, err := security.RetrieveRestrictedRequestContext(r) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err} + } + + access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to verify user authorizations to validate stack access", Err: err} + } + if !access { + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Access denied to resource", Err: httperrors.ErrResourceAccessDenied} + } + + var payload stackGitRedployPayload + err = request.DecodeAndValidateJSONPayload(r, &payload) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err} + } + + stack.GitConfig.ReferenceName = payload.RepositoryReferenceName + stack.Env = payload.Env + + backupProjectPath := fmt.Sprintf("%s-old", stack.ProjectPath) + err = filesystem.MoveDirectory(stack.ProjectPath, backupProjectPath) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to move git repository directory", Err: err} + } + + repositoryUsername := "" + repositoryPassword := "" + if payload.RepositoryAuthentication { + repositoryPassword = payload.RepositoryPassword + if repositoryPassword == "" && stack.GitConfig != nil && stack.GitConfig.Authentication != nil { + repositoryPassword = stack.GitConfig.Authentication.Password + } + repositoryUsername = payload.RepositoryUsername + } + + err = handler.GitService.CloneRepository(stack.ProjectPath, stack.GitConfig.URL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword) + if err != nil { + restoreError := filesystem.MoveDirectory(backupProjectPath, stack.ProjectPath) + if restoreError != nil { + log.Printf("[WARN] [http,stacks,git] [error: %s] [message: failed restoring backup folder]", restoreError) + } + + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err} + } + + defer func() { + err = handler.FileService.RemoveDirectory(backupProjectPath) + if err != nil { + log.Printf("[WARN] [http,stacks,git] [error: %s] [message: unable to remove git repository directory]", err) + } + }() + + httpErr := handler.deployStack(r, stack, endpoint) + if httpErr != nil { + return httpErr + } + + err = handler.DataStore.Stack().UpdateStack(stack.ID, stack) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack changes inside the database", Err: err} + } + + if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { + // sanitize password in the http response to minimise possible security leaks + stack.GitConfig.Authentication.Password = "" + } + + return response.JSON(w, stack) +} + +func (handler *Handler) deployStack(r *http.Request, stack *portainer.Stack, endpoint *portainer.Endpoint) *httperror.HandlerError { + if stack.Type == portainer.DockerSwarmStack { + config, httpErr := handler.createSwarmDeployConfig(r, stack, endpoint, false) + if httpErr != nil { + return httpErr + } + + err := handler.deploySwarmStack(config) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err} + } + + stack.UpdateDate = time.Now().Unix() + stack.UpdatedBy = config.user.Username + stack.Status = portainer.StackStatusActive + + return nil + } + + config, httpErr := handler.createComposeDeployConfig(r, stack, endpoint) + if httpErr != nil { + return httpErr + } + + err := handler.deployComposeStack(config) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err} + } + + stack.UpdateDate = time.Now().Unix() + stack.UpdatedBy = config.user.Username + stack.Status = portainer.StackStatusActive + + return nil +} diff --git a/api/http/handler/stacks/webhook_invoke.go b/api/http/handler/stacks/webhook_invoke.go new file mode 100644 index 000000000..01c9d701b --- /dev/null +++ b/api/http/handler/stacks/webhook_invoke.go @@ -0,0 +1,54 @@ +package stacks + +import ( + "log" + "net/http" + + "github.com/gofrs/uuid" + + "github.com/portainer/libhttp/response" + + bolterrors "github.com/portainer/portainer/api/bolt/errors" + "github.com/portainer/portainer/api/stacks" + + httperror "github.com/portainer/libhttp/error" + "github.com/portainer/libhttp/request" +) + +func (handler *Handler) webhookInvoke(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { + webhookID, err := retrieveUUIDRouteVariableValue(r, "webhookID") + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid webhook identifier route variable", Err: err} + } + + stack, err := handler.DataStore.Stack().StackByWebhookID(webhookID.String()) + if err != nil { + statusCode := http.StatusInternalServerError + if err == bolterrors.ErrObjectNotFound { + statusCode = http.StatusNotFound + } + return &httperror.HandlerError{StatusCode: statusCode, Message: "Unable to find the stack by webhook ID", Err: err} + } + + if err = stacks.RedeployWhenChanged(stack.ID, handler.StackDeployer, handler.DataStore, handler.GitService); err != nil { + log.Printf("[ERROR] %s\n", err) + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Failed to update the stack", Err: err} + } + + return response.Empty(w) +} + +func retrieveUUIDRouteVariableValue(r *http.Request, name string) (uuid.UUID, error) { + webhookID, err := request.RetrieveRouteVariableValue(r, name) + if err != nil { + return uuid.Nil, err + } + + uid, err := uuid.FromString(webhookID) + + if err != nil { + return uuid.Nil, err + } + + return uid, nil +} diff --git a/api/http/handler/stacks/webhook_invoke_test.go b/api/http/handler/stacks/webhook_invoke_test.go new file mode 100644 index 000000000..cc6656519 --- /dev/null +++ b/api/http/handler/stacks/webhook_invoke_test.go @@ -0,0 +1,59 @@ +package stacks + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/gofrs/uuid" + "github.com/stretchr/testify/assert" + + portainer "github.com/portainer/portainer/api" + + "github.com/portainer/portainer/api/bolt/bolttest" +) + +func TestHandler_webhookInvoke(t *testing.T) { + store, teardown := bolttest.MustNewTestStore(true) + defer teardown() + + webhookID := newGuidString(t) + store.StackService.CreateStack(&portainer.Stack{ + AutoUpdate: &portainer.StackAutoUpdate{ + Webhook: webhookID, + }, + }) + + h := NewHandler(nil) + h.DataStore = store + + t.Run("invalid uuid results in http.StatusBadRequest", func(t *testing.T) { + w := httptest.NewRecorder() + req := newRequest("notuuid") + h.Router.ServeHTTP(w, req) + assert.Equal(t, http.StatusBadRequest, w.Code) + }) + t.Run("registered webhook ID in http.StatusNoContent", func(t *testing.T) { + w := httptest.NewRecorder() + req := newRequest(webhookID) + h.Router.ServeHTTP(w, req) + assert.Equal(t, http.StatusNoContent, w.Code) + }) + t.Run("unregistered webhook ID in http.StatusNotFound", func(t *testing.T) { + w := httptest.NewRecorder() + req := newRequest(newGuidString(t)) + h.Router.ServeHTTP(w, req) + assert.Equal(t, http.StatusNotFound, w.Code) + }) +} + +func newGuidString(t *testing.T) string { + uuid, err := uuid.NewV4() + assert.NoError(t, err) + + return uuid.String() +} + +func newRequest(webhookID string) *http.Request { + return httptest.NewRequest(http.MethodPost, "/stacks/webhooks/"+webhookID, nil) +} diff --git a/api/http/server.go b/api/http/server.go index 13ab4dcda..a38c1922a 100644 --- a/api/http/server.go +++ b/api/http/server.go @@ -50,6 +50,8 @@ import ( "github.com/portainer/portainer/api/internal/authorization" "github.com/portainer/portainer/api/internal/ssl" "github.com/portainer/portainer/api/kubernetes/cli" + "github.com/portainer/portainer/api/scheduler" + stackdeployer "github.com/portainer/portainer/api/stacks" ) // Server implements the portainer.Server interface @@ -79,8 +81,10 @@ type Server struct { DockerClientFactory *docker.ClientFactory KubernetesClientFactory *cli.ClientFactory KubernetesDeployer portainer.KubernetesDeployer + Scheduler *scheduler.Scheduler ShutdownCtx context.Context ShutdownTrigger context.CancelFunc + StackDeployer stackdeployer.StackDeployer } // Start starts the HTTP server @@ -185,10 +189,12 @@ func (server *Server) Start() error { stackHandler.DataStore = server.DataStore stackHandler.DockerClientFactory = server.DockerClientFactory stackHandler.FileService = server.FileService - stackHandler.SwarmStackManager = server.SwarmStackManager - stackHandler.ComposeStackManager = server.ComposeStackManager stackHandler.KubernetesDeployer = server.KubernetesDeployer stackHandler.GitService = server.GitService + stackHandler.Scheduler = server.Scheduler + stackHandler.SwarmStackManager = server.SwarmStackManager + stackHandler.ComposeStackManager = server.ComposeStackManager + stackHandler.StackDeployer = server.StackDeployer var tagHandler = tags.NewHandler(requestBouncer) tagHandler.DataStore = server.DataStore diff --git a/api/internal/stackutils/stackutils.go b/api/internal/stackutils/stackutils.go index 5b1e9bf43..7e94bff17 100644 --- a/api/internal/stackutils/stackutils.go +++ b/api/internal/stackutils/stackutils.go @@ -2,6 +2,7 @@ package stackutils import ( "fmt" + "path" portainer "github.com/portainer/portainer/api" ) @@ -10,3 +11,12 @@ import ( func ResourceControlID(endpointID portainer.EndpointID, name string) string { return fmt.Sprintf("%d_%s", endpointID, name) } + +// GetStackFilePaths returns a list of file paths based on stack project path +func GetStackFilePaths(stack *portainer.Stack) []string { + var filePaths []string + for _, file := range append([]string{stack.EntryPoint}, stack.AdditionalFiles...) { + filePaths = append(filePaths, path.Join(stack.ProjectPath, file)) + } + return filePaths +} diff --git a/api/internal/stackutils/stackutils_test.go b/api/internal/stackutils/stackutils_test.go new file mode 100644 index 000000000..6af19d8af --- /dev/null +++ b/api/internal/stackutils/stackutils_test.go @@ -0,0 +1,26 @@ +package stackutils + +import ( + "testing" + + portainer "github.com/portainer/portainer/api" + "github.com/stretchr/testify/assert" +) + +func Test_GetStackFilePaths(t *testing.T) { + stack := &portainer.Stack{ + ProjectPath: "/tmp/stack/1", + EntryPoint: "file-one.yml", + } + + t.Run("stack doesn't have additional files", func(t *testing.T) { + expected := []string{"/tmp/stack/1/file-one.yml"} + assert.ElementsMatch(t, expected, GetStackFilePaths(stack)) + }) + + t.Run("stack has additional files", func(t *testing.T) { + stack.AdditionalFiles = []string{"file-two.yml", "file-three.yml"} + expected := []string{"/tmp/stack/1/file-one.yml", "/tmp/stack/1/file-two.yml", "/tmp/stack/1/file-three.yml"} + assert.ElementsMatch(t, expected, GetStackFilePaths(stack)) + }) +} diff --git a/api/libcompose/compose_stack.go b/api/libcompose/compose_stack.go index 41da7239d..f2dfff230 100644 --- a/api/libcompose/compose_stack.go +++ b/api/libcompose/compose_stack.go @@ -16,6 +16,7 @@ import ( "github.com/portainer/libcompose/project" "github.com/portainer/libcompose/project/options" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/internal/stackutils" ) const ( @@ -86,12 +87,12 @@ func (manager *ComposeStackManager) Up(stack *portainer.Stack, endpoint *portain for _, envvar := range stack.Env { env[envvar.Name] = envvar.Value } + filePaths := stackutils.GetStackFilePaths(stack) - composeFilePath := path.Join(stack.ProjectPath, stack.EntryPoint) proj, err := docker.NewProject(&ctx.Context{ ConfigDir: manager.dataPath, Context: project.Context{ - ComposeFiles: []string{composeFilePath}, + ComposeFiles: filePaths, EnvironmentLookup: &lookup.ComposableEnvLookup{ Lookups: []config.EnvironmentLookup{ &lookup.EnvfileLookup{ @@ -120,10 +121,13 @@ func (manager *ComposeStackManager) Down(stack *portainer.Stack, endpoint *porta return err } - composeFilePath := path.Join(stack.ProjectPath, stack.EntryPoint) + var composeFiles []string + for _, file := range append([]string{stack.EntryPoint}, stack.AdditionalFiles...) { + composeFiles = append(composeFiles, path.Join(stack.ProjectPath, file)) + } proj, err := docker.NewProject(&ctx.Context{ Context: project.Context{ - ComposeFiles: []string{composeFilePath}, + ComposeFiles: composeFiles, ProjectName: stack.Name, }, ClientFactory: clientFactory, @@ -134,3 +138,11 @@ func (manager *ComposeStackManager) Down(stack *portainer.Stack, endpoint *porta return proj.Down(context.Background(), options.Down{RemoveVolume: false, RemoveOrphans: true}) } + +func stackFilePaths(stack *portainer.Stack) []string { + var filePaths []string + for _, file := range append([]string{stack.EntryPoint}, stack.AdditionalFiles...) { + filePaths = append(filePaths, path.Join(stack.ProjectPath, file)) + } + return filePaths +} diff --git a/api/portainer.go b/api/portainer.go index 7b7ff45f3..73e286d6d 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -744,10 +744,24 @@ type ( UpdateDate int64 `example:"1587399600"` // The username which last updated this stack UpdatedBy string `example:"bob"` + // Only applies when deploying stack with multiple files + AdditionalFiles []string `json:"AdditionalFiles"` + // The auto update settings of a git stack + AutoUpdate *StackAutoUpdate `json:"AutoUpdate"` // The git config of this stack GitConfig *gittypes.RepoConfig } + //StackAutoUpdate represents the git auto sync config for stack deployment + StackAutoUpdate struct { + // Auto update interval + Interval string `example:"1m30s"` + // A UUID generated from client + Webhook string `example:"05de31a2-79fa-4644-9c12-faa67e5c49f0"` + // Autoupdate job id + JobID string `example:"15"` + } + // StackID represents a stack identifier (it must be composed of Name + "_" + SwarmID to create a unique identifier) StackID int @@ -1187,6 +1201,7 @@ type ( // GitService represents a service for managing Git GitService interface { CloneRepository(destination string, repositoryURL, referenceName, username, password string) error + LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) } // JWTService represents a service for managing JWT tokens @@ -1302,6 +1317,8 @@ type ( UpdateStack(ID StackID, stack *Stack) error DeleteStack(ID StackID) error GetNextIdentifier() int + StackByWebhookID(ID string) (*Stack, error) + RefreshableStacks() ([]Stack, error) } // SnapshotService represents a service for managing endpoint snapshots diff --git a/api/scheduler/scheduler.go b/api/scheduler/scheduler.go new file mode 100644 index 000000000..6568f753f --- /dev/null +++ b/api/scheduler/scheduler.go @@ -0,0 +1,73 @@ +package scheduler + +import ( + "context" + "log" + "strconv" + "time" + + "github.com/pkg/errors" + "github.com/robfig/cron/v3" +) + +type Scheduler struct { + crontab *cron.Cron + shutdownCtx context.Context +} + +func NewScheduler(ctx context.Context) *Scheduler { + crontab := cron.New(cron.WithChain(cron.Recover(cron.DefaultLogger))) + crontab.Start() + + s := &Scheduler{ + crontab: crontab, + } + + if ctx != nil { + go func() { + <-ctx.Done() + s.Shutdown() + }() + } + + return s +} + +// Shutdown stops the scheduler and waits for it to stop if it is running; otherwise does nothing. +func (s *Scheduler) Shutdown() error { + if s.crontab == nil { + return nil + } + + log.Println("[DEBUG] Stopping scheduler") + ctx := s.crontab.Stop() + <-ctx.Done() + + for _, j := range s.crontab.Entries() { + s.crontab.Remove(j.ID) + } + + err := ctx.Err() + if err == context.Canceled { + return nil + } + return err +} + +// StopJob stops the job from being run in the future +func (s *Scheduler) StopJob(jobID string) error { + id, err := strconv.Atoi(jobID) + if err != nil { + return errors.Wrapf(err, "failed convert jobID %q to int", jobID) + } + s.crontab.Remove(cron.EntryID(id)) + + return nil +} + +// StartJobEvery schedules a new periodic job with a given duration. +// Returns job id that could be used to stop the given job +func (s *Scheduler) StartJobEvery(duration time.Duration, job func()) string { + entryId := s.crontab.Schedule(cron.Every(duration), cron.FuncJob(job)) + return strconv.Itoa(int(entryId)) +} diff --git a/api/scheduler/scheduler_test.go b/api/scheduler/scheduler_test.go new file mode 100644 index 000000000..6d21e49ec --- /dev/null +++ b/api/scheduler/scheduler_test.go @@ -0,0 +1,57 @@ +package scheduler + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func Test_CanStartAndTerminate(t *testing.T) { + s := NewScheduler(context.Background()) + s.StartJobEvery(1*time.Minute, func() { fmt.Println("boop") }) + + err := s.Shutdown() + assert.NoError(t, err, "Shutdown should return no errors") + assert.Empty(t, s.crontab.Entries(), "all jobs should have been removed") +} + +func Test_CanTerminateByCancellingContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + s := NewScheduler(ctx) + s.StartJobEvery(1*time.Minute, func() { fmt.Println("boop") }) + + cancel() + + for i := 0; i < 100; i++ { + if len(s.crontab.Entries()) == 0 { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatal("all jobs are expected to be cleaned by now; it might be a timing issue, otherwise implementation defect") +} + +func Test_StartAndStopJob(t *testing.T) { + s := NewScheduler(context.Background()) + defer s.Shutdown() + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + + var jobOne string + var workDone bool + jobOne = s.StartJobEvery(time.Second, func() { + assert.Equal(t, 1, len(s.crontab.Entries()), "scheduler should have one active job") + workDone = true + + s.StopJob(jobOne) + cancel() + }) + + <-ctx.Done() + assert.True(t, workDone, "value should been set in the job") + assert.Equal(t, 0, len(s.crontab.Entries()), "scheduler should have no active jobs") + +} diff --git a/api/stacks/deploy.go b/api/stacks/deploy.go new file mode 100644 index 000000000..ccf5eb441 --- /dev/null +++ b/api/stacks/deploy.go @@ -0,0 +1,138 @@ +package stacks + +import ( + "strings" + "time" + + "github.com/pkg/errors" + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/http/security" +) + +func RedeployWhenChanged(stackID portainer.StackID, deployer StackDeployer, datastore portainer.DataStore, gitService portainer.GitService) error { + stack, err := datastore.Stack().Stack(stackID) + if err != nil { + return errors.WithMessagef(err, "failed to get the stack %v", stackID) + } + + if stack.GitConfig == nil { + return nil // do nothing if it isn't a git-based stack + } + + username, password := "", "" + if stack.GitConfig.Authentication != nil { + username, password = stack.GitConfig.Authentication.Username, stack.GitConfig.Authentication.Password + } + + newHash, err := gitService.LatestCommitID(stack.GitConfig.URL, stack.GitConfig.ReferenceName, username, password) + if err != nil { + return errors.WithMessagef(err, "failed to fetch latest commit id of the stack %v", stack.ID) + } + + if strings.EqualFold(newHash, string(stack.GitConfig.ConfigHash)) { + return nil + } + + cloneParams := &cloneRepositoryParameters{ + url: stack.GitConfig.URL, + ref: stack.GitConfig.ReferenceName, + toDir: stack.ProjectPath, + } + if stack.GitConfig.Authentication != nil { + cloneParams.auth = &gitAuth{ + username: username, + password: password, + } + } + + if err := cloneGitRepository(gitService, cloneParams); err != nil { + return errors.WithMessagef(err, "failed to do a fresh clone of the stack %v", stack.ID) + } + + endpoint, err := datastore.Endpoint().Endpoint(stack.EndpointID) + if err != nil { + return errors.WithMessagef(err, "failed to find the endpoint %v associated to the stack %v", stack.EndpointID, stack.ID) + } + + author := stack.UpdatedBy + if author == "" { + author = stack.CreatedBy + } + + registries, err := getUserRegistries(datastore, author, endpoint.ID) + if err != nil { + return err + } + + switch stack.Type { + case portainer.DockerComposeStack: + err := deployer.DeployComposeStack(stack, endpoint, registries) + if err != nil { + return errors.WithMessagef(err, "failed to deploy a docker compose stack %v", stackID) + } + case portainer.DockerSwarmStack: + err := deployer.DeploySwarmStack(stack, endpoint, registries, true) + if err != nil { + return errors.WithMessagef(err, "failed to deploy a docker compose stack %v", stackID) + } + default: + return errors.Errorf("cannot update stack, type %v is unsupported", stack.Type) + } + + stack.UpdateDate = time.Now().Unix() + stack.GitConfig.ConfigHash = newHash + if err := datastore.Stack().UpdateStack(stack.ID, stack); err != nil { + return errors.WithMessagef(err, "failed to update the stack %v", stack.ID) + } + + return nil +} + +func getUserRegistries(datastore portainer.DataStore, authorUsername string, endpointID portainer.EndpointID) ([]portainer.Registry, error) { + registries, err := datastore.Registry().Registries() + if err != nil { + return nil, errors.WithMessage(err, "unable to retrieve registries from the database") + } + + user, err := datastore.User().UserByUsername(authorUsername) + if err != nil { + return nil, errors.WithMessagef(err, "failed to fetch a stack's author [%s]", authorUsername) + } + + if user.Role == portainer.AdministratorRole { + return registries, nil + } + + userMemberships, err := datastore.TeamMembership().TeamMembershipsByUserID(user.ID) + if err != nil { + return nil, errors.WithMessagef(err, "failed to fetch memberships of the stack author [%s]", authorUsername) + } + + filteredRegistries := make([]portainer.Registry, 0, len(registries)) + for _, registry := range registries { + if security.AuthorizedRegistryAccess(®istry, user, userMemberships, endpointID) { + filteredRegistries = append(filteredRegistries, registry) + } + } + + return filteredRegistries, nil +} + +type cloneRepositoryParameters struct { + url string + ref string + toDir string + auth *gitAuth +} + +type gitAuth struct { + username string + password string +} + +func cloneGitRepository(gitService portainer.GitService, cloneParams *cloneRepositoryParameters) error { + if cloneParams.auth != nil { + return gitService.CloneRepository(cloneParams.toDir, cloneParams.url, cloneParams.ref, cloneParams.auth.username, cloneParams.auth.password) + } + return gitService.CloneRepository(cloneParams.toDir, cloneParams.url, cloneParams.ref, "", "") +} diff --git a/api/stacks/deploy_test.go b/api/stacks/deploy_test.go new file mode 100644 index 000000000..58b7de913 --- /dev/null +++ b/api/stacks/deploy_test.go @@ -0,0 +1,221 @@ +package stacks + +import ( + "errors" + "io/ioutil" + "strings" + "testing" + + portainer "github.com/portainer/portainer/api" + bolt "github.com/portainer/portainer/api/bolt/bolttest" + gittypes "github.com/portainer/portainer/api/git/types" + "github.com/stretchr/testify/assert" +) + +type gitService struct { + cloneErr error + id string +} + +func (g *gitService) CloneRepository(destination, repositoryURL, referenceName, username, password string) error { + return g.cloneErr +} + +func (g *gitService) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) { + return g.id, nil +} + +type noopDeployer struct{} + +func (s *noopDeployer) DeploySwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, prune bool) error { + return nil +} + +func (s *noopDeployer) DeployComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry) error { + return nil +} + +func Test_redeployWhenChanged_FailsWhenCannotFindStack(t *testing.T) { + store, teardown := bolt.MustNewTestStore(true) + defer teardown() + + err := RedeployWhenChanged(1, nil, store, nil) + assert.Error(t, err) + assert.Truef(t, strings.HasPrefix(err.Error(), "failed to get the stack"), "it isn't an error we expected: %v", err.Error()) +} + +func Test_redeployWhenChanged_DoesNothingWhenNotAGitBasedStack(t *testing.T) { + store, teardown := bolt.MustNewTestStore(true) + defer teardown() + + err := store.Stack().CreateStack(&portainer.Stack{ID: 1}) + assert.NoError(t, err, "failed to create a test stack") + + err = RedeployWhenChanged(1, nil, store, &gitService{nil, ""}) + assert.NoError(t, err) +} + +func Test_redeployWhenChanged_DoesNothingWhenNoGitChanges(t *testing.T) { + store, teardown := bolt.MustNewTestStore(true) + defer teardown() + + tmpDir, _ := ioutil.TempDir("", "stack") + + err := store.Stack().CreateStack(&portainer.Stack{ + ID: 1, + ProjectPath: tmpDir, + GitConfig: &gittypes.RepoConfig{ + URL: "url", + ReferenceName: "ref", + ConfigHash: "oldHash", + }}) + assert.NoError(t, err, "failed to create a test stack") + + err = RedeployWhenChanged(1, nil, store, &gitService{nil, "oldHash"}) + assert.NoError(t, err) +} + +func Test_redeployWhenChanged_FailsWhenCannotClone(t *testing.T) { + cloneErr := errors.New("failed to clone") + store, teardown := bolt.MustNewTestStore(true) + defer teardown() + + err := store.Stack().CreateStack(&portainer.Stack{ + ID: 1, + GitConfig: &gittypes.RepoConfig{ + URL: "url", + ReferenceName: "ref", + ConfigHash: "oldHash", + }}) + assert.NoError(t, err, "failed to create a test stack") + + err = RedeployWhenChanged(1, nil, store, &gitService{cloneErr, "newHash"}) + assert.Error(t, err) + assert.ErrorIs(t, err, cloneErr, "should failed to clone but didn't, check test setup") +} + +func Test_redeployWhenChanged(t *testing.T) { + store, teardown := bolt.MustNewTestStore(true) + defer teardown() + + tmpDir, _ := ioutil.TempDir("", "stack") + + err := store.Endpoint().CreateEndpoint(&portainer.Endpoint{ID: 1}) + assert.NoError(t, err, "error creating endpoint") + + username := "user" + err = store.User().CreateUser(&portainer.User{Username: username, Role: portainer.AdministratorRole}) + assert.NoError(t, err, "error creating a user") + + stack := portainer.Stack{ + ID: 1, + EndpointID: 1, + ProjectPath: tmpDir, + UpdatedBy: username, + GitConfig: &gittypes.RepoConfig{ + URL: "url", + ReferenceName: "ref", + ConfigHash: "oldHash", + }} + err = store.Stack().CreateStack(&stack) + assert.NoError(t, err, "failed to create a test stack") + + t.Run("can deploy docker compose stack", func(t *testing.T) { + stack.Type = portainer.DockerComposeStack + store.Stack().UpdateStack(stack.ID, &stack) + + err = RedeployWhenChanged(1, &noopDeployer{}, store, &gitService{nil, "newHash"}) + assert.NoError(t, err) + }) + + t.Run("can deploy docker swarm stack", func(t *testing.T) { + stack.Type = portainer.DockerSwarmStack + store.Stack().UpdateStack(stack.ID, &stack) + + err = RedeployWhenChanged(1, &noopDeployer{}, store, &gitService{nil, "newHash"}) + assert.NoError(t, err) + }) + + t.Run("can NOT deploy kube stack", func(t *testing.T) { + stack.Type = portainer.KubernetesStack + store.Stack().UpdateStack(stack.ID, &stack) + + err = RedeployWhenChanged(1, &noopDeployer{}, store, &gitService{nil, "newHash"}) + assert.EqualError(t, err, "cannot update stack, type 3 is unsupported") + }) +} + +func Test_getUserRegistries(t *testing.T) { + store, teardown := bolt.MustNewTestStore(true) + defer teardown() + + endpointID := 123 + + admin := portainer.User{ID: 1, Username: "admin", Role: portainer.AdministratorRole} + err := store.User().CreateUser(&admin) + assert.NoError(t, err, "error creating an admin") + + user := portainer.User{ID: 2, Username: "user", Role: portainer.StandardUserRole} + err = store.User().CreateUser(&user) + assert.NoError(t, err, "error creating a user") + + team := portainer.Team{ID: 1, Name: "team"} + + store.TeamMembership().CreateTeamMembership(&portainer.TeamMembership{ + ID: 1, + UserID: user.ID, + TeamID: team.ID, + Role: portainer.TeamMember, + }) + + registryReachableByUser := portainer.Registry{ + ID: 1, + RegistryAccesses: portainer.RegistryAccesses{ + portainer.EndpointID(endpointID): { + UserAccessPolicies: map[portainer.UserID]portainer.AccessPolicy{ + user.ID: {RoleID: portainer.RoleID(portainer.StandardUserRole)}, + }, + }, + }, + } + err = store.Registry().CreateRegistry(®istryReachableByUser) + assert.NoError(t, err, "couldn't create a registry") + + registryReachableByTeam := portainer.Registry{ + ID: 2, + RegistryAccesses: portainer.RegistryAccesses{ + portainer.EndpointID(endpointID): { + TeamAccessPolicies: map[portainer.TeamID]portainer.AccessPolicy{ + team.ID: {RoleID: portainer.RoleID(portainer.StandardUserRole)}, + }, + }, + }, + } + err = store.Registry().CreateRegistry(®istryReachableByTeam) + assert.NoError(t, err, "couldn't create a registry") + + registryRestricted := portainer.Registry{ + ID: 3, + RegistryAccesses: portainer.RegistryAccesses{ + portainer.EndpointID(endpointID): { + UserAccessPolicies: map[portainer.UserID]portainer.AccessPolicy{ + user.ID + 100: {RoleID: portainer.RoleID(portainer.StandardUserRole)}, + }, + }, + }, + } + err = store.Registry().CreateRegistry(®istryRestricted) + assert.NoError(t, err, "couldn't create a registry") + + t.Run("admin should has access to all registries", func(t *testing.T) { + registries, err := getUserRegistries(store, admin.Username, portainer.EndpointID(endpointID)) + assert.NoError(t, err) + assert.ElementsMatch(t, []portainer.Registry{registryReachableByUser, registryReachableByTeam, registryRestricted}, registries) + }) + + t.Run("regular user has access to registries allowed to him and/or his team", func(t *testing.T) { + registries, err := getUserRegistries(store, user.Username, portainer.EndpointID(endpointID)) + assert.NoError(t, err) + assert.ElementsMatch(t, []portainer.Registry{registryReachableByUser, registryReachableByTeam}, registries) + }) +} diff --git a/api/stacks/deployer.go b/api/stacks/deployer.go new file mode 100644 index 000000000..d38c50cbc --- /dev/null +++ b/api/stacks/deployer.go @@ -0,0 +1,46 @@ +package stacks + +import ( + "sync" + + portainer "github.com/portainer/portainer/api" +) + +type StackDeployer interface { + DeploySwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, prune bool) error + DeployComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry) error +} + +type stackDeployer struct { + lock *sync.Mutex + swarmStackManager portainer.SwarmStackManager + composeStackManager portainer.ComposeStackManager +} + +func NewStackDeployer(swarmStackManager portainer.SwarmStackManager, composeStackManager portainer.ComposeStackManager) *stackDeployer { + return &stackDeployer{ + lock: &sync.Mutex{}, + swarmStackManager: swarmStackManager, + composeStackManager: composeStackManager, + } +} + +func (d *stackDeployer) DeploySwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, prune bool) error { + d.lock.Lock() + defer d.lock.Unlock() + + d.swarmStackManager.Login(registries, endpoint) + defer d.swarmStackManager.Logout(endpoint) + + return d.swarmStackManager.Deploy(stack, prune, endpoint) +} + +func (d *stackDeployer) DeployComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry) error { + d.lock.Lock() + defer d.lock.Unlock() + + d.swarmStackManager.Login(registries, endpoint) + defer d.swarmStackManager.Logout(endpoint) + + return d.composeStackManager.Up(stack, endpoint) +} diff --git a/api/stacks/scheduled.go b/api/stacks/scheduled.go new file mode 100644 index 000000000..fb90ca22c --- /dev/null +++ b/api/stacks/scheduled.go @@ -0,0 +1,34 @@ +package stacks + +import ( + "log" + "time" + + "github.com/pkg/errors" + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/scheduler" +) + +func StartStackSchedules(scheduler *scheduler.Scheduler, stackdeployer StackDeployer, datastore portainer.DataStore, gitService portainer.GitService) error { + stacks, err := datastore.Stack().RefreshableStacks() + if err != nil { + return errors.Wrap(err, "failed to fetch refreshable stacks") + } + for _, stack := range stacks { + d, err := time.ParseDuration(stack.AutoUpdate.Interval) + if err != nil { + return errors.Wrap(err, "Unable to parse auto update interval") + } + jobID := scheduler.StartJobEvery(d, func() { + if err := RedeployWhenChanged(stack.ID, stackdeployer, datastore, gitService); err != nil { + log.Printf("[ERROR] %s\n", err) + } + }) + + stack.AutoUpdate.JobID = jobID + if err := datastore.Stack().UpdateStack(stack.ID, &stack); err != nil { + return errors.Wrap(err, "failed to update stack job id") + } + } + return nil +} diff --git a/app/docker/components/datatables/containers-datatable/containersDatatable.html b/app/docker/components/datatables/containers-datatable/containersDatatable.html index 3299d8c87..5cd726759 100644 --- a/app/docker/components/datatables/containers-datatable/containersDatatable.html +++ b/app/docker/components/datatables/containers-datatable/containersDatatable.html @@ -92,7 +92,7 @@ ng-model="$ctrl.state.textFilter" ng-change="$ctrl.onTextFilterChange()" placeholder="Search..." - auto-focus + focus-if="!$ctrl.notAutoFocus" ng-model-options="{ debounce: 300 }" /> diff --git a/app/docker/components/datatables/containers-datatable/containersDatatable.js b/app/docker/components/datatables/containers-datatable/containersDatatable.js index 0edc97de0..d748c27aa 100644 --- a/app/docker/components/datatables/containers-datatable/containersDatatable.js +++ b/app/docker/components/datatables/containers-datatable/containersDatatable.js @@ -12,5 +12,6 @@ angular.module('portainer.docker').component('containersDatatable', { showAddAction: '<', offlineMode: '<', refreshCallback: '<', + notAutoFocus: '<', }, }); diff --git a/app/docker/components/datatables/services-datatable/servicesDatatable.html b/app/docker/components/datatables/services-datatable/servicesDatatable.html index 3e72d09f4..cd7bb808e 100644 --- a/app/docker/components/datatables/services-datatable/servicesDatatable.html +++ b/app/docker/components/datatables/services-datatable/servicesDatatable.html @@ -56,7 +56,7 @@ ng-model="$ctrl.state.textFilter" ng-change="$ctrl.onTextFilterChange()" placeholder="Search..." - auto-focus + focus-if="!$ctrl.notAutoFocus" ng-model-options="{ debounce: 300 }" /> diff --git a/app/docker/components/datatables/services-datatable/servicesDatatable.js b/app/docker/components/datatables/services-datatable/servicesDatatable.js index 6e5bbfb77..0354eab2f 100644 --- a/app/docker/components/datatables/services-datatable/servicesDatatable.js +++ b/app/docker/components/datatables/services-datatable/servicesDatatable.js @@ -15,5 +15,6 @@ angular.module('portainer.docker').component('servicesDatatable', { showStackColumn: '<', showTaskLogsButton: '<', refreshCallback: '<', + notAutoFocus: '<', }, }); diff --git a/app/portainer/components/focusIf.js b/app/portainer/components/focusIf.js new file mode 100644 index 000000000..29c76b373 --- /dev/null +++ b/app/portainer/components/focusIf.js @@ -0,0 +1,22 @@ +import angular from 'angular'; +// ng-focus-if pkg from: https://github.com/hiebj/ng-focus-if +angular.module('portainer.app').directive('focusIf', function ($timeout) { + return { + restrict: 'A', + link: function ($scope, $element, $attrs) { + var dom = $element[0]; + if ($attrs.focusIf) { + $scope.$watch($attrs.focusIf, focus); + } else { + focus(true); + } + function focus(condition) { + if (condition) { + $timeout(function () { + dom.focus(); + }, $scope.$eval($attrs.focusDelay) || 0); + } + } + }, + }; +}); diff --git a/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-file-item/git-form-additional-file-item.controller.js b/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-file-item/git-form-additional-file-item.controller.js new file mode 100644 index 000000000..f712c9b61 --- /dev/null +++ b/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-file-item/git-form-additional-file-item.controller.js @@ -0,0 +1,22 @@ +class GitFormAdditionalFileItemController { + /* @ngInject */ + constructor() {} + + onChangePath(value) { + const fieldIsInvalid = typeof value === 'undefined'; + if (fieldIsInvalid) { + return; + } + this.onChange(this.index, { value }); + } + + removeValue() { + this.onChange(this.index); + } + + $onInit() { + this.formName = `variableForm${this.index}`; + } +} + +export default GitFormAdditionalFileItemController; diff --git a/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-file-item/git-form-additional-file-item.html b/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-file-item/git-form-additional-file-item.html new file mode 100644 index 000000000..3796ac4eb --- /dev/null +++ b/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-file-item/git-form-additional-file-item.html @@ -0,0 +1,20 @@ + +
+
+
+ path + +
+ +
+
+
+
+

Path is required.

+
+
+
+
+ diff --git a/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-file-item/index.js b/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-file-item/index.js new file mode 100644 index 000000000..f50efd346 --- /dev/null +++ b/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-file-item/index.js @@ -0,0 +1,14 @@ +import controller from './git-form-additional-file-item.controller.js'; + +export const gitFormAdditionalFileItem = { + templateUrl: './git-form-additional-file-item.html', + controller, + + bindings: { + variable: '<', + index: '<', + + onChange: '<', + onRemove: '<', + }, +}; diff --git a/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-files-panel.controller.js b/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-files-panel.controller.js new file mode 100644 index 000000000..8f3d4c22b --- /dev/null +++ b/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-files-panel.controller.js @@ -0,0 +1,26 @@ +class GitFormAutoUpdateFieldsetController { + /* @ngInject */ + constructor() { + this.add = this.add.bind(this); + this.onChangeVariable = this.onChangeVariable.bind(this); + } + + add() { + this.model.AdditionalFiles.push(''); + } + + onChangeVariable(index, variable) { + if (!variable) { + this.model.AdditionalFiles.splice(index, 1); + } else { + this.model.AdditionalFiles[index] = variable.value; + } + + this.onChange({ + ...this.model, + AdditionalFiles: this.model.AdditionalFiles, + }); + } +} + +export default GitFormAutoUpdateFieldsetController; diff --git a/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-files-panel.html b/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-files-panel.html new file mode 100644 index 000000000..b012c4a5d --- /dev/null +++ b/app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-files-panel.html @@ -0,0 +1,14 @@ +
+
+ + add file +
+
+ +
+
diff --git a/app/portainer/components/forms/git-form/git-form-additional-files-panel/index.js b/app/portainer/components/forms/git-form/git-form-additional-files-panel/index.js new file mode 100644 index 000000000..d6d525438 --- /dev/null +++ b/app/portainer/components/forms/git-form/git-form-additional-files-panel/index.js @@ -0,0 +1,10 @@ +import controller from './git-form-additional-files-panel.controller.js'; + +export const gitFormAdditionalFilesPanel = { + templateUrl: './git-form-additional-files-panel.html', + controller, + bindings: { + model: '<', + onChange: '<', + }, +}; diff --git a/app/portainer/components/forms/git-form/git-form-auth-fieldset/git-form-auth-fieldset.html b/app/portainer/components/forms/git-form/git-form-auth-fieldset/git-form-auth-fieldset.html index b884c248d..7f820783c 100644 --- a/app/portainer/components/forms/git-form/git-form-auth-fieldset/git-form-auth-fieldset.html +++ b/app/portainer/components/forms/git-form/git-form-auth-fieldset/git-form-auth-fieldset.html @@ -3,36 +3,38 @@ +
+ + Enabling authentication will store the credentials and it is advisable to use a git service account +
- - If your git account has 2FA enabled, you may receive an authentication required error when deploying your stack. In this case, you will need to provide a - personal-access token instead of your password. - -
-
- -
+ +
-
+
+ -
+
diff --git a/app/portainer/components/forms/git-form/git-form-auth-fieldset/index.js b/app/portainer/components/forms/git-form/git-form-auth-fieldset/index.js index 5c2373387..3000869c3 100644 --- a/app/portainer/components/forms/git-form/git-form-auth-fieldset/index.js +++ b/app/portainer/components/forms/git-form/git-form-auth-fieldset/index.js @@ -6,5 +6,6 @@ export const gitFormAuthFieldset = { bindings: { model: '<', onChange: '<', + isEdit: '<', }, }; diff --git a/app/portainer/components/forms/git-form/git-form-auto-update-fieldset/git-form-auto-update-fieldset.controller.js b/app/portainer/components/forms/git-form/git-form-auto-update-fieldset/git-form-auto-update-fieldset.controller.js new file mode 100644 index 000000000..9025ab0df --- /dev/null +++ b/app/portainer/components/forms/git-form/git-form-auto-update-fieldset/git-form-auto-update-fieldset.controller.js @@ -0,0 +1,26 @@ +class GitFormAutoUpdateFieldsetController { + /* @ngInject */ + constructor(clipboard) { + this.onChangeAutoUpdate = this.onChangeField('RepositoryAutomaticUpdates'); + this.onChangeMechanism = this.onChangeField('RepositoryMechanism'); + this.onChangeInterval = this.onChangeField('RepositoryFetchInterval'); + this.clipboard = clipboard; + } + + copyWebhook() { + this.clipboard.copyText(this.model.RepositoryWebhookURL); + $('#copyNotification').show(); + $('#copyNotification').fadeOut(2000); + } + + onChangeField(field) { + return (value) => { + this.onChange({ + ...this.model, + [field]: value, + }); + }; + } +} + +export default GitFormAutoUpdateFieldsetController; diff --git a/app/portainer/components/forms/git-form/git-form-auto-update-fieldset/git-form-auto-update-fieldset.html b/app/portainer/components/forms/git-form/git-form-auto-update-fieldset/git-form-auto-update-fieldset.html new file mode 100644 index 000000000..000b68e0f --- /dev/null +++ b/app/portainer/components/forms/git-form/git-form-auto-update-fieldset/git-form-auto-update-fieldset.html @@ -0,0 +1,69 @@ + +
+
+ +
+
+
+ + Any changes to this stack made locally in Portainer will be overriden by the definition in git and may cause service interruption. +
+
+ +
+
+
+ + +
+
+
+
+ +
+ +
+ {{ $ctrl.model.RepositoryWebhookURL | truncatelr }} + + + + +
+
+
+ +
+ +
+
+
+
+
+

This field is required.

+

Please enter a valid time interval.

+

Minimum interval is 1m

+
+
+
+ diff --git a/app/portainer/components/forms/git-form/git-form-auto-update-fieldset/index.js b/app/portainer/components/forms/git-form/git-form-auto-update-fieldset/index.js new file mode 100644 index 000000000..237cb6006 --- /dev/null +++ b/app/portainer/components/forms/git-form/git-form-auto-update-fieldset/index.js @@ -0,0 +1,10 @@ +import controller from './git-form-auto-update-fieldset.controller.js'; + +export const gitFormAutoUpdateFieldset = { + templateUrl: './git-form-auto-update-fieldset.html', + controller, + bindings: { + model: '<', + onChange: '<', + }, +}; diff --git a/app/portainer/components/forms/git-form/git-form-ref-field/git-form-ref-field.html b/app/portainer/components/forms/git-form/git-form-ref-field/git-form-ref-field.html index a12639d24..86164f650 100644 --- a/app/portainer/components/forms/git-form/git-form-ref-field/git-form-ref-field.html +++ b/app/portainer/components/forms/git-form/git-form-ref-field/git-form-ref-field.html @@ -5,8 +5,8 @@
- -
+ +
diff --git a/app/portainer/components/forms/git-form/git-form.html b/app/portainer/components/forms/git-form/git-form.html index 16b131008..ba35fcf67 100644 --- a/app/portainer/components/forms/git-form/git-form.html +++ b/app/portainer/components/forms/git-form/git-form.html @@ -5,5 +5,7 @@ - + + +
diff --git a/app/portainer/components/forms/git-form/git-form.js b/app/portainer/components/forms/git-form/git-form.js index 4a1c5733e..affa081dc 100644 --- a/app/portainer/components/forms/git-form/git-form.js +++ b/app/portainer/components/forms/git-form/git-form.js @@ -6,5 +6,8 @@ export const gitForm = { bindings: { model: '<', onChange: '<', + additionalFile: '<', + autoUpdate: '<', + showAuthExplanation: '<', }, }; diff --git a/app/portainer/components/forms/git-form/index.js b/app/portainer/components/forms/git-form/index.js index 323c1d271..60eff71e6 100644 --- a/app/portainer/components/forms/git-form/index.js +++ b/app/portainer/components/forms/git-form/index.js @@ -2,6 +2,9 @@ import angular from 'angular'; import { gitForm } from './git-form'; import { gitFormAuthFieldset } from './git-form-auth-fieldset'; +import { gitFormAdditionalFilesPanel } from './git-form-additional-files-panel'; +import { gitFormAdditionalFileItem } from './/git-form-additional-files-panel/git-form-additional-file-item'; +import { gitFormAutoUpdateFieldset } from './git-form-auto-update-fieldset'; import { gitFormComposePathField } from './git-form-compose-path-field'; import { gitFormRefField } from './git-form-ref-field'; import { gitFormUrlField } from './git-form-url-field'; @@ -12,4 +15,7 @@ export default angular .component('gitFormRefField', gitFormRefField) .component('gitForm', gitForm) .component('gitFormUrlField', gitFormUrlField) + .component('gitFormAdditionalFilesPanel', gitFormAdditionalFilesPanel) + .component('gitFormAdditionalFileItem', gitFormAdditionalFileItem) + .component('gitFormAutoUpdateFieldset', gitFormAutoUpdateFieldset) .component('gitFormAuthFieldset', gitFormAuthFieldset).name; diff --git a/app/portainer/components/forms/stack-redeploy-git-form/stack-redeploy-git-form.controller.js b/app/portainer/components/forms/stack-redeploy-git-form/stack-redeploy-git-form.controller.js index f489cc2d9..f4179817d 100644 --- a/app/portainer/components/forms/stack-redeploy-git-form/stack-redeploy-git-form.controller.js +++ b/app/portainer/components/forms/stack-redeploy-git-form/stack-redeploy-git-form.controller.js @@ -1,14 +1,20 @@ +import uuidv4 from 'uuid/v4'; class StackRedeployGitFormController { /* @ngInject */ - constructor($async, $state, StackService, ModalService, Notifications) { + constructor($async, $state, StackService, ModalService, Notifications, WebhookHelper, FormHelper) { this.$async = $async; this.$state = $state; this.StackService = StackService; this.ModalService = ModalService; this.Notifications = Notifications; + this.WebhookHelper = WebhookHelper; + this.FormHelper = FormHelper; this.state = { inProgress: false, + redeployInProgress: false, + showConfig: false, + isEdit: false, }; this.formValues = { @@ -16,10 +22,19 @@ class StackRedeployGitFormController { RepositoryAuthentication: false, RepositoryUsername: '', RepositoryPassword: '', + Env: [], + // auto upadte + AutoUpdate: { + RepositoryAutomaticUpdates: false, + RepositoryMechanism: 'Interval', + RepositoryFetchInterval: '5m', + RepositoryWebhookURL: '', + }, }; this.onChange = this.onChange.bind(this); this.onChangeRef = this.onChangeRef.bind(this); + this.handleEnvVarChange = this.handleEnvVarChange.bind(this); } onChangeRef(value) { @@ -50,13 +65,27 @@ class StackRedeployGitFormController { return; } - this.state.inProgress = true; + this.state.redeployInProgress = true; - await this.StackService.updateGit(this.stack.Id, this.stack.EndpointId, [], false, this.formValues); + await this.StackService.updateGit(this.stack.Id, this.stack.EndpointId, this.FormHelper.removeInvalidEnvVars(this.formValues.Env), false, this.formValues); await this.$state.reload(); } catch (err) { this.Notifications.error('Failure', err, 'Failed redeploying stack'); + } finally { + this.state.redeployInProgress = false; + } + }); + } + + async saveGitSettings() { + return this.$async(async () => { + try { + this.state.inProgress = true; + await this.StackService.updateGitStackSettings(this.stack.Id, this.stack.EndpointId, this.FormHelper.removeInvalidEnvVars(this.formValues.Env), this.formValues); + this.Notifications.success('Save stack settings successfully'); + } catch (err) { + this.Notifications.error('Failure', err, 'Unable to save stack settings'); } finally { this.state.inProgress = false; } @@ -64,11 +93,38 @@ class StackRedeployGitFormController { } isSubmitButtonDisabled() { - return this.state.inProgress; + return this.state.inProgress || this.state.redeployInProgress; + } + + handleEnvVarChange(value) { + this.formValues.Env = value; } $onInit() { this.formValues.RefName = this.model.ReferenceName; + this.formValues.Env = this.stack.Env; + // Init auto update + if (this.stack.AutoUpdate && (this.stack.AutoUpdate.Interval || this.stack.AutoUpdate.Webhook)) { + this.formValues.AutoUpdate.RepositoryAutomaticUpdates = true; + + if (this.stack.AutoUpdate.Interval) { + this.formValues.AutoUpdate.RepositoryMechanism = `Interval`; + this.formValues.AutoUpdate.RepositoryFetchInterval = this.stack.AutoUpdate.Interval; + } else if (this.stack.AutoUpdate.Webhook) { + this.formValues.AutoUpdate.RepositoryMechanism = `Webhook`; + this.formValues.AutoUpdate.RepositoryWebhookURL = this.WebhookHelper.returnStackWebhookUrl(this.stack.AutoUpdate.Webhook); + } + } + + if (!this.formValues.AutoUpdate.RepositoryWebhookURL) { + this.formValues.AutoUpdate.RepositoryWebhookURL = this.WebhookHelper.returnStackWebhookUrl(uuidv4()); + } + + if (this.stack.GitConfig && this.stack.GitConfig.Authentication) { + this.formValues.RepositoryUsername = this.stack.GitConfig.Authentication.Username; + this.formValues.RepositoryAuthentication = true; + this.state.isEdit = true; + } } } diff --git a/app/portainer/components/forms/stack-redeploy-git-form/stack-redeploy-git-form.html b/app/portainer/components/forms/stack-redeploy-git-form/stack-redeploy-git-form.html index 5b5efc42b..340400573 100644 --- a/app/portainer/components/forms/stack-redeploy-git-form/stack-redeploy-git-form.html +++ b/app/portainer/components/forms/stack-redeploy-git-form/stack-redeploy-git-form.html @@ -9,22 +9,62 @@ .

- Update {{ $ctrl.model.ConfigFilePath }} in git and pull from here to update the stack. + Update + {{ $ctrl.model.ConfigFilePath }},{{ $ctrl.stack.AdditionalFiles.join(',') }} + in git and pull from here to update the stack.

- - + +
+
+

+ + + {{ $ctrl.state.showConfig ? 'Hide' : 'Advanced' }} configuration + +

+
+
+ + + + + diff --git a/app/portainer/components/intervalFormat.js b/app/portainer/components/intervalFormat.js new file mode 100644 index 000000000..73126928e --- /dev/null +++ b/app/portainer/components/intervalFormat.js @@ -0,0 +1,26 @@ +import angular from 'angular'; +import parse from 'parse-duration'; + +angular.module('portainer.app').directive('intervalFormat', function () { + return { + restrict: 'A', + require: 'ngModel', + link: function ($scope, $element, $attrs, ngModel) { + ngModel.$validators.invalidIntervalFormat = function (modelValue) { + try { + return modelValue && modelValue.toUpperCase().match(/^P?(?!$)(\d+Y)?(\d+M)?(\d+W)?(\d+D)?(T?(?=\d+[HMS])(\d+H)?(\d+M)?(\d+S)?)?$/gm) !== null; + } catch (error) { + return false; + } + }; + + ngModel.$validators.minimumInterval = function (modelValue) { + try { + return modelValue && parse(modelValue, 'minute') >= 1; + } catch (error) { + return false; + } + }; + }, + }; +}); diff --git a/app/portainer/helpers/webhookHelper.js b/app/portainer/helpers/webhookHelper.js index 8411544fd..69a487c0e 100644 --- a/app/portainer/helpers/webhookHelper.js +++ b/app/portainer/helpers/webhookHelper.js @@ -1,16 +1,21 @@ angular.module('portainer.app').factory('WebhookHelper', [ '$location', 'API_ENDPOINT_WEBHOOKS', - function WebhookHelperFactory($location, API_ENDPOINT_WEBHOOKS) { + 'API_ENDPOINT_STACKS', + function WebhookHelperFactory($location, API_ENDPOINT_WEBHOOKS, API_ENDPOINT_STACKS) { 'use strict'; + var helper = {}; + const protocol = $location.protocol().toLowerCase(); + const port = $location.port(); + const displayPort = (protocol === 'http' && port === 80) || (protocol === 'https' && port === 443) ? '' : ':' + port; helper.returnWebhookUrl = function (token) { - var displayPort = - ($location.protocol().toLowerCase() === 'http' && $location.port() === 80) || ($location.protocol().toLowerCase() === 'https' && $location.port() === 443) - ? '' - : ':' + $location.port(); - return $location.protocol() + '://' + $location.host() + displayPort + '/' + API_ENDPOINT_WEBHOOKS + '/' + token; + return `${protocol}://${$location.host()}${displayPort}/${API_ENDPOINT_WEBHOOKS}/${token}`; + }; + + helper.returnStackWebhookUrl = function (token) { + return `${protocol}://${$location.host()}${displayPort}/${API_ENDPOINT_STACKS}/webhooks/${token}`; }; return helper; diff --git a/app/portainer/models/stack.js b/app/portainer/models/stack.js index 1b11fa432..41c32f70f 100644 --- a/app/portainer/models/stack.js +++ b/app/portainer/models/stack.js @@ -20,6 +20,8 @@ export function StackViewModel(data) { this.Orphaned = false; this.Checked = false; this.GitConfig = data.GitConfig; + this.AdditionalFiles = data.AdditionalFiles; + this.AutoUpdate = data.AutoUpdate; } export function ExternalStackViewModel(name, type, creationDate) { diff --git a/app/portainer/rest/stack.js b/app/portainer/rest/stack.js index 6327fc647..d41bdf04a 100644 --- a/app/portainer/rest/stack.js +++ b/app/portainer/rest/stack.js @@ -5,7 +5,7 @@ angular.module('portainer.app').factory('Stack', [ function StackFactory($resource, EndpointProvider, API_ENDPOINT_STACKS) { 'use strict'; return $resource( - API_ENDPOINT_STACKS + '/:id/:action', + API_ENDPOINT_STACKS + '/:id/:action/:subaction', {}, { get: { method: 'GET', params: { id: '@id' } }, @@ -18,7 +18,8 @@ angular.module('portainer.app').factory('Stack', [ migrate: { method: 'POST', params: { id: '@id', action: 'migrate', endpointId: '@endpointId' }, ignoreLoadingBar: true }, start: { method: 'POST', params: { id: '@id', action: 'start' } }, stop: { method: 'POST', params: { id: '@id', action: 'stop' } }, - updateGit: { method: 'PUT', params: { action: 'git' } }, + updateGit: { method: 'PUT', params: { id: '@id', action: 'git', subaction: 'redeploy' } }, + updateGitStackSettings: { method: 'POST', params: { id: '@id', action: 'git' }, ignoreLoadingBar: true }, } ); }, diff --git a/app/portainer/services/api/stackService.js b/app/portainer/services/api/stackService.js index a26c6c98b..86ad1905a 100644 --- a/app/portainer/services/api/stackService.js +++ b/app/portainer/services/api/stackService.js @@ -326,12 +326,18 @@ angular.module('portainer.app').factory('StackService', [ Name: name, RepositoryURL: repositoryOptions.RepositoryURL, RepositoryReferenceName: repositoryOptions.RepositoryReferenceName, - ComposeFilePathInRepository: repositoryOptions.ComposeFilePathInRepository, + ComposeFile: repositoryOptions.ComposeFilePathInRepository, + AdditionalFiles: repositoryOptions.AdditionalFiles, RepositoryAuthentication: repositoryOptions.RepositoryAuthentication, RepositoryUsername: repositoryOptions.RepositoryUsername, RepositoryPassword: repositoryOptions.RepositoryPassword, Env: env, }; + + if (repositoryOptions.AutoUpdate) { + payload.AutoUpdate = repositoryOptions.AutoUpdate; + } + return Stack.create({ method: 'repository', type: 2, endpointId: endpointId }, payload).$promise; }; @@ -346,12 +352,18 @@ angular.module('portainer.app').factory('StackService', [ SwarmID: swarm.Id, RepositoryURL: repositoryOptions.RepositoryURL, RepositoryReferenceName: repositoryOptions.RepositoryReferenceName, - ComposeFilePathInRepository: repositoryOptions.ComposeFilePathInRepository, + ComposeFile: repositoryOptions.ComposeFilePathInRepository, + AdditionalFiles: repositoryOptions.AdditionalFiles, RepositoryAuthentication: repositoryOptions.RepositoryAuthentication, RepositoryUsername: repositoryOptions.RepositoryUsername, RepositoryPassword: repositoryOptions.RepositoryPassword, Env: env, }; + + if (repositoryOptions.AutoUpdate) { + payload.AutoUpdate = repositoryOptions.AutoUpdate; + } + return Stack.create({ method: 'repository', type: 1, endpointId: endpointId }, payload).$promise; }) .then(function success(data) { @@ -405,6 +417,31 @@ angular.module('portainer.app').factory('StackService', [ ).$promise; } + service.updateGitStackSettings = function (id, endpointId, env, gitConfig) { + // prepare auto update + const autoUpdate = {}; + + if (gitConfig.AutoUpdate.RepositoryAutomaticUpdates) { + if (gitConfig.AutoUpdate.RepositoryMechanism === 'Interval') { + autoUpdate.Interval = gitConfig.AutoUpdate.RepositoryFetchInterval; + } else if (gitConfig.AutoUpdate.RepositoryMechanism === 'Webhook') { + autoUpdate.Webhook = gitConfig.AutoUpdate.RepositoryWebhookURL.split('/').reverse()[0]; + } + } + + return Stack.updateGitStackSettings( + { endpointId, id }, + { + AutoUpdate: autoUpdate, + Env: env, + RepositoryReferenceName: gitConfig.RefName, + RepositoryAuthentication: gitConfig.RepositoryAuthentication, + RepositoryUsername: gitConfig.RepositoryUsername, + RepositoryPassword: gitConfig.RepositoryPassword, + } + ).$promise; + }; + return service; }, ]); diff --git a/app/portainer/views/stacks/create/createStackController.js b/app/portainer/views/stacks/create/createStackController.js index 2b42600b2..168179142 100644 --- a/app/portainer/views/stacks/create/createStackController.js +++ b/app/portainer/views/stacks/create/createStackController.js @@ -1,6 +1,6 @@ import angular from 'angular'; import _ from 'lodash-es'; - +import uuidv4 from 'uuid/v4'; import { AccessControlFormData } from '../../../components/accessControlForm/porAccessControlFormModel'; angular @@ -21,7 +21,9 @@ angular StackHelper, ContainerHelper, CustomTemplateService, - ContainerService + ContainerService, + WebhookHelper, + clipboard ) { $scope.formValues = { Name: '', @@ -33,8 +35,13 @@ angular RepositoryUsername: '', RepositoryPassword: '', Env: [], + AdditionalFiles: [], ComposeFilePathInRepository: 'docker-compose.yml', AccessControlData: new AccessControlFormData(), + RepositoryAutomaticUpdates: true, + RepositoryMechanism: 'Interval', + RepositoryFetchInterval: '5m', + RepositoryWebhookURL: WebhookHelper.returnStackWebhookUrl(uuidv4()), }; $scope.state = { @@ -67,6 +74,14 @@ angular $scope.formValues.Env.splice(index, 1); }; + $scope.addAdditionalFiles = function () { + $scope.formValues.AdditionalFiles.push(''); + }; + + $scope.removeAdditionalFiles = function (index) { + $scope.formValues.AdditionalFiles.splice(index, 1); + }; + function validateForm(accessControlData, isAdmin) { $scope.state.formValidationError = ''; var error = ''; @@ -95,6 +110,7 @@ angular if (method === 'repository') { var repositoryOptions = { + AdditionalFiles: $scope.formValues.AdditionalFiles, RepositoryURL: $scope.formValues.RepositoryURL, RepositoryReferenceName: $scope.formValues.RepositoryReferenceName, ComposeFilePathInRepository: $scope.formValues.ComposeFilePathInRepository, @@ -102,10 +118,24 @@ angular RepositoryUsername: $scope.formValues.RepositoryUsername, RepositoryPassword: $scope.formValues.RepositoryPassword, }; + + getAutoUpdatesProperty(repositoryOptions); + return StackService.createSwarmStackFromGitRepository(name, repositoryOptions, env, endpointId); } } + function getAutoUpdatesProperty(repositoryOptions) { + if ($scope.formValues.RepositoryAutomaticUpdates) { + repositoryOptions.AutoUpdate = {}; + if ($scope.formValues.RepositoryMechanism === 'Interval') { + repositoryOptions.AutoUpdate.Interval = $scope.formValues.RepositoryFetchInterval; + } else if ($scope.formValues.RepositoryMechanism === 'Webhook') { + repositoryOptions.AutoUpdate.Webhook = $scope.formValues.RepositoryWebhookURL.split('/').reverse()[0]; + } + } + } + function createComposeStack(name, method) { var env = FormHelper.removeInvalidEnvVars($scope.formValues.Env); const endpointId = +$state.params.endpointId; @@ -118,6 +148,7 @@ angular return StackService.createComposeStackFromFileUpload(name, stackFile, env, endpointId); } else if (method === 'repository') { var repositoryOptions = { + AdditionalFiles: $scope.formValues.AdditionalFiles, RepositoryURL: $scope.formValues.RepositoryURL, RepositoryReferenceName: $scope.formValues.RepositoryReferenceName, ComposeFilePathInRepository: $scope.formValues.ComposeFilePathInRepository, @@ -125,10 +156,19 @@ angular RepositoryUsername: $scope.formValues.RepositoryUsername, RepositoryPassword: $scope.formValues.RepositoryPassword, }; + + getAutoUpdatesProperty(repositoryOptions); + return StackService.createComposeStackFromGitRepository(name, repositoryOptions, env, endpointId); } } + $scope.copyWebhook = function () { + clipboard.copyText($scope.formValues.RepositoryWebhookURL); + $('#copyNotification').show(); + $('#copyNotification').fadeOut(2000); + }; + $scope.handleEnvVarChange = handleEnvVarChange; function handleEnvVarChange(value) { $scope.formValues.Env = value; diff --git a/app/portainer/views/stacks/create/createstack.html b/app/portainer/views/stacks/create/createstack.html index 8e545b7e3..abb4c935b 100644 --- a/app/portainer/views/stacks/create/createstack.html +++ b/app/portainer/views/stacks/create/createstack.html @@ -130,7 +130,14 @@ - +
@@ -207,7 +214,7 @@ || (state.Method === 'editor' && (!formValues.StackFileContent || state.editorYamlValidationError)) || (state.Method === 'upload' && (!formValues.StackFile || state.uploadYamlValidationError)) || (state.Method === 'template' && (!formValues.StackFileContent || !selectedTemplate || state.editorYamlValidationError)) - || (state.Method === 'repository' && ((!formValues.RepositoryURL || !formValues.ComposeFilePathInRepository) || (formValues.RepositoryAuthentication && (!formValues.RepositoryUsername || !formValues.RepositoryPassword)))) + || (state.Method === 'repository' && ((!formValues.RepositoryURL || !formValues.ComposeFilePathInRepository) || (formValues.RepositoryAuthentication && !formValues.RepositoryPassword))) || !formValues.Name" ng-click="deployStack()" button-spinner="state.actionInProgress" diff --git a/app/portainer/views/stacks/edit/stack.html b/app/portainer/views/stacks/edit/stack.html index 88ae401b8..57fdc69f6 100644 --- a/app/portainer/views/stacks/edit/stack.html +++ b/app/portainer/views/stacks/edit/stack.html @@ -125,7 +125,7 @@ - + Editor
@@ -214,6 +214,7 @@ order-by="Status" show-host-column="false" show-add-action="false" + not-auto-focus="true" >
@@ -233,6 +234,7 @@ show-task-logs-button="applicationState.endpoint.apiVersion >= 1.30" show-add-action="false" show-stack-column="false" + not-auto-focus="true" >
diff --git a/package.json b/package.json index 56c9858c6..56f9cda6e 100644 --- a/package.json +++ b/package.json @@ -91,9 +91,10 @@ "jquery": "^3.5.1", "js-base64": "^3.6.0", "js-yaml": "^3.14.0", - "lodash-es": "^4.17.15", + "lodash-es": "^4.17.15", "moment": "^2.21.0", "ng-file-upload": "~12.2.13", + "parse-duration": "^1.0.0", "source-map-loader": "^1.1.2", "spinkit": "^2.0.1", "splitargs": "github:deviantony/splitargs#semver:~0.2.0", diff --git a/yarn.lock b/yarn.lock index 2f6daba4d..62b8a406f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7967,6 +7967,11 @@ parse-asn1@^5.0.0, parse-asn1@^5.1.5: pbkdf2 "^3.0.3" safe-buffer "^5.1.1" +parse-duration@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/parse-duration/-/parse-duration-1.0.0.tgz#8605651745f61088f6fb14045c887526c291858c" + integrity sha512-X4kUkCTHU1N/kEbwK9FpUJ0UZQa90VzeczfS704frR30gljxDG0pSziws06XlK+CGRSo/1wtG1mFIdBFQTMQNw== + parse-filepath@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/parse-filepath/-/parse-filepath-1.0.2.tgz#a632127f53aaf3d15876f5872f3ffac763d6c891"