feat(stacks): support automated sync for stacks [EE-248] (#5340)

api/backup/backup.go

@@ -10,6 +10,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/archive"
 	"github.com/portainer/portainer/api/crypto"
+	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/http/offlinegate"
 )
 
@@ -32,7 +33,7 @@ func CreateBackupArchive(password string, gate *offlinegate.OfflineGate, datasto
 	}
 
 	for _, filename := range filesToBackup {
-		err := copyPath(filepath.Join(filestorePath, filename), backupDirPath)
+		err := filesystem.CopyPath(filepath.Join(filestorePath, filename), backupDirPath)
 		if err != nil {
 			return "", errors.Wrap(err, "Failed to create backup file")
 		}

api/backup/copy_test.go

@@ -1,105 +0,0 @@
-package backup
-
-import (
-	"io/ioutil"
-	"os"
-	"path"
-	"path/filepath"
-	"testing"
-
-	"github.com/docker/docker/pkg/ioutils"
-	"github.com/stretchr/testify/assert"
-)
-
-func listFiles(dir string) []string {
-	items := make([]string, 0)
-	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
-		if path == dir {
-			return nil
-		}
-		items = append(items, path)
-		return nil
-	})
-
-	return items
-}
-
-func contains(t *testing.T, list []string, path string) {
-	assert.Contains(t, list, path)
-	copyContent, _ := ioutil.ReadFile(path)
-	assert.Equal(t, "content\n", string(copyContent))
-}
-
-func Test_copyFile_returnsError_whenSourceDoesNotExist(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
-	err := copyFile("does-not-exist", tmpdir)
-	assert.NotNil(t, err)
-}
-
-func Test_copyFile_shouldMakeAbackup(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
-	content := []byte("content")
-	ioutil.WriteFile(path.Join(tmpdir, "origin"), content, 0600)
-
-	err := copyFile(path.Join(tmpdir, "origin"), path.Join(tmpdir, "copy"))
-	assert.Nil(t, err)
-
-	copyContent, _ := ioutil.ReadFile(path.Join(tmpdir, "copy"))
-	assert.Equal(t, content, copyContent)
-}
-
-func Test_copyDir_shouldCopyAllFilesAndDirectories(t *testing.T) {
-	destination, _ := ioutils.TempDir("", "destination")
-	defer os.RemoveAll(destination)
-	err := copyDir("./test_assets/copy_test", destination)
-	assert.Nil(t, err)
-
-	createdFiles := listFiles(destination)
-
-	contains(t, createdFiles, filepath.Join(destination, "copy_test", "outer"))
-	contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", ".dotfile"))
-	contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", "inner"))
-}
-
-func Test_backupPath_shouldSkipWhenNotExist(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
-	err := copyPath("does-not-exists", tmpdir)
-	assert.Nil(t, err)
-
-	assert.Empty(t, listFiles(tmpdir))
-}
-
-func Test_backupPath_shouldCopyFile(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
-	content := []byte("content")
-	ioutil.WriteFile(path.Join(tmpdir, "file"), content, 0600)
-
-	os.MkdirAll(path.Join(tmpdir, "backup"), 0700)
-	err := copyPath(path.Join(tmpdir, "file"), path.Join(tmpdir, "backup"))
-	assert.Nil(t, err)
-
-	copyContent, err := ioutil.ReadFile(path.Join(tmpdir, "backup", "file"))
-	assert.Nil(t, err)
-	assert.Equal(t, content, copyContent)
-}
-
-func Test_backupPath_shouldCopyDir(t *testing.T) {
-	destination, _ := ioutils.TempDir("", "destination")
-	defer os.RemoveAll(destination)
-	err := copyPath("./test_assets/copy_test", destination)
-	assert.Nil(t, err)
-
-	createdFiles := listFiles(destination)
-
-	contains(t, createdFiles, filepath.Join(destination, "copy_test", "outer"))
-	contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", ".dotfile"))
-	contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", "inner"))
-}

api/backup/restore.go

@@ -11,6 +11,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/archive"
 	"github.com/portainer/portainer/api/crypto"
+	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/http/offlinegate"
 )
 
@@ -59,7 +60,7 @@ func extractArchive(r io.Reader, destinationDirPath string) error {
 
 func restoreFiles(srcDir string, destinationDir string) error {
 	for _, filename := range filesToRestore {
-		err := copyPath(filepath.Join(srcDir, filename), destinationDir)
+		err := filesystem.CopyPath(filepath.Join(srcDir, filename), destinationDir)
 		if err != nil {
 			return err
 		}

api/bolt/migrator/migrate_dbversion33.go

@@ -0,0 +1,32 @@
+package migrator
+
+import (
+	portainer "github.com/portainer/portainer/api"
+)
+
+func (m *Migrator) migrateDBVersionTo33() error {
+	err := migrateStackEntryPoint(m.stackService)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func migrateStackEntryPoint(stackService portainer.StackService) error {
+	stacks, err := stackService.Stacks()
+	if err != nil {
+		return err
+	}
+	for i := range stacks {
+		stack := &stacks[i]
+		if stack.GitConfig == nil {
+			continue
+		}
+		stack.GitConfig.ConfigFilePath = stack.EntryPoint
+		if err := stackService.UpdateStack(stack.ID, stack); err != nil {
+			return err
+		}
+	}
+	return nil
+}

api/bolt/migrator/migrate_dbversion33_test.go

@@ -0,0 +1,51 @@
+package migrator
+
+import (
+	"path"
+	"testing"
+	"time"
+
+	"github.com/boltdb/bolt"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+	"github.com/portainer/portainer/api/bolt/stack"
+	gittypes "github.com/portainer/portainer/api/git/types"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMigrateStackEntryPoint(t *testing.T) {
+	dbConn, err := bolt.Open(path.Join(t.TempDir(), "portainer-ee-mig-33.db"), 0600, &bolt.Options{Timeout: 1 * time.Second})
+	assert.NoError(t, err, "failed to init testing DB connection")
+	defer dbConn.Close()
+
+	stackService, err := stack.NewService(&internal.DbConnection{DB: dbConn})
+	assert.NoError(t, err, "failed to init testing Stack service")
+
+	stacks := []*portainer.Stack{
+		{
+			ID:         1,
+			EntryPoint: "dir/sub/compose.yml",
+		},
+		{
+			ID:         2,
+			EntryPoint: "dir/sub/compose.yml",
+			GitConfig:  &gittypes.RepoConfig{},
+		},
+	}
+
+	for _, s := range stacks {
+		err := stackService.CreateStack(s)
+		assert.NoError(t, err, "failed to create stack")
+	}
+
+	err = migrateStackEntryPoint(stackService)
+	assert.NoError(t, err, "failed to migrate entry point to Git ConfigFilePath")
+
+	s, err := stackService.Stack(1)
+	assert.NoError(t, err)
+	assert.Nil(t, s.GitConfig, "first stack should not have git config")
+
+	s, err = stackService.Stack(2)
+	assert.NoError(t, err)
+	assert.Equal(t, "dir/sub/compose.yml", s.GitConfig.ConfigFilePath, "second stack should have config file path migrated")
+}

api/bolt/migrator/migrator.go

@@ -381,5 +381,11 @@ func (m *Migrator) Migrate() error {
 		}
 	}
 
+	if m.currentDBVersion < 33 {
+		if err := m.migrateDBVersionTo33(); err != nil {
+			return err
+		}
+	}
+
 	return m.versionService.StoreDBVersion(portainer.DBVersion)
 }

api/bolt/stack/stack.go

@@ -1,11 +1,14 @@
 package stack
 
 import (
+	"strings"
+
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
+	pkgerrors "github.com/pkg/errors"
 )
 
 const (
@@ -133,3 +136,76 @@ func (service *Service) DeleteStack(ID portainer.StackID) error {
 	identifier := internal.Itob(int(ID))
 	return internal.DeleteObject(service.connection, BucketName, identifier)
 }
+
+// StackByWebhookID returns a pointer to a stack object by webhook ID.
+// It returns nil, errors.ErrObjectNotFound if there's no stack associated with the webhook ID.
+func (service *Service) StackByWebhookID(id string) (*portainer.Stack, error) {
+	if id == "" {
+		return nil, pkgerrors.New("webhook ID can't be empty string")
+	}
+	var stack portainer.Stack
+	found := false
+
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+		cursor := bucket.Cursor()
+
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var t struct {
+				AutoUpdate *struct {
+					WebhookID string `json:"Webhook"`
+				} `json:"AutoUpdate"`
+			}
+
+			err := internal.UnmarshalObject(v, &t)
+			if err != nil {
+				return err
+			}
+
+			if t.AutoUpdate != nil && strings.EqualFold(t.AutoUpdate.WebhookID, id) {
+				found = true
+				err := internal.UnmarshalObject(v, &stack)
+				if err != nil {
+					return err
+				}
+				break
+			}
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+	if !found {
+		return nil, errors.ErrObjectNotFound
+	}
+
+	return &stack, nil
+}
+
+// RefreshableStacks returns stacks that are configured for a periodic update
+func (service *Service) RefreshableStacks() ([]portainer.Stack, error) {
+	stacks := make([]portainer.Stack, 0)
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+		cursor := bucket.Cursor()
+
+		var stack portainer.Stack
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			err := internal.UnmarshalObject(v, &stack)
+			if err != nil {
+				return err
+			}
+
+			if stack.AutoUpdate != nil && stack.AutoUpdate.Interval != "" {
+				stacks = append(stacks, stack)
+			}
+		}
+
+		return nil
+	})
+
+	return stacks, err
+}

api/bolt/stack/tests/stack_test.go

@@ -0,0 +1,111 @@
+package tests
+
+import (
+	"testing"
+	"time"
+
+	"github.com/portainer/portainer/api/bolt"
+
+	bolterrors "github.com/portainer/portainer/api/bolt/errors"
+
+	"github.com/portainer/portainer/api/bolt/bolttest"
+
+	"github.com/gofrs/uuid"
+
+	"github.com/stretchr/testify/assert"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/filesystem"
+)
+
+func newGuidString(t *testing.T) string {
+	uuid, err := uuid.NewV4()
+	assert.NoError(t, err)
+
+	return uuid.String()
+}
+
+type stackBuilder struct {
+	t     *testing.T
+	count int
+	store *bolt.Store
+}
+
+func TestService_StackByWebhookID(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode. Normally takes ~1s to run.")
+	}
+	store, teardown := bolttest.MustNewTestStore(true)
+	defer teardown()
+
+	b := stackBuilder{t: t, store: store}
+	b.createNewStack(newGuidString(t))
+	for i := 0; i < 10; i++ {
+		b.createNewStack("")
+	}
+	webhookID := newGuidString(t)
+	stack := b.createNewStack(webhookID)
+
+	// can find a stack by webhook ID
+	got, err := store.StackService.StackByWebhookID(webhookID)
+	assert.NoError(t, err)
+	assert.Equal(t, stack, *got)
+
+	// returns nil and object not found error if there's no stack associated with the webhook
+	got, err = store.StackService.StackByWebhookID(newGuidString(t))
+	assert.Nil(t, got)
+	assert.ErrorIs(t, err, bolterrors.ErrObjectNotFound)
+}
+
+func (b *stackBuilder) createNewStack(webhookID string) portainer.Stack {
+	b.count++
+	stack := portainer.Stack{
+		ID:           portainer.StackID(b.count),
+		Name:         "Name",
+		Type:         portainer.DockerComposeStack,
+		EndpointID:   2,
+		EntryPoint:   filesystem.ComposeFileDefaultName,
+		Env:          []portainer.Pair{{"Name1", "Value1"}},
+		Status:       portainer.StackStatusActive,
+		CreationDate: time.Now().Unix(),
+		ProjectPath:  "/tmp/project",
+		CreatedBy:    "test",
+	}
+
+	if webhookID == "" {
+		if b.count%2 == 0 {
+			stack.AutoUpdate = &portainer.StackAutoUpdate{
+				Interval: "",
+				Webhook:  "",
+			}
+		} // else keep AutoUpdate nil
+	} else {
+		stack.AutoUpdate = &portainer.StackAutoUpdate{Webhook: webhookID}
+	}
+
+	err := b.store.StackService.CreateStack(&stack)
+	assert.NoError(b.t, err)
+
+	return stack
+}
+
+func Test_RefreshableStacks(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode. Normally takes ~1s to run.")
+	}
+	store, teardown := bolttest.MustNewTestStore(true)
+	defer teardown()
+
+	staticStack := portainer.Stack{ID: 1}
+	stackWithWebhook := portainer.Stack{ID: 2, AutoUpdate: &portainer.StackAutoUpdate{Webhook: "webhook"}}
+	refreshableStack := portainer.Stack{ID: 3, AutoUpdate: &portainer.StackAutoUpdate{Interval: "1m"}}
+
+	for _, stack := range []*portainer.Stack{&staticStack, &stackWithWebhook, &refreshableStack} {
+		err := store.Stack().CreateStack(stack)
+		assert.NoError(t, err)
+	}
+
+	stacks, err := store.Stack().RefreshableStacks()
+	assert.NoError(t, err)
+	assert.ElementsMatch(t, []portainer.Stack{refreshableStack}, stacks)
+}

api/cmd/portainer/main.go

@@ -6,7 +6,6 @@ import (
 	"os"
 	"strings"
 
-	wrapper "github.com/portainer/docker-compose-wrapper"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt"
 	"github.com/portainer/portainer/api/chisel"
@@ -31,6 +30,8 @@ import (
 	"github.com/portainer/portainer/api/ldap"
 	"github.com/portainer/portainer/api/libcompose"
 	"github.com/portainer/portainer/api/oauth"
+	"github.com/portainer/portainer/api/scheduler"
+	"github.com/portainer/portainer/api/stacks"
 )
 
 func initCLI() *portainer.CLIFlags {
@@ -88,12 +89,8 @@ func shutdownDatastore(shutdownCtx context.Context, datastore portainer.DataStor
 func initComposeStackManager(assetsPath string, dataStorePath string, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager {
 	composeWrapper, err := exec.NewComposeStackManager(assetsPath, dataStorePath, proxyManager)
 	if err != nil {
-		if err == wrapper.ErrBinaryNotFound {
-			log.Printf("[INFO] [message: docker-compose binary not found, falling back to libcompose]")
-			return libcompose.NewComposeStackManager(dataStorePath, reverseTunnelService)
-		}
-
-		log.Fatalf("failed initalizing compose stack manager; err=%s", err)
+		log.Printf("[INFO] [main,compose] [message: falling-back to libcompose] [error: %s]", err)
+		return libcompose.NewComposeStackManager(dataStorePath, reverseTunnelService)
 	}
 
 	return composeWrapper
@@ -525,6 +522,10 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatalf("failed to fetch ssl settings from DB")
 	}
 
+	scheduler := scheduler.NewScheduler(shutdownCtx)
+	stackDeployer := stacks.NewStackDeployer(swarmStackManager, composeStackManager)
+	stacks.StartStackSchedules(scheduler, stackDeployer, dataStore, gitService)
+
 	return &http.Server{
 		AuthorizationService:        authorizationService,
 		ReverseTunnelService:        reverseTunnelService,
@@ -550,8 +551,10 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		SSLService:                  sslService,
 		DockerClientFactory:         dockerClientFactory,
 		KubernetesClientFactory:     kubernetesClientFactory,
+		Scheduler:                   scheduler,
 		ShutdownCtx:                 shutdownCtx,
 		ShutdownTrigger:             shutdownTrigger,
+		StackDeployer:               stackDeployer,
 	}
 }
 

api/exec/compose_stack.go

@@ -7,8 +7,8 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/pkg/errors"
 	wrapper "github.com/portainer/docker-compose-wrapper"
-
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/http/proxy"
 	"github.com/portainer/portainer/api/http/proxy/factory"
@@ -35,12 +35,6 @@ func NewComposeStackManager(binaryPath string, configPath string, proxyManager *
 	}, nil
 }
 
-// NormalizeStackName returns a new stack name with unsupported characters replaced
-func (w *ComposeStackManager) NormalizeStackName(name string) string {
-	r := regexp.MustCompile("[^a-z0-9]+")
-	return r.ReplaceAllString(strings.ToLower(name), "")
-}
-
 // ComposeSyntaxMaxVersion returns the maximum supported version of the docker compose syntax
 func (w *ComposeStackManager) ComposeSyntaxMaxVersion() string {
 	return portainer.ComposeSyntaxMaxVersion
@@ -50,7 +44,7 @@ func (w *ComposeStackManager) ComposeSyntaxMaxVersion() string {
 func (w *ComposeStackManager) Up(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
 	url, proxy, err := w.fetchEndpointProxy(endpoint)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "failed to featch endpoint proxy")
 	}
 
 	if proxy != nil {
@@ -59,13 +53,12 @@ func (w *ComposeStackManager) Up(stack *portainer.Stack, endpoint *portainer.End
 
 	envFilePath, err := createEnvFile(stack)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "failed to create env file")
 	}
 
-	filePath := stackFilePath(stack)
-
-	_, err = w.wrapper.Up([]string{filePath}, url, stack.Name, envFilePath, w.configPath)
-	return err
+	filePaths := append([]string{stack.EntryPoint}, stack.AdditionalFiles...)
+	_, err = w.wrapper.Up(filePaths, stack.ProjectPath, url, stack.Name, envFilePath, w.configPath)
+	return errors.Wrap(err, "failed to deploy a stack")
 }
 
 // Down stops and removes containers, networks, images, and volumes. Wraps `docker-compose down --remove-orphans` command
@@ -78,14 +71,16 @@ func (w *ComposeStackManager) Down(stack *portainer.Stack, endpoint *portainer.E
 		defer proxy.Close()
 	}
 
-	filePath := stackFilePath(stack)
+	filePaths := append([]string{stack.EntryPoint}, stack.AdditionalFiles...)
 
-	_, err = w.wrapper.Down([]string{filePath}, url, stack.Name)
+	_, err = w.wrapper.Down(filePaths, stack.ProjectPath, url, stack.Name)
 	return err
 }
 
-func stackFilePath(stack *portainer.Stack) string {
-	return path.Join(stack.ProjectPath, stack.EntryPoint)
+// NormalizeStackName returns a new stack name with unsupported characters replaced
+func (w *ComposeStackManager) NormalizeStackName(name string) string {
+	r := regexp.MustCompile("[^a-z0-9]+")
+	return r.ReplaceAllString(strings.ToLower(name), "")
 }
 
 func (w *ComposeStackManager) fetchEndpointProxy(endpoint *portainer.Endpoint) (string, *factory.ProxyServer, error) {
@@ -118,5 +113,5 @@ func createEnvFile(stack *portainer.Stack) (string, error) {
 	}
 	envfile.Close()
 
-	return envFilePath, nil
+	return "stack.env", nil
 }

api/exec/compose_stack_integration_test.go

@@ -1,5 +1,3 @@
-// +build integration
-
 package exec
 
 import (

api/exec/compose_stack_test.go

@@ -10,47 +10,6 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func Test_stackFilePath(t *testing.T) {
-	tests := []struct {
-		name     string
-		stack    *portainer.Stack
-		expected string
-	}{
-		// {
-		// 	name:     "should return empty result if stack is missing",
-		// 	stack:    nil,
-		// 	expected: "",
-		// },
-		// {
-		// 	name:     "should return empty result if stack don't have entrypoint",
-		// 	stack:    &portainer.Stack{},
-		// 	expected: "",
-		// },
-		{
-			name: "should allow file name and dir",
-			stack: &portainer.Stack{
-				ProjectPath: "dir",
-				EntryPoint:  "file",
-			},
-			expected: path.Join("dir", "file"),
-		},
-		{
-			name: "should allow file name only",
-			stack: &portainer.Stack{
-				EntryPoint: "file",
-			},
-			expected: "file",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			result := stackFilePath(tt.stack)
-			assert.Equal(t, tt.expected, result)
-		})
-	}
-}
-
 func Test_createEnvFile(t *testing.T) {
 	dir := t.TempDir()
 
@@ -60,11 +19,6 @@ func Test_createEnvFile(t *testing.T) {
 		expected     string
 		expectedFile bool
 	}{
-		// {
-		// 	name:     "should not add env file option if stack is missing",
-		// 	stack:    nil,
-		// 	expected: "",
-		// },
 		{
 			name: "should not add env file option if stack doesn't have env variables",
 			stack: &portainer.Stack{
@@ -98,7 +52,7 @@ func Test_createEnvFile(t *testing.T) {
 			result, _ := createEnvFile(tt.stack)
 
 			if tt.expected != "" {
-				assert.Equal(t, path.Join(tt.stack.ProjectPath, "stack.env"), result)
+				assert.Equal(t, "stack.env", result)
 
 				f, _ := os.Open(path.Join(dir, "stack.env"))
 				content, _ := ioutil.ReadAll(f)

api/exec/swarm_stack.go

@@ -13,6 +13,7 @@ import (
 	"strings"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/stackutils"
 )
 
 // SwarmStackManager represents a service for managing stacks.
@@ -63,22 +64,23 @@ func (manager *SwarmStackManager) Logout(endpoint *portainer.Endpoint) error {
 
 // Deploy executes the docker stack deploy command.
 func (manager *SwarmStackManager) Deploy(stack *portainer.Stack, prune bool, endpoint *portainer.Endpoint) error {
-	stackFilePath := path.Join(stack.ProjectPath, stack.EntryPoint)
+	filePaths := stackutils.GetStackFilePaths(stack)
 	command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
 
 	if prune {
-		args = append(args, "stack", "deploy", "--prune", "--with-registry-auth", "--compose-file", stackFilePath, stack.Name)
+		args = append(args, "stack", "deploy", "--prune", "--with-registry-auth")
 	} else {
-		args = append(args, "stack", "deploy", "--with-registry-auth", "--compose-file", stackFilePath, stack.Name)
+		args = append(args, "stack", "deploy", "--with-registry-auth")
 	}
 
+	args = configureFilePaths(args, filePaths)
+	args = append(args, stack.Name)
+
 	env := make([]string, 0)
 	for _, envvar := range stack.Env {
 		env = append(env, envvar.Name+"="+envvar.Value)
 	}
-
-	stackFolder := path.Dir(stackFilePath)
-	return runCommandAndCaptureStdErr(command, args, env, stackFolder)
+	return runCommandAndCaptureStdErr(command, args, env, stack.ProjectPath)
 }
 
 // Remove executes the docker stack rm command.
@@ -191,3 +193,10 @@ func (manager *SwarmStackManager) NormalizeStackName(name string) string {
 	r := regexp.MustCompile("[^a-z0-9]+")
 	return r.ReplaceAllString(strings.ToLower(name), "")
 }
+
+func configureFilePaths(args []string, filePaths []string) []string {
+	for _, path := range filePaths {
+		args = append(args, "--compose-file", path)
+	}
+	return args
+}

api/exec/swarm_stack_test.go

@@ -0,0 +1,15 @@
+package exec
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConfigFilePaths(t *testing.T) {
+	args := []string{"stack", "deploy", "--with-registry-auth"}
+	filePaths := []string{"dir/file", "dir/file-two", "dir/file-three"}
+	expected := []string{"stack", "deploy", "--with-registry-auth", "--compose-file", "dir/file", "--compose-file", "dir/file-two", "--compose-file", "dir/file-three"}
+	output := configureFilePaths(args, filePaths)
+	assert.ElementsMatch(t, expected, output, "wrong output file paths")
+}

api/filesystem/copy.go

@@ -1,4 +1,4 @@
-package backup
+package filesystem
 
 import (
 	"errors"
@@ -8,7 +8,8 @@ import (
 	"strings"
 )
 
-func copyPath(path string, toDir string) error {
+// CopyPath copies file or directory defined by the path to the toDir path
+func CopyPath(path string, toDir string) error {
 	info, err := os.Stat(path)
 	if err != nil && errors.Is(err, os.ErrNotExist) {
 		// skip copy if file does not exist
@@ -20,17 +21,30 @@ func copyPath(path string, toDir string) error {
 		return copyFile(path, destination)
 	}
 
-	return copyDir(path, toDir)
+	return CopyDir(path, toDir, true)
 }
 
-func copyDir(fromDir, toDir string) error {
+// CopyDir copies contents of fromDir to toDir.
+// When keepParent is true, contents will be copied with their immediate parent dir,
+// i.e. given /from/dirA and /to/dirB with keepParent == true, result will be /to/dirB/dirA/<children>
+func CopyDir(fromDir, toDir string, keepParent bool) error {
 	cleanedSourcePath := filepath.Clean(fromDir)
 	parentDirectory := filepath.Dir(cleanedSourcePath)
 	err := filepath.Walk(cleanedSourcePath, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
-		destination := filepath.Join(toDir, strings.TrimPrefix(path, parentDirectory))
+		var destination string
+		if keepParent {
+			destination = filepath.Join(toDir, strings.TrimPrefix(path, parentDirectory))
+		} else {
+			destination = filepath.Join(toDir, strings.TrimPrefix(path, cleanedSourcePath))
+		}
+
+		if destination == "" {
+			return nil
+		}
+
 		if info.IsDir() {
 			return nil // skip directory creations
 		}

api/filesystem/copy_test.go

@@ -0,0 +1,92 @@
+package filesystem
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_copyFile_returnsError_whenSourceDoesNotExist(t *testing.T) {
+	tmpdir, _ := ioutil.TempDir("", "backup")
+	defer os.RemoveAll(tmpdir)
+
+	err := copyFile("does-not-exist", tmpdir)
+	assert.Error(t, err)
+}
+
+func Test_copyFile_shouldMakeAbackup(t *testing.T) {
+	tmpdir, _ := ioutil.TempDir("", "backup")
+	defer os.RemoveAll(tmpdir)
+
+	content := []byte("content")
+	ioutil.WriteFile(path.Join(tmpdir, "origin"), content, 0600)
+
+	err := copyFile(path.Join(tmpdir, "origin"), path.Join(tmpdir, "copy"))
+	assert.NoError(t, err)
+
+	copyContent, _ := ioutil.ReadFile(path.Join(tmpdir, "copy"))
+	assert.Equal(t, content, copyContent)
+}
+
+func Test_CopyDir_shouldCopyAllFilesAndDirectories(t *testing.T) {
+	destination, _ := ioutil.TempDir("", "destination")
+	defer os.RemoveAll(destination)
+	err := CopyDir("./testdata/copy_test", destination, true)
+	assert.NoError(t, err)
+
+	assert.FileExists(t, filepath.Join(destination, "copy_test", "outer"))
+	assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", ".dotfile"))
+	assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", "inner"))
+}
+
+func Test_CopyDir_shouldCopyOnlyDirContents(t *testing.T) {
+	destination, _ := ioutil.TempDir("", "destination")
+	defer os.RemoveAll(destination)
+	err := CopyDir("./testdata/copy_test", destination, false)
+	assert.NoError(t, err)
+
+	assert.FileExists(t, filepath.Join(destination, "outer"))
+	assert.FileExists(t, filepath.Join(destination, "dir", ".dotfile"))
+	assert.FileExists(t, filepath.Join(destination, "dir", "inner"))
+}
+
+func Test_CopyPath_shouldSkipWhenNotExist(t *testing.T) {
+	tmpdir, _ := ioutil.TempDir("", "backup")
+	defer os.RemoveAll(tmpdir)
+
+	err := CopyPath("does-not-exists", tmpdir)
+	assert.NoError(t, err)
+
+	assert.NoFileExists(t, tmpdir)
+}
+
+func Test_CopyPath_shouldCopyFile(t *testing.T) {
+	tmpdir, _ := ioutil.TempDir("", "backup")
+	defer os.RemoveAll(tmpdir)
+
+	content := []byte("content")
+	ioutil.WriteFile(path.Join(tmpdir, "file"), content, 0600)
+
+	os.MkdirAll(path.Join(tmpdir, "backup"), 0700)
+	err := CopyPath(path.Join(tmpdir, "file"), path.Join(tmpdir, "backup"))
+	assert.NoError(t, err)
+
+	copyContent, err := ioutil.ReadFile(path.Join(tmpdir, "backup", "file"))
+	assert.NoError(t, err)
+	assert.Equal(t, content, copyContent)
+}
+
+func Test_CopyPath_shouldCopyDir(t *testing.T) {
+	destination, _ := ioutil.TempDir("", "destination")
+	defer os.RemoveAll(destination)
+	err := CopyPath("./testdata/copy_test", destination)
+	assert.NoError(t, err)
+
+	assert.FileExists(t, filepath.Join(destination, "copy_test", "outer"))
+	assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", ".dotfile"))
+	assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", "inner"))
+}

api/git/azure.go

@@ -2,15 +2,17 @@ package git
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
-	"github.com/pkg/errors"
-	"github.com/portainer/portainer/api/archive"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
 	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/portainer/portainer/api/archive"
 )
 
 const (
@@ -37,7 +39,7 @@ type azureDownloader struct {
 
 func NewAzureDownloader(client *http.Client) *azureDownloader {
 	return &azureDownloader{
-		client: client,
+		client:  client,
 		baseUrl: "https://dev.azure.com",
 	}
 }
@@ -100,6 +102,57 @@ func (a *azureDownloader) downloadZipFromAzureDevOps(ctx context.Context, option
 	return zipFile.Name(), nil
 }
 
+func (a *azureDownloader) latestCommitID(ctx context.Context, options fetchOptions) (string, error) {
+	config, err := parseUrl(options.repositoryUrl)
+	if err != nil {
+		return "", errors.WithMessage(err, "failed to parse url")
+	}
+
+	refsUrl, err := a.buildRefsUrl(config, options.referenceName)
+	if err != nil {
+		return "", errors.WithMessage(err, "failed to build azure refs url")
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "GET", refsUrl, nil)
+	if options.username != "" || options.password != "" {
+		req.SetBasicAuth(options.username, options.password)
+	} else if config.username != "" || config.password != "" {
+		req.SetBasicAuth(config.username, config.password)
+	}
+
+	if err != nil {
+		return "", errors.WithMessage(err, "failed to create a new HTTP request")
+	}
+
+	resp, err := a.client.Do(req)
+	if err != nil {
+		return "", errors.WithMessage(err, "failed to make an HTTP request")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("failed to get repository refs with a status \"%v\"", resp.Status)
+	}
+
+	var refs struct {
+		Value []struct {
+			Name     string `json:"name"`
+			ObjectId string `json:"objectId"`
+		}
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&refs); err != nil {
+		return "", errors.Wrap(err, "could not parse Azure Refs response")
+	}
+
+	for _, ref := range refs.Value {
+		if strings.EqualFold(ref.Name, options.referenceName) {
+			return ref.ObjectId, nil
+		}
+	}
+
+	return "", errors.Errorf("could not find ref %q in the repository", options.referenceName)
+}
+
 func parseUrl(rawUrl string) (*azureOptions, error) {
 	if strings.HasPrefix(rawUrl, "https://") || strings.HasPrefix(rawUrl, "http://") {
 		return parseHttpUrl(rawUrl)
@@ -193,6 +246,27 @@ func (a *azureDownloader) buildDownloadUrl(config *azureOptions, referenceName s
 	return u.String(), nil
 }
 
+func (a *azureDownloader) buildRefsUrl(config *azureOptions, referenceName string) (string, error) {
+	rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/refs",
+		a.baseUrl,
+		url.PathEscape(config.organisation),
+		url.PathEscape(config.project),
+		url.PathEscape(config.repository))
+	u, err := url.Parse(rawUrl)
+
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to parse refs url path %s", rawUrl)
+	}
+
+	// filterContains=main&api-version=6.0
+	q := u.Query()
+	q.Set("filterContains", formatReferenceName(referenceName))
+	q.Set("api-version", "6.0")
+	u.RawQuery = q.Encode()
+
+	return u.String(), nil
+}
+
 const (
 	branchPrefix = "refs/heads/"
 	tagPrefix    = "refs/tags/"

api/git/azure_integration_test.go

@@ -78,6 +78,18 @@ func TestService_ClonePrivateRepository_Azure(t *testing.T) {
 	assert.FileExists(t, filepath.Join(dst, "README.md"))
 }
 
+func TestService_LatestCommitID_Azure(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	pat := getRequiredValue(t, "AZURE_DEVOPS_PAT")
+	service := NewService()
+
+	repositoryUrl := "https://portainer.visualstudio.com/Playground/_git/dev_integration"
+	id, err := service.LatestCommitID(repositoryUrl, "refs/heads/main", "", pat)
+	assert.NoError(t, err)
+	assert.NotEmpty(t, id, "cannot guarantee commit id, but it should be not empty")
+}
+
 func getRequiredValue(t *testing.T, name string) string {
 	value, ok := os.LookupEnv(name)
 	if !ok {

api/git/azure_test.go

@@ -2,11 +2,12 @@ package git
 
 import (
 	"context"
-	"github.com/stretchr/testify/assert"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func Test_buildDownloadUrl(t *testing.T) {
@@ -27,6 +28,23 @@ func Test_buildDownloadUrl(t *testing.T) {
 	}
 }
 
+func Test_buildRefsUrl(t *testing.T) {
+	a := NewAzureDownloader(nil)
+	u, err := a.buildRefsUrl(&azureOptions{
+		organisation: "organisation",
+		project:      "project",
+		repository:   "repository",
+	}, "refs/heads/main")
+
+	expectedUrl, _ := url.Parse("https://dev.azure.com/organisation/project/_apis/git/repositories/repository/refs?filterContains=main&api-version=6.0")
+	actualUrl, _ := url.Parse(u)
+	assert.NoError(t, err)
+	assert.Equal(t, expectedUrl.Host, actualUrl.Host)
+	assert.Equal(t, expectedUrl.Scheme, actualUrl.Scheme)
+	assert.Equal(t, expectedUrl.Path, actualUrl.Path)
+	assert.Equal(t, expectedUrl.Query(), actualUrl.Query())
+}
+
 func Test_parseAzureUrl(t *testing.T) {
 	type args struct {
 		url string
@@ -248,3 +266,110 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
 		})
 	}
 }
+
+func Test_azureDownloader_latestCommitID(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		response := `{
+			"value": [
+				{
+					"name": "refs/heads/feature/calcApp",
+					"objectId": "ffe9cba521f00d7f60e322845072238635edb451",
+					"creator": {
+						"displayName": "Normal Paulk",
+						"url": "https://vssps.dev.azure.com/fabrikam/_apis/Identities/ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"_links": {
+							"avatar": {
+								"href": "https://dev.azure.com/fabrikam/_apis/GraphProfile/MemberAvatars/aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5"
+							}
+						},
+						"id": "ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"uniqueName": "dev@mailserver.com",
+						"imageUrl": "https://dev.azure.com/fabrikam/_api/_common/identityImage?id=ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"descriptor": "aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5"
+					},
+					"url": "https://dev.azure.com/fabrikam/7484f783-66a3-4f27-b7cd-6b08b0b077ed/_apis/git/repositories/d3d1760b-311c-4175-a726-20dfc6a7f885/refs?filter=heads%2Ffeature%2FcalcApp"
+				},
+				{
+					"name": "refs/heads/feature/replacer",
+					"objectId": "917131a709996c5cfe188c3b57e9a6ad90e8b85c",
+					"creator": {
+						"displayName": "Normal Paulk",
+						"url": "https://vssps.dev.azure.com/fabrikam/_apis/Identities/ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"_links": {
+							"avatar": {
+								"href": "https://dev.azure.com/fabrikam/_apis/GraphProfile/MemberAvatars/aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5"
+							}
+						},
+						"id": "ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"uniqueName": "dev@mailserver.com",
+						"imageUrl": "https://dev.azure.com/fabrikam/_api/_common/identityImage?id=ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"descriptor": "aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5"
+					},
+					"url": "https://dev.azure.com/fabrikam/7484f783-66a3-4f27-b7cd-6b08b0b077ed/_apis/git/repositories/d3d1760b-311c-4175-a726-20dfc6a7f885/refs?filter=heads%2Ffeature%2Freplacer"
+				},
+				{
+					"name": "refs/heads/master",
+					"objectId": "ffe9cba521f00d7f60e322845072238635edb451",
+					"creator": {
+						"displayName": "Normal Paulk",
+						"url": "https://vssps.dev.azure.com/fabrikam/_apis/Identities/ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"_links": {
+							"avatar": {
+								"href": "https://dev.azure.com/fabrikam/_apis/GraphProfile/MemberAvatars/aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5"
+							}
+						},
+						"id": "ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"uniqueName": "dev@mailserver.com",
+						"imageUrl": "https://dev.azure.com/fabrikam/_api/_common/identityImage?id=ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"descriptor": "aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5"
+					},
+					"url": "https://dev.azure.com/fabrikam/7484f783-66a3-4f27-b7cd-6b08b0b077ed/_apis/git/repositories/d3d1760b-311c-4175-a726-20dfc6a7f885/refs?filter=heads%2Fmaster"
+				}
+			],
+			"count": 3
+		}`
+		w.Header().Set("Content-Type", "application/json")
+		w.Write([]byte(response))
+	}))
+	defer server.Close()
+
+	a := &azureDownloader{
+		client:  server.Client(),
+		baseUrl: server.URL,
+	}
+
+	tests := []struct {
+		name    string
+		args    fetchOptions
+		want    string
+		wantErr bool
+	}{
+		{
+			name: "should be able to parse response",
+			args: fetchOptions{
+				referenceName: "refs/heads/master",
+				repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository"},
+			want:    "ffe9cba521f00d7f60e322845072238635edb451",
+			wantErr: false,
+		},
+		{
+			name: "should be able to parse response",
+			args: fetchOptions{
+				referenceName: "refs/heads/unknown",
+				repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository"},
+			want:    "",
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			id, err := a.latestCommitID(context.Background(), tt.args)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("azureDownloader.latestCommitID() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			assert.Equal(t, tt.want, id)
+		})
+	}
+}

api/git/git.go

@@ -6,16 +6,26 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
 
 	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/transport/client"
 	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
+	"github.com/go-git/go-git/v5/storage/memory"
 )
 
+type fetchOptions struct {
+	repositoryUrl string
+	username      string
+	password      string
+	referenceName string
+}
+
 type cloneOptions struct {
 	repositoryUrl string
 	username      string
@@ -26,6 +36,7 @@ type cloneOptions struct {
 
 type downloader interface {
 	download(ctx context.Context, dst string, opt cloneOptions) error
+	latestCommitID(ctx context.Context, opt fetchOptions) (string, error)
 }
 
 type gitClient struct {
@@ -36,13 +47,7 @@ func (c gitClient) download(ctx context.Context, dst string, opt cloneOptions) e
 	gitOptions := git.CloneOptions{
 		URL:   opt.repositoryUrl,
 		Depth: opt.depth,
-	}
-
-	if opt.password != "" || opt.username != "" {
-		gitOptions.Auth = &githttp.BasicAuth{
-			Username: opt.username,
-			Password: opt.password,
-		}
+		Auth:  getAuth(opt.username, opt.password),
 	}
 
 	if opt.referenceName != "" {
@@ -62,6 +67,44 @@ func (c gitClient) download(ctx context.Context, dst string, opt cloneOptions) e
 	return nil
 }
 
+func (c gitClient) latestCommitID(ctx context.Context, opt fetchOptions) (string, error) {
+	remote := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{
+		Name: "origin",
+		URLs: []string{opt.repositoryUrl},
+	})
+
+	listOptions := &git.ListOptions{
+		Auth: getAuth(opt.username, opt.password),
+	}
+
+	refs, err := remote.List(listOptions)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to list repository refs")
+	}
+
+	for _, ref := range refs {
+		if strings.EqualFold(ref.Name().String(), opt.referenceName) {
+			return ref.Hash().String(), nil
+		}
+	}
+
+	return "", errors.Errorf("could not find ref %q in the repository", opt.referenceName)
+}
+
+func getAuth(username, password string) *githttp.BasicAuth {
+	if password != "" {
+		if username == "" {
+			username = "token"
+		}
+
+		return &githttp.BasicAuth{
+			Username: username,
+			Password: password,
+		}
+	}
+	return nil
+}
+
 // Service represents a service for managing Git.
 type Service struct {
 	httpsCli *http.Client
@@ -109,3 +152,19 @@ func (service *Service) cloneRepository(destination string, options cloneOptions
 
 	return service.git.download(context.TODO(), destination, options)
 }
+
+// LatestCommitID returns SHA1 of the latest commit of the specified reference
+func (service *Service) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) {
+	options := fetchOptions{
+		repositoryUrl: repositoryURL,
+		username:      username,
+		password:      password,
+		referenceName: referenceName,
+	}
+
+	if isAzureUrl(options.repositoryUrl) {
+		return service.azure.latestCommitID(context.TODO(), options)
+	}
+
+	return service.git.latestCommitID(context.TODO(), options)
+}

api/git/git_integration_test.go

@@ -12,7 +12,7 @@ import (
 func TestService_ClonePrivateRepository_GitHub(t *testing.T) {
 	ensureIntegrationTest(t)
 
-	pat := getRequiredValue(t, "GITHUB_PAT")
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
 	username := getRequiredValue(t, "GITHUB_USERNAME")
 	service := NewService()
 
@@ -21,7 +21,20 @@ func TestService_ClonePrivateRepository_GitHub(t *testing.T) {
 	defer os.RemoveAll(dst)
 
 	repositoryUrl := "https://github.com/portainer/private-test-repository.git"
-	err = service.CloneRepository(dst, repositoryUrl, "refs/heads/main", username, pat)
+	err = service.CloneRepository(dst, repositoryUrl, "refs/heads/main", username, accessToken)
 	assert.NoError(t, err)
 	assert.FileExists(t, filepath.Join(dst, "README.md"))
 }
+
+func TestService_LatestCommitID_GitHub(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
+	username := getRequiredValue(t, "GITHUB_USERNAME")
+	service := NewService()
+
+	repositoryUrl := "https://github.com/portainer/private-test-repository.git"
+	id, err := service.LatestCommitID(repositoryUrl, "refs/heads/main", username, accessToken)
+	assert.NoError(t, err)
+	assert.NotEmpty(t, id, "cannot guarantee commit id, but it should be not empty")
+}

api/git/git_test.go

@@ -105,7 +105,19 @@ func Test_cloneRepository(t *testing.T) {
 	})
 
 	assert.NoError(t, err)
-	assert.Equal(t, 3, getCommitHistoryLength(t, err, dir), "cloned repo has incorrect depth")
+	assert.Equal(t, 4, getCommitHistoryLength(t, err, dir), "cloned repo has incorrect depth")
+}
+
+func Test_latestCommitID(t *testing.T) {
+	service := Service{git: gitClient{preserveGitDirectory: true}} // no need for http client since the test access the repo via file system.
+
+	repositoryURL := bareRepoDir
+	referenceName := "refs/heads/main"
+
+	id, err := service.LatestCommitID(repositoryURL, referenceName, "", "")
+
+	assert.NoError(t, err)
+	assert.Equal(t, "68dcaa7bd452494043c64252ab90db0f98ecf8d2", id)
 }
 
 func getCommitHistoryLength(t *testing.T, err error, dir string) int {
@@ -137,6 +149,10 @@ func (t *testDownloader) download(_ context.Context, _ string, _ cloneOptions) e
 	return nil
 }
 
+func (t *testDownloader) latestCommitID(_ context.Context, _ fetchOptions) (string, error) {
+	return "", nil
+}
+
 func Test_cloneRepository_azure(t *testing.T) {
 	tests := []struct {
 		name   string

api/git/types/types.go

@@ -1,10 +1,20 @@
 package gittypes
 
+// RepoConfig represents a configuration for a repo
 type RepoConfig struct {
 	// The repo url
-	URL string `example:"https://github.com/portainer/portainer-ee.git"`
+	URL string `example:"https://github.com/portainer/portainer.git"`
 	// The reference name
 	ReferenceName string `example:"refs/heads/branch_name"`
 	// Path to where the config file is in this url/refName
 	ConfigFilePath string `example:"docker-compose.yml"`
+	// Git credentials
+	Authentication *GitAuthentication
+	// Repository hash
+	ConfigHash string `example:"bc4c183d756879ea4d173315338110b31004b8e0"`
+}
+
+type GitAuthentication struct {
+	Username string
+	Password string
 }

api/go.mod

@@ -27,10 +27,11 @@ require (
 	github.com/mitchellh/mapstructure v1.1.2 // indirect
 	github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6
 	github.com/pkg/errors v0.9.1
-	github.com/portainer/docker-compose-wrapper v0.0.0-20210527221011-0a1418224b92
+	github.com/portainer/docker-compose-wrapper v0.0.0-20210810234209-d01bc85eb481
 	github.com/portainer/libcompose v0.5.3
 	github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a
 	github.com/portainer/libhttp v0.0.0-20190806161843-ba068f58be33
+	github.com/robfig/cron/v3 v3.0.1
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.0
 	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2

api/go.sum

@@ -241,8 +241,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/portainer/docker-compose-wrapper v0.0.0-20210527221011-0a1418224b92 h1:Hh7SHCf3SJblVywU0TTn5lpTKsH5W23LAKH5sqWggig=
-github.com/portainer/docker-compose-wrapper v0.0.0-20210527221011-0a1418224b92/go.mod h1:PF2O2O4UNYWdtPcp6n/mIKpKk+f1jhFTezS8txbf+XM=
+github.com/portainer/docker-compose-wrapper v0.0.0-20210810234209-d01bc85eb481 h1:5c8N9Gh21Ja/9EIpfyHFmQvTCKgOjnRhosmo0ZshkFk=
+github.com/portainer/docker-compose-wrapper v0.0.0-20210810234209-d01bc85eb481/go.mod h1:WxDlJWZxCnicdLCPnLNEv7/gRhjeIVuCGmsv+iOPH3c=
 github.com/portainer/libcompose v0.5.3 h1:tE4WcPuGvo+NKeDkDWpwNavNLZ5GHIJ4RvuZXsI9uI8=
 github.com/portainer/libcompose v0.5.3/go.mod h1:7SKd/ho69rRKHDFSDUwkbMcol2TMKU5OslDsajr8Ro8=
 github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a h1:qY8TbocN75n5PDl16o0uVr5MevtM5IhdwSelXEd4nFM=
@@ -263,6 +263,8 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE=
 github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=

api/http/handler/stacks/autoupdate.go

@@ -0,0 +1,38 @@
+package stacks
+
+import (
+	"log"
+	"net/http"
+	"time"
+
+	httperror "github.com/portainer/libhttp/error"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/scheduler"
+	"github.com/portainer/portainer/api/stacks"
+)
+
+func startAutoupdate(stackID portainer.StackID, interval string, scheduler *scheduler.Scheduler, stackDeployer stacks.StackDeployer, datastore portainer.DataStore, gitService portainer.GitService) (jobID string, e *httperror.HandlerError) {
+	d, err := time.ParseDuration(interval)
+	if err != nil {
+		return "", &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Unable to parse stack's auto update interval", Err: err}
+	}
+
+	jobID = scheduler.StartJobEvery(d, func() {
+		if err := stacks.RedeployWhenChanged(stackID, stackDeployer, datastore, gitService); err != nil {
+			log.Printf("[ERROR] [http,stacks] [message: failed redeploying] [err: %s]\n", err)
+		}
+	})
+
+	return jobID, nil
+}
+
+func stopAutoupdate(stackID portainer.StackID, jobID string, scheduler scheduler.Scheduler) {
+	if jobID == "" {
+		return
+	}
+
+	if err := scheduler.StopJob(jobID); err != nil {
+		log.Printf("[WARN] could not stop the job for the stack %v", stackID)
+	}
+
+}

api/http/handler/stacks/create_compose_stack.go

@@ -1,7 +1,6 @@
 package stacks
 
 import (
-	"errors"
 	"fmt"
 	"net/http"
 	"path"
@@ -9,10 +8,12 @@ import (
 	"time"
 
 	"github.com/asaskevich/govalidator"
+	"github.com/pkg/errors"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/filesystem"
+	gittypes "github.com/portainer/portainer/api/git/types"
 	"github.com/portainer/portainer/api/http/security"
 )
 
@@ -100,7 +101,6 @@ func (handler *Handler) createComposeStackFromFileContent(w http.ResponseWriter,
 type composeStackFromGitRepositoryPayload struct {
 	// Name of the stack
 	Name string `example:"myStack" validate:"required"`
-
 	// URL of a Git repository hosting the Stack file
 	RepositoryURL string `example:"https://github.com/openfaas/faas" validate:"required"`
 	// Reference name of a Git repository hosting the Stack file
@@ -112,8 +112,11 @@ type composeStackFromGitRepositoryPayload struct {
 	// Password used in basic authentication. Required when RepositoryAuthentication is true.
 	RepositoryPassword string `example:"myGitPassword"`
 	// Path to the Stack file inside the Git repository
-	ComposeFilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"`
-
+	ComposeFile string `example:"docker-compose.yml" default:"docker-compose.yml"`
+	// Applicable when deploying with multiple stack files
+	AdditionalFiles []string `example:"[nz.compose.yml, uat.compose.yml]"`
+	// Optional auto update configuration
+	AutoUpdate *portainer.StackAutoUpdate
 	// A list of environment variables used during stack deployment
 	Env []portainer.Pair
 }
@@ -122,14 +125,18 @@ func (payload *composeStackFromGitRepositoryPayload) Validate(r *http.Request) e
 	if govalidator.IsNull(payload.Name) {
 		return errors.New("Invalid stack name")
 	}
-
 	if govalidator.IsNull(payload.RepositoryURL) || !govalidator.IsURL(payload.RepositoryURL) {
 		return errors.New("Invalid repository URL. Must correspond to a valid URL format")
 	}
-	if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) {
-		return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled")
+	if govalidator.IsNull(payload.RepositoryReferenceName) {
+		payload.RepositoryReferenceName = defaultGitReferenceName
+	}
+	if payload.RepositoryAuthentication && govalidator.IsNull(payload.RepositoryPassword) {
+		return errors.New("Invalid repository credentials. Password must be specified when authentication is enabled")
+	}
+	if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil {
+		return err
 	}
-
 	return nil
 }
 
@@ -141,42 +148,72 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite
 	}
 
 	payload.Name = handler.ComposeStackManager.NormalizeStackName(payload.Name)
-	if payload.ComposeFilePathInRepository == "" {
-		payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName
+	if payload.ComposeFile == "" {
+		payload.ComposeFile = filesystem.ComposeFileDefaultName
 	}
 
 	isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, false)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err}
 	}
 	if !isUnique {
-		errorMessage := fmt.Sprintf("A stack with the name '%s' already exists", payload.Name)
-		return &httperror.HandlerError{http.StatusConflict, errorMessage, errors.New(errorMessage)}
+		return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.Name), Err: errStackAlreadyExists}
+	}
+
+	//make sure the webhook ID is unique
+	if payload.AutoUpdate != nil && payload.AutoUpdate.Webhook != "" {
+		isUnique, err := handler.checkUniqueWebhookID(payload.AutoUpdate.Webhook)
+		if err != nil {
+			return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for webhook ID collision", Err: err}
+		}
+		if !isUnique {
+			return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("Webhook ID: %s already exists", payload.AutoUpdate.Webhook), Err: errWebhookIDAlreadyExists}
+		}
 	}
 
 	stackID := handler.DataStore.Stack().GetNextIdentifier()
 	stack := &portainer.Stack{
-		ID:           portainer.StackID(stackID),
-		Name:         payload.Name,
-		Type:         portainer.DockerComposeStack,
-		EndpointID:   endpoint.ID,
-		EntryPoint:   payload.ComposeFilePathInRepository,
-		Env:          payload.Env,
+		ID:              portainer.StackID(stackID),
+		Name:            payload.Name,
+		Type:            portainer.DockerComposeStack,
+		EndpointID:      endpoint.ID,
+		EntryPoint:      payload.ComposeFile,
+		AdditionalFiles: payload.AdditionalFiles,
+		AutoUpdate:      payload.AutoUpdate,
+		Env:             payload.Env,
+		GitConfig: &gittypes.RepoConfig{
+			URL:            payload.RepositoryURL,
+			ReferenceName:  payload.RepositoryReferenceName,
+			ConfigFilePath: payload.ComposeFile,
+		},
 		Status:       portainer.StackStatusActive,
 		CreationDate: time.Now().Unix(),
 	}
 
+	if payload.RepositoryAuthentication {
+		stack.GitConfig.Authentication = &gittypes.GitAuthentication{
+			Username: payload.RepositoryUsername,
+			Password: payload.RepositoryPassword,
+		}
+	}
+
 	projectPath := handler.FileService.GetStackProjectPath(strconv.Itoa(int(stack.ID)))
 	stack.ProjectPath = projectPath
 
 	doCleanUp := true
 	defer handler.cleanUp(stack, &doCleanUp)
 
-	err = handler.cloneAndSaveConfig(stack, projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, payload.ComposeFilePathInRepository, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword)
+	err = handler.clone(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword)
 	if err != nil {
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err}
 	}
 
+	commitId, err := handler.latestCommitID(payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to fetch git repository id", Err: err}
+	}
+	stack.GitConfig.ConfigHash = commitId
+
 	config, configErr := handler.createComposeDeployConfig(r, stack, endpoint)
 	if configErr != nil {
 		return configErr
@@ -187,6 +224,15 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err}
 	}
 
+	if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" {
+		jobID, e := startAutoupdate(stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService)
+		if e != nil {
+			return e
+		}
+
+		stack.AutoUpdate.JobID = jobID
+	}
+
 	stack.CreatedBy = config.user.Username
 
 	err = handler.DataStore.Stack().CreateStack(stack)
@@ -331,7 +377,7 @@ func (handler *Handler) createComposeDeployConfig(r *http.Request, stack *portai
 func (handler *Handler) deployComposeStack(config *composeStackDeploymentConfig) error {
 	isAdminOrEndpointAdmin, err := handler.userIsAdminOrEndpointAdmin(config.user, config.endpoint.ID)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "failed to check user priviliges deploying a stack")
 	}
 
 	securitySettings := &config.endpoint.SecuritySettings
@@ -344,15 +390,17 @@ func (handler *Handler) deployComposeStack(config *composeStackDeploymentConfig)
 		!securitySettings.AllowContainerCapabilitiesForRegularUsers) &&
 		!isAdminOrEndpointAdmin {
 
-		composeFilePath := path.Join(config.stack.ProjectPath, config.stack.EntryPoint)
-		stackContent, err := handler.FileService.GetFileContent(composeFilePath)
-		if err != nil {
-			return err
-		}
+		for _, file := range append([]string{config.stack.EntryPoint}, config.stack.AdditionalFiles...) {
+			path := path.Join(config.stack.ProjectPath, file)
+			stackContent, err := handler.FileService.GetFileContent(path)
+			if err != nil {
+				return errors.Wrapf(err, "failed to get stack file content `%q`", path)
+			}
 
-		err = handler.isValidStackFile(stackContent, securitySettings)
-		if err != nil {
-			return err
+			err = handler.isValidStackFile(stackContent, securitySettings)
+			if err != nil {
+				return errors.Wrap(err, "compose file is invalid")
+			}
 		}
 	}
 
@@ -363,7 +411,7 @@ func (handler *Handler) deployComposeStack(config *composeStackDeploymentConfig)
 
 	err = handler.ComposeStackManager.Up(config.stack, config.endpoint)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "failed to start up the stack")
 	}
 
 	return handler.SwarmStackManager.Logout(config.endpoint)

api/http/handler/stacks/create_kubernetes_stack_test.go

@@ -23,6 +23,10 @@ func (g *git) ClonePrivateRepositoryWithBasicAuth(repositoryURL, referenceName s
 	return g.ClonePublicRepository(repositoryURL, referenceName, destination)
 }
 
+func (g *git) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) {
+	return "", nil
+}
+
 func TestCloneAndConvertGitRepoFile(t *testing.T) {
 	dir, err := os.MkdirTemp("", "kube-create-stack")
 	assert.NoError(t, err, "failed to create a tmp dir")

api/http/handler/stacks/create_swarm_stack.go

@@ -13,6 +13,7 @@ import (
 	"github.com/portainer/libhttp/request"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/filesystem"
+	gittypes "github.com/portainer/portainer/api/git/types"
 	"github.com/portainer/portainer/api/http/security"
 )
 
@@ -121,7 +122,11 @@ type swarmStackFromGitRepositoryPayload struct {
 	// Password used in basic authentication. Required when RepositoryAuthentication is true.
 	RepositoryPassword string `example:"myGitPassword"`
 	// Path to the Stack file inside the Git repository
-	ComposeFilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"`
+	ComposeFile string `example:"docker-compose.yml" default:"docker-compose.yml"`
+	// Applicable when deploying with multiple stack files
+	AdditionalFiles []string `example:"[nz.compose.yml, uat.compose.yml]"`
+	// Optional auto update configuration
+	AutoUpdate *portainer.StackAutoUpdate
 }
 
 func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) error {
@@ -134,11 +139,14 @@ func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) err
 	if govalidator.IsNull(payload.RepositoryURL) || !govalidator.IsURL(payload.RepositoryURL) {
 		return errors.New("Invalid repository URL. Must correspond to a valid URL format")
 	}
-	if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) {
-		return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled")
+	if govalidator.IsNull(payload.RepositoryReferenceName) {
+		payload.RepositoryReferenceName = defaultGitReferenceName
 	}
-	if govalidator.IsNull(payload.ComposeFilePathInRepository) {
-		payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName
+	if payload.RepositoryAuthentication && govalidator.IsNull(payload.RepositoryPassword) {
+		return errors.New("Invalid repository credentials. Password must be specified when authentication is enabled")
+	}
+	if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil {
+		return err
 	}
 	return nil
 }
@@ -147,44 +155,74 @@ func (handler *Handler) createSwarmStackFromGitRepository(w http.ResponseWriter,
 	var payload swarmStackFromGitRepositoryPayload
 	err := request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err}
 	}
 
 	payload.Name = handler.SwarmStackManager.NormalizeStackName(payload.Name)
 
 	isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, true)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err}
 	}
 	if !isUnique {
-		errorMessage := fmt.Sprintf("A stack with the name '%s' is already running", payload.Name)
-		return &httperror.HandlerError{http.StatusConflict, errorMessage, errors.New(errorMessage)}
+		return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.Name), Err: errStackAlreadyExists}
+	}
+
+	//make sure the webhook ID is unique
+	if payload.AutoUpdate != nil && payload.AutoUpdate.Webhook != "" {
+		isUnique, err := handler.checkUniqueWebhookID(payload.AutoUpdate.Webhook)
+		if err != nil {
+			return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for webhook ID collision", Err: err}
+		}
+		if !isUnique {
+			return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("Webhook ID: %s already exists", payload.AutoUpdate.Webhook), Err: errWebhookIDAlreadyExists}
+		}
 	}
 
 	stackID := handler.DataStore.Stack().GetNextIdentifier()
 	stack := &portainer.Stack{
-		ID:           portainer.StackID(stackID),
-		Name:         payload.Name,
-		Type:         portainer.DockerSwarmStack,
-		SwarmID:      payload.SwarmID,
-		EndpointID:   endpoint.ID,
-		EntryPoint:   payload.ComposeFilePathInRepository,
+		ID:              portainer.StackID(stackID),
+		Name:            payload.Name,
+		Type:            portainer.DockerSwarmStack,
+		SwarmID:         payload.SwarmID,
+		EndpointID:      endpoint.ID,
+		EntryPoint:      payload.ComposeFile,
+		AdditionalFiles: payload.AdditionalFiles,
+		AutoUpdate:      payload.AutoUpdate,
+		GitConfig: &gittypes.RepoConfig{
+			URL:            payload.RepositoryURL,
+			ReferenceName:  payload.RepositoryReferenceName,
+			ConfigFilePath: payload.ComposeFile,
+		},
 		Env:          payload.Env,
 		Status:       portainer.StackStatusActive,
 		CreationDate: time.Now().Unix(),
 	}
 
+	if payload.RepositoryAuthentication {
+		stack.GitConfig.Authentication = &gittypes.GitAuthentication{
+			Username: payload.RepositoryUsername,
+			Password: payload.RepositoryPassword,
+		}
+	}
+
 	projectPath := handler.FileService.GetStackProjectPath(strconv.Itoa(int(stack.ID)))
 	stack.ProjectPath = projectPath
 
 	doCleanUp := true
 	defer handler.cleanUp(stack, &doCleanUp)
 
-	err = handler.cloneAndSaveConfig(stack, projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, payload.ComposeFilePathInRepository, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword)
+	err = handler.clone(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword)
 	if err != nil {
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err}
 	}
 
+	commitId, err := handler.latestCommitID(payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to fetch git repository id", Err: err}
+	}
+	stack.GitConfig.ConfigHash = commitId
+
 	config, configErr := handler.createSwarmDeployConfig(r, stack, endpoint, false)
 	if configErr != nil {
 		return configErr
@@ -192,14 +230,23 @@ func (handler *Handler) createSwarmStackFromGitRepository(w http.ResponseWriter,
 
 	err = handler.deploySwarmStack(config)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err}
+	}
+
+	if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" {
+		jobID, e := startAutoupdate(stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService)
+		if e != nil {
+			return e
+		}
+
+		stack.AutoUpdate.JobID = jobID
 	}
 
 	stack.CreatedBy = config.user.Username
 
 	err = handler.DataStore.Stack().CreateStack(stack)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack inside the database", Err: err}
 	}
 
 	doCleanUp = false
@@ -350,16 +397,17 @@ func (handler *Handler) deploySwarmStack(config *swarmStackDeploymentConfig) err
 	settings := &config.endpoint.SecuritySettings
 
 	if !settings.AllowBindMountsForRegularUsers && !isAdminOrEndpointAdmin {
-		composeFilePath := path.Join(config.stack.ProjectPath, config.stack.EntryPoint)
+		for _, file := range append([]string{config.stack.EntryPoint}, config.stack.AdditionalFiles...) {
+			path := path.Join(config.stack.ProjectPath, file)
+			stackContent, err := handler.FileService.GetFileContent(path)
+			if err != nil {
+				return err
+			}
 
-		stackContent, err := handler.FileService.GetFileContent(composeFilePath)
-		if err != nil {
-			return err
-		}
-
-		err = handler.isValidStackFile(stackContent, settings)
-		if err != nil {
-			return err
+			err = handler.isValidStackFile(stackContent, settings)
+			if err != nil {
+				return err
+			}
 		}
 	}
 

api/http/handler/stacks/handler.go

@@ -2,23 +2,30 @@ package stacks
 
 import (
 	"context"
-	"errors"
+	"fmt"
 	"net/http"
 	"strings"
 	"sync"
 
 	"github.com/docker/docker/api/types"
 	"github.com/gorilla/mux"
+	"github.com/pkg/errors"
 	httperror "github.com/portainer/libhttp/error"
 	portainer "github.com/portainer/portainer/api"
+	bolterrors "github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/docker"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/authorization"
+	"github.com/portainer/portainer/api/scheduler"
+	"github.com/portainer/portainer/api/stacks"
 )
 
+const defaultGitReferenceName = "refs/heads/master"
+
 var (
-	errStackAlreadyExists = errors.New("A stack already exists with this name")
-	errStackNotExternal   = errors.New("Not an external stack")
+	errStackAlreadyExists     = errors.New("A stack already exists with this name")
+	errWebhookIDAlreadyExists = errors.New("A webhook ID already exists")
+	errStackNotExternal       = errors.New("Not an external stack")
 )
 
 // Handler is the HTTP handler used to handle stack operations.
@@ -34,6 +41,8 @@ type Handler struct {
 	SwarmStackManager   portainer.SwarmStackManager
 	ComposeStackManager portainer.ComposeStackManager
 	KubernetesDeployer  portainer.KubernetesDeployer
+	Scheduler           *scheduler.Scheduler
+	StackDeployer       stacks.StackDeployer
 }
 
 // NewHandler creates a handler to manage stack operations.
@@ -57,7 +66,9 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 	h.Handle("/stacks/{id}",
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackUpdate))).Methods(http.MethodPut)
 	h.Handle("/stacks/{id}/git",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackUpdateGit))).Methods(http.MethodPut)
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackUpdateGit))).Methods(http.MethodPost)
+	h.Handle("/stacks/{id}/git/redeploy",
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackGitRedeploy))).Methods(http.MethodPut)
 	h.Handle("/stacks/{id}/file",
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackFile))).Methods(http.MethodGet)
 	h.Handle("/stacks/{id}/migrate",
@@ -66,6 +77,9 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackStart))).Methods(http.MethodPost)
 	h.Handle("/stacks/{id}/stop",
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackStop))).Methods(http.MethodPost)
+	h.Handle("/stacks/webhooks/{webhookID}",
+		httperror.LoggerHandler(h.webhookInvoke)).Methods(http.MethodPost)
+
 	return h
 }
 
@@ -159,3 +173,34 @@ func (handler *Handler) checkUniqueName(endpoint *portainer.Endpoint, name strin
 
 	return true, nil
 }
+
+func (handler *Handler) checkUniqueWebhookID(webhookID string) (bool, error) {
+	_, err := handler.DataStore.Stack().StackByWebhookID(webhookID)
+	if err == bolterrors.ErrObjectNotFound {
+		return true, nil
+	}
+	return false, err
+}
+
+func (handler *Handler) clone(projectPath, repositoryURL, refName string, auth bool, username, password string) error {
+	if !auth {
+		username = ""
+		password = ""
+	}
+
+	err := handler.GitService.CloneRepository(projectPath, repositoryURL, refName, username, password)
+	if err != nil {
+		return fmt.Errorf("unable to clone git repository: %w", err)
+	}
+
+	return nil
+}
+
+func (handler *Handler) latestCommitID(repositoryURL, refName string, auth bool, username, password string) (string, error) {
+	if !auth {
+		username = ""
+		password = ""
+	}
+
+	return handler.GitService.LatestCommitID(repositoryURL, refName, username, password)
+}

api/http/handler/stacks/helper.go

@@ -0,0 +1,24 @@
+package stacks
+
+import (
+	"time"
+
+	"github.com/asaskevich/govalidator"
+	"github.com/pkg/errors"
+	portainer "github.com/portainer/portainer/api"
+)
+
+func validateStackAutoUpdate(autoUpdate *portainer.StackAutoUpdate) error {
+	if autoUpdate == nil {
+		return nil
+	}
+	if autoUpdate.Webhook != "" && !govalidator.IsUUID(autoUpdate.Webhook) {
+		return errors.New("invalid Webhook format")
+	}
+	if autoUpdate.Interval != "" {
+		if _, err := time.ParseDuration(autoUpdate.Interval); err != nil {
+			return errors.New("invalid Interval format")
+		}
+	}
+	return nil
+}

api/http/handler/stacks/helper_test.go

@@ -0,0 +1,42 @@
+package stacks
+
+import (
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_ValidateStackAutoUpdate(t *testing.T) {
+	tests := []struct {
+		name    string
+		value   *portainer.StackAutoUpdate
+		wantErr bool
+	}{
+		{
+			name:    "webhook is not a valid UUID",
+			value:   &portainer.StackAutoUpdate{Webhook: "fake-webhook"},
+			wantErr: true,
+		},
+		{
+			name:    "incorrect interval value",
+			value:   &portainer.StackAutoUpdate{Interval: "1dd2hh3mm"},
+			wantErr: true,
+		},
+		{
+			name: "valid auto update",
+			value: &portainer.StackAutoUpdate{
+				Webhook:  "8dce8c2f-9ca1-482b-ad20-271e86536ada",
+				Interval: "5h30m40s10ms",
+			},
+			wantErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := validateStackAutoUpdate(tt.value)
+			assert.Equalf(t, tt.wantErr, err != nil, "received %+v", err)
+		})
+	}
+}

api/http/handler/stacks/stack_associate.go

@@ -2,6 +2,9 @@ package stacks
 
 import (
 	"fmt"
+	"net/http"
+	"time"
+
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
@@ -9,8 +12,6 @@ import (
 	bolterrors "github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/stackutils"
-	"net/http"
-	"time"
 )
 
 // PUT request on /api/stacks/:id/associate?endpointId=<endpointId>&swarmId=<swarmId>&orphanedRunning=<orphanedRunning>
@@ -87,5 +88,10 @@ func (handler *Handler) stackAssociate(w http.ResponseWriter, r *http.Request) *
 
 	stack.ResourceControl = resourceControl
 
+	if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" {
+		// sanitize password in the http response to minimise possible security leaks
+		stack.GitConfig.Authentication.Password = ""
+	}
+
 	return response.JSON(w, stack)
 }

api/http/handler/stacks/stack_create.go

@@ -1,19 +1,17 @@
 package stacks
 
 import (
-	"errors"
-	"fmt"
 	"log"
 	"net/http"
 
 	"github.com/docker/cli/cli/compose/loader"
 	"github.com/docker/cli/cli/compose/types"
+	"github.com/pkg/errors"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
 	bolterrors "github.com/portainer/portainer/api/bolt/errors"
-	gittypes "github.com/portainer/portainer/api/git/types"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/authorization"
 	"github.com/portainer/portainer/api/internal/endpointutils"
@@ -129,7 +127,7 @@ func (handler *Handler) createComposeStack(w http.ResponseWriter, r *http.Reques
 		return handler.createComposeStackFromFileUpload(w, r, endpoint, userID)
 	}
 
-	return &httperror.HandlerError{http.StatusBadRequest, "Invalid value for query parameter: method. Value must be one of: string, repository or file", errors.New(request.ErrInvalidQueryParameter)}
+	return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid value for query parameter: method. Value must be one of: string, repository or file", Err: errors.New(request.ErrInvalidQueryParameter)}
 }
 
 func (handler *Handler) createSwarmStack(w http.ResponseWriter, r *http.Request, method string, endpoint *portainer.Endpoint, userID portainer.UserID) *httperror.HandlerError {
@@ -142,7 +140,7 @@ func (handler *Handler) createSwarmStack(w http.ResponseWriter, r *http.Request,
 		return handler.createSwarmStackFromFileUpload(w, r, endpoint, userID)
 	}
 
-	return &httperror.HandlerError{http.StatusBadRequest, "Invalid value for query parameter: method. Value must be one of: string, repository or file", errors.New(request.ErrInvalidQueryParameter)}
+	return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid value for query parameter: method. Value must be one of: string, repository or file", Err: errors.New(request.ErrInvalidQueryParameter)}
 }
 
 func (handler *Handler) createKubernetesStack(w http.ResponseWriter, r *http.Request, method string, endpoint *portainer.Endpoint) *httperror.HandlerError {
@@ -232,24 +230,11 @@ func (handler *Handler) decorateStackResponse(w http.ResponseWriter, stack *port
 	}
 
 	stack.ResourceControl = resourceControl
+
+	if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" {
+		// sanitize password in the http response to minimise possible security leaks
+		stack.GitConfig.Authentication.Password = ""
+	}
+
 	return response.JSON(w, stack)
 }
-
-func (handler *Handler) cloneAndSaveConfig(stack *portainer.Stack, projectPath, repositoryURL, refName, configFilePath string, auth bool, username, password string) error {
-	if !auth {
-		username = ""
-		password = ""
-	}
-
-	err := handler.GitService.CloneRepository(projectPath, repositoryURL, refName, username, password)
-	if err != nil {
-		return fmt.Errorf("unable to clone git repository: %w", err)
-	}
-
-	stack.GitConfig = &gittypes.RepoConfig{
-		URL:            repositoryURL,
-		ReferenceName:  refName,
-		ConfigFilePath: configFilePath,
-	}
-	return nil
-}

api/http/handler/stacks/stack_create_test.go

@@ -1,29 +0,0 @@
-package stacks
-
-import (
-	"testing"
-
-	portainer "github.com/portainer/portainer/api"
-	gittypes "github.com/portainer/portainer/api/git/types"
-	"github.com/portainer/portainer/api/http/security"
-	"github.com/portainer/portainer/api/internal/testhelpers"
-	"github.com/stretchr/testify/assert"
-)
-
-func Test_stackHandler_cloneAndSaveConfig_shouldCallGitCloneAndSaveConfigOnStack(t *testing.T) {
-	handler := NewHandler(&security.RequestBouncer{})
-	handler.GitService = testhelpers.NewGitService()
-
-	url := "url"
-	refName := "ref"
-	configPath := "path"
-	stack := &portainer.Stack{}
-	err := handler.cloneAndSaveConfig(stack, "", url, refName, configPath, false, "", "")
-	assert.NoError(t, err, "clone and save should not fail")
-
-	assert.Equal(t, gittypes.RepoConfig{
-		URL:            url,
-		ReferenceName:  refName,
-		ConfigFilePath: configPath,
-	}, *stack.GitConfig)
-}

api/http/handler/stacks/stack_delete.go

@@ -96,6 +96,11 @@ func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *htt
 		}
 	}
 
+	// stop scheduler updates of the stack before removal
+	if stack.AutoUpdate != nil {
+		stopAutoupdate(stack.ID, stack.AutoUpdate.JobID, *handler.Scheduler)
+	}
+
 	err = handler.deleteStack(stack, endpoint)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err}

api/http/handler/stacks/stack_inspect.go

@@ -78,5 +78,10 @@ func (handler *Handler) stackInspect(w http.ResponseWriter, r *http.Request) *ht
 		}
 	}
 
+	if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" {
+		// sanitize password in the http response to minimise possible security leaks
+		stack.GitConfig.Authentication.Password = ""
+	}
+
 	return response.JSON(w, stack)
 }

api/http/handler/stacks/stack_list.go

@@ -1,9 +1,10 @@
 package stacks
 
 import (
-	httperrors "github.com/portainer/portainer/api/http/errors"
 	"net/http"
 
+	httperrors "github.com/portainer/portainer/api/http/errors"
+
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
@@ -80,6 +81,13 @@ func (handler *Handler) stackList(w http.ResponseWriter, r *http.Request) *httpe
 		stacks = authorization.FilterAuthorizedStacks(stacks, user, userTeamIDs)
 	}
 
+	for _, stack := range stacks {
+		if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" {
+			// sanitize password in the http response to minimise possible security leaks
+			stack.GitConfig.Authentication.Password = ""
+		}
+	}
+
 	return response.JSON(w, stacks)
 }
 

api/http/handler/stacks/stack_migrate.go

@@ -150,6 +150,11 @@ func (handler *Handler) stackMigrate(w http.ResponseWriter, r *http.Request) *ht
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack changes inside the database", err}
 	}
 
+	if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" {
+		// sanitize password in the http response to minimise possible security leaks
+		stack.GitConfig.Authentication.Password = ""
+	}
+
 	return response.JSON(w, stack)
 }
 

api/http/handler/stacks/stack_start.go

@@ -85,6 +85,17 @@ func (handler *Handler) stackStart(w http.ResponseWriter, r *http.Request) *http
 		return &httperror.HandlerError{http.StatusBadRequest, "Stack is already active", errors.New("Stack is already active")}
 	}
 
+	if stack.AutoUpdate != nil && stack.AutoUpdate.Interval != "" {
+		stopAutoupdate(stack.ID, stack.AutoUpdate.JobID, *handler.Scheduler)
+
+		jobID, e := startAutoupdate(stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService)
+		if e != nil {
+			return e
+		}
+
+		stack.AutoUpdate.JobID = jobID
+	}
+
 	err = handler.startStack(stack, endpoint)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to start stack", err}
@@ -96,6 +107,11 @@ func (handler *Handler) stackStart(w http.ResponseWriter, r *http.Request) *http
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update stack status", err}
 	}
 
+	if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" {
+		// sanitize password in the http response to minimise possible security leaks
+		stack.GitConfig.Authentication.Password = ""
+	}
+
 	return response.JSON(w, stack)
 }
 

api/http/handler/stacks/stack_stop.go

@@ -74,6 +74,12 @@ func (handler *Handler) stackStop(w http.ResponseWriter, r *http.Request) *httpe
 		return &httperror.HandlerError{http.StatusBadRequest, "Stack is already inactive", errors.New("Stack is already inactive")}
 	}
 
+	// stop scheduler updates of the stack before stopping
+	if stack.AutoUpdate != nil && stack.AutoUpdate.JobID != "" {
+		stopAutoupdate(stack.ID, stack.AutoUpdate.JobID, *handler.Scheduler)
+		stack.AutoUpdate.JobID = ""
+	}
+
 	err = handler.stopStack(stack, endpoint)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to stop stack", err}
@@ -85,6 +91,11 @@ func (handler *Handler) stackStop(w http.ResponseWriter, r *http.Request) *httpe
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update stack status", err}
 	}
 
+	if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" {
+		// sanitize password in the http response to minimise possible security leaks
+		stack.GitConfig.Authentication.Password = ""
+	}
+
 	return response.JSON(w, stack)
 }
 

api/http/handler/stacks/stack_update.go

@@ -128,6 +128,11 @@ func (handler *Handler) stackUpdate(w http.ResponseWriter, r *http.Request) *htt
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack changes inside the database", err}
 	}
 
+	if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" {
+		// sanitize password in the http response to minimise possible security leaks
+		stack.GitConfig.Authentication.Password = ""
+	}
+
 	return response.JSON(w, stack)
 }
 

api/http/handler/stacks/stack_update_git.go

@@ -2,10 +2,7 @@ package stacks
 
 import (
 	"errors"
-	"fmt"
-	"log"
 	"net/http"
-	"time"
 
 	"github.com/asaskevich/govalidator"
 	httperror "github.com/portainer/libhttp/error"
@@ -13,22 +10,28 @@ import (
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
 	bolterrors "github.com/portainer/portainer/api/bolt/errors"
-	"github.com/portainer/portainer/api/filesystem"
+	gittypes "github.com/portainer/portainer/api/git/types"
 	httperrors "github.com/portainer/portainer/api/http/errors"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/stackutils"
 )
 
-type updateStackGitPayload struct {
+type stackGitUpdatePayload struct {
+	AutoUpdate               *portainer.StackAutoUpdate
+	Env                      []portainer.Pair
 	RepositoryReferenceName  string
 	RepositoryAuthentication bool
 	RepositoryUsername       string
 	RepositoryPassword       string
 }
 
-func (payload *updateStackGitPayload) Validate(r *http.Request) error {
-	if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) {
-		return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled")
+func (payload *stackGitUpdatePayload) Validate(r *http.Request) error {
+	if govalidator.IsNull(payload.RepositoryReferenceName) {
+		payload.RepositoryReferenceName = defaultGitReferenceName
+	}
+
+	if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil {
+		return err
 	}
 	return nil
 }
@@ -53,18 +56,23 @@ func (payload *updateStackGitPayload) Validate(r *http.Request) error {
 func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	stackID, err := request.RetrieveNumericRouteVariableValue(r, "id")
 	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err}
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid stack identifier route variable", Err: err}
+	}
+
+	var payload stackGitUpdatePayload
+	err = request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err}
 	}
 
 	stack, err := handler.DataStore.Stack().Stack(portainer.StackID(stackID))
 	if err == bolterrors.ErrObjectNotFound {
-		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a stack with the specified identifier inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find a stack with the specified identifier inside the database", Err: err}
 	} else if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err}
-	}
-
-	if stack.GitConfig == nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Stack is not created from git", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find a stack with the specified identifier inside the database", Err: err}
+	} else if stack.GitConfig == nil {
+		msg := "No Git config in the found stack"
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: msg, Err: errors.New(msg)}
 	}
 
 	// TODO: this is a work-around for stacks created with Portainer version >= 1.17.1
@@ -72,7 +80,7 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) *
 	// can use the optional EndpointID query parameter to associate a valid endpoint identifier to the stack.
 	endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", true)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: endpointId", err}
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid query parameter: endpointId", Err: err}
 	}
 	if endpointID != int(stack.EndpointID) {
 		stack.EndpointID = portainer.EndpointID(endpointID)
@@ -80,117 +88,75 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) *
 
 	endpoint, err := handler.DataStore.Endpoint().Endpoint(stack.EndpointID)
 	if err == bolterrors.ErrObjectNotFound {
-		return &httperror.HandlerError{http.StatusNotFound, "Unable to find the endpoint associated to the stack inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err}
 	} else if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find the endpoint associated to the stack inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err}
 	}
 
 	err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
+		return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to access endpoint", Err: err}
 	}
 
 	resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve a resource control associated to the stack", Err: err}
 	}
 
 	securityContext, err := security.RetrieveRestrictedRequestContext(r)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err}
 	}
 
 	access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to verify user authorizations to validate stack access", Err: err}
 	}
 	if !access {
-		return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", httperrors.ErrResourceAccessDenied}
+		return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Access denied to resource", Err: httperrors.ErrResourceAccessDenied}
 	}
 
-	var payload updateStackGitPayload
-	err = request.DecodeAndValidateJSONPayload(r, &payload)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+	//stop the autoupdate job if there is any
+	if stack.AutoUpdate != nil {
+		stopAutoupdate(stack.ID, stack.AutoUpdate.JobID, *handler.Scheduler)
 	}
 
+	//update retrieved stack data based on the payload
 	stack.GitConfig.ReferenceName = payload.RepositoryReferenceName
+	stack.AutoUpdate = payload.AutoUpdate
+	stack.Env = payload.Env
 
-	backupProjectPath := fmt.Sprintf("%s-old", stack.ProjectPath)
-	err = filesystem.MoveDirectory(stack.ProjectPath, backupProjectPath)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to move git repository directory", err}
+	stack.GitConfig.Authentication = nil
+	if payload.RepositoryAuthentication {
+		password := payload.RepositoryPassword
+		if password == "" && stack.GitConfig != nil && stack.GitConfig.Authentication != nil {
+			password = stack.GitConfig.Authentication.Password
+		}
+		stack.GitConfig.Authentication = &gittypes.GitAuthentication{
+			Username: payload.RepositoryUsername,
+			Password: password,
+		}
 	}
 
-	repositoryUsername := payload.RepositoryUsername
-	repositoryPassword := payload.RepositoryPassword
-	if !payload.RepositoryAuthentication {
-		repositoryUsername = ""
-		repositoryPassword = ""
-	}
-
-	err = handler.GitService.CloneRepository(stack.ProjectPath, stack.GitConfig.URL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword)
-	if err != nil {
-		restoreError := filesystem.MoveDirectory(backupProjectPath, stack.ProjectPath)
-		if restoreError != nil {
-			log.Printf("[WARN] [http,stacks,git] [error: %s] [message: failed restoring backup folder]", restoreError)
+	if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" {
+		jobID, e := startAutoupdate(stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService)
+		if e != nil {
+			return e
 		}
 
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to clone git repository", err}
-	}
-
-	defer func() {
-		err = handler.FileService.RemoveDirectory(backupProjectPath)
-		if err != nil {
-			log.Printf("[WARN] [http,stacks,git] [error: %s] [message: unable to remove git repository directory]", err)
-		}
-	}()
-
-	httpErr := handler.deployStack(r, stack, endpoint)
-	if httpErr != nil {
-		return httpErr
+		stack.AutoUpdate.JobID = jobID
 	}
 
+	//save the updated stack to DB
 	err = handler.DataStore.Stack().UpdateStack(stack.ID, stack)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack changes inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack changes inside the database", Err: err}
+	}
+
+	if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" {
+		// sanitize password in the http response to minimise possible security leaks
+		stack.GitConfig.Authentication.Password = ""
 	}
 
 	return response.JSON(w, stack)
 }
-
-func (handler *Handler) deployStack(r *http.Request, stack *portainer.Stack, endpoint *portainer.Endpoint) *httperror.HandlerError {
-	if stack.Type == portainer.DockerSwarmStack {
-		config, httpErr := handler.createSwarmDeployConfig(r, stack, endpoint, false)
-		if httpErr != nil {
-			return httpErr
-		}
-
-		err := handler.deploySwarmStack(config)
-		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err}
-		}
-
-		stack.UpdateDate = time.Now().Unix()
-		stack.UpdatedBy = config.user.Username
-		stack.Status = portainer.StackStatusActive
-
-		return nil
-	}
-
-	config, httpErr := handler.createComposeDeployConfig(r, stack, endpoint)
-	if httpErr != nil {
-		return httpErr
-	}
-
-	err := handler.deployComposeStack(config)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err}
-	}
-
-	stack.UpdateDate = time.Now().Unix()
-	stack.UpdatedBy = config.user.Username
-	stack.Status = portainer.StackStatusActive
-
-	return nil
-}

api/http/handler/stacks/stack_update_git_redeploy.go

@@ -0,0 +1,190 @@
+package stacks
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"time"
+
+	"github.com/asaskevich/govalidator"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	portainer "github.com/portainer/portainer/api"
+	bolterrors "github.com/portainer/portainer/api/bolt/errors"
+	"github.com/portainer/portainer/api/filesystem"
+	httperrors "github.com/portainer/portainer/api/http/errors"
+	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/internal/stackutils"
+)
+
+type stackGitRedployPayload struct {
+	RepositoryReferenceName  string
+	RepositoryAuthentication bool
+	RepositoryUsername       string
+	RepositoryPassword       string
+	Env                      []portainer.Pair
+}
+
+func (payload *stackGitRedployPayload) Validate(r *http.Request) error {
+	if govalidator.IsNull(payload.RepositoryReferenceName) {
+		payload.RepositoryReferenceName = defaultGitReferenceName
+	}
+
+	return nil
+}
+
+// PUT request on /api/stacks/:id/git?endpointId=<endpointId>
+func (handler *Handler) stackGitRedeploy(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	stackID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid stack identifier route variable", Err: err}
+	}
+
+	stack, err := handler.DataStore.Stack().Stack(portainer.StackID(stackID))
+	if err == bolterrors.ErrObjectNotFound {
+		return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find a stack with the specified identifier inside the database", Err: err}
+	} else if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find a stack with the specified identifier inside the database", Err: err}
+	}
+
+	if stack.GitConfig == nil {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Stack is not created from git", Err: err}
+	}
+
+	// TODO: this is a work-around for stacks created with Portainer version >= 1.17.1
+	// The EndpointID property is not available for these stacks, this API endpoint
+	// can use the optional EndpointID query parameter to associate a valid endpoint identifier to the stack.
+	endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", true)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid query parameter: endpointId", Err: err}
+	}
+	if endpointID != int(stack.EndpointID) {
+		stack.EndpointID = portainer.EndpointID(endpointID)
+	}
+
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(stack.EndpointID)
+	if err == bolterrors.ErrObjectNotFound {
+		return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err}
+	} else if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err}
+	}
+
+	err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to access endpoint", Err: err}
+	}
+
+	resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve a resource control associated to the stack", Err: err}
+	}
+
+	securityContext, err := security.RetrieveRestrictedRequestContext(r)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err}
+	}
+
+	access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to verify user authorizations to validate stack access", Err: err}
+	}
+	if !access {
+		return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Access denied to resource", Err: httperrors.ErrResourceAccessDenied}
+	}
+
+	var payload stackGitRedployPayload
+	err = request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err}
+	}
+
+	stack.GitConfig.ReferenceName = payload.RepositoryReferenceName
+	stack.Env = payload.Env
+
+	backupProjectPath := fmt.Sprintf("%s-old", stack.ProjectPath)
+	err = filesystem.MoveDirectory(stack.ProjectPath, backupProjectPath)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to move git repository directory", Err: err}
+	}
+
+	repositoryUsername := ""
+	repositoryPassword := ""
+	if payload.RepositoryAuthentication {
+		repositoryPassword = payload.RepositoryPassword
+		if repositoryPassword == "" && stack.GitConfig != nil && stack.GitConfig.Authentication != nil {
+			repositoryPassword = stack.GitConfig.Authentication.Password
+		}
+		repositoryUsername = payload.RepositoryUsername
+	}
+
+	err = handler.GitService.CloneRepository(stack.ProjectPath, stack.GitConfig.URL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword)
+	if err != nil {
+		restoreError := filesystem.MoveDirectory(backupProjectPath, stack.ProjectPath)
+		if restoreError != nil {
+			log.Printf("[WARN] [http,stacks,git] [error: %s] [message: failed restoring backup folder]", restoreError)
+		}
+
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err}
+	}
+
+	defer func() {
+		err = handler.FileService.RemoveDirectory(backupProjectPath)
+		if err != nil {
+			log.Printf("[WARN] [http,stacks,git] [error: %s] [message: unable to remove git repository directory]", err)
+		}
+	}()
+
+	httpErr := handler.deployStack(r, stack, endpoint)
+	if httpErr != nil {
+		return httpErr
+	}
+
+	err = handler.DataStore.Stack().UpdateStack(stack.ID, stack)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack changes inside the database", Err: err}
+	}
+
+	if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" {
+		// sanitize password in the http response to minimise possible security leaks
+		stack.GitConfig.Authentication.Password = ""
+	}
+
+	return response.JSON(w, stack)
+}
+
+func (handler *Handler) deployStack(r *http.Request, stack *portainer.Stack, endpoint *portainer.Endpoint) *httperror.HandlerError {
+	if stack.Type == portainer.DockerSwarmStack {
+		config, httpErr := handler.createSwarmDeployConfig(r, stack, endpoint, false)
+		if httpErr != nil {
+			return httpErr
+		}
+
+		err := handler.deploySwarmStack(config)
+		if err != nil {
+			return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err}
+		}
+
+		stack.UpdateDate = time.Now().Unix()
+		stack.UpdatedBy = config.user.Username
+		stack.Status = portainer.StackStatusActive
+
+		return nil
+	}
+
+	config, httpErr := handler.createComposeDeployConfig(r, stack, endpoint)
+	if httpErr != nil {
+		return httpErr
+	}
+
+	err := handler.deployComposeStack(config)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err}
+	}
+
+	stack.UpdateDate = time.Now().Unix()
+	stack.UpdatedBy = config.user.Username
+	stack.Status = portainer.StackStatusActive
+
+	return nil
+}

api/http/handler/stacks/webhook_invoke.go

@@ -0,0 +1,54 @@
+package stacks
+
+import (
+	"log"
+	"net/http"
+
+	"github.com/gofrs/uuid"
+
+	"github.com/portainer/libhttp/response"
+
+	bolterrors "github.com/portainer/portainer/api/bolt/errors"
+	"github.com/portainer/portainer/api/stacks"
+
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+)
+
+func (handler *Handler) webhookInvoke(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	webhookID, err := retrieveUUIDRouteVariableValue(r, "webhookID")
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid webhook identifier route variable", Err: err}
+	}
+
+	stack, err := handler.DataStore.Stack().StackByWebhookID(webhookID.String())
+	if err != nil {
+		statusCode := http.StatusInternalServerError
+		if err == bolterrors.ErrObjectNotFound {
+			statusCode = http.StatusNotFound
+		}
+		return &httperror.HandlerError{StatusCode: statusCode, Message: "Unable to find the stack by webhook ID", Err: err}
+	}
+
+	if err = stacks.RedeployWhenChanged(stack.ID, handler.StackDeployer, handler.DataStore, handler.GitService); err != nil {
+		log.Printf("[ERROR] %s\n", err)
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Failed to update the stack", Err: err}
+	}
+
+	return response.Empty(w)
+}
+
+func retrieveUUIDRouteVariableValue(r *http.Request, name string) (uuid.UUID, error) {
+	webhookID, err := request.RetrieveRouteVariableValue(r, name)
+	if err != nil {
+		return uuid.Nil, err
+	}
+
+	uid, err := uuid.FromString(webhookID)
+
+	if err != nil {
+		return uuid.Nil, err
+	}
+
+	return uid, nil
+}

api/http/handler/stacks/webhook_invoke_test.go

@@ -0,0 +1,59 @@
+package stacks
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gofrs/uuid"
+	"github.com/stretchr/testify/assert"
+
+	portainer "github.com/portainer/portainer/api"
+
+	"github.com/portainer/portainer/api/bolt/bolttest"
+)
+
+func TestHandler_webhookInvoke(t *testing.T) {
+	store, teardown := bolttest.MustNewTestStore(true)
+	defer teardown()
+
+	webhookID := newGuidString(t)
+	store.StackService.CreateStack(&portainer.Stack{
+		AutoUpdate: &portainer.StackAutoUpdate{
+			Webhook: webhookID,
+		},
+	})
+
+	h := NewHandler(nil)
+	h.DataStore = store
+
+	t.Run("invalid uuid results in http.StatusBadRequest", func(t *testing.T) {
+		w := httptest.NewRecorder()
+		req := newRequest("notuuid")
+		h.Router.ServeHTTP(w, req)
+		assert.Equal(t, http.StatusBadRequest, w.Code)
+	})
+	t.Run("registered webhook ID in http.StatusNoContent", func(t *testing.T) {
+		w := httptest.NewRecorder()
+		req := newRequest(webhookID)
+		h.Router.ServeHTTP(w, req)
+		assert.Equal(t, http.StatusNoContent, w.Code)
+	})
+	t.Run("unregistered webhook ID in http.StatusNotFound", func(t *testing.T) {
+		w := httptest.NewRecorder()
+		req := newRequest(newGuidString(t))
+		h.Router.ServeHTTP(w, req)
+		assert.Equal(t, http.StatusNotFound, w.Code)
+	})
+}
+
+func newGuidString(t *testing.T) string {
+	uuid, err := uuid.NewV4()
+	assert.NoError(t, err)
+
+	return uuid.String()
+}
+
+func newRequest(webhookID string) *http.Request {
+	return httptest.NewRequest(http.MethodPost, "/stacks/webhooks/"+webhookID, nil)
+}

api/http/server.go

@@ -50,6 +50,8 @@ import (
 	"github.com/portainer/portainer/api/internal/authorization"
 	"github.com/portainer/portainer/api/internal/ssl"
 	"github.com/portainer/portainer/api/kubernetes/cli"
+	"github.com/portainer/portainer/api/scheduler"
+	stackdeployer "github.com/portainer/portainer/api/stacks"
 )
 
 // Server implements the portainer.Server interface
@@ -79,8 +81,10 @@ type Server struct {
 	DockerClientFactory         *docker.ClientFactory
 	KubernetesClientFactory     *cli.ClientFactory
 	KubernetesDeployer          portainer.KubernetesDeployer
+	Scheduler                   *scheduler.Scheduler
 	ShutdownCtx                 context.Context
 	ShutdownTrigger             context.CancelFunc
+	StackDeployer               stackdeployer.StackDeployer
 }
 
 // Start starts the HTTP server
@@ -185,10 +189,12 @@ func (server *Server) Start() error {
 	stackHandler.DataStore = server.DataStore
 	stackHandler.DockerClientFactory = server.DockerClientFactory
 	stackHandler.FileService = server.FileService
-	stackHandler.SwarmStackManager = server.SwarmStackManager
-	stackHandler.ComposeStackManager = server.ComposeStackManager
 	stackHandler.KubernetesDeployer = server.KubernetesDeployer
 	stackHandler.GitService = server.GitService
+	stackHandler.Scheduler = server.Scheduler
+	stackHandler.SwarmStackManager = server.SwarmStackManager
+	stackHandler.ComposeStackManager = server.ComposeStackManager
+	stackHandler.StackDeployer = server.StackDeployer
 
 	var tagHandler = tags.NewHandler(requestBouncer)
 	tagHandler.DataStore = server.DataStore

api/internal/stackutils/stackutils.go

@@ -2,6 +2,7 @@ package stackutils
 
 import (
 	"fmt"
+	"path"
 
 	portainer "github.com/portainer/portainer/api"
 )
@@ -10,3 +11,12 @@ import (
 func ResourceControlID(endpointID portainer.EndpointID, name string) string {
 	return fmt.Sprintf("%d_%s", endpointID, name)
 }
+
+// GetStackFilePaths returns a list of file paths based on stack project path
+func GetStackFilePaths(stack *portainer.Stack) []string {
+	var filePaths []string
+	for _, file := range append([]string{stack.EntryPoint}, stack.AdditionalFiles...) {
+		filePaths = append(filePaths, path.Join(stack.ProjectPath, file))
+	}
+	return filePaths
+}

api/internal/stackutils/stackutils_test.go

@@ -0,0 +1,26 @@
+package stackutils
+
+import (
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_GetStackFilePaths(t *testing.T) {
+	stack := &portainer.Stack{
+		ProjectPath: "/tmp/stack/1",
+		EntryPoint:  "file-one.yml",
+	}
+
+	t.Run("stack doesn't have additional files", func(t *testing.T) {
+		expected := []string{"/tmp/stack/1/file-one.yml"}
+		assert.ElementsMatch(t, expected, GetStackFilePaths(stack))
+	})
+
+	t.Run("stack has additional files", func(t *testing.T) {
+		stack.AdditionalFiles = []string{"file-two.yml", "file-three.yml"}
+		expected := []string{"/tmp/stack/1/file-one.yml", "/tmp/stack/1/file-two.yml", "/tmp/stack/1/file-three.yml"}
+		assert.ElementsMatch(t, expected, GetStackFilePaths(stack))
+	})
+}

api/libcompose/compose_stack.go

@@ -16,6 +16,7 @@ import (
 	"github.com/portainer/libcompose/project"
 	"github.com/portainer/libcompose/project/options"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/stackutils"
 )
 
 const (
@@ -86,12 +87,12 @@ func (manager *ComposeStackManager) Up(stack *portainer.Stack, endpoint *portain
 	for _, envvar := range stack.Env {
 		env[envvar.Name] = envvar.Value
 	}
+	filePaths := stackutils.GetStackFilePaths(stack)
 
-	composeFilePath := path.Join(stack.ProjectPath, stack.EntryPoint)
 	proj, err := docker.NewProject(&ctx.Context{
 		ConfigDir: manager.dataPath,
 		Context: project.Context{
-			ComposeFiles: []string{composeFilePath},
+			ComposeFiles: filePaths,
 			EnvironmentLookup: &lookup.ComposableEnvLookup{
 				Lookups: []config.EnvironmentLookup{
 					&lookup.EnvfileLookup{
@@ -120,10 +121,13 @@ func (manager *ComposeStackManager) Down(stack *portainer.Stack, endpoint *porta
 		return err
 	}
 
-	composeFilePath := path.Join(stack.ProjectPath, stack.EntryPoint)
+	var composeFiles []string
+	for _, file := range append([]string{stack.EntryPoint}, stack.AdditionalFiles...) {
+		composeFiles = append(composeFiles, path.Join(stack.ProjectPath, file))
+	}
 	proj, err := docker.NewProject(&ctx.Context{
 		Context: project.Context{
-			ComposeFiles: []string{composeFilePath},
+			ComposeFiles: composeFiles,
 			ProjectName:  stack.Name,
 		},
 		ClientFactory: clientFactory,
@@ -134,3 +138,11 @@ func (manager *ComposeStackManager) Down(stack *portainer.Stack, endpoint *porta
 
 	return proj.Down(context.Background(), options.Down{RemoveVolume: false, RemoveOrphans: true})
 }
+
+func stackFilePaths(stack *portainer.Stack) []string {
+	var filePaths []string
+	for _, file := range append([]string{stack.EntryPoint}, stack.AdditionalFiles...) {
+		filePaths = append(filePaths, path.Join(stack.ProjectPath, file))
+	}
+	return filePaths
+}

api/portainer.go

@@ -744,10 +744,24 @@ type (
 		UpdateDate int64 `example:"1587399600"`
 		// The username which last updated this stack
 		UpdatedBy string `example:"bob"`
+		// Only applies when deploying stack with multiple files
+		AdditionalFiles []string `json:"AdditionalFiles"`
+		// The auto update settings of a git stack
+		AutoUpdate *StackAutoUpdate `json:"AutoUpdate"`
 		// The git config of this stack
 		GitConfig *gittypes.RepoConfig
 	}
 
+	//StackAutoUpdate represents the git auto sync config for stack deployment
+	StackAutoUpdate struct {
+		// Auto update interval
+		Interval string `example:"1m30s"`
+		// A UUID generated from client
+		Webhook string `example:"05de31a2-79fa-4644-9c12-faa67e5c49f0"`
+		// Autoupdate job id
+		JobID string `example:"15"`
+	}
+
 	// StackID represents a stack identifier (it must be composed of Name + "_" + SwarmID to create a unique identifier)
 	StackID int
 
@@ -1187,6 +1201,7 @@ type (
 	// GitService represents a service for managing Git
 	GitService interface {
 		CloneRepository(destination string, repositoryURL, referenceName, username, password string) error
+		LatestCommitID(repositoryURL, referenceName, username, password string) (string, error)
 	}
 
 	// JWTService represents a service for managing JWT tokens
@@ -1302,6 +1317,8 @@ type (
 		UpdateStack(ID StackID, stack *Stack) error
 		DeleteStack(ID StackID) error
 		GetNextIdentifier() int
+		StackByWebhookID(ID string) (*Stack, error)
+		RefreshableStacks() ([]Stack, error)
 	}
 
 	// SnapshotService represents a service for managing endpoint snapshots

api/scheduler/scheduler.go

@@ -0,0 +1,73 @@
+package scheduler
+
+import (
+	"context"
+	"log"
+	"strconv"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/robfig/cron/v3"
+)
+
+type Scheduler struct {
+	crontab     *cron.Cron
+	shutdownCtx context.Context
+}
+
+func NewScheduler(ctx context.Context) *Scheduler {
+	crontab := cron.New(cron.WithChain(cron.Recover(cron.DefaultLogger)))
+	crontab.Start()
+
+	s := &Scheduler{
+		crontab: crontab,
+	}
+
+	if ctx != nil {
+		go func() {
+			<-ctx.Done()
+			s.Shutdown()
+		}()
+	}
+
+	return s
+}
+
+// Shutdown stops the scheduler and waits for it to stop if it is running; otherwise does nothing.
+func (s *Scheduler) Shutdown() error {
+	if s.crontab == nil {
+		return nil
+	}
+
+	log.Println("[DEBUG] Stopping scheduler")
+	ctx := s.crontab.Stop()
+	<-ctx.Done()
+
+	for _, j := range s.crontab.Entries() {
+		s.crontab.Remove(j.ID)
+	}
+
+	err := ctx.Err()
+	if err == context.Canceled {
+		return nil
+	}
+	return err
+}
+
+// StopJob stops the job from being run in the future
+func (s *Scheduler) StopJob(jobID string) error {
+	id, err := strconv.Atoi(jobID)
+	if err != nil {
+		return errors.Wrapf(err, "failed convert jobID %q to int", jobID)
+	}
+	s.crontab.Remove(cron.EntryID(id))
+
+	return nil
+}
+
+// StartJobEvery schedules a new periodic job with a given duration.
+// Returns job id that could be used to stop the given job
+func (s *Scheduler) StartJobEvery(duration time.Duration, job func()) string {
+	entryId := s.crontab.Schedule(cron.Every(duration), cron.FuncJob(job))
+	return strconv.Itoa(int(entryId))
+}

api/scheduler/scheduler_test.go

@@ -0,0 +1,57 @@
+package scheduler
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_CanStartAndTerminate(t *testing.T) {
+	s := NewScheduler(context.Background())
+	s.StartJobEvery(1*time.Minute, func() { fmt.Println("boop") })
+
+	err := s.Shutdown()
+	assert.NoError(t, err, "Shutdown should return no errors")
+	assert.Empty(t, s.crontab.Entries(), "all jobs should have been removed")
+}
+
+func Test_CanTerminateByCancellingContext(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	s := NewScheduler(ctx)
+	s.StartJobEvery(1*time.Minute, func() { fmt.Println("boop") })
+
+	cancel()
+
+	for i := 0; i < 100; i++ {
+		if len(s.crontab.Entries()) == 0 {
+			return
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	t.Fatal("all jobs are expected to be cleaned by now; it might be a timing issue, otherwise implementation defect")
+}
+
+func Test_StartAndStopJob(t *testing.T) {
+	s := NewScheduler(context.Background())
+	defer s.Shutdown()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+
+	var jobOne string
+	var workDone bool
+	jobOne = s.StartJobEvery(time.Second, func() {
+		assert.Equal(t, 1, len(s.crontab.Entries()), "scheduler should have one active job")
+		workDone = true
+
+		s.StopJob(jobOne)
+		cancel()
+	})
+
+	<-ctx.Done()
+	assert.True(t, workDone, "value should been set in the job")
+	assert.Equal(t, 0, len(s.crontab.Entries()), "scheduler should have no active jobs")
+
+}

api/stacks/deploy.go

@@ -0,0 +1,138 @@
+package stacks
+
+import (
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/http/security"
+)
+
+func RedeployWhenChanged(stackID portainer.StackID, deployer StackDeployer, datastore portainer.DataStore, gitService portainer.GitService) error {
+	stack, err := datastore.Stack().Stack(stackID)
+	if err != nil {
+		return errors.WithMessagef(err, "failed to get the stack %v", stackID)
+	}
+
+	if stack.GitConfig == nil {
+		return nil // do nothing if it isn't a git-based stack
+	}
+
+	username, password := "", ""
+	if stack.GitConfig.Authentication != nil {
+		username, password = stack.GitConfig.Authentication.Username, stack.GitConfig.Authentication.Password
+	}
+
+	newHash, err := gitService.LatestCommitID(stack.GitConfig.URL, stack.GitConfig.ReferenceName, username, password)
+	if err != nil {
+		return errors.WithMessagef(err, "failed to fetch latest commit id of the stack %v", stack.ID)
+	}
+
+	if strings.EqualFold(newHash, string(stack.GitConfig.ConfigHash)) {
+		return nil
+	}
+
+	cloneParams := &cloneRepositoryParameters{
+		url:   stack.GitConfig.URL,
+		ref:   stack.GitConfig.ReferenceName,
+		toDir: stack.ProjectPath,
+	}
+	if stack.GitConfig.Authentication != nil {
+		cloneParams.auth = &gitAuth{
+			username: username,
+			password: password,
+		}
+	}
+
+	if err := cloneGitRepository(gitService, cloneParams); err != nil {
+		return errors.WithMessagef(err, "failed to do a fresh clone of the stack %v", stack.ID)
+	}
+
+	endpoint, err := datastore.Endpoint().Endpoint(stack.EndpointID)
+	if err != nil {
+		return errors.WithMessagef(err, "failed to find the endpoint %v associated to the stack %v", stack.EndpointID, stack.ID)
+	}
+
+	author := stack.UpdatedBy
+	if author == "" {
+		author = stack.CreatedBy
+	}
+
+	registries, err := getUserRegistries(datastore, author, endpoint.ID)
+	if err != nil {
+		return err
+	}
+
+	switch stack.Type {
+	case portainer.DockerComposeStack:
+		err := deployer.DeployComposeStack(stack, endpoint, registries)
+		if err != nil {
+			return errors.WithMessagef(err, "failed to deploy a docker compose stack %v", stackID)
+		}
+	case portainer.DockerSwarmStack:
+		err := deployer.DeploySwarmStack(stack, endpoint, registries, true)
+		if err != nil {
+			return errors.WithMessagef(err, "failed to deploy a docker compose stack %v", stackID)
+		}
+	default:
+		return errors.Errorf("cannot update stack, type %v is unsupported", stack.Type)
+	}
+
+	stack.UpdateDate = time.Now().Unix()
+	stack.GitConfig.ConfigHash = newHash
+	if err := datastore.Stack().UpdateStack(stack.ID, stack); err != nil {
+		return errors.WithMessagef(err, "failed to update the stack %v", stack.ID)
+	}
+
+	return nil
+}
+
+func getUserRegistries(datastore portainer.DataStore, authorUsername string, endpointID portainer.EndpointID) ([]portainer.Registry, error) {
+	registries, err := datastore.Registry().Registries()
+	if err != nil {
+		return nil, errors.WithMessage(err, "unable to retrieve registries from the database")
+	}
+
+	user, err := datastore.User().UserByUsername(authorUsername)
+	if err != nil {
+		return nil, errors.WithMessagef(err, "failed to fetch a stack's author [%s]", authorUsername)
+	}
+
+	if user.Role == portainer.AdministratorRole {
+		return registries, nil
+	}
+
+	userMemberships, err := datastore.TeamMembership().TeamMembershipsByUserID(user.ID)
+	if err != nil {
+		return nil, errors.WithMessagef(err, "failed to fetch memberships of the stack author [%s]", authorUsername)
+	}
+
+	filteredRegistries := make([]portainer.Registry, 0, len(registries))
+	for _, registry := range registries {
+		if security.AuthorizedRegistryAccess(&registry, user, userMemberships, endpointID) {
+			filteredRegistries = append(filteredRegistries, registry)
+		}
+	}
+
+	return filteredRegistries, nil
+}
+
+type cloneRepositoryParameters struct {
+	url   string
+	ref   string
+	toDir string
+	auth  *gitAuth
+}
+
+type gitAuth struct {
+	username string
+	password string
+}
+
+func cloneGitRepository(gitService portainer.GitService, cloneParams *cloneRepositoryParameters) error {
+	if cloneParams.auth != nil {
+		return gitService.CloneRepository(cloneParams.toDir, cloneParams.url, cloneParams.ref, cloneParams.auth.username, cloneParams.auth.password)
+	}
+	return gitService.CloneRepository(cloneParams.toDir, cloneParams.url, cloneParams.ref, "", "")
+}

api/stacks/deploy_test.go

@@ -0,0 +1,221 @@
+package stacks
+
+import (
+	"errors"
+	"io/ioutil"
+	"strings"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	bolt "github.com/portainer/portainer/api/bolt/bolttest"
+	gittypes "github.com/portainer/portainer/api/git/types"
+	"github.com/stretchr/testify/assert"
+)
+
+type gitService struct {
+	cloneErr error
+	id       string
+}
+
+func (g *gitService) CloneRepository(destination, repositoryURL, referenceName, username, password string) error {
+	return g.cloneErr
+}
+
+func (g *gitService) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) {
+	return g.id, nil
+}
+
+type noopDeployer struct{}
+
+func (s *noopDeployer) DeploySwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, prune bool) error {
+	return nil
+}
+
+func (s *noopDeployer) DeployComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry) error {
+	return nil
+}
+
+func Test_redeployWhenChanged_FailsWhenCannotFindStack(t *testing.T) {
+	store, teardown := bolt.MustNewTestStore(true)
+	defer teardown()
+
+	err := RedeployWhenChanged(1, nil, store, nil)
+	assert.Error(t, err)
+	assert.Truef(t, strings.HasPrefix(err.Error(), "failed to get the stack"), "it isn't an error we expected: %v", err.Error())
+}
+
+func Test_redeployWhenChanged_DoesNothingWhenNotAGitBasedStack(t *testing.T) {
+	store, teardown := bolt.MustNewTestStore(true)
+	defer teardown()
+
+	err := store.Stack().CreateStack(&portainer.Stack{ID: 1})
+	assert.NoError(t, err, "failed to create a test stack")
+
+	err = RedeployWhenChanged(1, nil, store, &gitService{nil, ""})
+	assert.NoError(t, err)
+}
+
+func Test_redeployWhenChanged_DoesNothingWhenNoGitChanges(t *testing.T) {
+	store, teardown := bolt.MustNewTestStore(true)
+	defer teardown()
+
+	tmpDir, _ := ioutil.TempDir("", "stack")
+
+	err := store.Stack().CreateStack(&portainer.Stack{
+		ID:          1,
+		ProjectPath: tmpDir,
+		GitConfig: &gittypes.RepoConfig{
+			URL:           "url",
+			ReferenceName: "ref",
+			ConfigHash:    "oldHash",
+		}})
+	assert.NoError(t, err, "failed to create a test stack")
+
+	err = RedeployWhenChanged(1, nil, store, &gitService{nil, "oldHash"})
+	assert.NoError(t, err)
+}
+
+func Test_redeployWhenChanged_FailsWhenCannotClone(t *testing.T) {
+	cloneErr := errors.New("failed to clone")
+	store, teardown := bolt.MustNewTestStore(true)
+	defer teardown()
+
+	err := store.Stack().CreateStack(&portainer.Stack{
+		ID: 1,
+		GitConfig: &gittypes.RepoConfig{
+			URL:           "url",
+			ReferenceName: "ref",
+			ConfigHash:    "oldHash",
+		}})
+	assert.NoError(t, err, "failed to create a test stack")
+
+	err = RedeployWhenChanged(1, nil, store, &gitService{cloneErr, "newHash"})
+	assert.Error(t, err)
+	assert.ErrorIs(t, err, cloneErr, "should failed to clone but didn't, check test setup")
+}
+
+func Test_redeployWhenChanged(t *testing.T) {
+	store, teardown := bolt.MustNewTestStore(true)
+	defer teardown()
+
+	tmpDir, _ := ioutil.TempDir("", "stack")
+
+	err := store.Endpoint().CreateEndpoint(&portainer.Endpoint{ID: 1})
+	assert.NoError(t, err, "error creating endpoint")
+
+	username := "user"
+	err = store.User().CreateUser(&portainer.User{Username: username, Role: portainer.AdministratorRole})
+	assert.NoError(t, err, "error creating a user")
+
+	stack := portainer.Stack{
+		ID:          1,
+		EndpointID:  1,
+		ProjectPath: tmpDir,
+		UpdatedBy:   username,
+		GitConfig: &gittypes.RepoConfig{
+			URL:           "url",
+			ReferenceName: "ref",
+			ConfigHash:    "oldHash",
+		}}
+	err = store.Stack().CreateStack(&stack)
+	assert.NoError(t, err, "failed to create a test stack")
+
+	t.Run("can deploy docker compose stack", func(t *testing.T) {
+		stack.Type = portainer.DockerComposeStack
+		store.Stack().UpdateStack(stack.ID, &stack)
+
+		err = RedeployWhenChanged(1, &noopDeployer{}, store, &gitService{nil, "newHash"})
+		assert.NoError(t, err)
+	})
+
+	t.Run("can deploy docker swarm stack", func(t *testing.T) {
+		stack.Type = portainer.DockerSwarmStack
+		store.Stack().UpdateStack(stack.ID, &stack)
+
+		err = RedeployWhenChanged(1, &noopDeployer{}, store, &gitService{nil, "newHash"})
+		assert.NoError(t, err)
+	})
+
+	t.Run("can NOT deploy kube stack", func(t *testing.T) {
+		stack.Type = portainer.KubernetesStack
+		store.Stack().UpdateStack(stack.ID, &stack)
+
+		err = RedeployWhenChanged(1, &noopDeployer{}, store, &gitService{nil, "newHash"})
+		assert.EqualError(t, err, "cannot update stack, type 3 is unsupported")
+	})
+}
+
+func Test_getUserRegistries(t *testing.T) {
+	store, teardown := bolt.MustNewTestStore(true)
+	defer teardown()
+
+	endpointID := 123
+
+	admin := portainer.User{ID: 1, Username: "admin", Role: portainer.AdministratorRole}
+	err := store.User().CreateUser(&admin)
+	assert.NoError(t, err, "error creating an admin")
+
+	user := portainer.User{ID: 2, Username: "user", Role: portainer.StandardUserRole}
+	err = store.User().CreateUser(&user)
+	assert.NoError(t, err, "error creating a user")
+
+	team := portainer.Team{ID: 1, Name: "team"}
+
+	store.TeamMembership().CreateTeamMembership(&portainer.TeamMembership{
+		ID:     1,
+		UserID: user.ID,
+		TeamID: team.ID,
+		Role:   portainer.TeamMember,
+	})
+
+	registryReachableByUser := portainer.Registry{
+		ID: 1,
+		RegistryAccesses: portainer.RegistryAccesses{
+			portainer.EndpointID(endpointID): {
+				UserAccessPolicies: map[portainer.UserID]portainer.AccessPolicy{
+					user.ID: {RoleID: portainer.RoleID(portainer.StandardUserRole)},
+				},
+			},
+		},
+	}
+	err = store.Registry().CreateRegistry(&registryReachableByUser)
+	assert.NoError(t, err, "couldn't create a registry")
+
+	registryReachableByTeam := portainer.Registry{
+		ID: 2,
+		RegistryAccesses: portainer.RegistryAccesses{
+			portainer.EndpointID(endpointID): {
+				TeamAccessPolicies: map[portainer.TeamID]portainer.AccessPolicy{
+					team.ID: {RoleID: portainer.RoleID(portainer.StandardUserRole)},
+				},
+			},
+		},
+	}
+	err = store.Registry().CreateRegistry(&registryReachableByTeam)
+	assert.NoError(t, err, "couldn't create a registry")
+
+	registryRestricted := portainer.Registry{
+		ID: 3,
+		RegistryAccesses: portainer.RegistryAccesses{
+			portainer.EndpointID(endpointID): {
+				UserAccessPolicies: map[portainer.UserID]portainer.AccessPolicy{
+					user.ID + 100: {RoleID: portainer.RoleID(portainer.StandardUserRole)},
+				},
+			},
+		},
+	}
+	err = store.Registry().CreateRegistry(&registryRestricted)
+	assert.NoError(t, err, "couldn't create a registry")
+
+	t.Run("admin should has access to all registries", func(t *testing.T) {
+		registries, err := getUserRegistries(store, admin.Username, portainer.EndpointID(endpointID))
+		assert.NoError(t, err)
+		assert.ElementsMatch(t, []portainer.Registry{registryReachableByUser, registryReachableByTeam, registryRestricted}, registries)
+	})
+
+	t.Run("regular user has access to registries allowed to him and/or his team", func(t *testing.T) {
+		registries, err := getUserRegistries(store, user.Username, portainer.EndpointID(endpointID))
+		assert.NoError(t, err)
+		assert.ElementsMatch(t, []portainer.Registry{registryReachableByUser, registryReachableByTeam}, registries)
+	})
+}

api/stacks/deployer.go

@@ -0,0 +1,46 @@
+package stacks
+
+import (
+	"sync"
+
+	portainer "github.com/portainer/portainer/api"
+)
+
+type StackDeployer interface {
+	DeploySwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, prune bool) error
+	DeployComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry) error
+}
+
+type stackDeployer struct {
+	lock                *sync.Mutex
+	swarmStackManager   portainer.SwarmStackManager
+	composeStackManager portainer.ComposeStackManager
+}
+
+func NewStackDeployer(swarmStackManager portainer.SwarmStackManager, composeStackManager portainer.ComposeStackManager) *stackDeployer {
+	return &stackDeployer{
+		lock:                &sync.Mutex{},
+		swarmStackManager:   swarmStackManager,
+		composeStackManager: composeStackManager,
+	}
+}
+
+func (d *stackDeployer) DeploySwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, prune bool) error {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+
+	d.swarmStackManager.Login(registries, endpoint)
+	defer d.swarmStackManager.Logout(endpoint)
+
+	return d.swarmStackManager.Deploy(stack, prune, endpoint)
+}
+
+func (d *stackDeployer) DeployComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry) error {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+
+	d.swarmStackManager.Login(registries, endpoint)
+	defer d.swarmStackManager.Logout(endpoint)
+
+	return d.composeStackManager.Up(stack, endpoint)
+}

api/stacks/scheduled.go

@@ -0,0 +1,34 @@
+package stacks
+
+import (
+	"log"
+	"time"
+
+	"github.com/pkg/errors"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/scheduler"
+)
+
+func StartStackSchedules(scheduler *scheduler.Scheduler, stackdeployer StackDeployer, datastore portainer.DataStore, gitService portainer.GitService) error {
+	stacks, err := datastore.Stack().RefreshableStacks()
+	if err != nil {
+		return errors.Wrap(err, "failed to fetch refreshable stacks")
+	}
+	for _, stack := range stacks {
+		d, err := time.ParseDuration(stack.AutoUpdate.Interval)
+		if err != nil {
+			return errors.Wrap(err, "Unable to parse auto update interval")
+		}
+		jobID := scheduler.StartJobEvery(d, func() {
+			if err := RedeployWhenChanged(stack.ID, stackdeployer, datastore, gitService); err != nil {
+				log.Printf("[ERROR] %s\n", err)
+			}
+		})
+
+		stack.AutoUpdate.JobID = jobID
+		if err := datastore.Stack().UpdateStack(stack.ID, &stack); err != nil {
+			return errors.Wrap(err, "failed to update stack job id")
+		}
+	}
+	return nil
+}

app/docker/components/datatables/containers-datatable/containersDatatable.html

@@ -92,7 +92,7 @@
           ng-model="$ctrl.state.textFilter"
           ng-change="$ctrl.onTextFilterChange()"
           placeholder="Search..."
-          auto-focus
+          focus-if="!$ctrl.notAutoFocus"
           ng-model-options="{ debounce: 300 }"
         />
       </div>

app/docker/components/datatables/containers-datatable/containersDatatable.js

@@ -12,5 +12,6 @@ angular.module('portainer.docker').component('containersDatatable', {
     showAddAction: '<',
     offlineMode: '<',
     refreshCallback: '<',
+    notAutoFocus: '<',
   },
 });

app/docker/components/datatables/services-datatable/servicesDatatable.html

@@ -56,7 +56,7 @@
           ng-model="$ctrl.state.textFilter"
           ng-change="$ctrl.onTextFilterChange()"
           placeholder="Search..."
-          auto-focus
+          focus-if="!$ctrl.notAutoFocus"
           ng-model-options="{ debounce: 300 }"
         />
       </div>

app/docker/components/datatables/services-datatable/servicesDatatable.js

@@ -15,5 +15,6 @@ angular.module('portainer.docker').component('servicesDatatable', {
     showStackColumn: '<',
     showTaskLogsButton: '<',
     refreshCallback: '<',
+    notAutoFocus: '<',
   },
 });

app/portainer/components/focusIf.js

@@ -0,0 +1,22 @@
+import angular from 'angular';
+// ng-focus-if pkg from: https://github.com/hiebj/ng-focus-if
+angular.module('portainer.app').directive('focusIf', function ($timeout) {
+  return {
+    restrict: 'A',
+    link: function ($scope, $element, $attrs) {
+      var dom = $element[0];
+      if ($attrs.focusIf) {
+        $scope.$watch($attrs.focusIf, focus);
+      } else {
+        focus(true);
+      }
+      function focus(condition) {
+        if (condition) {
+          $timeout(function () {
+            dom.focus();
+          }, $scope.$eval($attrs.focusDelay) || 0);
+        }
+      }
+    },
+  };
+});

app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-file-item/git-form-additional-file-item.controller.js

@@ -0,0 +1,22 @@
+class GitFormAdditionalFileItemController {
+  /* @ngInject */
+  constructor() {}
+
+  onChangePath(value) {
+    const fieldIsInvalid = typeof value === 'undefined';
+    if (fieldIsInvalid) {
+      return;
+    }
+    this.onChange(this.index, { value });
+  }
+
+  removeValue() {
+    this.onChange(this.index);
+  }
+
+  $onInit() {
+    this.formName = `variableForm${this.index}`;
+  }
+}
+
+export default GitFormAdditionalFileItemController;

app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-file-item/git-form-additional-file-item.html

@@ -0,0 +1,20 @@
+<ng-form class="env-item form-horizontal" name="$ctrl.{{ $ctrl.formName }}">
+  <div class="form-group col-sm-12">
+    <div class="form-inline" style="margin-top: 10px;">
+      <div class="input-group col-sm-5 input-group-sm">
+        <span class="input-group-addon">path</span>
+        <input type="text" name="name" class="form-control" ng-model="$ctrl.variable" ng-change="$ctrl.onChangePath($ctrl.variable)" required />
+      </div>
+      <button class="btn btn-sm btn-danger" type="button" ng-click="$ctrl.removeValue()">
+        <i class="fa fa-trash" aria-hidden="true"></i>
+      </button>
+    </div>
+    <div ng-show="$ctrl[$ctrl.formName].name.$invalid">
+      <div class="col-sm-12 small text-warning">
+        <div ng-messages="$ctrl[$ctrl.formName].name.$error">
+          <p ng-message="required"> <i class="fa fa-exclamation-triangle" aria-hidden="true"></i> Path is required. </p>
+        </div>
+      </div>
+    </div>
+  </div>
+</ng-form>

app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-file-item/index.js

@@ -0,0 +1,14 @@
+import controller from './git-form-additional-file-item.controller.js';
+
+export const gitFormAdditionalFileItem = {
+  templateUrl: './git-form-additional-file-item.html',
+  controller,
+
+  bindings: {
+    variable: '<',
+    index: '<',
+
+    onChange: '<',
+    onRemove: '<',
+  },
+};

app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-files-panel.controller.js

@@ -0,0 +1,26 @@
+class GitFormAutoUpdateFieldsetController {
+  /* @ngInject */
+  constructor() {
+    this.add = this.add.bind(this);
+    this.onChangeVariable = this.onChangeVariable.bind(this);
+  }
+
+  add() {
+    this.model.AdditionalFiles.push('');
+  }
+
+  onChangeVariable(index, variable) {
+    if (!variable) {
+      this.model.AdditionalFiles.splice(index, 1);
+    } else {
+      this.model.AdditionalFiles[index] = variable.value;
+    }
+
+    this.onChange({
+      ...this.model,
+      AdditionalFiles: this.model.AdditionalFiles,
+    });
+  }
+}
+
+export default GitFormAutoUpdateFieldsetController;

app/portainer/components/forms/git-form/git-form-additional-files-panel/git-form-additional-files-panel.html

@@ -0,0 +1,14 @@
+<div class="form-group">
+  <div class="col-sm-12" style="margin-top: 5px;">
+    <label class="control-label text-left">Additional paths</label>
+    <span class="label label-default interactive" style="margin-left: 10px;" ng-click="$ctrl.add()"> <i class="fa fa-plus-circle" aria-hidden="true"></i> add file </span>
+  </div>
+  <div class="col-sm-12 form-inline" style="margin-top: 10px;">
+    <git-form-additional-file-item
+      ng-repeat="variable in $ctrl.model.AdditionalFiles track by $index"
+      variable="variable"
+      index="$index"
+      on-change="($ctrl.onChangeVariable)"
+    ></git-form-additional-file-item>
+  </div>
+</div>

app/portainer/components/forms/git-form/git-form-additional-files-panel/index.js

@@ -0,0 +1,10 @@
+import controller from './git-form-additional-files-panel.controller.js';
+
+export const gitFormAdditionalFilesPanel = {
+  templateUrl: './git-form-additional-files-panel.html',
+  controller,
+  bindings: {
+    model: '<',
+    onChange: '<',
+  },
+};

app/portainer/components/forms/git-form/git-form-auth-fieldset/git-form-auth-fieldset.html

@@ -3,36 +3,38 @@
     <por-switch-field ng-model="$ctrl.model.RepositoryAuthentication" label="Authentication" on-change="($ctrl.onChangeAuth)"></por-switch-field>
   </div>
 </div>
+<div class="small text-warning" style="margin: 5px 0 15px 0;" ng-if="$ctrl.model.RepositoryAuthentication && $ctrl.showAuthExplanation">
+  <i class="fa fa-exclamation-circle" aria-hidden="true"></i>
+  <span class="text-muted">Enabling authentication will store the credentials and it is advisable to use a git service account</span>
+</div>
 <div ng-if="$ctrl.model.RepositoryAuthentication">
   <div class="form-group">
-    <span class="col-sm-12 text-muted small">
-      If your git account has 2FA enabled, you may receive an <code>authentication required</code> error when deploying your stack. In this case, you will need to provide a
-      personal-access token instead of your password.
-    </span>
-  </div>
-  <div class="form-group">
-    <label for="repository_username" class="col-sm-1 control-label text-left">Username</label>
-    <div class="col-sm-11 col-md-5">
+    <label for="repository_username" class="col-sm-2 control-label text-left">Username</label>
+    <div class="col-sm-3">
       <input
         type="text"
         class="form-control"
         ng-model="$ctrl.model.RepositoryUsername"
         name="repository_username"
-        placeholder="myGitUser"
+        placeholder="git username"
         ng-change="$ctrl.onChangeUsername($ctrl.model.RepositoryUsername)"
       />
     </div>
-    <label for="repository_password" class="col-sm-1 control-label text-left">
-      Password
+  </div>
+  <div class="form-group">
+    <label for="repository_password" class="col-sm-2 control-label text-left">
+      Personal Access Token
+      <portainer-tooltip position="bottom" message="Provide a personal access token or password"></portainer-tooltip>
     </label>
-    <div class="col-sm-11 col-md-5">
+    <div class="col-sm-3">
       <input
         type="password"
         class="form-control"
         ng-model="$ctrl.model.RepositoryPassword"
         name="repository_password"
-        placeholder="myPassword"
+        placeholder="personal access token"
         ng-change="$ctrl.onChangePassword($ctrl.model.RepositoryPassword)"
+        ng-required="!$ctrl.isEdit"
       />
     </div>
   </div>

app/portainer/components/forms/git-form/git-form-auth-fieldset/index.js

@@ -6,5 +6,6 @@ export const gitFormAuthFieldset = {
   bindings: {
     model: '<',
     onChange: '<',
+    isEdit: '<',
   },
 };

app/portainer/components/forms/git-form/git-form-auto-update-fieldset/git-form-auto-update-fieldset.controller.js

@@ -0,0 +1,26 @@
+class GitFormAutoUpdateFieldsetController {
+  /* @ngInject */
+  constructor(clipboard) {
+    this.onChangeAutoUpdate = this.onChangeField('RepositoryAutomaticUpdates');
+    this.onChangeMechanism = this.onChangeField('RepositoryMechanism');
+    this.onChangeInterval = this.onChangeField('RepositoryFetchInterval');
+    this.clipboard = clipboard;
+  }
+
+  copyWebhook() {
+    this.clipboard.copyText(this.model.RepositoryWebhookURL);
+    $('#copyNotification').show();
+    $('#copyNotification').fadeOut(2000);
+  }
+
+  onChangeField(field) {
+    return (value) => {
+      this.onChange({
+        ...this.model,
+        [field]: value,
+      });
+    };
+  }
+}
+
+export default GitFormAutoUpdateFieldsetController;

app/portainer/components/forms/git-form/git-form-auto-update-fieldset/git-form-auto-update-fieldset.html

@@ -0,0 +1,69 @@
+<ng-form name="autoUpdateForm">
+  <div class="form-group">
+    <div class="col-sm-12">
+      <por-switch-field name="autoUpdate" ng-model="$ctrl.model.RepositoryAutomaticUpdates" label="Automatic updates" on-change="($ctrl.onChangeAutoUpdate)"></por-switch-field>
+    </div>
+  </div>
+  <div class="small text-warning" style="margin: 5px 0 10px 0;" ng-if="$ctrl.model.RepositoryAutomaticUpdates">
+    <i class="fa fa-exclamation-circle" aria-hidden="true"></i>
+    <span class="text-muted">Any changes to this stack made locally in Portainer will be overriden by the definition in git and may cause service interruption.</span>
+  </div>
+  <div class="form-group" ng-if="$ctrl.model.RepositoryAutomaticUpdates">
+    <label for="repository_mechanism" class="col-sm-1 control-label text-left">
+      Mechanism
+    </label>
+    <div class="col-sm-11">
+      <div class="input-group col-sm-10 input-group-sm">
+        <div class="btn-group btn-group-sm">
+          <label class="btn btn-primary" ng-click="$ctrl.onChangeMechanism($ctrl.model.RepositoryMechanism)" ng-model="$ctrl.model.RepositoryMechanism" uib-btn-radio="'Interval'"
+            >Polling</label
+          >
+          <label class="btn btn-primary" ng-click="$ctrl.onChangeMechanism($ctrl.model.RepositoryMechanism)" ng-model="$ctrl.model.RepositoryMechanism" uib-btn-radio="'Webhook'"
+            >Webhook</label
+          >
+        </div>
+      </div>
+    </div>
+  </div>
+
+  <div class="form-group" ng-if="$ctrl.model.RepositoryAutomaticUpdates && $ctrl.model.RepositoryMechanism === 'Webhook'">
+    <label for="repository_mechanism" class="col-sm-1 control-label text-left">
+      Webhook
+    </label>
+    <div class="col-sm-11">
+      <span class="text-muted"> {{ $ctrl.model.RepositoryWebhookURL | truncatelr }} </span>
+      <button type="button" class="btn btn-sm btn-primary btn-sm space-left" ng-if="$ctrl.model.RepositoryWebhookURL" ng-click="$ctrl.copyWebhook()">
+        <span><i class="fa fa-copy space-right" aria-hidden="true"></i>Copy link</span>
+      </button>
+      <span>
+        <i id="copyNotification" class="fa fa-check green-icon" aria-hidden="true" style="margin-left: 7px; display: none;"></i>
+      </span>
+    </div>
+  </div>
+  <div class="form-group" ng-if="$ctrl.model.RepositoryAutomaticUpdates && $ctrl.model.RepositoryMechanism === 'Interval'">
+    <label for="repository_fetch_interval" class="col-sm-1 control-label text-left">
+      Fetch interval
+    </label>
+    <div class="col-sm-11">
+      <input
+        type="text"
+        class="form-control"
+        ng-change="$ctrl.onChangeInterval($ctrl.model.RepositoryFetchInterval)"
+        ng-model="$ctrl.model.RepositoryFetchInterval"
+        name="repository_fetch_interval"
+        placeholder="5m"
+        required
+        interval-format
+      />
+    </div>
+  </div>
+  <div class="form-group col-md-12" ng-show="autoUpdateForm.repository_fetch_interval.$touched && autoUpdateForm.repository_fetch_interval.$invalid">
+    <div class="small text-warning">
+      <div ng-messages="autoUpdateForm.repository_fetch_interval.$error">
+        <p ng-message="required"> <i class="fa fa-exclamation-triangle" aria-hidden="true"></i> This field is required.</p>
+        <p ng-message="invalidIntervalFormat"> <i class="fa fa-exclamation-triangle" aria-hidden="true"></i> Please enter a valid time interval.</p>
+        <p ng-message="minimumInterval"> <i class="fa fa-exclamation-triangle" aria-hidden="true"></i> Minimum interval is 1m</p>
+      </div>
+    </div>
+  </div>
+</ng-form>

app/portainer/components/forms/git-form/git-form-auto-update-fieldset/index.js

@@ -0,0 +1,10 @@
+import controller from './git-form-auto-update-fieldset.controller.js';
+
+export const gitFormAutoUpdateFieldset = {
+  templateUrl: './git-form-auto-update-fieldset.html',
+  controller,
+  bindings: {
+    model: '<',
+    onChange: '<',
+  },
+};

app/portainer/components/forms/git-form/git-form-ref-field/git-form-ref-field.html

@@ -5,8 +5,8 @@
   </span>
 </div>
 <div class="form-group">
-  <label for="stack_repository_reference_name" class="col-sm-2 control-label text-left">Repository reference</label>
-  <div class="col-sm-10">
+  <label for="stack_repository_reference_name" class="col-sm-1 control-label text-left">Repository reference</label>
+  <div class="col-sm-11">
     <input type="text" class="form-control" ng-model="$ctrl.value" id="stack_repository_reference_name" placeholder="refs/heads/master" ng-change="$ctrl.onChange($ctrl.value)" />
   </div>
 </div>

app/portainer/components/forms/git-form/git-form.html

@@ -5,5 +5,7 @@
   <git-form-url-field value="$ctrl.model.RepositoryURL" on-change="($ctrl.onChangeURL)"></git-form-url-field>
   <git-form-ref-field value="$ctrl.model.RepositoryReferenceName" on-change="($ctrl.onChangeRefName)"></git-form-ref-field>
   <git-form-compose-path-field value="$ctrl.model.ComposeFilePathInRepository" on-change="($ctrl.onChangeComposePath)"></git-form-compose-path-field>
-  <git-form-auth-fieldset model="$ctrl.model" on-change="($ctrl.onChange)"></git-form-auth-fieldset>
+  <git-form-additional-files-panel ng-if="$ctrl.additionalFile" model="$ctrl.model" on-change="($ctrl.onChange)"></git-form-additional-files-panel>
+  <git-form-auth-fieldset model="$ctrl.model" on-change="($ctrl.onChange)" show-auth-explanation="$ctrl.showAuthExplanation"></git-form-auth-fieldset>
+  <git-form-auto-update-fieldset ng-if="$ctrl.autoUpdate" model="$ctrl.model" on-change="($ctrl.onChange)"></git-form-auto-update-fieldset>
 </div>

app/portainer/components/forms/git-form/git-form.js

@@ -6,5 +6,8 @@ export const gitForm = {
   bindings: {
     model: '<',
     onChange: '<',
+    additionalFile: '<',
+    autoUpdate: '<',
+    showAuthExplanation: '<',
   },
 };

app/portainer/components/forms/git-form/index.js

@@ -2,6 +2,9 @@ import angular from 'angular';
 
 import { gitForm } from './git-form';
 import { gitFormAuthFieldset } from './git-form-auth-fieldset';
+import { gitFormAdditionalFilesPanel } from './git-form-additional-files-panel';
+import { gitFormAdditionalFileItem } from './/git-form-additional-files-panel/git-form-additional-file-item';
+import { gitFormAutoUpdateFieldset } from './git-form-auto-update-fieldset';
 import { gitFormComposePathField } from './git-form-compose-path-field';
 import { gitFormRefField } from './git-form-ref-field';
 import { gitFormUrlField } from './git-form-url-field';
@@ -12,4 +15,7 @@ export default angular
   .component('gitFormRefField', gitFormRefField)
   .component('gitForm', gitForm)
   .component('gitFormUrlField', gitFormUrlField)
+  .component('gitFormAdditionalFilesPanel', gitFormAdditionalFilesPanel)
+  .component('gitFormAdditionalFileItem', gitFormAdditionalFileItem)
+  .component('gitFormAutoUpdateFieldset', gitFormAutoUpdateFieldset)
   .component('gitFormAuthFieldset', gitFormAuthFieldset).name;

app/portainer/components/forms/stack-redeploy-git-form/stack-redeploy-git-form.controller.js

@@ -1,14 +1,20 @@
+import uuidv4 from 'uuid/v4';
 class StackRedeployGitFormController {
   /* @ngInject */
-  constructor($async, $state, StackService, ModalService, Notifications) {
+  constructor($async, $state, StackService, ModalService, Notifications, WebhookHelper, FormHelper) {
     this.$async = $async;
     this.$state = $state;
     this.StackService = StackService;
     this.ModalService = ModalService;
     this.Notifications = Notifications;
+    this.WebhookHelper = WebhookHelper;
+    this.FormHelper = FormHelper;
 
     this.state = {
       inProgress: false,
+      redeployInProgress: false,
+      showConfig: false,
+      isEdit: false,
     };
 
     this.formValues = {
@@ -16,10 +22,19 @@ class StackRedeployGitFormController {
       RepositoryAuthentication: false,
       RepositoryUsername: '',
       RepositoryPassword: '',
+      Env: [],
+      // auto upadte
+      AutoUpdate: {
+        RepositoryAutomaticUpdates: false,
+        RepositoryMechanism: 'Interval',
+        RepositoryFetchInterval: '5m',
+        RepositoryWebhookURL: '',
+      },
     };
 
     this.onChange = this.onChange.bind(this);
     this.onChangeRef = this.onChangeRef.bind(this);
+    this.handleEnvVarChange = this.handleEnvVarChange.bind(this);
   }
 
   onChangeRef(value) {
@@ -50,13 +65,27 @@ class StackRedeployGitFormController {
           return;
         }
 
-        this.state.inProgress = true;
+        this.state.redeployInProgress = true;
 
-        await this.StackService.updateGit(this.stack.Id, this.stack.EndpointId, [], false, this.formValues);
+        await this.StackService.updateGit(this.stack.Id, this.stack.EndpointId, this.FormHelper.removeInvalidEnvVars(this.formValues.Env), false, this.formValues);
 
         await this.$state.reload();
       } catch (err) {
         this.Notifications.error('Failure', err, 'Failed redeploying stack');
+      } finally {
+        this.state.redeployInProgress = false;
+      }
+    });
+  }
+
+  async saveGitSettings() {
+    return this.$async(async () => {
+      try {
+        this.state.inProgress = true;
+        await this.StackService.updateGitStackSettings(this.stack.Id, this.stack.EndpointId, this.FormHelper.removeInvalidEnvVars(this.formValues.Env), this.formValues);
+        this.Notifications.success('Save stack settings successfully');
+      } catch (err) {
+        this.Notifications.error('Failure', err, 'Unable to save stack settings');
       } finally {
         this.state.inProgress = false;
       }
@@ -64,11 +93,38 @@ class StackRedeployGitFormController {
   }
 
   isSubmitButtonDisabled() {
-    return this.state.inProgress;
+    return this.state.inProgress || this.state.redeployInProgress;
+  }
+
+  handleEnvVarChange(value) {
+    this.formValues.Env = value;
   }
 
   $onInit() {
     this.formValues.RefName = this.model.ReferenceName;
+    this.formValues.Env = this.stack.Env;
+    // Init auto update
+    if (this.stack.AutoUpdate && (this.stack.AutoUpdate.Interval || this.stack.AutoUpdate.Webhook)) {
+      this.formValues.AutoUpdate.RepositoryAutomaticUpdates = true;
+
+      if (this.stack.AutoUpdate.Interval) {
+        this.formValues.AutoUpdate.RepositoryMechanism = `Interval`;
+        this.formValues.AutoUpdate.RepositoryFetchInterval = this.stack.AutoUpdate.Interval;
+      } else if (this.stack.AutoUpdate.Webhook) {
+        this.formValues.AutoUpdate.RepositoryMechanism = `Webhook`;
+        this.formValues.AutoUpdate.RepositoryWebhookURL = this.WebhookHelper.returnStackWebhookUrl(this.stack.AutoUpdate.Webhook);
+      }
+    }
+
+    if (!this.formValues.AutoUpdate.RepositoryWebhookURL) {
+      this.formValues.AutoUpdate.RepositoryWebhookURL = this.WebhookHelper.returnStackWebhookUrl(uuidv4());
+    }
+
+    if (this.stack.GitConfig && this.stack.GitConfig.Authentication) {
+      this.formValues.RepositoryUsername = this.stack.GitConfig.Authentication.Username;
+      this.formValues.RepositoryAuthentication = true;
+      this.state.isEdit = true;
+    }
   }
 }
 

app/portainer/components/forms/stack-redeploy-git-form/stack-redeploy-git-form.html

@@ -9,22 +9,62 @@
         .
       </p>
       <p>
-        Update <code>{{ $ctrl.model.ConfigFilePath }}</code> in git and pull from here to update the stack.
+        Update
+        <code
+          >{{ $ctrl.model.ConfigFilePath }}<span ng-if="$ctrl.stack.AdditionalFiles.length > 0">,{{ $ctrl.stack.AdditionalFiles.join(',') }}</span></code
+        >
+        in git and pull from here to update the stack.
       </p>
     </div>
   </div>
 
-  <git-form-ref-field value="$ctrl.formValues.RefName" on-change="($ctrl.onChangeRef)"></git-form-ref-field>
-  <git-form-auth-fieldset model="$ctrl.formValues" on-change="($ctrl.onChange)"></git-form-auth-fieldset>
+  <git-form-auto-update-fieldset model="$ctrl.formValues.AutoUpdate" on-change="($ctrl.onChange)"></git-form-auto-update-fieldset>
+  <div class="form-group">
+    <div class="col-sm-12">
+      <p>
+        <a class="small interactive" ng-click="$ctrl.state.showConfig = !$ctrl.state.showConfig">
+          <i ng-class="{ 'fa fa-minus space-right': $ctrl.state.showConfig, 'fa fa-plus space-right': !$ctrl.state.showConfig }" aria-hidden="true"></i>
+          {{ $ctrl.state.showConfig ? 'Hide' : 'Advanced' }} configuration
+        </a>
+      </p>
+    </div>
+  </div>
+  <git-form-ref-field ng-if="$ctrl.state.showConfig" value="$ctrl.formValues.RefName" on-change="($ctrl.onChangeRef)"></git-form-ref-field>
+  <git-form-auth-fieldset
+    ng-if="$ctrl.state.showConfig"
+    is-edit="$ctrl.state.isEdit"
+    model="$ctrl.formValues"
+    on-change="($ctrl.onChange)"
+    show-auth-explanation="true"
+  ></git-form-auth-fieldset>
+  <environment-variables-panel
+    ng-model="$ctrl.formValues.Env"
+    explanation="These values will be used as substitutions in the stack file"
+    on-change="($ctrl.handleEnvVarChange)"
+  ></environment-variables-panel>
 
   <button
     class="btn btn-sm btn-primary"
     ng-click="$ctrl.submit()"
-    ng-disabled="$ctrl.isSubmitButtonDisabled()"
+    ng-if="!$ctrl.formValues.AutoUpdate.RepositoryAutomaticUpdates"
+    ng-disabled="$ctrl.isSubmitButtonDisabled() || !$ctrl.redeployGitForm.$valid"
+    style="margin-top: 7px; margin-left: 0;"
+    button-spinner="$ctrl.state.redeployInProgress"
     style="margin-top: 7px; margin-left: 0;"
     button-spinner="$ctrl.state.inProgress"
   >
-    <span ng-hide="$ctrl.state.inProgress"> <i class="fa fa-sync space-right" aria-hidden="true"></i> Pull and redeploy </span>
+    <span ng-hide="$ctrl.state.redeployInProgress"> <i class="fa fa-sync space-right" aria-hidden="true"></i> Pull and redeploy </span>
+    <span ng-show="$ctrl.state.redeployInProgress">In progress...</span>
+  </button>
+
+  <button
+    class="btn btn-sm btn-primary"
+    ng-click="$ctrl.saveGitSettings()"
+    ng-disabled="$ctrl.isSubmitButtonDisabled() || !$ctrl.redeployGitForm.$valid"
+    style="margin-top: 7px; margin-left: 0;"
+    button-spinner="$ctrl.state.inProgress"
+  >
+    <span ng-hide="$ctrl.state.inProgress"> Save settings </span>
     <span ng-show="$ctrl.state.inProgress">In progress...</span>
   </button>
 </form>

app/portainer/components/intervalFormat.js

@@ -0,0 +1,26 @@
+import angular from 'angular';
+import parse from 'parse-duration';
+
+angular.module('portainer.app').directive('intervalFormat', function () {
+  return {
+    restrict: 'A',
+    require: 'ngModel',
+    link: function ($scope, $element, $attrs, ngModel) {
+      ngModel.$validators.invalidIntervalFormat = function (modelValue) {
+        try {
+          return modelValue && modelValue.toUpperCase().match(/^P?(?!$)(\d+Y)?(\d+M)?(\d+W)?(\d+D)?(T?(?=\d+[HMS])(\d+H)?(\d+M)?(\d+S)?)?$/gm) !== null;
+        } catch (error) {
+          return false;
+        }
+      };
+
+      ngModel.$validators.minimumInterval = function (modelValue) {
+        try {
+          return modelValue && parse(modelValue, 'minute') >= 1;
+        } catch (error) {
+          return false;
+        }
+      };
+    },
+  };
+});

app/portainer/helpers/webhookHelper.js

@@ -1,16 +1,21 @@
 angular.module('portainer.app').factory('WebhookHelper', [
   '$location',
   'API_ENDPOINT_WEBHOOKS',
-  function WebhookHelperFactory($location, API_ENDPOINT_WEBHOOKS) {
+  'API_ENDPOINT_STACKS',
+  function WebhookHelperFactory($location, API_ENDPOINT_WEBHOOKS, API_ENDPOINT_STACKS) {
     'use strict';
+
     var helper = {};
+    const protocol = $location.protocol().toLowerCase();
+    const port = $location.port();
+    const displayPort = (protocol === 'http' && port === 80) || (protocol === 'https' && port === 443) ? '' : ':' + port;
 
     helper.returnWebhookUrl = function (token) {
-      var displayPort =
-        ($location.protocol().toLowerCase() === 'http' && $location.port() === 80) || ($location.protocol().toLowerCase() === 'https' && $location.port() === 443)
-          ? ''
-          : ':' + $location.port();
-      return $location.protocol() + '://' + $location.host() + displayPort + '/' + API_ENDPOINT_WEBHOOKS + '/' + token;
+      return `${protocol}://${$location.host()}${displayPort}/${API_ENDPOINT_WEBHOOKS}/${token}`;
+    };
+
+    helper.returnStackWebhookUrl = function (token) {
+      return `${protocol}://${$location.host()}${displayPort}/${API_ENDPOINT_STACKS}/webhooks/${token}`;
     };
 
     return helper;

app/portainer/models/stack.js

@@ -20,6 +20,8 @@ export function StackViewModel(data) {
   this.Orphaned = false;
   this.Checked = false;
   this.GitConfig = data.GitConfig;
+  this.AdditionalFiles = data.AdditionalFiles;
+  this.AutoUpdate = data.AutoUpdate;
 }
 
 export function ExternalStackViewModel(name, type, creationDate) {

app/portainer/rest/stack.js

@@ -5,7 +5,7 @@ angular.module('portainer.app').factory('Stack', [
   function StackFactory($resource, EndpointProvider, API_ENDPOINT_STACKS) {
     'use strict';
     return $resource(
-      API_ENDPOINT_STACKS + '/:id/:action',
+      API_ENDPOINT_STACKS + '/:id/:action/:subaction',
       {},
       {
         get: { method: 'GET', params: { id: '@id' } },
@@ -18,7 +18,8 @@ angular.module('portainer.app').factory('Stack', [
         migrate: { method: 'POST', params: { id: '@id', action: 'migrate', endpointId: '@endpointId' }, ignoreLoadingBar: true },
         start: { method: 'POST', params: { id: '@id', action: 'start' } },
         stop: { method: 'POST', params: { id: '@id', action: 'stop' } },
-        updateGit: { method: 'PUT', params: { action: 'git' } },
+        updateGit: { method: 'PUT', params: { id: '@id', action: 'git', subaction: 'redeploy' } },
+        updateGitStackSettings: { method: 'POST', params: { id: '@id', action: 'git' }, ignoreLoadingBar: true },
       }
     );
   },

app/portainer/services/api/stackService.js

@@ -326,12 +326,18 @@ angular.module('portainer.app').factory('StackService', [
         Name: name,
         RepositoryURL: repositoryOptions.RepositoryURL,
         RepositoryReferenceName: repositoryOptions.RepositoryReferenceName,
-        ComposeFilePathInRepository: repositoryOptions.ComposeFilePathInRepository,
+        ComposeFile: repositoryOptions.ComposeFilePathInRepository,
+        AdditionalFiles: repositoryOptions.AdditionalFiles,
         RepositoryAuthentication: repositoryOptions.RepositoryAuthentication,
         RepositoryUsername: repositoryOptions.RepositoryUsername,
         RepositoryPassword: repositoryOptions.RepositoryPassword,
         Env: env,
       };
+
+      if (repositoryOptions.AutoUpdate) {
+        payload.AutoUpdate = repositoryOptions.AutoUpdate;
+      }
+
       return Stack.create({ method: 'repository', type: 2, endpointId: endpointId }, payload).$promise;
     };
 
@@ -346,12 +352,18 @@ angular.module('portainer.app').factory('StackService', [
             SwarmID: swarm.Id,
             RepositoryURL: repositoryOptions.RepositoryURL,
             RepositoryReferenceName: repositoryOptions.RepositoryReferenceName,
-            ComposeFilePathInRepository: repositoryOptions.ComposeFilePathInRepository,
+            ComposeFile: repositoryOptions.ComposeFilePathInRepository,
+            AdditionalFiles: repositoryOptions.AdditionalFiles,
             RepositoryAuthentication: repositoryOptions.RepositoryAuthentication,
             RepositoryUsername: repositoryOptions.RepositoryUsername,
             RepositoryPassword: repositoryOptions.RepositoryPassword,
             Env: env,
           };
+
+          if (repositoryOptions.AutoUpdate) {
+            payload.AutoUpdate = repositoryOptions.AutoUpdate;
+          }
+
           return Stack.create({ method: 'repository', type: 1, endpointId: endpointId }, payload).$promise;
         })
         .then(function success(data) {
@@ -405,6 +417,31 @@ angular.module('portainer.app').factory('StackService', [
       ).$promise;
     }
 
+    service.updateGitStackSettings = function (id, endpointId, env, gitConfig) {
+      // prepare auto update
+      const autoUpdate = {};
+
+      if (gitConfig.AutoUpdate.RepositoryAutomaticUpdates) {
+        if (gitConfig.AutoUpdate.RepositoryMechanism === 'Interval') {
+          autoUpdate.Interval = gitConfig.AutoUpdate.RepositoryFetchInterval;
+        } else if (gitConfig.AutoUpdate.RepositoryMechanism === 'Webhook') {
+          autoUpdate.Webhook = gitConfig.AutoUpdate.RepositoryWebhookURL.split('/').reverse()[0];
+        }
+      }
+
+      return Stack.updateGitStackSettings(
+        { endpointId, id },
+        {
+          AutoUpdate: autoUpdate,
+          Env: env,
+          RepositoryReferenceName: gitConfig.RefName,
+          RepositoryAuthentication: gitConfig.RepositoryAuthentication,
+          RepositoryUsername: gitConfig.RepositoryUsername,
+          RepositoryPassword: gitConfig.RepositoryPassword,
+        }
+      ).$promise;
+    };
+
     return service;
   },
 ]);

app/portainer/views/stacks/create/createStackController.js

@@ -1,6 +1,6 @@
 import angular from 'angular';
 import _ from 'lodash-es';
-
+import uuidv4 from 'uuid/v4';
 import { AccessControlFormData } from '../../../components/accessControlForm/porAccessControlFormModel';
 
 angular
@@ -21,7 +21,9 @@ angular
     StackHelper,
     ContainerHelper,
     CustomTemplateService,
-    ContainerService
+    ContainerService,
+    WebhookHelper,
+    clipboard
   ) {
     $scope.formValues = {
       Name: '',
@@ -33,8 +35,13 @@ angular
       RepositoryUsername: '',
       RepositoryPassword: '',
       Env: [],
+      AdditionalFiles: [],
       ComposeFilePathInRepository: 'docker-compose.yml',
       AccessControlData: new AccessControlFormData(),
+      RepositoryAutomaticUpdates: true,
+      RepositoryMechanism: 'Interval',
+      RepositoryFetchInterval: '5m',
+      RepositoryWebhookURL: WebhookHelper.returnStackWebhookUrl(uuidv4()),
     };
 
     $scope.state = {
@@ -67,6 +74,14 @@ angular
       $scope.formValues.Env.splice(index, 1);
     };
 
+    $scope.addAdditionalFiles = function () {
+      $scope.formValues.AdditionalFiles.push('');
+    };
+
+    $scope.removeAdditionalFiles = function (index) {
+      $scope.formValues.AdditionalFiles.splice(index, 1);
+    };
+
     function validateForm(accessControlData, isAdmin) {
       $scope.state.formValidationError = '';
       var error = '';
@@ -95,6 +110,7 @@ angular
 
       if (method === 'repository') {
         var repositoryOptions = {
+          AdditionalFiles: $scope.formValues.AdditionalFiles,
           RepositoryURL: $scope.formValues.RepositoryURL,
           RepositoryReferenceName: $scope.formValues.RepositoryReferenceName,
           ComposeFilePathInRepository: $scope.formValues.ComposeFilePathInRepository,
@@ -102,10 +118,24 @@ angular
           RepositoryUsername: $scope.formValues.RepositoryUsername,
           RepositoryPassword: $scope.formValues.RepositoryPassword,
         };
+
+        getAutoUpdatesProperty(repositoryOptions);
+
         return StackService.createSwarmStackFromGitRepository(name, repositoryOptions, env, endpointId);
       }
     }
 
+    function getAutoUpdatesProperty(repositoryOptions) {
+      if ($scope.formValues.RepositoryAutomaticUpdates) {
+        repositoryOptions.AutoUpdate = {};
+        if ($scope.formValues.RepositoryMechanism === 'Interval') {
+          repositoryOptions.AutoUpdate.Interval = $scope.formValues.RepositoryFetchInterval;
+        } else if ($scope.formValues.RepositoryMechanism === 'Webhook') {
+          repositoryOptions.AutoUpdate.Webhook = $scope.formValues.RepositoryWebhookURL.split('/').reverse()[0];
+        }
+      }
+    }
+
     function createComposeStack(name, method) {
       var env = FormHelper.removeInvalidEnvVars($scope.formValues.Env);
       const endpointId = +$state.params.endpointId;
@@ -118,6 +148,7 @@ angular
         return StackService.createComposeStackFromFileUpload(name, stackFile, env, endpointId);
       } else if (method === 'repository') {
         var repositoryOptions = {
+          AdditionalFiles: $scope.formValues.AdditionalFiles,
           RepositoryURL: $scope.formValues.RepositoryURL,
           RepositoryReferenceName: $scope.formValues.RepositoryReferenceName,
           ComposeFilePathInRepository: $scope.formValues.ComposeFilePathInRepository,
@@ -125,10 +156,19 @@ angular
           RepositoryUsername: $scope.formValues.RepositoryUsername,
           RepositoryPassword: $scope.formValues.RepositoryPassword,
         };
+
+        getAutoUpdatesProperty(repositoryOptions);
+
         return StackService.createComposeStackFromGitRepository(name, repositoryOptions, env, endpointId);
       }
     }
 
+    $scope.copyWebhook = function () {
+      clipboard.copyText($scope.formValues.RepositoryWebhookURL);
+      $('#copyNotification').show();
+      $('#copyNotification').fadeOut(2000);
+    };
+
     $scope.handleEnvVarChange = handleEnvVarChange;
     function handleEnvVarChange(value) {
       $scope.formValues.Env = value;

app/portainer/views/stacks/create/createstack.html

@@ -130,7 +130,14 @@
             </div>
           </div>
           <!-- !upload -->
-          <git-form ng-if="state.Method === 'repository'" model="formValues" on-change="(onChangeFormValues)"></git-form>
+          <git-form
+            ng-if="state.Method === 'repository'"
+            model="formValues"
+            on-change="(onChangeFormValues)"
+            additional-file="true"
+            auto-update="true"
+            show-auth-explanation="true"
+          ></git-form>
           <!-- custom-template -->
           <div ng-show="state.Method === 'template'">
             <div class="form-group">
@@ -207,7 +214,7 @@
               || (state.Method === 'editor' && (!formValues.StackFileContent || state.editorYamlValidationError))
               || (state.Method === 'upload' && (!formValues.StackFile || state.uploadYamlValidationError))
               || (state.Method === 'template' && (!formValues.StackFileContent || !selectedTemplate || state.editorYamlValidationError))
-              || (state.Method === 'repository' && ((!formValues.RepositoryURL || !formValues.ComposeFilePathInRepository) || (formValues.RepositoryAuthentication && (!formValues.RepositoryUsername || !formValues.RepositoryPassword))))
+              || (state.Method === 'repository' && ((!formValues.RepositoryURL || !formValues.ComposeFilePathInRepository) || (formValues.RepositoryAuthentication && !formValues.RepositoryPassword)))
               || !formValues.Name"
                 ng-click="deployStack()"
                 button-spinner="state.actionInProgress"

app/portainer/views/stacks/edit/stack.html

@@ -125,7 +125,7 @@
           </uib-tab>
           <!-- !tab-info -->
           <!-- tab-file -->
-          <uib-tab index="1" select="showEditor()" ng-if="!external">
+          <uib-tab index="1" select="showEditor()" ng-if="!external && !stack.GitConfig">
             <uib-tab-heading> <i class="fa fa-pencil-alt space-right" aria-hidden="true"></i> Editor </uib-tab-heading>
             <form class="form-horizontal" ng-if="state.showEditorTab" style="margin-top: 10px;" name="stackUpdateForm">
               <div class="form-group">
@@ -214,6 +214,7 @@
       order-by="Status"
       show-host-column="false"
       show-add-action="false"
+      not-auto-focus="true"
     ></containers-datatable>
   </div>
 </div>
@@ -233,6 +234,7 @@
       show-task-logs-button="applicationState.endpoint.apiVersion >= 1.30"
       show-add-action="false"
       show-stack-column="false"
+      not-auto-focus="true"
     ></services-datatable>
   </div>
 </div>

package.json

@@ -91,9 +91,10 @@
     "jquery": "^3.5.1",
     "js-base64": "^3.6.0",
     "js-yaml": "^3.14.0",
-    "lodash-es": "^4.17.15",
+    "lodash-es": "^4.17.15",  
     "moment": "^2.21.0",
     "ng-file-upload": "~12.2.13",
+    "parse-duration": "^1.0.0",
     "source-map-loader": "^1.1.2",
     "spinkit": "^2.0.1",
     "splitargs": "github:deviantony/splitargs#semver:~0.2.0",

yarn.lock

@@ -7967,6 +7967,11 @@ parse-asn1@^5.0.0, parse-asn1@^5.1.5:
     pbkdf2 "^3.0.3"
     safe-buffer "^5.1.1"
 
+parse-duration@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/parse-duration/-/parse-duration-1.0.0.tgz#8605651745f61088f6fb14045c887526c291858c"
+  integrity sha512-X4kUkCTHU1N/kEbwK9FpUJ0UZQa90VzeczfS704frR30gljxDG0pSziws06XlK+CGRSo/1wtG1mFIdBFQTMQNw==
+
 parse-filepath@^1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/parse-filepath/-/parse-filepath-1.0.2.tgz#a632127f53aaf3d15876f5872f3ffac763d6c891"