feat(compose): add docker-compose wrapper (#4713)

* feat(compose): add docker-compose wrapper

ce-187

* fix(compose): pick compose implementation upon startup

* Add static compose build for linux

* Fix wget

* Fix platofrm specific docker-compose download

* Keep amd64 architecture as download parameter

* Add tmp folder for docker-compose

* fix: line endings

* add proxy server

* logs

* Proxy

* Add lite transport for compose

* Fix local deployment

* refactor: pass proxyManager by ref

* fix: string conversion

* refactor: compose wrapper remove unused code

* fix: tests

* Add edge

* Fix merge issue

* refactor: remove unused code

* Move server to proxy implementation

* Cleanup wrapper and manager

* feat: pass max supported compose syntax version with each endpoint

* fix: pick compose syntax version

* fix: store wrapper version in portainer

* Get and show composeSyntaxMaxVersion at stack creation screen

* Get and show composeSyntaxMaxVersion at stack editor screen

* refactor: proxy server

* Fix used tmp

* Bump docker-compose to 1.28.0

* remove message for docker compose limitation

* fix: markup typo

* Rollback docker compose to 1.27.4

* * attempt to fix the windows build issue

* * attempt to debug grunt issue

* * use console log in grunt file

* fix: try to fix windows build by removing indirect deps from go.mod

* Remove tmp folder

* Remove builder stage

* feat(build/windows): add git for Docker Compose

* feat(build/windows): add git for Docker Compose

* feat(build/windows): add git for Docker Compose

* feat(build/windows): add git for Docker Compose

* feat(build/windows): add git for Docker Compose

* feat(build/windows): add git for Docker Compose - fixed verbose output

* refactor: renames

* fix(stack): get endpoint by EndpointProvider

* fix(stack): use margin to add space between line instead of using br tag

Co-authored-by: Stéphane Busso <stephane.busso@gmail.com>
Co-authored-by: Simon Meng <simon.meng@portainer.io>
Co-authored-by: yi-portainer <yi.chen@portainer.io>
Co-authored-by: Steven Kang <skan070@gmail.com>

api/cmd/portainer/main.go

@@ -17,6 +17,8 @@ import (
 	"github.com/portainer/portainer/api/git"
 	"github.com/portainer/portainer/api/http"
 	"github.com/portainer/portainer/api/http/client"
+	"github.com/portainer/portainer/api/http/proxy"
+	kubeproxy "github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
 	"github.com/portainer/portainer/api/internal/snapshot"
 	"github.com/portainer/portainer/api/jwt"
 	"github.com/portainer/portainer/api/kubernetes"
@@ -71,7 +73,12 @@ func initDataStore(dataStorePath string, fileService portainer.FileService) port
 	return store
 }
 
-func initComposeStackManager(dataStorePath string, reverseTunnelService portainer.ReverseTunnelService) portainer.ComposeStackManager {
+func initComposeStackManager(assetsPath string, dataStorePath string, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager {
+	composeWrapper := exec.NewComposeWrapper(assetsPath, proxyManager)
+	if composeWrapper != nil {
+		return composeWrapper
+	}
+
 	return libcompose.NewComposeStackManager(dataStorePath, reverseTunnelService)
 }
 
@@ -384,8 +391,10 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
+	kubernetesTokenCacheManager := kubeproxy.NewTokenCacheManager()
+	proxyManager := proxy.NewManager(dataStore, digitalSignatureService, reverseTunnelService, dockerClientFactory, kubernetesClientFactory, kubernetesTokenCacheManager)
 
-	composeStackManager := initComposeStackManager(*flags.Data, reverseTunnelService)
+	composeStackManager := initComposeStackManager(*flags.Assets, *flags.Data, reverseTunnelService, proxyManager)
 
 	kubernetesDeployer := initKubernetesDeployer(*flags.Assets)
 
@@ -466,6 +475,8 @@ func main() {
 		LDAPService:                 ldapService,
 		OAuthService:                oauthService,
 		GitService:                  gitService,
+		ProxyManager:                proxyManager,
+		KubernetesTokenCacheManager: kubernetesTokenCacheManager,
 		SignatureService:            digitalSignatureService,
 		SnapshotService:             snapshotService,
 		SSL:                         *flags.SSL,

api/exec/compose_wrapper.go

@@ -0,0 +1,132 @@
+package exec
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/http/proxy"
+)
+
+// ComposeWrapper is a wrapper for docker-compose binary
+type ComposeWrapper struct {
+	binaryPath   string
+	proxyManager *proxy.Manager
+}
+
+// NewComposeWrapper returns a docker-compose wrapper if corresponding binary present, otherwise nil
+func NewComposeWrapper(binaryPath string, proxyManager *proxy.Manager) *ComposeWrapper {
+	if !IsBinaryPresent(programPath(binaryPath, "docker-compose")) {
+		return nil
+	}
+
+	return &ComposeWrapper{
+		binaryPath:   binaryPath,
+		proxyManager: proxyManager,
+	}
+}
+
+// ComposeSyntaxMaxVersion returns the maximum supported version of the docker compose syntax
+func (w *ComposeWrapper) ComposeSyntaxMaxVersion() string {
+	return portainer.ComposeSyntaxMaxVersion
+}
+
+// Up builds, (re)creates and starts containers in the background. Wraps `docker-compose up -d` command
+func (w *ComposeWrapper) Up(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
+	_, err := w.command([]string{"up", "-d"}, stack, endpoint)
+	return err
+}
+
+// Down stops and removes containers, networks, images, and volumes. Wraps `docker-compose down --remove-orphans` command
+func (w *ComposeWrapper) Down(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
+	_, err := w.command([]string{"down", "--remove-orphans"}, stack, endpoint)
+	return err
+}
+
+func (w *ComposeWrapper) command(command []string, stack *portainer.Stack, endpoint *portainer.Endpoint) ([]byte, error) {
+	if endpoint == nil {
+		return nil, errors.New("cannot call a compose command on an empty endpoint")
+	}
+
+	program := programPath(w.binaryPath, "docker-compose")
+
+	options := setComposeFile(stack)
+
+	options = addProjectNameOption(options, stack)
+	options, err := addEnvFileOption(options, stack)
+	if err != nil {
+		return nil, err
+	}
+
+	if !(endpoint.URL == "" || strings.HasPrefix(endpoint.URL, "unix://") || strings.HasPrefix(endpoint.URL, "npipe://")) {
+
+		proxy, err := w.proxyManager.CreateComposeProxyServer(endpoint)
+		if err != nil {
+			return nil, err
+		}
+
+		defer proxy.Close()
+
+		options = append(options, "-H", fmt.Sprintf("http://127.0.0.1:%d", proxy.Port))
+	}
+
+	args := append(options, command...)
+
+	var stderr bytes.Buffer
+	cmd := exec.Command(program, args...)
+	cmd.Stderr = &stderr
+
+	out, err := cmd.Output()
+	if err != nil {
+		return out, errors.New(stderr.String())
+	}
+
+	return out, nil
+}
+
+func setComposeFile(stack *portainer.Stack) []string {
+	options := make([]string, 0)
+
+	if stack == nil || stack.EntryPoint == "" {
+		return options
+	}
+
+	composeFilePath := path.Join(stack.ProjectPath, stack.EntryPoint)
+	options = append(options, "-f", composeFilePath)
+	return options
+}
+
+func addProjectNameOption(options []string, stack *portainer.Stack) []string {
+	if stack == nil || stack.Name == "" {
+		return options
+	}
+
+	options = append(options, "-p", stack.Name)
+	return options
+}
+
+func addEnvFileOption(options []string, stack *portainer.Stack) ([]string, error) {
+	if stack == nil || stack.Env == nil || len(stack.Env) == 0 {
+		return options, nil
+	}
+
+	envFilePath := path.Join(stack.ProjectPath, "stack.env")
+
+	envfile, err := os.OpenFile(envFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return options, err
+	}
+
+	for _, v := range stack.Env {
+		envfile.WriteString(fmt.Sprintf("%s=%s\n", v.Name, v.Value))
+	}
+	envfile.Close()
+
+	options = append(options, "--env-file", envFilePath)
+	return options, nil
+}

api/exec/compose_wrapper_integration_test.go

@@ -0,0 +1,75 @@
+// +build integration
+
+package exec
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+)
+
+const composeFile = `version: "3.9"
+services:
+  busybox:
+    image: "alpine:latest"
+    container_name: "compose_wrapper_test"`
+const composedContainerName = "compose_wrapper_test"
+
+func setup(t *testing.T) (*portainer.Stack, *portainer.Endpoint) {
+	dir := t.TempDir()
+	composeFileName := "compose_wrapper_test.yml"
+	f, _ := os.Create(filepath.Join(dir, composeFileName))
+	f.WriteString(composeFile)
+
+	stack := &portainer.Stack{
+		ProjectPath: dir,
+		EntryPoint:  composeFileName,
+		Name:        "project-name",
+	}
+
+	endpoint := &portainer.Endpoint{}
+
+	return stack, endpoint
+}
+
+func Test_UpAndDown(t *testing.T) {
+
+	stack, endpoint := setup(t)
+
+	w := NewComposeWrapper("", nil)
+
+	err := w.Up(stack, endpoint)
+	if err != nil {
+		t.Fatalf("Error calling docker-compose up: %s", err)
+	}
+
+	if containerExists(composedContainerName) == false {
+		t.Fatal("container should exist")
+	}
+
+	err = w.Down(stack, endpoint)
+	if err != nil {
+		t.Fatalf("Error calling docker-compose down: %s", err)
+	}
+
+	if containerExists(composedContainerName) {
+		t.Fatal("container should be removed")
+	}
+}
+
+func containerExists(contaierName string) bool {
+	cmd := exec.Command(osProgram("docker"), "ps", "-a", "-f", fmt.Sprintf("name=%s", contaierName))
+
+	out, err := cmd.Output()
+	if err != nil {
+		log.Fatalf("failed to list containers: %s", err)
+	}
+
+	return strings.Contains(string(out), contaierName)
+}

api/exec/compose_wrapper_test.go

@@ -0,0 +1,143 @@
+package exec
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_setComposeFile(t *testing.T) {
+	tests := []struct {
+		name     string
+		stack    *portainer.Stack
+		expected []string
+	}{
+		{
+			name:     "should return empty result if stack is missing",
+			stack:    nil,
+			expected: []string{},
+		},
+		{
+			name:     "should return empty result if stack don't have entrypoint",
+			stack:    &portainer.Stack{},
+			expected: []string{},
+		},
+		{
+			name: "should allow file name and dir",
+			stack: &portainer.Stack{
+				ProjectPath: "dir",
+				EntryPoint:  "file",
+			},
+			expected: []string{"-f", path.Join("dir", "file")},
+		},
+		{
+			name: "should allow file name only",
+			stack: &portainer.Stack{
+				EntryPoint: "file",
+			},
+			expected: []string{"-f", "file"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := setComposeFile(tt.stack)
+			assert.ElementsMatch(t, tt.expected, result)
+		})
+	}
+}
+
+func Test_addProjectNameOption(t *testing.T) {
+	tests := []struct {
+		name     string
+		stack    *portainer.Stack
+		expected []string
+	}{
+		{
+			name:     "should not add project option if stack is missing",
+			stack:    nil,
+			expected: []string{},
+		},
+		{
+			name:     "should not add project option if stack doesn't have name",
+			stack:    &portainer.Stack{},
+			expected: []string{},
+		},
+		{
+			name: "should add project name option if stack has a name",
+			stack: &portainer.Stack{
+				Name: "project-name",
+			},
+			expected: []string{"-p", "project-name"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			options := []string{"-a", "b"}
+			result := addProjectNameOption(options, tt.stack)
+			assert.ElementsMatch(t, append(options, tt.expected...), result)
+		})
+	}
+}
+
+func Test_addEnvFileOption(t *testing.T) {
+	dir := t.TempDir()
+
+	tests := []struct {
+		name            string
+		stack           *portainer.Stack
+		expected        []string
+		expectedContent string
+	}{
+		{
+			name:     "should not add env file option if stack is missing",
+			stack:    nil,
+			expected: []string{},
+		},
+		{
+			name:     "should not add env file option if stack doesn't have env variables",
+			stack:    &portainer.Stack{},
+			expected: []string{},
+		},
+		{
+			name: "should not add env file option if stack's env variables are empty",
+			stack: &portainer.Stack{
+				ProjectPath: dir,
+				Env:         []portainer.Pair{},
+			},
+			expected: []string{},
+		},
+		{
+			name: "should add env file option if stack has env variables",
+			stack: &portainer.Stack{
+				ProjectPath: dir,
+				Env: []portainer.Pair{
+					{Name: "var1", Value: "value1"},
+					{Name: "var2", Value: "value2"},
+				},
+			},
+			expected:        []string{"--env-file", path.Join(dir, "stack.env")},
+			expectedContent: "var1=value1\nvar2=value2\n",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			options := []string{"-a", "b"}
+			result, _ := addEnvFileOption(options, tt.stack)
+			assert.ElementsMatch(t, append(options, tt.expected...), result)
+
+			if tt.expectedContent != "" {
+				f, _ := os.Open(path.Join(dir, "stack.env"))
+				content, _ := ioutil.ReadAll(f)
+
+				assert.Equal(t, tt.expectedContent, string(content))
+			}
+		})
+	}
+}

api/exec/utils.go

@@ -0,0 +1,24 @@
+package exec
+
+import (
+	"os/exec"
+	"path/filepath"
+	"runtime"
+)
+
+func osProgram(program string) string {
+	if runtime.GOOS == "windows" {
+		program += ".exe"
+	}
+	return program
+}
+
+func programPath(rootPath, program string) string {
+	return filepath.Join(rootPath, osProgram(program))
+}
+
+// IsBinaryPresent returns true if corresponding program exists on PATH
+func IsBinaryPresent(program string) bool {
+	_, err := exec.LookPath(program)
+	return err == nil
+}

api/exec/utils_test.go

@@ -0,0 +1,16 @@
+package exec
+
+import (
+	"testing"
+)
+
+func Test_isBinaryPresent(t *testing.T) {
+
+	if !IsBinaryPresent("docker") {
+		t.Error("expect docker binary to exist on the path")
+	}
+
+	if IsBinaryPresent("executable-with-this-name-should-not-exist") {
+		t.Error("expect binary with a random name to be missing on the path")
+	}
+}

api/go.mod

@@ -28,6 +28,7 @@ require (
 	github.com/portainer/libcompose v0.5.3
 	github.com/portainer/libcrypto v0.0.0-20190723020515-23ebe86ab2c2
 	github.com/portainer/libhttp v0.0.0-20190806161843-ba068f58be33
+	github.com/stretchr/testify v1.6.1 // indirect
 	golang.org/x/crypto v0.0.0-20191128160524-b544559bb6d1
 	golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933 // indirect
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45

api/go.sum

@@ -262,12 +262,15 @@ github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=
 github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce h1:fb190+cK2Xz/dvi9Hv8eCYJYvIGUTN2/KLq1pT6CjEc=
 github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4=
 github.com/urfave/cli v1.21.0/go.mod h1:lxDj6qX9Q6lWQxIrbrT0nwecwUtRnhVZAJjJZrVUZZQ=
@@ -392,6 +395,8 @@ gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

api/http/handler/endpoints/endpoint_inspect.go

@@ -6,7 +6,7 @@ import (
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/errors"
 )
 
@@ -30,6 +30,7 @@ func (handler *Handler) endpointInspect(w http.ResponseWriter, r *http.Request)
 	}
 
 	hideFields(endpoint)
+	endpoint.ComposeSyntaxMaxVersion = handler.ComposeStackManager.ComposeSyntaxMaxVersion()
 
 	return response.JSON(w, endpoint)
 }

api/http/handler/endpoints/endpoint_list.go

@@ -5,12 +5,11 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/portainer/portainer/api"
-
 	"github.com/portainer/libhttp/request"
 
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/response"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/http/security"
 )
 
@@ -89,6 +88,7 @@ func (handler *Handler) endpointList(w http.ResponseWriter, r *http.Request) *ht
 
 	for idx := range paginatedEndpoints {
 		hideFields(&paginatedEndpoints[idx])
+		paginatedEndpoints[idx].ComposeSyntaxMaxVersion = handler.ComposeStackManager.ComposeSyntaxMaxVersion()
 	}
 
 	w.Header().Set("X-Total-Count", strconv.Itoa(filteredEndpointCount))

api/http/handler/endpoints/handler.go

@@ -27,6 +27,7 @@ type Handler struct {
 	ProxyManager         *proxy.Manager
 	ReverseTunnelService portainer.ReverseTunnelService
 	SnapshotService      portainer.SnapshotService
+	ComposeStackManager  portainer.ComposeStackManager
 }
 
 // NewHandler creates a handler to manage endpoint operations.

api/http/handler/stacks/create_compose_stack.go

@@ -357,7 +357,6 @@ func (handler *Handler) deployComposeStack(config *composeStackDeploymentConfig)
 		!isAdminOrEndpointAdmin {
 
 		composeFilePath := path.Join(config.stack.ProjectPath, config.stack.EntryPoint)
-
 		stackContent, err := handler.FileService.GetFileContent(composeFilePath)
 		if err != nil {
 			return err

api/http/handler/stacks/handler.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/gorilla/mux"
 	httperror "github.com/portainer/libhttp/error"
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/authorization"
 )

api/http/handler/stacks/stack_delete.go

@@ -155,5 +155,6 @@ func (handler *Handler) deleteStack(stack *portainer.Stack, endpoint *portainer.
 	if stack.Type == portainer.DockerSwarmStack {
 		return handler.SwarmStackManager.Remove(stack, endpoint)
 	}
+
 	return handler.ComposeStackManager.Down(stack, endpoint)
 }

api/http/handler/stacks/stack_start.go

@@ -4,13 +4,13 @@ import (
 	"errors"
 	"net/http"
 
+	portainer "github.com/portainer/portainer/api"
 	httperrors "github.com/portainer/portainer/api/http/errors"
 	"github.com/portainer/portainer/api/http/security"
 
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
-	"github.com/portainer/portainer/api"
 	bolterrors "github.com/portainer/portainer/api/bolt/errors"
 )
 

api/http/handler/stacks/stack_stop.go

@@ -4,15 +4,14 @@ import (
 	"errors"
 	"net/http"
 
-	httperrors "github.com/portainer/portainer/api/http/errors"
-
-	"github.com/portainer/portainer/api/http/security"
-
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
-	"github.com/portainer/portainer/api"
+
+	portainer "github.com/portainer/portainer/api"
 	bolterrors "github.com/portainer/portainer/api/bolt/errors"
+	httperrors "github.com/portainer/portainer/api/http/errors"
+	"github.com/portainer/portainer/api/http/security"
 )
 
 // POST request on /api/stacks/:id/stop

api/http/proxy/factory/docker_compose.go

@@ -0,0 +1,88 @@
+package factory
+
+import (
+	"fmt"
+	"log"
+	"net"
+	"net/http"
+	"net/url"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/crypto"
+	"github.com/portainer/portainer/api/http/proxy/factory/dockercompose"
+)
+
+// ProxyServer provide an extedned proxy with a local server to forward requests
+type ProxyServer struct {
+	server *http.Server
+	Port   int
+}
+
+func (factory *ProxyFactory) NewDockerComposeAgentProxy(endpoint *portainer.Endpoint) (*ProxyServer, error) {
+
+	if endpoint.Type == portainer.EdgeAgentOnDockerEnvironment {
+		return &ProxyServer{
+			Port: factory.reverseTunnelService.GetTunnelDetails(endpoint.ID).Port,
+		}, nil
+	}
+
+	endpointURL, err := url.Parse(endpoint.URL)
+	if err != nil {
+		return nil, err
+	}
+
+	endpointURL.Scheme = "http"
+	httpTransport := &http.Transport{}
+
+	if endpoint.TLSConfig.TLS || endpoint.TLSConfig.TLSSkipVerify {
+		config, err := crypto.CreateTLSConfigurationFromDisk(endpoint.TLSConfig.TLSCACertPath, endpoint.TLSConfig.TLSCertPath, endpoint.TLSConfig.TLSKeyPath, endpoint.TLSConfig.TLSSkipVerify)
+		if err != nil {
+			return nil, err
+		}
+
+		httpTransport.TLSClientConfig = config
+		endpointURL.Scheme = "https"
+	}
+
+	proxy := newSingleHostReverseProxyWithHostHeader(endpointURL)
+
+	proxy.Transport = dockercompose.NewAgentTransport(factory.signatureService, httpTransport)
+
+	proxyServer := &ProxyServer{
+		&http.Server{
+			Handler: proxy,
+		},
+		0,
+	}
+
+	return proxyServer, proxyServer.start()
+}
+
+func (proxy *ProxyServer) start() error {
+	listener, err := net.Listen("tcp", ":0")
+	if err != nil {
+		return err
+	}
+
+	proxy.Port = listener.Addr().(*net.TCPAddr).Port
+	go func() {
+		proxyHost := fmt.Sprintf("127.0.0.1:%d", proxy.Port)
+		log.Printf("Starting Proxy server on %s...\n", proxyHost)
+
+		err := proxy.server.Serve(listener)
+		log.Printf("Exiting Proxy server %s\n", proxyHost)
+
+		if err != http.ErrServerClosed {
+			log.Printf("Proxy server %s exited with an error: %s\n", proxyHost, err)
+		}
+	}()
+
+	return nil
+}
+
+// Close shuts down the server
+func (proxy *ProxyServer) Close() {
+	if proxy.server != nil {
+		proxy.server.Close()
+	}
+}

api/http/proxy/factory/dockercompose/transport.go

@@ -0,0 +1,40 @@
+package dockercompose
+
+import (
+	"net/http"
+
+	portainer "github.com/portainer/portainer/api"
+)
+
+type (
+	// AgentTransport is an http.Transport wrapper that adds custom http headers to communicate to an Agent
+	AgentTransport struct {
+		httpTransport      *http.Transport
+		signatureService   portainer.DigitalSignatureService
+		endpointIdentifier portainer.EndpointID
+	}
+)
+
+// NewAgentTransport returns a new transport that can be used to send signed requests to a Portainer agent
+func NewAgentTransport(signatureService portainer.DigitalSignatureService, httpTransport *http.Transport) *AgentTransport {
+	transport := &AgentTransport{
+		httpTransport:    httpTransport,
+		signatureService: signatureService,
+	}
+
+	return transport
+}
+
+// RoundTrip is the implementation of the the http.RoundTripper interface
+func (transport *AgentTransport) RoundTrip(request *http.Request) (*http.Response, error) {
+
+	signature, err := transport.signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
+	if err != nil {
+		return nil, err
+	}
+
+	request.Header.Set(portainer.PortainerAgentPublicKeyHeader, transport.signatureService.EncodedPublicKey())
+	request.Header.Set(portainer.PortainerAgentSignatureHeader, signature)
+
+	return transport.httpTransport.RoundTrip(request)
+}

api/http/proxy/manager.go

@@ -1,6 +1,7 @@
 package proxy
 
 import (
+	"fmt"
 	"net/http"
 
 	"github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
@@ -43,13 +44,19 @@ func (manager *Manager) CreateAndRegisterEndpointProxy(endpoint *portainer.Endpo
 		return nil, err
 	}
 
-	manager.endpointProxies.Set(string(endpoint.ID), proxy)
+	manager.endpointProxies.Set(fmt.Sprint(endpoint.ID), proxy)
 	return proxy, nil
 }
 
+// CreateComposeProxyServer creates a new HTTP reverse proxy based on endpoint properties and and adds it to the registered proxies.
+// It can also be used to create a new HTTP reverse proxy and replace an already registered proxy.
+func (manager *Manager) CreateComposeProxyServer(endpoint *portainer.Endpoint) (*factory.ProxyServer, error) {
+	return manager.proxyFactory.NewDockerComposeAgentProxy(endpoint)
+}
+
 // GetEndpointProxy returns the proxy associated to a key
 func (manager *Manager) GetEndpointProxy(endpoint *portainer.Endpoint) http.Handler {
-	proxy, ok := manager.endpointProxies.Get(string(endpoint.ID))
+	proxy, ok := manager.endpointProxies.Get(fmt.Sprint(endpoint.ID))
 	if !ok {
 		return nil
 	}
@@ -61,7 +68,7 @@ func (manager *Manager) GetEndpointProxy(endpoint *portainer.Endpoint) http.Hand
 // and cleans the k8s endpoint client cache. DeleteEndpointProxy
 // is currently only called for edge connection clean up.
 func (manager *Manager) DeleteEndpointProxy(endpoint *portainer.Endpoint) {
-	manager.endpointProxies.Remove(string(endpoint.ID))
+	manager.endpointProxies.Remove(fmt.Sprint(endpoint.ID))
 	manager.k8sClientFactory.RemoveKubeClient(endpoint)
 }
 

api/http/server.go

@@ -39,6 +39,7 @@ import (
 	"github.com/portainer/portainer/api/http/proxy"
 	"github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
 	"github.com/portainer/portainer/api/http/security"
+
 	"github.com/portainer/portainer/api/kubernetes/cli"
 )
 
@@ -59,6 +60,8 @@ type Server struct {
 	LDAPService                 portainer.LDAPService
 	OAuthService                portainer.OAuthService
 	SwarmStackManager           portainer.SwarmStackManager
+	ProxyManager                *proxy.Manager
+	KubernetesTokenCacheManager *kubernetes.TokenCacheManager
 	Handler                     *handler.Handler
 	SSL                         bool
 	SSLCert                     string
@@ -70,8 +73,7 @@ type Server struct {
 
 // Start starts the HTTP server
 func (server *Server) Start() error {
-	kubernetesTokenCacheManager := kubernetes.NewTokenCacheManager()
+	kubernetesTokenCacheManager := server.KubernetesTokenCacheManager
-	proxyManager := proxy.NewManager(server.DataStore, server.SignatureService, server.ReverseTunnelService, server.DockerClientFactory, server.KubernetesClientFactory, kubernetesTokenCacheManager)
 
 	requestBouncer := security.NewRequestBouncer(server.DataStore, server.JWTService)
 
@@ -82,7 +84,7 @@ func (server *Server) Start() error {
 	authHandler.CryptoService = server.CryptoService
 	authHandler.JWTService = server.JWTService
 	authHandler.LDAPService = server.LDAPService
-	authHandler.ProxyManager = proxyManager
+	authHandler.ProxyManager = server.ProxyManager
 	authHandler.KubernetesTokenCacheManager = kubernetesTokenCacheManager
 	authHandler.OAuthService = server.OAuthService
 
@@ -116,10 +118,10 @@ func (server *Server) Start() error {
 	var endpointHandler = endpoints.NewHandler(requestBouncer)
 	endpointHandler.DataStore = server.DataStore
 	endpointHandler.FileService = server.FileService
-	endpointHandler.ProxyManager = proxyManager
+	endpointHandler.ProxyManager = server.ProxyManager
 	endpointHandler.SnapshotService = server.SnapshotService
-	endpointHandler.ProxyManager = proxyManager
 	endpointHandler.ReverseTunnelService = server.ReverseTunnelService
+	endpointHandler.ComposeStackManager = server.ComposeStackManager
 
 	var endpointEdgeHandler = endpointedge.NewHandler(requestBouncer)
 	endpointEdgeHandler.DataStore = server.DataStore
@@ -131,7 +133,7 @@ func (server *Server) Start() error {
 
 	var endpointProxyHandler = endpointproxy.NewHandler(requestBouncer)
 	endpointProxyHandler.DataStore = server.DataStore
-	endpointProxyHandler.ProxyManager = proxyManager
+	endpointProxyHandler.ProxyManager = server.ProxyManager
 	endpointProxyHandler.ReverseTunnelService = server.ReverseTunnelService
 
 	var fileHandler = file.NewHandler(filepath.Join(server.AssetsPath, "public"))
@@ -141,7 +143,7 @@ func (server *Server) Start() error {
 	var registryHandler = registries.NewHandler(requestBouncer)
 	registryHandler.DataStore = server.DataStore
 	registryHandler.FileService = server.FileService
-	registryHandler.ProxyManager = proxyManager
+	registryHandler.ProxyManager = server.ProxyManager
 
 	var resourceControlHandler = resourcecontrols.NewHandler(requestBouncer)
 	resourceControlHandler.DataStore = server.DataStore

api/libcompose/compose_stack.go

@@ -13,11 +13,12 @@ import (
 	"github.com/portainer/libcompose/lookup"
 	"github.com/portainer/libcompose/project"
 	"github.com/portainer/libcompose/project/options"
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 )
 
 const (
 	dockerClientVersion     = "1.24"
+	composeSyntaxMaxVersion = "2"
 )
 
 // ComposeStackManager represents a service for managing compose stacks.
@@ -58,6 +59,11 @@ func (manager *ComposeStackManager) createClient(endpoint *portainer.Endpoint) (
 	return client.NewDefaultFactory(clientOpts)
 }
 
+// ComposeSyntaxMaxVersion returns the maximum supported version of the docker compose syntax
+func (manager *ComposeStackManager) ComposeSyntaxMaxVersion() string {
+	return composeSyntaxMaxVersion
+}
+
 // Up will deploy a compose stack (equivalent of docker-compose up)
 func (manager *ComposeStackManager) Up(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
 

api/portainer.go

@@ -208,6 +208,7 @@ type (
 		EdgeKey                 string              `json:"EdgeKey"`
 		EdgeCheckinInterval     int                 `json:"EdgeCheckinInterval"`
 		Kubernetes              KubernetesData      `json:"Kubernetes"`
+		ComposeSyntaxMaxVersion string              `json:"ComposeSyntaxMaxVersion"`
 
 		// Deprecated fields
 		// Deprecated in DBVersion == 4
@@ -778,6 +779,7 @@ type (
 
 	// ComposeStackManager represents a service to manage Compose stacks
 	ComposeStackManager interface {
+		ComposeSyntaxMaxVersion() string
 		Up(stack *Stack, endpoint *Endpoint) error
 		Down(stack *Stack, endpoint *Endpoint) error
 	}
@@ -1126,6 +1128,8 @@ const (
 	APIVersion = "2.0.1"
 	// DBVersion is the version number of the Portainer database
 	DBVersion = 25
+	// ComposeSyntaxMaxVersion is a maximum supported version of the docker compose syntax
+	ComposeSyntaxMaxVersion = "3.9"
 	// AssetsServerURL represents the URL of the Portainer asset server
 	AssetsServerURL = "https://portainer-io-assets.sfo2.digitaloceanspaces.com"
 	// MessageOfTheDayURL represents the URL where Portainer MOTD message can be retrieved

app/portainer/views/stacks/create/createStackController.js

@@ -14,7 +14,8 @@ angular
     FormValidator,
     ResourceControlService,
     FormHelper,
-    CustomTemplateService
+    CustomTemplateService,
+    EndpointProvider
   ) {
     $scope.formValues = {
       Name: '',
@@ -166,6 +167,7 @@ angular
 
     async function initView() {
       var endpointMode = $scope.applicationState.endpoint.mode;
+      const endpointId = +$state.params.endpointId;
       $scope.state.StackType = 2;
       if (endpointMode.provider === 'DOCKER_SWARM_MODE' && endpointMode.role === 'MANAGER') {
         $scope.state.StackType = 1;
@@ -177,6 +179,13 @@ angular
       } catch (err) {
         Notifications.error('Failure', err, 'Unable to retrieve Custom Templates');
       }
+
+      try {
+        const endpoint = EndpointProvider.currentEndpoint();
+        $scope.composeSyntaxMaxVersion = endpoint.ComposeSyntaxMaxVersion;
+      } catch (err) {
+        Notifications.error('Failure', err, 'Unable to retrieve the ComposeSyntaxMaxVersion');
+      }
     }
 
     initView();

app/portainer/views/stacks/create/createstack.html

@@ -20,10 +20,15 @@
             <span class="col-sm-12 text-muted small" ng-if="state.StackType === 1">
               This stack will be deployed using the equivalent of the <code>docker stack deploy</code> command.
             </span>
-            <span class="col-sm-12 text-muted small" ng-if="state.StackType === 2">
+            <div class="col-sm-12 text-muted small" ng-if="state.StackType === 2 && composeSyntaxMaxVersion == 2">
-              This stack will be deployed using the equivalent of <code>docker-compose</code>. Only Compose file format version <b>2</b> is supported at the moment. <br /><br />
+              <div style="margin-bottom: 7px;">
+                This stack will be deployed using the equivalent of <code>docker-compose</code>. Only Compose file format version <b>2</b> is supported at the moment.
+              </div>
               <i class="fa fa-exclamation-circle orange-icon" aria-hidden="true" style="margin-right: 2px;"></i>
               Note: Due to a limitation of libcompose, the name of the stack will be standardized to remove all special characters and uppercase letters.
+            </div>
+            <span class="col-sm-12 text-muted small" ng-if="state.StackType === 2 && composeSyntaxMaxVersion > 2">
+              This stack will be deployed using <code>docker-compose</code>.
             </span>
           </div>
           <!-- build-method -->

app/portainer/views/stacks/edit/stack.html

@@ -94,6 +94,12 @@
             <uib-tab-heading> <i class="fa fa-pencil-alt space-right" aria-hidden="true"></i> Editor </uib-tab-heading>
             <form class="form-horizontal" ng-if="state.showEditorTab" style="margin-top: 10px;">
               <div class="form-group">
+                <span class="col-sm-12 text-muted small" style="margin-bottom: 7px;" ng-if="stackType == 2 && composeSyntaxMaxVersion == 2">
+                  This stack will be deployed using the equivalent of <code>docker-compose</code>. Only Compose file format version <b>2</b> is supported at the moment.
+                </span>
+                <span class="col-sm-12 text-muted small" style="margin-bottom: 7px;" ng-if="stackType == 2 && composeSyntaxMaxVersion > 2">
+                  This stack will be deployed using <code>docker-compose</code>.
+                </span>
                 <span class="col-sm-12 text-muted small">
                   You can get more information about Compose file format in the <a href="https://docs.docker.com/compose/compose-file/" target="_blank">official documentation</a>.
                 </span>

app/portainer/views/stacks/edit/stackController.js

@@ -359,7 +359,7 @@ angular.module('portainer.app').controller('StackController', [
         });
     }
 
-    function initView() {
+    async function initView() {
       var stackName = $transition$.params().name;
       $scope.stackName = stackName;
       var external = $transition$.params().external;
@@ -372,6 +372,15 @@ angular.module('portainer.app').controller('StackController', [
         var stackId = $transition$.params().id;
         loadStack(stackId);
       }
+
+      try {
+        const endpoint = EndpointProvider.currentEndpoint();
+        $scope.composeSyntaxMaxVersion = endpoint.ComposeSyntaxMaxVersion;
+      } catch (err) {
+        Notifications.error('Failure', err, 'Unable to retrieve the ComposeSyntaxMaxVersion');
+      }
+
+      $scope.stackType = $transition$.params().type;
     }
 
     initView();

build/download_docker_compose_binary.ps1

@@ -0,0 +1,8 @@
+param (
+  [string]$docker_compose_version
+)
+
+$ErrorActionPreference = "Stop";
+$ProgressPreference = "SilentlyContinue";
+
+Invoke-WebRequest -O "dist/docker-compose.exe" "https://github.com/docker/compose/releases/download/$($docker_compose_version)/docker-compose-Windows-x86_64.exe"

build/download_docker_compose_binary.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+PLATFORM=$1
+ARCH=$2
+DOCKER_COMPOSE_VERSION=$3
+
+if [ "${PLATFORM}" == 'linux' ] && [ "${ARCH}" == 'amd64' ]; then
+  wget -O "dist/docker-compose" "https://github.com/portainer/docker-compose-linux-amd64-static-binary/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose"
+  chmod +x "dist/docker-compose"
+elif [ "${PLATFORM}" == 'mac' ]; then
+  wget -O "dist/docker-compose" "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-Darwin-x86_64"
+  chmod +x "dist/docker-compose"
+fi
+
+exit 0

build/windows2016/nanoserver/Dockerfile

@@ -1,12 +1,24 @@
 FROM mcr.microsoft.com/windows/servercore:ltsc2019 as core
+
+ENV GIT_VERSION 2.30.0
+ENV GIT_PATCH_VERSION 2
+
+RUN powershell -Command $ErrorActionPreference = 'Stop' ; \
+    [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; \
+    Invoke-WebRequest $('https://github.com/git-for-windows/git/releases/download/v{0}.windows.{1}/MinGit-{0}.{1}-busybox-64-bit.zip' -f $env:GIT_VERSION, $env:GIT_PATCH_VERSION) -OutFile 'mingit.zip' -UseBasicParsing ; \
+    Expand-Archive mingit.zip -DestinationPath c:\mingit
+
 FROM mcr.microsoft.com/windows/nanoserver:1809-amd64
 
 USER ContainerAdministrator
 
 COPY --from=core /windows/system32/netapi32.dll /windows/system32/netapi32.dll
+COPY --from=core /mingit /mingit
 
 COPY dist /
 
+RUN setx /M path "C:\mingit\cmd;%path%"
+
 WORKDIR /
 
 EXPOSE 9000

gruntfile.js

@@ -19,6 +19,8 @@ module.exports = function (grunt) {
     binaries: {
       dockerLinuxVersion: '19.03.13',
       dockerWindowsVersion: '19-03-12',
+      dockerLinuxComposeVersion: '1.27.4',
+      dockerWindowsComposeVersion: '1.28.0',
       komposeVersion: 'v1.22.0',
       kubectlVersion: 'v1.18.0',
     },
@@ -37,6 +39,7 @@ module.exports = function (grunt) {
   grunt.registerTask('build:server', [
     'shell:build_binary:linux:' + arch,
     'shell:download_docker_binary:linux:' + arch,
+    'shell:download_docker_compose_binary:linux:' + arch,
     'shell:download_kompose_binary:linux:' + arch,
     'shell:download_kubectl_binary:linux:' + arch,
   ]);
@@ -63,6 +66,7 @@ module.exports = function (grunt) {
       'copy:assets',
       'shell:build_binary:' + p + ':' + a,
       'shell:download_docker_binary:' + p + ':' + a,
+      'shell:download_docker_compose_binary:' + p + ':' + a,
       'shell:download_kompose_binary:' + p + ':' + a,
       'shell:download_kubectl_binary:' + p + ':' + a,
       'webpack:prod',
@@ -77,6 +81,7 @@ module.exports = function (grunt) {
       'copy:assets',
       'shell:build_binary_azuredevops:' + p + ':' + a,
       'shell:download_docker_binary:' + p + ':' + a,
+      'shell:download_docker_compose_binary:' + p + ':' + a,
       'shell:download_kompose_binary:' + p + ':' + a,
       'shell:download_kubectl_binary:' + p + ':' + a,
       'webpack:prod',
@@ -138,6 +143,7 @@ gruntfile_cfg.shell = {
   download_docker_binary: { command: shell_download_docker_binary },
   download_kompose_binary: { command: shell_download_kompose_binary },
   download_kubectl_binary: { command: shell_download_kubectl_binary },
+  download_docker_compose_binary: { command: shell_download_docker_compose_binary },
   run_container: { command: shell_run_container },
   run_localserver: { command: shell_run_localserver, options: { async: true } },
   install_yarndeps: { command: shell_install_yarndeps },
@@ -171,7 +177,7 @@ function shell_run_container() {
     'docker rm -f portainer',
     'docker run -d -p 8000:8000 -p 9000:9000 -v $(pwd)/dist:/app -v ' +
       portainer_data +
-      ':/data -v /var/run/docker.sock:/var/run/docker.sock:z --name portainer portainer/base /app/portainer',
+      ':/data -v /var/run/docker.sock:/var/run/docker.sock:z -v /tmp:/tmp --name portainer portainer/base /app/portainer',
   ].join(';');
 }
 
@@ -204,6 +210,38 @@ function shell_download_docker_binary(p, a) {
   }
 }
 
+function shell_download_docker_compose_binary(p, a) {
+  console.log('request docker compose for ' + p + ':' + a);
+  var ps = { windows: 'win', darwin: 'mac' };
+  var as = { arm: 'armhf', arm64: 'aarch64' };
+  var ip = ps[p] || p;
+  var ia = as[a] || a;
+  console.log('download docker compose for ' + ip + ':' + ia);
+  var linuxBinaryVersion = '<%= binaries.dockerLinuxComposeVersion %>';
+  var windowsBinaryVersion = '<%= binaries.dockerWindowsComposeVersion %>';
+  console.log('download docker compose versions; Linux: ' + linuxBinaryVersion + ' Windows: ' + windowsBinaryVersion);
+
+  if (ip === 'linux' || ip === 'mac') {
+    return [
+      'if [ -f dist/docker-compose ]; then',
+      'echo "Docker Compose binary exists";',
+      'else',
+      'build/download_docker_compose_binary.sh ' + ip + ' ' + ia + ' ' + linuxBinaryVersion + ';',
+      'fi',
+    ].join(' ');
+  } else if (ip === 'win') {
+    return [
+      'powershell -Command "& {if (Test-Path -Path "dist/docker-compose.exe") {',
+      'Write-Host "Skipping download, Docker Compose binary exists"',
+      'return',
+      '} else {',
+      '& ".\\build\\download_docker_compose_binary.ps1" -docker_compose_version ' + windowsBinaryVersion + '',
+      '}}"',
+    ].join(' ');
+  }
+  console.log('docker compose is downloaded');
+}
+
 function shell_download_kompose_binary(p, a) {
   var binaryVersion = '<%= binaries.komposeVersion %>';
 

package.json

@@ -21,7 +21,8 @@
     "build:server": "grunt clean:server && grunt build:server",
     "build:client": "grunt clean:client && grunt build:client",
     "clean": "grunt clean:all",
-    "start": "grunt clean:all && grunt start",
+    "start": "grunt start",
+    "start:clean": "grunt clean:all && grunt start",
     "start:localserver": "grunt start:localserver",
     "start:server": "grunt clean:server && grunt start:server",
     "start:client": "grunt clean:client && grunt start:client",