From 33861a834b64045d819fbcc7752ccd3a8d8b6db1 Mon Sep 17 00:00:00 2001 From: Dmitry Salakhov Date: Mon, 4 Jul 2022 13:16:04 +1200 Subject: [PATCH] fix(compose): merge default and in-place stack env vars [EE-2860] (#7076) --- api/exec/compose_stack.go | 202 ++++----------------------------- api/exec/compose_stack_test.go | 58 ++-------- api/go.mod | 2 +- api/go.sum | 4 +- 4 files changed, 37 insertions(+), 229 deletions(-) diff --git a/api/exec/compose_stack.go b/api/exec/compose_stack.go index f80af924d..386cc8c6e 100644 --- a/api/exec/compose_stack.go +++ b/api/exec/compose_stack.go @@ -6,7 +6,6 @@ import ( "io" "os" "path" - "regexp" "strings" "github.com/pkg/errors" @@ -14,7 +13,6 @@ import ( libstack "github.com/portainer/docker-compose-wrapper" "github.com/portainer/docker-compose-wrapper/compose" - "github.com/docker/cli/cli/compose/loader" portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/http/proxy" "github.com/portainer/portainer/api/http/proxy/factory" @@ -56,13 +54,13 @@ func (manager *ComposeStackManager) Up(ctx context.Context, stack *portainer.Sta defer proxy.Close() } - envFilePath, err := createEnvFile(stack) + envFile, err := createEnvFile(stack) if err != nil { return errors.Wrap(err, "failed to create env file") } filePaths := stackutils.GetStackFilePaths(stack) - err = manager.deployer.Deploy(ctx, stack.ProjectPath, url, stack.Name, filePaths, envFilePath, forceRereate) + err = manager.deployer.Deploy(ctx, stack.ProjectPath, url, stack.Name, filePaths, envFile, forceRereate) return errors.Wrap(err, "failed to deploy a stack") } @@ -76,12 +74,14 @@ func (manager *ComposeStackManager) Down(ctx context.Context, stack *portainer.S defer proxy.Close() } - if err := updateNetworkEnvFile(stack); err != nil { - return err + envFile, err := createEnvFile(stack) + if err != nil { + return errors.Wrap(err, "failed to create env file") } filePaths := stackutils.GetStackFilePaths(stack) - err = manager.deployer.Remove(ctx, stack.ProjectPath, url, stack.Name, filePaths) + + err = manager.deployer.Remove(ctx, stack.ProjectPath, url, stack.Name, filePaths, envFile) return errors.Wrap(err, "failed to remove a stack") } @@ -103,200 +103,42 @@ func (manager *ComposeStackManager) fetchEndpointProxy(endpoint *portainer.Endpo return fmt.Sprintf("tcp://127.0.0.1:%d", proxy.Port), proxy, nil } +// createEnvFile creates a file that would hold both "in-place" and default environment variables. +// It will return the name of the file if the stack has "in-place" env vars, otherwise empty string. func createEnvFile(stack *portainer.Stack) (string, error) { - // workaround for EE-1862. It will have to be removed when - // docker/compose upgraded to v2.x. - if err := createNetworkEnvFile(stack); err != nil { - return "", errors.Wrap(err, "failed to create network env file") - } - if stack.Env == nil || len(stack.Env) == 0 { return "", nil } envFilePath := path.Join(stack.ProjectPath, "stack.env") - envfile, err := os.OpenFile(envFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { return "", err } + defer envfile.Close() + + copyDefaultEnvFile(stack, envfile) for _, v := range stack.Env { envfile.WriteString(fmt.Sprintf("%s=%s\n", v.Name, v.Value)) } - envfile.Close() return "stack.env", nil } -func fileNotExist(filePath string) bool { - if _, err := os.Stat(filePath); errors.Is(err, os.ErrNotExist) { - return true - } - - return false -} - -func updateNetworkEnvFile(stack *portainer.Stack) error { - envFilePath := path.Join(stack.ProjectPath, ".env") - stackFilePath := path.Join(stack.ProjectPath, "stack.env") - if fileNotExist(envFilePath) { - if fileNotExist(stackFilePath) { - return nil - } - - flags := os.O_WRONLY | os.O_SYNC | os.O_CREATE - envFile, err := os.OpenFile(envFilePath, flags, 0666) - if err != nil { - return err - } - - defer envFile.Close() - - stackFile, err := os.Open(stackFilePath) - if err != nil { - return err - } - - defer stackFile.Close() - - _, err = io.Copy(envFile, stackFile) - return err - } - - return nil -} - -func createNetworkEnvFile(stack *portainer.Stack) error { - networkNameSet := NewStringSet() - - for _, filePath := range stackutils.GetStackFilePaths(stack) { - networkNames, err := extractNetworkNames(filePath) - if err != nil { - return errors.Wrap(err, "failed to extract network name") - } - - if networkNames == nil || networkNames.Len() == 0 { - continue - } - - networkNameSet.Union(networkNames) - } - - for _, s := range networkNameSet.List() { - if _, ok := os.LookupEnv(s); ok { - networkNameSet.Remove(s) - } - } - - if networkNameSet.Len() == 0 && stack.Env == nil { - return nil - } - - envfile, err := os.OpenFile(path.Join(stack.ProjectPath, ".env"), - os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) +// copyDefaultEnvFile copies the default .env file if it exists to the provided writer +func copyDefaultEnvFile(stack *portainer.Stack, w io.Writer) { + defaultEnvFile, err := os.Open(path.Join(path.Join(stack.ProjectPath, path.Dir(stack.EntryPoint)), ".env")) if err != nil { - return errors.Wrap(err, "failed to open env file") + // If cannot open a default file, then don't need to copy it. + // We could as well stat it and check if it exists, but this is more efficient. + return } - defer envfile.Close() + defer defaultEnvFile.Close() - var scanEnvSettingFunc = func(name string) (string, bool) { - if stack.Env != nil { - for _, v := range stack.Env { - if name == v.Name { - return v.Value, true - } - } - } - - return "", false + if _, err = io.Copy(w, defaultEnvFile); err == nil { + io.WriteString(w, "\n") } - - for _, s := range networkNameSet.List() { - if _, ok := scanEnvSettingFunc(s); !ok { - stack.Env = append(stack.Env, portainer.Pair{ - Name: s, - Value: "None", - }) - } - } - - if stack.Env != nil { - for _, v := range stack.Env { - envfile.WriteString( - fmt.Sprintf("%s=%s\n", v.Name, v.Value)) - } - } - - return nil -} - -func extractNetworkNames(filePath string) (StringSet, error) { - if info, err := os.Stat(filePath); errors.Is(err, - os.ErrNotExist) || info.IsDir() { - return nil, nil - } - - stackFileContent, err := os.ReadFile(filePath) - if err != nil { - return nil, errors.Wrap(err, "failed to open yaml file") - } - - config, err := loader.ParseYAML(stackFileContent) - if err != nil { - // invalid stack file - return nil, errors.Wrap(err, "invalid stack file") - } - - var version string - if _, ok := config["version"]; ok { - version, _ = config["version"].(string) - } - - var networks map[string]interface{} - if value, ok := config["networks"]; ok { - if value == nil { - return nil, nil - } - - if networks, ok = value.(map[string]interface{}); !ok { - return nil, nil - } - } else { - return nil, nil - } - - networkContent, err := loader.LoadNetworks(networks, version) - if err != nil { - return nil, nil // skip the error - } - - re := regexp.MustCompile(`^\$\{?([^\}]+)\}?$`) - networkNames := NewStringSet() - - for _, v := range networkContent { - matched := re.FindAllStringSubmatch(v.Name, -1) - if matched != nil && matched[0] != nil { - if strings.Contains(matched[0][1], ":-") { - continue - } - - if strings.Contains(matched[0][1], "?") { - continue - } - - if strings.Contains(matched[0][1], "-") { - continue - } - - networkNames.Add(matched[0][1]) - } - } - - if networkNames.Len() == 0 { - return nil, nil - } - - return networkNames, nil + // If couldn't copy the .env file, then ignore the error and try to continue } diff --git a/api/exec/compose_stack_test.go b/api/exec/compose_stack_test.go index e827cdb19..733eb7fb2 100644 --- a/api/exec/compose_stack_test.go +++ b/api/exec/compose_stack_test.go @@ -65,56 +65,22 @@ func Test_createEnvFile(t *testing.T) { } } -func Test_createNetworkEnvFile(t *testing.T) { +func Test_createEnvFile_mergesDefultAndInplaceEnvVars(t *testing.T) { dir := t.TempDir() - buf := []byte(` -version: '3.6' -services: - nginx-example: - image: nginx:latest -networks: - default: - name: ${test} - driver: bridge -`) - if err := ioutil.WriteFile(path.Join(dir, - "docker-compose.yml"), buf, 0644); err != nil { - t.Fatalf("Failed to create yaml file: %s", err) - } - - stackWithoutEnv := &portainer.Stack{ + os.WriteFile(path.Join(dir, ".env"), []byte("VAR1=VAL1\nVAR2=VAL2\n"), 0600) + stack := &portainer.Stack{ ProjectPath: dir, - EntryPoint: "docker-compose.yml", - Env: []portainer.Pair{}, - } - - if err := createNetworkEnvFile(stackWithoutEnv); err != nil { - t.Fatalf("Failed to create network env file: %s", err) - } - - content, err := ioutil.ReadFile(path.Join(dir, ".env")) - if err != nil { - t.Fatalf("Failed to read network env file: %s", err) - } - - assert.Equal(t, "test=None\n", string(content)) - - stackWithEnv := &portainer.Stack{ - ProjectPath: dir, - EntryPoint: "docker-compose.yml", Env: []portainer.Pair{ - {Name: "test", Value: "test-value"}, + {Name: "VAR1", Value: "NEW_VAL1"}, + {Name: "VAR3", Value: "VAL3"}, }, } + result, err := createEnvFile(stack) + assert.Equal(t, "stack.env", result) + assert.NoError(t, err) + assert.FileExists(t, path.Join(dir, "stack.env")) + f, _ := os.Open(path.Join(dir, "stack.env")) + content, _ := ioutil.ReadAll(f) - if err := createNetworkEnvFile(stackWithEnv); err != nil { - t.Fatalf("Failed to create network env file: %s", err) - } - - content, err = ioutil.ReadFile(path.Join(dir, ".env")) - if err != nil { - t.Fatalf("Failed to read network env file: %s", err) - } - - assert.Equal(t, "test=test-value\n", string(content)) + assert.Equal(t, []byte("VAR1=VAL1\nVAR2=VAL2\n\nVAR1=NEW_VAL1\nVAR3=VAL3\n"), content) } diff --git a/api/go.mod b/api/go.mod index 59d14de08..00b0c64b4 100644 --- a/api/go.mod +++ b/api/go.mod @@ -32,7 +32,7 @@ require ( github.com/koding/websocketproxy v0.0.0-20181220232114-7ed82d81a28c github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6 github.com/pkg/errors v0.9.1 - github.com/portainer/docker-compose-wrapper v0.0.0-20220531190153-c597b853e410 + github.com/portainer/docker-compose-wrapper v0.0.0-20220703222411-e3cf664b39c6 github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a github.com/portainer/libhelm v0.0.0-20210929000907-825e93d62108 github.com/portainer/libhttp v0.0.0-20211208103139-07a5f798eb3f diff --git a/api/go.sum b/api/go.sum index 9cb1ed1ee..c86ba9835 100644 --- a/api/go.sum +++ b/api/go.sum @@ -809,8 +809,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/portainer/docker-compose-wrapper v0.0.0-20220531190153-c597b853e410 h1:LjxLd8UGR8ae73ov/vLrt/0jedj/nh98XnONkr8DJj8= -github.com/portainer/docker-compose-wrapper v0.0.0-20220531190153-c597b853e410/go.mod h1:WxDlJWZxCnicdLCPnLNEv7/gRhjeIVuCGmsv+iOPH3c= +github.com/portainer/docker-compose-wrapper v0.0.0-20220703222411-e3cf664b39c6 h1:6VQZsYaJGfEq1LSKiNQ8HIW3olB04MpnW6HTnLnpMSQ= +github.com/portainer/docker-compose-wrapper v0.0.0-20220703222411-e3cf664b39c6/go.mod h1:WxDlJWZxCnicdLCPnLNEv7/gRhjeIVuCGmsv+iOPH3c= github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a h1:qY8TbocN75n5PDl16o0uVr5MevtM5IhdwSelXEd4nFM= github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a/go.mod h1:n54EEIq+MM0NNtqLeCby8ljL+l275VpolXO0ibHegLE= github.com/portainer/libhelm v0.0.0-20210929000907-825e93d62108 h1:5e8KAnDa2G3cEHK7aV/ue8lOaoQwBZUzoALslwWkR04=