mirror of https://github.com/portainer/portainer
fix(compose): merge default and in-place stack env vars [EE-2860] (#7076)
Dmitry Salakhov
GitHub
parent
dd4d126934
commit
33861a834b
|
|
@ -6,7 +6,6 @@ import (
|
|||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
|
@ -14,7 +13,6 @@ import (
|
|||
libstack "github.com/portainer/docker-compose-wrapper"
|
||||
"github.com/portainer/docker-compose-wrapper/compose"
|
||||
|
||||
"github.com/docker/cli/cli/compose/loader"
|
||||
portainer "github.com/portainer/portainer/api"
|
||||
"github.com/portainer/portainer/api/http/proxy"
|
||||
"github.com/portainer/portainer/api/http/proxy/factory"
|
||||
|
|
@ -56,13 +54,13 @@ func (manager *ComposeStackManager) Up(ctx context.Context, stack *portainer.Sta
|
|||
defer proxy.Close()
|
||||
}
|
||||
|
||||
envFilePath, err := createEnvFile(stack)
|
||||
envFile, err := createEnvFile(stack)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to create env file")
|
||||
}
|
||||
|
||||
filePaths := stackutils.GetStackFilePaths(stack)
|
||||
err = manager.deployer.Deploy(ctx, stack.ProjectPath, url, stack.Name, filePaths, envFilePath, forceRereate)
|
||||
err = manager.deployer.Deploy(ctx, stack.ProjectPath, url, stack.Name, filePaths, envFile, forceRereate)
|
||||
return errors.Wrap(err, "failed to deploy a stack")
|
||||
}
|
||||
|
||||
|
|
@ -76,12 +74,14 @@ func (manager *ComposeStackManager) Down(ctx context.Context, stack *portainer.S
|
|||
defer proxy.Close()
|
||||
}
|
||||
|
||||
if err := updateNetworkEnvFile(stack); err != nil {
|
||||
return err
|
||||
envFile, err := createEnvFile(stack)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to create env file")
|
||||
}
|
||||
|
||||
filePaths := stackutils.GetStackFilePaths(stack)
|
||||
err = manager.deployer.Remove(ctx, stack.ProjectPath, url, stack.Name, filePaths)
|
||||
|
||||
err = manager.deployer.Remove(ctx, stack.ProjectPath, url, stack.Name, filePaths, envFile)
|
||||
return errors.Wrap(err, "failed to remove a stack")
|
||||
}
|
||||
|
||||
|
|
@ -103,200 +103,42 @@ func (manager *ComposeStackManager) fetchEndpointProxy(endpoint *portainer.Endpo
|
|||
return fmt.Sprintf("tcp://127.0.0.1:%d", proxy.Port), proxy, nil
|
||||
}
|
||||
|
||||
// createEnvFile creates a file that would hold both "in-place" and default environment variables.
|
||||
// It will return the name of the file if the stack has "in-place" env vars, otherwise empty string.
|
||||
func createEnvFile(stack *portainer.Stack) (string, error) {
|
||||
// workaround for EE-1862. It will have to be removed when
|
||||
// docker/compose upgraded to v2.x.
|
||||
if err := createNetworkEnvFile(stack); err != nil {
|
||||
return "", errors.Wrap(err, "failed to create network env file")
|
||||
}
|
||||
|
||||
if stack.Env == nil || len(stack.Env) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
envFilePath := path.Join(stack.ProjectPath, "stack.env")
|
||||
|
||||
envfile, err := os.OpenFile(envFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer envfile.Close()
|
||||
|
||||
copyDefaultEnvFile(stack, envfile)
|
||||
|
||||
for _, v := range stack.Env {
|
||||
envfile.WriteString(fmt.Sprintf("%s=%s\n", v.Name, v.Value))
|
||||
}
|
||||
envfile.Close()
|
||||
|
||||
return "stack.env", nil
|
||||
}
|
||||
|
||||
func fileNotExist(filePath string) bool {
|
||||
if _, err := os.Stat(filePath); errors.Is(err, os.ErrNotExist) {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func updateNetworkEnvFile(stack *portainer.Stack) error {
|
||||
envFilePath := path.Join(stack.ProjectPath, ".env")
|
||||
stackFilePath := path.Join(stack.ProjectPath, "stack.env")
|
||||
if fileNotExist(envFilePath) {
|
||||
if fileNotExist(stackFilePath) {
|
||||
return nil
|
||||
}
|
||||
|
||||
flags := os.O_WRONLY | os.O_SYNC | os.O_CREATE
|
||||
envFile, err := os.OpenFile(envFilePath, flags, 0666)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer envFile.Close()
|
||||
|
||||
stackFile, err := os.Open(stackFilePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer stackFile.Close()
|
||||
|
||||
_, err = io.Copy(envFile, stackFile)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func createNetworkEnvFile(stack *portainer.Stack) error {
|
||||
networkNameSet := NewStringSet()
|
||||
|
||||
for _, filePath := range stackutils.GetStackFilePaths(stack) {
|
||||
networkNames, err := extractNetworkNames(filePath)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to extract network name")
|
||||
}
|
||||
|
||||
if networkNames == nil || networkNames.Len() == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
networkNameSet.Union(networkNames)
|
||||
}
|
||||
|
||||
for _, s := range networkNameSet.List() {
|
||||
if _, ok := os.LookupEnv(s); ok {
|
||||
networkNameSet.Remove(s)
|
||||
}
|
||||
}
|
||||
|
||||
if networkNameSet.Len() == 0 && stack.Env == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
envfile, err := os.OpenFile(path.Join(stack.ProjectPath, ".env"),
|
||||
os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
|
||||
// copyDefaultEnvFile copies the default .env file if it exists to the provided writer
|
||||
func copyDefaultEnvFile(stack *portainer.Stack, w io.Writer) {
|
||||
defaultEnvFile, err := os.Open(path.Join(path.Join(stack.ProjectPath, path.Dir(stack.EntryPoint)), ".env"))
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to open env file")
|
||||
// If cannot open a default file, then don't need to copy it.
|
||||
// We could as well stat it and check if it exists, but this is more efficient.
|
||||
return
|
||||
}
|
||||
|
||||
defer envfile.Close()
|
||||
defer defaultEnvFile.Close()
|
||||
|
||||
var scanEnvSettingFunc = func(name string) (string, bool) {
|
||||
if stack.Env != nil {
|
||||
for _, v := range stack.Env {
|
||||
if name == v.Name {
|
||||
return v.Value, true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return "", false
|
||||
if _, err = io.Copy(w, defaultEnvFile); err == nil {
|
||||
io.WriteString(w, "\n")
|
||||
}
|
||||
|
||||
for _, s := range networkNameSet.List() {
|
||||
if _, ok := scanEnvSettingFunc(s); !ok {
|
||||
stack.Env = append(stack.Env, portainer.Pair{
|
||||
Name: s,
|
||||
Value: "None",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if stack.Env != nil {
|
||||
for _, v := range stack.Env {
|
||||
envfile.WriteString(
|
||||
fmt.Sprintf("%s=%s\n", v.Name, v.Value))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func extractNetworkNames(filePath string) (StringSet, error) {
|
||||
if info, err := os.Stat(filePath); errors.Is(err,
|
||||
os.ErrNotExist) || info.IsDir() {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
stackFileContent, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to open yaml file")
|
||||
}
|
||||
|
||||
config, err := loader.ParseYAML(stackFileContent)
|
||||
if err != nil {
|
||||
// invalid stack file
|
||||
return nil, errors.Wrap(err, "invalid stack file")
|
||||
}
|
||||
|
||||
var version string
|
||||
if _, ok := config["version"]; ok {
|
||||
version, _ = config["version"].(string)
|
||||
}
|
||||
|
||||
var networks map[string]interface{}
|
||||
if value, ok := config["networks"]; ok {
|
||||
if value == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if networks, ok = value.(map[string]interface{}); !ok {
|
||||
return nil, nil
|
||||
}
|
||||
} else {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
networkContent, err := loader.LoadNetworks(networks, version)
|
||||
if err != nil {
|
||||
return nil, nil // skip the error
|
||||
}
|
||||
|
||||
re := regexp.MustCompile(`^\$\{?([^\}]+)\}?$`)
|
||||
networkNames := NewStringSet()
|
||||
|
||||
for _, v := range networkContent {
|
||||
matched := re.FindAllStringSubmatch(v.Name, -1)
|
||||
if matched != nil && matched[0] != nil {
|
||||
if strings.Contains(matched[0][1], ":-") {
|
||||
continue
|
||||
}
|
||||
|
||||
if strings.Contains(matched[0][1], "?") {
|
||||
continue
|
||||
}
|
||||
|
||||
if strings.Contains(matched[0][1], "-") {
|
||||
continue
|
||||
}
|
||||
|
||||
networkNames.Add(matched[0][1])
|
||||
}
|
||||
}
|
||||
|
||||
if networkNames.Len() == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return networkNames, nil
|
||||
// If couldn't copy the .env file, then ignore the error and try to continue
|
||||
}
|
||||
|
|
|
|||
|
|
@ -65,56 +65,22 @@ func Test_createEnvFile(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func Test_createNetworkEnvFile(t *testing.T) {
|
||||
func Test_createEnvFile_mergesDefultAndInplaceEnvVars(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
buf := []byte(`
|
||||
version: '3.6'
|
||||
services:
|
||||
nginx-example:
|
||||
image: nginx:latest
|
||||
networks:
|
||||
default:
|
||||
name: ${test}
|
||||
driver: bridge
|
||||
`)
|
||||
if err := ioutil.WriteFile(path.Join(dir,
|
||||
"docker-compose.yml"), buf, 0644); err != nil {
|
||||
t.Fatalf("Failed to create yaml file: %s", err)
|
||||
}
|
||||
|
||||
stackWithoutEnv := &portainer.Stack{
|
||||
os.WriteFile(path.Join(dir, ".env"), []byte("VAR1=VAL1\nVAR2=VAL2\n"), 0600)
|
||||
stack := &portainer.Stack{
|
||||
ProjectPath: dir,
|
||||
EntryPoint: "docker-compose.yml",
|
||||
Env: []portainer.Pair{},
|
||||
}
|
||||
|
||||
if err := createNetworkEnvFile(stackWithoutEnv); err != nil {
|
||||
t.Fatalf("Failed to create network env file: %s", err)
|
||||
}
|
||||
|
||||
content, err := ioutil.ReadFile(path.Join(dir, ".env"))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read network env file: %s", err)
|
||||
}
|
||||
|
||||
assert.Equal(t, "test=None\n", string(content))
|
||||
|
||||
stackWithEnv := &portainer.Stack{
|
||||
ProjectPath: dir,
|
||||
EntryPoint: "docker-compose.yml",
|
||||
Env: []portainer.Pair{
|
||||
{Name: "test", Value: "test-value"},
|
||||
{Name: "VAR1", Value: "NEW_VAL1"},
|
||||
{Name: "VAR3", Value: "VAL3"},
|
||||
},
|
||||
}
|
||||
result, err := createEnvFile(stack)
|
||||
assert.Equal(t, "stack.env", result)
|
||||
assert.NoError(t, err)
|
||||
assert.FileExists(t, path.Join(dir, "stack.env"))
|
||||
f, _ := os.Open(path.Join(dir, "stack.env"))
|
||||
content, _ := ioutil.ReadAll(f)
|
||||
|
||||
if err := createNetworkEnvFile(stackWithEnv); err != nil {
|
||||
t.Fatalf("Failed to create network env file: %s", err)
|
||||
}
|
||||
|
||||
content, err = ioutil.ReadFile(path.Join(dir, ".env"))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read network env file: %s", err)
|
||||
}
|
||||
|
||||
assert.Equal(t, "test=test-value\n", string(content))
|
||||
assert.Equal(t, []byte("VAR1=VAL1\nVAR2=VAL2\n\nVAR1=NEW_VAL1\nVAR3=VAL3\n"), content)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ require (
|
|||
github.com/koding/websocketproxy v0.0.0-20181220232114-7ed82d81a28c
|
||||
github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/portainer/docker-compose-wrapper v0.0.0-20220531190153-c597b853e410
|
||||
github.com/portainer/docker-compose-wrapper v0.0.0-20220703222411-e3cf664b39c6
|
||||
github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a
|
||||
github.com/portainer/libhelm v0.0.0-20210929000907-825e93d62108
|
||||
github.com/portainer/libhttp v0.0.0-20211208103139-07a5f798eb3f
|
||||
|
|
|
|||
|
|
@ -809,8 +809,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
|||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/portainer/docker-compose-wrapper v0.0.0-20220531190153-c597b853e410 h1:LjxLd8UGR8ae73ov/vLrt/0jedj/nh98XnONkr8DJj8=
|
||||
github.com/portainer/docker-compose-wrapper v0.0.0-20220531190153-c597b853e410/go.mod h1:WxDlJWZxCnicdLCPnLNEv7/gRhjeIVuCGmsv+iOPH3c=
|
||||
github.com/portainer/docker-compose-wrapper v0.0.0-20220703222411-e3cf664b39c6 h1:6VQZsYaJGfEq1LSKiNQ8HIW3olB04MpnW6HTnLnpMSQ=
|
||||
github.com/portainer/docker-compose-wrapper v0.0.0-20220703222411-e3cf664b39c6/go.mod h1:WxDlJWZxCnicdLCPnLNEv7/gRhjeIVuCGmsv+iOPH3c=
|
||||
github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a h1:qY8TbocN75n5PDl16o0uVr5MevtM5IhdwSelXEd4nFM=
|
||||
github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a/go.mod h1:n54EEIq+MM0NNtqLeCby8ljL+l275VpolXO0ibHegLE=
|
||||
github.com/portainer/libhelm v0.0.0-20210929000907-825e93d62108 h1:5e8KAnDa2G3cEHK7aV/ue8lOaoQwBZUzoALslwWkR04=
|
||||
|
|
|
|||
Loading…
Reference in New Issue