From 2c05496962c73026fd5246ef784440074004fa58 Mon Sep 17 00:00:00 2001 From: andres-portainer <91705312+andres-portainer@users.noreply.github.com> Date: Sat, 15 Mar 2025 10:09:22 -0300 Subject: [PATCH] feat(edgeconfigs): parse .env config files for interpolation BE-11673 (#514) --- api/filesystem/serialize_per_dev_configs.go | 33 +++-- .../serialize_per_dev_configs_test.go | 133 ++++++++++-------- 2 files changed, 96 insertions(+), 70 deletions(-) diff --git a/api/filesystem/serialize_per_dev_configs.go b/api/filesystem/serialize_per_dev_configs.go index 55881885a..7ae653934 100644 --- a/api/filesystem/serialize_per_dev_configs.go +++ b/api/filesystem/serialize_per_dev_configs.go @@ -15,15 +15,19 @@ type MultiFilterArgs []struct { } // MultiFilterDirForPerDevConfigs filers the given dirEntries with multiple filter args, returns the merged entries for the given device -func MultiFilterDirForPerDevConfigs(dirEntries []DirEntry, configPath string, multiFilterArgs MultiFilterArgs) []DirEntry { +func MultiFilterDirForPerDevConfigs(dirEntries []DirEntry, configPath string, multiFilterArgs MultiFilterArgs) ([]DirEntry, []string) { var filteredDirEntries []DirEntry + var envFiles []string + for _, multiFilterArg := range multiFilterArgs { - tmp := FilterDirForPerDevConfigs(dirEntries, multiFilterArg.FilterKey, configPath, multiFilterArg.FilterType) + tmp, efs := FilterDirForPerDevConfigs(dirEntries, multiFilterArg.FilterKey, configPath, multiFilterArg.FilterType) filteredDirEntries = append(filteredDirEntries, tmp...) + + envFiles = append(envFiles, efs...) } - return deduplicate(filteredDirEntries) + return deduplicate(filteredDirEntries), envFiles } func deduplicate(dirEntries []DirEntry) []DirEntry { @@ -32,8 +36,7 @@ func deduplicate(dirEntries []DirEntry) []DirEntry { marks := make(map[string]struct{}) for _, dirEntry := range dirEntries { - _, ok := marks[dirEntry.Name] - if !ok { + if _, ok := marks[dirEntry.Name]; !ok { marks[dirEntry.Name] = struct{}{} deduplicatedDirEntries = append(deduplicatedDirEntries, dirEntry) } @@ -50,20 +53,25 @@ func deduplicate(dirEntries []DirEntry) []DirEntry { // 3. For filterType dir: // dir entry: A/B/C/ // all entries: A/B/C//* -func FilterDirForPerDevConfigs(dirEntries []DirEntry, deviceName, configPath string, filterType portainer.PerDevConfigsFilterType) []DirEntry { +func FilterDirForPerDevConfigs(dirEntries []DirEntry, deviceName, configPath string, filterType portainer.PerDevConfigsFilterType) ([]DirEntry, []string) { var filteredDirEntries []DirEntry + var envFiles []string + for _, dirEntry := range dirEntries { if shouldIncludeEntry(dirEntry, deviceName, configPath, filterType) { filteredDirEntries = append(filteredDirEntries, dirEntry) + + if shouldParseEnvVars(dirEntry, deviceName, configPath, filterType) { + envFiles = append(envFiles, dirEntry.Name) + } } } - return filteredDirEntries + return filteredDirEntries, envFiles } func shouldIncludeEntry(dirEntry DirEntry, deviceName, configPath string, filterType portainer.PerDevConfigsFilterType) bool { - // Include all entries outside of dir A if !isInConfigDir(dirEntry, configPath) { return true @@ -120,6 +128,15 @@ func shouldIncludeDir(dirEntry DirEntry, deviceName, configPath string) bool { return strings.HasPrefix(dirEntry.Name, filterPrefix) } +func shouldParseEnvVars(dirEntry DirEntry, deviceName, configPath string, filterType portainer.PerDevConfigsFilterType) bool { + if !dirEntry.IsFile { + return false + } + + return isInConfigDir(dirEntry, configPath) && + filepath.Base(dirEntry.Name) == deviceName+".env" +} + func appendTailSeparator(path string) string { return fmt.Sprintf("%s%c", path, os.PathSeparator) } diff --git a/api/filesystem/serialize_per_dev_configs_test.go b/api/filesystem/serialize_per_dev_configs_test.go index 55a1a4643..2330db68d 100644 --- a/api/filesystem/serialize_per_dev_configs_test.go +++ b/api/filesystem/serialize_per_dev_configs_test.go @@ -4,14 +4,17 @@ import ( "testing" portainer "github.com/portainer/portainer/api" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestMultiFilterDirForPerDevConfigs(t *testing.T) { - type args struct { - dirEntries []DirEntry - configPath string - multiFilterArgs MultiFilterArgs + f := func(dirEntries []DirEntry, configPath string, multiFilterArgs MultiFilterArgs, wantDirEntries []DirEntry) { + t.Helper() + + dirEntries, _ = MultiFilterDirForPerDevConfigs(dirEntries, configPath, multiFilterArgs) + require.Equal(t, wantDirEntries, dirEntries) } baseDirEntries := []DirEntry{ @@ -26,69 +29,75 @@ func TestMultiFilterDirForPerDevConfigs(t *testing.T) { {"configs/folder2/config2", "", true, 420}, } - tests := []struct { - name string - args args - want []DirEntry - }{ - { - name: "filter file1", - args: args{ - baseDirEntries, - "configs", - MultiFilterArgs{{"file1", portainer.PerDevConfigsTypeFile}}, - }, - want: []DirEntry{baseDirEntries[0], baseDirEntries[1], baseDirEntries[2], baseDirEntries[3]}, + // Filter file1 + f( + baseDirEntries, + "configs", + MultiFilterArgs{{"file1", portainer.PerDevConfigsTypeFile}}, + []DirEntry{baseDirEntries[0], baseDirEntries[1], baseDirEntries[2], baseDirEntries[3]}, + ) + + // Filter folder1 + f( + baseDirEntries, + "configs", + MultiFilterArgs{{"folder1", portainer.PerDevConfigsTypeDir}}, + []DirEntry{baseDirEntries[0], baseDirEntries[1], baseDirEntries[2], baseDirEntries[5], baseDirEntries[6]}, + ) + + // Filter file1 and folder1 + f( + baseDirEntries, + "configs", + MultiFilterArgs{{"folder1", portainer.PerDevConfigsTypeDir}}, + []DirEntry{baseDirEntries[0], baseDirEntries[1], baseDirEntries[2], baseDirEntries[5], baseDirEntries[6]}, + ) + + // Filter file1 and file2 + f( + baseDirEntries, + "configs", + MultiFilterArgs{ + {"file1", portainer.PerDevConfigsTypeFile}, + {"file2", portainer.PerDevConfigsTypeFile}, }, - { - name: "filter folder1", - args: args{ - baseDirEntries, - "configs", - MultiFilterArgs{{"folder1", portainer.PerDevConfigsTypeDir}}, - }, - want: []DirEntry{baseDirEntries[0], baseDirEntries[1], baseDirEntries[2], baseDirEntries[5], baseDirEntries[6]}, - }, - { - name: "filter file1 and folder1", - args: args{ - baseDirEntries, - "configs", - MultiFilterArgs{{"folder1", portainer.PerDevConfigsTypeDir}}, - }, - want: []DirEntry{baseDirEntries[0], baseDirEntries[1], baseDirEntries[2], baseDirEntries[5], baseDirEntries[6]}, - }, - { - name: "filter file1 and file2", - args: args{ - baseDirEntries, - "configs", - MultiFilterArgs{ - {"file1", portainer.PerDevConfigsTypeFile}, - {"file2", portainer.PerDevConfigsTypeFile}, - }, - }, - want: []DirEntry{baseDirEntries[0], baseDirEntries[1], baseDirEntries[2], baseDirEntries[3], baseDirEntries[4]}, - }, - { - name: "filter folder1 and folder2", - args: args{ - baseDirEntries, - "configs", - MultiFilterArgs{ - {"folder1", portainer.PerDevConfigsTypeDir}, - {"folder2", portainer.PerDevConfigsTypeDir}, - }, - }, - want: []DirEntry{baseDirEntries[0], baseDirEntries[1], baseDirEntries[2], baseDirEntries[5], baseDirEntries[6], baseDirEntries[7], baseDirEntries[8]}, + []DirEntry{baseDirEntries[0], baseDirEntries[1], baseDirEntries[2], baseDirEntries[3], baseDirEntries[4]}, + ) + + // Filter folder1 and folder2 + f( + baseDirEntries, + "configs", + MultiFilterArgs{ + {"folder1", portainer.PerDevConfigsTypeDir}, + {"folder2", portainer.PerDevConfigsTypeDir}, }, + []DirEntry{baseDirEntries[0], baseDirEntries[1], baseDirEntries[2], baseDirEntries[5], baseDirEntries[6], baseDirEntries[7], baseDirEntries[8]}, + ) +} + +func TestMultiFilterDirForPerDevConfigsEnvFiles(t *testing.T) { + f := func(dirEntries []DirEntry, configPath string, multiFilterArgs MultiFilterArgs, wantEnvFiles []string) { + t.Helper() + + _, envFiles := MultiFilterDirForPerDevConfigs(dirEntries, configPath, multiFilterArgs) + require.Equal(t, wantEnvFiles, envFiles) } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equalf(t, tt.want, MultiFilterDirForPerDevConfigs(tt.args.dirEntries, tt.args.configPath, tt.args.multiFilterArgs), "MultiFilterDirForPerDevConfigs(%v, %v, %v)", tt.args.dirEntries, tt.args.configPath, tt.args.multiFilterArgs) - }) + baseDirEntries := []DirEntry{ + {".env", "", true, 420}, + {"docker-compose.yaml", "", true, 420}, + {"configs", "", false, 420}, + {"configs/edge-id/edge-id.env", "", true, 420}, } + + f( + baseDirEntries, + "configs", + MultiFilterArgs{{"edge-id", portainer.PerDevConfigsTypeDir}}, + []string{"configs/edge-id/edge-id.env"}, + ) + } func TestIsInConfigDir(t *testing.T) {