feat(edge-stack): per-device-configs-for-edge-stack EE-5461 (#9203)

api/filesystem/serialize_per_dev_configs.go

@@ -0,0 +1,106 @@
+package filesystem
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/portainer/portainer/api"
+)
+
+// FilterDirForPerDevConfigs filers the given dirEntries, returns entries for the given device
+// For given configPath A/B/C, return entries:
+//  1. all entries outside of dir A
+//  2. dir entries A, A/B, A/B/C
+//  3. For filterType file:
+//     file entries: A/B/C/<deviceName> and A/B/C/<deviceName>.*
+//  4. For filterType dir:
+//     dir entry:   A/B/C/<deviceName>
+//     all entries: A/B/C/<deviceName>/*
+func FilterDirForPerDevConfigs(dirEntries []DirEntry, deviceName, configPath string, filterType portainer.PerDevConfigsFilterType) []DirEntry {
+	var filteredDirEntries []DirEntry
+
+	for _, dirEntry := range dirEntries {
+		if shouldIncludeEntry(dirEntry, deviceName, configPath, filterType) {
+			filteredDirEntries = append(filteredDirEntries, dirEntry)
+		}
+	}
+
+	return filteredDirEntries
+}
+
+func shouldIncludeEntry(dirEntry DirEntry, deviceName, configPath string, filterType portainer.PerDevConfigsFilterType) bool {
+
+	// Include all entries outside of dir A
+	if !isInConfigRootDir(dirEntry, configPath) {
+		return true
+	}
+
+	// Include dir entries A, A/B, A/B/C
+	if isParentDir(dirEntry, configPath) {
+		return true
+	}
+
+	if filterType == portainer.PerDevConfigsTypeFile {
+		// Include file entries A/B/C/<deviceName> or A/B/C/<deviceName>.*
+		return shouldIncludeFile(dirEntry, deviceName, configPath)
+	}
+
+	// Include:
+	// dir entry A/B/C/<deviceName>
+	// all entries A/B/C/<deviceName>/*
+	return shouldIncludeDir(dirEntry, deviceName, configPath)
+}
+
+func isInConfigRootDir(dirEntry DirEntry, configPath string) bool {
+	// get the first element of the configPath
+	rootDir := strings.Split(configPath, string(os.PathSeparator))[0]
+
+	// return true if entry name starts with "A/"
+	return strings.HasPrefix(dirEntry.Name, appendTailSeparator(rootDir))
+}
+
+func isParentDir(dirEntry DirEntry, configPath string) bool {
+	if dirEntry.IsFile {
+		return false
+	}
+
+	// return true for dir entries A, A/B, A/B/C
+	return strings.HasPrefix(appendTailSeparator(configPath), appendTailSeparator(dirEntry.Name))
+}
+
+func shouldIncludeFile(dirEntry DirEntry, deviceName, configPath string) bool {
+	if !dirEntry.IsFile {
+		return false
+	}
+
+	// example: A/B/C/<deviceName>
+	filterEqual := filepath.Join(configPath, deviceName)
+
+	// example: A/B/C/<deviceName>/
+	filterPrefix := fmt.Sprintf("%s.", filterEqual)
+
+	// include file entries: A/B/C/<deviceName> or A/B/C/<deviceName>.*
+	return dirEntry.Name == filterEqual || strings.HasPrefix(dirEntry.Name, filterPrefix)
+}
+
+func shouldIncludeDir(dirEntry DirEntry, deviceName, configPath string) bool {
+	// example: A/B/C/'/<deviceName>
+	filterEqual := filepath.Join(configPath, deviceName)
+
+	// example: A/B/C/<deviceName>/
+	filterPrefix := appendTailSeparator(filterEqual)
+
+	// include dir entry: A/B/C/<deviceName>
+	if !dirEntry.IsFile && dirEntry.Name == filterEqual {
+		return true
+	}
+
+	// include all entries A/B/C/<deviceName>/*
+	return strings.HasPrefix(dirEntry.Name, filterPrefix)
+}
+
+func appendTailSeparator(path string) string {
+	return fmt.Sprintf("%s%c", path, os.PathSeparator)
+}

api/git/azure.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/go-git/go-git/v5/plumbing/filemode"
 	"github.com/portainer/portainer/api/archive"
 	"github.com/portainer/portainer/api/crypto"
 	gittypes "github.com/portainer/portainer/api/git/types"
@@ -491,6 +492,7 @@ func (a *azureClient) listFiles(ctx context.Context, opt fetchOption) ([]string,
 	var tree struct {
 		TreeEntries []struct {
 			RelativePath string `json:"relativePath"`
+			Mode         string `json:"mode"`
 		} `json:"treeEntries"`
 	}
 
@@ -500,7 +502,11 @@ func (a *azureClient) listFiles(ctx context.Context, opt fetchOption) ([]string,
 
 	var allPaths []string
 	for _, treeEntry := range tree.TreeEntries {
-		allPaths = append(allPaths, treeEntry.RelativePath)
+		mode, _ := filemode.New(treeEntry.Mode)
+		isDir := filemode.Dir == mode
+		if opt.dirOnly == isDir {
+			allPaths = append(allPaths, treeEntry.RelativePath)
+		}
 	}
 
 	return allPaths, nil

api/git/azure_integration_test.go

@@ -247,7 +247,7 @@ func TestService_ListFiles_Azure(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, tt.extensions, false)
+			paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, false, tt.extensions, false)
 			if tt.expect.shouldFail {
 				assert.Error(t, err)
 				if tt.expect.err != nil {
@@ -270,8 +270,8 @@ func TestService_ListFiles_Azure_Concurrently(t *testing.T) {
 	username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
 	service := newService(context.TODO(), repositoryCacheSize, 200*time.Millisecond)
 
-	go service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, []string{}, false)
+	go service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, false, []string{}, false)
-	service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, []string{}, false)
+	service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, false, []string{}, false)
 
 	time.Sleep(2 * time.Second)
 }

api/git/git.go

@@ -9,6 +9,7 @@ import (
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/filemode"
 	"github.com/go-git/go-git/v5/plumbing/object"
 	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
 	"github.com/go-git/go-git/v5/storage/memory"
@@ -165,10 +166,18 @@ func (c *gitClient) listFiles(ctx context.Context, opt fetchOption) ([]string, e
 	}
 
 	var allPaths []string
-	tree.Files().ForEach(func(f *object.File) error {
+	w := object.NewTreeWalker(tree, true, nil)
-		allPaths = append(allPaths, f.Name)
+	for {
-		return nil
+		name, entry, err := w.Next()
-	})
+		if err != nil {
+			break
+		}
+
+		isDir := entry.Mode == filemode.Dir
+		if opt.dirOnly == isDir {
+			allPaths = append(allPaths, name)
+		}
+	}
 
 	return allPaths, nil
 }

api/git/git_integration_test.go

@@ -202,7 +202,7 @@ func TestService_ListFiles_GitHub(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, tt.extensions, false)
+			paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, false, tt.extensions, false)
 			if tt.expect.shouldFail {
 				assert.Error(t, err)
 				if tt.expect.err != nil {
@@ -226,8 +226,8 @@ func TestService_ListFiles_Github_Concurrently(t *testing.T) {
 	username := getRequiredValue(t, "GITHUB_USERNAME")
 	service := newService(context.TODO(), repositoryCacheSize, 200*time.Millisecond)
 
-	go service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false)
+	go service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, false, []string{}, false)
-	service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false)
+	service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, false, []string{}, false)
 
 	time.Sleep(2 * time.Second)
 }
@@ -241,7 +241,7 @@ func TestService_purgeCache_Github(t *testing.T) {
 	service := NewService(context.TODO())
 
 	service.ListRefs(repositoryUrl, username, accessToken, false, false)
-	service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false)
+	service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, false, []string{}, false)
 
 	assert.Equal(t, 1, service.repoRefCache.Len())
 	assert.Equal(t, 1, service.repoFileCache.Len())
@@ -262,7 +262,7 @@ func TestService_purgeCacheByTTL_Github(t *testing.T) {
 	service := newService(context.TODO(), 2, 40*timeout)
 
 	service.ListRefs(repositoryUrl, username, accessToken, false, false)
-	service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false)
+	service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, false, []string{}, false)
 	assert.Equal(t, 1, service.repoRefCache.Len())
 	assert.Equal(t, 1, service.repoFileCache.Len())
 
@@ -316,12 +316,12 @@ func TestService_HardRefresh_ListRefs_And_RemoveAllCaches_GitHub(t *testing.T) {
 	assert.GreaterOrEqual(t, len(refs), 1)
 	assert.Equal(t, 1, service.repoRefCache.Len())
 
-	files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false)
+	files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, false, []string{}, false)
 	assert.NoError(t, err)
 	assert.GreaterOrEqual(t, len(files), 1)
 	assert.Equal(t, 1, service.repoFileCache.Len())
 
-	files, err = service.ListFiles(repositoryUrl, "refs/heads/test", username, accessToken, false, []string{}, false)
+	files, err = service.ListFiles(repositoryUrl, "refs/heads/test", username, accessToken, false, false, []string{}, false)
 	assert.NoError(t, err)
 	assert.GreaterOrEqual(t, len(files), 1)
 	assert.Equal(t, 2, service.repoFileCache.Len())
@@ -344,12 +344,12 @@ func TestService_HardRefresh_ListFiles_GitHub(t *testing.T) {
 	accessToken := getRequiredValue(t, "GITHUB_PAT")
 	username := getRequiredValue(t, "GITHUB_USERNAME")
 	repositoryUrl := privateGitRepoURL
-	files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false)
+	files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, false, []string{}, false)
 	assert.NoError(t, err)
 	assert.GreaterOrEqual(t, len(files), 1)
 	assert.Equal(t, 1, service.repoFileCache.Len())
 
-	_, err = service.ListFiles(repositoryUrl, "refs/heads/main", username, "fake-token", true, []string{}, false)
+	_, err = service.ListFiles(repositoryUrl, "refs/heads/main", username, "fake-token", false, true, []string{}, false)
 	assert.Error(t, err)
 	assert.Equal(t, 0, service.repoFileCache.Len())
 }

api/git/service.go

@@ -28,6 +28,7 @@ type baseOption struct {
 type fetchOption struct {
 	baseOption
 	referenceName string
+	dirOnly       bool
 }
 
 // cloneOption allows to add a history truncated to the specified number of commits
@@ -224,8 +225,8 @@ func (service *Service) ListRefs(repositoryURL, username, password string, hardR
 
 // ListFiles will list all the files of the target repository with specific extensions.
 // If extension is not provided, it will list all the files under the target repository
-func (service *Service) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string, tlsSkipVerify bool) ([]string, error) {
+func (service *Service) ListFiles(repositoryURL, referenceName, username, password string, dirOnly, hardRefresh bool, includedExts []string, tlsSkipVerify bool) ([]string, error) {
-	repoKey := generateCacheKey(repositoryURL, referenceName, username, password, strconv.FormatBool(tlsSkipVerify))
+	repoKey := generateCacheKey(repositoryURL, referenceName, username, password, strconv.FormatBool(tlsSkipVerify), strconv.FormatBool(dirOnly))
 
 	if service.cacheEnabled && hardRefresh {
 		// Should remove the cache explicitly, so that the following normal list can show the correct result
@@ -254,6 +255,7 @@ func (service *Service) ListFiles(repositoryURL, referenceName, username, passwo
 			tlsSkipVerify: tlsSkipVerify,
 		},
 		referenceName: referenceName,
+		dirOnly:       dirOnly,
 	}
 
 	var (

api/internal/testhelpers/git_service.go

@@ -27,6 +27,6 @@ func (g *gitService) ListRefs(repositoryURL, username, password string, hardRefr
 	return nil, nil
 }
 
-func (g *gitService) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string, tlsSkipVerify bool) ([]string, error) {
+func (g *gitService) ListFiles(repositoryURL, referenceName, username, password string, dirOnly, hardRefresh bool, includedExts []string, tlsSkipVerify bool) ([]string, error) {
 	return nil, nil
 }

api/portainer.go

@@ -1429,7 +1429,7 @@ type (
 		CloneRepository(destination string, repositoryURL, referenceName, username, password string, tlsSkipVerify bool) error
 		LatestCommitID(repositoryURL, referenceName, username, password string, tlsSkipVerify bool) (string, error)
 		ListRefs(repositoryURL, username, password string, hardRefresh bool, tlsSkipVerify bool) ([]string, error)
-		ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includeExts []string, tlsSkipVerify bool) ([]string, error)
+		ListFiles(repositoryURL, referenceName, username, password string, dirOnly, hardRefresh bool, includeExts []string, tlsSkipVerify bool) ([]string, error)
 	}
 
 	// OpenAMTService represents a service for managing OpenAMT
@@ -2040,3 +2040,10 @@ const (
 	AzurePathContainerGroups = "/subscriptions/*/providers/Microsoft.ContainerInstance/containerGroups"
 	AzurePathContainerGroup  = "/subscriptions/*/resourceGroups/*/providers/Microsoft.ContainerInstance/containerGroups/*"
 )
+
+type PerDevConfigsFilterType string
+
+const (
+	PerDevConfigsTypeFile PerDevConfigsFilterType = "file"
+	PerDevConfigsTypeDir  PerDevConfigsFilterType = "dir"
+)