From db61fb149b419fe12cf61c486f178648206e8c79 Mon Sep 17 00:00:00 2001 From: cmeng Date: Fri, 14 Jul 2023 06:41:47 +1200 Subject: [PATCH] feat(edge-stack): per-device-configs-for-edge-stack EE-5461 (#9203) --- api/filesystem/serialize_per_dev_configs.go | 106 ++++++++++++++++++++ api/git/azure.go | 8 +- api/git/azure_integration_test.go | 6 +- api/git/git.go | 17 +++- api/git/git_integration_test.go | 18 ++-- api/git/service.go | 6 +- api/internal/testhelpers/git_service.go | 2 +- api/portainer.go | 9 +- 8 files changed, 151 insertions(+), 21 deletions(-) create mode 100644 api/filesystem/serialize_per_dev_configs.go diff --git a/api/filesystem/serialize_per_dev_configs.go b/api/filesystem/serialize_per_dev_configs.go new file mode 100644 index 000000000..241ecb0ba --- /dev/null +++ b/api/filesystem/serialize_per_dev_configs.go @@ -0,0 +1,106 @@ +package filesystem + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/portainer/portainer/api" +) + +// FilterDirForPerDevConfigs filers the given dirEntries, returns entries for the given device +// For given configPath A/B/C, return entries: +// 1. all entries outside of dir A +// 2. dir entries A, A/B, A/B/C +// 3. For filterType file: +// file entries: A/B/C/ and A/B/C/.* +// 4. For filterType dir: +// dir entry: A/B/C/ +// all entries: A/B/C//* +func FilterDirForPerDevConfigs(dirEntries []DirEntry, deviceName, configPath string, filterType portainer.PerDevConfigsFilterType) []DirEntry { + var filteredDirEntries []DirEntry + + for _, dirEntry := range dirEntries { + if shouldIncludeEntry(dirEntry, deviceName, configPath, filterType) { + filteredDirEntries = append(filteredDirEntries, dirEntry) + } + } + + return filteredDirEntries +} + +func shouldIncludeEntry(dirEntry DirEntry, deviceName, configPath string, filterType portainer.PerDevConfigsFilterType) bool { + + // Include all entries outside of dir A + if !isInConfigRootDir(dirEntry, configPath) { + return true + } + + // Include dir entries A, A/B, A/B/C + if isParentDir(dirEntry, configPath) { + return true + } + + if filterType == portainer.PerDevConfigsTypeFile { + // Include file entries A/B/C/ or A/B/C/.* + return shouldIncludeFile(dirEntry, deviceName, configPath) + } + + // Include: + // dir entry A/B/C/ + // all entries A/B/C//* + return shouldIncludeDir(dirEntry, deviceName, configPath) +} + +func isInConfigRootDir(dirEntry DirEntry, configPath string) bool { + // get the first element of the configPath + rootDir := strings.Split(configPath, string(os.PathSeparator))[0] + + // return true if entry name starts with "A/" + return strings.HasPrefix(dirEntry.Name, appendTailSeparator(rootDir)) +} + +func isParentDir(dirEntry DirEntry, configPath string) bool { + if dirEntry.IsFile { + return false + } + + // return true for dir entries A, A/B, A/B/C + return strings.HasPrefix(appendTailSeparator(configPath), appendTailSeparator(dirEntry.Name)) +} + +func shouldIncludeFile(dirEntry DirEntry, deviceName, configPath string) bool { + if !dirEntry.IsFile { + return false + } + + // example: A/B/C/ + filterEqual := filepath.Join(configPath, deviceName) + + // example: A/B/C// + filterPrefix := fmt.Sprintf("%s.", filterEqual) + + // include file entries: A/B/C/ or A/B/C/.* + return dirEntry.Name == filterEqual || strings.HasPrefix(dirEntry.Name, filterPrefix) +} + +func shouldIncludeDir(dirEntry DirEntry, deviceName, configPath string) bool { + // example: A/B/C/'/ + filterEqual := filepath.Join(configPath, deviceName) + + // example: A/B/C// + filterPrefix := appendTailSeparator(filterEqual) + + // include dir entry: A/B/C/ + if !dirEntry.IsFile && dirEntry.Name == filterEqual { + return true + } + + // include all entries A/B/C//* + return strings.HasPrefix(dirEntry.Name, filterPrefix) +} + +func appendTailSeparator(path string) string { + return fmt.Sprintf("%s%c", path, os.PathSeparator) +} diff --git a/api/git/azure.go b/api/git/azure.go index 03a531dc2..b85ffe7b4 100644 --- a/api/git/azure.go +++ b/api/git/azure.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "github.com/go-git/go-git/v5/plumbing/filemode" "github.com/portainer/portainer/api/archive" "github.com/portainer/portainer/api/crypto" gittypes "github.com/portainer/portainer/api/git/types" @@ -491,6 +492,7 @@ func (a *azureClient) listFiles(ctx context.Context, opt fetchOption) ([]string, var tree struct { TreeEntries []struct { RelativePath string `json:"relativePath"` + Mode string `json:"mode"` } `json:"treeEntries"` } @@ -500,7 +502,11 @@ func (a *azureClient) listFiles(ctx context.Context, opt fetchOption) ([]string, var allPaths []string for _, treeEntry := range tree.TreeEntries { - allPaths = append(allPaths, treeEntry.RelativePath) + mode, _ := filemode.New(treeEntry.Mode) + isDir := filemode.Dir == mode + if opt.dirOnly == isDir { + allPaths = append(allPaths, treeEntry.RelativePath) + } } return allPaths, nil diff --git a/api/git/azure_integration_test.go b/api/git/azure_integration_test.go index 214acc31d..3e297a129 100644 --- a/api/git/azure_integration_test.go +++ b/api/git/azure_integration_test.go @@ -247,7 +247,7 @@ func TestService_ListFiles_Azure(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, tt.extensions, false) + paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, false, tt.extensions, false) if tt.expect.shouldFail { assert.Error(t, err) if tt.expect.err != nil { @@ -270,8 +270,8 @@ func TestService_ListFiles_Azure_Concurrently(t *testing.T) { username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME") service := newService(context.TODO(), repositoryCacheSize, 200*time.Millisecond) - go service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, []string{}, false) - service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, []string{}, false) + go service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, false, []string{}, false) + service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, false, []string{}, false) time.Sleep(2 * time.Second) } diff --git a/api/git/git.go b/api/git/git.go index 7024deaf8..66fe17d8a 100644 --- a/api/git/git.go +++ b/api/git/git.go @@ -9,6 +9,7 @@ import ( "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/filemode" "github.com/go-git/go-git/v5/plumbing/object" githttp "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/go-git/go-git/v5/storage/memory" @@ -165,10 +166,18 @@ func (c *gitClient) listFiles(ctx context.Context, opt fetchOption) ([]string, e } var allPaths []string - tree.Files().ForEach(func(f *object.File) error { - allPaths = append(allPaths, f.Name) - return nil - }) + w := object.NewTreeWalker(tree, true, nil) + for { + name, entry, err := w.Next() + if err != nil { + break + } + + isDir := entry.Mode == filemode.Dir + if opt.dirOnly == isDir { + allPaths = append(allPaths, name) + } + } return allPaths, nil } diff --git a/api/git/git_integration_test.go b/api/git/git_integration_test.go index 7b5bda468..add10afd6 100644 --- a/api/git/git_integration_test.go +++ b/api/git/git_integration_test.go @@ -202,7 +202,7 @@ func TestService_ListFiles_GitHub(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, tt.extensions, false) + paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, false, tt.extensions, false) if tt.expect.shouldFail { assert.Error(t, err) if tt.expect.err != nil { @@ -226,8 +226,8 @@ func TestService_ListFiles_Github_Concurrently(t *testing.T) { username := getRequiredValue(t, "GITHUB_USERNAME") service := newService(context.TODO(), repositoryCacheSize, 200*time.Millisecond) - go service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false) - service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false) + go service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, false, []string{}, false) + service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, false, []string{}, false) time.Sleep(2 * time.Second) } @@ -241,7 +241,7 @@ func TestService_purgeCache_Github(t *testing.T) { service := NewService(context.TODO()) service.ListRefs(repositoryUrl, username, accessToken, false, false) - service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false) + service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, false, []string{}, false) assert.Equal(t, 1, service.repoRefCache.Len()) assert.Equal(t, 1, service.repoFileCache.Len()) @@ -262,7 +262,7 @@ func TestService_purgeCacheByTTL_Github(t *testing.T) { service := newService(context.TODO(), 2, 40*timeout) service.ListRefs(repositoryUrl, username, accessToken, false, false) - service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false) + service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, false, []string{}, false) assert.Equal(t, 1, service.repoRefCache.Len()) assert.Equal(t, 1, service.repoFileCache.Len()) @@ -316,12 +316,12 @@ func TestService_HardRefresh_ListRefs_And_RemoveAllCaches_GitHub(t *testing.T) { assert.GreaterOrEqual(t, len(refs), 1) assert.Equal(t, 1, service.repoRefCache.Len()) - files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false) + files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, false, []string{}, false) assert.NoError(t, err) assert.GreaterOrEqual(t, len(files), 1) assert.Equal(t, 1, service.repoFileCache.Len()) - files, err = service.ListFiles(repositoryUrl, "refs/heads/test", username, accessToken, false, []string{}, false) + files, err = service.ListFiles(repositoryUrl, "refs/heads/test", username, accessToken, false, false, []string{}, false) assert.NoError(t, err) assert.GreaterOrEqual(t, len(files), 1) assert.Equal(t, 2, service.repoFileCache.Len()) @@ -344,12 +344,12 @@ func TestService_HardRefresh_ListFiles_GitHub(t *testing.T) { accessToken := getRequiredValue(t, "GITHUB_PAT") username := getRequiredValue(t, "GITHUB_USERNAME") repositoryUrl := privateGitRepoURL - files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false) + files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, false, []string{}, false) assert.NoError(t, err) assert.GreaterOrEqual(t, len(files), 1) assert.Equal(t, 1, service.repoFileCache.Len()) - _, err = service.ListFiles(repositoryUrl, "refs/heads/main", username, "fake-token", true, []string{}, false) + _, err = service.ListFiles(repositoryUrl, "refs/heads/main", username, "fake-token", false, true, []string{}, false) assert.Error(t, err) assert.Equal(t, 0, service.repoFileCache.Len()) } diff --git a/api/git/service.go b/api/git/service.go index 6b08e2d19..7b680bdb1 100644 --- a/api/git/service.go +++ b/api/git/service.go @@ -28,6 +28,7 @@ type baseOption struct { type fetchOption struct { baseOption referenceName string + dirOnly bool } // cloneOption allows to add a history truncated to the specified number of commits @@ -224,8 +225,8 @@ func (service *Service) ListRefs(repositoryURL, username, password string, hardR // ListFiles will list all the files of the target repository with specific extensions. // If extension is not provided, it will list all the files under the target repository -func (service *Service) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string, tlsSkipVerify bool) ([]string, error) { - repoKey := generateCacheKey(repositoryURL, referenceName, username, password, strconv.FormatBool(tlsSkipVerify)) +func (service *Service) ListFiles(repositoryURL, referenceName, username, password string, dirOnly, hardRefresh bool, includedExts []string, tlsSkipVerify bool) ([]string, error) { + repoKey := generateCacheKey(repositoryURL, referenceName, username, password, strconv.FormatBool(tlsSkipVerify), strconv.FormatBool(dirOnly)) if service.cacheEnabled && hardRefresh { // Should remove the cache explicitly, so that the following normal list can show the correct result @@ -254,6 +255,7 @@ func (service *Service) ListFiles(repositoryURL, referenceName, username, passwo tlsSkipVerify: tlsSkipVerify, }, referenceName: referenceName, + dirOnly: dirOnly, } var ( diff --git a/api/internal/testhelpers/git_service.go b/api/internal/testhelpers/git_service.go index fae9caec6..6af1b6459 100644 --- a/api/internal/testhelpers/git_service.go +++ b/api/internal/testhelpers/git_service.go @@ -27,6 +27,6 @@ func (g *gitService) ListRefs(repositoryURL, username, password string, hardRefr return nil, nil } -func (g *gitService) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string, tlsSkipVerify bool) ([]string, error) { +func (g *gitService) ListFiles(repositoryURL, referenceName, username, password string, dirOnly, hardRefresh bool, includedExts []string, tlsSkipVerify bool) ([]string, error) { return nil, nil } diff --git a/api/portainer.go b/api/portainer.go index 47eb27217..c342b5a9f 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -1429,7 +1429,7 @@ type ( CloneRepository(destination string, repositoryURL, referenceName, username, password string, tlsSkipVerify bool) error LatestCommitID(repositoryURL, referenceName, username, password string, tlsSkipVerify bool) (string, error) ListRefs(repositoryURL, username, password string, hardRefresh bool, tlsSkipVerify bool) ([]string, error) - ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includeExts []string, tlsSkipVerify bool) ([]string, error) + ListFiles(repositoryURL, referenceName, username, password string, dirOnly, hardRefresh bool, includeExts []string, tlsSkipVerify bool) ([]string, error) } // OpenAMTService represents a service for managing OpenAMT @@ -2040,3 +2040,10 @@ const ( AzurePathContainerGroups = "/subscriptions/*/providers/Microsoft.ContainerInstance/containerGroups" AzurePathContainerGroup = "/subscriptions/*/resourceGroups/*/providers/Microsoft.ContainerInstance/containerGroups/*" ) + +type PerDevConfigsFilterType string + +const ( + PerDevConfigsTypeFile PerDevConfigsFilterType = "file" + PerDevConfigsTypeDir PerDevConfigsFilterType = "dir" +)