github
/
portainer
mirror of https://github.com/portainer/portainer
1
0
Fork 0
Code Issues Releases Wiki Activity

feat(edge-stack): per-device-configs-for-edge-stack EE-5461 (#9203)

pull/9208/head
cmeng 2023-07-14 06:41:47 +12:00 committed by GitHub
parent 76b871d8a0
commit db61fb149b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 151 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
api/filesystem/serialize_per_dev_configs.go Normal file
View File

@ -0,0 +1,106 @@
package filesystem
import (
"fmt"
"os"
"path/filepath"
"strings"
"github.com/portainer/portainer/api"
)
// FilterDirForPerDevConfigs filers the given dirEntries, returns entries for the given device
// For given configPath A/B/C, return entries:
// 1. all entries outside of dir A
// 2. dir entries A, A/B, A/B/C
// 3. For filterType file:
// file entries: A/B/C/<deviceName> and A/B/C/<deviceName>.*
// 4. For filterType dir:
// dir entry: A/B/C/<deviceName>
// all entries: A/B/C/<deviceName>/*
func FilterDirForPerDevConfigs(dirEntries []DirEntry, deviceName, configPath string, filterType portainer.PerDevConfigsFilterType) []DirEntry {
var filteredDirEntries []DirEntry
for _, dirEntry := range dirEntries {
if shouldIncludeEntry(dirEntry, deviceName, configPath, filterType) {
filteredDirEntries = append(filteredDirEntries, dirEntry)
}
}
return filteredDirEntries
}
func shouldIncludeEntry(dirEntry DirEntry, deviceName, configPath string, filterType portainer.PerDevConfigsFilterType) bool {
// Include all entries outside of dir A
if !isInConfigRootDir(dirEntry, configPath) {
return true
}
// Include dir entries A, A/B, A/B/C
if isParentDir(dirEntry, configPath) {
return true
}
if filterType == portainer.PerDevConfigsTypeFile {
// Include file entries A/B/C/<deviceName> or A/B/C/<deviceName>.*
return shouldIncludeFile(dirEntry, deviceName, configPath)
}
// Include:
// dir entry A/B/C/<deviceName>
// all entries A/B/C/<deviceName>/*
return shouldIncludeDir(dirEntry, deviceName, configPath)
}
func isInConfigRootDir(dirEntry DirEntry, configPath string) bool {
// get the first element of the configPath
rootDir := strings.Split(configPath, string(os.PathSeparator))[0]
// return true if entry name starts with "A/"
return strings.HasPrefix(dirEntry.Name, appendTailSeparator(rootDir))
}
func isParentDir(dirEntry DirEntry, configPath string) bool {
if dirEntry.IsFile {
return false
}
// return true for dir entries A, A/B, A/B/C
return strings.HasPrefix(appendTailSeparator(configPath), appendTailSeparator(dirEntry.Name))
}
func shouldIncludeFile(dirEntry DirEntry, deviceName, configPath string) bool {
if !dirEntry.IsFile {
return false
}
// example: A/B/C/<deviceName>
filterEqual := filepath.Join(configPath, deviceName)
// example: A/B/C/<deviceName>/
filterPrefix := fmt.Sprintf("%s.", filterEqual)
// include file entries: A/B/C/<deviceName> or A/B/C/<deviceName>.*
return dirEntry.Name == filterEqual || strings.HasPrefix(dirEntry.Name, filterPrefix)
}
func shouldIncludeDir(dirEntry DirEntry, deviceName, configPath string) bool {
// example: A/B/C/'/<deviceName>
filterEqual := filepath.Join(configPath, deviceName)
// example: A/B/C/<deviceName>/
filterPrefix := appendTailSeparator(filterEqual)
// include dir entry: A/B/C/<deviceName>
if !dirEntry.IsFile && dirEntry.Name == filterEqual {
return true
}
// include all entries A/B/C/<deviceName>/*
return strings.HasPrefix(dirEntry.Name, filterPrefix)
}
func appendTailSeparator(path string) string {
return fmt.Sprintf("%s%c", path, os.PathSeparator)
}

8
api/git/azure.go
View File

@ -11,6 +11,7 @@ import (
"strings"
"time"
"github.com/go-git/go-git/v5/plumbing/filemode"
"github.com/portainer/portainer/api/archive"
"github.com/portainer/portainer/api/crypto"
gittypes "github.com/portainer/portainer/api/git/types"
@ -491,6 +492,7 @@ func (a *azureClient) listFiles(ctx context.Context, opt fetchOption) ([]string,
var tree struct {
TreeEntries []struct {
RelativePath string `json:"relativePath"`
Mode string `json:"mode"`
} `json:"treeEntries"`
}
@ -500,7 +502,11 @@ func (a *azureClient) listFiles(ctx context.Context, opt fetchOption) ([]string,
var allPaths []string
for _, treeEntry := range tree.TreeEntries {
allPaths = append(allPaths, treeEntry.RelativePath)
mode, _ := filemode.New(treeEntry.Mode)
isDir := filemode.Dir == mode
if opt.dirOnly == isDir {
allPaths = append(allPaths, treeEntry.RelativePath)
}
}
return allPaths, nil

6
api/git/azure_integration_test.go
View File

@ -247,7 +247,7 @@ func TestService_ListFiles_Azure(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, tt.extensions, false)
paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, false, tt.extensions, false)
if tt.expect.shouldFail {
assert.Error(t, err)
if tt.expect.err != nil {
@ -270,8 +270,8 @@ func TestService_ListFiles_Azure_Concurrently(t *testing.T) {
username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
service := newService(context.TODO(), repositoryCacheSize, 200*time.Millisecond)
go service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, []string{}, false)
service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, []string{}, false)
go service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, false, []string{}, false)
service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, false, []string{}, false)
time.Sleep(2 * time.Second)
}

17
api/git/git.go
View File

@ -9,6 +9,7 @@ import (
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/filemode"
"github.com/go-git/go-git/v5/plumbing/object"
githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/go-git/go-git/v5/storage/memory"
@ -165,10 +166,18 @@ func (c *gitClient) listFiles(ctx context.Context, opt fetchOption) ([]string, e
}
var allPaths []string
tree.Files().ForEach(func(f *object.File) error {
allPaths = append(allPaths, f.Name)
return nil
})
w := object.NewTreeWalker(tree, true, nil)
for {
name, entry, err := w.Next()
if err != nil {
break
}
isDir := entry.Mode == filemode.Dir
if opt.dirOnly == isDir {
allPaths = append(allPaths, name)
}
}
return allPaths, nil
}

18
api/git/git_integration_test.go
View File

@ -202,7 +202,7 @@ func TestService_ListFiles_GitHub(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, tt.extensions, false)
paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, false, tt.extensions, false)
if tt.expect.shouldFail {
assert.Error(t, err)
if tt.expect.err != nil {
@ -226,8 +226,8 @@ func TestService_ListFiles_Github_Concurrently(t *testing.T) {
username := getRequiredValue(t, "GITHUB_USERNAME")
service := newService(context.TODO(), repositoryCacheSize, 200*time.Millisecond)
go service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false)
service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false)
go service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, false, []string{}, false)
service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, false, []string{}, false)
time.Sleep(2 * time.Second)
}
@ -241,7 +241,7 @@ func TestService_purgeCache_Github(t *testing.T) {
service := NewService(context.TODO())
service.ListRefs(repositoryUrl, username, accessToken, false, false)
service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false)
service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, false, []string{}, false)
assert.Equal(t, 1, service.repoRefCache.Len())
assert.Equal(t, 1, service.repoFileCache.Len())
@ -262,7 +262,7 @@ func TestService_purgeCacheByTTL_Github(t *testing.T) {
service := newService(context.TODO(), 2, 40*timeout)
service.ListRefs(repositoryUrl, username, accessToken, false, false)
service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false)
service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, false, []string{}, false)
assert.Equal(t, 1, service.repoRefCache.Len())
assert.Equal(t, 1, service.repoFileCache.Len())
@ -316,12 +316,12 @@ func TestService_HardRefresh_ListRefs_And_RemoveAllCaches_GitHub(t *testing.T) {
assert.GreaterOrEqual(t, len(refs), 1)
assert.Equal(t, 1, service.repoRefCache.Len())
files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false)
files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, false, []string{}, false)
assert.NoError(t, err)
assert.GreaterOrEqual(t, len(files), 1)
assert.Equal(t, 1, service.repoFileCache.Len())
files, err = service.ListFiles(repositoryUrl, "refs/heads/test", username, accessToken, false, []string{}, false)
files, err = service.ListFiles(repositoryUrl, "refs/heads/test", username, accessToken, false, false, []string{}, false)
assert.NoError(t, err)
assert.GreaterOrEqual(t, len(files), 1)
assert.Equal(t, 2, service.repoFileCache.Len())
@ -344,12 +344,12 @@ func TestService_HardRefresh_ListFiles_GitHub(t *testing.T) {
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
repositoryUrl := privateGitRepoURL
files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false)
files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, false, []string{}, false)
assert.NoError(t, err)
assert.GreaterOrEqual(t, len(files), 1)
assert.Equal(t, 1, service.repoFileCache.Len())
_, err = service.ListFiles(repositoryUrl, "refs/heads/main", username, "fake-token", true, []string{}, false)
_, err = service.ListFiles(repositoryUrl, "refs/heads/main", username, "fake-token", false, true, []string{}, false)
assert.Error(t, err)
assert.Equal(t, 0, service.repoFileCache.Len())
}

6
api/git/service.go
View File

@ -28,6 +28,7 @@ type baseOption struct {
type fetchOption struct {
baseOption
referenceName string
dirOnly bool
}
// cloneOption allows to add a history truncated to the specified number of commits
@ -224,8 +225,8 @@ func (service *Service) ListRefs(repositoryURL, username, password string, hardR
// ListFiles will list all the files of the target repository with specific extensions.
// If extension is not provided, it will list all the files under the target repository
func (service *Service) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string, tlsSkipVerify bool) ([]string, error) {
repoKey := generateCacheKey(repositoryURL, referenceName, username, password, strconv.FormatBool(tlsSkipVerify))
func (service *Service) ListFiles(repositoryURL, referenceName, username, password string, dirOnly, hardRefresh bool, includedExts []string, tlsSkipVerify bool) ([]string, error) {
repoKey := generateCacheKey(repositoryURL, referenceName, username, password, strconv.FormatBool(tlsSkipVerify), strconv.FormatBool(dirOnly))
if service.cacheEnabled && hardRefresh {
// Should remove the cache explicitly, so that the following normal list can show the correct result
@ -254,6 +255,7 @@ func (service *Service) ListFiles(repositoryURL, referenceName, username, passwo
tlsSkipVerify: tlsSkipVerify,
},
referenceName: referenceName,
dirOnly: dirOnly,
}
var (

2
api/internal/testhelpers/git_service.go
View File

@ -27,6 +27,6 @@ func (g *gitService) ListRefs(repositoryURL, username, password string, hardRefr
return nil, nil
}
func (g *gitService) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string, tlsSkipVerify bool) ([]string, error) {
func (g *gitService) ListFiles(repositoryURL, referenceName, username, password string, dirOnly, hardRefresh bool, includedExts []string, tlsSkipVerify bool) ([]string, error) {
return nil, nil
}

9
api/portainer.go
View File

@ -1429,7 +1429,7 @@ type (
CloneRepository(destination string, repositoryURL, referenceName, username, password string, tlsSkipVerify bool) error
LatestCommitID(repositoryURL, referenceName, username, password string, tlsSkipVerify bool) (string, error)
ListRefs(repositoryURL, username, password string, hardRefresh bool, tlsSkipVerify bool) ([]string, error)
ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includeExts []string, tlsSkipVerify bool) ([]string, error)
ListFiles(repositoryURL, referenceName, username, password string, dirOnly, hardRefresh bool, includeExts []string, tlsSkipVerify bool) ([]string, error)
}
// OpenAMTService represents a service for managing OpenAMT
@ -2040,3 +2040,10 @@ const (
AzurePathContainerGroups = "/subscriptions/*/providers/Microsoft.ContainerInstance/containerGroups"
AzurePathContainerGroup = "/subscriptions/*/resourceGroups/*/providers/Microsoft.ContainerInstance/containerGroups/*"
)
type PerDevConfigsFilterType string
const (
PerDevConfigsTypeFile PerDevConfigsFilterType = "file"
PerDevConfigsTypeDir PerDevConfigsFilterType = "dir"
)