diff --git a/api/git/git.go b/api/git/git.go index 66fe17d8a..1bdc0a3d7 100644 --- a/api/git/git.go +++ b/api/git/git.go @@ -143,6 +143,7 @@ func (c *gitClient) listFiles(ctx context.Context, opt fetchOption) ([]string, e ReferenceName: plumbing.ReferenceName(opt.referenceName), Auth: getAuth(opt.username, opt.password), InsecureSkipTLS: opt.tlsSkipVerify, + Tags: git.NoTags, } repo, err := git.Clone(memory.NewStorage(), nil, cloneOption) @@ -166,7 +167,10 @@ func (c *gitClient) listFiles(ctx context.Context, opt fetchOption) ([]string, e } var allPaths []string + w := object.NewTreeWalker(tree, true, nil) + defer w.Close() + for { name, entry, err := w.Next() if err != nil { diff --git a/api/git/git_test.go b/api/git/git_test.go index 1a824e7f7..0bce90d56 100644 --- a/api/git/git_test.go +++ b/api/git/git_test.go @@ -91,6 +91,29 @@ func Test_latestCommitID(t *testing.T) { assert.Equal(t, "68dcaa7bd452494043c64252ab90db0f98ecf8d2", id) } +func Test_ListRefs(t *testing.T) { + service := Service{git: NewGitClient(true)} + + repositoryURL := setup(t) + + fs, err := service.ListRefs(repositoryURL, "", "", false, false) + + assert.NoError(t, err) + assert.Equal(t, []string{"refs/heads/main"}, fs) +} + +func Test_ListFiles(t *testing.T) { + service := Service{git: NewGitClient(true)} + + repositoryURL := setup(t) + referenceName := "refs/heads/main" + + fs, err := service.ListFiles(repositoryURL, referenceName, "", "", false, false, []string{".yml"}, false) + + assert.NoError(t, err) + assert.Equal(t, []string{"docker-compose.yml"}, fs) +} + func getCommitHistoryLength(t *testing.T, err error, dir string) int { repo, err := git.PlainOpen(dir) if err != nil { diff --git a/api/git/service.go b/api/git/service.go index 7b680bdb1..5919941bb 100644 --- a/api/git/service.go +++ b/api/git/service.go @@ -9,6 +9,7 @@ import ( lru "github.com/hashicorp/golang-lru" "github.com/rs/zerolog/log" + "golang.org/x/sync/singleflight" ) const ( @@ -223,11 +224,23 @@ func (service *Service) ListRefs(repositoryURL, username, password string, hardR return refs, nil } +var singleflightGroup = &singleflight.Group{} + // ListFiles will list all the files of the target repository with specific extensions. // If extension is not provided, it will list all the files under the target repository func (service *Service) ListFiles(repositoryURL, referenceName, username, password string, dirOnly, hardRefresh bool, includedExts []string, tlsSkipVerify bool) ([]string, error) { repoKey := generateCacheKey(repositoryURL, referenceName, username, password, strconv.FormatBool(tlsSkipVerify), strconv.FormatBool(dirOnly)) + fs, err, _ := singleflightGroup.Do(repoKey, func() (any, error) { + return service.listFiles(repositoryURL, referenceName, username, password, dirOnly, hardRefresh, tlsSkipVerify) + }) + + return filterFiles(fs.([]string), includedExts), err +} + +func (service *Service) listFiles(repositoryURL, referenceName, username, password string, dirOnly, hardRefresh bool, tlsSkipVerify bool) ([]string, error) { + repoKey := generateCacheKey(repositoryURL, referenceName, username, password, strconv.FormatBool(tlsSkipVerify), strconv.FormatBool(dirOnly)) + if service.cacheEnabled && hardRefresh { // Should remove the cache explicitly, so that the following normal list can show the correct result service.repoFileCache.Remove(repoKey) @@ -235,14 +248,9 @@ func (service *Service) ListFiles(repositoryURL, referenceName, username, passwo if service.repoFileCache != nil { // lookup the files cache first - cache, ok := service.repoFileCache.Get(repoKey) - if ok { - files, success := cache.([]string) - if success { - // For the case while searching files in a repository without include extensions for the first time, - // but with include extensions for the second time - includedFiles := filterFiles(files, includedExts) - return includedFiles, nil + if cache, ok := service.repoFileCache.Get(repoKey); ok { + if files, ok := cache.([]string); ok { + return files, nil } } } @@ -274,12 +282,11 @@ func (service *Service) ListFiles(repositoryURL, referenceName, username, passwo } } - includedFiles := filterFiles(files, includedExts) if service.cacheEnabled && service.repoFileCache != nil { - service.repoFileCache.Add(repoKey, includedFiles) - return includedFiles, nil + service.repoFileCache.Add(repoKey, files) } - return includedFiles, nil + + return files, nil } func (service *Service) purgeCache() {