fix(git): optimize listFiles() BE-11184 (#12161)

api/git/git.go

@@ -143,6 +143,7 @@ func (c *gitClient) listFiles(ctx context.Context, opt fetchOption) ([]string, e
 		ReferenceName:   plumbing.ReferenceName(opt.referenceName),
 		Auth:            getAuth(opt.username, opt.password),
 		InsecureSkipTLS: opt.tlsSkipVerify,
+		Tags:            git.NoTags,
 	}
 
 	repo, err := git.Clone(memory.NewStorage(), nil, cloneOption)
@@ -166,7 +167,10 @@ func (c *gitClient) listFiles(ctx context.Context, opt fetchOption) ([]string, e
 	}
 
 	var allPaths []string
+
 	w := object.NewTreeWalker(tree, true, nil)
+	defer w.Close()
+
 	for {
 		name, entry, err := w.Next()
 		if err != nil {

api/git/git_test.go

@@ -91,6 +91,29 @@ func Test_latestCommitID(t *testing.T) {
 	assert.Equal(t, "68dcaa7bd452494043c64252ab90db0f98ecf8d2", id)
 }
 
+func Test_ListRefs(t *testing.T) {
+	service := Service{git: NewGitClient(true)}
+
+	repositoryURL := setup(t)
+
+	fs, err := service.ListRefs(repositoryURL, "", "", false, false)
+
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"refs/heads/main"}, fs)
+}
+
+func Test_ListFiles(t *testing.T) {
+	service := Service{git: NewGitClient(true)}
+
+	repositoryURL := setup(t)
+	referenceName := "refs/heads/main"
+
+	fs, err := service.ListFiles(repositoryURL, referenceName, "", "", false, false, []string{".yml"}, false)
+
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"docker-compose.yml"}, fs)
+}
+
 func getCommitHistoryLength(t *testing.T, err error, dir string) int {
 	repo, err := git.PlainOpen(dir)
 	if err != nil {

api/git/service.go

@@ -9,6 +9,7 @@ import (
 
 	lru "github.com/hashicorp/golang-lru"
 	"github.com/rs/zerolog/log"
+	"golang.org/x/sync/singleflight"
 )
 
 const (
@@ -223,11 +224,23 @@ func (service *Service) ListRefs(repositoryURL, username, password string, hardR
 	return refs, nil
 }
 
+var singleflightGroup = &singleflight.Group{}
+
 // ListFiles will list all the files of the target repository with specific extensions.
 // If extension is not provided, it will list all the files under the target repository
 func (service *Service) ListFiles(repositoryURL, referenceName, username, password string, dirOnly, hardRefresh bool, includedExts []string, tlsSkipVerify bool) ([]string, error) {
 	repoKey := generateCacheKey(repositoryURL, referenceName, username, password, strconv.FormatBool(tlsSkipVerify), strconv.FormatBool(dirOnly))
 
+	fs, err, _ := singleflightGroup.Do(repoKey, func() (any, error) {
+		return service.listFiles(repositoryURL, referenceName, username, password, dirOnly, hardRefresh, tlsSkipVerify)
+	})
+
+	return filterFiles(fs.([]string), includedExts), err
+}
+
+func (service *Service) listFiles(repositoryURL, referenceName, username, password string, dirOnly, hardRefresh bool, tlsSkipVerify bool) ([]string, error) {
+	repoKey := generateCacheKey(repositoryURL, referenceName, username, password, strconv.FormatBool(tlsSkipVerify), strconv.FormatBool(dirOnly))
+
 	if service.cacheEnabled && hardRefresh {
 		// Should remove the cache explicitly, so that the following normal list can show the correct result
 		service.repoFileCache.Remove(repoKey)
@@ -235,14 +248,9 @@ func (service *Service) ListFiles(repositoryURL, referenceName, username, passwo
 
 	if service.repoFileCache != nil {
 		// lookup the files cache first
-		cache, ok := service.repoFileCache.Get(repoKey)
-		if ok {
-			files, success := cache.([]string)
-			if success {
-				// For the case while searching files in a repository without include extensions for the first time,
-				// but with include extensions for the second time
-				includedFiles := filterFiles(files, includedExts)
-				return includedFiles, nil
+		if cache, ok := service.repoFileCache.Get(repoKey); ok {
+			if files, ok := cache.([]string); ok {
+				return files, nil
 			}
 		}
 	}
@@ -274,12 +282,11 @@ func (service *Service) ListFiles(repositoryURL, referenceName, username, passwo
 		}
 	}
 
-	includedFiles := filterFiles(files, includedExts)
 	if service.cacheEnabled && service.repoFileCache != nil {
-		service.repoFileCache.Add(repoKey, includedFiles)
-		return includedFiles, nil
+		service.repoFileCache.Add(repoKey, files)
 	}
-	return includedFiles, nil
+
+	return files, nil
 }
 
 func (service *Service) purgeCache() {