package git import ( "context" "strconv" "strings" "sync" "time" lru "github.com/hashicorp/golang-lru" "github.com/rs/zerolog/log" ) var ( REPOSITORY_CACHE_SIZE = 4 REPOSITORY_CACHE_TTL = 5 * time.Minute ) // baseOption provides a minimum group of information to operate a git repository, like git-remote type baseOption struct { repositoryUrl string username string password string tlsSkipVerify bool } // fetchOption allows to specify the reference name of the target repository type fetchOption struct { baseOption referenceName string } // cloneOption allows to add a history truncated to the specified number of commits type cloneOption struct { fetchOption depth int } type repoManager interface { download(ctx context.Context, dst string, opt cloneOption) error latestCommitID(ctx context.Context, opt fetchOption) (string, error) listRefs(ctx context.Context, opt baseOption) ([]string, error) listFiles(ctx context.Context, opt fetchOption) ([]string, error) } // Service represents a service for managing Git. type Service struct { shutdownCtx context.Context azure repoManager git repoManager timerStopped bool mut sync.Mutex cacheEnabled bool // Cache the result of repository refs, key is repository URL repoRefCache *lru.Cache // Cache the result of repository file tree, key is the concatenated string of repository URL and ref value repoFileCache *lru.Cache } // NewService initializes a new service. func NewService(ctx context.Context) *Service { return newService(ctx, REPOSITORY_CACHE_SIZE, REPOSITORY_CACHE_TTL) } func newService(ctx context.Context, cacheSize int, cacheTTL time.Duration) *Service { service := &Service{ shutdownCtx: ctx, azure: NewAzureClient(), git: NewGitClient(false), timerStopped: false, cacheEnabled: cacheSize > 0, } if service.cacheEnabled { var err error service.repoRefCache, err = lru.New(cacheSize) if err != nil { log.Debug().Err(err).Msg("failed to create ref cache") } service.repoFileCache, err = lru.New(cacheSize) if err != nil { log.Debug().Err(err).Msg("failed to create file cache") } if cacheTTL > 0 { go service.startCacheCleanTimer(cacheTTL) } } return service } // startCacheCleanTimer starts a timer to purge caches periodically func (service *Service) startCacheCleanTimer(d time.Duration) { ticker := time.NewTicker(d) for { select { case <-ticker.C: service.purgeCache() case <-service.shutdownCtx.Done(): ticker.Stop() service.mut.Lock() service.timerStopped = true service.mut.Unlock() return } } } // timerHasStopped shows the CacheClean timer state with thread-safe way func (service *Service) timerHasStopped() bool { service.mut.Lock() defer service.mut.Unlock() ret := service.timerStopped return ret } // CloneRepository clones a git repository using the specified URL in the specified // destination folder. func (service *Service) CloneRepository(destination, repositoryURL, referenceName, username, password string, tlsSkipVerify bool) error { options := cloneOption{ fetchOption: fetchOption{ baseOption: baseOption{ repositoryUrl: repositoryURL, username: username, password: password, tlsSkipVerify: tlsSkipVerify, }, referenceName: referenceName, }, depth: 1, } return service.cloneRepository(destination, options) } func (service *Service) cloneRepository(destination string, options cloneOption) error { if isAzureUrl(options.repositoryUrl) { return service.azure.download(context.TODO(), destination, options) } return service.git.download(context.TODO(), destination, options) } // LatestCommitID returns SHA1 of the latest commit of the specified reference func (service *Service) LatestCommitID(repositoryURL, referenceName, username, password string, tlsSkipVerify bool) (string, error) { options := fetchOption{ baseOption: baseOption{ repositoryUrl: repositoryURL, username: username, password: password, tlsSkipVerify: tlsSkipVerify, }, referenceName: referenceName, } if isAzureUrl(options.repositoryUrl) { return service.azure.latestCommitID(context.TODO(), options) } return service.git.latestCommitID(context.TODO(), options) } // ListRefs will list target repository's references without cloning the repository func (service *Service) ListRefs(repositoryURL, username, password string, hardRefresh bool, tlsSkipVerify bool) ([]string, error) { refCacheKey := generateCacheKey(repositoryURL, username, password, strconv.FormatBool(tlsSkipVerify)) if service.cacheEnabled && hardRefresh { // Should remove the cache explicitly, so that the following normal list can show the correct result service.repoRefCache.Remove(refCacheKey) // Remove file caches pointed to the same repository for _, fileCacheKey := range service.repoFileCache.Keys() { key, ok := fileCacheKey.(string) if ok { if strings.HasPrefix(key, repositoryURL) { service.repoFileCache.Remove(key) } } } } if service.repoRefCache != nil { // Lookup the refs cache first cache, ok := service.repoRefCache.Get(refCacheKey) if ok { refs, success := cache.([]string) if success { return refs, nil } } } options := baseOption{ repositoryUrl: repositoryURL, username: username, password: password, tlsSkipVerify: tlsSkipVerify, } var ( refs []string err error ) if isAzureUrl(options.repositoryUrl) { refs, err = service.azure.listRefs(context.TODO(), options) if err != nil { return nil, err } } else { refs, err = service.git.listRefs(context.TODO(), options) if err != nil { return nil, err } } if service.cacheEnabled && service.repoRefCache != nil { service.repoRefCache.Add(refCacheKey, refs) } return refs, nil } // ListFiles will list all the files of the target repository with specific extensions. // If extension is not provided, it will list all the files under the target repository func (service *Service) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string, tlsSkipVerify bool) ([]string, error) { repoKey := generateCacheKey(repositoryURL, referenceName, username, password, strconv.FormatBool(tlsSkipVerify)) if service.cacheEnabled && hardRefresh { // Should remove the cache explicitly, so that the following normal list can show the correct result service.repoFileCache.Remove(repoKey) } if service.repoFileCache != nil { // lookup the files cache first cache, ok := service.repoFileCache.Get(repoKey) if ok { files, success := cache.([]string) if success { // For the case while searching files in a repository without include extensions for the first time, // but with include extensions for the second time includedFiles := filterFiles(files, includedExts) return includedFiles, nil } } } options := fetchOption{ baseOption: baseOption{ repositoryUrl: repositoryURL, username: username, password: password, tlsSkipVerify: tlsSkipVerify, }, referenceName: referenceName, } var ( files []string err error ) if isAzureUrl(options.repositoryUrl) { files, err = service.azure.listFiles(context.TODO(), options) if err != nil { return nil, err } } else { files, err = service.git.listFiles(context.TODO(), options) if err != nil { return nil, err } } includedFiles := filterFiles(files, includedExts) if service.cacheEnabled && service.repoFileCache != nil { service.repoFileCache.Add(repoKey, includedFiles) return includedFiles, nil } return includedFiles, nil } func (service *Service) purgeCache() { if service.repoRefCache != nil { service.repoRefCache.Purge() } if service.repoFileCache != nil { service.repoFileCache.Purge() } } func generateCacheKey(names ...string) string { return strings.Join(names, "-") } func matchExtensions(target string, exts []string) bool { if len(exts) == 0 { return true } for _, ext := range exts { if strings.HasSuffix(target, ext) { return true } } return false } func filterFiles(paths []string, includedExts []string) []string { if len(includedExts) == 0 { return paths } var includedFiles []string for _, filename := range paths { // filter out the filenames with non-included extension if matchExtensions(filename, includedExts) { includedFiles = append(includedFiles, filename) } } return includedFiles }