From 5777c182971bec4b0fe51ee56ba3be228cb3c1e1 Mon Sep 17 00:00:00 2001 From: Oscar Zhou <100548325+oscarzhou-portainer@users.noreply.github.com> Date: Wed, 21 Sep 2022 17:47:02 +1200 Subject: [PATCH] feat(gitops): support to list git repository refs and file tree [EE-2673] (#7100) --- api/cmd/portainer/main.go | 7 +- api/git/azure.go | 255 ++++++++++++-- api/git/azure_integration_test.go | 211 +++++++++++- api/git/azure_test.go | 326 +++++++++++++++++- api/git/git.go | 159 ++++----- api/git/git_integration_test.go | 322 ++++++++++++++++- api/git/git_test.go | 225 +++++++++--- api/git/service.go | 320 +++++++++++++++++ api/http/handler/edgestacks/edgestack_test.go | 8 + .../proxy/factory/docker/transport_test.go | 11 +- api/portainer.go | 2 + api/stacks/deploy_test.go | 8 + .../components/forms/git-form/git-form.html | 6 +- 13 files changed, 1673 insertions(+), 187 deletions(-) create mode 100644 api/git/service.go diff --git a/api/cmd/portainer/main.go b/api/cmd/portainer/main.go index 29e9fffa5..e0a2c50c4 100644 --- a/api/cmd/portainer/main.go +++ b/api/cmd/portainer/main.go @@ -210,8 +210,8 @@ func initOAuthService() portainer.OAuthService { return oauth.NewService() } -func initGitService() portainer.GitService { - return git.NewService() +func initGitService(ctx context.Context) portainer.GitService { + return git.NewService(ctx) } func initSSLService(addr, certPath, keyPath string, fileService portainer.FileService, dataStore dataservices.DataStore, shutdownTrigger context.CancelFunc) (*ssl.Service, error) { @@ -580,8 +580,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server { ldapService := initLDAPService() oauthService := initOAuthService() - - gitService := initGitService() + gitService := initGitService(shutdownCtx) openAMTService := openamt.NewService() diff --git a/api/git/azure.go b/api/git/azure.go index 36f495d5f..c46b5ba9d 100644 --- a/api/git/azure.go +++ b/api/git/azure.go @@ -2,6 +2,7 @@ package git import ( "context" + "crypto/tls" "encoding/json" "fmt" "io" @@ -10,7 +11,10 @@ import ( "net/url" "os" "strings" + "time" + "github.com/go-git/go-git/v5/plumbing/transport/client" + githttp "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/pkg/errors" "github.com/portainer/portainer/api/archive" ) @@ -32,20 +36,47 @@ type azureOptions struct { username, password string } -type azureDownloader struct { +// azureRef abstracts from the response of https://docs.microsoft.com/en-us/rest/api/azure/devops/git/refs/list?view=azure-devops-rest-6.0#refs +type azureRef struct { + Name string `json:"name"` + ObjectID string `json:"objectId"` +} + +// azureItem abstracts from the response of https://docs.microsoft.com/en-us/rest/api/azure/devops/git/items/get?view=azure-devops-rest-6.0#download +type azureItem struct { + ObjectID string `json:"objectId"` + CommitId string `json:"commitId"` + Path string `json:"path"` +} + +type azureClient struct { client *http.Client baseUrl string } -func NewAzureDownloader(client *http.Client) *azureDownloader { - return &azureDownloader{ - client: client, +func NewAzureClient() *azureClient { + httpsCli := newHttpClientForAzure() + return &azureClient{ + client: httpsCli, baseUrl: "https://dev.azure.com", } } -func (a *azureDownloader) download(ctx context.Context, destination string, options cloneOptions) error { - zipFilepath, err := a.downloadZipFromAzureDevOps(ctx, options) +func newHttpClientForAzure() *http.Client { + httpsCli := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + Proxy: http.ProxyFromEnvironment, + }, + Timeout: 300 * time.Second, + } + + client.InstallProtocol("https", githttp.NewClient(httpsCli)) + return httpsCli +} + +func (a *azureClient) download(ctx context.Context, destination string, opt cloneOption) error { + zipFilepath, err := a.downloadZipFromAzureDevOps(ctx, opt) if err != nil { return errors.Wrap(err, "failed to download a zip file from Azure DevOps") } @@ -59,12 +90,12 @@ func (a *azureDownloader) download(ctx context.Context, destination string, opti return nil } -func (a *azureDownloader) downloadZipFromAzureDevOps(ctx context.Context, options cloneOptions) (string, error) { - config, err := parseUrl(options.repositoryUrl) +func (a *azureClient) downloadZipFromAzureDevOps(ctx context.Context, opt cloneOption) (string, error) { + config, err := parseUrl(opt.repositoryUrl) if err != nil { return "", errors.WithMessage(err, "failed to parse url") } - downloadUrl, err := a.buildDownloadUrl(config, options.referenceName) + downloadUrl, err := a.buildDownloadUrl(config, opt.referenceName) if err != nil { return "", errors.WithMessage(err, "failed to build download url") } @@ -75,8 +106,8 @@ func (a *azureDownloader) downloadZipFromAzureDevOps(ctx context.Context, option defer zipFile.Close() req, err := http.NewRequestWithContext(ctx, "GET", downloadUrl, nil) - if options.username != "" || options.password != "" { - req.SetBasicAuth(options.username, options.password) + if opt.username != "" || opt.password != "" { + req.SetBasicAuth(opt.username, opt.password) } else if config.username != "" || config.password != "" { req.SetBasicAuth(config.username, config.password) } @@ -102,53 +133,58 @@ func (a *azureDownloader) downloadZipFromAzureDevOps(ctx context.Context, option return zipFile.Name(), nil } -func (a *azureDownloader) latestCommitID(ctx context.Context, options fetchOptions) (string, error) { - config, err := parseUrl(options.repositoryUrl) +func (a *azureClient) latestCommitID(ctx context.Context, opt fetchOption) (string, error) { + rootItem, err := a.getRootItem(ctx, opt) if err != nil { - return "", errors.WithMessage(err, "failed to parse url") + return "", err + } + return rootItem.CommitId, nil +} + +func (a *azureClient) getRootItem(ctx context.Context, opt fetchOption) (*azureItem, error) { + config, err := parseUrl(opt.repositoryUrl) + if err != nil { + return nil, errors.WithMessage(err, "failed to parse url") } - rootItemUrl, err := a.buildRootItemUrl(config, options.referenceName) + rootItemUrl, err := a.buildRootItemUrl(config, opt.referenceName) if err != nil { - return "", errors.WithMessage(err, "failed to build azure root item url") + return nil, errors.WithMessage(err, "failed to build azure root item url") } req, err := http.NewRequestWithContext(ctx, "GET", rootItemUrl, nil) - if options.username != "" || options.password != "" { - req.SetBasicAuth(options.username, options.password) + if opt.username != "" || opt.password != "" { + req.SetBasicAuth(opt.username, opt.password) } else if config.username != "" || config.password != "" { req.SetBasicAuth(config.username, config.password) } if err != nil { - return "", errors.WithMessage(err, "failed to create a new HTTP request") + return nil, errors.WithMessage(err, "failed to create a new HTTP request") } resp, err := a.client.Do(req) if err != nil { - return "", errors.WithMessage(err, "failed to make an HTTP request") + return nil, errors.WithMessage(err, "failed to make an HTTP request") } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("failed to get repository root item with a status \"%v\"", resp.Status) + return nil, checkAzureStatusCode(fmt.Errorf("failed to get repository root item with a status \"%v\"", resp.Status), resp.StatusCode) } var items struct { - Value []struct { - CommitId string `json:"commitId"` - } + Value []azureItem } if err := json.NewDecoder(resp.Body).Decode(&items); err != nil { - return "", errors.Wrap(err, "could not parse Azure items response") + return nil, errors.Wrap(err, "could not parse Azure items response") } if len(items.Value) == 0 || items.Value[0].CommitId == "" { - return "", errors.Errorf("failed to get latest commitID in the repository") + return nil, errors.Errorf("failed to get latest commitID in the repository") } - - return items.Value[0].CommitId, nil + return &items.Value[0], nil } func parseUrl(rawUrl string) (*azureOptions, error) { @@ -219,7 +255,7 @@ func parseHttpUrl(rawUrl string) (*azureOptions, error) { return &opt, nil } -func (a *azureDownloader) buildDownloadUrl(config *azureOptions, referenceName string) (string, error) { +func (a *azureClient) buildDownloadUrl(config *azureOptions, referenceName string) (string, error) { rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/items", a.baseUrl, url.PathEscape(config.organisation), @@ -246,7 +282,7 @@ func (a *azureDownloader) buildDownloadUrl(config *azureOptions, referenceName s return u.String(), nil } -func (a *azureDownloader) buildRootItemUrl(config *azureOptions, referenceName string) (string, error) { +func (a *azureClient) buildRootItemUrl(config *azureOptions, referenceName string) (string, error) { rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/items", a.baseUrl, url.PathEscape(config.organisation), @@ -270,6 +306,49 @@ func (a *azureDownloader) buildRootItemUrl(config *azureOptions, referenceName s return u.String(), nil } +func (a *azureClient) buildRefsUrl(config *azureOptions) (string, error) { + // ref@https://docs.microsoft.com/en-us/rest/api/azure/devops/git/refs/list?view=azure-devops-rest-6.0#gitref + rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/refs", + a.baseUrl, + url.PathEscape(config.organisation), + url.PathEscape(config.project), + url.PathEscape(config.repository)) + u, err := url.Parse(rawUrl) + + if err != nil { + return "", errors.Wrapf(err, "failed to parse list refs url path %s", rawUrl) + } + + q := u.Query() + q.Set("api-version", "6.0") + u.RawQuery = q.Encode() + + return u.String(), nil +} + +func (a *azureClient) buildTreeUrl(config *azureOptions, rootObjectHash string) (string, error) { + // ref@https://docs.microsoft.com/en-us/rest/api/azure/devops/git/trees/get?view=azure-devops-rest-6.0 + rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/trees/%s", + a.baseUrl, + url.PathEscape(config.organisation), + url.PathEscape(config.project), + url.PathEscape(config.repository), + url.PathEscape(rootObjectHash), + ) + u, err := url.Parse(rawUrl) + + if err != nil { + return "", errors.Wrapf(err, "failed to parse list tree url path %s", rawUrl) + } + q := u.Query() + // projectId={projectId}&recursive=true&fileName={fileName}&$format={$format}&api-version=6.0 + q.Set("recursive", "true") + q.Set("api-version", "6.0") + u.RawQuery = q.Encode() + + return u.String(), nil +} + const ( branchPrefix = "refs/heads/" tagPrefix = "refs/tags/" @@ -294,3 +373,119 @@ func getVersionType(name string) string { } return "commit" } + +func (a *azureClient) listRefs(ctx context.Context, opt baseOption) ([]string, error) { + config, err := parseUrl(opt.repositoryUrl) + if err != nil { + return nil, errors.WithMessage(err, "failed to parse url") + } + + listRefsUrl, err := a.buildRefsUrl(config) + if err != nil { + return nil, errors.WithMessage(err, "failed to build list refs url") + } + + req, err := http.NewRequestWithContext(ctx, "GET", listRefsUrl, nil) + if opt.username != "" || opt.password != "" { + req.SetBasicAuth(opt.username, opt.password) + } else if config.username != "" || config.password != "" { + req.SetBasicAuth(config.username, config.password) + } + + if err != nil { + return nil, errors.WithMessage(err, "failed to create a new HTTP request") + } + + resp, err := a.client.Do(req) + if err != nil { + return nil, errors.WithMessage(err, "failed to make an HTTP request") + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, checkAzureStatusCode(fmt.Errorf("failed to list refs with a status \"%v\"", resp.Status), resp.StatusCode) + } + + var refs struct { + Value []azureRef + } + + if err := json.NewDecoder(resp.Body).Decode(&refs); err != nil { + return nil, errors.Wrap(err, "could not parse Azure refs response") + } + + var ret []string + for _, value := range refs.Value { + if value.Name == "HEAD" { + continue + } + ret = append(ret, value.Name) + } + + return ret, nil +} + +// listFiles list all filenames under the specific repository +func (a *azureClient) listFiles(ctx context.Context, opt fetchOption) ([]string, error) { + rootItem, err := a.getRootItem(ctx, opt) + if err != nil { + return nil, err + } + + config, err := parseUrl(opt.repositoryUrl) + if err != nil { + return nil, errors.WithMessage(err, "failed to parse url") + } + + listTreeUrl, err := a.buildTreeUrl(config, rootItem.ObjectID) + if err != nil { + return nil, errors.WithMessage(err, "failed to build list tree url") + } + + req, err := http.NewRequestWithContext(ctx, "GET", listTreeUrl, nil) + if opt.username != "" || opt.password != "" { + req.SetBasicAuth(opt.username, opt.password) + } else if config.username != "" || config.password != "" { + req.SetBasicAuth(config.username, config.password) + } + + if err != nil { + return nil, errors.WithMessage(err, "failed to create a new HTTP request") + } + + resp, err := a.client.Do(req) + if err != nil { + return nil, errors.WithMessage(err, "failed to make an HTTP request") + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to list tree url with a status \"%v\"", resp.Status) + } + + var tree struct { + TreeEntries []struct { + RelativePath string `json:"relativePath"` + } `json:"treeEntries"` + } + + if err := json.NewDecoder(resp.Body).Decode(&tree); err != nil { + return nil, errors.Wrap(err, "could not parse Azure tree response") + } + + var allPaths []string + for _, treeEntry := range tree.TreeEntries { + allPaths = append(allPaths, treeEntry.RelativePath) + } + + return allPaths, nil +} + +func checkAzureStatusCode(err error, code int) error { + if code == http.StatusNotFound { + return ErrIncorrectRepositoryURL + } else if code == http.StatusUnauthorized || code == http.StatusNonAuthoritativeInfo { + return ErrAuthenticationFailure + } + return err +} diff --git a/api/git/azure_integration_test.go b/api/git/azure_integration_test.go index aef1ff5eb..876e59496 100644 --- a/api/git/azure_integration_test.go +++ b/api/git/azure_integration_test.go @@ -1,20 +1,26 @@ package git import ( + "context" "fmt" "os" "path/filepath" "testing" + "time" _ "github.com/joho/godotenv/autoload" "github.com/stretchr/testify/assert" ) +var ( + privateAzureRepoURL = "https://portainer.visualstudio.com/gitops-test/_git/gitops-test" +) + func TestService_ClonePublicRepository_Azure(t *testing.T) { ensureIntegrationTest(t) pat := getRequiredValue(t, "AZURE_DEVOPS_PAT") - service := NewService() + service := NewService(context.TODO()) type args struct { repositoryURLFormat string @@ -30,7 +36,7 @@ func TestService_ClonePublicRepository_Azure(t *testing.T) { { name: "Clone Azure DevOps repo branch", args: args{ - repositoryURLFormat: "https://:%s@portainer.visualstudio.com/Playground/_git/dev_integration", + repositoryURLFormat: "https://:%s@portainer.visualstudio.com/gitops-test/_git/gitops-test", referenceName: "refs/heads/main", username: "", password: pat, @@ -40,8 +46,8 @@ func TestService_ClonePublicRepository_Azure(t *testing.T) { { name: "Clone Azure DevOps repo tag", args: args{ - repositoryURLFormat: "https://:%s@portainer.visualstudio.com/Playground/_git/dev_integration", - referenceName: "refs/tags/v1.1", + repositoryURLFormat: "https://:%s@portainer.visualstudio.com/gitops-test/_git/gitops-test", + referenceName: "refs/heads/tags/v1.1", username: "", password: pat, }, @@ -63,12 +69,11 @@ func TestService_ClonePrivateRepository_Azure(t *testing.T) { ensureIntegrationTest(t) pat := getRequiredValue(t, "AZURE_DEVOPS_PAT") - service := NewService() + service := NewService(context.TODO()) dst := t.TempDir() - repositoryUrl := "https://portainer.visualstudio.com/Playground/_git/dev_integration" - err := service.CloneRepository(dst, repositoryUrl, "refs/heads/main", "", pat) + err := service.CloneRepository(dst, privateAzureRepoURL, "refs/heads/main", "", pat) assert.NoError(t, err) assert.FileExists(t, filepath.Join(dst, "README.md")) } @@ -77,14 +82,200 @@ func TestService_LatestCommitID_Azure(t *testing.T) { ensureIntegrationTest(t) pat := getRequiredValue(t, "AZURE_DEVOPS_PAT") - service := NewService() + service := NewService(context.TODO()) - repositoryUrl := "https://portainer.visualstudio.com/Playground/_git/dev_integration" - id, err := service.LatestCommitID(repositoryUrl, "refs/heads/main", "", pat) + id, err := service.LatestCommitID(privateAzureRepoURL, "refs/heads/main", "", pat) assert.NoError(t, err) assert.NotEmpty(t, id, "cannot guarantee commit id, but it should be not empty") } +func TestService_ListRefs_Azure(t *testing.T) { + ensureIntegrationTest(t) + + accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT") + username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME") + service := NewService(context.TODO()) + + refs, err := service.ListRefs(privateAzureRepoURL, username, accessToken, false) + assert.NoError(t, err) + assert.GreaterOrEqual(t, len(refs), 1) +} + +func TestService_ListRefs_Azure_Concurrently(t *testing.T) { + ensureIntegrationTest(t) + + accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT") + username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME") + service := newService(context.TODO(), REPOSITORY_CACHE_SIZE, 200*time.Millisecond) + + go service.ListRefs(privateAzureRepoURL, username, accessToken, false) + service.ListRefs(privateAzureRepoURL, username, accessToken, false) + + time.Sleep(2 * time.Second) +} + +func TestService_ListFiles_Azure(t *testing.T) { + ensureIntegrationTest(t) + + type expectResult struct { + shouldFail bool + err error + matchedCount int + } + service := newService(context.TODO(), 0, 0) + accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT") + username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME") + + tests := []struct { + name string + args fetchOption + extensions []string + expect expectResult + }{ + { + name: "list tree with real repository and head ref but incorrect credential", + args: fetchOption{ + baseOption: baseOption{ + repositoryUrl: privateAzureRepoURL, + username: "test-username", + password: "test-token", + }, + referenceName: "refs/heads/main", + }, + extensions: []string{}, + expect: expectResult{ + shouldFail: true, + err: ErrAuthenticationFailure, + }, + }, + { + name: "list tree with real repository and head ref but no credential", + args: fetchOption{ + baseOption: baseOption{ + repositoryUrl: privateAzureRepoURL, + username: "", + password: "", + }, + referenceName: "refs/heads/main", + }, + extensions: []string{}, + expect: expectResult{ + shouldFail: true, + err: ErrAuthenticationFailure, + }, + }, + { + name: "list tree with real repository and head ref", + args: fetchOption{ + baseOption: baseOption{ + repositoryUrl: privateAzureRepoURL, + username: username, + password: accessToken, + }, + referenceName: "refs/heads/main", + }, + extensions: []string{}, + expect: expectResult{ + err: nil, + matchedCount: 19, + }, + }, + { + name: "list tree with real repository and head ref and existing file extension", + args: fetchOption{ + baseOption: baseOption{ + repositoryUrl: privateAzureRepoURL, + username: username, + password: accessToken, + }, + referenceName: "refs/heads/main", + }, + extensions: []string{"yml"}, + expect: expectResult{ + err: nil, + matchedCount: 2, + }, + }, + { + name: "list tree with real repository and head ref and non-existing file extension", + args: fetchOption{ + baseOption: baseOption{ + repositoryUrl: privateAzureRepoURL, + username: username, + password: accessToken, + }, + referenceName: "refs/heads/main", + }, + extensions: []string{"hcl"}, + expect: expectResult{ + err: nil, + matchedCount: 2, + }, + }, + { + name: "list tree with real repository but non-existing ref", + args: fetchOption{ + baseOption: baseOption{ + repositoryUrl: privateAzureRepoURL, + username: username, + password: accessToken, + }, + referenceName: "refs/fake/feature", + }, + extensions: []string{}, + expect: expectResult{ + shouldFail: true, + }, + }, + { + name: "list tree with fake repository ", + args: fetchOption{ + baseOption: baseOption{ + repositoryUrl: privateAzureRepoURL + "fake", + username: username, + password: accessToken, + }, + referenceName: "refs/fake/feature", + }, + extensions: []string{}, + expect: expectResult{ + shouldFail: true, + err: ErrIncorrectRepositoryURL, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, tt.extensions) + if tt.expect.shouldFail { + assert.Error(t, err) + if tt.expect.err != nil { + assert.Equal(t, tt.expect.err, err) + } + } else { + assert.NoError(t, err) + if tt.expect.matchedCount > 0 { + assert.Greater(t, len(paths), 0) + } + } + }) + } +} + +func TestService_ListFiles_Azure_Concurrently(t *testing.T) { + ensureIntegrationTest(t) + + accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT") + username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME") + service := newService(context.TODO(), REPOSITORY_CACHE_SIZE, 200*time.Millisecond) + + go service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, []string{}) + service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, []string{}) + + time.Sleep(2 * time.Second) +} + func getRequiredValue(t *testing.T, name string) string { value, ok := os.LookupEnv(name) if !ok { diff --git a/api/git/azure_test.go b/api/git/azure_test.go index 0cf4099bc..dfe81a440 100644 --- a/api/git/azure_test.go +++ b/api/git/azure_test.go @@ -11,7 +11,7 @@ import ( ) func Test_buildDownloadUrl(t *testing.T) { - a := NewAzureDownloader(nil) + a := NewAzureClient() u, err := a.buildDownloadUrl(&azureOptions{ organisation: "organisation", project: "project", @@ -29,7 +29,7 @@ func Test_buildDownloadUrl(t *testing.T) { } func Test_buildRootItemUrl(t *testing.T) { - a := NewAzureDownloader(nil) + a := NewAzureClient() u, err := a.buildRootItemUrl(&azureOptions{ organisation: "organisation", project: "project", @@ -45,6 +45,40 @@ func Test_buildRootItemUrl(t *testing.T) { assert.Equal(t, expectedUrl.Query(), actualUrl.Query()) } +func Test_buildRefsUrl(t *testing.T) { + a := NewAzureClient() + u, err := a.buildRefsUrl(&azureOptions{ + organisation: "organisation", + project: "project", + repository: "repository", + }) + + expectedUrl, _ := url.Parse("https://dev.azure.com/organisation/project/_apis/git/repositories/repository/refs?api-version=6.0") + actualUrl, _ := url.Parse(u) + assert.NoError(t, err) + assert.Equal(t, expectedUrl.Host, actualUrl.Host) + assert.Equal(t, expectedUrl.Scheme, actualUrl.Scheme) + assert.Equal(t, expectedUrl.Path, actualUrl.Path) + assert.Equal(t, expectedUrl.Query(), actualUrl.Query()) +} + +func Test_buildTreeUrl(t *testing.T) { + a := NewAzureClient() + u, err := a.buildTreeUrl(&azureOptions{ + organisation: "organisation", + project: "project", + repository: "repository", + }, "sha1") + + expectedUrl, _ := url.Parse("https://dev.azure.com/organisation/project/_apis/git/repositories/repository/trees/sha1?api-version=6.0&recursive=true") + actualUrl, _ := url.Parse(u) + assert.NoError(t, err) + assert.Equal(t, expectedUrl.Host, actualUrl.Host) + assert.Equal(t, expectedUrl.Scheme, actualUrl.Scheme) + assert.Equal(t, expectedUrl.Path, actualUrl.Path) + assert.Equal(t, expectedUrl.Query(), actualUrl.Query()) +} + func Test_parseAzureUrl(t *testing.T) { type args struct { url string @@ -200,7 +234,7 @@ func Test_isAzureUrl(t *testing.T) { func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) { type args struct { - options cloneOptions + options baseOption } type basicAuth struct { username, password string @@ -213,7 +247,7 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) { { name: "username, password embedded", args: args{ - options: cloneOptions{ + options: baseOption{ repositoryUrl: "https://username:password@dev.azure.com/Organisation/Project/_git/Repository", }, }, @@ -225,7 +259,7 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) { { name: "username, password embedded, clone options take precedence", args: args{ - options: cloneOptions{ + options: baseOption{ repositoryUrl: "https://username:password@dev.azure.com/Organisation/Project/_git/Repository", username: "u", password: "p", @@ -239,7 +273,7 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) { { name: "no credentials", args: args{ - options: cloneOptions{ + options: baseOption{ repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository", }, }, @@ -256,11 +290,17 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) { })) defer server.Close() - a := &azureDownloader{ + a := &azureClient{ client: server.Client(), baseUrl: server.URL, } - _, err := a.downloadZipFromAzureDevOps(context.Background(), tt.args.options) + + option := cloneOption{ + fetchOption: fetchOption{ + baseOption: tt.args.options, + }, + } + _, err := a.downloadZipFromAzureDevOps(context.Background(), option) assert.Error(t, err) assert.Equal(t, tt.want, zipRequestAuth) }) @@ -287,22 +327,25 @@ func Test_azureDownloader_latestCommitID(t *testing.T) { })) defer server.Close() - a := &azureDownloader{ + a := &azureClient{ client: server.Client(), baseUrl: server.URL, } tests := []struct { name string - args fetchOptions + args fetchOption want string wantErr bool }{ { name: "should be able to parse response", - args: fetchOptions{ + args: fetchOption{ + baseOption: baseOption{ + repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository", + }, referenceName: "", - repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository"}, + }, want: "27104ad7549d9e66685e115a497533f18024be9c", wantErr: false, }, @@ -319,3 +362,262 @@ func Test_azureDownloader_latestCommitID(t *testing.T) { }) } } + +type testRepoManager struct { + called bool +} + +func (t *testRepoManager) download(_ context.Context, _ string, _ cloneOption) error { + t.called = true + return nil +} + +func (t *testRepoManager) latestCommitID(_ context.Context, _ fetchOption) (string, error) { + return "", nil +} + +func (t *testRepoManager) listRefs(_ context.Context, _ baseOption) ([]string, error) { + return nil, nil +} + +func (t *testRepoManager) listFiles(_ context.Context, _ fetchOption) ([]string, error) { + return nil, nil +} +func Test_cloneRepository_azure(t *testing.T) { + tests := []struct { + name string + url string + called bool + }{ + { + name: "Azure HTTP URL", + url: "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository", + called: true, + }, + { + name: "Azure SSH URL", + url: "git@ssh.dev.azure.com:v3/Organisation/Project/Repository", + called: true, + }, + { + name: "Something else", + url: "https://example.com", + called: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + azure := &testRepoManager{} + git := &testRepoManager{} + + s := &Service{azure: azure, git: git} + s.cloneRepository("", cloneOption{ + fetchOption: fetchOption{ + baseOption: baseOption{ + + repositoryUrl: tt.url, + }, + }, + depth: 1, + }) + + // if azure API is called, git isn't and vice versa + assert.Equal(t, tt.called, azure.called) + assert.Equal(t, tt.called, !git.called) + }) + } +} + +func Test_listRefs_azure(t *testing.T) { + ensureIntegrationTest(t) + + client := NewAzureClient() + + type expectResult struct { + err error + refsCount int + } + + accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT") + username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME") + tests := []struct { + name string + args baseOption + expect expectResult + }{ + { + name: "list refs of a real repository", + args: baseOption{ + repositoryUrl: privateAzureRepoURL, + username: username, + password: accessToken, + }, + expect: expectResult{ + err: nil, + refsCount: 2, + }, + }, + { + name: "list refs of a real repository with incorrect credential", + args: baseOption{ + repositoryUrl: privateAzureRepoURL, + username: "test-username", + password: "test-token", + }, + expect: expectResult{ + err: ErrAuthenticationFailure, + }, + }, + { + name: "list refs of a real repository without providing credential", + args: baseOption{ + repositoryUrl: privateAzureRepoURL, + username: "", + password: "", + }, + expect: expectResult{ + err: ErrAuthenticationFailure, + }, + }, + { + name: "list refs of a fake repository", + args: baseOption{ + repositoryUrl: privateAzureRepoURL + "fake", + username: username, + password: accessToken, + }, + expect: expectResult{ + err: ErrIncorrectRepositoryURL, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + refs, err := client.listRefs(context.TODO(), tt.args) + if tt.expect.err == nil { + assert.NoError(t, err) + if tt.expect.refsCount > 0 { + assert.Greater(t, len(refs), 0) + } + } else { + assert.Error(t, err) + assert.Equal(t, tt.expect.err, err) + } + }) + } + +} + +func Test_listFiles_azure(t *testing.T) { + ensureIntegrationTest(t) + + client := NewAzureClient() + + type expectResult struct { + shouldFail bool + err error + matchedCount int + } + + accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT") + username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME") + tests := []struct { + name string + args fetchOption + expect expectResult + }{ + { + name: "list tree with real repository and head ref but incorrect credential", + args: fetchOption{ + baseOption: baseOption{ + repositoryUrl: privateAzureRepoURL, + username: "test-username", + password: "test-token", + }, + referenceName: "refs/heads/main", + }, + expect: expectResult{ + shouldFail: true, + err: ErrAuthenticationFailure, + }, + }, + { + name: "list tree with real repository and head ref but no credential", + args: fetchOption{ + baseOption: baseOption{ + repositoryUrl: privateAzureRepoURL, + username: "", + password: "", + }, + referenceName: "refs/heads/main", + }, + expect: expectResult{ + shouldFail: true, + err: ErrAuthenticationFailure, + }, + }, + { + name: "list tree with real repository and head ref", + args: fetchOption{ + baseOption: baseOption{ + repositoryUrl: privateAzureRepoURL, + username: username, + password: accessToken, + }, + referenceName: "refs/heads/main", + }, + expect: expectResult{ + err: nil, + matchedCount: 19, + }, + }, + { + name: "list tree with real repository but non-existing ref", + args: fetchOption{ + baseOption: baseOption{ + repositoryUrl: privateAzureRepoURL, + username: username, + password: accessToken, + }, + referenceName: "refs/fake/feature", + }, + expect: expectResult{ + shouldFail: true, + }, + }, + { + name: "list tree with fake repository ", + args: fetchOption{ + baseOption: baseOption{ + repositoryUrl: privateAzureRepoURL + "fake", + username: username, + password: accessToken, + }, + referenceName: "refs/fake/feature", + }, + expect: expectResult{ + shouldFail: true, + err: ErrIncorrectRepositoryURL, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + paths, err := client.listFiles(context.TODO(), tt.args) + if tt.expect.shouldFail { + assert.Error(t, err) + if tt.expect.err != nil { + assert.Equal(t, tt.expect.err, err) + } + } else { + assert.NoError(t, err) + if tt.expect.matchedCount > 0 { + assert.Greater(t, len(paths), 0) + } + } + }) + } +} diff --git a/api/git/git.go b/api/git/git.go index e79c60009..5f187cbb3 100644 --- a/api/git/git.go +++ b/api/git/git.go @@ -2,52 +2,31 @@ package git import ( "context" - "crypto/tls" - "net/http" "os" "path/filepath" "strings" - "time" "github.com/pkg/errors" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/transport/client" + "github.com/go-git/go-git/v5/plumbing/object" githttp "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/go-git/go-git/v5/storage/memory" ) -var ( - ErrAuthenticationFailure = errors.New("Authentication failed, please ensure that the git credentials are correct.") -) - -type fetchOptions struct { - repositoryUrl string - username string - password string - referenceName string -} - -type cloneOptions struct { - repositoryUrl string - username string - password string - referenceName string - depth int -} - -type downloader interface { - download(ctx context.Context, dst string, opt cloneOptions) error - latestCommitID(ctx context.Context, opt fetchOptions) (string, error) -} - type gitClient struct { preserveGitDirectory bool } -func (c gitClient) download(ctx context.Context, dst string, opt cloneOptions) error { +func NewGitClient(preserveGitDir bool) *gitClient { + return &gitClient{ + preserveGitDirectory: preserveGitDir, + } +} + +func (c *gitClient) download(ctx context.Context, dst string, opt cloneOption) error { gitOptions := git.CloneOptions{ URL: opt.repositoryUrl, Depth: opt.depth, @@ -74,7 +53,7 @@ func (c gitClient) download(ctx context.Context, dst string, opt cloneOptions) e return nil } -func (c gitClient) latestCommitID(ctx context.Context, opt fetchOptions) (string, error) { +func (c *gitClient) latestCommitID(ctx context.Context, opt fetchOption) (string, error) { remote := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{ Name: "origin", URLs: []string{opt.repositoryUrl}, @@ -124,66 +103,78 @@ func getAuth(username, password string) *githttp.BasicAuth { return nil } -// Service represents a service for managing Git. -type Service struct { - httpsCli *http.Client - azure downloader - git downloader +func (c *gitClient) listRefs(ctx context.Context, opt baseOption) ([]string, error) { + rem := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{ + Name: "origin", + URLs: []string{opt.repositoryUrl}, + }) + + listOptions := &git.ListOptions{ + Auth: getAuth(opt.username, opt.password), + } + + refs, err := rem.List(listOptions) + if err != nil { + return nil, checkGitError(err) + } + + var ret []string + for _, ref := range refs { + if ref.Name().String() == "HEAD" { + continue + } + ret = append(ret, ref.Name().String()) + } + + return ret, nil } -// NewService initializes a new service. -func NewService() *Service { - httpsCli := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - Proxy: http.ProxyFromEnvironment, - }, - Timeout: 300 * time.Second, +// listFiles list all filenames under the specific repository +func (c *gitClient) listFiles(ctx context.Context, opt fetchOption) ([]string, error) { + cloneOption := &git.CloneOptions{ + URL: opt.repositoryUrl, + NoCheckout: true, + Depth: 1, + SingleBranch: true, + ReferenceName: plumbing.ReferenceName(opt.referenceName), + Auth: getAuth(opt.username, opt.password), } - client.InstallProtocol("https", githttp.NewClient(httpsCli)) - - return &Service{ - httpsCli: httpsCli, - azure: NewAzureDownloader(httpsCli), - git: gitClient{}, + repo, err := git.Clone(memory.NewStorage(), nil, cloneOption) + if err != nil { + return nil, checkGitError(err) } + + head, err := repo.Head() + if err != nil { + return nil, err + } + + commit, err := repo.CommitObject(head.Hash()) + if err != nil { + return nil, err + } + + tree, err := commit.Tree() + if err != nil { + return nil, err + } + + var allPaths []string + tree.Files().ForEach(func(f *object.File) error { + allPaths = append(allPaths, f.Name) + return nil + }) + + return allPaths, nil } -// CloneRepository clones a git repository using the specified URL in the specified -// destination folder. -func (service *Service) CloneRepository(destination, repositoryURL, referenceName, username, password string) error { - options := cloneOptions{ - repositoryUrl: repositoryURL, - username: username, - password: password, - referenceName: referenceName, - depth: 1, +func checkGitError(err error) error { + errMsg := err.Error() + if errMsg == "repository not found" { + return ErrIncorrectRepositoryURL + } else if errMsg == "authentication required" { + return ErrAuthenticationFailure } - - return service.cloneRepository(destination, options) -} - -func (service *Service) cloneRepository(destination string, options cloneOptions) error { - if isAzureUrl(options.repositoryUrl) { - return service.azure.download(context.TODO(), destination, options) - } - - return service.git.download(context.TODO(), destination, options) -} - -// LatestCommitID returns SHA1 of the latest commit of the specified reference -func (service *Service) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) { - options := fetchOptions{ - repositoryUrl: repositoryURL, - username: username, - password: password, - referenceName: referenceName, - } - - if isAzureUrl(options.repositoryUrl) { - return service.azure.latestCommitID(context.TODO(), options) - } - - return service.git.latestCommitID(context.TODO(), options) + return err } diff --git a/api/git/git_integration_test.go b/api/git/git_integration_test.go index cf26ac816..977d5ba80 100644 --- a/api/git/git_integration_test.go +++ b/api/git/git_integration_test.go @@ -1,22 +1,28 @@ package git import ( + "context" "path/filepath" "testing" + "time" "github.com/stretchr/testify/assert" ) +const ( + privateGitRepoURL string = "https://github.com/portainer/private-test-repository.git" +) + func TestService_ClonePrivateRepository_GitHub(t *testing.T) { ensureIntegrationTest(t) accessToken := getRequiredValue(t, "GITHUB_PAT") username := getRequiredValue(t, "GITHUB_USERNAME") - service := NewService() + service := newService(context.TODO(), 0, 0) dst := t.TempDir() - repositoryUrl := "https://github.com/portainer/private-test-repository.git" + repositoryUrl := privateGitRepoURL err := service.CloneRepository(dst, repositoryUrl, "refs/heads/main", username, accessToken) assert.NoError(t, err) assert.FileExists(t, filepath.Join(dst, "README.md")) @@ -27,10 +33,318 @@ func TestService_LatestCommitID_GitHub(t *testing.T) { accessToken := getRequiredValue(t, "GITHUB_PAT") username := getRequiredValue(t, "GITHUB_USERNAME") - service := NewService() + service := newService(context.TODO(), 0, 0) - repositoryUrl := "https://github.com/portainer/private-test-repository.git" + repositoryUrl := privateGitRepoURL id, err := service.LatestCommitID(repositoryUrl, "refs/heads/main", username, accessToken) assert.NoError(t, err) assert.NotEmpty(t, id, "cannot guarantee commit id, but it should be not empty") } + +func TestService_ListRefs_GitHub(t *testing.T) { + ensureIntegrationTest(t) + + accessToken := getRequiredValue(t, "GITHUB_PAT") + username := getRequiredValue(t, "GITHUB_USERNAME") + service := newService(context.TODO(), 0, 0) + + repositoryUrl := privateGitRepoURL + refs, err := service.ListRefs(repositoryUrl, username, accessToken, false) + assert.NoError(t, err) + assert.GreaterOrEqual(t, len(refs), 1) +} + +func TestService_ListRefs_Github_Concurrently(t *testing.T) { + ensureIntegrationTest(t) + + accessToken := getRequiredValue(t, "GITHUB_PAT") + username := getRequiredValue(t, "GITHUB_USERNAME") + service := newService(context.TODO(), REPOSITORY_CACHE_SIZE, 200*time.Millisecond) + + repositoryUrl := privateGitRepoURL + go service.ListRefs(repositoryUrl, username, accessToken, false) + service.ListRefs(repositoryUrl, username, accessToken, false) + + time.Sleep(2 * time.Second) +} + +func TestService_ListFiles_GitHub(t *testing.T) { + ensureIntegrationTest(t) + + type expectResult struct { + shouldFail bool + err error + matchedCount int + } + service := newService(context.TODO(), 0, 0) + accessToken := getRequiredValue(t, "GITHUB_PAT") + username := getRequiredValue(t, "GITHUB_USERNAME") + + tests := []struct { + name string + args fetchOption + extensions []string + expect expectResult + }{ + { + name: "list tree with real repository and head ref but incorrect credential", + args: fetchOption{ + baseOption: baseOption{ + repositoryUrl: privateGitRepoURL, + username: "test-username", + password: "test-token", + }, + referenceName: "refs/heads/main", + }, + extensions: []string{}, + expect: expectResult{ + shouldFail: true, + err: ErrAuthenticationFailure, + }, + }, + { + name: "list tree with real repository and head ref but no credential", + args: fetchOption{ + baseOption: baseOption{ + repositoryUrl: privateGitRepoURL + "fake", + username: "", + password: "", + }, + referenceName: "refs/heads/main", + }, + extensions: []string{}, + expect: expectResult{ + shouldFail: true, + err: ErrAuthenticationFailure, + }, + }, + { + name: "list tree with real repository and head ref", + args: fetchOption{ + baseOption: baseOption{ + repositoryUrl: privateGitRepoURL, + username: username, + password: accessToken, + }, + referenceName: "refs/heads/main", + }, + extensions: []string{}, + expect: expectResult{ + err: nil, + matchedCount: 15, + }, + }, + { + name: "list tree with real repository and head ref and existing file extension", + args: fetchOption{ + baseOption: baseOption{ + repositoryUrl: privateGitRepoURL, + username: username, + password: accessToken, + }, + referenceName: "refs/heads/main", + }, + extensions: []string{"yml"}, + expect: expectResult{ + err: nil, + matchedCount: 2, + }, + }, + { + name: "list tree with real repository and head ref and non-existing file extension", + args: fetchOption{ + baseOption: baseOption{ + repositoryUrl: privateGitRepoURL, + username: username, + password: accessToken, + }, + referenceName: "refs/heads/main", + }, + extensions: []string{"hcl"}, + expect: expectResult{ + err: nil, + matchedCount: 2, + }, + }, + { + name: "list tree with real repository but non-existing ref", + args: fetchOption{ + baseOption: baseOption{ + repositoryUrl: privateGitRepoURL, + username: username, + password: accessToken, + }, + referenceName: "refs/fake/feature", + }, + extensions: []string{}, + expect: expectResult{ + shouldFail: true, + }, + }, + { + name: "list tree with fake repository ", + args: fetchOption{ + baseOption: baseOption{ + repositoryUrl: privateGitRepoURL + "fake", + username: username, + password: accessToken, + }, + referenceName: "refs/fake/feature", + }, + extensions: []string{}, + expect: expectResult{ + shouldFail: true, + err: ErrIncorrectRepositoryURL, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, tt.extensions) + if tt.expect.shouldFail { + assert.Error(t, err) + if tt.expect.err != nil { + assert.Equal(t, tt.expect.err, err) + } + } else { + assert.NoError(t, err) + if tt.expect.matchedCount > 0 { + assert.Greater(t, len(paths), 0) + } + } + }) + } +} + +func TestService_ListFiles_Github_Concurrently(t *testing.T) { + ensureIntegrationTest(t) + + repositoryUrl := privateGitRepoURL + accessToken := getRequiredValue(t, "GITHUB_PAT") + username := getRequiredValue(t, "GITHUB_USERNAME") + service := newService(context.TODO(), REPOSITORY_CACHE_SIZE, 200*time.Millisecond) + + go service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}) + service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}) + + time.Sleep(2 * time.Second) +} + +func TestService_purgeCache_Github(t *testing.T) { + ensureIntegrationTest(t) + + repositoryUrl := privateGitRepoURL + accessToken := getRequiredValue(t, "GITHUB_PAT") + username := getRequiredValue(t, "GITHUB_USERNAME") + service := NewService(context.TODO()) + + service.ListRefs(repositoryUrl, username, accessToken, false) + service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}) + + assert.Equal(t, 1, service.repoRefCache.Len()) + assert.Equal(t, 1, service.repoFileCache.Len()) + + service.purgeCache() + assert.Equal(t, 0, service.repoRefCache.Len()) + assert.Equal(t, 0, service.repoFileCache.Len()) +} + +func TestService_purgeCacheByTTL_Github(t *testing.T) { + ensureIntegrationTest(t) + + timeout := 100 * time.Millisecond + repositoryUrl := privateGitRepoURL + accessToken := getRequiredValue(t, "GITHUB_PAT") + username := getRequiredValue(t, "GITHUB_USERNAME") + // 40*timeout is designed for giving enough time for ListRefs and ListFiles to cache the result + service := newService(context.TODO(), 2, 40*timeout) + + service.ListRefs(repositoryUrl, username, accessToken, false) + service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}) + assert.Equal(t, 1, service.repoRefCache.Len()) + assert.Equal(t, 1, service.repoFileCache.Len()) + + // 40*timeout is designed for giving enough time for TTL being activated + time.Sleep(40 * timeout) + assert.Equal(t, 0, service.repoRefCache.Len()) + assert.Equal(t, 0, service.repoFileCache.Len()) +} + +func TestService_canStopCacheCleanTimer_whenContextDone(t *testing.T) { + timeout := 10 * time.Millisecond + deadlineCtx, _ := context.WithDeadline(context.TODO(), time.Now().Add(10*timeout)) + + service := NewService(deadlineCtx) + assert.False(t, service.timerHasStopped(), "timer should not be stopped") + + <-time.After(20 * timeout) + + assert.True(t, service.timerHasStopped(), "timer should be stopped") +} + +func TestService_HardRefresh_ListRefs_GitHub(t *testing.T) { + ensureIntegrationTest(t) + + accessToken := getRequiredValue(t, "GITHUB_PAT") + username := getRequiredValue(t, "GITHUB_USERNAME") + service := newService(context.TODO(), 2, 0) + + repositoryUrl := privateGitRepoURL + refs, err := service.ListRefs(repositoryUrl, username, accessToken, false) + assert.NoError(t, err) + assert.GreaterOrEqual(t, len(refs), 1) + assert.Equal(t, 1, service.repoRefCache.Len()) + + refs, err = service.ListRefs(repositoryUrl, username, "fake-token", true) + assert.Error(t, err) + assert.Equal(t, 0, service.repoRefCache.Len()) +} + +func TestService_HardRefresh_ListRefs_And_RemoveAllCaches_GitHub(t *testing.T) { + ensureIntegrationTest(t) + + accessToken := getRequiredValue(t, "GITHUB_PAT") + username := getRequiredValue(t, "GITHUB_USERNAME") + service := newService(context.TODO(), 2, 0) + + repositoryUrl := privateGitRepoURL + refs, err := service.ListRefs(repositoryUrl, username, accessToken, false) + assert.NoError(t, err) + assert.GreaterOrEqual(t, len(refs), 1) + assert.Equal(t, 1, service.repoRefCache.Len()) + + files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}) + assert.NoError(t, err) + assert.GreaterOrEqual(t, len(files), 1) + assert.Equal(t, 1, service.repoFileCache.Len()) + + files, err = service.ListFiles(repositoryUrl, "refs/heads/test", username, accessToken, false, []string{}) + assert.NoError(t, err) + assert.GreaterOrEqual(t, len(files), 1) + assert.Equal(t, 2, service.repoFileCache.Len()) + + refs, err = service.ListRefs(repositoryUrl, username, "fake-token", true) + assert.Error(t, err) + assert.Equal(t, 0, service.repoRefCache.Len()) + + // The relevant file caches should be removed too + assert.Equal(t, 0, service.repoFileCache.Len()) +} + +func TestService_HardRefresh_ListFiles_GitHub(t *testing.T) { + ensureIntegrationTest(t) + + service := newService(context.TODO(), 2, 0) + accessToken := getRequiredValue(t, "GITHUB_PAT") + username := getRequiredValue(t, "GITHUB_USERNAME") + repositoryUrl := privateGitRepoURL + files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}) + assert.NoError(t, err) + assert.GreaterOrEqual(t, len(files), 1) + assert.Equal(t, 1, service.repoFileCache.Len()) + + files, err = service.ListFiles(repositoryUrl, "refs/heads/main", username, "fake-token", true, []string{}) + assert.Error(t, err) + assert.Equal(t, 0, service.repoFileCache.Len()) +} diff --git a/api/git/git_test.go b/api/git/git_test.go index fe8cae2f3..ad97d6afb 100644 --- a/api/git/git_test.go +++ b/api/git/git_test.go @@ -32,7 +32,7 @@ func setup(t *testing.T) string { } func Test_ClonePublicRepository_Shallow(t *testing.T) { - service := Service{git: gitClient{preserveGitDirectory: true}} // no need for http client since the test access the repo via file system. + service := Service{git: NewGitClient(true)} // no need for http client since the test access the repo via file system. repositoryURL := setup(t) referenceName := "refs/heads/main" @@ -44,7 +44,7 @@ func Test_ClonePublicRepository_Shallow(t *testing.T) { } func Test_ClonePublicRepository_NoGitDirectory(t *testing.T) { - service := Service{git: gitClient{preserveGitDirectory: false}} // no need for http client since the test access the repo via file system. + service := Service{git: NewGitClient(false)} // no need for http client since the test access the repo via file system. repositoryURL := setup(t) referenceName := "refs/heads/main" @@ -56,7 +56,7 @@ func Test_ClonePublicRepository_NoGitDirectory(t *testing.T) { } func Test_cloneRepository(t *testing.T) { - service := Service{git: gitClient{preserveGitDirectory: true}} // no need for http client since the test access the repo via file system. + service := Service{git: NewGitClient(true)} // no need for http client since the test access the repo via file system. repositoryURL := setup(t) referenceName := "refs/heads/main" @@ -64,10 +64,14 @@ func Test_cloneRepository(t *testing.T) { dir := t.TempDir() t.Logf("Cloning into %s", dir) - err := service.cloneRepository(dir, cloneOptions{ - repositoryUrl: repositoryURL, - referenceName: referenceName, - depth: 10, + err := service.cloneRepository(dir, cloneOption{ + fetchOption: fetchOption{ + baseOption: baseOption{ + repositoryUrl: repositoryURL, + }, + referenceName: referenceName, + }, + depth: 10, }) assert.NoError(t, err) @@ -75,7 +79,7 @@ func Test_cloneRepository(t *testing.T) { } func Test_latestCommitID(t *testing.T) { - service := Service{git: gitClient{preserveGitDirectory: true}} // no need for http client since the test access the repo via file system. + service := Service{git: NewGitClient(true)} // no need for http client since the test access the repo via file system. repositoryURL := setup(t) referenceName := "refs/heads/main" @@ -106,53 +110,196 @@ func getCommitHistoryLength(t *testing.T, err error, dir string) int { return count } -type testDownloader struct { - called bool -} +func Test_listRefsPrivateRepository(t *testing.T) { + ensureIntegrationTest(t) -func (t *testDownloader) download(_ context.Context, _ string, _ cloneOptions) error { - t.called = true - return nil -} + accessToken := getRequiredValue(t, "GITHUB_PAT") + username := getRequiredValue(t, "GITHUB_USERNAME") -func (t *testDownloader) latestCommitID(_ context.Context, _ fetchOptions) (string, error) { - return "", nil -} + client := NewGitClient(false) + + type expectResult struct { + err error + refsCount int + } -func Test_cloneRepository_azure(t *testing.T) { tests := []struct { name string - url string - called bool + args baseOption + expect expectResult }{ { - name: "Azure HTTP URL", - url: "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository", - called: true, + name: "list refs of a real private repository", + args: baseOption{ + repositoryUrl: privateGitRepoURL, + username: username, + password: accessToken, + }, + expect: expectResult{ + err: nil, + refsCount: 2, + }, }, { - name: "Azure SSH URL", - url: "git@ssh.dev.azure.com:v3/Organisation/Project/Repository", - called: true, + name: "list refs of a real private repository with incorrect credential", + args: baseOption{ + repositoryUrl: privateGitRepoURL, + username: "test-username", + password: "test-token", + }, + expect: expectResult{ + err: ErrAuthenticationFailure, + }, }, { - name: "Something else", - url: "https://example.com", - called: false, + name: "list refs of a fake repository without providing credential", + args: baseOption{ + repositoryUrl: privateGitRepoURL + "fake", + username: "", + password: "", + }, + expect: expectResult{ + err: ErrAuthenticationFailure, + }, + }, + { + name: "list refs of a fake repository", + args: baseOption{ + repositoryUrl: privateGitRepoURL + "fake", + username: username, + password: accessToken, + }, + expect: expectResult{ + err: ErrIncorrectRepositoryURL, + }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - azure := &testDownloader{} - git := &testDownloader{} - - s := &Service{azure: azure, git: git} - s.cloneRepository("", cloneOptions{repositoryUrl: tt.url, depth: 1}) - - // if azure API is called, git isn't and vice versa - assert.Equal(t, tt.called, azure.called) - assert.Equal(t, tt.called, !git.called) + refs, err := client.listRefs(context.TODO(), tt.args) + if tt.expect.err == nil { + assert.NoError(t, err) + if tt.expect.refsCount > 0 { + assert.Greater(t, len(refs), 0) + } + } else { + assert.Error(t, err) + assert.Equal(t, tt.expect.err, err) + } + }) + } +} + +func Test_listFilesPrivateRepository(t *testing.T) { + ensureIntegrationTest(t) + + client := NewGitClient(false) + + type expectResult struct { + shouldFail bool + err error + matchedCount int + } + + accessToken := getRequiredValue(t, "GITHUB_PAT") + username := getRequiredValue(t, "GITHUB_USERNAME") + + tests := []struct { + name string + args fetchOption + expect expectResult + }{ + { + name: "list tree with real repository and head ref but incorrect credential", + args: fetchOption{ + baseOption: baseOption{ + repositoryUrl: privateGitRepoURL, + username: "test-username", + password: "test-token", + }, + referenceName: "refs/heads/main", + }, + expect: expectResult{ + shouldFail: true, + err: ErrAuthenticationFailure, + }, + }, + { + name: "list tree with real repository and head ref but no credential", + args: fetchOption{ + baseOption: baseOption{ + repositoryUrl: privateGitRepoURL + "fake", + username: "", + password: "", + }, + referenceName: "refs/heads/main", + }, + expect: expectResult{ + shouldFail: true, + err: ErrAuthenticationFailure, + }, + }, + { + name: "list tree with real repository and head ref", + args: fetchOption{ + baseOption: baseOption{ + repositoryUrl: privateGitRepoURL, + username: username, + password: accessToken, + }, + referenceName: "refs/heads/main", + }, + expect: expectResult{ + err: nil, + matchedCount: 15, + }, + }, + { + name: "list tree with real repository but non-existing ref", + args: fetchOption{ + baseOption: baseOption{ + repositoryUrl: privateGitRepoURL, + username: username, + password: accessToken, + }, + referenceName: "refs/fake/feature", + }, + expect: expectResult{ + shouldFail: true, + }, + }, + { + name: "list tree with fake repository ", + args: fetchOption{ + baseOption: baseOption{ + repositoryUrl: privateGitRepoURL + "fake", + username: username, + password: accessToken, + }, + referenceName: "refs/fake/feature", + }, + expect: expectResult{ + shouldFail: true, + err: ErrIncorrectRepositoryURL, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + paths, err := client.listFiles(context.TODO(), tt.args) + if tt.expect.shouldFail { + assert.Error(t, err) + if tt.expect.err != nil { + assert.Equal(t, tt.expect.err, err) + } + } else { + assert.NoError(t, err) + if tt.expect.matchedCount > 0 { + assert.Greater(t, len(paths), 0) + } + } }) } } diff --git a/api/git/service.go b/api/git/service.go new file mode 100644 index 000000000..f7cd55222 --- /dev/null +++ b/api/git/service.go @@ -0,0 +1,320 @@ +package git + +import ( + "context" + "errors" + "log" + "strings" + "sync" + "time" + + lru "github.com/hashicorp/golang-lru" +) + +var ( + ErrIncorrectRepositoryURL = errors.New("Git repository could not be found, please ensure that the URL is correct.") + ErrAuthenticationFailure = errors.New("Authentication failed, please ensure that the git credentials are correct.") + + REPOSITORY_CACHE_SIZE = 4 + REPOSITORY_CACHE_TTL = 5 * time.Minute +) + +// baseOption provides a minimum group of information to operate a git repository, like git-remote +type baseOption struct { + repositoryUrl string + username string + password string +} + +// fetchOption allows to specify the reference name of the target repository +type fetchOption struct { + baseOption + referenceName string +} + +// cloneOption allows to add a history truncated to the specified number of commits +type cloneOption struct { + fetchOption + depth int +} + +type repoManager interface { + download(ctx context.Context, dst string, opt cloneOption) error + latestCommitID(ctx context.Context, opt fetchOption) (string, error) + listRefs(ctx context.Context, opt baseOption) ([]string, error) + listFiles(ctx context.Context, opt fetchOption) ([]string, error) +} + +// Service represents a service for managing Git. +type Service struct { + shutdownCtx context.Context + azure repoManager + git repoManager + timerStopped bool + mut sync.Mutex + + cacheEnabled bool + // Cache the result of repository refs, key is repository URL + repoRefCache *lru.Cache + // Cache the result of repository file tree, key is the concatenated string of repository URL and ref value + repoFileCache *lru.Cache +} + +// NewService initializes a new service. +func NewService(ctx context.Context) *Service { + return newService(ctx, REPOSITORY_CACHE_SIZE, REPOSITORY_CACHE_TTL) +} + +func newService(ctx context.Context, cacheSize int, cacheTTL time.Duration) *Service { + service := &Service{ + shutdownCtx: ctx, + azure: NewAzureClient(), + git: NewGitClient(false), + timerStopped: false, + cacheEnabled: cacheSize > 0, + } + + if service.cacheEnabled { + var err error + service.repoRefCache, err = lru.New(cacheSize) + if err != nil { + log.Printf("[DEBUG] [git] [message: failed to create ref cache: %v\n", err) + } + + service.repoFileCache, err = lru.New(cacheSize) + if err != nil { + log.Printf("[DEBUG] [git] [message: failed to create file cache: %v\n", err) + } + + if cacheTTL > 0 { + go service.startCacheCleanTimer(cacheTTL) + } + } + + return service +} + +// startCacheCleanTimer starts a timer to purge caches periodically +func (service *Service) startCacheCleanTimer(d time.Duration) { + ticker := time.NewTicker(d) + + for { + select { + case <-ticker.C: + service.purgeCache() + + case <-service.shutdownCtx.Done(): + ticker.Stop() + service.mut.Lock() + service.timerStopped = true + service.mut.Unlock() + return + } + } +} + +// timerHasStopped shows the CacheClean timer state with thread-safe way +func (service *Service) timerHasStopped() bool { + service.mut.Lock() + defer service.mut.Unlock() + ret := service.timerStopped + return ret +} + +// CloneRepository clones a git repository using the specified URL in the specified +// destination folder. +func (service *Service) CloneRepository(destination, repositoryURL, referenceName, username, password string) error { + options := cloneOption{ + fetchOption: fetchOption{ + baseOption: baseOption{ + repositoryUrl: repositoryURL, + username: username, + password: password, + }, + referenceName: referenceName, + }, + depth: 1, + } + + return service.cloneRepository(destination, options) +} + +func (service *Service) cloneRepository(destination string, options cloneOption) error { + if isAzureUrl(options.repositoryUrl) { + return service.azure.download(context.TODO(), destination, options) + } + + return service.git.download(context.TODO(), destination, options) +} + +// LatestCommitID returns SHA1 of the latest commit of the specified reference +func (service *Service) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) { + options := fetchOption{ + baseOption: baseOption{ + repositoryUrl: repositoryURL, + username: username, + password: password, + }, + referenceName: referenceName, + } + + if isAzureUrl(options.repositoryUrl) { + return service.azure.latestCommitID(context.TODO(), options) + } + + return service.git.latestCommitID(context.TODO(), options) +} + +// ListRefs will list target repository's references without cloning the repository +func (service *Service) ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error) { + if service.cacheEnabled && hardRefresh { + // Should remove the cache explicitly, so that the following normal list can show the correct result + service.repoRefCache.Remove(repositoryURL) + // Remove file caches pointed to the same repository + for _, fileCacheKey := range service.repoFileCache.Keys() { + key, ok := fileCacheKey.(string) + if ok { + if strings.HasPrefix(key, repositoryURL) { + service.repoFileCache.Remove(key) + } + } + } + } + + if service.repoRefCache != nil { + // Lookup the refs cache first + cache, ok := service.repoRefCache.Get(repositoryURL) + if ok { + refs, success := cache.([]string) + if success { + return refs, nil + } + } + } + + options := baseOption{ + repositoryUrl: repositoryURL, + username: username, + password: password, + } + + var ( + refs []string + err error + ) + if isAzureUrl(options.repositoryUrl) { + refs, err = service.azure.listRefs(context.TODO(), options) + if err != nil { + return nil, err + } + } else { + refs, err = service.git.listRefs(context.TODO(), options) + if err != nil { + return nil, err + } + } + + if service.cacheEnabled && service.repoRefCache != nil { + service.repoRefCache.Add(options.repositoryUrl, refs) + } + return refs, nil +} + +// ListFiles will list all the files of the target repository with specific extensions. +// If extension is not provided, it will list all the files under the target repository +func (service *Service) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string) ([]string, error) { + repoKey := generateCacheKey(repositoryURL, referenceName) + + if service.cacheEnabled && hardRefresh { + // Should remove the cache explicitly, so that the following normal list can show the correct result + service.repoFileCache.Remove(repoKey) + } + + if service.repoFileCache != nil { + // lookup the files cache first + cache, ok := service.repoFileCache.Get(repoKey) + if ok { + files, success := cache.([]string) + if success { + // For the case while searching files in a repository without include extensions for the first time, + // but with include extensions for the second time + includedFiles := filterFiles(files, includedExts) + return includedFiles, nil + } + } + } + + options := fetchOption{ + baseOption: baseOption{ + repositoryUrl: repositoryURL, + username: username, + password: password, + }, + referenceName: referenceName, + } + + var ( + files []string + err error + ) + if isAzureUrl(options.repositoryUrl) { + files, err = service.azure.listFiles(context.TODO(), options) + if err != nil { + return nil, err + } + } else { + files, err = service.git.listFiles(context.TODO(), options) + if err != nil { + return nil, err + } + } + + includedFiles := filterFiles(files, includedExts) + if service.cacheEnabled && service.repoFileCache != nil { + service.repoFileCache.Add(repoKey, includedFiles) + return includedFiles, nil + } + return includedFiles, nil +} + +func (service *Service) purgeCache() { + if service.repoRefCache != nil { + service.repoRefCache.Purge() + } + + if service.repoFileCache != nil { + service.repoFileCache.Purge() + } +} + +func generateCacheKey(names ...string) string { + return strings.Join(names, "-") +} + +func matchExtensions(target string, exts []string) bool { + if len(exts) == 0 { + return true + } + + for _, ext := range exts { + if strings.HasSuffix(target, ext) { + return true + } + } + return false +} + +func filterFiles(paths []string, includedExts []string) []string { + if len(includedExts) == 0 { + return paths + } + + var includedFiles []string + for _, filename := range paths { + // filter out the filenames with non-included extension + if matchExtensions(filename, includedExts) { + includedFiles = append(includedFiles, filename) + } + } + return includedFiles +} diff --git a/api/http/handler/edgestacks/edgestack_test.go b/api/http/handler/edgestacks/edgestack_test.go index 1d02c8f8a..cc9267b22 100644 --- a/api/http/handler/edgestacks/edgestack_test.go +++ b/api/http/handler/edgestacks/edgestack_test.go @@ -34,6 +34,14 @@ func (g *gitService) LatestCommitID(repositoryURL, referenceName, username, pass return g.id, nil } +func (g *gitService) ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error) { + return nil, nil +} + +func (g *gitService) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string) ([]string, error) { + return nil, nil +} + // Helpers func setupHandler(t *testing.T) (*Handler, string, func()) { t.Helper() diff --git a/api/http/proxy/factory/docker/transport_test.go b/api/http/proxy/factory/docker/transport_test.go index f8f066d99..cff9dd7fe 100644 --- a/api/http/proxy/factory/docker/transport_test.go +++ b/api/http/proxy/factory/docker/transport_test.go @@ -1,11 +1,12 @@ package docker import ( - portainer "github.com/portainer/portainer/api" - "github.com/stretchr/testify/assert" "net/http" "net/http/httptest" "testing" + + portainer "github.com/portainer/portainer/api" + "github.com/stretchr/testify/assert" ) type noopGitService struct{} @@ -16,6 +17,12 @@ func (s *noopGitService) CloneRepository(destination string, repositoryURL, refe func (s *noopGitService) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) { return "my-latest-commit-id", nil } +func (g *noopGitService) ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error) { + return nil, nil +} +func (g *noopGitService) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string) ([]string, error) { + return nil, nil +} func TestTransport_updateDefaultGitBranch(t *testing.T) { type fields struct { diff --git a/api/portainer.go b/api/portainer.go index b148e49f9..48e9ba073 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -1323,6 +1323,8 @@ type ( GitService interface { CloneRepository(destination string, repositoryURL, referenceName, username, password string) error LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) + ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error) + ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includeExts []string) ([]string, error) } // OpenAMTService represents a service for managing OpenAMT diff --git a/api/stacks/deploy_test.go b/api/stacks/deploy_test.go index 4a45a4b18..2148e8da8 100644 --- a/api/stacks/deploy_test.go +++ b/api/stacks/deploy_test.go @@ -25,6 +25,14 @@ func (g *gitService) LatestCommitID(repositoryURL, referenceName, username, pass return g.id, nil } +func (g *gitService) ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error) { + return nil, nil +} + +func (g *gitService) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string) ([]string, error) { + return nil, nil +} + type noopDeployer struct{} func (s *noopDeployer) DeploySwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, prune bool, pullImage bool) error { diff --git a/app/portainer/components/forms/git-form/git-form.html b/app/portainer/components/forms/git-form/git-form.html index 8d241e702..00d2c2d1e 100644 --- a/app/portainer/components/forms/git-form/git-form.html +++ b/app/portainer/components/forms/git-form/git-form.html @@ -1,7 +1,11 @@
Git repository
+ + + + - -