github
/
portainer
mirror of https://github.com/portainer/portainer
1
0
Fork 0
Code Issues Releases Wiki Activity

feat(gitops): support to list git repository refs and file tree [EE-2673] (#7100)

pull/7203/head
Oscar Zhou 2022-09-21 17:47:02 +12:00 committed by GitHub
parent ef1d648c07
commit 5777c18297
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 1673 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
api/cmd/portainer/main.go
View File

@ -210,8 +210,8 @@ func initOAuthService() portainer.OAuthService {
return oauth.NewService()
}
func initGitService() portainer.GitService {
return git.NewService()
func initGitService(ctx context.Context) portainer.GitService {
return git.NewService(ctx)
}
func initSSLService(addr, certPath, keyPath string, fileService portainer.FileService, dataStore dataservices.DataStore, shutdownTrigger context.CancelFunc) (*ssl.Service, error) {
@ -580,8 +580,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
ldapService := initLDAPService()
oauthService := initOAuthService()
gitService := initGitService()
gitService := initGitService(shutdownCtx)
openAMTService := openamt.NewService()

255
api/git/azure.go
View File

@ -2,6 +2,7 @@ package git
import (
"context"
"crypto/tls"
"encoding/json"
"fmt"
"io"
@ -10,7 +11,10 @@ import (
"net/url"
"os"
"strings"
"time"
"github.com/go-git/go-git/v5/plumbing/transport/client"
githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/pkg/errors"
"github.com/portainer/portainer/api/archive"
)
@ -32,20 +36,47 @@ type azureOptions struct {
username, password string
}
type azureDownloader struct {
// azureRef abstracts from the response of https://docs.microsoft.com/en-us/rest/api/azure/devops/git/refs/list?view=azure-devops-rest-6.0#refs
type azureRef struct {
Name string `json:"name"`
ObjectID string `json:"objectId"`
}
// azureItem abstracts from the response of https://docs.microsoft.com/en-us/rest/api/azure/devops/git/items/get?view=azure-devops-rest-6.0#download
type azureItem struct {
ObjectID string `json:"objectId"`
CommitId string `json:"commitId"`
Path string `json:"path"`
}
type azureClient struct {
client *http.Client
baseUrl string
}
func NewAzureDownloader(client *http.Client) *azureDownloader {
return &azureDownloader{
client: client,
func NewAzureClient() *azureClient {
httpsCli := newHttpClientForAzure()
return &azureClient{
client: httpsCli,
baseUrl: "https://dev.azure.com",
}
}
func (a *azureDownloader) download(ctx context.Context, destination string, options cloneOptions) error {
zipFilepath, err := a.downloadZipFromAzureDevOps(ctx, options)
func newHttpClientForAzure() *http.Client {
httpsCli := &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
Proxy: http.ProxyFromEnvironment,
},
Timeout: 300 * time.Second,
}
client.InstallProtocol("https", githttp.NewClient(httpsCli))
return httpsCli
}
func (a *azureClient) download(ctx context.Context, destination string, opt cloneOption) error {
zipFilepath, err := a.downloadZipFromAzureDevOps(ctx, opt)
if err != nil {
return errors.Wrap(err, "failed to download a zip file from Azure DevOps")
}
@ -59,12 +90,12 @@ func (a *azureDownloader) download(ctx context.Context, destination string, opti
return nil
}
func (a *azureDownloader) downloadZipFromAzureDevOps(ctx context.Context, options cloneOptions) (string, error) {
config, err := parseUrl(options.repositoryUrl)
func (a *azureClient) downloadZipFromAzureDevOps(ctx context.Context, opt cloneOption) (string, error) {
config, err := parseUrl(opt.repositoryUrl)
if err != nil {
return "", errors.WithMessage(err, "failed to parse url")
}
downloadUrl, err := a.buildDownloadUrl(config, options.referenceName)
downloadUrl, err := a.buildDownloadUrl(config, opt.referenceName)
if err != nil {
return "", errors.WithMessage(err, "failed to build download url")
}
@ -75,8 +106,8 @@ func (a *azureDownloader) downloadZipFromAzureDevOps(ctx context.Context, option
defer zipFile.Close()
req, err := http.NewRequestWithContext(ctx, "GET", downloadUrl, nil)
if options.username != "" || options.password != "" {
req.SetBasicAuth(options.username, options.password)
if opt.username != "" || opt.password != "" {
req.SetBasicAuth(opt.username, opt.password)
} else if config.username != "" || config.password != "" {
req.SetBasicAuth(config.username, config.password)
}
@ -102,53 +133,58 @@ func (a *azureDownloader) downloadZipFromAzureDevOps(ctx context.Context, option
return zipFile.Name(), nil
}
func (a *azureDownloader) latestCommitID(ctx context.Context, options fetchOptions) (string, error) {
config, err := parseUrl(options.repositoryUrl)
func (a *azureClient) latestCommitID(ctx context.Context, opt fetchOption) (string, error) {
rootItem, err := a.getRootItem(ctx, opt)
if err != nil {
return "", errors.WithMessage(err, "failed to parse url")
return "", err
}
return rootItem.CommitId, nil
}
func (a *azureClient) getRootItem(ctx context.Context, opt fetchOption) (*azureItem, error) {
config, err := parseUrl(opt.repositoryUrl)
if err != nil {
return nil, errors.WithMessage(err, "failed to parse url")
}
rootItemUrl, err := a.buildRootItemUrl(config, options.referenceName)
rootItemUrl, err := a.buildRootItemUrl(config, opt.referenceName)
if err != nil {
return "", errors.WithMessage(err, "failed to build azure root item url")
return nil, errors.WithMessage(err, "failed to build azure root item url")
}
req, err := http.NewRequestWithContext(ctx, "GET", rootItemUrl, nil)
if options.username != "" || options.password != "" {
req.SetBasicAuth(options.username, options.password)
if opt.username != "" || opt.password != "" {
req.SetBasicAuth(opt.username, opt.password)
} else if config.username != "" || config.password != "" {
req.SetBasicAuth(config.username, config.password)
}
if err != nil {
return "", errors.WithMessage(err, "failed to create a new HTTP request")
return nil, errors.WithMessage(err, "failed to create a new HTTP request")
}
resp, err := a.client.Do(req)
if err != nil {
return "", errors.WithMessage(err, "failed to make an HTTP request")
return nil, errors.WithMessage(err, "failed to make an HTTP request")
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("failed to get repository root item with a status \"%v\"", resp.Status)
return nil, checkAzureStatusCode(fmt.Errorf("failed to get repository root item with a status \"%v\"", resp.Status), resp.StatusCode)
}
var items struct {
Value []struct {
CommitId string `json:"commitId"`
}
Value []azureItem
}
if err := json.NewDecoder(resp.Body).Decode(&items); err != nil {
return "", errors.Wrap(err, "could not parse Azure items response")
return nil, errors.Wrap(err, "could not parse Azure items response")
}
if len(items.Value) == 0 || items.Value[0].CommitId == "" {
return "", errors.Errorf("failed to get latest commitID in the repository")
return nil, errors.Errorf("failed to get latest commitID in the repository")
}
return items.Value[0].CommitId, nil
return &items.Value[0], nil
}
func parseUrl(rawUrl string) (*azureOptions, error) {
@ -219,7 +255,7 @@ func parseHttpUrl(rawUrl string) (*azureOptions, error) {
return &opt, nil
}
func (a *azureDownloader) buildDownloadUrl(config *azureOptions, referenceName string) (string, error) {
func (a *azureClient) buildDownloadUrl(config *azureOptions, referenceName string) (string, error) {
rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/items",
a.baseUrl,
url.PathEscape(config.organisation),
@ -246,7 +282,7 @@ func (a *azureDownloader) buildDownloadUrl(config *azureOptions, referenceName s
return u.String(), nil
}
func (a *azureDownloader) buildRootItemUrl(config *azureOptions, referenceName string) (string, error) {
func (a *azureClient) buildRootItemUrl(config *azureOptions, referenceName string) (string, error) {
rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/items",
a.baseUrl,
url.PathEscape(config.organisation),
@ -270,6 +306,49 @@ func (a *azureDownloader) buildRootItemUrl(config *azureOptions, referenceName s
return u.String(), nil
}
func (a *azureClient) buildRefsUrl(config *azureOptions) (string, error) {
// ref@https://docs.microsoft.com/en-us/rest/api/azure/devops/git/refs/list?view=azure-devops-rest-6.0#gitref
rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/refs",
a.baseUrl,
url.PathEscape(config.organisation),
url.PathEscape(config.project),
url.PathEscape(config.repository))
u, err := url.Parse(rawUrl)
if err != nil {
return "", errors.Wrapf(err, "failed to parse list refs url path %s", rawUrl)
}
q := u.Query()
q.Set("api-version", "6.0")
u.RawQuery = q.Encode()
return u.String(), nil
}
func (a *azureClient) buildTreeUrl(config *azureOptions, rootObjectHash string) (string, error) {
// ref@https://docs.microsoft.com/en-us/rest/api/azure/devops/git/trees/get?view=azure-devops-rest-6.0
rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/trees/%s",
a.baseUrl,
url.PathEscape(config.organisation),
url.PathEscape(config.project),
url.PathEscape(config.repository),
url.PathEscape(rootObjectHash),
)
u, err := url.Parse(rawUrl)
if err != nil {
return "", errors.Wrapf(err, "failed to parse list tree url path %s", rawUrl)
}
q := u.Query()
// projectId={projectId}&recursive=true&fileName={fileName}&$format={$format}&api-version=6.0
q.Set("recursive", "true")
q.Set("api-version", "6.0")
u.RawQuery = q.Encode()
return u.String(), nil
}
const (
branchPrefix = "refs/heads/"
tagPrefix = "refs/tags/"
@ -294,3 +373,119 @@ func getVersionType(name string) string {
}
return "commit"
}
func (a *azureClient) listRefs(ctx context.Context, opt baseOption) ([]string, error) {
config, err := parseUrl(opt.repositoryUrl)
if err != nil {
return nil, errors.WithMessage(err, "failed to parse url")
}
listRefsUrl, err := a.buildRefsUrl(config)
if err != nil {
return nil, errors.WithMessage(err, "failed to build list refs url")
}
req, err := http.NewRequestWithContext(ctx, "GET", listRefsUrl, nil)
if opt.username != "" || opt.password != "" {
req.SetBasicAuth(opt.username, opt.password)
} else if config.username != "" || config.password != "" {
req.SetBasicAuth(config.username, config.password)
}
if err != nil {
return nil, errors.WithMessage(err, "failed to create a new HTTP request")
}
resp, err := a.client.Do(req)
if err != nil {
return nil, errors.WithMessage(err, "failed to make an HTTP request")
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, checkAzureStatusCode(fmt.Errorf("failed to list refs with a status \"%v\"", resp.Status), resp.StatusCode)
}
var refs struct {
Value []azureRef
}
if err := json.NewDecoder(resp.Body).Decode(&refs); err != nil {
return nil, errors.Wrap(err, "could not parse Azure refs response")
}
var ret []string
for _, value := range refs.Value {
if value.Name == "HEAD" {
continue
}
ret = append(ret, value.Name)
}
return ret, nil
}
// listFiles list all filenames under the specific repository
func (a *azureClient) listFiles(ctx context.Context, opt fetchOption) ([]string, error) {
rootItem, err := a.getRootItem(ctx, opt)
if err != nil {
return nil, err
}
config, err := parseUrl(opt.repositoryUrl)
if err != nil {
return nil, errors.WithMessage(err, "failed to parse url")
}
listTreeUrl, err := a.buildTreeUrl(config, rootItem.ObjectID)
if err != nil {
return nil, errors.WithMessage(err, "failed to build list tree url")
}
req, err := http.NewRequestWithContext(ctx, "GET", listTreeUrl, nil)
if opt.username != "" || opt.password != "" {
req.SetBasicAuth(opt.username, opt.password)
} else if config.username != "" || config.password != "" {
req.SetBasicAuth(config.username, config.password)
}
if err != nil {
return nil, errors.WithMessage(err, "failed to create a new HTTP request")
}
resp, err := a.client.Do(req)
if err != nil {
return nil, errors.WithMessage(err, "failed to make an HTTP request")
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("failed to list tree url with a status \"%v\"", resp.Status)
}
var tree struct {
TreeEntries []struct {
RelativePath string `json:"relativePath"`
} `json:"treeEntries"`
}
if err := json.NewDecoder(resp.Body).Decode(&tree); err != nil {
return nil, errors.Wrap(err, "could not parse Azure tree response")
}
var allPaths []string
for _, treeEntry := range tree.TreeEntries {
allPaths = append(allPaths, treeEntry.RelativePath)
}
return allPaths, nil
}
func checkAzureStatusCode(err error, code int) error {
if code == http.StatusNotFound {
return ErrIncorrectRepositoryURL
} else if code == http.StatusUnauthorized || code == http.StatusNonAuthoritativeInfo {
return ErrAuthenticationFailure
}
return err
}

211
api/git/azure_integration_test.go
View File

@ -1,20 +1,26 @@
package git
import (
"context"
"fmt"
"os"
"path/filepath"
"testing"
"time"
_ "github.com/joho/godotenv/autoload"
"github.com/stretchr/testify/assert"
)
var (
privateAzureRepoURL = "https://portainer.visualstudio.com/gitops-test/_git/gitops-test"
)
func TestService_ClonePublicRepository_Azure(t *testing.T) {
ensureIntegrationTest(t)
pat := getRequiredValue(t, "AZURE_DEVOPS_PAT")
service := NewService()
service := NewService(context.TODO())
type args struct {
repositoryURLFormat string
@ -30,7 +36,7 @@ func TestService_ClonePublicRepository_Azure(t *testing.T) {
{
name: "Clone Azure DevOps repo branch",
args: args{
repositoryURLFormat: "https://:%s@portainer.visualstudio.com/Playground/_git/dev_integration",
repositoryURLFormat: "https://:%s@portainer.visualstudio.com/gitops-test/_git/gitops-test",
referenceName: "refs/heads/main",
username: "",
password: pat,
@ -40,8 +46,8 @@ func TestService_ClonePublicRepository_Azure(t *testing.T) {
{
name: "Clone Azure DevOps repo tag",
args: args{
repositoryURLFormat: "https://:%s@portainer.visualstudio.com/Playground/_git/dev_integration",
referenceName: "refs/tags/v1.1",
repositoryURLFormat: "https://:%s@portainer.visualstudio.com/gitops-test/_git/gitops-test",
referenceName: "refs/heads/tags/v1.1",
username: "",
password: pat,
},
@ -63,12 +69,11 @@ func TestService_ClonePrivateRepository_Azure(t *testing.T) {
ensureIntegrationTest(t)
pat := getRequiredValue(t, "AZURE_DEVOPS_PAT")
service := NewService()
service := NewService(context.TODO())
dst := t.TempDir()
repositoryUrl := "https://portainer.visualstudio.com/Playground/_git/dev_integration"
err := service.CloneRepository(dst, repositoryUrl, "refs/heads/main", "", pat)
err := service.CloneRepository(dst, privateAzureRepoURL, "refs/heads/main", "", pat)
assert.NoError(t, err)
assert.FileExists(t, filepath.Join(dst, "README.md"))
}
@ -77,14 +82,200 @@ func TestService_LatestCommitID_Azure(t *testing.T) {
ensureIntegrationTest(t)
pat := getRequiredValue(t, "AZURE_DEVOPS_PAT")
service := NewService()
service := NewService(context.TODO())
repositoryUrl := "https://portainer.visualstudio.com/Playground/_git/dev_integration"
id, err := service.LatestCommitID(repositoryUrl, "refs/heads/main", "", pat)
id, err := service.LatestCommitID(privateAzureRepoURL, "refs/heads/main", "", pat)
assert.NoError(t, err)
assert.NotEmpty(t, id, "cannot guarantee commit id, but it should be not empty")
}
func TestService_ListRefs_Azure(t *testing.T) {
ensureIntegrationTest(t)
accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
service := NewService(context.TODO())
refs, err := service.ListRefs(privateAzureRepoURL, username, accessToken, false)
assert.NoError(t, err)
assert.GreaterOrEqual(t, len(refs), 1)
}
func TestService_ListRefs_Azure_Concurrently(t *testing.T) {
ensureIntegrationTest(t)
accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
service := newService(context.TODO(), REPOSITORY_CACHE_SIZE, 200*time.Millisecond)
go service.ListRefs(privateAzureRepoURL, username, accessToken, false)
service.ListRefs(privateAzureRepoURL, username, accessToken, false)
time.Sleep(2 * time.Second)
}
func TestService_ListFiles_Azure(t *testing.T) {
ensureIntegrationTest(t)
type expectResult struct {
shouldFail bool
err error
matchedCount int
}
service := newService(context.TODO(), 0, 0)
accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
tests := []struct {
name string
args fetchOption
extensions []string
expect expectResult
}{
{
name: "list tree with real repository and head ref but incorrect credential",
args: fetchOption{
baseOption: baseOption{
repositoryUrl: privateAzureRepoURL,
username: "test-username",
password: "test-token",
},
referenceName: "refs/heads/main",
},
extensions: []string{},
expect: expectResult{
shouldFail: true,
err: ErrAuthenticationFailure,
},
},
{
name: "list tree with real repository and head ref but no credential",
args: fetchOption{
baseOption: baseOption{
repositoryUrl: privateAzureRepoURL,
username: "",
password: "",
},
referenceName: "refs/heads/main",
},
extensions: []string{},
expect: expectResult{
shouldFail: true,
err: ErrAuthenticationFailure,
},
},
{
name: "list tree with real repository and head ref",
args: fetchOption{
baseOption: baseOption{
repositoryUrl: privateAzureRepoURL,
username: username,
password: accessToken,
},
referenceName: "refs/heads/main",
},
extensions: []string{},
expect: expectResult{
err: nil,
matchedCount: 19,
},
},
{
name: "list tree with real repository and head ref and existing file extension",
args: fetchOption{
baseOption: baseOption{
repositoryUrl: privateAzureRepoURL,
username: username,
password: accessToken,
},
referenceName: "refs/heads/main",
},
extensions: []string{"yml"},
expect: expectResult{
err: nil,
matchedCount: 2,
},
},
{
name: "list tree with real repository and head ref and non-existing file extension",
args: fetchOption{
baseOption: baseOption{
repositoryUrl: privateAzureRepoURL,
username: username,
password: accessToken,
},
referenceName: "refs/heads/main",
},
extensions: []string{"hcl"},
expect: expectResult{
err: nil,
matchedCount: 2,
},
},
{
name: "list tree with real repository but non-existing ref",
args: fetchOption{
baseOption: baseOption{
repositoryUrl: privateAzureRepoURL,
username: username,
password: accessToken,
},
referenceName: "refs/fake/feature",
},
extensions: []string{},
expect: expectResult{
shouldFail: true,
},
},
{
name: "list tree with fake repository ",
args: fetchOption{
baseOption: baseOption{
repositoryUrl: privateAzureRepoURL + "fake",
username: username,
password: accessToken,
},
referenceName: "refs/fake/feature",
},
extensions: []string{},
expect: expectResult{
shouldFail: true,
err: ErrIncorrectRepositoryURL,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, tt.extensions)
if tt.expect.shouldFail {
assert.Error(t, err)
if tt.expect.err != nil {
assert.Equal(t, tt.expect.err, err)
}
} else {
assert.NoError(t, err)
if tt.expect.matchedCount > 0 {
assert.Greater(t, len(paths), 0)
}
}
})
}
}
func TestService_ListFiles_Azure_Concurrently(t *testing.T) {
ensureIntegrationTest(t)
accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
service := newService(context.TODO(), REPOSITORY_CACHE_SIZE, 200*time.Millisecond)
go service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, []string{})
service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, []string{})
time.Sleep(2 * time.Second)
}
func getRequiredValue(t *testing.T, name string) string {
value, ok := os.LookupEnv(name)
if !ok {

326
api/git/azure_test.go
View File

@ -11,7 +11,7 @@ import (
)
func Test_buildDownloadUrl(t *testing.T) {
a := NewAzureDownloader(nil)
a := NewAzureClient()
u, err := a.buildDownloadUrl(&azureOptions{
organisation: "organisation",
project: "project",
@ -29,7 +29,7 @@ func Test_buildDownloadUrl(t *testing.T) {
}
func Test_buildRootItemUrl(t *testing.T) {
a := NewAzureDownloader(nil)
a := NewAzureClient()
u, err := a.buildRootItemUrl(&azureOptions{
organisation: "organisation",
project: "project",
@ -45,6 +45,40 @@ func Test_buildRootItemUrl(t *testing.T) {
assert.Equal(t, expectedUrl.Query(), actualUrl.Query())
}
func Test_buildRefsUrl(t *testing.T) {
a := NewAzureClient()
u, err := a.buildRefsUrl(&azureOptions{
organisation: "organisation",
project: "project",
repository: "repository",
})
expectedUrl, _ := url.Parse("https://dev.azure.com/organisation/project/_apis/git/repositories/repository/refs?api-version=6.0")
actualUrl, _ := url.Parse(u)
assert.NoError(t, err)
assert.Equal(t, expectedUrl.Host, actualUrl.Host)
assert.Equal(t, expectedUrl.Scheme, actualUrl.Scheme)
assert.Equal(t, expectedUrl.Path, actualUrl.Path)
assert.Equal(t, expectedUrl.Query(), actualUrl.Query())
}
func Test_buildTreeUrl(t *testing.T) {
a := NewAzureClient()
u, err := a.buildTreeUrl(&azureOptions{
organisation: "organisation",
project: "project",
repository: "repository",
}, "sha1")
expectedUrl, _ := url.Parse("https://dev.azure.com/organisation/project/_apis/git/repositories/repository/trees/sha1?api-version=6.0&recursive=true")
actualUrl, _ := url.Parse(u)
assert.NoError(t, err)
assert.Equal(t, expectedUrl.Host, actualUrl.Host)
assert.Equal(t, expectedUrl.Scheme, actualUrl.Scheme)
assert.Equal(t, expectedUrl.Path, actualUrl.Path)
assert.Equal(t, expectedUrl.Query(), actualUrl.Query())
}
func Test_parseAzureUrl(t *testing.T) {
type args struct {
url string
@ -200,7 +234,7 @@ func Test_isAzureUrl(t *testing.T) {
func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
type args struct {
options cloneOptions
options baseOption
}
type basicAuth struct {
username, password string
@ -213,7 +247,7 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
{
name: "username, password embedded",
args: args{
options: cloneOptions{
options: baseOption{
repositoryUrl: "https://username:password@dev.azure.com/Organisation/Project/_git/Repository",
},
},
@ -225,7 +259,7 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
{
name: "username, password embedded, clone options take precedence",
args: args{
options: cloneOptions{
options: baseOption{
repositoryUrl: "https://username:password@dev.azure.com/Organisation/Project/_git/Repository",
username: "u",
password: "p",
@ -239,7 +273,7 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
{
name: "no credentials",
args: args{
options: cloneOptions{
options: baseOption{
repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository",
},
},
@ -256,11 +290,17 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
}))
defer server.Close()
a := &azureDownloader{
a := &azureClient{
client: server.Client(),
baseUrl: server.URL,
}
_, err := a.downloadZipFromAzureDevOps(context.Background(), tt.args.options)
option := cloneOption{
fetchOption: fetchOption{
baseOption: tt.args.options,
},
}
_, err := a.downloadZipFromAzureDevOps(context.Background(), option)
assert.Error(t, err)
assert.Equal(t, tt.want, zipRequestAuth)
})
@ -287,22 +327,25 @@ func Test_azureDownloader_latestCommitID(t *testing.T) {
}))
defer server.Close()
a := &azureDownloader{
a := &azureClient{
client: server.Client(),
baseUrl: server.URL,
}
tests := []struct {
name string
args fetchOptions
args fetchOption
want string
wantErr bool
}{
{
name: "should be able to parse response",
args: fetchOptions{
args: fetchOption{
baseOption: baseOption{
repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository",
},
referenceName: "",
repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository"},
},
want: "27104ad7549d9e66685e115a497533f18024be9c",
wantErr: false,
},
@ -319,3 +362,262 @@ func Test_azureDownloader_latestCommitID(t *testing.T) {
})
}
}
type testRepoManager struct {
called bool
}
func (t *testRepoManager) download(_ context.Context, _ string, _ cloneOption) error {
t.called = true
return nil
}
func (t *testRepoManager) latestCommitID(_ context.Context, _ fetchOption) (string, error) {
return "", nil
}
func (t *testRepoManager) listRefs(_ context.Context, _ baseOption) ([]string, error) {
return nil, nil
}
func (t *testRepoManager) listFiles(_ context.Context, _ fetchOption) ([]string, error) {
return nil, nil
}
func Test_cloneRepository_azure(t *testing.T) {
tests := []struct {
name string
url string
called bool
}{
{
name: "Azure HTTP URL",
url: "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository",
called: true,
},
{
name: "Azure SSH URL",
url: "git@ssh.dev.azure.com:v3/Organisation/Project/Repository",
called: true,
},
{
name: "Something else",
url: "https://example.com",
called: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
azure := &testRepoManager{}
git := &testRepoManager{}
s := &Service{azure: azure, git: git}
s.cloneRepository("", cloneOption{
fetchOption: fetchOption{
baseOption: baseOption{
repositoryUrl: tt.url,
},
},
depth: 1,
})
// if azure API is called, git isn't and vice versa
assert.Equal(t, tt.called, azure.called)
assert.Equal(t, tt.called, !git.called)
})
}
}
func Test_listRefs_azure(t *testing.T) {
ensureIntegrationTest(t)
client := NewAzureClient()
type expectResult struct {
err error
refsCount int
}
accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
tests := []struct {
name string
args baseOption
expect expectResult
}{
{
name: "list refs of a real repository",
args: baseOption{
repositoryUrl: privateAzureRepoURL,
username: username,
password: accessToken,
},
expect: expectResult{
err: nil,
refsCount: 2,
},
},
{
name: "list refs of a real repository with incorrect credential",
args: baseOption{
repositoryUrl: privateAzureRepoURL,
username: "test-username",
password: "test-token",
},
expect: expectResult{
err: ErrAuthenticationFailure,
},
},
{
name: "list refs of a real repository without providing credential",
args: baseOption{
repositoryUrl: privateAzureRepoURL,
username: "",
password: "",
},
expect: expectResult{
err: ErrAuthenticationFailure,
},
},
{
name: "list refs of a fake repository",
args: baseOption{
repositoryUrl: privateAzureRepoURL + "fake",
username: username,
password: accessToken,
},
expect: expectResult{
err: ErrIncorrectRepositoryURL,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
refs, err := client.listRefs(context.TODO(), tt.args)
if tt.expect.err == nil {
assert.NoError(t, err)
if tt.expect.refsCount > 0 {
assert.Greater(t, len(refs), 0)
}
} else {
assert.Error(t, err)
assert.Equal(t, tt.expect.err, err)
}
})
}
}
func Test_listFiles_azure(t *testing.T) {
ensureIntegrationTest(t)
client := NewAzureClient()
type expectResult struct {
shouldFail bool
err error
matchedCount int
}
accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
tests := []struct {
name string
args fetchOption
expect expectResult
}{
{
name: "list tree with real repository and head ref but incorrect credential",
args: fetchOption{
baseOption: baseOption{
repositoryUrl: privateAzureRepoURL,
username: "test-username",
password: "test-token",
},
referenceName: "refs/heads/main",
},
expect: expectResult{
shouldFail: true,
err: ErrAuthenticationFailure,
},
},
{
name: "list tree with real repository and head ref but no credential",
args: fetchOption{
baseOption: baseOption{
repositoryUrl: privateAzureRepoURL,
username: "",
password: "",
},
referenceName: "refs/heads/main",
},
expect: expectResult{
shouldFail: true,
err: ErrAuthenticationFailure,
},
},
{
name: "list tree with real repository and head ref",
args: fetchOption{
baseOption: baseOption{
repositoryUrl: privateAzureRepoURL,
username: username,
password: accessToken,
},
referenceName: "refs/heads/main",
},
expect: expectResult{
err: nil,
matchedCount: 19,
},
},
{
name: "list tree with real repository but non-existing ref",
args: fetchOption{
baseOption: baseOption{
repositoryUrl: privateAzureRepoURL,
username: username,
password: accessToken,
},
referenceName: "refs/fake/feature",
},
expect: expectResult{
shouldFail: true,
},
},
{
name: "list tree with fake repository ",
args: fetchOption{
baseOption: baseOption{
repositoryUrl: privateAzureRepoURL + "fake",
username: username,
password: accessToken,
},
referenceName: "refs/fake/feature",
},
expect: expectResult{
shouldFail: true,
err: ErrIncorrectRepositoryURL,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
paths, err := client.listFiles(context.TODO(), tt.args)
if tt.expect.shouldFail {
assert.Error(t, err)
if tt.expect.err != nil {
assert.Equal(t, tt.expect.err, err)
}
} else {
assert.NoError(t, err)
if tt.expect.matchedCount > 0 {
assert.Greater(t, len(paths), 0)
}
}
})
}
}

159
api/git/git.go
View File

@ -2,52 +2,31 @@ package git
import (
"context"
"crypto/tls"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"github.com/pkg/errors"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/transport/client"
"github.com/go-git/go-git/v5/plumbing/object"
githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/go-git/go-git/v5/storage/memory"
)
var (
ErrAuthenticationFailure = errors.New("Authentication failed, please ensure that the git credentials are correct.")
)
type fetchOptions struct {
repositoryUrl string
username string
password string
referenceName string
}
type cloneOptions struct {
repositoryUrl string
username string
password string
referenceName string
depth int
}
type downloader interface {
download(ctx context.Context, dst string, opt cloneOptions) error
latestCommitID(ctx context.Context, opt fetchOptions) (string, error)
}
type gitClient struct {
preserveGitDirectory bool
}
func (c gitClient) download(ctx context.Context, dst string, opt cloneOptions) error {
func NewGitClient(preserveGitDir bool) *gitClient {
return &gitClient{
preserveGitDirectory: preserveGitDir,
}
}
func (c *gitClient) download(ctx context.Context, dst string, opt cloneOption) error {
gitOptions := git.CloneOptions{
URL: opt.repositoryUrl,
Depth: opt.depth,
@ -74,7 +53,7 @@ func (c gitClient) download(ctx context.Context, dst string, opt cloneOptions) e
return nil
}
func (c gitClient) latestCommitID(ctx context.Context, opt fetchOptions) (string, error) {
func (c *gitClient) latestCommitID(ctx context.Context, opt fetchOption) (string, error) {
remote := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{
Name: "origin",
URLs: []string{opt.repositoryUrl},
@ -124,66 +103,78 @@ func getAuth(username, password string) *githttp.BasicAuth {
return nil
}
// Service represents a service for managing Git.
type Service struct {
httpsCli *http.Client
azure downloader
git downloader
func (c *gitClient) listRefs(ctx context.Context, opt baseOption) ([]string, error) {
rem := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{
Name: "origin",
URLs: []string{opt.repositoryUrl},
})
listOptions := &git.ListOptions{
Auth: getAuth(opt.username, opt.password),
}
refs, err := rem.List(listOptions)
if err != nil {
return nil, checkGitError(err)
}
var ret []string
for _, ref := range refs {
if ref.Name().String() == "HEAD" {
continue
}
ret = append(ret, ref.Name().String())
}
return ret, nil
}
// NewService initializes a new service.
func NewService() *Service {
httpsCli := &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
Proxy: http.ProxyFromEnvironment,
},
Timeout: 300 * time.Second,
// listFiles list all filenames under the specific repository
func (c *gitClient) listFiles(ctx context.Context, opt fetchOption) ([]string, error) {
cloneOption := &git.CloneOptions{
URL: opt.repositoryUrl,
NoCheckout: true,
Depth: 1,
SingleBranch: true,
ReferenceName: plumbing.ReferenceName(opt.referenceName),
Auth: getAuth(opt.username, opt.password),
}
client.InstallProtocol("https", githttp.NewClient(httpsCli))
return &Service{
httpsCli: httpsCli,
azure: NewAzureDownloader(httpsCli),
git: gitClient{},
repo, err := git.Clone(memory.NewStorage(), nil, cloneOption)
if err != nil {
return nil, checkGitError(err)
}
head, err := repo.Head()
if err != nil {
return nil, err
}
commit, err := repo.CommitObject(head.Hash())
if err != nil {
return nil, err
}
tree, err := commit.Tree()
if err != nil {
return nil, err
}
var allPaths []string
tree.Files().ForEach(func(f *object.File) error {
allPaths = append(allPaths, f.Name)
return nil
})
return allPaths, nil
}
// CloneRepository clones a git repository using the specified URL in the specified
// destination folder.
func (service *Service) CloneRepository(destination, repositoryURL, referenceName, username, password string) error {
options := cloneOptions{
repositoryUrl: repositoryURL,
username: username,
password: password,
referenceName: referenceName,
depth: 1,
func checkGitError(err error) error {
errMsg := err.Error()
if errMsg == "repository not found" {
return ErrIncorrectRepositoryURL
} else if errMsg == "authentication required" {
return ErrAuthenticationFailure
}
return service.cloneRepository(destination, options)
}
func (service *Service) cloneRepository(destination string, options cloneOptions) error {
if isAzureUrl(options.repositoryUrl) {
return service.azure.download(context.TODO(), destination, options)
}
return service.git.download(context.TODO(), destination, options)
}
// LatestCommitID returns SHA1 of the latest commit of the specified reference
func (service *Service) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) {
options := fetchOptions{
repositoryUrl: repositoryURL,
username: username,
password: password,
referenceName: referenceName,
}
if isAzureUrl(options.repositoryUrl) {
return service.azure.latestCommitID(context.TODO(), options)
}
return service.git.latestCommitID(context.TODO(), options)
return err
}

322
api/git/git_integration_test.go
View File

@ -1,22 +1,28 @@
package git
import (
"context"
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
const (
privateGitRepoURL string = "https://github.com/portainer/private-test-repository.git"
)
func TestService_ClonePrivateRepository_GitHub(t *testing.T) {
ensureIntegrationTest(t)
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
service := NewService()
service := newService(context.TODO(), 0, 0)
dst := t.TempDir()
repositoryUrl := "https://github.com/portainer/private-test-repository.git"
repositoryUrl := privateGitRepoURL
err := service.CloneRepository(dst, repositoryUrl, "refs/heads/main", username, accessToken)
assert.NoError(t, err)
assert.FileExists(t, filepath.Join(dst, "README.md"))
@ -27,10 +33,318 @@ func TestService_LatestCommitID_GitHub(t *testing.T) {
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
service := NewService()
service := newService(context.TODO(), 0, 0)
repositoryUrl := "https://github.com/portainer/private-test-repository.git"
repositoryUrl := privateGitRepoURL
id, err := service.LatestCommitID(repositoryUrl, "refs/heads/main", username, accessToken)
assert.NoError(t, err)
assert.NotEmpty(t, id, "cannot guarantee commit id, but it should be not empty")
}
func TestService_ListRefs_GitHub(t *testing.T) {
ensureIntegrationTest(t)
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
service := newService(context.TODO(), 0, 0)
repositoryUrl := privateGitRepoURL
refs, err := service.ListRefs(repositoryUrl, username, accessToken, false)
assert.NoError(t, err)
assert.GreaterOrEqual(t, len(refs), 1)
}
func TestService_ListRefs_Github_Concurrently(t *testing.T) {
ensureIntegrationTest(t)
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
service := newService(context.TODO(), REPOSITORY_CACHE_SIZE, 200*time.Millisecond)
repositoryUrl := privateGitRepoURL
go service.ListRefs(repositoryUrl, username, accessToken, false)
service.ListRefs(repositoryUrl, username, accessToken, false)
time.Sleep(2 * time.Second)
}
func TestService_ListFiles_GitHub(t *testing.T) {
ensureIntegrationTest(t)
type expectResult struct {
shouldFail bool
err error
matchedCount int
}
service := newService(context.TODO(), 0, 0)
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
tests := []struct {
name string
args fetchOption
extensions []string
expect expectResult
}{
{
name: "list tree with real repository and head ref but incorrect credential",
args: fetchOption{
baseOption: baseOption{
repositoryUrl: privateGitRepoURL,
username: "test-username",
password: "test-token",
},
referenceName: "refs/heads/main",
},
extensions: []string{},
expect: expectResult{
shouldFail: true,
err: ErrAuthenticationFailure,
},
},
{
name: "list tree with real repository and head ref but no credential",
args: fetchOption{
baseOption: baseOption{
repositoryUrl: privateGitRepoURL + "fake",
username: "",
password: "",
},
referenceName: "refs/heads/main",
},
extensions: []string{},
expect: expectResult{
shouldFail: true,
err: ErrAuthenticationFailure,
},
},
{
name: "list tree with real repository and head ref",
args: fetchOption{
baseOption: baseOption{
repositoryUrl: privateGitRepoURL,
username: username,
password: accessToken,
},
referenceName: "refs/heads/main",
},
extensions: []string{},
expect: expectResult{
err: nil,
matchedCount: 15,
},
},
{
name: "list tree with real repository and head ref and existing file extension",
args: fetchOption{
baseOption: baseOption{
repositoryUrl: privateGitRepoURL,
username: username,
password: accessToken,
},
referenceName: "refs/heads/main",
},
extensions: []string{"yml"},
expect: expectResult{
err: nil,
matchedCount: 2,
},
},
{
name: "list tree with real repository and head ref and non-existing file extension",
args: fetchOption{
baseOption: baseOption{
repositoryUrl: privateGitRepoURL,
username: username,
password: accessToken,
},
referenceName: "refs/heads/main",
},
extensions: []string{"hcl"},
expect: expectResult{
err: nil,
matchedCount: 2,
},
},
{
name: "list tree with real repository but non-existing ref",
args: fetchOption{
baseOption: baseOption{
repositoryUrl: privateGitRepoURL,
username: username,
password: accessToken,
},
referenceName: "refs/fake/feature",
},
extensions: []string{},
expect: expectResult{
shouldFail: true,
},
},
{
name: "list tree with fake repository ",
args: fetchOption{
baseOption: baseOption{
repositoryUrl: privateGitRepoURL + "fake",
username: username,
password: accessToken,
},
referenceName: "refs/fake/feature",
},
extensions: []string{},
expect: expectResult{
shouldFail: true,
err: ErrIncorrectRepositoryURL,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, tt.extensions)
if tt.expect.shouldFail {
assert.Error(t, err)
if tt.expect.err != nil {
assert.Equal(t, tt.expect.err, err)
}
} else {
assert.NoError(t, err)
if tt.expect.matchedCount > 0 {
assert.Greater(t, len(paths), 0)
}
}
})
}
}
func TestService_ListFiles_Github_Concurrently(t *testing.T) {
ensureIntegrationTest(t)
repositoryUrl := privateGitRepoURL
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
service := newService(context.TODO(), REPOSITORY_CACHE_SIZE, 200*time.Millisecond)
go service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{})
service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{})
time.Sleep(2 * time.Second)
}
func TestService_purgeCache_Github(t *testing.T) {
ensureIntegrationTest(t)
repositoryUrl := privateGitRepoURL
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
service := NewService(context.TODO())
service.ListRefs(repositoryUrl, username, accessToken, false)
service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{})
assert.Equal(t, 1, service.repoRefCache.Len())
assert.Equal(t, 1, service.repoFileCache.Len())
service.purgeCache()
assert.Equal(t, 0, service.repoRefCache.Len())
assert.Equal(t, 0, service.repoFileCache.Len())
}
func TestService_purgeCacheByTTL_Github(t *testing.T) {
ensureIntegrationTest(t)
timeout := 100 * time.Millisecond
repositoryUrl := privateGitRepoURL
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
// 40*timeout is designed for giving enough time for ListRefs and ListFiles to cache the result
service := newService(context.TODO(), 2, 40*timeout)
service.ListRefs(repositoryUrl, username, accessToken, false)
service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{})
assert.Equal(t, 1, service.repoRefCache.Len())
assert.Equal(t, 1, service.repoFileCache.Len())
// 40*timeout is designed for giving enough time for TTL being activated
time.Sleep(40 * timeout)
assert.Equal(t, 0, service.repoRefCache.Len())
assert.Equal(t, 0, service.repoFileCache.Len())
}
func TestService_canStopCacheCleanTimer_whenContextDone(t *testing.T) {
timeout := 10 * time.Millisecond
deadlineCtx, _ := context.WithDeadline(context.TODO(), time.Now().Add(10*timeout))
service := NewService(deadlineCtx)
assert.False(t, service.timerHasStopped(), "timer should not be stopped")
<-time.After(20 * timeout)
assert.True(t, service.timerHasStopped(), "timer should be stopped")
}
func TestService_HardRefresh_ListRefs_GitHub(t *testing.T) {
ensureIntegrationTest(t)
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
service := newService(context.TODO(), 2, 0)
repositoryUrl := privateGitRepoURL
refs, err := service.ListRefs(repositoryUrl, username, accessToken, false)
assert.NoError(t, err)
assert.GreaterOrEqual(t, len(refs), 1)
assert.Equal(t, 1, service.repoRefCache.Len())
refs, err = service.ListRefs(repositoryUrl, username, "fake-token", true)
assert.Error(t, err)
assert.Equal(t, 0, service.repoRefCache.Len())
}
func TestService_HardRefresh_ListRefs_And_RemoveAllCaches_GitHub(t *testing.T) {
ensureIntegrationTest(t)
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
service := newService(context.TODO(), 2, 0)
repositoryUrl := privateGitRepoURL
refs, err := service.ListRefs(repositoryUrl, username, accessToken, false)
assert.NoError(t, err)
assert.GreaterOrEqual(t, len(refs), 1)
assert.Equal(t, 1, service.repoRefCache.Len())
files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{})
assert.NoError(t, err)
assert.GreaterOrEqual(t, len(files), 1)
assert.Equal(t, 1, service.repoFileCache.Len())
files, err = service.ListFiles(repositoryUrl, "refs/heads/test", username, accessToken, false, []string{})
assert.NoError(t, err)
assert.GreaterOrEqual(t, len(files), 1)
assert.Equal(t, 2, service.repoFileCache.Len())
refs, err = service.ListRefs(repositoryUrl, username, "fake-token", true)
assert.Error(t, err)
assert.Equal(t, 0, service.repoRefCache.Len())
// The relevant file caches should be removed too
assert.Equal(t, 0, service.repoFileCache.Len())
}
func TestService_HardRefresh_ListFiles_GitHub(t *testing.T) {
ensureIntegrationTest(t)
service := newService(context.TODO(), 2, 0)
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
repositoryUrl := privateGitRepoURL
files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{})
assert.NoError(t, err)
assert.GreaterOrEqual(t, len(files), 1)
assert.Equal(t, 1, service.repoFileCache.Len())
files, err = service.ListFiles(repositoryUrl, "refs/heads/main", username, "fake-token", true, []string{})
assert.Error(t, err)
assert.Equal(t, 0, service.repoFileCache.Len())
}

225
api/git/git_test.go
View File

@ -32,7 +32,7 @@ func setup(t *testing.T) string {
}
func Test_ClonePublicRepository_Shallow(t *testing.T) {
service := Service{git: gitClient{preserveGitDirectory: true}} // no need for http client since the test access the repo via file system.
service := Service{git: NewGitClient(true)} // no need for http client since the test access the repo via file system.
repositoryURL := setup(t)
referenceName := "refs/heads/main"
@ -44,7 +44,7 @@ func Test_ClonePublicRepository_Shallow(t *testing.T) {
}
func Test_ClonePublicRepository_NoGitDirectory(t *testing.T) {
service := Service{git: gitClient{preserveGitDirectory: false}} // no need for http client since the test access the repo via file system.
service := Service{git: NewGitClient(false)} // no need for http client since the test access the repo via file system.
repositoryURL := setup(t)
referenceName := "refs/heads/main"
@ -56,7 +56,7 @@ func Test_ClonePublicRepository_NoGitDirectory(t *testing.T) {
}
func Test_cloneRepository(t *testing.T) {
service := Service{git: gitClient{preserveGitDirectory: true}} // no need for http client since the test access the repo via file system.
service := Service{git: NewGitClient(true)} // no need for http client since the test access the repo via file system.
repositoryURL := setup(t)
referenceName := "refs/heads/main"
@ -64,10 +64,14 @@ func Test_cloneRepository(t *testing.T) {
dir := t.TempDir()
t.Logf("Cloning into %s", dir)
err := service.cloneRepository(dir, cloneOptions{
repositoryUrl: repositoryURL,
referenceName: referenceName,
depth: 10,
err := service.cloneRepository(dir, cloneOption{
fetchOption: fetchOption{
baseOption: baseOption{
repositoryUrl: repositoryURL,
},
referenceName: referenceName,
},
depth: 10,
})
assert.NoError(t, err)
@ -75,7 +79,7 @@ func Test_cloneRepository(t *testing.T) {
}
func Test_latestCommitID(t *testing.T) {
service := Service{git: gitClient{preserveGitDirectory: true}} // no need for http client since the test access the repo via file system.
service := Service{git: NewGitClient(true)} // no need for http client since the test access the repo via file system.
repositoryURL := setup(t)
referenceName := "refs/heads/main"
@ -106,53 +110,196 @@ func getCommitHistoryLength(t *testing.T, err error, dir string) int {
return count
}
type testDownloader struct {
called bool
}
func Test_listRefsPrivateRepository(t *testing.T) {
ensureIntegrationTest(t)
func (t *testDownloader) download(_ context.Context, _ string, _ cloneOptions) error {
t.called = true
return nil
}
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
func (t *testDownloader) latestCommitID(_ context.Context, _ fetchOptions) (string, error) {
return "", nil
}
client := NewGitClient(false)
type expectResult struct {
err error
refsCount int
}
func Test_cloneRepository_azure(t *testing.T) {
tests := []struct {
name string
url string
called bool
args baseOption
expect expectResult
}{
{
name: "Azure HTTP URL",
url: "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository",
called: true,
name: "list refs of a real private repository",
args: baseOption{
repositoryUrl: privateGitRepoURL,
username: username,
password: accessToken,
},
expect: expectResult{
err: nil,
refsCount: 2,
},
},
{
name: "Azure SSH URL",
url: "git@ssh.dev.azure.com:v3/Organisation/Project/Repository",
called: true,
name: "list refs of a real private repository with incorrect credential",
args: baseOption{
repositoryUrl: privateGitRepoURL,
username: "test-username",
password: "test-token",
},
expect: expectResult{
err: ErrAuthenticationFailure,
},
},
{
name: "Something else",
url: "https://example.com",
called: false,
name: "list refs of a fake repository without providing credential",
args: baseOption{
repositoryUrl: privateGitRepoURL + "fake",
username: "",
password: "",
},
expect: expectResult{
err: ErrAuthenticationFailure,
},
},
{
name: "list refs of a fake repository",
args: baseOption{
repositoryUrl: privateGitRepoURL + "fake",
username: username,
password: accessToken,
},
expect: expectResult{
err: ErrIncorrectRepositoryURL,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
azure := &testDownloader{}
git := &testDownloader{}
s := &Service{azure: azure, git: git}
s.cloneRepository("", cloneOptions{repositoryUrl: tt.url, depth: 1})
// if azure API is called, git isn't and vice versa
assert.Equal(t, tt.called, azure.called)
assert.Equal(t, tt.called, !git.called)
refs, err := client.listRefs(context.TODO(), tt.args)
if tt.expect.err == nil {
assert.NoError(t, err)
if tt.expect.refsCount > 0 {
assert.Greater(t, len(refs), 0)
}
} else {
assert.Error(t, err)
assert.Equal(t, tt.expect.err, err)
}
})
}
}
func Test_listFilesPrivateRepository(t *testing.T) {
ensureIntegrationTest(t)
client := NewGitClient(false)
type expectResult struct {
shouldFail bool
err error
matchedCount int
}
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
tests := []struct {
name string
args fetchOption
expect expectResult
}{
{
name: "list tree with real repository and head ref but incorrect credential",
args: fetchOption{
baseOption: baseOption{
repositoryUrl: privateGitRepoURL,
username: "test-username",
password: "test-token",
},
referenceName: "refs/heads/main",
},
expect: expectResult{
shouldFail: true,
err: ErrAuthenticationFailure,
},
},
{
name: "list tree with real repository and head ref but no credential",
args: fetchOption{
baseOption: baseOption{
repositoryUrl: privateGitRepoURL + "fake",
username: "",
password: "",
},
referenceName: "refs/heads/main",
},
expect: expectResult{
shouldFail: true,
err: ErrAuthenticationFailure,
},
},
{
name: "list tree with real repository and head ref",
args: fetchOption{
baseOption: baseOption{
repositoryUrl: privateGitRepoURL,
username: username,
password: accessToken,
},
referenceName: "refs/heads/main",
},
expect: expectResult{
err: nil,
matchedCount: 15,
},
},
{
name: "list tree with real repository but non-existing ref",
args: fetchOption{
baseOption: baseOption{
repositoryUrl: privateGitRepoURL,
username: username,
password: accessToken,
},
referenceName: "refs/fake/feature",
},
expect: expectResult{
shouldFail: true,
},
},
{
name: "list tree with fake repository ",
args: fetchOption{
baseOption: baseOption{
repositoryUrl: privateGitRepoURL + "fake",
username: username,
password: accessToken,
},
referenceName: "refs/fake/feature",
},
expect: expectResult{
shouldFail: true,
err: ErrIncorrectRepositoryURL,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
paths, err := client.listFiles(context.TODO(), tt.args)
if tt.expect.shouldFail {
assert.Error(t, err)
if tt.expect.err != nil {
assert.Equal(t, tt.expect.err, err)
}
} else {
assert.NoError(t, err)
if tt.expect.matchedCount > 0 {
assert.Greater(t, len(paths), 0)
}
}
})
}
}

320
api/git/service.go Normal file
View File

@ -0,0 +1,320 @@
package git
import (
"context"
"errors"
"log"
"strings"
"sync"
"time"
lru "github.com/hashicorp/golang-lru"
)
var (
ErrIncorrectRepositoryURL = errors.New("Git repository could not be found, please ensure that the URL is correct.")
ErrAuthenticationFailure = errors.New("Authentication failed, please ensure that the git credentials are correct.")
REPOSITORY_CACHE_SIZE = 4
REPOSITORY_CACHE_TTL = 5 * time.Minute
)
// baseOption provides a minimum group of information to operate a git repository, like git-remote
type baseOption struct {
repositoryUrl string
username string
password string
}
// fetchOption allows to specify the reference name of the target repository
type fetchOption struct {
baseOption
referenceName string
}
// cloneOption allows to add a history truncated to the specified number of commits
type cloneOption struct {
fetchOption
depth int
}
type repoManager interface {
download(ctx context.Context, dst string, opt cloneOption) error
latestCommitID(ctx context.Context, opt fetchOption) (string, error)
listRefs(ctx context.Context, opt baseOption) ([]string, error)
listFiles(ctx context.Context, opt fetchOption) ([]string, error)
}
// Service represents a service for managing Git.
type Service struct {
shutdownCtx context.Context
azure repoManager
git repoManager
timerStopped bool
mut sync.Mutex
cacheEnabled bool
// Cache the result of repository refs, key is repository URL
repoRefCache *lru.Cache
// Cache the result of repository file tree, key is the concatenated string of repository URL and ref value
repoFileCache *lru.Cache
}
// NewService initializes a new service.
func NewService(ctx context.Context) *Service {
return newService(ctx, REPOSITORY_CACHE_SIZE, REPOSITORY_CACHE_TTL)
}
func newService(ctx context.Context, cacheSize int, cacheTTL time.Duration) *Service {
service := &Service{
shutdownCtx: ctx,
azure: NewAzureClient(),
git: NewGitClient(false),
timerStopped: false,
cacheEnabled: cacheSize > 0,
}
if service.cacheEnabled {
var err error
service.repoRefCache, err = lru.New(cacheSize)
if err != nil {
log.Printf("[DEBUG] [git] [message: failed to create ref cache: %v\n", err)
}
service.repoFileCache, err = lru.New(cacheSize)
if err != nil {
log.Printf("[DEBUG] [git] [message: failed to create file cache: %v\n", err)
}
if cacheTTL > 0 {
go service.startCacheCleanTimer(cacheTTL)
}
}
return service
}
// startCacheCleanTimer starts a timer to purge caches periodically
func (service *Service) startCacheCleanTimer(d time.Duration) {
ticker := time.NewTicker(d)
for {
select {
case <-ticker.C:
service.purgeCache()
case <-service.shutdownCtx.Done():
ticker.Stop()
service.mut.Lock()
service.timerStopped = true
service.mut.Unlock()
return
}
}
}
// timerHasStopped shows the CacheClean timer state with thread-safe way
func (service *Service) timerHasStopped() bool {
service.mut.Lock()
defer service.mut.Unlock()
ret := service.timerStopped
return ret
}
// CloneRepository clones a git repository using the specified URL in the specified
// destination folder.
func (service *Service) CloneRepository(destination, repositoryURL, referenceName, username, password string) error {
options := cloneOption{
fetchOption: fetchOption{
baseOption: baseOption{
repositoryUrl: repositoryURL,
username: username,
password: password,
},
referenceName: referenceName,
},
depth: 1,
}
return service.cloneRepository(destination, options)
}
func (service *Service) cloneRepository(destination string, options cloneOption) error {
if isAzureUrl(options.repositoryUrl) {
return service.azure.download(context.TODO(), destination, options)
}
return service.git.download(context.TODO(), destination, options)
}
// LatestCommitID returns SHA1 of the latest commit of the specified reference
func (service *Service) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) {
options := fetchOption{
baseOption: baseOption{
repositoryUrl: repositoryURL,
username: username,
password: password,
},
referenceName: referenceName,
}
if isAzureUrl(options.repositoryUrl) {
return service.azure.latestCommitID(context.TODO(), options)
}
return service.git.latestCommitID(context.TODO(), options)
}
// ListRefs will list target repository's references without cloning the repository
func (service *Service) ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error) {
if service.cacheEnabled && hardRefresh {
// Should remove the cache explicitly, so that the following normal list can show the correct result
service.repoRefCache.Remove(repositoryURL)
// Remove file caches pointed to the same repository
for _, fileCacheKey := range service.repoFileCache.Keys() {
key, ok := fileCacheKey.(string)
if ok {
if strings.HasPrefix(key, repositoryURL) {
service.repoFileCache.Remove(key)
}
}
}
}
if service.repoRefCache != nil {
// Lookup the refs cache first
cache, ok := service.repoRefCache.Get(repositoryURL)
if ok {
refs, success := cache.([]string)
if success {
return refs, nil
}
}
}
options := baseOption{
repositoryUrl: repositoryURL,
username: username,
password: password,
}
var (
refs []string
err error
)
if isAzureUrl(options.repositoryUrl) {
refs, err = service.azure.listRefs(context.TODO(), options)
if err != nil {
return nil, err
}
} else {
refs, err = service.git.listRefs(context.TODO(), options)
if err != nil {
return nil, err
}
}
if service.cacheEnabled && service.repoRefCache != nil {
service.repoRefCache.Add(options.repositoryUrl, refs)
}
return refs, nil
}
// ListFiles will list all the files of the target repository with specific extensions.
// If extension is not provided, it will list all the files under the target repository
func (service *Service) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string) ([]string, error) {
repoKey := generateCacheKey(repositoryURL, referenceName)
if service.cacheEnabled && hardRefresh {
// Should remove the cache explicitly, so that the following normal list can show the correct result
service.repoFileCache.Remove(repoKey)
}
if service.repoFileCache != nil {
// lookup the files cache first
cache, ok := service.repoFileCache.Get(repoKey)
if ok {
files, success := cache.([]string)
if success {
// For the case while searching files in a repository without include extensions for the first time,
// but with include extensions for the second time
includedFiles := filterFiles(files, includedExts)
return includedFiles, nil
}
}
}
options := fetchOption{
baseOption: baseOption{
repositoryUrl: repositoryURL,
username: username,
password: password,
},
referenceName: referenceName,
}
var (
files []string
err error
)
if isAzureUrl(options.repositoryUrl) {
files, err = service.azure.listFiles(context.TODO(), options)
if err != nil {
return nil, err
}
} else {
files, err = service.git.listFiles(context.TODO(), options)
if err != nil {
return nil, err
}
}
includedFiles := filterFiles(files, includedExts)
if service.cacheEnabled && service.repoFileCache != nil {
service.repoFileCache.Add(repoKey, includedFiles)
return includedFiles, nil
}
return includedFiles, nil
}
func (service *Service) purgeCache() {
if service.repoRefCache != nil {
service.repoRefCache.Purge()
}
if service.repoFileCache != nil {
service.repoFileCache.Purge()
}
}
func generateCacheKey(names ...string) string {
return strings.Join(names, "-")
}
func matchExtensions(target string, exts []string) bool {
if len(exts) == 0 {
return true
}
for _, ext := range exts {
if strings.HasSuffix(target, ext) {
return true
}
}
return false
}
func filterFiles(paths []string, includedExts []string) []string {
if len(includedExts) == 0 {
return paths
}
var includedFiles []string
for _, filename := range paths {
// filter out the filenames with non-included extension
if matchExtensions(filename, includedExts) {
includedFiles = append(includedFiles, filename)
}
}
return includedFiles
}

8
api/http/handler/edgestacks/edgestack_test.go
View File

@ -34,6 +34,14 @@ func (g *gitService) LatestCommitID(repositoryURL, referenceName, username, pass
return g.id, nil
}
func (g *gitService) ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error) {
return nil, nil
}
func (g *gitService) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string) ([]string, error) {
return nil, nil
}
// Helpers
func setupHandler(t *testing.T) (*Handler, string, func()) {
t.Helper()

11
api/http/proxy/factory/docker/transport_test.go
View File

@ -1,11 +1,12 @@
package docker
import (
portainer "github.com/portainer/portainer/api"
"github.com/stretchr/testify/assert"
"net/http"
"net/http/httptest"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/stretchr/testify/assert"
)
type noopGitService struct{}
@ -16,6 +17,12 @@ func (s *noopGitService) CloneRepository(destination string, repositoryURL, refe
func (s *noopGitService) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) {
return "my-latest-commit-id", nil
}
func (g *noopGitService) ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error) {
return nil, nil
}
func (g *noopGitService) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string) ([]string, error) {
return nil, nil
}
func TestTransport_updateDefaultGitBranch(t *testing.T) {
type fields struct {

2
api/portainer.go
View File

@ -1323,6 +1323,8 @@ type (
GitService interface {
CloneRepository(destination string, repositoryURL, referenceName, username, password string) error
LatestCommitID(repositoryURL, referenceName, username, password string) (string, error)
ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error)
ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includeExts []string) ([]string, error)
}
// OpenAMTService represents a service for managing OpenAMT

8
api/stacks/deploy_test.go
View File

@ -25,6 +25,14 @@ func (g *gitService) LatestCommitID(repositoryURL, referenceName, username, pass
return g.id, nil
}
func (g *gitService) ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error) {
return nil, nil
}
func (g *gitService) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string) ([]string, error) {
return nil, nil
}
type noopDeployer struct{}
func (s *noopDeployer) DeploySwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, prune bool, pullImage bool) error {

6
app/portainer/components/forms/git-form/git-form.html
View File

@ -1,7 +1,11 @@
<ng-form name="$ctrl.gitForm">
<div class="col-sm-12 form-section-title"> Git repository </div>
<git-form-auth-fieldset model="$ctrl.model" on-change="($ctrl.onChange)"></git-form-auth-fieldset>
<git-form-url-field value="$ctrl.model.RepositoryURL" on-change="($ctrl.onChangeURL)"></git-form-url-field>
<git-form-ref-field value="$ctrl.model.RepositoryReferenceName" on-change="($ctrl.onChangeRefName)"></git-form-ref-field>
<git-form-compose-path-field
value="$ctrl.model.ComposeFilePathInRepository"
on-change="($ctrl.onChangeComposePath)"
@ -10,8 +14,6 @@
<git-form-additional-files-panel ng-if="$ctrl.additionalFile" model="$ctrl.model" on-change="($ctrl.onChange)"></git-form-additional-files-panel>
<git-form-auth-fieldset model="$ctrl.model" on-change="($ctrl.onChange)"></git-form-auth-fieldset>
<git-form-auto-update-fieldset
ng-if="$ctrl.autoUpdate"
model="$ctrl.model"