fix(git) EE-2026 git default branch (#6876)

fix(git) EE-2026 git default branch
pull/6040/head
congs 2022-05-16 09:35:11 +12:00 committed by GitHub
parent 0ffb84aaa6
commit df05914fac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 167 additions and 127 deletions

View File

@ -607,7 +607,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
kubeClusterAccessService := kubernetes.NewKubeClusterAccessService(*flags.BaseURL, *flags.AddrHTTPS, sslSettings.CertPath)
proxyManager := proxy.NewManager(dataStore, digitalSignatureService, reverseTunnelService, dockerClientFactory, kubernetesClientFactory, kubernetesTokenCacheManager)
proxyManager := proxy.NewManager(dataStore, digitalSignatureService, reverseTunnelService, dockerClientFactory, kubernetesClientFactory, kubernetesTokenCacheManager, gitService)
reverseTunnelService.ProxyManager = proxyManager

View File

@ -108,12 +108,12 @@ func (a *azureDownloader) latestCommitID(ctx context.Context, options fetchOptio
return "", errors.WithMessage(err, "failed to parse url")
}
refsUrl, err := a.buildRefsUrl(config, options.referenceName)
rootItemUrl, err := a.buildRootItemUrl(config, options.referenceName)
if err != nil {
return "", errors.WithMessage(err, "failed to build azure refs url")
return "", errors.WithMessage(err, "failed to build azure root item url")
}
req, err := http.NewRequestWithContext(ctx, "GET", refsUrl, nil)
req, err := http.NewRequestWithContext(ctx, "GET", rootItemUrl, nil)
if options.username != "" || options.password != "" {
req.SetBasicAuth(options.username, options.password)
} else if config.username != "" || config.password != "" {
@ -131,26 +131,24 @@ func (a *azureDownloader) latestCommitID(ctx context.Context, options fetchOptio
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("failed to get repository refs with a status \"%v\"", resp.Status)
return "", fmt.Errorf("failed to get repository root item with a status \"%v\"", resp.Status)
}
var refs struct {
var items struct {
Value []struct {
Name string `json:"name"`
ObjectId string `json:"objectId"`
}
}
if err := json.NewDecoder(resp.Body).Decode(&refs); err != nil {
return "", errors.Wrap(err, "could not parse Azure Refs response")
}
for _, ref := range refs.Value {
if strings.EqualFold(ref.Name, options.referenceName) {
return ref.ObjectId, nil
CommitId string `json:"commitId"`
}
}
return "", errors.Errorf("could not find ref %q in the repository", options.referenceName)
if err := json.NewDecoder(resp.Body).Decode(&items); err != nil {
return "", errors.Wrap(err, "could not parse Azure items response")
}
if len(items.Value) == 0 || items.Value[0].CommitId == "" {
return "", errors.Errorf("failed to get latest commitID in the repository")
}
return items.Value[0].CommitId, nil
}
func parseUrl(rawUrl string) (*azureOptions, error) {
@ -236,8 +234,10 @@ func (a *azureDownloader) buildDownloadUrl(config *azureOptions, referenceName s
// scopePath=/&download=true&versionDescriptor.version=main&$format=zip&recursionLevel=full&api-version=6.0
q.Set("scopePath", "/")
q.Set("download", "true")
q.Set("versionDescriptor.versionType", getVersionType(referenceName))
q.Set("versionDescriptor.version", formatReferenceName(referenceName))
if referenceName != "" {
q.Set("versionDescriptor.versionType", getVersionType(referenceName))
q.Set("versionDescriptor.version", formatReferenceName(referenceName))
}
q.Set("$format", "zip")
q.Set("recursionLevel", "full")
q.Set("api-version", "6.0")
@ -246,8 +246,8 @@ func (a *azureDownloader) buildDownloadUrl(config *azureOptions, referenceName s
return u.String(), nil
}
func (a *azureDownloader) buildRefsUrl(config *azureOptions, referenceName string) (string, error) {
rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/refs",
func (a *azureDownloader) buildRootItemUrl(config *azureOptions, referenceName string) (string, error) {
rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/items",
a.baseUrl,
url.PathEscape(config.organisation),
url.PathEscape(config.project),
@ -255,12 +255,15 @@ func (a *azureDownloader) buildRefsUrl(config *azureOptions, referenceName strin
u, err := url.Parse(rawUrl)
if err != nil {
return "", errors.Wrapf(err, "failed to parse refs url path %s", rawUrl)
return "", errors.Wrapf(err, "failed to parse root item url path %s", rawUrl)
}
// filterContains=main&api-version=6.0
q := u.Query()
q.Set("filterContains", formatReferenceName(referenceName))
q.Set("scopePath", "/")
if referenceName != "" {
q.Set("versionDescriptor.versionType", getVersionType(referenceName))
q.Set("versionDescriptor.version", formatReferenceName(referenceName))
}
q.Set("api-version", "6.0")
u.RawQuery = q.Encode()

View File

@ -28,15 +28,15 @@ func Test_buildDownloadUrl(t *testing.T) {
}
}
func Test_buildRefsUrl(t *testing.T) {
func Test_buildRootItemUrl(t *testing.T) {
a := NewAzureDownloader(nil)
u, err := a.buildRefsUrl(&azureOptions{
u, err := a.buildRootItemUrl(&azureOptions{
organisation: "organisation",
project: "project",
repository: "repository",
}, "refs/heads/main")
expectedUrl, _ := url.Parse("https://dev.azure.com/organisation/project/_apis/git/repositories/repository/refs?filterContains=main&api-version=6.0")
expectedUrl, _ := url.Parse("https://dev.azure.com/organisation/project/_apis/git/repositories/repository/items?scopePath=/&api-version=6.0&versionDescriptor.version=main&versionDescriptor.versionType=branch")
actualUrl, _ := url.Parse(u)
assert.NoError(t, err)
assert.Equal(t, expectedUrl.Host, actualUrl.Host)
@ -270,63 +270,17 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
func Test_azureDownloader_latestCommitID(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
response := `{
"value": [
{
"name": "refs/heads/feature/calcApp",
"objectId": "ffe9cba521f00d7f60e322845072238635edb451",
"creator": {
"displayName": "Normal Paulk",
"url": "https://vssps.dev.azure.com/fabrikam/_apis/Identities/ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
"_links": {
"avatar": {
"href": "https://dev.azure.com/fabrikam/_apis/GraphProfile/MemberAvatars/aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5"
}
},
"id": "ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
"uniqueName": "dev@mailserver.com",
"imageUrl": "https://dev.azure.com/fabrikam/_api/_common/identityImage?id=ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
"descriptor": "aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5"
},
"url": "https://dev.azure.com/fabrikam/7484f783-66a3-4f27-b7cd-6b08b0b077ed/_apis/git/repositories/d3d1760b-311c-4175-a726-20dfc6a7f885/refs?filter=heads%2Ffeature%2FcalcApp"
},
{
"name": "refs/heads/feature/replacer",
"objectId": "917131a709996c5cfe188c3b57e9a6ad90e8b85c",
"creator": {
"displayName": "Normal Paulk",
"url": "https://vssps.dev.azure.com/fabrikam/_apis/Identities/ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
"_links": {
"avatar": {
"href": "https://dev.azure.com/fabrikam/_apis/GraphProfile/MemberAvatars/aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5"
}
},
"id": "ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
"uniqueName": "dev@mailserver.com",
"imageUrl": "https://dev.azure.com/fabrikam/_api/_common/identityImage?id=ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
"descriptor": "aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5"
},
"url": "https://dev.azure.com/fabrikam/7484f783-66a3-4f27-b7cd-6b08b0b077ed/_apis/git/repositories/d3d1760b-311c-4175-a726-20dfc6a7f885/refs?filter=heads%2Ffeature%2Freplacer"
},
{
"name": "refs/heads/master",
"objectId": "ffe9cba521f00d7f60e322845072238635edb451",
"creator": {
"displayName": "Normal Paulk",
"url": "https://vssps.dev.azure.com/fabrikam/_apis/Identities/ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
"_links": {
"avatar": {
"href": "https://dev.azure.com/fabrikam/_apis/GraphProfile/MemberAvatars/aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5"
}
},
"id": "ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
"uniqueName": "dev@mailserver.com",
"imageUrl": "https://dev.azure.com/fabrikam/_api/_common/identityImage?id=ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
"descriptor": "aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5"
},
"url": "https://dev.azure.com/fabrikam/7484f783-66a3-4f27-b7cd-6b08b0b077ed/_apis/git/repositories/d3d1760b-311c-4175-a726-20dfc6a7f885/refs?filter=heads%2Fmaster"
}
],
"count": 3
"count": 1,
"value": [
{
"objectId": "1a5630f017127db7de24d8771da0f536ff98fc9b",
"gitObjectType": "tree",
"commitId": "27104ad7549d9e66685e115a497533f18024be9c",
"path": "/",
"isFolder": true,
"url": "https://dev.azure.com/simonmeng0474/4b546a97-c481-4506-bdd5-976e9592f91a/_apis/git/repositories/a22247ad-053f-43bc-88a7-62ff4846bb97/items?path=%2F&versionType=Branch&versionOptions=None"
}
]
}`
w.Header().Set("Content-Type", "application/json")
w.Write([]byte(response))
@ -347,19 +301,11 @@ func Test_azureDownloader_latestCommitID(t *testing.T) {
{
name: "should be able to parse response",
args: fetchOptions{
referenceName: "refs/heads/master",
referenceName: "",
repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository"},
want: "ffe9cba521f00d7f60e322845072238635edb451",
want: "27104ad7549d9e66685e115a497533f18024be9c",
wantErr: false,
},
{
name: "should be able to parse response",
args: fetchOptions{
referenceName: "refs/heads/unknown",
repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository"},
want: "",
wantErr: true,
},
}
for _, tt := range tests {

View File

@ -82,8 +82,17 @@ func (c gitClient) latestCommitID(ctx context.Context, opt fetchOptions) (string
return "", errors.Wrap(err, "failed to list repository refs")
}
referenceName := opt.referenceName
if referenceName == "" {
for _, ref := range refs {
if strings.EqualFold(ref.Name().String(), "HEAD") {
referenceName = ref.Target().String()
}
}
}
for _, ref := range refs {
if strings.EqualFold(ref.Name().String(), opt.referenceName) {
if strings.EqualFold(ref.Name().String(), referenceName) {
return ref.Hash().String(), nil
}
}

View File

@ -177,9 +177,6 @@ func (payload *composeStackFromGitRepositoryPayload) Validate(r *http.Request) e
if govalidator.IsNull(payload.RepositoryURL) || !govalidator.IsURL(payload.RepositoryURL) {
return errors.New("Invalid repository URL. Must correspond to a valid URL format")
}
if govalidator.IsNull(payload.RepositoryReferenceName) {
payload.RepositoryReferenceName = defaultGitReferenceName
}
if payload.RepositoryAuthentication && govalidator.IsNull(payload.RepositoryPassword) {
return errors.New("Invalid repository credentials. Password must be specified when authentication is enabled")
}

View File

@ -70,9 +70,6 @@ func (payload *kubernetesGitDeploymentPayload) Validate(r *http.Request) error {
if govalidator.IsNull(payload.ManifestFile) {
return errors.New("Invalid manifest file in repository")
}
if govalidator.IsNull(payload.RepositoryReferenceName) {
payload.RepositoryReferenceName = defaultGitReferenceName
}
if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil {
return err
}

View File

@ -144,9 +144,6 @@ func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) err
if govalidator.IsNull(payload.RepositoryURL) || !govalidator.IsURL(payload.RepositoryURL) {
return errors.New("Invalid repository URL. Must correspond to a valid URL format")
}
if govalidator.IsNull(payload.RepositoryReferenceName) {
payload.RepositoryReferenceName = defaultGitReferenceName
}
if payload.RepositoryAuthentication && govalidator.IsNull(payload.RepositoryPassword) {
return errors.New("Invalid repository credentials. Password must be specified when authentication is enabled")
}

View File

@ -21,8 +21,6 @@ import (
"github.com/portainer/portainer/api/stacks"
)
const defaultGitReferenceName = "refs/heads/master"
var (
errStackAlreadyExists = errors.New("A stack already exists with this name")
errWebhookIDAlreadyExists = errors.New("A webhook ID already exists")

View File

@ -4,7 +4,6 @@ import (
"net/http"
"time"
"github.com/asaskevich/govalidator"
"github.com/pkg/errors"
httperror "github.com/portainer/libhttp/error"
"github.com/portainer/libhttp/request"
@ -26,10 +25,6 @@ type stackGitUpdatePayload struct {
}
func (payload *stackGitUpdatePayload) Validate(r *http.Request) error {
if govalidator.IsNull(payload.RepositoryReferenceName) {
payload.RepositoryReferenceName = defaultGitReferenceName
}
if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil {
return err
}

View File

@ -6,7 +6,6 @@ import (
"net/http"
"time"
"github.com/asaskevich/govalidator"
"github.com/pkg/errors"
httperror "github.com/portainer/libhttp/error"
"github.com/portainer/libhttp/request"
@ -28,9 +27,6 @@ type stackGitRedployPayload struct {
}
func (payload *stackGitRedployPayload) Validate(r *http.Request) error {
if govalidator.IsNull(payload.RepositoryReferenceName) {
payload.RepositoryReferenceName = defaultGitReferenceName
}
return nil
}

View File

@ -38,9 +38,6 @@ func (payload *kubernetesFileStackUpdatePayload) Validate(r *http.Request) error
}
func (payload *kubernetesGitStackUpdatePayload) Validate(r *http.Request) error {
if govalidator.IsNull(payload.RepositoryReferenceName) {
payload.RepositoryReferenceName = defaultGitReferenceName
}
if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil {
return err
}

View File

@ -64,7 +64,7 @@ func (factory *ProxyFactory) newDockerHTTPProxy(endpoint *portainer.Endpoint) (h
DockerClientFactory: factory.dockerClientFactory,
}
dockerTransport, err := docker.NewTransport(transportParameters, httpTransport)
dockerTransport, err := docker.NewTransport(transportParameters, httpTransport, factory.gitService)
if err != nil {
return nil, err
}

View File

@ -35,6 +35,7 @@ type (
signatureService portainer.DigitalSignatureService
reverseTunnelService portainer.ReverseTunnelService
dockerClientFactory *docker.ClientFactory
gitService portainer.GitService
}
// TransportParameters is used to create a new Transport
@ -62,7 +63,7 @@ type (
)
// NewTransport returns a pointer to a new Transport instance.
func NewTransport(parameters *TransportParameters, httpTransport *http.Transport) (*Transport, error) {
func NewTransport(parameters *TransportParameters, httpTransport *http.Transport, gitService portainer.GitService) (*Transport, error) {
transport := &Transport{
endpoint: parameters.Endpoint,
dataStore: parameters.DataStore,
@ -70,6 +71,7 @@ func NewTransport(parameters *TransportParameters, httpTransport *http.Transport
reverseTunnelService: parameters.ReverseTunnelService,
dockerClientFactory: parameters.DockerClientFactory,
HTTPTransport: httpTransport,
gitService: gitService,
}
return transport, nil
@ -381,9 +383,31 @@ func (transport *Transport) proxyTaskRequest(request *http.Request) (*http.Respo
}
func (transport *Transport) proxyBuildRequest(request *http.Request) (*http.Response, error) {
err := transport.updateDefaultGitBranch(request)
if err != nil {
return nil, err
}
return transport.interceptAndRewriteRequest(request, buildOperation)
}
func (transport *Transport) updateDefaultGitBranch(request *http.Request) error {
remote := request.URL.Query().Get("remote")
if strings.HasSuffix(remote, ".git") {
repositoryURL := remote[:len(remote)-4]
latestCommitID, err := transport.gitService.LatestCommitID(repositoryURL, "", "", "")
if err != nil {
return err
}
newRemote := fmt.Sprintf("%s#%s", remote, latestCommitID)
q := request.URL.Query()
q.Set("remote", newRemote)
request.URL.RawQuery = q.Encode()
}
return nil
}
func (transport *Transport) proxyImageRequest(request *http.Request) (*http.Response, error) {
switch requestPath := request.URL.Path; requestPath {
case "/images/create":

View File

@ -0,0 +1,73 @@
package docker
import (
portainer "github.com/portainer/portainer/api"
"github.com/stretchr/testify/assert"
"net/http"
"net/http/httptest"
"testing"
)
type noopGitService struct{}
func (s *noopGitService) CloneRepository(destination string, repositoryURL, referenceName, username, password string) error {
return nil
}
func (s *noopGitService) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) {
return "my-latest-commit-id", nil
}
func TestTransport_updateDefaultGitBranch(t *testing.T) {
type fields struct {
gitService portainer.GitService
}
type args struct {
request *http.Request
}
defaultFields := fields{
gitService: &noopGitService{},
}
tests := []struct {
name string
fields fields
args args
wantErr bool
expectedQuery string
}{
{
name: "append commit ID",
fields: defaultFields,
args: args{
request: httptest.NewRequest(http.MethodPost, "http://unixsocket/build?dockerfile=Dockerfile&remote=https://my-host.com/my-user/my-repo.git&t=my-image", nil),
},
wantErr: false,
expectedQuery: "dockerfile=Dockerfile&remote=https%3A%2F%2Fmy-host.com%2Fmy-user%2Fmy-repo.git%23my-latest-commit-id&t=my-image",
},
{
name: "not append commit ID",
fields: defaultFields,
args: args{
request: httptest.NewRequest(http.MethodPost, "http://unixsocket/build?dockerfile=Dockerfile&remote=https://my-host.com/my-user/my-repo/my-file&t=my-image", nil),
},
wantErr: false,
expectedQuery: "dockerfile=Dockerfile&remote=https://my-host.com/my-user/my-repo/my-file&t=my-image",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
transport := &Transport{
gitService: tt.fields.gitService,
}
err := transport.updateDefaultGitBranch(tt.args.request)
if (err != nil) != tt.wantErr {
t.Errorf("updateDefaultGitBranch() error = %v, wantErr %v", err, tt.wantErr)
return
}
assert.Equal(t, tt.expectedQuery, tt.args.request.URL.RawQuery)
})
}
}

View File

@ -22,7 +22,7 @@ func (factory ProxyFactory) newOSBasedLocalProxy(path string, endpoint *portaine
proxy := &dockerLocalProxy{}
dockerTransport, err := docker.NewTransport(transportParameters, newSocketTransport(path))
dockerTransport, err := docker.NewTransport(transportParameters, newSocketTransport(path), factory.gitService)
if err != nil {
return nil, err
}

View File

@ -23,7 +23,7 @@ func (factory ProxyFactory) newOSBasedLocalProxy(path string, endpoint *portaine
proxy := &dockerLocalProxy{}
dockerTransport, err := docker.NewTransport(transportParameters, newNamedPipeTransport(path))
dockerTransport, err := docker.NewTransport(transportParameters, newNamedPipeTransport(path), factory.gitService)
if err != nil {
return nil, err
}

View File

@ -23,11 +23,12 @@ type (
dockerClientFactory *docker.ClientFactory
kubernetesClientFactory *cli.ClientFactory
kubernetesTokenCacheManager *kubernetes.TokenCacheManager
gitService portainer.GitService
}
)
// NewProxyFactory returns a pointer to a new instance of a ProxyFactory
func NewProxyFactory(dataStore dataservices.DataStore, signatureService portainer.DigitalSignatureService, tunnelService portainer.ReverseTunnelService, clientFactory *docker.ClientFactory, kubernetesClientFactory *cli.ClientFactory, kubernetesTokenCacheManager *kubernetes.TokenCacheManager) *ProxyFactory {
func NewProxyFactory(dataStore dataservices.DataStore, signatureService portainer.DigitalSignatureService, tunnelService portainer.ReverseTunnelService, clientFactory *docker.ClientFactory, kubernetesClientFactory *cli.ClientFactory, kubernetesTokenCacheManager *kubernetes.TokenCacheManager, gitService portainer.GitService) *ProxyFactory {
return &ProxyFactory{
dataStore: dataStore,
signatureService: signatureService,
@ -35,6 +36,7 @@ func NewProxyFactory(dataStore dataservices.DataStore, signatureService portaine
dockerClientFactory: clientFactory,
kubernetesClientFactory: kubernetesClientFactory,
kubernetesTokenCacheManager: kubernetesTokenCacheManager,
gitService: gitService,
}
}

View File

@ -25,11 +25,11 @@ type (
)
// NewManager initializes a new proxy Service
func NewManager(dataStore dataservices.DataStore, signatureService portainer.DigitalSignatureService, tunnelService portainer.ReverseTunnelService, clientFactory *docker.ClientFactory, kubernetesClientFactory *cli.ClientFactory, kubernetesTokenCacheManager *kubernetes.TokenCacheManager) *Manager {
func NewManager(dataStore dataservices.DataStore, signatureService portainer.DigitalSignatureService, tunnelService portainer.ReverseTunnelService, clientFactory *docker.ClientFactory, kubernetesClientFactory *cli.ClientFactory, kubernetesTokenCacheManager *kubernetes.TokenCacheManager, gitService portainer.GitService) *Manager {
return &Manager{
endpointProxies: cmap.New(),
k8sClientFactory: kubernetesClientFactory,
proxyFactory: factory.NewProxyFactory(dataStore, signatureService, tunnelService, clientFactory, kubernetesClientFactory, kubernetesTokenCacheManager),
proxyFactory: factory.NewProxyFactory(dataStore, signatureService, tunnelService, clientFactory, kubernetesClientFactory, kubernetesTokenCacheManager, gitService),
}
}

View File

@ -166,7 +166,13 @@
<div class="form-group">
<label for="image_url" class="col-sm-2 control-label text-left">URL</label>
<div class="col-sm-10">
<input type="text" class="form-control" ng-model="formValues.URL" id="image_url" placeholder="https://myhost.mydomain/myimage.tar.gz" />
<input
type="text"
class="form-control"
ng-model="formValues.URL"
id="image_url"
placeholder="https://myhost.mydomain/myimage.tar.gz or https://github.com/myname/myrepo.git#mybranch"
/>
</div>
</div>
<div class="form-group">