package git import ( "context" "encoding/json" "fmt" "io" "net/http" "net/url" "os" "strings" "time" "github.com/go-git/go-git/v5/plumbing/filemode" "github.com/portainer/portainer/api/archive" "github.com/portainer/portainer/api/crypto" gittypes "github.com/portainer/portainer/api/git/types" "github.com/pkg/errors" ) const ( azureDevOpsHost = "dev.azure.com" visualStudioHostSuffix = ".visualstudio.com" ) func isAzureUrl(s string) bool { return strings.Contains(s, azureDevOpsHost) || strings.Contains(s, visualStudioHostSuffix) } type azureOptions struct { organisation, project, repository string // a user may pass credentials in a repository URL, // for example https://:@/ username, password string } // azureRef abstracts from the response of https://docs.microsoft.com/en-us/rest/api/azure/devops/git/refs/list?view=azure-devops-rest-6.0#refs type azureRef struct { Name string `json:"name"` ObjectID string `json:"objectId"` } // azureItem abstracts from the response of https://docs.microsoft.com/en-us/rest/api/azure/devops/git/items/get?view=azure-devops-rest-6.0#download type azureItem struct { ObjectID string `json:"objectId"` CommitId string `json:"commitId"` Path string `json:"path"` } type azureClient struct { baseUrl string } func NewAzureClient() *azureClient { return &azureClient{ baseUrl: "https://dev.azure.com", } } func newHttpClientForAzure(insecureSkipVerify bool) *http.Client { tlsConfig := crypto.CreateTLSConfiguration() if insecureSkipVerify { tlsConfig.InsecureSkipVerify = true } httpsCli := &http.Client{ Transport: &http.Transport{ TLSClientConfig: tlsConfig, Proxy: http.ProxyFromEnvironment, }, Timeout: 300 * time.Second, } return httpsCli } func (a *azureClient) download(ctx context.Context, destination string, opt cloneOption) error { zipFilepath, err := a.downloadZipFromAzureDevOps(ctx, opt) if err != nil { return errors.Wrap(err, "failed to download a zip file from Azure DevOps") } defer os.Remove(zipFilepath) err = archive.UnzipFile(zipFilepath, destination) if err != nil { return errors.Wrap(err, "failed to unzip file") } return nil } func (a *azureClient) downloadZipFromAzureDevOps(ctx context.Context, opt cloneOption) (string, error) { config, err := parseUrl(opt.repositoryUrl) if err != nil { return "", errors.WithMessage(err, "failed to parse url") } downloadUrl, err := a.buildDownloadUrl(config, opt.referenceName) if err != nil { return "", errors.WithMessage(err, "failed to build download url") } zipFile, err := os.CreateTemp("", "azure-git-repo-*.zip") if err != nil { return "", errors.WithMessage(err, "failed to create temp file") } defer zipFile.Close() req, err := http.NewRequestWithContext(ctx, "GET", downloadUrl, nil) if opt.username != "" || opt.password != "" { req.SetBasicAuth(opt.username, opt.password) } else if config.username != "" || config.password != "" { req.SetBasicAuth(config.username, config.password) } if err != nil { return "", errors.WithMessage(err, "failed to create a new HTTP request") } client := newHttpClientForAzure(opt.tlsSkipVerify) defer client.CloseIdleConnections() res, err := client.Do(req) if err != nil { return "", errors.WithMessage(err, "failed to make an HTTP request") } defer res.Body.Close() if res.StatusCode != http.StatusOK { return "", fmt.Errorf("failed to download zip with a status \"%v\"", res.Status) } _, err = io.Copy(zipFile, res.Body) if err != nil { return "", errors.WithMessage(err, "failed to save HTTP response to a file") } return zipFile.Name(), nil } func (a *azureClient) latestCommitID(ctx context.Context, opt fetchOption) (string, error) { rootItem, err := a.getRootItem(ctx, opt) if err != nil { return "", err } return rootItem.CommitId, nil } func (a *azureClient) getRootItem(ctx context.Context, opt fetchOption) (*azureItem, error) { config, err := parseUrl(opt.repositoryUrl) if err != nil { return nil, errors.WithMessage(err, "failed to parse url") } rootItemUrl, err := a.buildRootItemUrl(config, opt.referenceName) if err != nil { return nil, errors.WithMessage(err, "failed to build azure root item url") } req, err := http.NewRequestWithContext(ctx, "GET", rootItemUrl, nil) if opt.username != "" || opt.password != "" { req.SetBasicAuth(opt.username, opt.password) } else if config.username != "" || config.password != "" { req.SetBasicAuth(config.username, config.password) } if err != nil { return nil, errors.WithMessage(err, "failed to create a new HTTP request") } client := newHttpClientForAzure(opt.tlsSkipVerify) defer client.CloseIdleConnections() resp, err := client.Do(req) if err != nil { return nil, errors.WithMessage(err, "failed to make an HTTP request") } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, checkAzureStatusCode(fmt.Errorf("failed to get repository root item with a status \"%v\"", resp.Status), resp.StatusCode) } var items struct { Value []azureItem } if err := json.NewDecoder(resp.Body).Decode(&items); err != nil { return nil, errors.Wrap(err, "could not parse Azure items response") } if len(items.Value) == 0 || items.Value[0].CommitId == "" { return nil, errors.Errorf("failed to get latest commitID in the repository") } return &items.Value[0], nil } func parseUrl(rawUrl string) (*azureOptions, error) { if strings.HasPrefix(rawUrl, "https://") || strings.HasPrefix(rawUrl, "http://") { return parseHttpUrl(rawUrl) } if strings.HasPrefix(rawUrl, "git@ssh") { return parseSshUrl(rawUrl) } if strings.HasPrefix(rawUrl, "ssh://") { r := []rune(rawUrl) return parseSshUrl(string(r[6:])) // remove the prefix } return nil, errors.Errorf("supported url schemes are https and ssh; recevied URL %s rawUrl", rawUrl) } const expectedSshUrl = "git@ssh.dev.azure.com:v3/Organisation/Project/Repository" func parseSshUrl(rawUrl string) (*azureOptions, error) { path := strings.Split(rawUrl, "/") unexpectedUrlErr := errors.Errorf("want url %s, got %s", expectedSshUrl, rawUrl) if len(path) != 4 { return nil, unexpectedUrlErr } return &azureOptions{ organisation: path[1], project: path[2], repository: path[3], }, nil } const expectedAzureDevOpsHttpUrl = "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository" const expectedVisualStudioHttpUrl = "https://organisation.visualstudio.com/project/_git/repository" func parseHttpUrl(rawUrl string) (*azureOptions, error) { u, err := url.Parse(rawUrl) if err != nil { return nil, errors.Wrap(err, "failed to parse HTTP url") } opt := azureOptions{} switch { case u.Host == azureDevOpsHost: path := strings.Split(u.Path, "/") if len(path) != 5 { return nil, errors.Errorf("want url %s, got %s", expectedAzureDevOpsHttpUrl, u) } opt.organisation = path[1] opt.project = path[2] opt.repository = path[4] case strings.HasSuffix(u.Host, visualStudioHostSuffix): path := strings.Split(u.Path, "/") if len(path) != 4 { return nil, errors.Errorf("want url %s, got %s", expectedVisualStudioHttpUrl, u) } opt.organisation = strings.TrimSuffix(u.Host, visualStudioHostSuffix) opt.project = path[1] opt.repository = path[3] default: return nil, errors.Errorf("unknown azure host in url \"%s\"", rawUrl) } opt.username = u.User.Username() opt.password, _ = u.User.Password() return &opt, nil } func (a *azureClient) buildDownloadUrl(config *azureOptions, referenceName string) (string, error) { rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/items", a.baseUrl, url.PathEscape(config.organisation), url.PathEscape(config.project), url.PathEscape(config.repository)) u, err := url.Parse(rawUrl) if err != nil { return "", errors.Wrapf(err, "failed to parse download url path %s", rawUrl) } q := u.Query() // scopePath=/&download=true&versionDescriptor.version=main&$format=zip&recursionLevel=full&api-version=6.0 q.Set("scopePath", "/") q.Set("download", "true") if referenceName != "" { q.Set("versionDescriptor.versionType", getVersionType(referenceName)) q.Set("versionDescriptor.version", formatReferenceName(referenceName)) } q.Set("$format", "zip") q.Set("recursionLevel", "full") q.Set("api-version", "6.0") u.RawQuery = q.Encode() return u.String(), nil } func (a *azureClient) buildRootItemUrl(config *azureOptions, referenceName string) (string, error) { rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/items", a.baseUrl, url.PathEscape(config.organisation), url.PathEscape(config.project), url.PathEscape(config.repository)) u, err := url.Parse(rawUrl) if err != nil { return "", errors.Wrapf(err, "failed to parse root item url path %s", rawUrl) } q := u.Query() q.Set("scopePath", "/") if referenceName != "" { q.Set("versionDescriptor.versionType", getVersionType(referenceName)) q.Set("versionDescriptor.version", formatReferenceName(referenceName)) } q.Set("api-version", "6.0") u.RawQuery = q.Encode() return u.String(), nil } func (a *azureClient) buildRefsUrl(config *azureOptions) (string, error) { // ref@https://docs.microsoft.com/en-us/rest/api/azure/devops/git/refs/list?view=azure-devops-rest-6.0#gitref rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/refs", a.baseUrl, url.PathEscape(config.organisation), url.PathEscape(config.project), url.PathEscape(config.repository)) u, err := url.Parse(rawUrl) if err != nil { return "", errors.Wrapf(err, "failed to parse list refs url path %s", rawUrl) } q := u.Query() q.Set("api-version", "6.0") u.RawQuery = q.Encode() return u.String(), nil } func (a *azureClient) buildTreeUrl(config *azureOptions, rootObjectHash string) (string, error) { // ref@https://docs.microsoft.com/en-us/rest/api/azure/devops/git/trees/get?view=azure-devops-rest-6.0 rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/trees/%s", a.baseUrl, url.PathEscape(config.organisation), url.PathEscape(config.project), url.PathEscape(config.repository), url.PathEscape(rootObjectHash), ) u, err := url.Parse(rawUrl) if err != nil { return "", errors.Wrapf(err, "failed to parse list tree url path %s", rawUrl) } q := u.Query() // projectId={projectId}&recursive=true&fileName={fileName}&$format={$format}&api-version=6.0 q.Set("recursive", "true") q.Set("api-version", "6.0") u.RawQuery = q.Encode() return u.String(), nil } const ( branchPrefix = "refs/heads/" tagPrefix = "refs/tags/" ) func formatReferenceName(name string) string { if strings.HasPrefix(name, branchPrefix) { return strings.TrimPrefix(name, branchPrefix) } if strings.HasPrefix(name, tagPrefix) { return strings.TrimPrefix(name, tagPrefix) } return name } func getVersionType(name string) string { if strings.HasPrefix(name, branchPrefix) { return "branch" } if strings.HasPrefix(name, tagPrefix) { return "tag" } return "commit" } func (a *azureClient) listRefs(ctx context.Context, opt baseOption) ([]string, error) { config, err := parseUrl(opt.repositoryUrl) if err != nil { return nil, errors.WithMessage(err, "failed to parse url") } listRefsUrl, err := a.buildRefsUrl(config) if err != nil { return nil, errors.WithMessage(err, "failed to build list refs url") } req, err := http.NewRequestWithContext(ctx, "GET", listRefsUrl, nil) if opt.username != "" || opt.password != "" { req.SetBasicAuth(opt.username, opt.password) } else if config.username != "" || config.password != "" { req.SetBasicAuth(config.username, config.password) } if err != nil { return nil, errors.WithMessage(err, "failed to create a new HTTP request") } client := newHttpClientForAzure(opt.tlsSkipVerify) defer client.CloseIdleConnections() resp, err := client.Do(req) if err != nil { return nil, errors.WithMessage(err, "failed to make an HTTP request") } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, checkAzureStatusCode(fmt.Errorf("failed to list refs with a status \"%v\"", resp.Status), resp.StatusCode) } var refs struct { Value []azureRef } if err := json.NewDecoder(resp.Body).Decode(&refs); err != nil { return nil, errors.Wrap(err, "could not parse Azure refs response") } var ret []string for _, value := range refs.Value { if value.Name == "HEAD" { continue } ret = append(ret, value.Name) } return ret, nil } // listFiles list all filenames under the specific repository func (a *azureClient) listFiles(ctx context.Context, opt fetchOption) ([]string, error) { rootItem, err := a.getRootItem(ctx, opt) if err != nil { return nil, err } config, err := parseUrl(opt.repositoryUrl) if err != nil { return nil, errors.WithMessage(err, "failed to parse url") } listTreeUrl, err := a.buildTreeUrl(config, rootItem.ObjectID) if err != nil { return nil, errors.WithMessage(err, "failed to build list tree url") } req, err := http.NewRequestWithContext(ctx, "GET", listTreeUrl, nil) if opt.username != "" || opt.password != "" { req.SetBasicAuth(opt.username, opt.password) } else if config.username != "" || config.password != "" { req.SetBasicAuth(config.username, config.password) } if err != nil { return nil, errors.WithMessage(err, "failed to create a new HTTP request") } client := newHttpClientForAzure(opt.tlsSkipVerify) defer client.CloseIdleConnections() resp, err := client.Do(req) if err != nil { return nil, errors.WithMessage(err, "failed to make an HTTP request") } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("failed to list tree url with a status \"%v\"", resp.Status) } var tree struct { TreeEntries []struct { RelativePath string `json:"relativePath"` Mode string `json:"mode"` } `json:"treeEntries"` } if err := json.NewDecoder(resp.Body).Decode(&tree); err != nil { return nil, errors.Wrap(err, "could not parse Azure tree response") } var allPaths []string for _, treeEntry := range tree.TreeEntries { mode, _ := filemode.New(treeEntry.Mode) isDir := filemode.Dir == mode if opt.dirOnly == isDir { allPaths = append(allPaths, treeEntry.RelativePath) } } return allPaths, nil } func checkAzureStatusCode(err error, code int) error { if code == http.StatusNotFound { return gittypes.ErrIncorrectRepositoryURL } else if code == http.StatusUnauthorized || code == http.StatusNonAuthoritativeInfo { return gittypes.ErrAuthenticationFailure } return err }