feat: add support for Onedrive Sharelink driver (#6793)

* feat: add support for Onedrive Sharelink driver

* fix(Onedrive Sharelink): use internal UA

drivers/all.go

@@ -36,6 +36,7 @@ import (
 	_ "github.com/alist-org/alist/v3/drivers/netease_music"
 	_ "github.com/alist-org/alist/v3/drivers/onedrive"
 	_ "github.com/alist-org/alist/v3/drivers/onedrive_app"
+	_ "github.com/alist-org/alist/v3/drivers/onedrive_sharelink"
 	_ "github.com/alist-org/alist/v3/drivers/pikpak"
 	_ "github.com/alist-org/alist/v3/drivers/pikpak_share"
 	_ "github.com/alist-org/alist/v3/drivers/quark_uc"

drivers/onedrive_sharelink/driver.go

@@ -0,0 +1,131 @@
+package onedrive_sharelink
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"github.com/alist-org/alist/v3/internal/driver"
+	"github.com/alist-org/alist/v3/internal/errs"
+	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/pkg/cron"
+	"github.com/alist-org/alist/v3/pkg/utils"
+	log "github.com/sirupsen/logrus"
+)
+
+type OnedriveSharelink struct {
+	model.Storage
+	cron *cron.Cron
+	Addition
+}
+
+func (d *OnedriveSharelink) Config() driver.Config {
+	return config
+}
+
+func (d *OnedriveSharelink) GetAddition() driver.Additional {
+	return &d.Addition
+}
+
+func (d *OnedriveSharelink) Init(ctx context.Context) error {
+	// Initialize error variable
+	var err error
+
+	// If there is "-my" in the URL, it is NOT a SharePoint link
+	d.IsSharepoint = !strings.Contains(d.ShareLinkURL, "-my")
+
+	// Initialize cron job to run every hour
+	d.cron = cron.NewCron(time.Hour * 1)
+	d.cron.Do(func() {
+		var err error
+		d.Headers, err = d.getHeaders()
+		if err != nil {
+			log.Errorf("%+v", err)
+		}
+	})
+
+	// Get initial headers
+	d.Headers, err = d.getHeaders()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (d *OnedriveSharelink) Drop(ctx context.Context) error {
+	return nil
+}
+
+func (d *OnedriveSharelink) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
+	path := dir.GetPath()
+	files, err := d.getFiles(path)
+	if err != nil {
+		return nil, err
+	}
+
+	// Convert the slice of files to the required model.Obj format
+	return utils.SliceConvert(files, func(src Item) (model.Obj, error) {
+		return fileToObj(src), nil
+	})
+}
+
+func (d *OnedriveSharelink) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
+	// Get the unique ID of the file
+	uniqueId := file.GetID()
+	// Cut the first char and the last char
+	uniqueId = uniqueId[1 : len(uniqueId)-1]
+	url := d.downloadLinkPrefix + uniqueId
+	header := d.Headers
+
+	// If the headers are older than 30 minutes, get new headers
+	if d.HeaderTime < time.Now().Unix()-1800 {
+		var err error
+		log.Debug("headers are older than 30 minutes, get new headers")
+		header, err = d.getHeaders()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return &model.Link{
+		URL:    url,
+		Header: header,
+	}, nil
+}
+
+func (d *OnedriveSharelink) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error {
+	// TODO create folder, optional
+	return errs.NotImplement
+}
+
+func (d *OnedriveSharelink) Move(ctx context.Context, srcObj, dstDir model.Obj) error {
+	// TODO move obj, optional
+	return errs.NotImplement
+}
+
+func (d *OnedriveSharelink) Rename(ctx context.Context, srcObj model.Obj, newName string) error {
+	// TODO rename obj, optional
+	return errs.NotImplement
+}
+
+func (d *OnedriveSharelink) Copy(ctx context.Context, srcObj, dstDir model.Obj) error {
+	// TODO copy obj, optional
+	return errs.NotImplement
+}
+
+func (d *OnedriveSharelink) Remove(ctx context.Context, obj model.Obj) error {
+	// TODO remove obj, optional
+	return errs.NotImplement
+}
+
+func (d *OnedriveSharelink) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error {
+	// TODO upload file, optional
+	return errs.NotImplement
+}
+
+//func (d *OnedriveSharelink) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) {
+//	return nil, errs.NotSupport
+//}
+
+var _ driver.Driver = (*OnedriveSharelink)(nil)

drivers/onedrive_sharelink/meta.go

@@ -0,0 +1,32 @@
+package onedrive_sharelink
+
+import (
+	"net/http"
+
+	"github.com/alist-org/alist/v3/internal/driver"
+	"github.com/alist-org/alist/v3/internal/op"
+)
+
+type Addition struct {
+	driver.RootPath
+	ShareLinkURL       string `json:"url" required:"true"`
+	ShareLinkPassword  string `json:"password"`
+	IsSharepoint       bool
+	downloadLinkPrefix string
+	Headers            http.Header
+	HeaderTime         int64
+}
+
+var config = driver.Config{
+	Name:        "Onedrive Sharelink",
+	OnlyProxy:   true,
+	NoUpload:    true,
+	DefaultRoot: "/",
+	CheckStatus: false,
+}
+
+func init() {
+	op.RegisterDriver(func() driver.Driver {
+		return &OnedriveSharelink{}
+	})
+}

drivers/onedrive_sharelink/types.go

@@ -0,0 +1,77 @@
+package onedrive_sharelink
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/alist-org/alist/v3/internal/model"
+)
+
+// FolderResp represents the structure of the folder response from the OneDrive API.
+type FolderResp struct {
+	// Data holds the nested structure of the response.
+	Data struct {
+		Legacy struct {
+			RenderListData struct {
+				ListData struct {
+					Items []Item `json:"Row"` // Items contains the list of items in the folder.
+				} `json:"ListData"`
+			} `json:"renderListDataAsStream"`
+		} `json:"legacy"`
+	} `json:"data"`
+}
+
+// Item represents an individual item in the folder.
+type Item struct {
+	ObjType      string    `json:"FSObjType"`       // ObjType indicates if the item is a file or folder.
+	Name         string    `json:"FileLeafRef"`     // Name is the name of the item.
+	ModifiedTime time.Time `json:"Modified."`       // ModifiedTime is the last modified time of the item.
+	Size         string    `json:"File_x0020_Size"` // Size is the size of the item in string format.
+	Id           string    `json:"UniqueId"`        // Id is the unique identifier of the item.
+}
+
+// fileToObj converts an Item to an ObjThumb.
+func fileToObj(f Item) *model.ObjThumb {
+	// Convert Size from string to int64.
+	size, _ := strconv.ParseInt(f.Size, 10, 64)
+	// Convert ObjType from string to int.
+	objtype, _ := strconv.Atoi(f.ObjType)
+
+	// Create a new ObjThumb with the converted values.
+	file := &model.ObjThumb{
+		Object: model.Object{
+			Name:     f.Name,
+			Modified: f.ModifiedTime,
+			Size:     size,
+			IsFolder: objtype == 1, // Check if the item is a folder.
+			ID:       f.Id,
+		},
+		Thumbnail: model.Thumbnail{},
+	}
+	return file
+}
+
+// GraphQLNEWRequest represents the structure of a new GraphQL request.
+type GraphQLNEWRequest struct {
+	ListData struct {
+		NextHref string `json:"NextHref"` // NextHref is the link to the next set of data.
+		Row      []Item `json:"Row"`      // Row contains the list of items.
+	} `json:"ListData"`
+}
+
+// GraphQLRequest represents the structure of a GraphQL request.
+type GraphQLRequest struct {
+	Data struct {
+		Legacy struct {
+			RenderListDataAsStream struct {
+				ListData struct {
+					NextHref string `json:"NextHref"` // NextHref is the link to the next set of data.
+					Row      []Item `json:"Row"`      // Row contains the list of items.
+				} `json:"ListData"`
+				ViewMetadata struct {
+					ListViewXml string `json:"ListViewXml"` // ListViewXml contains the XML of the list view.
+				} `json:"ViewMetadata"`
+			} `json:"renderListDataAsStream"`
+		} `json:"legacy"`
+	} `json:"data"`
+}

drivers/onedrive_sharelink/util.go

@@ -0,0 +1,363 @@
+package onedrive_sharelink
+
+import (
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/alist-org/alist/v3/drivers/base"
+	"github.com/alist-org/alist/v3/internal/conf"
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/net/html"
+)
+
+// NewNoRedirectClient creates an HTTP client that doesn't follow redirects
+func NewNoRedirectCLient() *http.Client {
+	return &http.Client{
+		Timeout: time.Hour * 48,
+		Transport: &http.Transport{
+			Proxy:           http.ProxyFromEnvironment,
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify},
+		},
+		// Prevent following redirects
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+}
+
+// getCookiesWithPassword fetches cookies required for authenticated access using the provided password
+func getCookiesWithPassword(link, password string) (string, error) {
+	// Send GET request
+	resp, err := http.Get(link)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	// Parse the HTML response
+	doc, err := html.Parse(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	// Initialize variables to store form data
+	var viewstate, eventvalidation, postAction string
+
+	// Recursive function to find input fields by their IDs
+	var findInputFields func(*html.Node)
+	findInputFields = func(n *html.Node) {
+		if n.Type == html.ElementNode && n.Data == "input" {
+			for _, attr := range n.Attr {
+				if attr.Key == "id" {
+					switch attr.Val {
+					case "__VIEWSTATE":
+						viewstate = getAttrValue(n, "value")
+					case "__EVENTVALIDATION":
+						eventvalidation = getAttrValue(n, "value")
+					}
+				}
+			}
+		}
+		if n.Type == html.ElementNode && n.Data == "form" {
+			for _, attr := range n.Attr {
+				if attr.Key == "id" && attr.Val == "inputForm" {
+					postAction = getAttrValue(n, "action")
+				}
+			}
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			findInputFields(c)
+		}
+	}
+	findInputFields(doc)
+
+	// Prepare the new URL for the POST request
+	linkParts, err := url.Parse(link)
+	if err != nil {
+		return "", err
+	}
+
+	newURL := fmt.Sprintf("%s://%s%s", linkParts.Scheme, linkParts.Host, postAction)
+
+	// Prepare the request body
+	data := url.Values{
+		"txtPassword":          []string{password},
+		"__EVENTVALIDATION":    []string{eventvalidation},
+		"__VIEWSTATE":          []string{viewstate},
+		"__VIEWSTATEENCRYPTED": []string{""},
+	}
+
+	client := &http.Client{
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+	// Send the POST request, preventing redirects
+	resp, err = client.PostForm(newURL, data)
+	if err != nil {
+		return "", err
+	}
+
+	// Extract the desired cookie value
+	cookie := resp.Cookies()
+	var fedAuthCookie string
+	for _, c := range cookie {
+		if c.Name == "FedAuth" {
+			fedAuthCookie = c.Value
+			break
+		}
+	}
+	if fedAuthCookie == "" {
+		return "", fmt.Errorf("wrong password")
+	}
+	return fmt.Sprintf("FedAuth=%s;", fedAuthCookie), nil
+}
+
+// getAttrValue retrieves the value of the specified attribute from an HTML node
+func getAttrValue(n *html.Node, key string) string {
+	for _, attr := range n.Attr {
+		if attr.Key == key {
+			return attr.Val
+		}
+	}
+	return ""
+}
+
+// getHeaders constructs and returns the necessary HTTP headers for accessing the OneDrive share link
+func (d *OnedriveSharelink) getHeaders() (http.Header, error) {
+	header := http.Header{}
+	header.Set("User-Agent", base.UserAgent)
+	header.Set("accept-language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6")
+
+	// Save current timestamp to d.HeaderTime
+	d.HeaderTime = time.Now().Unix()
+
+	if d.ShareLinkPassword == "" {
+		// Create a no-redirect client
+		clientNoDirect := NewNoRedirectCLient()
+		req, err := http.NewRequest("GET", d.ShareLinkURL, nil)
+		if err != nil {
+			return nil, err
+		}
+		// Set headers for the request
+		req.Header = header
+		answerNoRedirect, err := clientNoDirect.Do(req)
+		if err != nil {
+			return nil, err
+		}
+		redirectUrl := answerNoRedirect.Header.Get("Location")
+		log.Debugln("redirectUrl:", redirectUrl)
+		if redirectUrl == "" {
+			return nil, fmt.Errorf("password protected link. Please provide password")
+		}
+		header.Set("Cookie", answerNoRedirect.Header.Get("Set-Cookie"))
+		header.Set("Referer", redirectUrl)
+
+		// Extract the host part of the redirect URL and set it as the authority
+		u, err := url.Parse(redirectUrl)
+		if err != nil {
+			return nil, err
+		}
+		header.Set("authority", u.Host)
+		return header, nil
+	} else {
+		cookie, err := getCookiesWithPassword(d.ShareLinkURL, d.ShareLinkPassword)
+		if err != nil {
+			return nil, err
+		}
+		header.Set("Cookie", cookie)
+		header.Set("Referer", d.ShareLinkURL)
+		header.Set("authority", strings.Split(strings.Split(d.ShareLinkURL, "//")[1], "/")[0])
+		return header, nil
+	}
+}
+
+// getFiles retrieves the files from the OneDrive share link at the specified path
+func (d *OnedriveSharelink) getFiles(path string) ([]Item, error) {
+	clientNoDirect := NewNoRedirectCLient()
+	req, err := http.NewRequest("GET", d.ShareLinkURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	header := req.Header
+	redirectUrl := ""
+	if d.ShareLinkPassword == "" {
+		header.Set("User-Agent", base.UserAgent)
+		header.Set("accept-language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6")
+		req.Header = header
+		answerNoRedirect, err := clientNoDirect.Do(req)
+		if err != nil {
+			return nil, err
+		}
+		redirectUrl = answerNoRedirect.Header.Get("Location")
+	} else {
+		header = d.Headers
+		req.Header = header
+		answerNoRedirect, err := clientNoDirect.Do(req)
+		if err != nil {
+			return nil, err
+		}
+		redirectUrl = answerNoRedirect.Header.Get("Location")
+	}
+	redirectSplitURL := strings.Split(redirectUrl, "/")
+	req.Header = d.Headers
+	downloadLinkPrefix := ""
+	rootFolderPre := ""
+
+	// Determine the appropriate URL and root folder based on whether the link is SharePoint
+	if d.IsSharepoint {
+		// update req url
+		req.URL, err = url.Parse(redirectUrl)
+		if err != nil {
+			return nil, err
+		}
+		// Get redirectUrl
+		answer, err := clientNoDirect.Do(req)
+		if err != nil {
+			d.Headers, err = d.getHeaders()
+			if err != nil {
+				return nil, err
+			}
+			return d.getFiles(path)
+		}
+		defer answer.Body.Close()
+		re := regexp.MustCompile(`templateUrl":"(.*?)"`)
+		body, err := io.ReadAll(answer.Body)
+		if err != nil {
+			return nil, err
+		}
+		template := re.FindString(string(body))
+		template = template[strings.Index(template, "templateUrl\":\"")+len("templateUrl\":\""):]
+		template = template[:strings.Index(template, "?id=")]
+		template = template[:strings.LastIndex(template, "/")]
+		downloadLinkPrefix = template + "/download.aspx?UniqueId="
+		params, err := url.ParseQuery(redirectUrl[strings.Index(redirectUrl, "?")+1:])
+		if err != nil {
+			return nil, err
+		}
+		rootFolderPre = params.Get("id")
+	} else {
+		redirectUrlCut := redirectUrl[:strings.LastIndex(redirectUrl, "/")]
+		downloadLinkPrefix = redirectUrlCut + "/download.aspx?UniqueId="
+		params, err := url.ParseQuery(redirectUrl[strings.Index(redirectUrl, "?")+1:])
+		if err != nil {
+			return nil, err
+		}
+		rootFolderPre = params.Get("id")
+	}
+	d.downloadLinkPrefix = downloadLinkPrefix
+	rootFolder, err := url.QueryUnescape(rootFolderPre)
+	if err != nil {
+		return nil, err
+	}
+	log.Debugln("rootFolder:", rootFolder)
+	// Extract the relative path up to and including "Documents"
+	relativePath := strings.Split(rootFolder, "Documents")[0] + "Documents"
+
+	// URL encode the relative path
+	relativeUrl := url.QueryEscape(relativePath)
+	// Replace underscores and hyphens in the encoded relative path
+	relativeUrl = strings.Replace(relativeUrl, "_", "%5F", -1)
+	relativeUrl = strings.Replace(relativeUrl, "-", "%2D", -1)
+
+	// If the path is not the root, append the path to the root folder
+	if path != "/" {
+		rootFolder = rootFolder + path
+	}
+
+	// URL encode the full root folder path
+	rootFolderUrl := url.QueryEscape(rootFolder)
+	// Replace underscores and hyphens in the encoded root folder URL
+	rootFolderUrl = strings.Replace(rootFolderUrl, "_", "%5F", -1)
+	rootFolderUrl = strings.Replace(rootFolderUrl, "-", "%2D", -1)
+
+	log.Debugln("relativePath:", relativePath, "relativeUrl:", relativeUrl, "rootFolder:", rootFolder, "rootFolderUrl:", rootFolderUrl)
+
+	// Construct the GraphQL query with the encoded paths
+	graphqlVar := fmt.Sprintf(`{"query":"query (\n        $listServerRelativeUrl: String!,$renderListDataAsStreamParameters: RenderListDataAsStreamParameters!,$renderListDataAsStreamQueryString: String!\n        )\n      {\n      \n      legacy {\n      \n      renderListDataAsStream(\n      listServerRelativeUrl: $listServerRelativeUrl,\n      parameters: $renderListDataAsStreamParameters,\n      queryString: $renderListDataAsStreamQueryString\n      )\n    }\n      \n      \n  perf {\n    executionTime\n    overheadTime\n    parsingTime\n    queryCount\n    validationTime\n    resolvers {\n      name\n      queryCount\n      resolveTime\n      waitTime\n    }\n  }\n    }","variables":{"listServerRelativeUrl":"%s","renderListDataAsStreamParameters":{"renderOptions":5707527,"allowMultipleValueFilterForTaxonomyFields":true,"addRequiredFields":true,"folderServerRelativeUrl":"%s"},"renderListDataAsStreamQueryString":"@a1=\'%s\'&RootFolder=%s&TryNewExperienceSingle=TRUE"}}`, relativePath, rootFolder, relativeUrl, rootFolderUrl)
+	tempHeader := make(http.Header)
+	for k, v := range d.Headers {
+		tempHeader[k] = v
+	}
+	tempHeader["Content-Type"] = []string{"application/json;odata=verbose"}
+
+	client := &http.Client{}
+	postUrl := strings.Join(redirectSplitURL[:len(redirectSplitURL)-3], "/") + "/_api/v2.1/graphql"
+	req, err = http.NewRequest("POST", postUrl, strings.NewReader(graphqlVar))
+	if err != nil {
+		return nil, err
+	}
+	req.Header = tempHeader
+
+	resp, err := client.Do(req)
+	if err != nil {
+		d.Headers, err = d.getHeaders()
+		if err != nil {
+			return nil, err
+		}
+		return d.getFiles(path)
+	}
+	defer resp.Body.Close()
+	var graphqlReq GraphQLRequest
+	json.NewDecoder(resp.Body).Decode(&graphqlReq)
+	log.Debugln("graphqlReq:", graphqlReq)
+	filesData := graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.Row
+	if graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.NextHref != "" {
+		nextHref := graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.NextHref + "&@a1=REPLACEME&TryNewExperienceSingle=TRUE"
+		nextHref = strings.Replace(nextHref, "REPLACEME", "%27"+relativeUrl+"%27", -1)
+		log.Debugln("nextHref:", nextHref)
+		filesData = append(filesData, graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.Row...)
+
+		listViewXml := graphqlReq.Data.Legacy.RenderListDataAsStream.ViewMetadata.ListViewXml
+		log.Debugln("listViewXml:", listViewXml)
+		renderListDataAsStreamVar := `{"parameters":{"__metadata":{"type":"SP.RenderListDataParameters"},"RenderOptions":1216519,"ViewXml":"REPLACEME","AllowMultipleValueFilterForTaxonomyFields":true,"AddRequiredFields":true}}`
+		listViewXml = strings.Replace(listViewXml, `"`, `\"`, -1)
+		renderListDataAsStreamVar = strings.Replace(renderListDataAsStreamVar, "REPLACEME", listViewXml, -1)
+
+		graphqlReqNEW := GraphQLNEWRequest{}
+		postUrl = strings.Join(redirectSplitURL[:len(redirectSplitURL)-3], "/") + "/_api/web/GetListUsingPath(DecodedUrl=@a1)/RenderListDataAsStream" + nextHref
+		req, _ = http.NewRequest("POST", postUrl, strings.NewReader(renderListDataAsStreamVar))
+		req.Header = tempHeader
+
+		resp, err := client.Do(req)
+		if err != nil {
+			d.Headers, err = d.getHeaders()
+			if err != nil {
+				return nil, err
+			}
+			return d.getFiles(path)
+		}
+		defer resp.Body.Close()
+		json.NewDecoder(resp.Body).Decode(&graphqlReqNEW)
+		for graphqlReqNEW.ListData.NextHref != "" {
+			graphqlReqNEW = GraphQLNEWRequest{}
+			postUrl = strings.Join(redirectSplitURL[:len(redirectSplitURL)-3], "/") + "/_api/web/GetListUsingPath(DecodedUrl=@a1)/RenderListDataAsStream" + nextHref
+			req, _ = http.NewRequest("POST", postUrl, strings.NewReader(renderListDataAsStreamVar))
+			req.Header = tempHeader
+			resp, err := client.Do(req)
+			if err != nil {
+				d.Headers, err = d.getHeaders()
+				if err != nil {
+					return nil, err
+				}
+				return d.getFiles(path)
+			}
+			defer resp.Body.Close()
+			json.NewDecoder(resp.Body).Decode(&graphqlReqNEW)
+			nextHref = graphqlReqNEW.ListData.NextHref + "&@a1=REPLACEME&TryNewExperienceSingle=TRUE"
+			nextHref = strings.Replace(nextHref, "REPLACEME", "%27"+relativeUrl+"%27", -1)
+			filesData = append(filesData, graphqlReqNEW.ListData.Row...)
+		}
+		filesData = append(filesData, graphqlReqNEW.ListData.Row...)
+	} else {
+		filesData = append(filesData, graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.Row...)
+	}
+	return filesData, nil
+}