feat: add support for Onedrive Sharelink driver (#6793)

* feat: add support for Onedrive Sharelink driver

* fix(Onedrive Sharelink): use internal UA
pull/6801/head
Sakura-Byte 2024-07-17 12:21:06 +08:00 committed by GitHub
parent 049575b5a5
commit cee00005ab
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 604 additions and 0 deletions

View File

@ -36,6 +36,7 @@ import (
_ "github.com/alist-org/alist/v3/drivers/netease_music"
_ "github.com/alist-org/alist/v3/drivers/onedrive"
_ "github.com/alist-org/alist/v3/drivers/onedrive_app"
_ "github.com/alist-org/alist/v3/drivers/onedrive_sharelink"
_ "github.com/alist-org/alist/v3/drivers/pikpak"
_ "github.com/alist-org/alist/v3/drivers/pikpak_share"
_ "github.com/alist-org/alist/v3/drivers/quark_uc"

View File

@ -0,0 +1,131 @@
package onedrive_sharelink
import (
"context"
"strings"
"time"
"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/pkg/cron"
"github.com/alist-org/alist/v3/pkg/utils"
log "github.com/sirupsen/logrus"
)
type OnedriveSharelink struct {
model.Storage
cron *cron.Cron
Addition
}
func (d *OnedriveSharelink) Config() driver.Config {
return config
}
func (d *OnedriveSharelink) GetAddition() driver.Additional {
return &d.Addition
}
func (d *OnedriveSharelink) Init(ctx context.Context) error {
// Initialize error variable
var err error
// If there is "-my" in the URL, it is NOT a SharePoint link
d.IsSharepoint = !strings.Contains(d.ShareLinkURL, "-my")
// Initialize cron job to run every hour
d.cron = cron.NewCron(time.Hour * 1)
d.cron.Do(func() {
var err error
d.Headers, err = d.getHeaders()
if err != nil {
log.Errorf("%+v", err)
}
})
// Get initial headers
d.Headers, err = d.getHeaders()
if err != nil {
return err
}
return nil
}
func (d *OnedriveSharelink) Drop(ctx context.Context) error {
return nil
}
func (d *OnedriveSharelink) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
path := dir.GetPath()
files, err := d.getFiles(path)
if err != nil {
return nil, err
}
// Convert the slice of files to the required model.Obj format
return utils.SliceConvert(files, func(src Item) (model.Obj, error) {
return fileToObj(src), nil
})
}
func (d *OnedriveSharelink) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
// Get the unique ID of the file
uniqueId := file.GetID()
// Cut the first char and the last char
uniqueId = uniqueId[1 : len(uniqueId)-1]
url := d.downloadLinkPrefix + uniqueId
header := d.Headers
// If the headers are older than 30 minutes, get new headers
if d.HeaderTime < time.Now().Unix()-1800 {
var err error
log.Debug("headers are older than 30 minutes, get new headers")
header, err = d.getHeaders()
if err != nil {
return nil, err
}
}
return &model.Link{
URL: url,
Header: header,
}, nil
}
func (d *OnedriveSharelink) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error {
// TODO create folder, optional
return errs.NotImplement
}
func (d *OnedriveSharelink) Move(ctx context.Context, srcObj, dstDir model.Obj) error {
// TODO move obj, optional
return errs.NotImplement
}
func (d *OnedriveSharelink) Rename(ctx context.Context, srcObj model.Obj, newName string) error {
// TODO rename obj, optional
return errs.NotImplement
}
func (d *OnedriveSharelink) Copy(ctx context.Context, srcObj, dstDir model.Obj) error {
// TODO copy obj, optional
return errs.NotImplement
}
func (d *OnedriveSharelink) Remove(ctx context.Context, obj model.Obj) error {
// TODO remove obj, optional
return errs.NotImplement
}
func (d *OnedriveSharelink) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error {
// TODO upload file, optional
return errs.NotImplement
}
//func (d *OnedriveSharelink) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) {
// return nil, errs.NotSupport
//}
var _ driver.Driver = (*OnedriveSharelink)(nil)

View File

@ -0,0 +1,32 @@
package onedrive_sharelink
import (
"net/http"
"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/op"
)
type Addition struct {
driver.RootPath
ShareLinkURL string `json:"url" required:"true"`
ShareLinkPassword string `json:"password"`
IsSharepoint bool
downloadLinkPrefix string
Headers http.Header
HeaderTime int64
}
var config = driver.Config{
Name: "Onedrive Sharelink",
OnlyProxy: true,
NoUpload: true,
DefaultRoot: "/",
CheckStatus: false,
}
func init() {
op.RegisterDriver(func() driver.Driver {
return &OnedriveSharelink{}
})
}

View File

@ -0,0 +1,77 @@
package onedrive_sharelink
import (
"strconv"
"time"
"github.com/alist-org/alist/v3/internal/model"
)
// FolderResp represents the structure of the folder response from the OneDrive API.
type FolderResp struct {
// Data holds the nested structure of the response.
Data struct {
Legacy struct {
RenderListData struct {
ListData struct {
Items []Item `json:"Row"` // Items contains the list of items in the folder.
} `json:"ListData"`
} `json:"renderListDataAsStream"`
} `json:"legacy"`
} `json:"data"`
}
// Item represents an individual item in the folder.
type Item struct {
ObjType string `json:"FSObjType"` // ObjType indicates if the item is a file or folder.
Name string `json:"FileLeafRef"` // Name is the name of the item.
ModifiedTime time.Time `json:"Modified."` // ModifiedTime is the last modified time of the item.
Size string `json:"File_x0020_Size"` // Size is the size of the item in string format.
Id string `json:"UniqueId"` // Id is the unique identifier of the item.
}
// fileToObj converts an Item to an ObjThumb.
func fileToObj(f Item) *model.ObjThumb {
// Convert Size from string to int64.
size, _ := strconv.ParseInt(f.Size, 10, 64)
// Convert ObjType from string to int.
objtype, _ := strconv.Atoi(f.ObjType)
// Create a new ObjThumb with the converted values.
file := &model.ObjThumb{
Object: model.Object{
Name: f.Name,
Modified: f.ModifiedTime,
Size: size,
IsFolder: objtype == 1, // Check if the item is a folder.
ID: f.Id,
},
Thumbnail: model.Thumbnail{},
}
return file
}
// GraphQLNEWRequest represents the structure of a new GraphQL request.
type GraphQLNEWRequest struct {
ListData struct {
NextHref string `json:"NextHref"` // NextHref is the link to the next set of data.
Row []Item `json:"Row"` // Row contains the list of items.
} `json:"ListData"`
}
// GraphQLRequest represents the structure of a GraphQL request.
type GraphQLRequest struct {
Data struct {
Legacy struct {
RenderListDataAsStream struct {
ListData struct {
NextHref string `json:"NextHref"` // NextHref is the link to the next set of data.
Row []Item `json:"Row"` // Row contains the list of items.
} `json:"ListData"`
ViewMetadata struct {
ListViewXml string `json:"ListViewXml"` // ListViewXml contains the XML of the list view.
} `json:"ViewMetadata"`
} `json:"renderListDataAsStream"`
} `json:"legacy"`
} `json:"data"`
}

View File

@ -0,0 +1,363 @@
package onedrive_sharelink
import (
"crypto/tls"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"regexp"
"strings"
"time"
"github.com/alist-org/alist/v3/drivers/base"
"github.com/alist-org/alist/v3/internal/conf"
log "github.com/sirupsen/logrus"
"golang.org/x/net/html"
)
// NewNoRedirectClient creates an HTTP client that doesn't follow redirects
func NewNoRedirectCLient() *http.Client {
return &http.Client{
Timeout: time.Hour * 48,
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify},
},
// Prevent following redirects
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
},
}
}
// getCookiesWithPassword fetches cookies required for authenticated access using the provided password
func getCookiesWithPassword(link, password string) (string, error) {
// Send GET request
resp, err := http.Get(link)
if err != nil {
return "", err
}
defer resp.Body.Close()
// Parse the HTML response
doc, err := html.Parse(resp.Body)
if err != nil {
return "", err
}
// Initialize variables to store form data
var viewstate, eventvalidation, postAction string
// Recursive function to find input fields by their IDs
var findInputFields func(*html.Node)
findInputFields = func(n *html.Node) {
if n.Type == html.ElementNode && n.Data == "input" {
for _, attr := range n.Attr {
if attr.Key == "id" {
switch attr.Val {
case "__VIEWSTATE":
viewstate = getAttrValue(n, "value")
case "__EVENTVALIDATION":
eventvalidation = getAttrValue(n, "value")
}
}
}
}
if n.Type == html.ElementNode && n.Data == "form" {
for _, attr := range n.Attr {
if attr.Key == "id" && attr.Val == "inputForm" {
postAction = getAttrValue(n, "action")
}
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
findInputFields(c)
}
}
findInputFields(doc)
// Prepare the new URL for the POST request
linkParts, err := url.Parse(link)
if err != nil {
return "", err
}
newURL := fmt.Sprintf("%s://%s%s", linkParts.Scheme, linkParts.Host, postAction)
// Prepare the request body
data := url.Values{
"txtPassword": []string{password},
"__EVENTVALIDATION": []string{eventvalidation},
"__VIEWSTATE": []string{viewstate},
"__VIEWSTATEENCRYPTED": []string{""},
}
client := &http.Client{
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
},
}
// Send the POST request, preventing redirects
resp, err = client.PostForm(newURL, data)
if err != nil {
return "", err
}
// Extract the desired cookie value
cookie := resp.Cookies()
var fedAuthCookie string
for _, c := range cookie {
if c.Name == "FedAuth" {
fedAuthCookie = c.Value
break
}
}
if fedAuthCookie == "" {
return "", fmt.Errorf("wrong password")
}
return fmt.Sprintf("FedAuth=%s;", fedAuthCookie), nil
}
// getAttrValue retrieves the value of the specified attribute from an HTML node
func getAttrValue(n *html.Node, key string) string {
for _, attr := range n.Attr {
if attr.Key == key {
return attr.Val
}
}
return ""
}
// getHeaders constructs and returns the necessary HTTP headers for accessing the OneDrive share link
func (d *OnedriveSharelink) getHeaders() (http.Header, error) {
header := http.Header{}
header.Set("User-Agent", base.UserAgent)
header.Set("accept-language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6")
// Save current timestamp to d.HeaderTime
d.HeaderTime = time.Now().Unix()
if d.ShareLinkPassword == "" {
// Create a no-redirect client
clientNoDirect := NewNoRedirectCLient()
req, err := http.NewRequest("GET", d.ShareLinkURL, nil)
if err != nil {
return nil, err
}
// Set headers for the request
req.Header = header
answerNoRedirect, err := clientNoDirect.Do(req)
if err != nil {
return nil, err
}
redirectUrl := answerNoRedirect.Header.Get("Location")
log.Debugln("redirectUrl:", redirectUrl)
if redirectUrl == "" {
return nil, fmt.Errorf("password protected link. Please provide password")
}
header.Set("Cookie", answerNoRedirect.Header.Get("Set-Cookie"))
header.Set("Referer", redirectUrl)
// Extract the host part of the redirect URL and set it as the authority
u, err := url.Parse(redirectUrl)
if err != nil {
return nil, err
}
header.Set("authority", u.Host)
return header, nil
} else {
cookie, err := getCookiesWithPassword(d.ShareLinkURL, d.ShareLinkPassword)
if err != nil {
return nil, err
}
header.Set("Cookie", cookie)
header.Set("Referer", d.ShareLinkURL)
header.Set("authority", strings.Split(strings.Split(d.ShareLinkURL, "//")[1], "/")[0])
return header, nil
}
}
// getFiles retrieves the files from the OneDrive share link at the specified path
func (d *OnedriveSharelink) getFiles(path string) ([]Item, error) {
clientNoDirect := NewNoRedirectCLient()
req, err := http.NewRequest("GET", d.ShareLinkURL, nil)
if err != nil {
return nil, err
}
header := req.Header
redirectUrl := ""
if d.ShareLinkPassword == "" {
header.Set("User-Agent", base.UserAgent)
header.Set("accept-language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6")
req.Header = header
answerNoRedirect, err := clientNoDirect.Do(req)
if err != nil {
return nil, err
}
redirectUrl = answerNoRedirect.Header.Get("Location")
} else {
header = d.Headers
req.Header = header
answerNoRedirect, err := clientNoDirect.Do(req)
if err != nil {
return nil, err
}
redirectUrl = answerNoRedirect.Header.Get("Location")
}
redirectSplitURL := strings.Split(redirectUrl, "/")
req.Header = d.Headers
downloadLinkPrefix := ""
rootFolderPre := ""
// Determine the appropriate URL and root folder based on whether the link is SharePoint
if d.IsSharepoint {
// update req url
req.URL, err = url.Parse(redirectUrl)
if err != nil {
return nil, err
}
// Get redirectUrl
answer, err := clientNoDirect.Do(req)
if err != nil {
d.Headers, err = d.getHeaders()
if err != nil {
return nil, err
}
return d.getFiles(path)
}
defer answer.Body.Close()
re := regexp.MustCompile(`templateUrl":"(.*?)"`)
body, err := io.ReadAll(answer.Body)
if err != nil {
return nil, err
}
template := re.FindString(string(body))
template = template[strings.Index(template, "templateUrl\":\"")+len("templateUrl\":\""):]
template = template[:strings.Index(template, "?id=")]
template = template[:strings.LastIndex(template, "/")]
downloadLinkPrefix = template + "/download.aspx?UniqueId="
params, err := url.ParseQuery(redirectUrl[strings.Index(redirectUrl, "?")+1:])
if err != nil {
return nil, err
}
rootFolderPre = params.Get("id")
} else {
redirectUrlCut := redirectUrl[:strings.LastIndex(redirectUrl, "/")]
downloadLinkPrefix = redirectUrlCut + "/download.aspx?UniqueId="
params, err := url.ParseQuery(redirectUrl[strings.Index(redirectUrl, "?")+1:])
if err != nil {
return nil, err
}
rootFolderPre = params.Get("id")
}
d.downloadLinkPrefix = downloadLinkPrefix
rootFolder, err := url.QueryUnescape(rootFolderPre)
if err != nil {
return nil, err
}
log.Debugln("rootFolder:", rootFolder)
// Extract the relative path up to and including "Documents"
relativePath := strings.Split(rootFolder, "Documents")[0] + "Documents"
// URL encode the relative path
relativeUrl := url.QueryEscape(relativePath)
// Replace underscores and hyphens in the encoded relative path
relativeUrl = strings.Replace(relativeUrl, "_", "%5F", -1)
relativeUrl = strings.Replace(relativeUrl, "-", "%2D", -1)
// If the path is not the root, append the path to the root folder
if path != "/" {
rootFolder = rootFolder + path
}
// URL encode the full root folder path
rootFolderUrl := url.QueryEscape(rootFolder)
// Replace underscores and hyphens in the encoded root folder URL
rootFolderUrl = strings.Replace(rootFolderUrl, "_", "%5F", -1)
rootFolderUrl = strings.Replace(rootFolderUrl, "-", "%2D", -1)
log.Debugln("relativePath:", relativePath, "relativeUrl:", relativeUrl, "rootFolder:", rootFolder, "rootFolderUrl:", rootFolderUrl)
// Construct the GraphQL query with the encoded paths
graphqlVar := fmt.Sprintf(`{"query":"query (\n $listServerRelativeUrl: String!,$renderListDataAsStreamParameters: RenderListDataAsStreamParameters!,$renderListDataAsStreamQueryString: String!\n )\n {\n \n legacy {\n \n renderListDataAsStream(\n listServerRelativeUrl: $listServerRelativeUrl,\n parameters: $renderListDataAsStreamParameters,\n queryString: $renderListDataAsStreamQueryString\n )\n }\n \n \n perf {\n executionTime\n overheadTime\n parsingTime\n queryCount\n validationTime\n resolvers {\n name\n queryCount\n resolveTime\n waitTime\n }\n }\n }","variables":{"listServerRelativeUrl":"%s","renderListDataAsStreamParameters":{"renderOptions":5707527,"allowMultipleValueFilterForTaxonomyFields":true,"addRequiredFields":true,"folderServerRelativeUrl":"%s"},"renderListDataAsStreamQueryString":"@a1=\'%s\'&RootFolder=%s&TryNewExperienceSingle=TRUE"}}`, relativePath, rootFolder, relativeUrl, rootFolderUrl)
tempHeader := make(http.Header)
for k, v := range d.Headers {
tempHeader[k] = v
}
tempHeader["Content-Type"] = []string{"application/json;odata=verbose"}
client := &http.Client{}
postUrl := strings.Join(redirectSplitURL[:len(redirectSplitURL)-3], "/") + "/_api/v2.1/graphql"
req, err = http.NewRequest("POST", postUrl, strings.NewReader(graphqlVar))
if err != nil {
return nil, err
}
req.Header = tempHeader
resp, err := client.Do(req)
if err != nil {
d.Headers, err = d.getHeaders()
if err != nil {
return nil, err
}
return d.getFiles(path)
}
defer resp.Body.Close()
var graphqlReq GraphQLRequest
json.NewDecoder(resp.Body).Decode(&graphqlReq)
log.Debugln("graphqlReq:", graphqlReq)
filesData := graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.Row
if graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.NextHref != "" {
nextHref := graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.NextHref + "&@a1=REPLACEME&TryNewExperienceSingle=TRUE"
nextHref = strings.Replace(nextHref, "REPLACEME", "%27"+relativeUrl+"%27", -1)
log.Debugln("nextHref:", nextHref)
filesData = append(filesData, graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.Row...)
listViewXml := graphqlReq.Data.Legacy.RenderListDataAsStream.ViewMetadata.ListViewXml
log.Debugln("listViewXml:", listViewXml)
renderListDataAsStreamVar := `{"parameters":{"__metadata":{"type":"SP.RenderListDataParameters"},"RenderOptions":1216519,"ViewXml":"REPLACEME","AllowMultipleValueFilterForTaxonomyFields":true,"AddRequiredFields":true}}`
listViewXml = strings.Replace(listViewXml, `"`, `\"`, -1)
renderListDataAsStreamVar = strings.Replace(renderListDataAsStreamVar, "REPLACEME", listViewXml, -1)
graphqlReqNEW := GraphQLNEWRequest{}
postUrl = strings.Join(redirectSplitURL[:len(redirectSplitURL)-3], "/") + "/_api/web/GetListUsingPath(DecodedUrl=@a1)/RenderListDataAsStream" + nextHref
req, _ = http.NewRequest("POST", postUrl, strings.NewReader(renderListDataAsStreamVar))
req.Header = tempHeader
resp, err := client.Do(req)
if err != nil {
d.Headers, err = d.getHeaders()
if err != nil {
return nil, err
}
return d.getFiles(path)
}
defer resp.Body.Close()
json.NewDecoder(resp.Body).Decode(&graphqlReqNEW)
for graphqlReqNEW.ListData.NextHref != "" {
graphqlReqNEW = GraphQLNEWRequest{}
postUrl = strings.Join(redirectSplitURL[:len(redirectSplitURL)-3], "/") + "/_api/web/GetListUsingPath(DecodedUrl=@a1)/RenderListDataAsStream" + nextHref
req, _ = http.NewRequest("POST", postUrl, strings.NewReader(renderListDataAsStreamVar))
req.Header = tempHeader
resp, err := client.Do(req)
if err != nil {
d.Headers, err = d.getHeaders()
if err != nil {
return nil, err
}
return d.getFiles(path)
}
defer resp.Body.Close()
json.NewDecoder(resp.Body).Decode(&graphqlReqNEW)
nextHref = graphqlReqNEW.ListData.NextHref + "&@a1=REPLACEME&TryNewExperienceSingle=TRUE"
nextHref = strings.Replace(nextHref, "REPLACEME", "%27"+relativeUrl+"%27", -1)
filesData = append(filesData, graphqlReqNEW.ListData.Row...)
}
filesData = append(filesData, graphqlReqNEW.ListData.Row...)
} else {
filesData = append(filesData, graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.Row...)
}
return filesData, nil
}