diff --git a/.github/workflows/issue_rm_working.yml b/.github/workflows/issue_on_close.yml similarity index 90% rename from .github/workflows/issue_rm_working.yml rename to .github/workflows/issue_on_close.yml index 56181366..c90c2c0c 100644 --- a/.github/workflows/issue_rm_working.yml +++ b/.github/workflows/issue_on_close.yml @@ -14,4 +14,4 @@ jobs: actions: 'remove-labels' token: ${{ secrets.GITHUB_TOKEN }} issue-number: ${{ github.event.issue.number }} - labels: 'working' \ No newline at end of file + labels: 'working,pr-welcome' diff --git a/README.md b/README.md index d5e88e43..3f8fc4ee 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ English | [中文](./README_cn.md)| [日本語](./README_ja.md) | [Contributing] ## Features -- [x] Multiple storage +- [x] Multiple storages - [x] Local storage - [x] [Aliyundrive](https://www.aliyundrive.com/) - [x] OneDrive / Sharepoint ([global](https://www.office.com/), [cn](https://portal.partner.microsoftonline.cn),de,us) @@ -86,7 +86,7 @@ English | [中文](./README_cn.md)| [日本語](./README_ja.md) | [Contributing] - [x] Protected routes (password protection and authentication) - [x] WebDav (see https://alist.nn.ci/guide/webdav.html for details) - [x] [Docker Deploy](https://hub.docker.com/r/xhofe/alist) -- [x] Cloudflare workers proxy +- [x] Cloudflare Workers proxy - [x] File/Folder package download - [x] Web upload(Can allow visitors to upload), delete, mkdir, rename, move and copy - [x] Offline download @@ -103,7 +103,7 @@ English | [中文](./README_cn.md)| [日本語](./README_ja.md) | [Contributing] ## Discussion -Please go to our [discussion forum](https://github.com/Xhofe/alist/discussions) for general questions, **issues are for bug reports and feature request only.** +Please go to our [discussion forum](https://github.com/Xhofe/alist/discussions) for general questions, **issues are for bug reports and feature requests only.** ## Sponsor @@ -127,7 +127,7 @@ Thanks goes to these wonderful people: The `AList` is open-source software licensed under the AGPL-3.0 license. ## Disclaimer -- This program is a free and open source project. It is designed to share files on the network disk, which is convenient for downloading and learning golang. Please abide by relevant laws and regulations when using it, and do not abuse it; +- This program is a free and open source project. It is designed to share files on the network disk, which is convenient for downloading and learning Golang. Please abide by relevant laws and regulations when using it, and do not abuse it; - This program is implemented by calling the official sdk/interface, without destroying the official interface behavior; - This program only does 302 redirect/traffic forwarding, and does not intercept, store, or tamper with any user data; - Before using this program, you should understand and bear the corresponding risks, including but not limited to account ban, download speed limit, etc., which is none of this program's business; diff --git a/drivers/115/driver.go b/drivers/115/driver.go index d4c8f553..15f6b408 100644 --- a/drivers/115/driver.go +++ b/drivers/115/driver.go @@ -2,19 +2,22 @@ package _115 import ( "context" + "strings" + driver115 "github.com/SheltonZhu/115driver/pkg/driver" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" "github.com/pkg/errors" - "strings" + "golang.org/x/time/rate" ) type Pan115 struct { model.Storage Addition - client *driver115.Pan115Client + client *driver115.Pan115Client + limiter *rate.Limiter } func (d *Pan115) Config() driver.Config { @@ -26,14 +29,27 @@ func (d *Pan115) GetAddition() driver.Additional { } func (d *Pan115) Init(ctx context.Context) error { + if d.LimitRate > 0 { + d.limiter = rate.NewLimiter(rate.Limit(d.LimitRate), 1) + } return d.login() } +func (d *Pan115) WaitLimit(ctx context.Context) error { + if d.limiter != nil { + return d.limiter.Wait(ctx) + } + return nil +} + func (d *Pan115) Drop(ctx context.Context) error { return nil } func (d *Pan115) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } files, err := d.getFiles(dir.GetID()) if err != nil && !errors.Is(err, driver115.ErrNotExist) { return nil, err @@ -44,6 +60,9 @@ func (d *Pan115) List(ctx context.Context, dir model.Obj, args model.ListArgs) ( } func (d *Pan115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } downloadInfo, err := d.client. DownloadWithUA(file.(*FileObj).PickCode, driver115.UA115Browser) if err != nil { @@ -57,6 +76,9 @@ func (d *Pan115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) } func (d *Pan115) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } if _, err := d.client.Mkdir(parentDir.GetID(), dirName); err != nil { return err } @@ -64,22 +86,38 @@ func (d *Pan115) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin } func (d *Pan115) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } return d.client.Move(dstDir.GetID(), srcObj.GetID()) } func (d *Pan115) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } return d.client.Rename(srcObj.GetID(), newName) } func (d *Pan115) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } return d.client.Copy(dstDir.GetID(), srcObj.GetID()) } func (d *Pan115) Remove(ctx context.Context, obj model.Obj) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } return d.client.Delete(obj.GetID()) } func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } + var ( fastInfo *driver115.UploadInitResp dirID = dstDir.GetID() diff --git a/drivers/115/meta.go b/drivers/115/meta.go index cd39c2be..16ec22cd 100644 --- a/drivers/115/meta.go +++ b/drivers/115/meta.go @@ -6,9 +6,10 @@ import ( ) type Addition struct { - Cookie string `json:"cookie" type:"text" help:"one of QR code token and cookie required"` - QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"` - PageSize int64 `json:"page_size" type:"number" default:"56" help:"list api per page size of 115 driver"` + Cookie string `json:"cookie" type:"text" help:"one of QR code token and cookie required"` + QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"` + PageSize int64 `json:"page_size" type:"number" default:"56" help:"list api per page size of 115 driver"` + LimitRate float64 `json:"limit_rate" type:"number" default:"2" help:"limit all api request rate (1r/[limit_rate]s)"` driver.RootID } diff --git a/drivers/aliyundrive_open/meta.go b/drivers/aliyundrive_open/meta.go index af4d1257..bd69211c 100644 --- a/drivers/aliyundrive_open/meta.go +++ b/drivers/aliyundrive_open/meta.go @@ -11,7 +11,7 @@ type Addition struct { RefreshToken string `json:"refresh_token" required:"true"` OrderBy string `json:"order_by" type:"select" options:"name,size,updated_at,created_at"` OrderDirection string `json:"order_direction" type:"select" options:"ASC,DESC"` - OauthTokenURL string `json:"oauth_token_url" default:"https://api.xhofe.top/alist/ali_open/token"` + OauthTokenURL string `json:"oauth_token_url" default:"https://api.nn.ci/alist/ali_open/token"` ClientID string `json:"client_id" required:"false" help:"Keep it empty if you don't have one"` ClientSecret string `json:"client_secret" required:"false" help:"Keep it empty if you don't have one"` RemoveWay string `json:"remove_way" required:"true" type:"select" options:"trash,delete"` diff --git a/drivers/aliyundrive_open/upload.go b/drivers/aliyundrive_open/upload.go index e4a0cf7e..73c37dcf 100644 --- a/drivers/aliyundrive_open/upload.go +++ b/drivers/aliyundrive_open/upload.go @@ -5,7 +5,6 @@ import ( "context" "encoding/base64" "fmt" - "github.com/alist-org/alist/v3/pkg/http_range" "io" "math" "net/http" @@ -16,6 +15,7 @@ import ( "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" @@ -258,6 +258,7 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m return nil, err } offset += partSize + up(i * 100 / count) } } else { log.Debugf("[aliyundrive_open] rapid upload success, file id: %s", createResp.FileId) diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index 5a4716d0..20810a76 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -5,7 +5,6 @@ import ( "crypto/md5" "encoding/hex" "errors" - "fmt" "io" "math" "net/url" @@ -28,10 +27,9 @@ type BaiduNetdisk struct { Addition uploadThread int + vipType int // 会员类型，0普通用户(4G/4M)、1普通会员(10G/16M)、2超级会员(20G/32M) } -const DefaultSliceSize int64 = 4 * utils.MB - func (d *BaiduNetdisk) Config() driver.Config { return config } @@ -54,7 +52,11 @@ func (d *BaiduNetdisk) Init(ctx context.Context) error { "method": "uinfo", }, nil) log.Debugf("[baidu] get uinfo: %s", string(res)) - return err + if err != nil { + return err + } + d.vipType = utils.Json.Get(res, "vip_type").ToInt() + return nil } func (d *BaiduNetdisk) Drop(ctx context.Context) error { @@ -153,20 +155,13 @@ func (d *BaiduNetdisk) PutRapid(ctx context.Context, dstDir model.Obj, stream mo } streamSize := stream.GetSize() - rawPath := stdpath.Join(dstDir.GetPath(), stream.GetName()) - path := encodeURIComponent(rawPath) + path := stdpath.Join(dstDir.GetPath(), stream.GetName()) mtime := stream.ModTime().Unix() ctime := stream.CreateTime().Unix() blockList, _ := utils.Json.MarshalToString([]string{contentMd5}) - data := fmt.Sprintf("path=%s&size=%d&isdir=0&rtype=3&block_list=%s&local_mtime=%d&local_ctime=%d", - path, streamSize, blockList, mtime, ctime) - params := map[string]string{ - "method": "create", - } - log.Debugf("[baidu_netdisk] precreate data: %s", data) var newFile File - _, err := d.post("/xpan/file", params, data, &newFile) + _, err := d.create(path, streamSize, 0, "", blockList, &newFile, mtime, ctime) if err != nil { return nil, err } @@ -185,17 +180,18 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F } streamSize := stream.GetSize() - count := int(math.Max(math.Ceil(float64(streamSize)/float64(DefaultSliceSize)), 1)) - lastBlockSize := streamSize % DefaultSliceSize + sliceSize := d.getSliceSize() + count := int(math.Max(math.Ceil(float64(streamSize)/float64(sliceSize)), 1)) + lastBlockSize := streamSize % sliceSize if streamSize > 0 && lastBlockSize == 0 { - lastBlockSize = DefaultSliceSize + lastBlockSize = sliceSize } //cal md5 for first 256k data const SliceSize int64 = 256 * 1024 // cal md5 blockList := make([]string, 0, count) - byteSize := DefaultSliceSize + byteSize := sliceSize fileMd5H := md5.New() sliceMd5H := md5.New() sliceMd5H2 := md5.New() @@ -218,9 +214,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F contentMd5 := hex.EncodeToString(fileMd5H.Sum(nil)) sliceMd5 := hex.EncodeToString(sliceMd5H2.Sum(nil)) blockListStr, _ := utils.Json.MarshalToString(blockList) - - rawPath := stdpath.Join(dstDir.GetPath(), stream.GetName()) - path := encodeURIComponent(rawPath) + path := stdpath.Join(dstDir.GetPath(), stream.GetName()) mtime := stream.ModTime().Unix() ctime := stream.CreateTime().Unix() @@ -228,13 +222,23 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F // 尝试获取之前的进度 precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, d.AccessToken, contentMd5) if !ok { - data := fmt.Sprintf("path=%s&size=%d&isdir=0&autoinit=1&rtype=3&block_list=%s&content-md5=%s&slice-md5=%s&local_mtime=%d&local_ctime=%d", - path, streamSize, blockListStr, contentMd5, sliceMd5, mtime, ctime) params := map[string]string{ "method": "precreate", } - log.Debugf("[baidu_netdisk] precreate data: %s", data) - _, err = d.post("/xpan/file", params, data, &precreateResp) + form := map[string]string{ + "path": path, + "size": strconv.FormatInt(streamSize, 10), + "isdir": "0", + "autoinit": "1", + "rtype": "3", + "block_list": blockListStr, + "content-md5": contentMd5, + "slice-md5": sliceMd5, + } + joinTime(form, ctime, mtime) + + log.Debugf("[baidu_netdisk] precreate data: %s", form) + _, err = d.postForm("/xpan/file", params, form, &precreateResp) if err != nil { return nil, err } @@ -257,7 +261,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F break } - i, partseq, offset, byteSize := i, partseq, int64(partseq)*DefaultSliceSize, DefaultSliceSize + i, partseq, offset, byteSize := i, partseq, int64(partseq)*sliceSize, sliceSize if partseq+1 == count { byteSize = lastBlockSize } @@ -290,7 +294,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F // step.3 创建文件 var newFile File - _, err = d.create(rawPath, streamSize, 0, precreateResp.Uploadid, blockListStr, &newFile, mtime, ctime) + _, err = d.create(path, streamSize, 0, precreateResp.Uploadid, blockListStr, &newFile, mtime, ctime) if err != nil { return nil, err } diff --git a/drivers/baidu_netdisk/types.go b/drivers/baidu_netdisk/types.go index 81a49112..cbec0bcf 100644 --- a/drivers/baidu_netdisk/types.go +++ b/drivers/baidu_netdisk/types.go @@ -1,7 +1,6 @@ package baidu_netdisk import ( - "github.com/alist-org/alist/v3/pkg/utils" "path" "strconv" "time" @@ -71,7 +70,9 @@ func fileToObj(f File) *model.ObjThumb { Modified: time.Unix(f.LocalMtime, 0), Ctime: time.Unix(f.LocalCtime, 0), IsFolder: f.Isdir == 1, - HashInfo: utils.NewHashInfo(utils.MD5, f.Md5), + + // 直接获取的MD5是错误的 + // HashInfo: utils.NewHashInfo(utils.MD5, f.Md5), }, Thumbnail: model.Thumbnail{Thumbnail: f.Thumbs.Url3}, } diff --git a/drivers/baidu_netdisk/util.go b/drivers/baidu_netdisk/util.go index b867fbaa..6c51156c 100644 --- a/drivers/baidu_netdisk/util.go +++ b/drivers/baidu_netdisk/util.go @@ -4,9 +4,7 @@ import ( "errors" "fmt" "net/http" - "net/url" "strconv" - "strings" "time" "github.com/alist-org/alist/v3/drivers/base" @@ -96,10 +94,10 @@ func (d *BaiduNetdisk) get(pathname string, params map[string]string, resp inter }, resp) } -func (d *BaiduNetdisk) post(pathname string, params map[string]string, data interface{}, resp interface{}) ([]byte, error) { +func (d *BaiduNetdisk) postForm(pathname string, params map[string]string, form map[string]string, resp interface{}) ([]byte, error) { return d.request("https://pan.baidu.com/rest/2.0"+pathname, http.MethodPost, func(req *resty.Request) { req.SetQueryParams(params) - req.SetBody(data) + req.SetFormData(form) }, resp) } @@ -154,6 +152,9 @@ func (d *BaiduNetdisk) linkOfficial(file model.Obj, args model.LinkArgs) (*model //if res.StatusCode() == 302 { u = res.Header().Get("location") //} + + updateObjMd5(file, "pan.baidu.com", u) + return &model.Link{ URL: u, Header: http.Header{ @@ -176,6 +177,9 @@ func (d *BaiduNetdisk) linkCrack(file model.Obj, args model.LinkArgs) (*model.Li if err != nil { return nil, err } + + updateObjMd5(file, d.CustomCrackUA, resp.Info[0].Dlink) + return &model.Link{ URL: resp.Info[0].Dlink, Header: http.Header{ @@ -190,29 +194,73 @@ func (d *BaiduNetdisk) manage(opera string, filelist any) ([]byte, error) { "opera": opera, } marshal, _ := utils.Json.MarshalToString(filelist) - data := fmt.Sprintf("async=0&filelist=%s&ondup=fail", marshal) - return d.post("/xpan/file", params, data, nil) + return d.postForm("/xpan/file", params, map[string]string{ + "async": "0", + "filelist": marshal, + "ondup": "fail", + }, nil) } func (d *BaiduNetdisk) create(path string, size int64, isdir int, uploadid, block_list string, resp any, mtime, ctime int64) ([]byte, error) { params := map[string]string{ "method": "create", } - data := "" - if mtime == 0 || ctime == 0 { - data = fmt.Sprintf("path=%s&size=%d&isdir=%d&rtype=3", encodeURIComponent(path), size, isdir) - } else { - data = fmt.Sprintf("path=%s&size=%d&isdir=%d&rtype=3&local_mtime=%d&local_ctime=%d", encodeURIComponent(path), size, isdir, mtime, ctime) + form := map[string]string{ + "path": path, + "size": strconv.FormatInt(size, 10), + "isdir": strconv.Itoa(isdir), + "rtype": "3", + } + if mtime != 0 && ctime != 0 { + joinTime(form, ctime, mtime) } if uploadid != "" { - data += fmt.Sprintf("&uploadid=%s&block_list=%s", uploadid, block_list) + form["uploadid"] = uploadid } - return d.post("/xpan/file", params, data, resp) + if block_list != "" { + form["block_list"] = block_list + } + return d.postForm("/xpan/file", params, form, resp) } -func encodeURIComponent(str string) string { - r := url.QueryEscape(str) - r = strings.ReplaceAll(r, "+", "%20") - return r +func joinTime(form map[string]string, ctime, mtime int64) { + form["local_mtime"] = strconv.FormatInt(mtime, 10) + form["local_ctime"] = strconv.FormatInt(ctime, 10) } + +func updateObjMd5(obj model.Obj, userAgent, u string) { + object := model.GetRawObject(obj) + if object != nil { + req, _ := http.NewRequest(http.MethodHead, u, nil) + req.Header.Add("User-Agent", userAgent) + resp, _ := base.HttpClient.Do(req) + if resp != nil { + contentMd5 := resp.Header.Get("Content-Md5") + object.HashInfo = utils.NewHashInfo(utils.MD5, contentMd5) + } + } +} + +const ( + DefaultSliceSize int64 = 4 * utils.MB + VipSliceSize = 16 * utils.MB + SVipSliceSize = 32 * utils.MB +) + +func (d *BaiduNetdisk) getSliceSize() int64 { + switch d.vipType { + case 1: + return VipSliceSize + case 2: + return SVipSliceSize + default: + return DefaultSliceSize + } +} + +// func encodeURIComponent(str string) string { +// r := url.QueryEscape(str) +// r = strings.ReplaceAll(r, "+", "%20") +// return r +// } diff --git a/drivers/cloudreve/driver.go b/drivers/cloudreve/driver.go index 030de7c2..2a22380e 100644 --- a/drivers/cloudreve/driver.go +++ b/drivers/cloudreve/driver.go @@ -49,7 +49,11 @@ func (d *Cloudreve) List(ctx context.Context, dir model.Obj, args model.ListArgs } return utils.SliceConvert(r.Objects, func(src Object) (model.Obj, error) { - return objectToObj(src), nil + thumb, err := d.GetThumb(src) + if err != nil { + return nil, err + } + return objectToObj(src, thumb), nil }) } diff --git a/drivers/cloudreve/types.go b/drivers/cloudreve/types.go index 114afd3c..e2567382 100644 --- a/drivers/cloudreve/types.go +++ b/drivers/cloudreve/types.go @@ -44,13 +44,16 @@ type Object struct { SourceEnabled bool `json:"source_enabled"` } -func objectToObj(f Object) *model.Object { - return &model.Object{ - ID: f.Id, - Name: f.Name, - Size: int64(f.Size), - Modified: f.Date, - IsFolder: f.Type == "dir", +func objectToObj(f Object, t model.Thumbnail) *model.ObjThumb { + return &model.ObjThumb{ + Object: model.Object{ + ID: f.Id, + Name: f.Name, + Size: int64(f.Size), + Modified: f.Date, + IsFolder: f.Type == "dir", + }, + Thumbnail: t, } } diff --git a/drivers/cloudreve/util.go b/drivers/cloudreve/util.go index 1b8778be..ed879466 100644 --- a/drivers/cloudreve/util.go +++ b/drivers/cloudreve/util.go @@ -149,3 +149,23 @@ func convertSrc(obj model.Obj) map[string]interface{} { m["items"] = items return m } + +func (d *Cloudreve) GetThumb(file Object) (model.Thumbnail, error) { + ua := d.CustomUA + if ua == "" { + ua = base.UserAgent + } + req := base.NoRedirectClient.R() + req.SetHeaders(map[string]string{ + "Cookie": "cloudreve-session=" + d.Cookie, + "Accept": "image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8", + "User-Agent": ua, + }) + resp, err := req.Execute(http.MethodGet, d.Address+"/api/v3/file/thumb/"+file.Id) + if err != nil { + return model.Thumbnail{}, err + } + return model.Thumbnail{ + Thumbnail: resp.Header().Get("Location"), + }, nil +} diff --git a/drivers/google_drive/types.go b/drivers/google_drive/types.go index 796c1321..07545932 100644 --- a/drivers/google_drive/types.go +++ b/drivers/google_drive/types.go @@ -5,6 +5,7 @@ import ( "time" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" log "github.com/sirupsen/logrus" ) @@ -23,12 +24,17 @@ type File struct { Name string `json:"name"` MimeType string `json:"mimeType"` ModifiedTime time.Time `json:"modifiedTime"` + CreatedTime time.Time `json:"createdTime"` Size string `json:"size"` ThumbnailLink string `json:"thumbnailLink"` ShortcutDetails struct { TargetId string `json:"targetId"` TargetMimeType string `json:"targetMimeType"` } `json:"shortcutDetails"` + + MD5Checksum string `json:"md5Checksum"` + SHA1Checksum string `json:"sha1Checksum"` + SHA256Checksum string `json:"sha256Checksum"` } func fileToObj(f File) *model.ObjThumb { @@ -39,10 +45,18 @@ func fileToObj(f File) *model.ObjThumb { ID: f.Id, Name: f.Name, Size: size, + Ctime: f.CreatedTime, Modified: f.ModifiedTime, IsFolder: f.MimeType == "application/vnd.google-apps.folder", + HashInfo: utils.NewHashInfoByMap(map[*utils.HashType]string{ + utils.MD5: f.MD5Checksum, + utils.SHA1: f.SHA1Checksum, + utils.SHA256: f.SHA256Checksum, + }), + }, + Thumbnail: model.Thumbnail{ + Thumbnail: f.ThumbnailLink, }, - Thumbnail: model.Thumbnail{}, } if f.MimeType == "application/vnd.google-apps.shortcut" { obj.ID = f.ShortcutDetails.TargetId diff --git a/drivers/google_drive/util.go b/drivers/google_drive/util.go index 5637d00e..2c1f13eb 100644 --- a/drivers/google_drive/util.go +++ b/drivers/google_drive/util.go @@ -5,7 +5,6 @@ import ( "crypto/x509" "encoding/pem" "fmt" - "github.com/alist-org/alist/v3/pkg/http_range" "io/ioutil" "net/http" "os" @@ -13,6 +12,8 @@ import ( "strconv" "time" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" @@ -195,7 +196,7 @@ func (d *GoogleDrive) getFiles(id string) ([]File, error) { } query := map[string]string{ "orderBy": orderBy, - "fields": "files(id,name,mimeType,size,modifiedTime,thumbnailLink,shortcutDetails),nextPageToken", + "fields": "files(id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum),nextPageToken", "pageSize": "1000", "q": fmt.Sprintf("'%s' in parents and trashed = false", id), //"includeItemsFromAllDrives": "true", diff --git a/drivers/onedrive_app/util.go b/drivers/onedrive_app/util.go index 6a061f1f..28b34837 100644 --- a/drivers/onedrive_app/util.go +++ b/drivers/onedrive_app/util.go @@ -71,8 +71,8 @@ func (d *OnedriveAPP) _accessToken() error { "grant_type": "client_credentials", "client_id": d.ClientID, "client_secret": d.ClientSecret, - "resource": "https://graph.microsoft.com/", - "scope": "https://graph.microsoft.com/.default", + "resource": onedriveHostMap[d.Region].Api + "/", + "scope": onedriveHostMap[d.Region].Api + "/.default", }).Post(url) if err != nil { return err diff --git a/drivers/webdav/util.go b/drivers/webdav/util.go index 92557c4f..84eebb2e 100644 --- a/drivers/webdav/util.go +++ b/drivers/webdav/util.go @@ -2,6 +2,7 @@ package webdav import ( "net/http" + "net/http/cookiejar" "github.com/alist-org/alist/v3/drivers/webdav/odrvcookie" "github.com/alist-org/alist/v3/internal/model" @@ -26,6 +27,13 @@ func (d *WebDav) setClient() error { } else { return err } + } else { + cookieJar, err := cookiejar.New(nil) + if err == nil { + c.SetJar(cookieJar) + } else { + return err + } } d.client = c return nil diff --git a/go.mod b/go.mod index 7f066e1c..f185aef9 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/SheltonZhu/115driver v1.0.16 github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 - github.com/Xhofe/wopan-sdk-go v0.1.1 - github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible + github.com/Xhofe/wopan-sdk-go v0.1.2 + github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.44.327 github.com/blevesearch/bleve/v2 v2.3.10 @@ -24,7 +24,7 @@ require ( github.com/foxxorcat/weiyun-sdk-go v0.1.2 github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 - github.com/go-resty/resty/v2 v2.8.0 + github.com/go-resty/resty/v2 v2.9.1 github.com/go-webauthn/webauthn v0.8.6 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/uuid v1.3.1 @@ -47,10 +47,10 @@ require ( github.com/u2takey/ffmpeg-go v0.5.0 github.com/upyun/go-sdk/v3 v3.0.4 github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 - golang.org/x/crypto v0.13.0 + golang.org/x/crypto v0.14.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/image v0.11.0 - golang.org/x/net v0.15.0 + golang.org/x/net v0.16.0 golang.org/x/oauth2 v0.12.0 gorm.io/driver/mysql v1.4.7 gorm.io/driver/postgres v1.4.8 @@ -186,8 +186,8 @@ require ( go.etcd.io/bbolt v1.3.7 // indirect golang.org/x/arch v0.3.0 // indirect golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.12.0 // indirect - golang.org/x/term v0.12.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect google.golang.org/api v0.134.0 // indirect diff --git a/go.sum b/go.sum index 49cdc805..0fb9a131 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,8 @@ github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 h1:WnvifFgYyogPz2ZFvaV github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4/go.mod h1:8pWlL2rpusvx7Xa6yYaIWOJ8bR3gPdFBUT7OystyGOY= github.com/Xhofe/wopan-sdk-go v0.1.1 h1:dSrTxNYclqNuo9libjtC+R6C4RCen/inh/dUXd12vpM= github.com/Xhofe/wopan-sdk-go v0.1.1/go.mod h1:xWcUS7PoFLDD9gy2BK2VQfilEsZngLMz2Vkx3oF2zJY= +github.com/Xhofe/wopan-sdk-go v0.1.2 h1:6Gh4YTT7b7YHN0OoJ33j7Jm9ru/ckuvcDxPnRmH07jc= +github.com/Xhofe/wopan-sdk-go v0.1.2/go.mod h1:ktLYb4t7rnPFq1AshLaPXq5kZER+DkEagT6/i/in0uo= github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0= github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM= github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ= @@ -27,6 +29,8 @@ github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible h1:QoRMR0TCctLDqBCMyOu1e github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible h1:KpbJFXwhVeuxNtBJ74MCGbIoaBok2uZvkD7QXp2+Wis= github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= +github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible h1:Sg/2xHwDrioHpxTN6WMiwbXTpUEinBpHsN7mG21Rc2k= +github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/andreburgaud/crypt2go v1.1.0 h1:eitZxTPY1krUsxinsng3Qvt/Ud7q/aQmmYRh8p4hyPw= github.com/andreburgaud/crypt2go v1.1.0/go.mod h1:4qhZPzarj1dCIRmCkpdgCklwp+hBq9yEt0zPe9Ayuhc= github.com/andreburgaud/crypt2go v1.2.0 h1:oly/ENAodeqTYpUafgd4r3v+VKLQnmOKUyfpj+TxHbE= @@ -184,6 +188,8 @@ github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPr github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= github.com/go-resty/resty/v2 v2.8.0 h1:J29d0JFWwSWrDCysnOK/YjsPMLQTx0TvgJEHVGvf2L8= github.com/go-resty/resty/v2 v2.8.0/go.mod h1:UCui0cMHekLrSntoMyofdSTaPpinlRHFtPpizuyDW2w= +github.com/go-resty/resty/v2 v2.9.1 h1:PIgGx4VrHvag0juCJ4dDv3MiFRlDmP0vicBucwf+gLM= +github.com/go-resty/resty/v2 v2.9.1/go.mod h1:4/GYJVjh9nhkhGR6AUNW3XhpDYNUr+Uvy9gV/VGZIy4= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-webauthn/webauthn v0.8.6 h1:bKMtL1qzd2WTFkf1mFTVbreYrwn7dsYmEPjTq6QN90E= @@ -506,6 +512,8 @@ golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b h1:r+vk0EmXNmekl0S0BascoeeoHk/L7wmaW2QF90K+kYI= golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ= @@ -536,6 +544,8 @@ golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos= +golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= @@ -573,6 +583,8 @@ golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -584,6 +596,8 @@ golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -599,6 +613,7 @@ golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/internal/model/obj.go b/internal/model/obj.go index cb46201f..77c0700a 100644 --- a/internal/model/obj.go +++ b/internal/model/obj.go @@ -1,14 +1,15 @@ package model import ( - "github.com/alist-org/alist/v3/pkg/http_range" - "github.com/alist-org/alist/v3/pkg/utils" "io" "regexp" "sort" "strings" "time" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + mapset "github.com/deckarep/golang-set/v2" "github.com/maruel/natural" @@ -146,6 +147,20 @@ func GetUrl(obj Obj) (url string, ok bool) { return url, false } +func GetRawObject(obj Obj) *Object { + switch v := obj.(type) { + case *ObjThumbURL: + return &v.Object + case *ObjThumb: + return &v.Object + case *ObjectURL: + return &v.Object + case *Object: + return v + } + return nil +} + // Merge func NewObjMerge() *ObjMerge { return &ObjMerge{ diff --git a/internal/stream/util.go b/internal/stream/util.go index 2cb13daa..7d2b7ef7 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -3,13 +3,14 @@ package stream import ( "context" "fmt" + "io" + "net/http" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/net" "github.com/alist-org/alist/v3/pkg/http_range" log "github.com/sirupsen/logrus" - "io" - "net/http" ) func GetRangeReadCloserFromLink(size int64, link *model.Link) (model.RangeReadCloserIF, error) { @@ -40,6 +41,9 @@ func GetRangeReadCloserFromLink(size int64, link *model.Link) (model.RangeReadCl if len(link.URL) > 0 { response, err := RequestRangedHttp(ctx, link, r.Start, r.Length) if err != nil { + if response == nil { + return nil, fmt.Errorf("http request failure, err:%s", err) + } return nil, fmt.Errorf("http request failure,status: %d err:%s", response.StatusCode, err) } if r.Start == 0 && (r.Length == -1 || r.Length == size) || response.StatusCode == http.StatusPartialContent || diff --git a/pkg/gowebdav/client.go b/pkg/gowebdav/client.go index 6e12289c..2fca0b7f 100644 --- a/pkg/gowebdav/client.go +++ b/pkg/gowebdav/client.go @@ -83,6 +83,11 @@ func (c *Client) SetTransport(transport http.RoundTripper) { c.c.Transport = transport } +// SetJar exposes the ability to set a cookie jar to the client. +func (c *Client) SetJar(jar http.CookieJar) { + c.c.Jar = jar +} + // Connect connects to our dav server func (c *Client) Connect() error { rs, err := c.options("/") @@ -351,6 +356,11 @@ func (c *Client) Link(path string) (string, http.Header, error) { return "", nil, newPathErrorErr("Link", path, err) } + if c.c.Jar != nil { + for _, cookie := range c.c.Jar.Cookies(r.URL) { + r.AddCookie(cookie) + } + } for k, vals := range c.headers { for _, v := range vals { r.Header.Add(k, v) diff --git a/pkg/utils/hash.go b/pkg/utils/hash.go index bbb8769b..8f8aaa26 100644 --- a/pkg/utils/hash.go +++ b/pkg/utils/hash.go @@ -184,6 +184,10 @@ type HashInfo struct { h map[*HashType]string `json:"hashInfo"` } +func NewHashInfoByMap(h map[*HashType]string) HashInfo { + return HashInfo{h} +} + func NewHashInfo(ht *HashType, str string) HashInfo { m := make(map[*HashType]string) if ht != nil { diff --git a/pkg/utils/hash/gcid.go b/pkg/utils/hash/gcid.go index 8e1b98d6..f6eccef7 100644 --- a/pkg/utils/hash/gcid.go +++ b/pkg/utils/hash/gcid.go @@ -72,11 +72,13 @@ func (h *gcid) Write(p []byte) (n int, err error) { } func (h *gcid) Sum(b []byte) []byte { - if hashm, ok := h.hash.(encoding.BinaryMarshaler); ok { - if hashum, ok := h.hash.(encoding.BinaryUnmarshaler); ok { - tempData, _ := hashm.MarshalBinary() - h.hash.Write(h.hashState.Sum(nil)) - defer hashum.UnmarshalBinary(tempData) + if h.offset != 0 { + if hashm, ok := h.hash.(encoding.BinaryMarshaler); ok { + if hashum, ok := h.hash.(encoding.BinaryUnmarshaler); ok { + tempData, _ := hashm.MarshalBinary() + defer hashum.UnmarshalBinary(tempData) + h.hash.Write(h.hashState.Sum(nil)) + } } } return h.hash.Sum(b)