From a008f54f4d5eda5738abfd54bf1abf1e18c08430 Mon Sep 17 00:00:00 2001 From: nkh0472 <67589323+nkh0472@users.noreply.github.com> Date: Thu, 5 Oct 2023 13:10:51 +0800 Subject: [PATCH 01/15] docs: minor language improvements (#5329) [skip ci] --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d5e88e43..3f8fc4ee 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ English | [中文](./README_cn.md)| [日本語](./README_ja.md) | [Contributing] ## Features -- [x] Multiple storage +- [x] Multiple storages - [x] Local storage - [x] [Aliyundrive](https://www.aliyundrive.com/) - [x] OneDrive / Sharepoint ([global](https://www.office.com/), [cn](https://portal.partner.microsoftonline.cn),de,us) @@ -86,7 +86,7 @@ English | [中文](./README_cn.md)| [日本語](./README_ja.md) | [Contributing] - [x] Protected routes (password protection and authentication) - [x] WebDav (see https://alist.nn.ci/guide/webdav.html for details) - [x] [Docker Deploy](https://hub.docker.com/r/xhofe/alist) -- [x] Cloudflare workers proxy +- [x] Cloudflare Workers proxy - [x] File/Folder package download - [x] Web upload(Can allow visitors to upload), delete, mkdir, rename, move and copy - [x] Offline download @@ -103,7 +103,7 @@ English | [中文](./README_cn.md)| [日本語](./README_ja.md) | [Contributing] ## Discussion -Please go to our [discussion forum](https://github.com/Xhofe/alist/discussions) for general questions, **issues are for bug reports and feature request only.** +Please go to our [discussion forum](https://github.com/Xhofe/alist/discussions) for general questions, **issues are for bug reports and feature requests only.** ## Sponsor @@ -127,7 +127,7 @@ Thanks goes to these wonderful people: The `AList` is open-source software licensed under the AGPL-3.0 license. ## Disclaimer -- This program is a free and open source project. It is designed to share files on the network disk, which is convenient for downloading and learning golang. Please abide by relevant laws and regulations when using it, and do not abuse it; +- This program is a free and open source project. It is designed to share files on the network disk, which is convenient for downloading and learning Golang. Please abide by relevant laws and regulations when using it, and do not abuse it; - This program is implemented by calling the official sdk/interface, without destroying the official interface behavior; - This program only does 302 redirect/traffic forwarding, and does not intercept, store, or tamper with any user data; - Before using this program, you should understand and bear the corresponding risks, including but not limited to account ban, download speed limit, etc., which is none of this program's business; From 1a283bb2720eff6d1b0c1dd6f1667a6449905a9b Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Fri, 6 Oct 2023 16:04:39 +0800 Subject: [PATCH 02/15] feat(google_drive): add `hash_info`, `ctime`, `thumbnail` (#5334) --- drivers/google_drive/types.go | 16 +++++++++++++++- drivers/google_drive/util.go | 5 +++-- pkg/utils/hash.go | 4 ++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/drivers/google_drive/types.go b/drivers/google_drive/types.go index 796c1321..07545932 100644 --- a/drivers/google_drive/types.go +++ b/drivers/google_drive/types.go @@ -5,6 +5,7 @@ import ( "time" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" log "github.com/sirupsen/logrus" ) @@ -23,12 +24,17 @@ type File struct { Name string `json:"name"` MimeType string `json:"mimeType"` ModifiedTime time.Time `json:"modifiedTime"` + CreatedTime time.Time `json:"createdTime"` Size string `json:"size"` ThumbnailLink string `json:"thumbnailLink"` ShortcutDetails struct { TargetId string `json:"targetId"` TargetMimeType string `json:"targetMimeType"` } `json:"shortcutDetails"` + + MD5Checksum string `json:"md5Checksum"` + SHA1Checksum string `json:"sha1Checksum"` + SHA256Checksum string `json:"sha256Checksum"` } func fileToObj(f File) *model.ObjThumb { @@ -39,10 +45,18 @@ func fileToObj(f File) *model.ObjThumb { ID: f.Id, Name: f.Name, Size: size, + Ctime: f.CreatedTime, Modified: f.ModifiedTime, IsFolder: f.MimeType == "application/vnd.google-apps.folder", + HashInfo: utils.NewHashInfoByMap(map[*utils.HashType]string{ + utils.MD5: f.MD5Checksum, + utils.SHA1: f.SHA1Checksum, + utils.SHA256: f.SHA256Checksum, + }), + }, + Thumbnail: model.Thumbnail{ + Thumbnail: f.ThumbnailLink, }, - Thumbnail: model.Thumbnail{}, } if f.MimeType == "application/vnd.google-apps.shortcut" { obj.ID = f.ShortcutDetails.TargetId diff --git a/drivers/google_drive/util.go b/drivers/google_drive/util.go index 5637d00e..2c1f13eb 100644 --- a/drivers/google_drive/util.go +++ b/drivers/google_drive/util.go @@ -5,7 +5,6 @@ import ( "crypto/x509" "encoding/pem" "fmt" - "github.com/alist-org/alist/v3/pkg/http_range" "io/ioutil" "net/http" "os" @@ -13,6 +12,8 @@ import ( "strconv" "time" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" @@ -195,7 +196,7 @@ func (d *GoogleDrive) getFiles(id string) ([]File, error) { } query := map[string]string{ "orderBy": orderBy, - "fields": "files(id,name,mimeType,size,modifiedTime,thumbnailLink,shortcutDetails),nextPageToken", + "fields": "files(id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum),nextPageToken", "pageSize": "1000", "q": fmt.Sprintf("'%s' in parents and trashed = false", id), //"includeItemsFromAllDrives": "true", diff --git a/pkg/utils/hash.go b/pkg/utils/hash.go index bbb8769b..8f8aaa26 100644 --- a/pkg/utils/hash.go +++ b/pkg/utils/hash.go @@ -184,6 +184,10 @@ type HashInfo struct { h map[*HashType]string `json:"hashInfo"` } +func NewHashInfoByMap(h map[*HashType]string) HashInfo { + return HashInfo{h} +} + func NewHashInfo(ht *HashType, str string) HashInfo { m := make(map[*HashType]string) if ht != nil { From 35d672217dde69e65b41b1fcd9786c1cfebcdc45 Mon Sep 17 00:00:00 2001 From: jeffmingup <1960588251@qq.com> Date: Sun, 8 Oct 2023 19:29:45 +0800 Subject: [PATCH 03/15] fix(onedrive_app): incorrect api on `_accessToken` (#5346) --- drivers/onedrive_app/util.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/onedrive_app/util.go b/drivers/onedrive_app/util.go index a40424bd..525a451d 100644 --- a/drivers/onedrive_app/util.go +++ b/drivers/onedrive_app/util.go @@ -71,8 +71,8 @@ func (d *OnedriveAPP) _accessToken() error { "grant_type": "client_credentials", "client_id": d.ClientID, "client_secret": d.ClientSecret, - "resource": "https://graph.microsoft.com/", - "scope": "https://graph.microsoft.com/.default", + "resource": onedriveHostMap[d.Region].Api + "/", + "scope": onedriveHostMap[d.Region].Api + "/.default", }).Post(url) if err != nil { return err From 8ef89ad0a496d5acc398794c0afa4f77c67ad371 Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Tue, 10 Oct 2023 18:08:27 +0800 Subject: [PATCH 04/15] fix(baidu_netdisk): hash and `error 2` (#5356) * fix(baidu):hash and error:2 * fix:invalid memory address --- drivers/baidu_netdisk/driver.go | 36 +++++++++--------- drivers/baidu_netdisk/types.go | 5 ++- drivers/baidu_netdisk/util.go | 65 ++++++++++++++++++++++++--------- internal/model/obj.go | 19 +++++++++- internal/stream/util.go | 8 +++- 5 files changed, 92 insertions(+), 41 deletions(-) diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index 141e2fc0..654c8b01 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -5,7 +5,6 @@ import ( "crypto/md5" "encoding/hex" "errors" - "fmt" "io" "math" "net/url" @@ -153,20 +152,13 @@ func (d *BaiduNetdisk) PutRapid(ctx context.Context, dstDir model.Obj, stream mo } streamSize := stream.GetSize() - rawPath := stdpath.Join(dstDir.GetPath(), stream.GetName()) - path := encodeURIComponent(rawPath) + path := stdpath.Join(dstDir.GetPath(), stream.GetName()) mtime := stream.ModTime().Unix() ctime := stream.CreateTime().Unix() blockList, _ := utils.Json.MarshalToString([]string{contentMd5}) - data := fmt.Sprintf("path=%s&size=%d&isdir=0&rtype=3&block_list=%s&local_mtime=%d&local_ctime=%d", - path, streamSize, blockList, mtime, ctime) - params := map[string]string{ - "method": "create", - } - log.Debugf("[baidu_netdisk] precreate data: %s", data) var newFile File - _, err := d.post("/xpan/file", params, data, &newFile) + _, err := d.create(path, streamSize, 0, "", blockList, &newFile, mtime, ctime) if err != nil { return nil, err } @@ -218,9 +210,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F contentMd5 := hex.EncodeToString(fileMd5H.Sum(nil)) sliceMd5 := hex.EncodeToString(sliceMd5H2.Sum(nil)) blockListStr, _ := utils.Json.MarshalToString(blockList) - - rawPath := stdpath.Join(dstDir.GetPath(), stream.GetName()) - path := encodeURIComponent(rawPath) + path := stdpath.Join(dstDir.GetPath(), stream.GetName()) mtime := stream.ModTime().Unix() ctime := stream.CreateTime().Unix() @@ -228,13 +218,23 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F // 尝试获取之前的进度 precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, d.AccessToken, contentMd5) if !ok { - data := fmt.Sprintf("path=%s&size=%d&isdir=0&autoinit=1&rtype=3&block_list=%s&content-md5=%s&slice-md5=%s&local_mtime=%d&local_ctime=%d", - path, streamSize, blockListStr, contentMd5, sliceMd5, mtime, ctime) params := map[string]string{ "method": "precreate", } - log.Debugf("[baidu_netdisk] precreate data: %s", data) - _, err = d.post("/xpan/file", params, data, &precreateResp) + form := map[string]string{ + "path": path, + "size": strconv.FormatInt(streamSize, 10), + "isdir": "0", + "autoinit": "1", + "rtype": "3", + "block_list": blockListStr, + "content-md5": contentMd5, + "slice-md5": sliceMd5, + } + joinTime(form, ctime, mtime) + + log.Debugf("[baidu_netdisk] precreate data: %s", form) + _, err = d.postForm("/xpan/file", params, form, &precreateResp) if err != nil { return nil, err } @@ -290,7 +290,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F // step.3 创建文件 var newFile File - _, err = d.create(rawPath, streamSize, 0, precreateResp.Uploadid, blockListStr, &newFile, mtime, ctime) + _, err = d.create(path, streamSize, 0, precreateResp.Uploadid, blockListStr, &newFile, mtime, ctime) if err != nil { return nil, err } diff --git a/drivers/baidu_netdisk/types.go b/drivers/baidu_netdisk/types.go index 81a49112..cbec0bcf 100644 --- a/drivers/baidu_netdisk/types.go +++ b/drivers/baidu_netdisk/types.go @@ -1,7 +1,6 @@ package baidu_netdisk import ( - "github.com/alist-org/alist/v3/pkg/utils" "path" "strconv" "time" @@ -71,7 +70,9 @@ func fileToObj(f File) *model.ObjThumb { Modified: time.Unix(f.LocalMtime, 0), Ctime: time.Unix(f.LocalCtime, 0), IsFolder: f.Isdir == 1, - HashInfo: utils.NewHashInfo(utils.MD5, f.Md5), + + // 直接获取的MD5是错误的 + // HashInfo: utils.NewHashInfo(utils.MD5, f.Md5), }, Thumbnail: model.Thumbnail{Thumbnail: f.Thumbs.Url3}, } diff --git a/drivers/baidu_netdisk/util.go b/drivers/baidu_netdisk/util.go index b867fbaa..d972eb83 100644 --- a/drivers/baidu_netdisk/util.go +++ b/drivers/baidu_netdisk/util.go @@ -4,9 +4,7 @@ import ( "errors" "fmt" "net/http" - "net/url" "strconv" - "strings" "time" "github.com/alist-org/alist/v3/drivers/base" @@ -96,10 +94,10 @@ func (d *BaiduNetdisk) get(pathname string, params map[string]string, resp inter }, resp) } -func (d *BaiduNetdisk) post(pathname string, params map[string]string, data interface{}, resp interface{}) ([]byte, error) { +func (d *BaiduNetdisk) postForm(pathname string, params map[string]string, form map[string]string, resp interface{}) ([]byte, error) { return d.request("https://pan.baidu.com/rest/2.0"+pathname, http.MethodPost, func(req *resty.Request) { req.SetQueryParams(params) - req.SetBody(data) + req.SetFormData(form) }, resp) } @@ -154,6 +152,9 @@ func (d *BaiduNetdisk) linkOfficial(file model.Obj, args model.LinkArgs) (*model //if res.StatusCode() == 302 { u = res.Header().Get("location") //} + + updateObjMd5(file, "pan.baidu.com", u) + return &model.Link{ URL: u, Header: http.Header{ @@ -176,6 +177,9 @@ func (d *BaiduNetdisk) linkCrack(file model.Obj, args model.LinkArgs) (*model.Li if err != nil { return nil, err } + + updateObjMd5(file, d.CustomCrackUA, resp.Info[0].Dlink) + return &model.Link{ URL: resp.Info[0].Dlink, Header: http.Header{ @@ -190,29 +194,56 @@ func (d *BaiduNetdisk) manage(opera string, filelist any) ([]byte, error) { "opera": opera, } marshal, _ := utils.Json.MarshalToString(filelist) - data := fmt.Sprintf("async=0&filelist=%s&ondup=fail", marshal) - return d.post("/xpan/file", params, data, nil) + return d.postForm("/xpan/file", params, map[string]string{ + "async": "0", + "filelist": marshal, + "ondup": "fail", + }, nil) } func (d *BaiduNetdisk) create(path string, size int64, isdir int, uploadid, block_list string, resp any, mtime, ctime int64) ([]byte, error) { params := map[string]string{ "method": "create", } - data := "" - if mtime == 0 || ctime == 0 { - data = fmt.Sprintf("path=%s&size=%d&isdir=%d&rtype=3", encodeURIComponent(path), size, isdir) - } else { - data = fmt.Sprintf("path=%s&size=%d&isdir=%d&rtype=3&local_mtime=%d&local_ctime=%d", encodeURIComponent(path), size, isdir, mtime, ctime) + form := map[string]string{ + "path": path, + "size": strconv.FormatInt(size, 10), + "isdir": strconv.Itoa(isdir), + "rtype": "3", + } + if mtime != 0 && ctime != 0 { + joinTime(form, ctime, mtime) } if uploadid != "" { - data += fmt.Sprintf("&uploadid=%s&block_list=%s", uploadid, block_list) + form["uploadid"] = uploadid } - return d.post("/xpan/file", params, data, resp) + if block_list != "" { + form["block_list"] = block_list + } + return d.postForm("/xpan/file", params, form, resp) } -func encodeURIComponent(str string) string { - r := url.QueryEscape(str) - r = strings.ReplaceAll(r, "+", "%20") - return r +func joinTime(form map[string]string, ctime, mtime int64) { + form["local_mtime"] = strconv.FormatInt(mtime, 10) + form["local_ctime"] = strconv.FormatInt(ctime, 10) } + +func updateObjMd5(obj model.Obj, userAgent, u string) { + object := model.GetRawObject(obj) + if object != nil { + req, _ := http.NewRequest(http.MethodHead, u, nil) + req.Header.Add("User-Agent", userAgent) + resp, _ := base.HttpClient.Do(req) + if resp != nil { + contentMd5 := resp.Header.Get("Content-Md5") + object.HashInfo = utils.NewHashInfo(utils.MD5, contentMd5) + } + } +} + +// func encodeURIComponent(str string) string { +// r := url.QueryEscape(str) +// r = strings.ReplaceAll(r, "+", "%20") +// return r +// } diff --git a/internal/model/obj.go b/internal/model/obj.go index cb46201f..77c0700a 100644 --- a/internal/model/obj.go +++ b/internal/model/obj.go @@ -1,14 +1,15 @@ package model import ( - "github.com/alist-org/alist/v3/pkg/http_range" - "github.com/alist-org/alist/v3/pkg/utils" "io" "regexp" "sort" "strings" "time" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + mapset "github.com/deckarep/golang-set/v2" "github.com/maruel/natural" @@ -146,6 +147,20 @@ func GetUrl(obj Obj) (url string, ok bool) { return url, false } +func GetRawObject(obj Obj) *Object { + switch v := obj.(type) { + case *ObjThumbURL: + return &v.Object + case *ObjThumb: + return &v.Object + case *ObjectURL: + return &v.Object + case *Object: + return v + } + return nil +} + // Merge func NewObjMerge() *ObjMerge { return &ObjMerge{ diff --git a/internal/stream/util.go b/internal/stream/util.go index 2cb13daa..7d2b7ef7 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -3,13 +3,14 @@ package stream import ( "context" "fmt" + "io" + "net/http" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/net" "github.com/alist-org/alist/v3/pkg/http_range" log "github.com/sirupsen/logrus" - "io" - "net/http" ) func GetRangeReadCloserFromLink(size int64, link *model.Link) (model.RangeReadCloserIF, error) { @@ -40,6 +41,9 @@ func GetRangeReadCloserFromLink(size int64, link *model.Link) (model.RangeReadCl if len(link.URL) > 0 { response, err := RequestRangedHttp(ctx, link, r.Start, r.Length) if err != nil { + if response == nil { + return nil, fmt.Errorf("http request failure, err:%s", err) + } return nil, fmt.Errorf("http request failure,status: %d err:%s", response.StatusCode, err) } if r.Start == 0 && (r.Length == -1 || r.Length == size) || response.StatusCode == http.StatusPartialContent || From 7600dc28df137c439e538b4257731c33a63db9b5 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Tue, 10 Oct 2023 18:13:58 +0800 Subject: [PATCH 05/15] fix(aliyundrive_open): change default api to raw server (close #5358) --- drivers/aliyundrive_open/meta.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/aliyundrive_open/meta.go b/drivers/aliyundrive_open/meta.go index af4d1257..bd69211c 100644 --- a/drivers/aliyundrive_open/meta.go +++ b/drivers/aliyundrive_open/meta.go @@ -11,7 +11,7 @@ type Addition struct { RefreshToken string `json:"refresh_token" required:"true"` OrderBy string `json:"order_by" type:"select" options:"name,size,updated_at,created_at"` OrderDirection string `json:"order_direction" type:"select" options:"ASC,DESC"` - OauthTokenURL string `json:"oauth_token_url" default:"https://api.xhofe.top/alist/ali_open/token"` + OauthTokenURL string `json:"oauth_token_url" default:"https://api.nn.ci/alist/ali_open/token"` ClientID string `json:"client_id" required:"false" help:"Keep it empty if you don't have one"` ClientSecret string `json:"client_secret" required:"false" help:"Keep it empty if you don't have one"` RemoveWay string `json:"remove_way" required:"true" type:"select" options:"trash,delete"` From 66e2324cac75cb3ef05af45dbdd10b124d534aff Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Tue, 10 Oct 2023 18:23:11 +0800 Subject: [PATCH 06/15] chore(deps): upgrade dependencies --- go.mod | 10 +++++----- go.sum | 11 +++++++++++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 7f066e1c..c50c1dac 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/foxxorcat/weiyun-sdk-go v0.1.2 github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 - github.com/go-resty/resty/v2 v2.8.0 + github.com/go-resty/resty/v2 v2.9.1 github.com/go-webauthn/webauthn v0.8.6 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/uuid v1.3.1 @@ -47,10 +47,10 @@ require ( github.com/u2takey/ffmpeg-go v0.5.0 github.com/upyun/go-sdk/v3 v3.0.4 github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 - golang.org/x/crypto v0.13.0 + golang.org/x/crypto v0.14.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/image v0.11.0 - golang.org/x/net v0.15.0 + golang.org/x/net v0.16.0 golang.org/x/oauth2 v0.12.0 gorm.io/driver/mysql v1.4.7 gorm.io/driver/postgres v1.4.8 @@ -186,8 +186,8 @@ require ( go.etcd.io/bbolt v1.3.7 // indirect golang.org/x/arch v0.3.0 // indirect golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.12.0 // indirect - golang.org/x/term v0.12.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect google.golang.org/api v0.134.0 // indirect diff --git a/go.sum b/go.sum index 49cdc805..679d91eb 100644 --- a/go.sum +++ b/go.sum @@ -184,6 +184,8 @@ github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPr github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= github.com/go-resty/resty/v2 v2.8.0 h1:J29d0JFWwSWrDCysnOK/YjsPMLQTx0TvgJEHVGvf2L8= github.com/go-resty/resty/v2 v2.8.0/go.mod h1:UCui0cMHekLrSntoMyofdSTaPpinlRHFtPpizuyDW2w= +github.com/go-resty/resty/v2 v2.9.1 h1:PIgGx4VrHvag0juCJ4dDv3MiFRlDmP0vicBucwf+gLM= +github.com/go-resty/resty/v2 v2.9.1/go.mod h1:4/GYJVjh9nhkhGR6AUNW3XhpDYNUr+Uvy9gV/VGZIy4= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-webauthn/webauthn v0.8.6 h1:bKMtL1qzd2WTFkf1mFTVbreYrwn7dsYmEPjTq6QN90E= @@ -506,6 +508,8 @@ golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b h1:r+vk0EmXNmekl0S0BascoeeoHk/L7wmaW2QF90K+kYI= golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ= @@ -536,6 +540,8 @@ golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos= +golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= @@ -573,6 +579,8 @@ golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -584,6 +592,8 @@ golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -599,6 +609,7 @@ golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 1ebc96a4e5220c979fd581bb3b5640e9436f6665 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Tue, 10 Oct 2023 18:32:00 +0800 Subject: [PATCH 07/15] fix(wopan): fatal error concurrent map writes (close #5352) --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index c50c1dac..89f13e94 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/SheltonZhu/115driver v1.0.16 github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 - github.com/Xhofe/wopan-sdk-go v0.1.1 + github.com/Xhofe/wopan-sdk-go v0.1.2 github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.44.327 diff --git a/go.sum b/go.sum index 679d91eb..ea5641fe 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,8 @@ github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 h1:WnvifFgYyogPz2ZFvaV github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4/go.mod h1:8pWlL2rpusvx7Xa6yYaIWOJ8bR3gPdFBUT7OystyGOY= github.com/Xhofe/wopan-sdk-go v0.1.1 h1:dSrTxNYclqNuo9libjtC+R6C4RCen/inh/dUXd12vpM= github.com/Xhofe/wopan-sdk-go v0.1.1/go.mod h1:xWcUS7PoFLDD9gy2BK2VQfilEsZngLMz2Vkx3oF2zJY= +github.com/Xhofe/wopan-sdk-go v0.1.2 h1:6Gh4YTT7b7YHN0OoJ33j7Jm9ru/ckuvcDxPnRmH07jc= +github.com/Xhofe/wopan-sdk-go v0.1.2/go.mod h1:ktLYb4t7rnPFq1AshLaPXq5kZER+DkEagT6/i/in0uo= github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0= github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM= github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ= From 69a98eaef612b58596e5c26c341b6d7cedecdf19 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 11 Oct 2023 22:01:55 +0800 Subject: [PATCH 08/15] fix(deps): update module github.com/aliyun/aliyun-oss-go-sdk to v2.2.9+incompatible (#5141) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 89f13e94..f185aef9 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 github.com/Xhofe/wopan-sdk-go v0.1.2 - github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible + github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.44.327 github.com/blevesearch/bleve/v2 v2.3.10 diff --git a/go.sum b/go.sum index ea5641fe..0fb9a131 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,8 @@ github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible h1:QoRMR0TCctLDqBCMyOu1e github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible h1:KpbJFXwhVeuxNtBJ74MCGbIoaBok2uZvkD7QXp2+Wis= github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= +github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible h1:Sg/2xHwDrioHpxTN6WMiwbXTpUEinBpHsN7mG21Rc2k= +github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/andreburgaud/crypt2go v1.1.0 h1:eitZxTPY1krUsxinsng3Qvt/Ud7q/aQmmYRh8p4hyPw= github.com/andreburgaud/crypt2go v1.1.0/go.mod h1:4qhZPzarj1dCIRmCkpdgCklwp+hBq9yEt0zPe9Ayuhc= github.com/andreburgaud/crypt2go v1.2.0 h1:oly/ENAodeqTYpUafgd4r3v+VKLQnmOKUyfpj+TxHbE= From b9e192b29cffddf14a0dfb2d3885def57a56ce16 Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Thu, 12 Oct 2023 20:57:12 +0800 Subject: [PATCH 09/15] fix(115): limit request rate (#5367 close #5275) * fix(115):limit request rate * chore(115): fix unit of `limit_rate` --------- Co-authored-by: Andy Hsu --- drivers/115/driver.go | 42 ++++++++++++++++++++++++++++++++++++++++-- drivers/115/meta.go | 7 ++++--- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/drivers/115/driver.go b/drivers/115/driver.go index d4c8f553..15f6b408 100644 --- a/drivers/115/driver.go +++ b/drivers/115/driver.go @@ -2,19 +2,22 @@ package _115 import ( "context" + "strings" + driver115 "github.com/SheltonZhu/115driver/pkg/driver" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" "github.com/pkg/errors" - "strings" + "golang.org/x/time/rate" ) type Pan115 struct { model.Storage Addition - client *driver115.Pan115Client + client *driver115.Pan115Client + limiter *rate.Limiter } func (d *Pan115) Config() driver.Config { @@ -26,14 +29,27 @@ func (d *Pan115) GetAddition() driver.Additional { } func (d *Pan115) Init(ctx context.Context) error { + if d.LimitRate > 0 { + d.limiter = rate.NewLimiter(rate.Limit(d.LimitRate), 1) + } return d.login() } +func (d *Pan115) WaitLimit(ctx context.Context) error { + if d.limiter != nil { + return d.limiter.Wait(ctx) + } + return nil +} + func (d *Pan115) Drop(ctx context.Context) error { return nil } func (d *Pan115) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } files, err := d.getFiles(dir.GetID()) if err != nil && !errors.Is(err, driver115.ErrNotExist) { return nil, err @@ -44,6 +60,9 @@ func (d *Pan115) List(ctx context.Context, dir model.Obj, args model.ListArgs) ( } func (d *Pan115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } downloadInfo, err := d.client. DownloadWithUA(file.(*FileObj).PickCode, driver115.UA115Browser) if err != nil { @@ -57,6 +76,9 @@ func (d *Pan115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) } func (d *Pan115) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } if _, err := d.client.Mkdir(parentDir.GetID(), dirName); err != nil { return err } @@ -64,22 +86,38 @@ func (d *Pan115) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin } func (d *Pan115) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } return d.client.Move(dstDir.GetID(), srcObj.GetID()) } func (d *Pan115) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } return d.client.Rename(srcObj.GetID(), newName) } func (d *Pan115) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } return d.client.Copy(dstDir.GetID(), srcObj.GetID()) } func (d *Pan115) Remove(ctx context.Context, obj model.Obj) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } return d.client.Delete(obj.GetID()) } func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } + var ( fastInfo *driver115.UploadInitResp dirID = dstDir.GetID() diff --git a/drivers/115/meta.go b/drivers/115/meta.go index cd39c2be..16ec22cd 100644 --- a/drivers/115/meta.go +++ b/drivers/115/meta.go @@ -6,9 +6,10 @@ import ( ) type Addition struct { - Cookie string `json:"cookie" type:"text" help:"one of QR code token and cookie required"` - QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"` - PageSize int64 `json:"page_size" type:"number" default:"56" help:"list api per page size of 115 driver"` + Cookie string `json:"cookie" type:"text" help:"one of QR code token and cookie required"` + QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"` + PageSize int64 `json:"page_size" type:"number" default:"56" help:"list api per page size of 115 driver"` + LimitRate float64 `json:"limit_rate" type:"number" default:"2" help:"limit all api request rate (1r/[limit_rate]s)"` driver.RootID } From 7f7335435c2f32a3eef76fac4c4f783d9d8624fd Mon Sep 17 00:00:00 2001 From: itsHenry <2671230065@qq.com> Date: Sat, 14 Oct 2023 13:12:46 +0800 Subject: [PATCH 10/15] feat(cloudreve): support thumbnail (#5373 close #5348) * feat(cloudreve): support thumbnail * chore: remove unnecessary code --- drivers/cloudreve/driver.go | 6 +++++- drivers/cloudreve/types.go | 17 ++++++++++------- drivers/cloudreve/util.go | 20 ++++++++++++++++++++ 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/drivers/cloudreve/driver.go b/drivers/cloudreve/driver.go index 030de7c2..2a22380e 100644 --- a/drivers/cloudreve/driver.go +++ b/drivers/cloudreve/driver.go @@ -49,7 +49,11 @@ func (d *Cloudreve) List(ctx context.Context, dir model.Obj, args model.ListArgs } return utils.SliceConvert(r.Objects, func(src Object) (model.Obj, error) { - return objectToObj(src), nil + thumb, err := d.GetThumb(src) + if err != nil { + return nil, err + } + return objectToObj(src, thumb), nil }) } diff --git a/drivers/cloudreve/types.go b/drivers/cloudreve/types.go index 114afd3c..e2567382 100644 --- a/drivers/cloudreve/types.go +++ b/drivers/cloudreve/types.go @@ -44,13 +44,16 @@ type Object struct { SourceEnabled bool `json:"source_enabled"` } -func objectToObj(f Object) *model.Object { - return &model.Object{ - ID: f.Id, - Name: f.Name, - Size: int64(f.Size), - Modified: f.Date, - IsFolder: f.Type == "dir", +func objectToObj(f Object, t model.Thumbnail) *model.ObjThumb { + return &model.ObjThumb{ + Object: model.Object{ + ID: f.Id, + Name: f.Name, + Size: int64(f.Size), + Modified: f.Date, + IsFolder: f.Type == "dir", + }, + Thumbnail: t, } } diff --git a/drivers/cloudreve/util.go b/drivers/cloudreve/util.go index 1b8778be..ed879466 100644 --- a/drivers/cloudreve/util.go +++ b/drivers/cloudreve/util.go @@ -149,3 +149,23 @@ func convertSrc(obj model.Obj) map[string]interface{} { m["items"] = items return m } + +func (d *Cloudreve) GetThumb(file Object) (model.Thumbnail, error) { + ua := d.CustomUA + if ua == "" { + ua = base.UserAgent + } + req := base.NoRedirectClient.R() + req.SetHeaders(map[string]string{ + "Cookie": "cloudreve-session=" + d.Cookie, + "Accept": "image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8", + "User-Agent": ua, + }) + resp, err := req.Execute(http.MethodGet, d.Address+"/api/v3/file/thumb/"+file.Id) + if err != nil { + return model.Thumbnail{}, err + } + return model.Thumbnail{ + Thumbnail: resp.Header().Get("Location"), + }, nil +} From 94d028743abf8e0d736f80c0ec4fb294a1cc064c Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sat, 14 Oct 2023 13:17:51 +0800 Subject: [PATCH 11/15] ci: remove `pr-welcome` label when close issue [skip ci] --- .github/workflows/{issue_rm_working.yml => issue_on_close.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{issue_rm_working.yml => issue_on_close.yml} (90%) diff --git a/.github/workflows/issue_rm_working.yml b/.github/workflows/issue_on_close.yml similarity index 90% rename from .github/workflows/issue_rm_working.yml rename to .github/workflows/issue_on_close.yml index 56181366..c90c2c0c 100644 --- a/.github/workflows/issue_rm_working.yml +++ b/.github/workflows/issue_on_close.yml @@ -14,4 +14,4 @@ jobs: actions: 'remove-labels' token: ${{ secrets.GITHUB_TOKEN }} issue-number: ${{ github.event.issue.number }} - labels: 'working' \ No newline at end of file + labels: 'working,pr-welcome' From cdfbe6dcf2b361e4c93c2703c2f8c9bddeac0ee6 Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Wed, 18 Oct 2023 16:27:07 +0800 Subject: [PATCH 12/15] fix: hash gcid empty file (#5394) --- pkg/utils/hash/gcid.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pkg/utils/hash/gcid.go b/pkg/utils/hash/gcid.go index 8e1b98d6..f6eccef7 100644 --- a/pkg/utils/hash/gcid.go +++ b/pkg/utils/hash/gcid.go @@ -72,11 +72,13 @@ func (h *gcid) Write(p []byte) (n int, err error) { } func (h *gcid) Sum(b []byte) []byte { - if hashm, ok := h.hash.(encoding.BinaryMarshaler); ok { - if hashum, ok := h.hash.(encoding.BinaryUnmarshaler); ok { - tempData, _ := hashm.MarshalBinary() - h.hash.Write(h.hashState.Sum(nil)) - defer hashum.UnmarshalBinary(tempData) + if h.offset != 0 { + if hashm, ok := h.hash.(encoding.BinaryMarshaler); ok { + if hashum, ok := h.hash.(encoding.BinaryUnmarshaler); ok { + tempData, _ := hashm.MarshalBinary() + defer hashum.UnmarshalBinary(tempData) + h.hash.Write(h.hashState.Sum(nil)) + } } } return h.hash.Sum(b) From 8ef8023c20bfeee97ec82155b52eae0d80b1410e Mon Sep 17 00:00:00 2001 From: NewbieOrange Date: Thu, 19 Oct 2023 19:17:09 +0800 Subject: [PATCH 13/15] fix(aliyundrive_open): upload progress for normal upload (#5398) --- drivers/aliyundrive_open/upload.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/aliyundrive_open/upload.go b/drivers/aliyundrive_open/upload.go index e4a0cf7e..73c37dcf 100644 --- a/drivers/aliyundrive_open/upload.go +++ b/drivers/aliyundrive_open/upload.go @@ -5,7 +5,6 @@ import ( "context" "encoding/base64" "fmt" - "github.com/alist-org/alist/v3/pkg/http_range" "io" "math" "net/http" @@ -16,6 +15,7 @@ import ( "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" @@ -258,6 +258,7 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m return nil, err } offset += partSize + up(i * 100 / count) } } else { log.Debugf("[aliyundrive_open] rapid upload success, file id: %s", createResp.FileId) From aaffaee2b54fc067d240ea0c20ea3c2f39615d6e Mon Sep 17 00:00:00 2001 From: gmugu <94156510@qq.com> Date: Thu, 19 Oct 2023 19:17:53 +0800 Subject: [PATCH 14/15] perf(webdav): support request with cookies (#5391) --- drivers/webdav/util.go | 8 ++++++++ pkg/gowebdav/client.go | 10 ++++++++++ 2 files changed, 18 insertions(+) diff --git a/drivers/webdav/util.go b/drivers/webdav/util.go index 92557c4f..84eebb2e 100644 --- a/drivers/webdav/util.go +++ b/drivers/webdav/util.go @@ -2,6 +2,7 @@ package webdav import ( "net/http" + "net/http/cookiejar" "github.com/alist-org/alist/v3/drivers/webdav/odrvcookie" "github.com/alist-org/alist/v3/internal/model" @@ -26,6 +27,13 @@ func (d *WebDav) setClient() error { } else { return err } + } else { + cookieJar, err := cookiejar.New(nil) + if err == nil { + c.SetJar(cookieJar) + } else { + return err + } } d.client = c return nil diff --git a/pkg/gowebdav/client.go b/pkg/gowebdav/client.go index 6e12289c..2fca0b7f 100644 --- a/pkg/gowebdav/client.go +++ b/pkg/gowebdav/client.go @@ -83,6 +83,11 @@ func (c *Client) SetTransport(transport http.RoundTripper) { c.c.Transport = transport } +// SetJar exposes the ability to set a cookie jar to the client. +func (c *Client) SetJar(jar http.CookieJar) { + c.c.Jar = jar +} + // Connect connects to our dav server func (c *Client) Connect() error { rs, err := c.options("/") @@ -351,6 +356,11 @@ func (c *Client) Link(path string) (string, http.Header, error) { return "", nil, newPathErrorErr("Link", path, err) } + if c.c.Jar != nil { + for _, cookie := range c.c.Jar.Cookies(r.URL) { + r.AddCookie(cookie) + } + } for k, vals := range c.headers { for _, v := range vals { r.Header.Add(k, v) From 4fc0a77565702f9bf498485d42336502f2ee9776 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Fri, 20 Oct 2023 21:06:25 +0800 Subject: [PATCH 15/15] fix(baidu_netdisk): upload file > 4GB (close #5392) --- drivers/baidu_netdisk/driver.go | 20 ++++++++++++-------- drivers/baidu_netdisk/util.go | 17 +++++++++++++++++ 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index 654c8b01..d5f71814 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -27,10 +27,9 @@ type BaiduNetdisk struct { Addition uploadThread int + vipType int // 会员类型，0普通用户(4G/4M)、1普通会员(10G/16M)、2超级会员(20G/32M) } -const DefaultSliceSize int64 = 4 * utils.MB - func (d *BaiduNetdisk) Config() driver.Config { return config } @@ -53,7 +52,11 @@ func (d *BaiduNetdisk) Init(ctx context.Context) error { "method": "uinfo", }, nil) log.Debugf("[baidu] get uinfo: %s", string(res)) - return err + if err != nil { + return err + } + d.vipType = utils.Json.Get(res, "vip_type").ToInt() + return nil } func (d *BaiduNetdisk) Drop(ctx context.Context) error { @@ -177,17 +180,18 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F } streamSize := stream.GetSize() - count := int(math.Max(math.Ceil(float64(streamSize)/float64(DefaultSliceSize)), 1)) - lastBlockSize := streamSize % DefaultSliceSize + sliceSize := d.getSliceSize() + count := int(math.Max(math.Ceil(float64(streamSize)/float64(sliceSize)), 1)) + lastBlockSize := streamSize % sliceSize if streamSize > 0 && lastBlockSize == 0 { - lastBlockSize = DefaultSliceSize + lastBlockSize = sliceSize } //cal md5 for first 256k data const SliceSize int64 = 256 * 1024 // cal md5 blockList := make([]string, 0, count) - byteSize := DefaultSliceSize + byteSize := sliceSize fileMd5H := md5.New() sliceMd5H := md5.New() sliceMd5H2 := md5.New() @@ -257,7 +261,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F break } - i, partseq, offset, byteSize := i, partseq, int64(partseq)*DefaultSliceSize, DefaultSliceSize + i, partseq, offset, byteSize := i, partseq, int64(partseq)*sliceSize, sliceSize if partseq+1 == count { byteSize = lastBlockSize } diff --git a/drivers/baidu_netdisk/util.go b/drivers/baidu_netdisk/util.go index d972eb83..6c51156c 100644 --- a/drivers/baidu_netdisk/util.go +++ b/drivers/baidu_netdisk/util.go @@ -242,6 +242,23 @@ func updateObjMd5(obj model.Obj, userAgent, u string) { } } +const ( + DefaultSliceSize int64 = 4 * utils.MB + VipSliceSize = 16 * utils.MB + SVipSliceSize = 32 * utils.MB +) + +func (d *BaiduNetdisk) getSliceSize() int64 { + switch d.vipType { + case 1: + return VipSliceSize + case 2: + return SVipSliceSize + default: + return DefaultSliceSize + } +} + // func encodeURIComponent(str string) string { // r := url.QueryEscape(str) // r = strings.ReplaceAll(r, "+", "%20")