From 2f19d4a8346b52b36752b70d5f52fe20254bf5d8 Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Sun, 8 Jan 2023 21:31:35 +0800 Subject: [PATCH] perf(lanzou): optimize the use of list cache (#2956) * fix:local sort not cache * perf(lanzou): Optimize the use of list cache --- drivers/lanzou/driver.go | 49 ++++++++++++++++++++++++++++------------ drivers/lanzou/help.go | 15 ++++++++++++ drivers/lanzou/types.go | 11 +++++---- drivers/lanzou/util.go | 8 +++---- internal/fs/list.go | 7 ------ internal/op/fs.go | 29 +++++++++++++++++++++++- pkg/utils/time.go | 31 ++++++++++++++++++++++++- 7 files changed, 117 insertions(+), 33 deletions(-) diff --git a/drivers/lanzou/driver.go b/drivers/lanzou/driver.go index 4e96ad1d..8551bef2 100644 --- a/drivers/lanzou/driver.go +++ b/drivers/lanzou/driver.go @@ -11,6 +11,7 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" ) @@ -108,18 +109,19 @@ func (d *LanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) } } } - + exp := GetExpirationTime(dfile.Url) return &model.Link{ URL: dfile.Url, Header: http.Header{ "User-Agent": []string{base.UserAgent}, }, + Expiration: &exp, }, nil } -func (d *LanZou) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { +func (d *LanZou) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { if d.IsCookie() { - _, err := d.doupload(func(req *resty.Request) { + data, err := d.doupload(func(req *resty.Request) { req.SetContext(ctx) req.SetFormData(map[string]string{ "task": "2", @@ -128,12 +130,18 @@ func (d *LanZou) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin "folder_description": "", }) }, nil) - return err + if err != nil { + return nil, err + } + return &FileOrFolder{ + Name: dirName, + FolID: utils.Json.Get(data, "text").ToString(), + }, nil } - return errs.NotImplement + return nil, errs.NotImplement } -func (d *LanZou) Move(ctx context.Context, srcObj, dstDir model.Obj) error { +func (d *LanZou) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { if d.IsCookie() { if !srcObj.IsDir() { _, err := d.doupload(func(req *resty.Request) { @@ -144,13 +152,16 @@ func (d *LanZou) Move(ctx context.Context, srcObj, dstDir model.Obj) error { "file_id": srcObj.GetID(), }) }, nil) - return err + if err != nil { + return nil, err + } + return srcObj, nil } } - return errs.NotImplement + return nil, errs.NotImplement } -func (d *LanZou) Rename(ctx context.Context, srcObj model.Obj, newName string) error { +func (d *LanZou) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { if d.IsCookie() { if !srcObj.IsDir() { _, err := d.doupload(func(req *resty.Request) { @@ -162,10 +173,14 @@ func (d *LanZou) Rename(ctx context.Context, srcObj model.Obj, newName string) e "type": "2", }) }, nil) - return err + if err != nil { + return nil, err + } + srcObj.(*FileOrFolder).NameAll = newName + return srcObj, nil } } - return errs.NotImplement + return nil, errs.NotImplement } func (d *LanZou) Remove(ctx context.Context, obj model.Obj) error { @@ -189,8 +204,9 @@ func (d *LanZou) Remove(ctx context.Context, obj model.Obj) error { return errs.NotImplement } -func (d *LanZou) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { +func (d *LanZou) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { if d.IsCookie() { + var resp RespText[[]FileOrFolder] _, err := d._post(d.BaseUrl+"/fileup.php", func(req *resty.Request) { req.SetFormData(map[string]string{ "task": "1", @@ -198,8 +214,11 @@ func (d *LanZou) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr "name": stream.GetName(), "folder_id": dstDir.GetID(), }).SetFileReader("upload_file", stream.GetName(), stream).SetContext(ctx) - }, nil, true) - return err + }, &resp, true) + if err != nil { + return nil, err + } + return &resp.Text[0], nil } - return errs.NotImplement + return nil, errs.NotImplement } diff --git a/drivers/lanzou/help.go b/drivers/lanzou/help.go index 4cb646c4..ae9b0ac2 100644 --- a/drivers/lanzou/help.go +++ b/drivers/lanzou/help.go @@ -175,3 +175,18 @@ func formToMap(from string) map[string]string { } return param } + +var regExpirationTime = regexp.MustCompile(`e=(\d+)`) + +func GetExpirationTime(url string) (etime time.Duration) { + exps := regExpirationTime.FindStringSubmatch(url) + if len(exps) < 2 { + return + } + timestamp, err := strconv.ParseInt(exps[1], 10, 64) + if err != nil { + return + } + etime = time.Duration(timestamp-time.Now().Unix()) * time.Second + return +} diff --git a/drivers/lanzou/types.go b/drivers/lanzou/types.go index 9044b296..8be8dd34 100644 --- a/drivers/lanzou/types.go +++ b/drivers/lanzou/types.go @@ -9,8 +9,12 @@ import ( var ErrFileShareCancel = errors.New("file sharing cancellation") var ErrFileNotExist = errors.New("file does not exist") -type FilesOrFoldersResp struct { - Text []FileOrFolder `json:"text"` +type RespText[T any] struct { + Text T `json:"text"` +} + +type RespInfo[T any] struct { + Info T `json:"info"` } type FileOrFolder struct { @@ -81,9 +85,6 @@ func (f *FileOrFolder) GetShareInfo() *FileShare { } /* 通过ID获取文件/文件夹分享信息 */ -type FileShareResp struct { - Info FileShare `json:"info"` -} type FileShare struct { Pwd string `json:"pwd"` Onof string `json:"onof"` diff --git a/drivers/lanzou/util.go b/drivers/lanzou/util.go index 7bdc0799..0803937c 100644 --- a/drivers/lanzou/util.go +++ b/drivers/lanzou/util.go @@ -114,7 +114,7 @@ func (d *LanZou) GetAllFiles(folderID string) ([]model.Obj, error) { // 通过ID获取文件夹 func (d *LanZou) GetFolders(folderID string) ([]FileOrFolder, error) { - var resp FilesOrFoldersResp + var resp RespText[[]FileOrFolder] _, err := d.doupload(func(req *resty.Request) { req.SetFormData(map[string]string{ "task": "47", @@ -131,7 +131,7 @@ func (d *LanZou) GetFolders(folderID string) ([]FileOrFolder, error) { func (d *LanZou) GetFiles(folderID string) ([]FileOrFolder, error) { files := make([]FileOrFolder, 0) for pg := 1; ; pg++ { - var resp FilesOrFoldersResp + var resp RespText[[]FileOrFolder] _, err := d.doupload(func(req *resty.Request) { req.SetFormData(map[string]string{ "task": "5", @@ -152,7 +152,7 @@ func (d *LanZou) GetFiles(folderID string) ([]FileOrFolder, error) { // 通过ID获取文件夹分享地址 func (d *LanZou) getFolderShareUrlByID(fileID string) (*FileShare, error) { - var resp FileShareResp + var resp RespInfo[FileShare] _, err := d.doupload(func(req *resty.Request) { req.SetFormData(map[string]string{ "task": "18", @@ -167,7 +167,7 @@ func (d *LanZou) getFolderShareUrlByID(fileID string) (*FileShare, error) { // 通过ID获取文件分享地址 func (d *LanZou) getFileShareUrlByID(fileID string) (*FileShare, error) { - var resp FileShareResp + var resp RespInfo[FileShare] _, err := d.doupload(func(req *resty.Request) { req.SetFormData(map[string]string{ "task": "22", diff --git a/internal/fs/list.go b/internal/fs/list.go index 0cdf0591..51775efc 100644 --- a/internal/fs/list.go +++ b/internal/fs/list.go @@ -38,13 +38,6 @@ func list(ctx context.Context, path string, refresh ...bool) ([]model.Obj, error om.InitHideReg(meta.Hide) } objs := om.Merge(virtualFiles, _objs...) - // sort objs - if storage != nil { - if storage.Config().LocalSort { - model.SortFiles(objs, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection) - } - model.ExtractFolder(objs, storage.GetStorage().ExtractFolder) - } return objs, nil } diff --git a/internal/op/fs.go b/internal/op/fs.go index e175ec2c..f24214d7 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -10,6 +10,7 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/generic_sync" "github.com/alist-org/alist/v3/pkg/singleflight" "github.com/alist-org/alist/v3/pkg/utils" "github.com/pkg/errors" @@ -49,6 +50,8 @@ func delCacheObj(storage driver.Driver, path string, obj model.Obj) { } } +var addSortDebounceMap generic_sync.MapOf[string, func(func())] + func addCacheObj(storage driver.Driver, path string, newObj model.Obj) { key := Key(storage, path) objs, ok := listCache.Get(key) @@ -59,7 +62,24 @@ func addCacheObj(storage driver.Driver, path string, newObj model.Obj) { return } } - objs = append(objs, newObj) + + // Simple separation of files and folders + if len(objs) > 0 && objs[len(objs)-1].IsDir() == newObj.IsDir() { + objs = append(objs, newObj) + } else { + objs = append([]model.Obj{newObj}, objs...) + } + + if storage.Config().LocalSort { + debounce, _ := addSortDebounceMap.LoadOrStore(key, utils.NewDebounce(time.Minute)) + log.Debug("addCacheObj: wait start sort") + debounce(func() { + log.Debug("addCacheObj: start sort") + model.SortFiles(objs, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection) + addSortDebounceMap.Delete(key) + }) + } + listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) } } @@ -113,6 +133,13 @@ func List(ctx context.Context, storage driver.Driver, path string, args model.Li hook(args.ReqPath, files) } }(args.ReqPath, files) + + // sort objs + if storage.Config().LocalSort { + model.SortFiles(files, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection) + } + model.ExtractFolder(files, storage.GetStorage().ExtractFolder) + if !storage.Config().NoCache { if len(files) > 0 { log.Debugf("set cache: %s => %+v", key, files) diff --git a/pkg/utils/time.go b/pkg/utils/time.go index 2f81e200..41632e40 100644 --- a/pkg/utils/time.go +++ b/pkg/utils/time.go @@ -1,8 +1,37 @@ package utils -import "time" +import ( + "sync" + "time" +) func MustParseCNTime(str string) time.Time { lastOpTime, _ := time.ParseInLocation("2006-01-02 15:04:05 -07", str+" +08", time.Local) return lastOpTime } + +func NewDebounce(interval time.Duration) func(f func()) { + var timer *time.Timer + var lock sync.Mutex + return func(f func()) { + lock.Lock() + defer lock.Unlock() + if timer != nil { + timer.Stop() + } + timer = time.AfterFunc(interval, f) + } +} + +func NewDebounce2(interval time.Duration, f func()) func() { + var timer *time.Timer + var lock sync.Mutex + return func() { + lock.Lock() + defer lock.Unlock() + if timer == nil { + timer = time.AfterFunc(interval, f) + } + (*time.Timer)(timer).Reset(interval) + } +}