github
/
alist
mirror of https://github.com/Xhofe/alist
1
0
Fork 0
Code Issues Releases Wiki Activity

perf(lanzou): optimize the use of list cache (#2956) 

* fix:local sort not cache

* perf(lanzou): Optimize the use of list cache
pull/2964/head
foxxorcat 2023-01-08 21:31:35 +08:00 committed by GitHub
parent 99a186d01b
commit 2f19d4a834
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 117 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
drivers/lanzou/driver.go
View File

@ -11,6 +11,7 @@ import (
"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/go-resty/resty/v2"
)
@ -108,18 +109,19 @@ func (d *LanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs)
}
}
}
exp := GetExpirationTime(dfile.Url)
return &model.Link{
URL: dfile.Url,
Header: http.Header{
"User-Agent": []string{base.UserAgent},
},
Expiration: &exp,
}, nil
}
func (d *LanZou) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error {
func (d *LanZou) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) {
if d.IsCookie() {
_, err := d.doupload(func(req *resty.Request) {
data, err := d.doupload(func(req *resty.Request) {
req.SetContext(ctx)
req.SetFormData(map[string]string{
"task": "2",
@ -128,12 +130,18 @@ func (d *LanZou) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin
"folder_description": "",
})
}, nil)
return err
if err != nil {
return nil, err
}
return &FileOrFolder{
Name: dirName,
FolID: utils.Json.Get(data, "text").ToString(),
}, nil
}
return errs.NotImplement
return nil, errs.NotImplement
}
func (d *LanZou) Move(ctx context.Context, srcObj, dstDir model.Obj) error {
func (d *LanZou) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) {
if d.IsCookie() {
if !srcObj.IsDir() {
_, err := d.doupload(func(req *resty.Request) {
@ -144,13 +152,16 @@ func (d *LanZou) Move(ctx context.Context, srcObj, dstDir model.Obj) error {
"file_id": srcObj.GetID(),
})
}, nil)
return err
if err != nil {
return nil, err
}
return srcObj, nil
}
}
return errs.NotImplement
return nil, errs.NotImplement
}
func (d *LanZou) Rename(ctx context.Context, srcObj model.Obj, newName string) error {
func (d *LanZou) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) {
if d.IsCookie() {
if !srcObj.IsDir() {
_, err := d.doupload(func(req *resty.Request) {
@ -162,10 +173,14 @@ func (d *LanZou) Rename(ctx context.Context, srcObj model.Obj, newName string) e
"type": "2",
})
}, nil)
return err
if err != nil {
return nil, err
}
srcObj.(*FileOrFolder).NameAll = newName
return srcObj, nil
}
}
return errs.NotImplement
return nil, errs.NotImplement
}
func (d *LanZou) Remove(ctx context.Context, obj model.Obj) error {
@ -189,8 +204,9 @@ func (d *LanZou) Remove(ctx context.Context, obj model.Obj) error {
return errs.NotImplement
}
func (d *LanZou) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error {
func (d *LanZou) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) {
if d.IsCookie() {
var resp RespText[[]FileOrFolder]
_, err := d._post(d.BaseUrl+"/fileup.php", func(req *resty.Request) {
req.SetFormData(map[string]string{
"task": "1",
@ -198,8 +214,11 @@ func (d *LanZou) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
"name": stream.GetName(),
"folder_id": dstDir.GetID(),
}).SetFileReader("upload_file", stream.GetName(), stream).SetContext(ctx)
}, nil, true)
return err
}, &resp, true)
if err != nil {
return nil, err
}
return &resp.Text[0], nil
}
return errs.NotImplement
return nil, errs.NotImplement
}

15
drivers/lanzou/help.go
View File

@ -175,3 +175,18 @@ func formToMap(from string) map[string]string {
}
return param
}
var regExpirationTime = regexp.MustCompile(`e=(\d+)`)
func GetExpirationTime(url string) (etime time.Duration) {
exps := regExpirationTime.FindStringSubmatch(url)
if len(exps) < 2 {
return
}
timestamp, err := strconv.ParseInt(exps[1], 10, 64)
if err != nil {
return
}
etime = time.Duration(timestamp-time.Now().Unix()) * time.Second
return
}

11
drivers/lanzou/types.go
View File

@ -9,8 +9,12 @@ import (
var ErrFileShareCancel = errors.New("file sharing cancellation")
var ErrFileNotExist = errors.New("file does not exist")
type FilesOrFoldersResp struct {
Text []FileOrFolder `json:"text"`
type RespText[T any] struct {
Text T `json:"text"`
}
type RespInfo[T any] struct {
Info T `json:"info"`
}
type FileOrFolder struct {
@ -81,9 +85,6 @@ func (f *FileOrFolder) GetShareInfo() *FileShare {
}
/* 通过ID获取文件/文件夹分享信息 */
type FileShareResp struct {
Info FileShare `json:"info"`
}
type FileShare struct {
Pwd string `json:"pwd"`
Onof string `json:"onof"`

8
drivers/lanzou/util.go
View File

@ -114,7 +114,7 @@ func (d *LanZou) GetAllFiles(folderID string) ([]model.Obj, error) {
// 通过ID获取文件夹
func (d *LanZou) GetFolders(folderID string) ([]FileOrFolder, error) {
var resp FilesOrFoldersResp
var resp RespText[[]FileOrFolder]
_, err := d.doupload(func(req *resty.Request) {
req.SetFormData(map[string]string{
"task": "47",
@ -131,7 +131,7 @@ func (d *LanZou) GetFolders(folderID string) ([]FileOrFolder, error) {
func (d *LanZou) GetFiles(folderID string) ([]FileOrFolder, error) {
files := make([]FileOrFolder, 0)
for pg := 1; ; pg++ {
var resp FilesOrFoldersResp
var resp RespText[[]FileOrFolder]
_, err := d.doupload(func(req *resty.Request) {
req.SetFormData(map[string]string{
"task": "5",
@ -152,7 +152,7 @@ func (d *LanZou) GetFiles(folderID string) ([]FileOrFolder, error) {
// 通过ID获取文件夹分享地址
func (d *LanZou) getFolderShareUrlByID(fileID string) (*FileShare, error) {
var resp FileShareResp
var resp RespInfo[FileShare]
_, err := d.doupload(func(req *resty.Request) {
req.SetFormData(map[string]string{
"task": "18",
@ -167,7 +167,7 @@ func (d *LanZou) getFolderShareUrlByID(fileID string) (*FileShare, error) {
// 通过ID获取文件分享地址
func (d *LanZou) getFileShareUrlByID(fileID string) (*FileShare, error) {
var resp FileShareResp
var resp RespInfo[FileShare]
_, err := d.doupload(func(req *resty.Request) {
req.SetFormData(map[string]string{
"task": "22",

7
internal/fs/list.go
View File

@ -38,13 +38,6 @@ func list(ctx context.Context, path string, refresh ...bool) ([]model.Obj, error
om.InitHideReg(meta.Hide)
}
objs := om.Merge(virtualFiles, _objs...)
// sort objs
if storage != nil {
if storage.Config().LocalSort {
model.SortFiles(objs, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection)
}
model.ExtractFolder(objs, storage.GetStorage().ExtractFolder)
}
return objs, nil
}

29
internal/op/fs.go
View File

@ -10,6 +10,7 @@ import (
"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/pkg/generic_sync"
"github.com/alist-org/alist/v3/pkg/singleflight"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/pkg/errors"
@ -49,6 +50,8 @@ func delCacheObj(storage driver.Driver, path string, obj model.Obj) {
}
}
var addSortDebounceMap generic_sync.MapOf[string, func(func())]
func addCacheObj(storage driver.Driver, path string, newObj model.Obj) {
key := Key(storage, path)
objs, ok := listCache.Get(key)
@ -59,7 +62,24 @@ func addCacheObj(storage driver.Driver, path string, newObj model.Obj) {
return
}
}
objs = append(objs, newObj)
// Simple separation of files and folders
if len(objs) > 0 && objs[len(objs)-1].IsDir() == newObj.IsDir() {
objs = append(objs, newObj)
} else {
objs = append([]model.Obj{newObj}, objs...)
}
if storage.Config().LocalSort {
debounce, _ := addSortDebounceMap.LoadOrStore(key, utils.NewDebounce(time.Minute))
log.Debug("addCacheObj: wait start sort")
debounce(func() {
log.Debug("addCacheObj: start sort")
model.SortFiles(objs, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection)
addSortDebounceMap.Delete(key)
})
}
listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))
}
}
@ -113,6 +133,13 @@ func List(ctx context.Context, storage driver.Driver, path string, args model.Li
hook(args.ReqPath, files)
}
}(args.ReqPath, files)
// sort objs
if storage.Config().LocalSort {
model.SortFiles(files, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection)
}
model.ExtractFolder(files, storage.GetStorage().ExtractFolder)
if !storage.Config().NoCache {
if len(files) > 0 {
log.Debugf("set cache: %s => %+v", key, files)

31
pkg/utils/time.go
View File

@ -1,8 +1,37 @@
package utils
import "time"
import (
"sync"
"time"
)
func MustParseCNTime(str string) time.Time {
lastOpTime, _ := time.ParseInLocation("2006-01-02 15:04:05 -07", str+" +08", time.Local)
return lastOpTime
}
func NewDebounce(interval time.Duration) func(f func()) {
var timer *time.Timer
var lock sync.Mutex
return func(f func()) {
lock.Lock()
defer lock.Unlock()
if timer != nil {
timer.Stop()
}
timer = time.AfterFunc(interval, f)
}
}
func NewDebounce2(interval time.Duration, f func()) func() {
var timer *time.Timer
var lock sync.Mutex
return func() {
lock.Lock()
defer lock.Unlock()
if timer == nil {
timer = time.AfterFunc(interval, f)
}
(*time.Timer)(timer).Reset(interval)
}
}