From fb64f0064078fa3719ad27870839313319c66b67 Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Sat, 17 Dec 2022 19:49:05 +0800 Subject: [PATCH] refactor: obj name mapping and internal path processing (#2733) * refactor:Prepare to remove the get interface * feat:add obj Unwarp interface * refactor:obj name mapping and program internal path processing * chore: fix typo * feat: unwrap get * fix: no use op.Get to get parent id * fix: set the path uniformly Co-authored-by: Noah Hsu --- drivers/123/driver.go | 5 - drivers/123/types.go | 3 +- drivers/139/driver.go | 5 - drivers/189/driver.go | 5 - drivers/189pc/types.go | 6 +- drivers/alist_v2/driver.go | 5 - drivers/alist_v3/driver.go | 5 - drivers/aliyundrive/driver.go | 5 - drivers/aliyundrive_share/driver.go | 5 - drivers/baidu_netdisk/driver.go | 5 - drivers/baidu_photo/types.go | 4 +- drivers/ftp/driver.go | 5 - drivers/google_drive/driver.go | 10 +- drivers/google_photo/driver.go | 5 - drivers/local/driver.go | 52 +++-------- drivers/mediatrack/driver.go | 22 ++--- drivers/mediatrack/types.go | 12 ++- drivers/mega/driver.go | 30 +++--- drivers/onedrive/driver.go | 18 ++-- drivers/onedrive/types.go | 28 ++++-- drivers/quark/driver.go | 5 - drivers/s3/driver.go | 5 - drivers/sftp/driver.go | 5 - drivers/smb/driver.go | 5 - drivers/teambition/driver.go | 5 - drivers/template/driver.go | 5 - drivers/thunder/types.go | 4 +- drivers/uss/driver.go | 5 - drivers/webdav/driver.go | 5 - drivers/yandex_disk/driver.go | 5 - internal/db/meta.go | 2 +- internal/db/searchnode.go | 3 +- internal/driver/driver.go | 2 +- internal/fs/fs.go | 4 +- internal/fs/get.go | 2 +- internal/fs/put.go | 4 +- internal/fs/util.go | 2 +- internal/model/obj.go | 14 +++ internal/model/object.go | 18 +++- internal/model/stream.go | 5 + internal/op/const.go | 5 +- internal/op/fs.go | 140 +++++++++++++++------------- internal/op/path.go | 28 ++---- internal/op/storage.go | 96 ++++++++----------- pkg/utils/file.go | 5 +- pkg/utils/path.go | 41 ++++---- pkg/utils/path_test.go | 15 +++ server/handles/meta.go | 4 +- server/middlewares/down.go | 2 +- server/static/config.go | 2 +- 50 files changed, 297 insertions(+), 381 deletions(-) diff --git a/drivers/123/driver.go b/drivers/123/driver.go index 53799cb8..38546c34 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -58,11 +58,6 @@ func (d *Pan123) List(ctx context.Context, dir model.Obj, args model.ListArgs) ( }) } -//func (d *Pan123) Get(ctx context.Context, path string) (model.Obj, error) { -// // this is optional -// return nil, errs.NotImplement -//} - func (d *Pan123) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { if f, ok := file.(File); ok { //var resp DownResp diff --git a/drivers/123/types.go b/drivers/123/types.go index 1986c7f9..06456dce 100644 --- a/drivers/123/types.go +++ b/drivers/123/types.go @@ -5,7 +5,6 @@ import ( "time" "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/pkg/utils" ) //type BaseResp struct { @@ -40,7 +39,7 @@ func (f File) GetSize() int64 { } func (f File) GetName() string { - return utils.MappingName(f.FileName) + return f.FileName } func (f File) ModTime() time.Time { diff --git a/drivers/139/driver.go b/drivers/139/driver.go index 82a85ce0..0200e355 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -53,11 +53,6 @@ func (d *Yun139) List(ctx context.Context, dir model.Obj, args model.ListArgs) ( } } -//func (d *Yun139) Get(ctx context.Context, path string) (model.Obj, error) { -// // this is optional -// return nil, errs.NotImplement -//} - func (d *Yun139) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { u, err := d.getLink(file.GetID()) if err != nil { diff --git a/drivers/189/driver.go b/drivers/189/driver.go index 68b392de..c5a88543 100644 --- a/drivers/189/driver.go +++ b/drivers/189/driver.go @@ -46,11 +46,6 @@ func (d *Cloud189) List(ctx context.Context, dir model.Obj, args model.ListArgs) return d.getFiles(dir.GetID()) } -//func (d *Cloud189) Get(ctx context.Context, path string) (model.Obj, error) { -// // this is optional -// return nil, errs.NotImplement -//} - func (d *Cloud189) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var resp DownResp u := "https://cloud.189.cn/api/portal/getFileInfo.action" diff --git a/drivers/189pc/types.go b/drivers/189pc/types.go index a3c1a4ed..0a6d4f9f 100644 --- a/drivers/189pc/types.go +++ b/drivers/189pc/types.go @@ -6,8 +6,6 @@ import ( "sort" "strings" "time" - - "github.com/alist-org/alist/v3/pkg/utils" ) // 居然有四种返回方式 @@ -136,7 +134,7 @@ type Cloud189File struct { } func (c *Cloud189File) GetSize() int64 { return c.Size } -func (c *Cloud189File) GetName() string { return utils.MappingName(c.Name) } +func (c *Cloud189File) GetName() string { return c.Name } func (c *Cloud189File) ModTime() time.Time { if c.parseTime == nil { c.parseTime = MustParseTime(c.LastOpTime) @@ -168,7 +166,7 @@ type Cloud189Folder struct { } func (c *Cloud189Folder) GetSize() int64 { return 0 } -func (c *Cloud189Folder) GetName() string { return utils.MappingName(c.Name) } +func (c *Cloud189Folder) GetName() string { return c.Name } func (c *Cloud189Folder) ModTime() time.Time { if c.parseTime == nil { c.parseTime = MustParseTime(c.LastOpTime) diff --git a/drivers/alist_v2/driver.go b/drivers/alist_v2/driver.go index 1e9117ca..a48588cd 100644 --- a/drivers/alist_v2/driver.go +++ b/drivers/alist_v2/driver.go @@ -67,11 +67,6 @@ func (d *AListV2) List(ctx context.Context, dir model.Obj, args model.ListArgs) return files, nil } -//func (d *AList) Get(ctx context.Context, path string) (model.Obj, error) { -// // this is optional -// return nil, errs.NotImplement -//} - func (d *AListV2) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { url := d.Address + "/api/public/path" var resp common.Resp[PathResp] diff --git a/drivers/alist_v3/driver.go b/drivers/alist_v3/driver.go index 8077f958..7e6fb088 100644 --- a/drivers/alist_v3/driver.go +++ b/drivers/alist_v3/driver.go @@ -71,11 +71,6 @@ func (d *AListV3) List(ctx context.Context, dir model.Obj, args model.ListArgs) return files, nil } -//func (d *AList) Get(ctx context.Context, path string) (model.Obj, error) { -// // this is optional -// return nil, errs.NotImplement -//} - func (d *AListV3) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { url := d.Address + "/api/fs/get" var resp common.Resp[FsGetResp] diff --git a/drivers/aliyundrive/driver.go b/drivers/aliyundrive/driver.go index 34a58beb..98bec318 100644 --- a/drivers/aliyundrive/driver.go +++ b/drivers/aliyundrive/driver.go @@ -81,11 +81,6 @@ func (d *AliDrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) }) } -//func (d *AliDrive) Get(ctx context.Context, path string) (model.Obj, error) { -// // TODO this is optional -// return nil, errs.NotImplement -//} - func (d *AliDrive) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { data := base.Json{ "drive_id": d.DriveId, diff --git a/drivers/aliyundrive_share/driver.go b/drivers/aliyundrive_share/driver.go index d0902196..b3a083c6 100644 --- a/drivers/aliyundrive_share/driver.go +++ b/drivers/aliyundrive_share/driver.go @@ -68,11 +68,6 @@ func (d *AliyundriveShare) List(ctx context.Context, dir model.Obj, args model.L }) } -//func (d *AliyundriveShare) Get(ctx context.Context, path string) (model.Obj, error) { -// // this is optional -// return nil, errs.NotImplement -//} - func (d *AliyundriveShare) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { data := base.Json{ "drive_id": d.DriveId, diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index be4e8d39..aedb05bd 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -52,11 +52,6 @@ func (d *BaiduNetdisk) List(ctx context.Context, dir model.Obj, args model.ListA }) } -//func (d *BaiduNetdisk) Get(ctx context.Context, path string) (model.Obj, error) { -// // this is optional -// return nil, errs.NotImplement -//} - func (d *BaiduNetdisk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { if d.DownloadAPI == "crack" { return d.linkCrack(file, args) diff --git a/drivers/baidu_photo/types.go b/drivers/baidu_photo/types.go index c20318e6..4701fc51 100644 --- a/drivers/baidu_photo/types.go +++ b/drivers/baidu_photo/types.go @@ -3,8 +3,6 @@ package baiduphoto import ( "fmt" "time" - - "github.com/alist-org/alist/v3/pkg/utils" ) type TokenErrResp struct { @@ -102,7 +100,7 @@ type ( ) func (a *Album) GetSize() int64 { return 0 } -func (a *Album) GetName() string { return utils.MappingName(a.Title) } +func (a *Album) GetName() string { return a.Title } func (a *Album) ModTime() time.Time { if a.parseTime == nil { a.parseTime = toTime(a.Mtime) diff --git a/drivers/ftp/driver.go b/drivers/ftp/driver.go index 1e799260..f3c9b43d 100644 --- a/drivers/ftp/driver.go +++ b/drivers/ftp/driver.go @@ -60,11 +60,6 @@ func (d *FTP) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]m return res, nil } -//func (d *FTP) Get(ctx context.Context, path string) (model.Obj, error) { -// // this is optional -// return nil, errs.NotImplement -//} - func (d *FTP) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { if err := d.login(); err != nil { return nil, err diff --git a/drivers/google_drive/driver.go b/drivers/google_drive/driver.go index c8678db0..dc2d9a43 100644 --- a/drivers/google_drive/driver.go +++ b/drivers/google_drive/driver.go @@ -4,14 +4,12 @@ import ( "context" "fmt" "net/http" - stdpath "path" "strconv" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" ) @@ -51,11 +49,6 @@ func (d *GoogleDrive) List(ctx context.Context, dir model.Obj, args model.ListAr }) } -//func (d *GoogleDrive) Get(ctx context.Context, path string) (model.Obj, error) { -// // this is optional -// return nil, errs.NotImplement -//} - func (d *GoogleDrive) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { url := fmt.Sprintf("https://www.googleapis.com/drive/v3/files/%s?includeItemsFromAllDrives=true&supportsAllDrives=true", file.GetID()) _, err := d.request(url, http.MethodGet, nil, nil) @@ -117,8 +110,7 @@ func (d *GoogleDrive) Remove(ctx context.Context, obj model.Obj) error { } func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - obj, _ := op.Get(ctx, d, stdpath.Join(dstDir.GetPath(), stream.GetName())) - + obj := stream.GetOld() var ( e Error url string diff --git a/drivers/google_photo/driver.go b/drivers/google_photo/driver.go index 40abeaf5..f2d5acea 100644 --- a/drivers/google_photo/driver.go +++ b/drivers/google_photo/driver.go @@ -47,11 +47,6 @@ func (d *GooglePhoto) List(ctx context.Context, dir model.Obj, args model.ListAr }) } -//func (d *GooglePhoto) Get(ctx context.Context, path string) (model.Obj, error) { -// // this is optional -// return nil, errs.NotImplement -//} - func (d *GooglePhoto) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { f, err := d.getMedia(file.GetID()) if err != nil { diff --git a/drivers/local/driver.go b/drivers/local/driver.go index a0f9d721..3a1cd4af 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -16,7 +16,6 @@ import ( "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" - "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/pkg/utils" @@ -34,19 +33,18 @@ func (d *Local) Config() driver.Config { return config } -func (d *Local) Init(ctx context.Context) (err error) { +func (d *Local) Init(ctx context.Context) error { if !utils.Exists(d.GetRootPath()) { - err = fmt.Errorf("root folder %s not exists", d.GetRootPath()) - } else { - if !filepath.IsAbs(d.GetRootPath()) { - abs, err := filepath.Abs(d.GetRootPath()) - if err != nil { - return err - } - d.SetRootPath(abs) - } + return fmt.Errorf("root folder %s not exists", d.GetRootPath()) } - return err + if !filepath.IsAbs(d.GetRootPath()) { + abs, err := filepath.Abs(d.GetRootPath()) + if err != nil { + return err + } + d.Addition.RootFolderPath = abs + } + return nil } func (d *Local) Drop(ctx context.Context) error { @@ -75,12 +73,13 @@ func (d *Local) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ thumb += "?type=thumb&sign=" + sign.Sign(stdpath.Join(args.ReqPath, f.Name())) } isFolder := f.IsDir() || isSymlinkDir(f, fullPath) - size := f.Size() - if isFolder { - size = 0 + var size int64 + if !isFolder { + size = f.Size() } file := model.ObjThumb{ Object: model.Object{ + Path: filepath.Join(dir.GetPath(), f.Name()), Name: f.Name(), Modified: f.ModTime(), Size: size, @@ -95,29 +94,6 @@ func (d *Local) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ return files, nil } -func (d *Local) Get(ctx context.Context, path string) (model.Obj, error) { - f, err := os.Stat(path) - if err != nil { - if strings.Contains(err.Error(), "cannot find the file") { - return nil, errs.ObjectNotFound - } - return nil, err - } - isFolder := f.IsDir() || isSymlinkDir(f, path) - size := f.Size() - if isFolder { - size = 0 - } - file := model.Object{ - Path: path, - Name: f.Name(), - Modified: f.ModTime(), - Size: size, - IsFolder: isFolder, - } - return &file, nil -} - func (d *Local) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { fullPath := file.GetPath() var link model.Link diff --git a/drivers/mediatrack/driver.go b/drivers/mediatrack/driver.go index 3a7188b7..2246aaa2 100644 --- a/drivers/mediatrack/driver.go +++ b/drivers/mediatrack/driver.go @@ -8,14 +8,12 @@ import ( "io" "net/http" "os" - "path" "strconv" "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" @@ -59,7 +57,7 @@ func (d *MediaTrack) List(ctx context.Context, dir model.Obj, args model.ListArg if f.File != nil && f.File.Cover != "" { thumb = "https://nano.mtres.cn/" + f.File.Cover } - return &model.ObjThumb{ + return &Object{ Object: model.Object{ ID: f.ID, Name: f.Title, @@ -68,15 +66,11 @@ func (d *MediaTrack) List(ctx context.Context, dir model.Obj, args model.ListArg Size: size, }, Thumbnail: model.Thumbnail{Thumbnail: thumb}, + ParentID: dir.GetID(), }, nil }) } -//func (d *MediaTrack) Get(ctx context.Context, path string) (model.Obj, error) { -// // this is optional -// return nil, errs.NotImplement -//} - func (d *MediaTrack) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { url := fmt.Sprintf("https://kayn.api.mediatrack.cn/v1/download_token/asset?asset_id=%s&source_type=project&password=&source_id=%s", file.GetID(), d.ProjectID) @@ -151,16 +145,18 @@ func (d *MediaTrack) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *MediaTrack) Remove(ctx context.Context, obj model.Obj) error { - dir, err := op.Get(ctx, d, path.Dir(obj.GetPath())) - if err != nil { - return err + var parentID string + if o, ok := obj.(*Object); ok { + parentID = o.ParentID + } else { + return fmt.Errorf("obj is not local Object") } data := base.Json{ - "origin_id": dir.GetID(), + "origin_id": parentID, "ids": []string{obj.GetID()}, } url := "https://jayce.api.mediatrack.cn/v4/assets/batch/delete" - _, err = d.request(url, http.MethodDelete, func(req *resty.Request) { + _, err := d.request(url, http.MethodDelete, func(req *resty.Request) { req.SetBody(data) }, nil) return err diff --git a/drivers/mediatrack/types.go b/drivers/mediatrack/types.go index 1adbfed9..e8805275 100644 --- a/drivers/mediatrack/types.go +++ b/drivers/mediatrack/types.go @@ -1,6 +1,10 @@ package mediatrack -import "time" +import ( + "time" + + "github.com/alist-org/alist/v3/internal/model" +) type BaseResp struct { Status string `json:"status"` @@ -60,3 +64,9 @@ type UploadResp struct { TraceID string `json:"trace_id"` RequestID string `json:"requestId"` } + +type Object struct { + model.Object + model.Thumbnail + ParentID string +} diff --git a/drivers/mega/driver.go b/drivers/mega/driver.go index a177133a..ad0cfe6d 100644 --- a/drivers/mega/driver.go +++ b/drivers/mega/driver.go @@ -9,7 +9,6 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/pkg/chanio" log "github.com/sirupsen/logrus" "github.com/t3rm1n4l/go-mega" ) @@ -56,13 +55,10 @@ func (d *Mega) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([] return nil, fmt.Errorf("unable to convert dir to mega node") } -func (d *Mega) Get(ctx context.Context, path string) (model.Obj, error) { - if path == "/" { - n := d.c.FS.GetRoot() - log.Debugf("mega root: %+v", *n) - return &MegaNode{n}, nil - } - return nil, errs.NotSupport +func (d *Mega) GetRoot(ctx context.Context) (model.Obj, error) { + n := d.c.FS.GetRoot() + log.Debugf("mega root: %+v", *n) + return &MegaNode{n}, nil } func (d *Mega) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { @@ -79,17 +75,21 @@ func (d *Mega) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* //u := down.GetResourceUrl() //u = strings.Replace(u, "http", "https", 1) //return &model.Link{URL: u}, nil - c := chanio.New() + r, w := io.Pipe() go func() { defer func() { _ = recover() }() log.Debugf("chunk size: %d", down.Chunks()) + var ( + chunk []byte + err error + ) for id := 0; id < down.Chunks(); id++ { - chunk, err := down.DownloadChunk(id) + chunk, err = down.DownloadChunk(id) if err != nil { log.Errorf("mega down: %+v", err) - return + break } log.Debugf("id: %d,len: %d", id, len(chunk)) //_, _, err = down.ChunkLocation(id) @@ -98,14 +98,16 @@ func (d *Mega) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* // return //} //_, err = c.Write(chunk) - _, err = c.Write(chunk) + if _, err = w.Write(chunk); err != nil { + break + } } - err := c.Close() + err = w.CloseWithError(err) if err != nil { log.Errorf("mega down: %+v", err) } }() - return &model.Link{Data: c}, nil + return &model.Link{Data: r}, nil } return nil, fmt.Errorf("unable to convert dir to mega node") } diff --git a/drivers/onedrive/driver.go b/drivers/onedrive/driver.go index 899b760d..14d4f69f 100644 --- a/drivers/onedrive/driver.go +++ b/drivers/onedrive/driver.go @@ -2,14 +2,13 @@ package onedrive import ( "context" + "fmt" "net/http" - stdpath "path" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" ) @@ -45,7 +44,7 @@ func (d *Onedrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) return nil, err } return utils.SliceConvert(files, func(src File) (model.Obj, error) { - return fileToObj(src), nil + return fileToObj(src, dir.GetID()), nil }) } @@ -90,18 +89,21 @@ func (d *Onedrive) Move(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *Onedrive) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - dstDir, err := op.Get(ctx, d, stdpath.Dir(srcObj.GetPath())) - if err != nil { - return err + //dstDir, err := op.GetUnwrap(ctx, d, stdpath.Dir(srcObj.GetPath())) + var parentID string + if o, ok := srcObj.(*Object); ok { + parentID = o.ParentID + } else { + return fmt.Errorf("srcObj is not Object") } data := base.Json{ "parentReference": base.Json{ - "id": dstDir.GetID(), + "id": parentID, }, "name": newName, } url := d.GetMetaUrl(false, srcObj.GetPath()) - _, err = d.Request(url, http.MethodPatch, func(req *resty.Request) { + _, err := d.Request(url, http.MethodPatch, func(req *resty.Request) { req.SetBody(data) }, nil) return err diff --git a/drivers/onedrive/types.go b/drivers/onedrive/types.go index e1bd7620..1899640b 100644 --- a/drivers/onedrive/types.go +++ b/drivers/onedrive/types.go @@ -42,21 +42,29 @@ type File struct { } `json:"parentReference"` } -func fileToObj(f File) *model.ObjThumbURL { +type Object struct { + model.ObjThumbURL + ParentID string +} + +func fileToObj(f File, parentID string) *Object { thumb := "" if len(f.Thumbnails) > 0 { thumb = f.Thumbnails[0].Medium.Url } - return &model.ObjThumbURL{ - Object: model.Object{ - ID: f.Id, - Name: f.Name, - Size: f.Size, - Modified: f.LastModifiedDateTime, - IsFolder: f.File == nil, + return &Object{ + ObjThumbURL: model.ObjThumbURL{ + Object: model.Object{ + ID: f.Id, + Name: f.Name, + Size: f.Size, + Modified: f.LastModifiedDateTime, + IsFolder: f.File == nil, + }, + Thumbnail: model.Thumbnail{Thumbnail: thumb}, + Url: model.Url{Url: f.Url}, }, - Thumbnail: model.Thumbnail{Thumbnail: thumb}, - Url: model.Url{Url: f.Url}, + ParentID: parentID, } } diff --git a/drivers/quark/driver.go b/drivers/quark/driver.go index 36194149..fb3731fc 100644 --- a/drivers/quark/driver.go +++ b/drivers/quark/driver.go @@ -51,11 +51,6 @@ func (d *Quark) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ }) } -//func (d *Quark) Get(ctx context.Context, path string) (model.Obj, error) { -// // TODO this is optional -// return nil, errs.NotImplement -//} - func (d *Quark) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { data := base.Json{ "fids": []string{file.GetID()}, diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go index e819a375..9ab3c4c8 100644 --- a/drivers/s3/driver.go +++ b/drivers/s3/driver.go @@ -57,11 +57,6 @@ func (d *S3) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]mo return d.listV1(dir.GetPath()) } -//func (d *S3) Get(ctx context.Context, path string) (model.Obj, error) { -// // this is optional -// return nil, errs.NotImplement -//} - func (d *S3) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { path := getKey(file.GetPath(), false) disposition := fmt.Sprintf(`attachment;filename="%s"`, url.QueryEscape(stdpath.Base(path))) diff --git a/drivers/sftp/driver.go b/drivers/sftp/driver.go index 7cd4e2a2..b9d3467b 100644 --- a/drivers/sftp/driver.go +++ b/drivers/sftp/driver.go @@ -47,11 +47,6 @@ func (d *SFTP) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([] }) } -//func (d *SFTP) Get(ctx context.Context, path string) (model.Obj, error) { -// // this is optional -// return nil, errs.NotImplement -//} - func (d *SFTP) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { remoteFile, err := d.client.Open(file.GetPath()) if err != nil { diff --git a/drivers/smb/driver.go b/drivers/smb/driver.go index c8bef00a..5392f130 100644 --- a/drivers/smb/driver.go +++ b/drivers/smb/driver.go @@ -69,11 +69,6 @@ func (d *SMB) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]m return files, nil } -//func (d *SMB) Get(ctx context.Context, path string) (model.Obj, error) { -// // this is optional -// return nil, errs.NotImplement -//} - func (d *SMB) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { if err := d.checkConn(); err != nil { return nil, err diff --git a/drivers/teambition/driver.go b/drivers/teambition/driver.go index 041e7cb3..62855f0c 100644 --- a/drivers/teambition/driver.go +++ b/drivers/teambition/driver.go @@ -37,11 +37,6 @@ func (d *Teambition) List(ctx context.Context, dir model.Obj, args model.ListArg return d.getFiles(dir.GetID()) } -//func (d *Teambition) Get(ctx context.Context, path string) (model.Obj, error) { -// // this is optional -// return nil, errs.NotImplement -//} - func (d *Teambition) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { if u, ok := file.(model.URL); ok { url := u.URL() diff --git a/drivers/template/driver.go b/drivers/template/driver.go index b1a0d673..e067b81d 100644 --- a/drivers/template/driver.go +++ b/drivers/template/driver.go @@ -36,11 +36,6 @@ func (d *Template) List(ctx context.Context, dir model.Obj, args model.ListArgs) return nil, errs.NotImplement } -//func (d *Template) Get(ctx context.Context, path string) (model.Obj, error) { -// // this is optional -// return nil, errs.NotImplement -//} - func (d *Template) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { // TODO return link of file return nil, errs.NotImplement diff --git a/drivers/thunder/types.go b/drivers/thunder/types.go index 03a5b365..0c60dc56 100644 --- a/drivers/thunder/types.go +++ b/drivers/thunder/types.go @@ -4,8 +4,6 @@ import ( "fmt" "strconv" "time" - - "github.com/alist-org/alist/v3/pkg/utils" ) type ErrResp struct { @@ -149,7 +147,7 @@ type Files struct { } func (c *Files) GetSize() int64 { size, _ := strconv.ParseInt(c.Size, 10, 64); return size } -func (c *Files) GetName() string { return utils.MappingName(c.Name) } +func (c *Files) GetName() string { return c.Name } func (c *Files) ModTime() time.Time { return c.ModifiedTime } func (c *Files) IsDir() bool { return c.Kind == FOLDER } func (c *Files) GetID() string { return c.ID } diff --git a/drivers/uss/driver.go b/drivers/uss/driver.go index 9f12ca3e..bce18b46 100644 --- a/drivers/uss/driver.go +++ b/drivers/uss/driver.go @@ -71,11 +71,6 @@ func (d *USS) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]m return res, err } -//func (d *USS) Get(ctx context.Context, path string) (model.Obj, error) { -// // this is optional -// return nil, errs.NotImplement -//} - func (d *USS) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { key := getKey(file.GetPath(), false) host := d.CustomHost diff --git a/drivers/webdav/driver.go b/drivers/webdav/driver.go index 761fca52..7c29bcb8 100644 --- a/drivers/webdav/driver.go +++ b/drivers/webdav/driver.go @@ -62,11 +62,6 @@ func (d *WebDav) List(ctx context.Context, dir model.Obj, args model.ListArgs) ( }) } -//func (d *WebDav) Get(ctx context.Context, path string) (model.Obj, error) { -// // this is optional -// return nil, errs.NotImplement -//} - func (d *WebDav) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { url, header, err := d.client.Link(file.GetPath()) if err != nil { diff --git a/drivers/yandex_disk/driver.go b/drivers/yandex_disk/driver.go index ef12516f..ddccf057 100644 --- a/drivers/yandex_disk/driver.go +++ b/drivers/yandex_disk/driver.go @@ -45,11 +45,6 @@ func (d *YandexDisk) List(ctx context.Context, dir model.Obj, args model.ListArg }) } -//func (d *YandexDisk) Get(ctx context.Context, path string) (model.Obj, error) { -// // this is optional -// return nil, errs.NotImplement -//} - func (d *YandexDisk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var resp DownResp _, err := d.request("/download", http.MethodGet, func(req *resty.Request) { diff --git a/internal/db/meta.go b/internal/db/meta.go index 0f9c0e30..8fd54fd3 100644 --- a/internal/db/meta.go +++ b/internal/db/meta.go @@ -19,7 +19,7 @@ var metaCache = cache.NewMemCache(cache.WithShards[*model.Meta](2)) var metaG singleflight.Group[*model.Meta] func GetNearestMeta(path string) (*model.Meta, error) { - path = utils.StandardizePath(path) + path = utils.FixAndCleanPath(path) meta, err := GetMetaByPath(path) if err == nil { return meta, nil diff --git a/internal/db/searchnode.go b/internal/db/searchnode.go index 97e00e9f..c134cbdf 100644 --- a/internal/db/searchnode.go +++ b/internal/db/searchnode.go @@ -30,6 +30,7 @@ func BatchCreateSearchNodes(nodes *[]model.SearchNode) error { } func DeleteSearchNodesByParent(prefix string) error { + prefix = utils.FixAndCleanPath(prefix) err := db.Where(whereInParent(prefix)).Delete(&model.SearchNode{}).Error if err != nil { return err @@ -37,7 +38,7 @@ func DeleteSearchNodesByParent(prefix string) error { dir, name := path.Split(prefix) return db.Where(fmt.Sprintf("%s = ? AND %s = ?", columnName("parent"), columnName("name")), - utils.StandardizePath(dir), name).Delete(&model.SearchNode{}).Error + dir, name).Delete(&model.SearchNode{}).Error } func ClearSearchNodes() error { diff --git a/internal/driver/driver.go b/internal/driver/driver.go index d4e4ea62..3e617eb9 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -39,7 +39,7 @@ type Reader interface { } type Getter interface { - Get(ctx context.Context, path string) (model.Obj, error) + GetRoot(ctx context.Context) (model.Obj, error) } type Writer interface { diff --git a/internal/fs/fs.go b/internal/fs/fs.go index 4b9ce5a3..05429f7e 100644 --- a/internal/fs/fs.go +++ b/internal/fs/fs.go @@ -80,7 +80,7 @@ func Remove(ctx context.Context, path string) error { return err } -func PutDirectly(ctx context.Context, dstDirPath string, file model.FileStreamer) error { +func PutDirectly(ctx context.Context, dstDirPath string, file *model.FileStream) error { err := putDirectly(ctx, dstDirPath, file) if err != nil { log.Errorf("failed put %s: %+v", dstDirPath, err) @@ -88,7 +88,7 @@ func PutDirectly(ctx context.Context, dstDirPath string, file model.FileStreamer return err } -func PutAsTask(dstDirPath string, file model.FileStreamer) error { +func PutAsTask(dstDirPath string, file *model.FileStream) error { err := putAsTask(dstDirPath, file) if err != nil { log.Errorf("failed put %s: %+v", dstDirPath, err) diff --git a/internal/fs/get.go b/internal/fs/get.go index 08f0222b..17c202b7 100644 --- a/internal/fs/get.go +++ b/internal/fs/get.go @@ -12,7 +12,7 @@ import ( ) func get(ctx context.Context, path string) (model.Obj, error) { - path = utils.StandardizePath(path) + path = utils.FixAndCleanPath(path) // maybe a virtual file if path != "/" { virtualFiles := op.GetStorageVirtualFilesByPath(stdpath.Dir(path)) diff --git a/internal/fs/put.go b/internal/fs/put.go index 035e8cd2..5adfb7f7 100644 --- a/internal/fs/put.go +++ b/internal/fs/put.go @@ -18,7 +18,7 @@ var UploadTaskManager = task.NewTaskManager(3, func(tid *uint64) { }) // putAsTask add as a put task and return immediately -func putAsTask(dstDirPath string, file model.FileStreamer) error { +func putAsTask(dstDirPath string, file *model.FileStream) error { storage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) if err != nil { return errors.WithMessage(err, "failed get storage") @@ -43,7 +43,7 @@ func putAsTask(dstDirPath string, file model.FileStreamer) error { } // putDirect put the file and return after finish -func putDirectly(ctx context.Context, dstDirPath string, file model.FileStreamer) error { +func putDirectly(ctx context.Context, dstDirPath string, file *model.FileStream) error { storage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) if err != nil { return errors.WithMessage(err, "failed get storage") diff --git a/internal/fs/util.go b/internal/fs/util.go index 9e59e844..b5d0cb4b 100644 --- a/internal/fs/util.go +++ b/internal/fs/util.go @@ -35,7 +35,7 @@ func containsByName(files []model.Obj, file model.Obj) bool { var httpClient = &http.Client{} -func getFileStreamFromLink(file model.Obj, link *model.Link) (model.FileStreamer, error) { +func getFileStreamFromLink(file model.Obj, link *model.Link) (*model.FileStream, error) { var rc io.ReadCloser mimetype := utils.GetMimeType(file.GetName()) if link.Data != nil { diff --git a/internal/model/obj.go b/internal/model/obj.go index 54fbb497..3a069858 100644 --- a/internal/model/obj.go +++ b/internal/model/obj.go @@ -8,11 +8,18 @@ import ( "github.com/maruel/natural" ) +type UnwrapObj interface { + Unwrap() Obj +} + type Obj interface { GetSize() int64 GetName() string ModTime() time.Time IsDir() bool + + // The internal information of the driver. + // If you want to use it, please understand what it means GetID() string GetPath() string } @@ -24,6 +31,7 @@ type FileStreamer interface { SetReadCloser(io.ReadCloser) NeedStore() bool GetReadCloser() io.ReadCloser + GetOld() Obj } type URL interface { @@ -86,3 +94,9 @@ func ExtractFolder(objs []Obj, extractFolder string) { return false }) } + +func WrapObjsName(objs []Obj) { + for i := 0; i < len(objs); i++ { + objs[i] = &ObjWrapName{Obj: objs[i]} + } +} diff --git a/internal/model/object.go b/internal/model/object.go index 8b0513a1..768fdda4 100644 --- a/internal/model/object.go +++ b/internal/model/object.go @@ -6,6 +6,22 @@ import ( "github.com/alist-org/alist/v3/pkg/utils" ) +type ObjWrapName struct { + Name string + Obj +} + +func (o *ObjWrapName) Unwrap() Obj { + return o.Obj +} + +func (o *ObjWrapName) GetName() string { + if o.Name == "" { + o.Name = utils.MappingName(o.Obj.GetName()) + } + return o.Name +} + type Object struct { ID string Path string @@ -16,7 +32,7 @@ type Object struct { } func (o *Object) GetName() string { - return utils.MappingName(o.Name) + return o.Name } func (o *Object) GetSize() int64 { diff --git a/internal/model/stream.go b/internal/model/stream.go index 92d7ee2f..cd7c3363 100644 --- a/internal/model/stream.go +++ b/internal/model/stream.go @@ -9,6 +9,7 @@ type FileStream struct { io.ReadCloser Mimetype string WebPutAsTask bool + Old Obj } func (f *FileStream) GetMimetype() string { @@ -26,3 +27,7 @@ func (f *FileStream) GetReadCloser() io.ReadCloser { func (f *FileStream) SetReadCloser(rc io.ReadCloser) { f.ReadCloser = rc } + +func (f *FileStream) GetOld() Obj { + return f.Old +} diff --git a/internal/op/const.go b/internal/op/const.go index 601dd00d..a465735f 100644 --- a/internal/op/const.go +++ b/internal/op/const.go @@ -1,3 +1,6 @@ package op -var WORK = "work" +const ( + WORK = "work" + RootName = "root" +) diff --git a/internal/op/fs.go b/internal/op/fs.go index cbd4d74d..b5bd4e6f 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -4,7 +4,6 @@ import ( "context" "os" stdpath "path" - "strings" "time" "github.com/Xhofe/go-cache" @@ -23,12 +22,11 @@ var listCache = cache.NewMemCache(cache.WithShards[[]model.Obj](64)) var listG singleflight.Group[[]model.Obj] func ClearCache(storage driver.Driver, path string) { - key := stdpath.Join(storage.GetStorage().MountPath, path) - listCache.Del(key) + listCache.Del(Key(storage, path)) } func Key(storage driver.Driver, path string) string { - return stdpath.Join(storage.GetStorage().MountPath, utils.StandardizePath(path)) + return stdpath.Join(storage.GetStorage().MountPath, utils.FixAndCleanPath(path)) } // List files in storage, not contains virtual file @@ -36,7 +34,7 @@ func List(ctx context.Context, storage driver.Driver, path string, args model.Li if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) } - path = utils.StandardizePath(path) + path = utils.FixAndCleanPath(path) log.Debugf("op.List %s", path) key := Key(storage, path) if len(refresh) == 0 || !refresh[0] { @@ -45,7 +43,7 @@ func List(ctx context.Context, storage driver.Driver, path string, args model.Li return files, nil } } - dir, err := Get(ctx, storage, path) + dir, err := GetUnwrap(ctx, storage, path) if err != nil { return nil, errors.WithMessage(err, "failed get dir") } @@ -58,6 +56,14 @@ func List(ctx context.Context, storage driver.Driver, path string, args model.Li if err != nil { return nil, errors.Wrapf(err, "failed to list objs") } + // set path + for _, f := range files { + if s, ok := f.(model.SetPath); ok && f.GetPath() == "" && dir.GetPath() != "" { + s.SetPath(stdpath.Join(dir.GetPath(), f.GetName())) + } + } + // warp obj name + model.WrapObjsName(files) // call hooks go func(reqPath string, files []model.Obj) { for _, hook := range objsUpdateHooks { @@ -78,48 +84,49 @@ func List(ctx context.Context, storage driver.Driver, path string, args model.Li return objs, err } -func isRoot(path, rootFolderPath string) bool { - if utils.PathEqual(path, rootFolderPath) { - return true - } - rootFolderPath = strings.TrimSuffix(rootFolderPath, "/") - rootFolderPath = strings.TrimPrefix(rootFolderPath, "\\") - // relative path, this shouldn't happen, because root folder path is absolute - if utils.PathEqual(path, "/") && rootFolderPath == "." { - return true - } - return false -} - // Get object from list of files func Get(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) { - path = utils.StandardizePath(path) + path = utils.FixAndCleanPath(path) log.Debugf("op.Get %s", path) - if g, ok := storage.(driver.Getter); ok { - obj, err := g.Get(ctx, path) - if err == nil { - return obj, nil - } - } + // is root folder - if r, ok := storage.GetAddition().(driver.IRootId); ok && utils.PathEqual(path, "/") { - return &model.Object{ - ID: r.GetRootId(), - Name: "root", - Size: 0, - Modified: storage.GetStorage().Modified, - IsFolder: true, - }, nil - } - if r, ok := storage.GetAddition().(driver.IRootPath); ok && isRoot(path, r.GetRootPath()) { - return &model.Object{ - Path: r.GetRootPath(), - Name: "root", - Size: 0, - Modified: storage.GetStorage().Modified, - IsFolder: true, + if utils.PathEqual(path, "/") { + var rootObj model.Obj + switch r := storage.GetAddition().(type) { + case driver.IRootId: + rootObj = &model.Object{ + ID: r.GetRootId(), + Name: RootName, + Size: 0, + Modified: storage.GetStorage().Modified, + IsFolder: true, + } + case driver.IRootPath: + rootObj = &model.Object{ + Path: r.GetRootPath(), + Name: RootName, + Size: 0, + Modified: storage.GetStorage().Modified, + IsFolder: true, + } + default: + if storage, ok := storage.(driver.Getter); ok { + obj, err := storage.GetRoot(ctx) + if err != nil { + return nil, errors.WithMessage(err, "failed get root obj") + } + rootObj = obj + } + } + if rootObj == nil { + return nil, errors.Errorf("please implement IRootPath or IRootId or Getter method") + } + return &model.ObjWrapName{ + Name: RootName, + Obj: rootObj, }, nil } + // not root folder dir, name := stdpath.Split(path) files, err := List(ctx, storage, dir, model.ListArgs{}) @@ -129,13 +136,6 @@ func Get(ctx context.Context, storage driver.Driver, path string) (model.Obj, er for _, f := range files { // TODO maybe copy obj here if f.GetName() == name { - // use path as id, why don't set id in List function? - // because files maybe cache, set id here can reduce memory usage - if f.GetPath() == "" { - if s, ok := f.(model.SetPath); ok { - s.SetPath(path) - } - } return f, nil } } @@ -143,6 +143,17 @@ func Get(ctx context.Context, storage driver.Driver, path string) (model.Obj, er return nil, errors.WithStack(errs.ObjectNotFound) } +func GetUnwrap(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) { + obj, err := Get(ctx, storage, path) + if err != nil { + return nil, err + } + if unwrap, ok := obj.(model.UnwrapObj); ok { + obj = unwrap.Unwrap() + } + return obj, err +} + var linkCache = cache.NewMemCache(cache.WithShards[*model.Link](16)) var linkG singleflight.Group[*model.Link] @@ -151,7 +162,7 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return nil, nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) } - file, err := Get(ctx, storage, path) + file, err := GetUnwrap(ctx, storage, path) if err != nil { return nil, nil, errors.WithMessage(err, "failed to get file") } @@ -178,7 +189,7 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li // Other api func Other(ctx context.Context, storage driver.Driver, args model.FsOtherArgs) (interface{}, error) { - obj, err := Get(ctx, storage, args.Path) + obj, err := GetUnwrap(ctx, storage, args.Path) if err != nil { return nil, errors.WithMessagef(err, "failed to get obj") } @@ -199,11 +210,11 @@ func MakeDir(ctx context.Context, storage driver.Driver, path string) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return errors.Errorf("storage not init: %s", storage.GetStorage().Status) } - path = utils.StandardizePath(path) + path = utils.FixAndCleanPath(path) key := Key(storage, path) _, err, _ := mkdirG.Do(key, func() (interface{}, error) { // check if dir exists - f, err := Get(ctx, storage, path) + f, err := GetUnwrap(ctx, storage, path) if err != nil { if errs.IsObjectNotFound(err) { parentPath, dirName := stdpath.Split(path) @@ -211,7 +222,7 @@ func MakeDir(ctx context.Context, storage driver.Driver, path string) error { if err != nil { return nil, errors.WithMessagef(err, "failed to make parent dir [%s]", parentPath) } - parentDir, err := Get(ctx, storage, parentPath) + parentDir, err := GetUnwrap(ctx, storage, parentPath) // this should not happen if err != nil { return nil, errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath) @@ -242,11 +253,11 @@ func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return errors.Errorf("storage not init: %s", storage.GetStorage().Status) } - srcObj, err := Get(ctx, storage, srcPath) + srcObj, err := GetUnwrap(ctx, storage, srcPath) if err != nil { return errors.WithMessage(err, "failed to get src object") } - dstDir, err := Get(ctx, storage, dstDirPath) + dstDir, err := GetUnwrap(ctx, storage, dstDirPath) if err != nil { return errors.WithMessage(err, "failed to get dst dir") } @@ -257,7 +268,7 @@ func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string) if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return errors.Errorf("storage not init: %s", storage.GetStorage().Status) } - srcObj, err := Get(ctx, storage, srcPath) + srcObj, err := GetUnwrap(ctx, storage, srcPath) if err != nil { return errors.WithMessage(err, "failed to get src object") } @@ -269,11 +280,11 @@ func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return errors.Errorf("storage not init: %s", storage.GetStorage().Status) } - srcObj, err := Get(ctx, storage, srcPath) + srcObj, err := GetUnwrap(ctx, storage, srcPath) if err != nil { return errors.WithMessage(err, "failed to get src object") } - dstDir, err := Get(ctx, storage, dstDirPath) + dstDir, err := GetUnwrap(ctx, storage, dstDirPath) return errors.WithStack(storage.Copy(ctx, srcObj, dstDir)) } @@ -281,7 +292,7 @@ func Remove(ctx context.Context, storage driver.Driver, path string) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return errors.Errorf("storage not init: %s", storage.GetStorage().Status) } - obj, err := Get(ctx, storage, path) + obj, err := GetUnwrap(ctx, storage, path) if err != nil { // if object not found, it's ok if errs.IsObjectNotFound(err) { @@ -313,7 +324,7 @@ func Remove(ctx context.Context, storage driver.Driver, path string) error { return errors.WithStack(err) } -func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file model.FileStreamer, up driver.UpdateProgress) error { +func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file *model.FileStream, up driver.UpdateProgress) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return errors.Errorf("storage not init: %s", storage.GetStorage().Status) } @@ -332,21 +343,22 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod }() // if file exist and size = 0, delete it dstPath := stdpath.Join(dstDirPath, file.GetName()) - fi, err := Get(ctx, storage, dstPath) + fi, err := GetUnwrap(ctx, storage, dstPath) if err == nil { if fi.GetSize() == 0 { err = Remove(ctx, storage, dstPath) if err != nil { return errors.WithMessagef(err, "failed remove file that exist and have size 0") } + } else { + file.Old = fi } } - err = MakeDir(ctx, storage, dstDirPath) if err != nil { return errors.WithMessagef(err, "failed to make dir [%s]", dstDirPath) } - parentDir, err := Get(ctx, storage, dstDirPath) + parentDir, err := GetUnwrap(ctx, storage, dstDirPath) // this should not happen if err != nil { return errors.WithMessagef(err, "failed to get dir [%s]", dstDirPath) diff --git a/internal/op/path.go b/internal/op/path.go index 8f394363..9909fbc2 100644 --- a/internal/op/path.go +++ b/internal/op/path.go @@ -1,7 +1,6 @@ package op import ( - stdpath "path" "strings" "github.com/alist-org/alist/v3/internal/driver" @@ -10,30 +9,17 @@ import ( log "github.com/sirupsen/logrus" ) -// ActualPath Get the actual path -// !!! maybe and \ in the path when use windows local -func ActualPath(storage driver.Additional, rawPath string) string { - if i, ok := storage.(driver.IRootPath); ok { - rawPath = stdpath.Join(i.GetRootPath(), rawPath) - } - return utils.StandardizePath(rawPath) -} - // GetStorageAndActualPath Get the corresponding storage and actual path // for path: remove the mount path prefix and join the actual root folder if exists -func GetStorageAndActualPath(rawPath string) (driver.Driver, string, error) { - rawPath = utils.StandardizePath(rawPath) - // why can remove this check? because reqPath has joined the base_path of user, no relative path - //if strings.Contains(rawPath, "..") { - // return nil, "", errors.WithStack(errs.RelativePath) - //} - storage := GetBalancedStorage(rawPath) +func GetStorageAndActualPath(rawPath string) (storage driver.Driver, actualPath string, err error) { + rawPath = utils.FixAndCleanPath(rawPath) + storage = GetBalancedStorage(rawPath) if storage == nil { - return nil, "", errors.Errorf("can't find storage with rawPath: %s", rawPath) + err = errors.Errorf("can't find storage with rawPath: %s", rawPath) + return } log.Debugln("use storage: ", storage.GetStorage().MountPath) virtualPath := utils.GetActualVirtualPath(storage.GetStorage().MountPath) - actualPath := strings.TrimPrefix(rawPath, virtualPath) - actualPath = ActualPath(storage.GetAddition(), actualPath) - return storage, actualPath, nil + actualPath = utils.FixAndCleanPath(strings.TrimPrefix(rawPath, virtualPath)) + return } diff --git a/internal/op/storage.go b/internal/op/storage.go index e3e45022..20006ca2 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -11,6 +11,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/generic_sync" "github.com/alist-org/alist/v3/pkg/utils" + mapset "github.com/deckarep/golang-set/v2" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) @@ -25,10 +26,11 @@ func GetAllStorages() []driver.Driver { } func HasStorage(mountPath string) bool { - return storagesMap.Has(mountPath) + return storagesMap.Has(utils.FixAndCleanPath(mountPath)) } func GetStorageByVirtualPath(virtualPath string) (driver.Driver, error) { + virtualPath = utils.FixAndCleanPath(virtualPath) storageDriver, ok := storagesMap.Load(virtualPath) if !ok { return nil, errors.Errorf("no mount path for an storage is: %s", virtualPath) @@ -40,7 +42,7 @@ func GetStorageByVirtualPath(virtualPath string) (driver.Driver, error) { // then instantiate corresponding driver and save it in memory func CreateStorage(ctx context.Context, storage model.Storage) (uint, error) { storage.Modified = time.Now() - storage.MountPath = utils.StandardizePath(storage.MountPath) + storage.MountPath = utils.FixAndCleanPath(storage.MountPath) var err error // check driver first driverName := storage.Driver @@ -65,7 +67,7 @@ func CreateStorage(ctx context.Context, storage model.Storage) (uint, error) { // LoadStorage load exist storage in db to memory func LoadStorage(ctx context.Context, storage model.Storage) error { - storage.MountPath = utils.StandardizePath(storage.MountPath) + storage.MountPath = utils.FixAndCleanPath(storage.MountPath) // check driver first driverName := storage.Driver driverNew, err := GetDriverNew(driverName) @@ -158,7 +160,7 @@ func UpdateStorage(ctx context.Context, storage model.Storage) error { return errors.Errorf("driver cannot be changed") } storage.Modified = time.Now() - storage.MountPath = utils.StandardizePath(storage.MountPath) + storage.MountPath = utils.FixAndCleanPath(storage.MountPath) err = db.UpdateStorage(&storage) if err != nil { return errors.WithMessage(err, "failed update storage in database") @@ -237,25 +239,20 @@ func saveDriverStorage(driver driver.Driver) error { func getStoragesByPath(path string) []driver.Driver { storages := make([]driver.Driver, 0) curSlashCount := 0 - storagesMap.Range(func(key string, value driver.Driver) bool { - virtualPath := utils.GetActualVirtualPath(value.GetStorage().MountPath) - if virtualPath == "/" { - virtualPath = "" + storagesMap.Range(func(mountPath string, value driver.Driver) bool { + virtualPath := utils.GetActualVirtualPath(mountPath) + // is this path + if strings.HasPrefix(path, virtualPath) { + slashCount := strings.Count(utils.PathAddSeparatorSuffix(virtualPath), "/") + // not the longest match + if slashCount > curSlashCount { + storages = storages[:0] + curSlashCount = slashCount + } + if slashCount == curSlashCount { + storages = append(storages, value) + } } - // not this - if path != virtualPath && !strings.HasPrefix(path, virtualPath+"/") { - return true - } - slashCount := strings.Count(virtualPath, "/") - // not the longest match - if slashCount < curSlashCount { - return true - } - if slashCount > curSlashCount { - storages = storages[:0] - curSlashCount = slashCount - } - storages = append(storages, value) return true }) // make sure the order is the same for same input @@ -277,36 +274,25 @@ func GetStorageVirtualFilesByPath(prefix string) []model.Obj { } return storages[i].GetStorage().Order < storages[j].GetStorage().Order }) - prefix = utils.StandardizePath(prefix) - if prefix != "/" { - prefix += "/" - } - set := make(map[string]interface{}) + + prefix = utils.FixAndCleanPath(prefix) + set := mapset.NewSet[string]() for _, v := range storages { - // TODO should save a balanced storage - // balance storage - if utils.IsBalance(v.GetStorage().MountPath) { + virtualPath := utils.GetActualVirtualPath(v.GetStorage().MountPath) + // Exclude prefix itself and non prefix + if len(prefix) >= len(virtualPath) || !strings.HasPrefix(virtualPath, prefix) { continue } - virtualPath := v.GetStorage().MountPath - if len(virtualPath) <= len(prefix) { - continue + + name := strings.SplitN(strings.TrimPrefix(virtualPath[len(prefix):], "/"), "/", 1)[0] + if set.Add(name) { + files = append(files, &model.Object{ + Name: name, + Size: 0, + Modified: v.GetStorage().Modified, + IsFolder: true, + }) } - // not prefixed with `prefix` - if !strings.HasPrefix(virtualPath, prefix) { - continue - } - name := strings.Split(strings.TrimPrefix(virtualPath, prefix), "/")[0] - if _, ok := set[name]; ok { - continue - } - files = append(files, &model.Object{ - Name: name, - Size: 0, - Modified: v.GetStorage().Modified, - IsFolder: true, - }) - set[name] = nil } return files } @@ -315,7 +301,7 @@ var balanceMap generic_sync.MapOf[string, int] // GetBalancedStorage get storage by path func GetBalancedStorage(path string) driver.Driver { - path = utils.StandardizePath(path) + path = utils.FixAndCleanPath(path) storages := getStoragesByPath(path) storageNum := len(storages) switch storageNum { @@ -325,15 +311,9 @@ func GetBalancedStorage(path string) driver.Driver { return storages[0] default: virtualPath := utils.GetActualVirtualPath(storages[0].GetStorage().MountPath) - cur, ok := balanceMap.Load(virtualPath) - i := 0 - if ok { - i = cur - i = (i + 1) % storageNum - balanceMap.Store(virtualPath, i) - } else { - balanceMap.Store(virtualPath, i) - } + i, _ := balanceMap.LoadOrStore(virtualPath, 0) + i = (i + 1) % storageNum + balanceMap.Store(virtualPath, i) return storages[i] } } diff --git a/pkg/utils/file.go b/pkg/utils/file.go index 985be6da..6bfe3527 100644 --- a/pkg/utils/file.go +++ b/pkg/utils/file.go @@ -3,7 +3,6 @@ package utils import ( "fmt" "io" - "io/ioutil" "mime" "os" "path" @@ -43,7 +42,7 @@ func CopyFile(src, dst string) error { // CopyDir Dir copies a whole directory recursively func CopyDir(src string, dst string) error { var err error - var fds []os.FileInfo + var fds []os.DirEntry var srcinfo os.FileInfo if srcinfo, err = os.Stat(src); err != nil { @@ -52,7 +51,7 @@ func CopyDir(src string, dst string) error { if err = os.MkdirAll(dst, srcinfo.Mode()); err != nil { return err } - if fds, err = ioutil.ReadDir(src); err != nil { + if fds, err = os.ReadDir(src); err != nil { return err } for _, fd := range fds { diff --git a/pkg/utils/path.go b/pkg/utils/path.go index 0be9bb4a..27328fbb 100644 --- a/pkg/utils/path.go +++ b/pkg/utils/path.go @@ -3,33 +3,37 @@ package utils import ( "net/url" stdpath "path" - "path/filepath" - "runtime" "strings" - - "github.com/alist-org/alist/v3/internal/errs" ) -// StandardizePath convert path like '/' '/root' '/a/b' -func StandardizePath(path string) string { - path = strings.TrimSuffix(path, "/") - // abs path - if filepath.IsAbs(path) && runtime.GOOS == "windows" { - return path - } - // relative path with prefix '..' - if strings.HasPrefix(path, ".") { - return path - } +// FixAndCleanPath +// The upper layer of the root directory is still the root directory. +// So ".." And "." will be cleared +// for example +// 1. ".." or "." => "/" +// 2. "../..." or "./..." => "/..." +// 3. "../.x." or "./.x." => "/.x." +// 4. "x//\\y" = > "/z/x" +func FixAndCleanPath(path string) string { + path = strings.ReplaceAll(path, "\\", "/") if !strings.HasPrefix(path, "/") { path = "/" + path } + return stdpath.Clean(path) +} + +// PathAddSeparatorSuffix Add path '/' suffix +// for example /root => /root/ +func PathAddSeparatorSuffix(path string) string { + if !strings.HasSuffix(path, "/") { + path = path + "/" + } return path } // PathEqual judge path is equal func PathEqual(path1, path2 string) bool { - return StandardizePath(path1) == StandardizePath(path2) + return FixAndCleanPath(path1) == FixAndCleanPath(path2) } func Ext(path string) string { @@ -64,8 +68,5 @@ func EncodePath(path string, all ...bool) string { } func JoinBasePath(basePath, reqPath string) (string, error) { - if strings.HasSuffix(reqPath, "..") || strings.Contains(reqPath, "../") { - return "", errs.RelativePath - } - return stdpath.Join(basePath, reqPath), nil + return stdpath.Join(FixAndCleanPath(basePath), FixAndCleanPath(reqPath)), nil } diff --git a/pkg/utils/path_test.go b/pkg/utils/path_test.go index 4eb20850..f42f2f8b 100644 --- a/pkg/utils/path_test.go +++ b/pkg/utils/path_test.go @@ -5,3 +5,18 @@ import "testing" func TestEncodePath(t *testing.T) { t.Log(EncodePath("http://localhost:5244/d/123#.png")) } + +func TestFixAndCleanPath(t *testing.T) { + datas := map[string]string{ + "": "/", + ".././": "/", + "../../.../": "/...", + "x//\\y/": "/x/y", + ".././.x/.y/.//..x../..y..": "/.x/.y/..x../..y..", + } + for key, value := range datas { + if FixAndCleanPath(key) != value { + t.Logf("raw %s fix fail", key) + } + } +} diff --git a/server/handles/meta.go b/server/handles/meta.go index 16019ed2..34825907 100644 --- a/server/handles/meta.go +++ b/server/handles/meta.go @@ -44,7 +44,7 @@ func CreateMeta(c *gin.Context) { common.ErrorStrResp(c, fmt.Sprintf("%s is illegal: %s", r, err.Error()), 400) return } - req.Path = utils.StandardizePath(req.Path) + req.Path = utils.FixAndCleanPath(req.Path) if err := db.CreateMeta(&req); err != nil { common.ErrorResp(c, err, 500, true) } else { @@ -63,7 +63,7 @@ func UpdateMeta(c *gin.Context) { common.ErrorStrResp(c, fmt.Sprintf("%s is illegal: %s", r, err.Error()), 400) return } - req.Path = utils.StandardizePath(req.Path) + req.Path = utils.FixAndCleanPath(req.Path) if err := db.UpdateMeta(&req); err != nil { common.ErrorResp(c, err, 500, true) } else { diff --git a/server/middlewares/down.go b/server/middlewares/down.go index 576fcfa3..411d053c 100644 --- a/server/middlewares/down.go +++ b/server/middlewares/down.go @@ -42,7 +42,7 @@ func Down(c *gin.Context) { // TODO: implement // path maybe contains # ? etc. func parsePath(path string) string { - return utils.StandardizePath(path) + return utils.FixAndCleanPath(path) } func needSign(meta *model.Meta, path string) bool { diff --git a/server/static/config.go b/server/static/config.go index ebad8a6f..513e086c 100644 --- a/server/static/config.go +++ b/server/static/config.go @@ -31,7 +31,7 @@ func getSiteConfig() SiteConfig { siteConfig.BasePath = setting.GetStr(conf.BasePath) } if siteConfig.BasePath != "" { - siteConfig.BasePath = utils.StandardizePath(siteConfig.BasePath) + siteConfig.BasePath = utils.FixAndCleanPath(siteConfig.BasePath) } if siteConfig.Cdn == "" { siteConfig.Cdn = siteConfig.BasePath