From fc1204c914e663dba4d3c2ebaffcf960e358f2a8 Mon Sep 17 00:00:00 2001 From: Noah Hsu Date: Sun, 10 Jul 2022 14:45:39 +0800 Subject: [PATCH] chore: rename account to storage --- drivers/local/driver.go | 10 +- internal/aria2/add.go | 12 +- internal/aria2/aria2_test.go | 4 +- internal/aria2/monitor.go | 8 +- internal/bootstrap/data/dev.go | 4 +- internal/db/account.go | 52 ++--- internal/db/db.go | 2 +- internal/driver/driver.go | 6 +- internal/errs/errors.go | 2 +- internal/fs/copy.go | 44 ++-- internal/fs/fs.go | 6 +- internal/fs/get.go | 10 +- internal/fs/link.go | 6 +- internal/fs/list.go | 20 +- internal/fs/put.go | 18 +- internal/fs/util.go | 4 +- internal/fs/write.go | 32 +-- internal/model/{account.go => storage.go} | 6 +- internal/operations/account.go | 245 ---------------------- internal/operations/account_test.go | 85 -------- internal/operations/fs.go | 90 ++++---- internal/operations/path.go | 22 +- internal/operations/storage.go | 245 ++++++++++++++++++++++ internal/operations/storage_test.go | 85 ++++++++ pkg/task/task.go | 2 +- server/controllers/account.go | 22 +- server/controllers/down.go | 22 +- server/controllers/fsmanage.go | 4 +- server/controllers/fsread.go | 12 +- server/router.go | 10 +- server/webdav/webdav.go | 6 +- 31 files changed, 548 insertions(+), 548 deletions(-) rename internal/model/{account.go => storage.go} (91%) delete mode 100644 internal/operations/account.go delete mode 100644 internal/operations/account_test.go create mode 100644 internal/operations/storage.go create mode 100644 internal/operations/storage_test.go diff --git a/drivers/local/driver.go b/drivers/local/driver.go index 64099b2c..a733ebca 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -16,7 +16,7 @@ import ( ) type Driver struct { - model.Account + model.Storage Addition } @@ -24,9 +24,9 @@ func (d Driver) Config() driver.Config { return config } -func (d *Driver) Init(ctx context.Context, account model.Account) error { - d.Account = account - err := utils.Json.UnmarshalFromString(d.Account.Addition, &d.Addition) +func (d *Driver) Init(ctx context.Context, storage model.Storage) error { + d.Storage = storage + err := utils.Json.UnmarshalFromString(d.Storage.Addition, &d.Addition) if err != nil { return errors.Wrap(err, "error while unmarshal addition") } @@ -42,7 +42,7 @@ func (d *Driver) Init(ctx context.Context, account model.Account) error { } d.SetStatus("OK") } - operations.MustSaveDriverAccount(d) + operations.MustSaveDriverStorage(d) return err } diff --git a/internal/aria2/add.go b/internal/aria2/add.go index 1fb897cc..5c5490af 100644 --- a/internal/aria2/add.go +++ b/internal/aria2/add.go @@ -13,17 +13,17 @@ import ( ) func AddURI(ctx context.Context, uri string, dstDirPath string) error { - // check account - account, dstDirActualPath, err := operations.GetAccountAndActualPath(dstDirPath) + // check storage + storage, dstDirActualPath, err := operations.GetStorageAndActualPath(dstDirPath) if err != nil { - return errors.WithMessage(err, "failed get account") + return errors.WithMessage(err, "failed get storage") } // check is it could upload - if account.Config().NoUpload { + if storage.Config().NoUpload { return errors.WithStack(errs.UploadNotSupported) } // check path is valid - obj, err := operations.Get(ctx, account, dstDirActualPath) + obj, err := operations.Get(ctx, storage, dstDirActualPath) if err != nil { if !errs.IsObjectNotFound(err) { return errors.WithMessage(err, "failed get object") @@ -45,7 +45,7 @@ func AddURI(ctx context.Context, uri string, dstDirPath string) error { } DownTaskManager.Submit(task.WithCancelCtx(&task.Task[string]{ ID: gid, - Name: fmt.Sprintf("download %s to [%s](%s)", uri, account.GetAccount().VirtualPath, dstDirActualPath), + Name: fmt.Sprintf("download %s to [%s](%s)", uri, storage.GetStorage().VirtualPath, dstDirActualPath), Func: func(tsk *task.Task[string]) error { m := &Monitor{ tsk: tsk, diff --git a/internal/aria2/aria2_test.go b/internal/aria2/aria2_test.go index 52d742b8..68ed235a 100644 --- a/internal/aria2/aria2_test.go +++ b/internal/aria2/aria2_test.go @@ -37,7 +37,7 @@ func TestConnect(t *testing.T) { func TestDown(t *testing.T) { TestConnect(t) - err := operations.CreateAccount(context.Background(), model.Account{ + err := operations.CreateStorage(context.Background(), model.Storage{ ID: 0, VirtualPath: "/", Index: 0, @@ -47,7 +47,7 @@ func TestDown(t *testing.T) { Remark: "", }) if err != nil { - t.Fatalf("failed to create account: %+v", err) + t.Fatalf("failed to create storage: %+v", err) } err = AddURI(context.Background(), "https://nodejs.org/dist/index.json", "/test") if err != nil { diff --git a/internal/aria2/monitor.go b/internal/aria2/monitor.go index cc141cec..5a510ce9 100644 --- a/internal/aria2/monitor.go +++ b/internal/aria2/monitor.go @@ -113,9 +113,9 @@ var TransferTaskManager = task.NewTaskManager[uint64](3, func(k *uint64) { func (m *Monitor) Complete() error { // check dstDir again - account, dstDirActualPath, err := operations.GetAccountAndActualPath(m.dstDirPath) + storage, dstDirActualPath, err := operations.GetStorageAndActualPath(m.dstDirPath) if err != nil { - return errors.WithMessage(err, "failed get account") + return errors.WithMessage(err, "failed get storage") } // get files files, err := client.GetFiles(m.tsk.ID) @@ -135,7 +135,7 @@ func (m *Monitor) Complete() error { }() for _, file := range files { TransferTaskManager.Submit(task.WithCancelCtx[uint64](&task.Task[uint64]{ - Name: fmt.Sprintf("transfer %s to [%s](%s)", file.Path, account.GetAccount().VirtualPath, dstDirActualPath), + Name: fmt.Sprintf("transfer %s to [%s](%s)", file.Path, storage.GetStorage().VirtualPath, dstDirActualPath), Func: func(tsk *task.Task[uint64]) error { defer wg.Done() size, _ := strconv.ParseInt(file.Length, 10, 64) @@ -157,7 +157,7 @@ func (m *Monitor) Complete() error { ReadCloser: f, Mimetype: mimetype, } - return operations.Put(tsk.Ctx, account, dstDirActualPath, stream, tsk.SetProgress) + return operations.Put(tsk.Ctx, storage, dstDirActualPath, stream, tsk.SetProgress) }, })) } diff --git a/internal/bootstrap/data/dev.go b/internal/bootstrap/data/dev.go index cb290bd3..de98c4ef 100644 --- a/internal/bootstrap/data/dev.go +++ b/internal/bootstrap/data/dev.go @@ -11,7 +11,7 @@ import ( ) func initDevData() { - err := operations.CreateAccount(context.Background(), model.Account{ + err := operations.CreateStorage(context.Background(), model.Storage{ VirtualPath: "/", Index: 0, Driver: "Local", @@ -19,7 +19,7 @@ func initDevData() { Addition: `{"root_folder":"."}`, }) if err != nil { - log.Fatalf("failed to create account: %+v", err) + log.Fatalf("failed to create storage: %+v", err) } err = db.CreateUser(&model.User{ Username: "Noah", diff --git a/internal/db/account.go b/internal/db/account.go index 3a0c2437..1d03b10f 100644 --- a/internal/db/account.go +++ b/internal/db/account.go @@ -5,46 +5,46 @@ import ( "github.com/pkg/errors" ) -// why don't need `cache` for account? -// because all account store in `operations.accountsMap` -// the most of the read operation is from `operations.accountsMap` +// why don't need `cache` for storage? +// because all storage store in `operations.storagesMap` +// the most of the read operation is from `operations.storagesMap` // just for persistence in database -// CreateAccount just insert account to database -func CreateAccount(account *model.Account) error { - return errors.WithStack(db.Create(account).Error) +// CreateStorage just insert storage to database +func CreateStorage(storage *model.Storage) error { + return errors.WithStack(db.Create(storage).Error) } -// UpdateAccount just update account in database -func UpdateAccount(account *model.Account) error { - return errors.WithStack(db.Save(account).Error) +// UpdateStorage just update storage in database +func UpdateStorage(storage *model.Storage) error { + return errors.WithStack(db.Save(storage).Error) } -// DeleteAccountById just delete account from database by id -func DeleteAccountById(id uint) error { - return errors.WithStack(db.Delete(&model.Account{}, id).Error) +// DeleteStorageById just delete storage from database by id +func DeleteStorageById(id uint) error { + return errors.WithStack(db.Delete(&model.Storage{}, id).Error) } -// GetAccounts Get all accounts from database order by index -func GetAccounts(pageIndex, pageSize int) ([]model.Account, int64, error) { - accountDB := db.Model(&model.Account{}) +// GetStorages Get all storages from database order by index +func GetStorages(pageIndex, pageSize int) ([]model.Storage, int64, error) { + storageDB := db.Model(&model.Storage{}) var count int64 - if err := accountDB.Count(&count).Error; err != nil { - return nil, 0, errors.Wrapf(err, "failed get accounts count") + if err := storageDB.Count(&count).Error; err != nil { + return nil, 0, errors.Wrapf(err, "failed get storages count") } - var accounts []model.Account - if err := accountDB.Order(columnName("index")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&accounts).Error; err != nil { + var storages []model.Storage + if err := storageDB.Order(columnName("index")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&storages).Error; err != nil { return nil, 0, errors.WithStack(err) } - return accounts, count, nil + return storages, count, nil } -// GetAccountById Get Account by id, used to update account usually -func GetAccountById(id uint) (*model.Account, error) { - var account model.Account - account.ID = id - if err := db.First(&account).Error; err != nil { +// GetStorageById Get Storage by id, used to update storage usually +func GetStorageById(id uint) (*model.Storage, error) { + var storage model.Storage + storage.ID = id + if err := db.First(&storage).Error; err != nil { return nil, errors.WithStack(err) } - return &account, nil + return &storage, nil } diff --git a/internal/db/db.go b/internal/db/db.go index 6e24a3f5..b7f910e3 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -10,7 +10,7 @@ var db gorm.DB func Init(d *gorm.DB) { db = *d - err := db.AutoMigrate(new(model.Account), new(model.User), new(model.Meta), new(model.SettingItem)) + err := db.AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem)) if err != nil { log.Fatalf("failed migrate database: %s", err.Error()) } diff --git a/internal/driver/driver.go b/internal/driver/driver.go index 629d67f9..2e1ce82f 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -17,10 +17,10 @@ type Meta interface { Config() Config // Init If already initialized, drop first // need to unmarshal string to addition first - Init(ctx context.Context, account model.Account) error + Init(ctx context.Context, storage model.Storage) error Drop(ctx context.Context) error - // GetAccount just get raw account - GetAccount() model.Account + // GetStorage just get raw storage + GetStorage() model.Storage GetAddition() Additional } diff --git a/internal/errs/errors.go b/internal/errs/errors.go index fc6051d6..3fa31569 100644 --- a/internal/errs/errors.go +++ b/internal/errs/errors.go @@ -9,7 +9,7 @@ var ( NotSupport = errors.New("not support") RelativePath = errors.New("access using relative path is not allowed") - MoveBetweenTwoAccounts = errors.New("can't move files between two account, try to copy") + MoveBetweenTwoStorages = errors.New("can't move files between two storages, try to copy") UploadNotSupported = errors.New("upload not supported") MetaNotFound = errors.New("meta not found") diff --git a/internal/fs/copy.go b/internal/fs/copy.go index 1e53f3f7..51660d9b 100644 --- a/internal/fs/copy.go +++ b/internal/fs/copy.go @@ -19,40 +19,40 @@ var CopyTaskManager = task.NewTaskManager(3, func(tid *uint64) { atomic.AddUint64(tid, 1) }) -// Copy if in an account, call move method +// Copy if in the same storage, call move method // if not, add copy task func _copy(ctx context.Context, srcObjPath, dstDirPath string) (bool, error) { - srcAccount, srcObjActualPath, err := operations.GetAccountAndActualPath(srcObjPath) + srcStorage, srcObjActualPath, err := operations.GetStorageAndActualPath(srcObjPath) if err != nil { - return false, errors.WithMessage(err, "failed get src account") + return false, errors.WithMessage(err, "failed get src storage") } - dstAccount, dstDirActualPath, err := operations.GetAccountAndActualPath(dstDirPath) + dstStorage, dstDirActualPath, err := operations.GetStorageAndActualPath(dstDirPath) if err != nil { - return false, errors.WithMessage(err, "failed get dst account") + return false, errors.WithMessage(err, "failed get dst storage") } - // copy if in an account, just call driver.Copy - if srcAccount.GetAccount() == dstAccount.GetAccount() { - return false, operations.Copy(ctx, srcAccount, srcObjActualPath, dstDirActualPath) + // copy if in the same storage, just call driver.Copy + if srcStorage.GetStorage() == dstStorage.GetStorage() { + return false, operations.Copy(ctx, srcStorage, srcObjActualPath, dstDirActualPath) } - // not in an account + // not in the same storage CopyTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{ - Name: fmt.Sprintf("copy [%s](%s) to [%s](%s)", srcAccount.GetAccount().VirtualPath, srcObjActualPath, dstAccount.GetAccount().VirtualPath, dstDirActualPath), + Name: fmt.Sprintf("copy [%s](%s) to [%s](%s)", srcStorage.GetStorage().VirtualPath, srcObjActualPath, dstStorage.GetStorage().VirtualPath, dstDirActualPath), Func: func(task *task.Task[uint64]) error { - return copyBetween2Accounts(task, srcAccount, dstAccount, srcObjActualPath, dstDirActualPath) + return copyBetween2Storages(task, srcStorage, dstStorage, srcObjActualPath, dstDirActualPath) }, })) return true, nil } -func copyBetween2Accounts(t *task.Task[uint64], srcAccount, dstAccount driver.Driver, srcObjPath, dstDirPath string) error { +func copyBetween2Storages(t *task.Task[uint64], srcStorage, dstStorage driver.Driver, srcObjPath, dstDirPath string) error { t.SetStatus("getting src object") - srcObj, err := operations.Get(t.Ctx, srcAccount, srcObjPath) + srcObj, err := operations.Get(t.Ctx, srcStorage, srcObjPath) if err != nil { return errors.WithMessagef(err, "failed get src [%s] file", srcObjPath) } if srcObj.IsDir() { t.SetStatus("src object is dir, listing objs") - objs, err := operations.List(t.Ctx, srcAccount, srcObjPath) + objs, err := operations.List(t.Ctx, srcStorage, srcObjPath) if err != nil { return errors.WithMessagef(err, "failed list src [%s] objs", srcObjPath) } @@ -63,29 +63,29 @@ func copyBetween2Accounts(t *task.Task[uint64], srcAccount, dstAccount driver.Dr srcObjPath := stdpath.Join(srcObjPath, obj.GetName()) dstObjPath := stdpath.Join(dstDirPath, obj.GetName()) CopyTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{ - Name: fmt.Sprintf("copy [%s](%s) to [%s](%s)", srcAccount.GetAccount().VirtualPath, srcObjPath, dstAccount.GetAccount().VirtualPath, dstObjPath), + Name: fmt.Sprintf("copy [%s](%s) to [%s](%s)", srcStorage.GetStorage().VirtualPath, srcObjPath, dstStorage.GetStorage().VirtualPath, dstObjPath), Func: func(t *task.Task[uint64]) error { - return copyBetween2Accounts(t, srcAccount, dstAccount, srcObjPath, dstObjPath) + return copyBetween2Storages(t, srcStorage, dstStorage, srcObjPath, dstObjPath) }, })) } } else { CopyTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{ - Name: fmt.Sprintf("copy [%s](%s) to [%s](%s)", srcAccount.GetAccount().VirtualPath, srcObjPath, dstAccount.GetAccount().VirtualPath, dstDirPath), + Name: fmt.Sprintf("copy [%s](%s) to [%s](%s)", srcStorage.GetStorage().VirtualPath, srcObjPath, dstStorage.GetStorage().VirtualPath, dstDirPath), Func: func(t *task.Task[uint64]) error { - return copyFileBetween2Accounts(t, srcAccount, dstAccount, srcObjPath, dstDirPath) + return copyFileBetween2Storages(t, srcStorage, dstStorage, srcObjPath, dstDirPath) }, })) } return nil } -func copyFileBetween2Accounts(tsk *task.Task[uint64], srcAccount, dstAccount driver.Driver, srcFilePath, dstDirPath string) error { - srcFile, err := operations.Get(tsk.Ctx, srcAccount, srcFilePath) +func copyFileBetween2Storages(tsk *task.Task[uint64], srcStorage, dstStorage driver.Driver, srcFilePath, dstDirPath string) error { + srcFile, err := operations.Get(tsk.Ctx, srcStorage, srcFilePath) if err != nil { return errors.WithMessagef(err, "failed get src [%s] file", srcFilePath) } - link, _, err := operations.Link(tsk.Ctx, srcAccount, srcFilePath, model.LinkArgs{}) + link, _, err := operations.Link(tsk.Ctx, srcStorage, srcFilePath, model.LinkArgs{}) if err != nil { return errors.WithMessagef(err, "failed get [%s] link", srcFilePath) } @@ -93,5 +93,5 @@ func copyFileBetween2Accounts(tsk *task.Task[uint64], srcAccount, dstAccount dri if err != nil { return errors.WithMessagef(err, "failed get [%s] stream", srcFilePath) } - return operations.Put(tsk.Ctx, dstAccount, dstDirPath, stream, tsk.SetProgress) + return operations.Put(tsk.Ctx, dstStorage, dstDirPath, stream, tsk.SetProgress) } diff --git a/internal/fs/fs.go b/internal/fs/fs.go index c898789a..b811e9be 100644 --- a/internal/fs/fs.go +++ b/internal/fs/fs.go @@ -95,10 +95,10 @@ func PutAsTask(dstDirPath string, file model.FileStreamer) error { return err } -func GetAccount(path string) (driver.Driver, error) { - accountDriver, _, err := operations.GetAccountAndActualPath(path) +func GetStorage(path string) (driver.Driver, error) { + storageDriver, _, err := operations.GetStorageAndActualPath(path) if err != nil { return nil, err } - return accountDriver, nil + return storageDriver, nil } diff --git a/internal/fs/get.go b/internal/fs/get.go index a41bb89c..aedd799d 100644 --- a/internal/fs/get.go +++ b/internal/fs/get.go @@ -14,16 +14,16 @@ func get(ctx context.Context, path string) (model.Obj, error) { path = utils.StandardizePath(path) // maybe a virtual file if path != "/" { - virtualFiles := operations.GetAccountVirtualFilesByPath(stdpath.Dir(path)) + virtualFiles := operations.GetStorageVirtualFilesByPath(stdpath.Dir(path)) for _, f := range virtualFiles { if f.GetName() == stdpath.Base(path) { return f, nil } } } - account, actualPath, err := operations.GetAccountAndActualPath(path) + storage, actualPath, err := operations.GetStorageAndActualPath(path) if err != nil { - // if there are no account prefix with path, maybe root folder + // if there are no storage prefix with path, maybe root folder if path == "/" { return model.Object{ Name: "root", @@ -32,7 +32,7 @@ func get(ctx context.Context, path string) (model.Obj, error) { IsFolder: true, }, nil } - return nil, errors.WithMessage(err, "failed get account") + return nil, errors.WithMessage(err, "failed get storage") } - return operations.Get(ctx, account, actualPath) + return operations.Get(ctx, storage, actualPath) } diff --git a/internal/fs/link.go b/internal/fs/link.go index 9b971a99..12e55478 100644 --- a/internal/fs/link.go +++ b/internal/fs/link.go @@ -8,9 +8,9 @@ import ( ) func link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, model.Obj, error) { - account, actualPath, err := operations.GetAccountAndActualPath(path) + storage, actualPath, err := operations.GetStorageAndActualPath(path) if err != nil { - return nil, nil, errors.WithMessage(err, "failed get account") + return nil, nil, errors.WithMessage(err, "failed get storage") } - return operations.Link(ctx, account, actualPath, args) + return operations.Link(ctx, storage, actualPath, args) } diff --git a/internal/fs/list.go b/internal/fs/list.go index 38a8b0cb..5947c783 100644 --- a/internal/fs/list.go +++ b/internal/fs/list.go @@ -15,15 +15,15 @@ import ( func list(ctx context.Context, path string) ([]model.Obj, error) { meta := ctx.Value("meta").(*model.Meta) user := ctx.Value("user").(*model.User) - account, actualPath, err := operations.GetAccountAndActualPath(path) - virtualFiles := operations.GetAccountVirtualFilesByPath(path) + storage, actualPath, err := operations.GetStorageAndActualPath(path) + virtualFiles := operations.GetStorageVirtualFilesByPath(path) if err != nil { if len(virtualFiles) != 0 { return virtualFiles, nil } - return nil, errors.WithMessage(err, "failed get account") + return nil, errors.WithMessage(err, "failed get storage") } - objs, err := operations.List(ctx, account, actualPath) + objs, err := operations.List(ctx, storage, actualPath) if err != nil { log.Errorf("%+v", err) if len(virtualFiles) != 0 { @@ -31,19 +31,19 @@ func list(ctx context.Context, path string) ([]model.Obj, error) { } return nil, errors.WithMessage(err, "failed get objs") } - for _, accountFile := range virtualFiles { - if !containsByName(objs, accountFile) { - objs = append(objs, accountFile) + for _, storageFile := range virtualFiles { + if !containsByName(objs, storageFile) { + objs = append(objs, storageFile) } } if whetherHide(user, meta, path) { objs = hide(objs, meta) } // sort objs - if account.Config().LocalSort { - model.SortFiles(objs, account.GetAccount().OrderBy, account.GetAccount().OrderDirection) + if storage.Config().LocalSort { + model.SortFiles(objs, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection) } - model.ExtractFolder(objs, account.GetAccount().ExtractFolder) + model.ExtractFolder(objs, storage.GetStorage().ExtractFolder) return objs, nil } diff --git a/internal/fs/put.go b/internal/fs/put.go index 46bf47b6..6155e2f0 100644 --- a/internal/fs/put.go +++ b/internal/fs/put.go @@ -18,12 +18,12 @@ var UploadTaskManager = task.NewTaskManager[uint64](3, func(tid *uint64) { // putAsTask add as a put task and return immediately func putAsTask(dstDirPath string, file model.FileStreamer) error { - account, dstDirActualPath, err := operations.GetAccountAndActualPath(dstDirPath) - if account.Config().NoUpload { + storage, dstDirActualPath, err := operations.GetStorageAndActualPath(dstDirPath) + if storage.Config().NoUpload { return errors.WithStack(errs.UploadNotSupported) } if err != nil { - return errors.WithMessage(err, "failed get account") + return errors.WithMessage(err, "failed get storage") } if file.NeedStore() { tempFile, err := utils.CreateTempFile(file) @@ -33,9 +33,9 @@ func putAsTask(dstDirPath string, file model.FileStreamer) error { file.SetReadCloser(tempFile) } UploadTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{ - Name: fmt.Sprintf("upload %s to [%s](%s)", file.GetName(), account.GetAccount().VirtualPath, dstDirActualPath), + Name: fmt.Sprintf("upload %s to [%s](%s)", file.GetName(), storage.GetStorage().VirtualPath, dstDirActualPath), Func: func(task *task.Task[uint64]) error { - return operations.Put(task.Ctx, account, dstDirActualPath, file, nil) + return operations.Put(task.Ctx, storage, dstDirActualPath, file, nil) }, })) return nil @@ -43,12 +43,12 @@ func putAsTask(dstDirPath string, file model.FileStreamer) error { // putDirect put the file and return after finish func putDirectly(ctx context.Context, dstDirPath string, file model.FileStreamer) error { - account, dstDirActualPath, err := operations.GetAccountAndActualPath(dstDirPath) - if account.Config().NoUpload { + storage, dstDirActualPath, err := operations.GetStorageAndActualPath(dstDirPath) + if storage.Config().NoUpload { return errors.WithStack(errs.UploadNotSupported) } if err != nil { - return errors.WithMessage(err, "failed get account") + return errors.WithMessage(err, "failed get storage") } - return operations.Put(ctx, account, dstDirActualPath, file, nil) + return operations.Put(ctx, storage, dstDirActualPath, file, nil) } diff --git a/internal/fs/util.go b/internal/fs/util.go index bca19097..4a6e1787 100644 --- a/internal/fs/util.go +++ b/internal/fs/util.go @@ -14,11 +14,11 @@ import ( ) func ClearCache(path string) { - account, actualPath, err := operations.GetAccountAndActualPath(path) + storage, actualPath, err := operations.GetStorageAndActualPath(path) if err != nil { return } - operations.ClearCache(account, actualPath) + operations.ClearCache(storage, actualPath) } func containsByName(files []model.Obj, file model.Obj) bool { diff --git a/internal/fs/write.go b/internal/fs/write.go index 531b17b4..624414aa 100644 --- a/internal/fs/write.go +++ b/internal/fs/write.go @@ -8,40 +8,40 @@ import ( ) func makeDir(ctx context.Context, path string) error { - account, actualPath, err := operations.GetAccountAndActualPath(path) + storage, actualPath, err := operations.GetStorageAndActualPath(path) if err != nil { - return errors.WithMessage(err, "failed get account") + return errors.WithMessage(err, "failed get storage") } - return operations.MakeDir(ctx, account, actualPath) + return operations.MakeDir(ctx, storage, actualPath) } func move(ctx context.Context, srcPath, dstDirPath string) error { - srcAccount, srcActualPath, err := operations.GetAccountAndActualPath(srcPath) + srcStorage, srcActualPath, err := operations.GetStorageAndActualPath(srcPath) if err != nil { - return errors.WithMessage(err, "failed get src account") + return errors.WithMessage(err, "failed get src storage") } - dstAccount, dstDirActualPath, err := operations.GetAccountAndActualPath(dstDirPath) + dstStorage, dstDirActualPath, err := operations.GetStorageAndActualPath(dstDirPath) if err != nil { - return errors.WithMessage(err, "failed get dst account") + return errors.WithMessage(err, "failed get dst storage") } - if srcAccount.GetAccount() != dstAccount.GetAccount() { - return errors.WithStack(errs.MoveBetweenTwoAccounts) + if srcStorage.GetStorage() != dstStorage.GetStorage() { + return errors.WithStack(errs.MoveBetweenTwoStorages) } - return operations.Move(ctx, srcAccount, srcActualPath, dstDirActualPath) + return operations.Move(ctx, srcStorage, srcActualPath, dstDirActualPath) } func rename(ctx context.Context, srcPath, dstName string) error { - account, srcActualPath, err := operations.GetAccountAndActualPath(srcPath) + storage, srcActualPath, err := operations.GetStorageAndActualPath(srcPath) if err != nil { - return errors.WithMessage(err, "failed get account") + return errors.WithMessage(err, "failed get storage") } - return operations.Rename(ctx, account, srcActualPath, dstName) + return operations.Rename(ctx, storage, srcActualPath, dstName) } func remove(ctx context.Context, path string) error { - account, actualPath, err := operations.GetAccountAndActualPath(path) + storage, actualPath, err := operations.GetStorageAndActualPath(path) if err != nil { - return errors.WithMessage(err, "failed get account") + return errors.WithMessage(err, "failed get storage") } - return operations.Remove(ctx, account, actualPath) + return operations.Remove(ctx, storage, actualPath) } diff --git a/internal/model/account.go b/internal/model/storage.go similarity index 91% rename from internal/model/account.go rename to internal/model/storage.go index 27482503..1e4327be 100644 --- a/internal/model/account.go +++ b/internal/model/storage.go @@ -2,7 +2,7 @@ package model import "time" -type Account struct { +type Storage struct { ID uint `json:"id" gorm:"primaryKey"` // unique key VirtualPath string `json:"virtual_path" gorm:"unique" binding:"required"` // must be standardized Index int `json:"index"` // use to sort @@ -27,11 +27,11 @@ type Proxy struct { DownProxyUrl string `json:"down_proxy_url"` } -func (a Account) GetAccount() Account { +func (a Storage) GetStorage() Storage { return a } -func (a *Account) SetStatus(status string) { +func (a *Storage) SetStatus(status string) { a.Status = status } diff --git a/internal/operations/account.go b/internal/operations/account.go deleted file mode 100644 index 57c9ba74..00000000 --- a/internal/operations/account.go +++ /dev/null @@ -1,245 +0,0 @@ -package operations - -import ( - "context" - log "github.com/sirupsen/logrus" - "sort" - "strings" - "time" - - "github.com/alist-org/alist/v3/internal/db" - "github.com/alist-org/alist/v3/internal/driver" - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/pkg/generic_sync" - "github.com/alist-org/alist/v3/pkg/utils" - "github.com/pkg/errors" -) - -// Although the driver type is stored, -// there is an account in each driver, -// so it should actually be an account, just wrapped by the driver -var accountsMap generic_sync.MapOf[string, driver.Driver] - -func GetAccountByVirtualPath(virtualPath string) (driver.Driver, error) { - accountDriver, ok := accountsMap.Load(virtualPath) - if !ok { - return nil, errors.Errorf("no virtual path for an account is: %s", virtualPath) - } - return accountDriver, nil -} - -// CreateAccount Save the account to database so account can get an id -// then instantiate corresponding driver and save it in memory -func CreateAccount(ctx context.Context, account model.Account) error { - account.Modified = time.Now() - account.VirtualPath = utils.StandardizePath(account.VirtualPath) - var err error - // check driver first - driverName := account.Driver - driverNew, err := GetDriverNew(driverName) - if err != nil { - return errors.WithMessage(err, "failed get driver new") - } - accountDriver := driverNew() - // insert account to database - err = db.CreateAccount(&account) - if err != nil { - return errors.WithMessage(err, "failed create account in database") - } - // already has an id - err = accountDriver.Init(ctx, account) - if err != nil { - return errors.WithMessage(err, "failed init account but account is already created") - } - log.Debugf("account %+v is created", accountDriver) - accountsMap.Store(account.VirtualPath, accountDriver) - return nil -} - -// UpdateAccount update account -// get old account first -// drop the account then reinitialize -func UpdateAccount(ctx context.Context, account model.Account) error { - oldAccount, err := db.GetAccountById(account.ID) - if err != nil { - return errors.WithMessage(err, "failed get old account") - } - if oldAccount.Driver != account.Driver { - return errors.Errorf("driver cannot be changed") - } - account.Modified = time.Now() - account.VirtualPath = utils.StandardizePath(account.VirtualPath) - err = db.UpdateAccount(&account) - if err != nil { - return errors.WithMessage(err, "failed update account in database") - } - accountDriver, err := GetAccountByVirtualPath(oldAccount.VirtualPath) - if oldAccount.VirtualPath != account.VirtualPath { - // virtual path renamed, need to drop the account - accountsMap.Delete(oldAccount.VirtualPath) - } - if err != nil { - return errors.WithMessage(err, "failed get account driver") - } - err = accountDriver.Drop(ctx) - if err != nil { - return errors.WithMessage(err, "failed drop account") - } - err = accountDriver.Init(ctx, account) - if err != nil { - return errors.WithMessage(err, "failed init account") - } - accountsMap.Store(account.VirtualPath, accountDriver) - return nil -} - -func DeleteAccountById(ctx context.Context, id uint) error { - account, err := db.GetAccountById(id) - if err != nil { - return errors.WithMessage(err, "failed get account") - } - accountDriver, err := GetAccountByVirtualPath(account.VirtualPath) - if err != nil { - return errors.WithMessage(err, "failed get account driver") - } - // drop the account in the driver - if err := accountDriver.Drop(ctx); err != nil { - return errors.WithMessage(err, "failed drop account") - } - // delete the account in the database - if err := db.DeleteAccountById(id); err != nil { - return errors.WithMessage(err, "failed delete account in database") - } - // delete the account in the memory - accountsMap.Delete(account.VirtualPath) - return nil -} - -// MustSaveDriverAccount call from specific driver -func MustSaveDriverAccount(driver driver.Driver) { - err := saveDriverAccount(driver) - if err != nil { - log.Errorf("failed save driver account: %s", err) - } -} - -func saveDriverAccount(driver driver.Driver) error { - account := driver.GetAccount() - addition := driver.GetAddition() - bytes, err := utils.Json.Marshal(addition) - if err != nil { - return errors.Wrap(err, "error while marshal addition") - } - account.Addition = string(bytes) - err = db.UpdateAccount(&account) - if err != nil { - return errors.WithMessage(err, "failed update account in database") - } - return nil -} - -// GetAccountsByPath get account by longest match path, contains balance account. -// for example, there is /a/b,/a/c,/a/d/e,/a/d/e.balance -// GetAccountsByPath(/a/d/e/f) => /a/d/e,/a/d/e.balance -func getAccountsByPath(path string) []driver.Driver { - accounts := make([]driver.Driver, 0) - curSlashCount := 0 - accountsMap.Range(func(key string, value driver.Driver) bool { - virtualPath := utils.GetActualVirtualPath(value.GetAccount().VirtualPath) - if virtualPath == "/" { - virtualPath = "" - } - // not this - if path != virtualPath && !strings.HasPrefix(path, virtualPath+"/") { - return true - } - slashCount := strings.Count(virtualPath, "/") - // not the longest match - if slashCount < curSlashCount { - return true - } - if slashCount > curSlashCount { - accounts = accounts[:0] - curSlashCount = slashCount - } - accounts = append(accounts, value) - return true - }) - // make sure the order is the same for same input - sort.Slice(accounts, func(i, j int) bool { - return accounts[i].GetAccount().VirtualPath < accounts[j].GetAccount().VirtualPath - }) - return accounts -} - -// GetAccountVirtualFilesByPath Obtain the virtual file generated by the account according to the path -// for example, there are: /a/b,/a/c,/a/d/e,/a/b.balance1,/av -// GetAccountVirtualFilesByPath(/a) => b,c,d -func GetAccountVirtualFilesByPath(prefix string) []model.Obj { - files := make([]model.Obj, 0) - accounts := accountsMap.Values() - sort.Slice(accounts, func(i, j int) bool { - if accounts[i].GetAccount().Index == accounts[j].GetAccount().Index { - return accounts[i].GetAccount().VirtualPath < accounts[j].GetAccount().VirtualPath - } - return accounts[i].GetAccount().Index < accounts[j].GetAccount().Index - }) - prefix = utils.StandardizePath(prefix) - if prefix == "/" { - prefix = "" - } - set := make(map[string]interface{}) - for _, v := range accounts { - // TODO should save a balanced account - // balance account - if utils.IsBalance(v.GetAccount().VirtualPath) { - continue - } - virtualPath := v.GetAccount().VirtualPath - if len(virtualPath) <= len(prefix) { - continue - } - // not prefixed with `prefix` - if !strings.HasPrefix(virtualPath, prefix+"/") { - continue - } - name := strings.Split(strings.TrimPrefix(virtualPath, prefix), "/")[1] - if _, ok := set[name]; ok { - continue - } - files = append(files, model.Object{ - Name: name, - Size: 0, - Modified: v.GetAccount().Modified, - }) - set[name] = nil - } - return files -} - -var balanceMap generic_sync.MapOf[string, int] - -// GetBalancedAccount get account by path -func GetBalancedAccount(path string) driver.Driver { - path = utils.StandardizePath(path) - accounts := getAccountsByPath(path) - accountNum := len(accounts) - switch accountNum { - case 0: - return nil - case 1: - return accounts[0] - default: - virtualPath := utils.GetActualVirtualPath(accounts[0].GetAccount().VirtualPath) - cur, ok := balanceMap.Load(virtualPath) - i := 0 - if ok { - i = cur - i = (i + 1) % accountNum - balanceMap.Store(virtualPath, i) - } else { - balanceMap.Store(virtualPath, i) - } - return accounts[i] - } -} diff --git a/internal/operations/account_test.go b/internal/operations/account_test.go deleted file mode 100644 index 462f7016..00000000 --- a/internal/operations/account_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package operations_test - -import ( - "context" - "github.com/alist-org/alist/v3/internal/db" - "testing" - - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/operations" - "github.com/alist-org/alist/v3/pkg/utils" - "gorm.io/driver/sqlite" - "gorm.io/gorm" -) - -func init() { - dB, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{}) - if err != nil { - panic("failed to connect database") - } - db.Init(dB) -} - -func TestCreateAccount(t *testing.T) { - var accounts = []struct { - account model.Account - iserr bool - }{ - {account: model.Account{Driver: "Local", VirtualPath: "/local", Addition: `{"root_folder":"."}`}, iserr: false}, - {account: model.Account{Driver: "Local", VirtualPath: "/local", Addition: `{"root_folder":"."}`}, iserr: true}, - {account: model.Account{Driver: "None", VirtualPath: "/none", Addition: `{"root_folder":"."}`}, iserr: true}, - } - for _, account := range accounts { - err := operations.CreateAccount(context.Background(), account.account) - if err != nil { - if !account.iserr { - t.Errorf("failed to create account: %+v", err) - } else { - t.Logf("expect failed to create account: %+v", err) - } - } - } -} - -func TestGetAccountVirtualFilesByPath(t *testing.T) { - setupAccounts(t) - virtualFiles := operations.GetAccountVirtualFilesByPath("/a") - var names []string - for _, virtualFile := range virtualFiles { - names = append(names, virtualFile.GetName()) - } - var expectedNames = []string{"b", "c", "d"} - if utils.SliceEqual(names, expectedNames) { - t.Logf("passed") - } else { - t.Errorf("expected: %+v, got: %+v", expectedNames, names) - } -} - -func TestGetBalancedAccount(t *testing.T) { - setupAccounts(t) - account := operations.GetBalancedAccount("/a/d/e") - if account.GetAccount().VirtualPath != "/a/d/e" { - t.Errorf("expected: /a/d/e, got: %+v", account.GetAccount().VirtualPath) - } - account = operations.GetBalancedAccount("/a/d/e") - if account.GetAccount().VirtualPath != "/a/d/e.balance" { - t.Errorf("expected: /a/d/e.balance, got: %+v", account.GetAccount().VirtualPath) - } -} - -func setupAccounts(t *testing.T) { - var accounts = []model.Account{ - {Driver: "Local", VirtualPath: "/a/b", Index: 0, Addition: `{"root_folder":"."}`}, - {Driver: "Local", VirtualPath: "/a/c", Index: 1, Addition: `{"root_folder":"."}`}, - {Driver: "Local", VirtualPath: "/a/d", Index: 2, Addition: `{"root_folder":"."}`}, - {Driver: "Local", VirtualPath: "/a/d/e", Index: 3, Addition: `{"root_folder":"."}`}, - {Driver: "Local", VirtualPath: "/a/d/e.balance", Index: 4, Addition: `{"root_folder":"."}`}, - } - for _, account := range accounts { - err := operations.CreateAccount(context.Background(), account) - if err != nil { - t.Fatalf("failed to create account: %+v", err) - } - } -} diff --git a/internal/operations/fs.go b/internal/operations/fs.go index f0ff941b..bf5bc67c 100644 --- a/internal/operations/fs.go +++ b/internal/operations/fs.go @@ -23,37 +23,37 @@ import ( var filesCache = cache.NewMemCache(cache.WithShards[[]model.Obj](64)) var filesG singleflight.Group[[]model.Obj] -func ClearCache(account driver.Driver, path string) { - key := stdpath.Join(account.GetAccount().VirtualPath, path) +func ClearCache(storage driver.Driver, path string) { + key := stdpath.Join(storage.GetStorage().VirtualPath, path) filesCache.Del(key) } // List files in storage, not contains virtual file -func List(ctx context.Context, account driver.Driver, path string, refresh ...bool) ([]model.Obj, error) { +func List(ctx context.Context, storage driver.Driver, path string, refresh ...bool) ([]model.Obj, error) { path = utils.StandardizePath(path) log.Debugf("operations.List %s", path) - dir, err := Get(ctx, account, path) + dir, err := Get(ctx, storage, path) if err != nil { return nil, errors.WithMessage(err, "failed get dir") } if !dir.IsDir() { return nil, errors.WithStack(errs.NotFolder) } - if account.Config().NoCache { - return account.List(ctx, dir) + if storage.Config().NoCache { + return storage.List(ctx, dir) } - key := stdpath.Join(account.GetAccount().VirtualPath, path) + key := stdpath.Join(storage.GetStorage().VirtualPath, path) if len(refresh) == 0 || !refresh[0] { if files, ok := filesCache.Get(key); ok { return files, nil } } files, err, _ := filesG.Do(key, func() ([]model.Obj, error) { - files, err := account.List(ctx, dir) + files, err := storage.List(ctx, dir) if err != nil { return nil, errors.WithMessage(err, "failed to list files") } - // TODO: maybe can get duration from account's config + // TODO: maybe can get duration from storage's config filesCache.Set(key, files, cache.WithEx[[]model.Obj](time.Minute*time.Duration(conf.Conf.CaCheExpiration))) return files, nil }) @@ -74,34 +74,34 @@ func isRoot(path, rootFolderPath string) bool { } // Get object from list of files -func Get(ctx context.Context, account driver.Driver, path string) (model.Obj, error) { +func Get(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) { path = utils.StandardizePath(path) log.Debugf("operations.Get %s", path) - if g, ok := account.(driver.Getter); ok { + if g, ok := storage.(driver.Getter); ok { return g.Get(ctx, path) } // is root folder - if r, ok := account.GetAddition().(driver.IRootFolderId); ok && utils.PathEqual(path, "/") { + if r, ok := storage.GetAddition().(driver.IRootFolderId); ok && utils.PathEqual(path, "/") { return model.Object{ ID: r.GetRootFolderId(), Name: "root", Size: 0, - Modified: account.GetAccount().Modified, + Modified: storage.GetStorage().Modified, IsFolder: true, }, nil } - if r, ok := account.GetAddition().(driver.IRootFolderPath); ok && isRoot(path, r.GetRootFolderPath()) { + if r, ok := storage.GetAddition().(driver.IRootFolderPath); ok && isRoot(path, r.GetRootFolderPath()) { return model.Object{ ID: r.GetRootFolderPath(), Name: "root", Size: 0, - Modified: account.GetAccount().Modified, + Modified: storage.GetStorage().Modified, IsFolder: true, }, nil } // not root folder dir, name := stdpath.Split(path) - files, err := List(ctx, account, dir) + files, err := List(ctx, storage, dir) if err != nil { return nil, errors.WithMessage(err, "failed get parent list") } @@ -124,20 +124,20 @@ var linkCache = cache.NewMemCache(cache.WithShards[*model.Link](16)) var linkG singleflight.Group[*model.Link] // Link get link, if is an url. should have an expiry time -func Link(ctx context.Context, account driver.Driver, path string, args model.LinkArgs) (*model.Link, model.Obj, error) { - file, err := Get(ctx, account, path) +func Link(ctx context.Context, storage driver.Driver, path string, args model.LinkArgs) (*model.Link, model.Obj, error) { + file, err := Get(ctx, storage, path) if err != nil { return nil, nil, errors.WithMessage(err, "failed to get file") } if file.IsDir() { return nil, nil, errors.WithStack(errs.NotFile) } - key := stdpath.Join(account.GetAccount().VirtualPath, path) + key := stdpath.Join(storage.GetStorage().VirtualPath, path) if link, ok := linkCache.Get(key); ok { return link, file, nil } fn := func() (*model.Link, error) { - link, err := account.Link(ctx, file, args) + link, err := storage.Link(ctx, file, args) if err != nil { return nil, errors.WithMessage(err, "failed get link") } @@ -150,22 +150,22 @@ func Link(ctx context.Context, account driver.Driver, path string, args model.Li return link, file, err } -func MakeDir(ctx context.Context, account driver.Driver, path string) error { +func MakeDir(ctx context.Context, storage driver.Driver, path string) error { // check if dir exists - f, err := Get(ctx, account, path) + f, err := Get(ctx, storage, path) if err != nil { if errs.IsObjectNotFound(err) { parentPath, dirName := stdpath.Split(path) - err = MakeDir(ctx, account, parentPath) + err = MakeDir(ctx, storage, parentPath) if err != nil { return errors.WithMessagef(err, "failed to make parent dir [%s]", parentPath) } - parentDir, err := Get(ctx, account, parentPath) + parentDir, err := Get(ctx, storage, parentPath) // this should not happen if err != nil { return errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath) } - return account.MakeDir(ctx, parentDir, dirName) + return storage.MakeDir(ctx, parentDir, dirName) } else { return errors.WithMessage(err, "failed to check if dir exists") } @@ -180,38 +180,38 @@ func MakeDir(ctx context.Context, account driver.Driver, path string) error { } } -func Move(ctx context.Context, account driver.Driver, srcPath, dstDirPath string) error { - srcObj, err := Get(ctx, account, srcPath) +func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string) error { + srcObj, err := Get(ctx, storage, srcPath) if err != nil { return errors.WithMessage(err, "failed to get src object") } - dstDir, err := Get(ctx, account, dstDirPath) + dstDir, err := Get(ctx, storage, dstDirPath) if err != nil { return errors.WithMessage(err, "failed to get dst dir") } - return account.Move(ctx, srcObj, dstDir) + return storage.Move(ctx, srcObj, dstDir) } -func Rename(ctx context.Context, account driver.Driver, srcPath, dstName string) error { - srcObj, err := Get(ctx, account, srcPath) +func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string) error { + srcObj, err := Get(ctx, storage, srcPath) if err != nil { return errors.WithMessage(err, "failed to get src object") } - return account.Rename(ctx, srcObj, dstName) + return storage.Rename(ctx, srcObj, dstName) } -// Copy Just copy file[s] in an account -func Copy(ctx context.Context, account driver.Driver, srcPath, dstDirPath string) error { - srcObj, err := Get(ctx, account, srcPath) +// Copy Just copy file[s] in a storage +func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string) error { + srcObj, err := Get(ctx, storage, srcPath) if err != nil { return errors.WithMessage(err, "failed to get src object") } - dstDir, err := Get(ctx, account, dstDirPath) - return account.Copy(ctx, srcObj, dstDir) + dstDir, err := Get(ctx, storage, dstDirPath) + return storage.Copy(ctx, srcObj, dstDir) } -func Remove(ctx context.Context, account driver.Driver, path string) error { - obj, err := Get(ctx, account, path) +func Remove(ctx context.Context, storage driver.Driver, path string) error { + obj, err := Get(ctx, storage, path) if err != nil { // if object not found, it's ok if errs.IsObjectNotFound(err) { @@ -219,10 +219,10 @@ func Remove(ctx context.Context, account driver.Driver, path string) error { } return errors.WithMessage(err, "failed to get object") } - return account.Remove(ctx, obj) + return storage.Remove(ctx, obj) } -func Put(ctx context.Context, account driver.Driver, dstDirPath string, file model.FileStreamer, up driver.UpdateProgress) error { +func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file model.FileStreamer, up driver.UpdateProgress) error { defer func() { if f, ok := file.GetReadCloser().(*os.File); ok { err := os.RemoveAll(f.Name()) @@ -236,11 +236,11 @@ func Put(ctx context.Context, account driver.Driver, dstDirPath string, file mod log.Errorf("failed to close file streamer, %v", err) } }() - err := MakeDir(ctx, account, dstDirPath) + err := MakeDir(ctx, storage, dstDirPath) if err != nil { return errors.WithMessagef(err, "failed to make dir [%s]", dstDirPath) } - parentDir, err := Get(ctx, account, dstDirPath) + parentDir, err := Get(ctx, storage, dstDirPath) // this should not happen if err != nil { return errors.WithMessagef(err, "failed to get dir [%s]", dstDirPath) @@ -249,11 +249,11 @@ func Put(ctx context.Context, account driver.Driver, dstDirPath string, file mod if up == nil { up = func(p int) {} } - err = account.Put(ctx, parentDir, file, up) + err = storage.Put(ctx, parentDir, file, up) log.Debugf("put file [%s] done", file.GetName()) if err == nil { // clear cache - key := stdpath.Join(account.GetAccount().VirtualPath, dstDirPath) + key := stdpath.Join(storage.GetStorage().VirtualPath, dstDirPath) filesCache.Del(key) } return err diff --git a/internal/operations/path.go b/internal/operations/path.go index cb6bd429..d2a69a8e 100644 --- a/internal/operations/path.go +++ b/internal/operations/path.go @@ -13,27 +13,27 @@ import ( // ActualPath Get the actual path // !!! maybe and \ in the path when use windows local -func ActualPath(account driver.Additional, rawPath string) string { - if i, ok := account.(driver.IRootFolderPath); ok { +func ActualPath(storage driver.Additional, rawPath string) string { + if i, ok := storage.(driver.IRootFolderPath); ok { rawPath = stdpath.Join(i.GetRootFolderPath(), rawPath) } return utils.StandardizePath(rawPath) } -// GetAccountAndActualPath Get the corresponding account +// GetStorageAndActualPath Get the corresponding storage and actual path // for path: remove the virtual path prefix and join the actual root folder if exists -func GetAccountAndActualPath(rawPath string) (driver.Driver, string, error) { +func GetStorageAndActualPath(rawPath string) (driver.Driver, string, error) { rawPath = utils.StandardizePath(rawPath) if strings.Contains(rawPath, "..") { return nil, "", errors.WithStack(errs.RelativePath) } - account := GetBalancedAccount(rawPath) - if account == nil { - return nil, "", errors.Errorf("can't find account with rawPath: %s", rawPath) + storage := GetBalancedStorage(rawPath) + if storage == nil { + return nil, "", errors.Errorf("can't find storage with rawPath: %s", rawPath) } - log.Debugln("use account: ", account.GetAccount().VirtualPath) - virtualPath := utils.GetActualVirtualPath(account.GetAccount().VirtualPath) + log.Debugln("use storage: ", storage.GetStorage().VirtualPath) + virtualPath := utils.GetActualVirtualPath(storage.GetStorage().VirtualPath) actualPath := strings.TrimPrefix(rawPath, virtualPath) - actualPath = ActualPath(account.GetAddition(), actualPath) - return account, actualPath, nil + actualPath = ActualPath(storage.GetAddition(), actualPath) + return storage, actualPath, nil } diff --git a/internal/operations/storage.go b/internal/operations/storage.go new file mode 100644 index 00000000..80ce127d --- /dev/null +++ b/internal/operations/storage.go @@ -0,0 +1,245 @@ +package operations + +import ( + "context" + log "github.com/sirupsen/logrus" + "sort" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/generic_sync" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/pkg/errors" +) + +// Although the driver type is stored, +// there is a storage in each driver, +// so it should actually be a storage, just wrapped by the driver +var storagesMap generic_sync.MapOf[string, driver.Driver] + +func GetStorageByVirtualPath(virtualPath string) (driver.Driver, error) { + storageDriver, ok := storagesMap.Load(virtualPath) + if !ok { + return nil, errors.Errorf("no virtual path for an storage is: %s", virtualPath) + } + return storageDriver, nil +} + +// CreateStorage Save the storage to database so storage can get an id +// then instantiate corresponding driver and save it in memory +func CreateStorage(ctx context.Context, storage model.Storage) error { + storage.Modified = time.Now() + storage.VirtualPath = utils.StandardizePath(storage.VirtualPath) + var err error + // check driver first + driverName := storage.Driver + driverNew, err := GetDriverNew(driverName) + if err != nil { + return errors.WithMessage(err, "failed get driver new") + } + storageDriver := driverNew() + // insert storage to database + err = db.CreateStorage(&storage) + if err != nil { + return errors.WithMessage(err, "failed create storage in database") + } + // already has an id + err = storageDriver.Init(ctx, storage) + if err != nil { + return errors.WithMessage(err, "failed init storage but storage is already created") + } + log.Debugf("storage %+v is created", storageDriver) + storagesMap.Store(storage.VirtualPath, storageDriver) + return nil +} + +// UpdateStorage update storage +// get old storage first +// drop the storage then reinitialize +func UpdateStorage(ctx context.Context, storage model.Storage) error { + oldStorage, err := db.GetStorageById(storage.ID) + if err != nil { + return errors.WithMessage(err, "failed get old storage") + } + if oldStorage.Driver != storage.Driver { + return errors.Errorf("driver cannot be changed") + } + storage.Modified = time.Now() + storage.VirtualPath = utils.StandardizePath(storage.VirtualPath) + err = db.UpdateStorage(&storage) + if err != nil { + return errors.WithMessage(err, "failed update storage in database") + } + storageDriver, err := GetStorageByVirtualPath(oldStorage.VirtualPath) + if oldStorage.VirtualPath != storage.VirtualPath { + // virtual path renamed, need to drop the storage + storagesMap.Delete(oldStorage.VirtualPath) + } + if err != nil { + return errors.WithMessage(err, "failed get storage driver") + } + err = storageDriver.Drop(ctx) + if err != nil { + return errors.WithMessage(err, "failed drop storage") + } + err = storageDriver.Init(ctx, storage) + if err != nil { + return errors.WithMessage(err, "failed init storage") + } + storagesMap.Store(storage.VirtualPath, storageDriver) + return nil +} + +func DeleteStorageById(ctx context.Context, id uint) error { + storage, err := db.GetStorageById(id) + if err != nil { + return errors.WithMessage(err, "failed get storage") + } + storageDriver, err := GetStorageByVirtualPath(storage.VirtualPath) + if err != nil { + return errors.WithMessage(err, "failed get storage driver") + } + // drop the storage in the driver + if err := storageDriver.Drop(ctx); err != nil { + return errors.WithMessage(err, "failed drop storage") + } + // delete the storage in the database + if err := db.DeleteStorageById(id); err != nil { + return errors.WithMessage(err, "failed delete storage in database") + } + // delete the storage in the memory + storagesMap.Delete(storage.VirtualPath) + return nil +} + +// MustSaveDriverStorage call from specific driver +func MustSaveDriverStorage(driver driver.Driver) { + err := saveDriverStorage(driver) + if err != nil { + log.Errorf("failed save driver storage: %s", err) + } +} + +func saveDriverStorage(driver driver.Driver) error { + storage := driver.GetStorage() + addition := driver.GetAddition() + bytes, err := utils.Json.Marshal(addition) + if err != nil { + return errors.Wrap(err, "error while marshal addition") + } + storage.Addition = string(bytes) + err = db.UpdateStorage(&storage) + if err != nil { + return errors.WithMessage(err, "failed update storage in database") + } + return nil +} + +// getStoragesByPath get storage by longest match path, contains balance storage. +// for example, there is /a/b,/a/c,/a/d/e,/a/d/e.balance +// getStoragesByPath(/a/d/e/f) => /a/d/e,/a/d/e.balance +func getStoragesByPath(path string) []driver.Driver { + storages := make([]driver.Driver, 0) + curSlashCount := 0 + storagesMap.Range(func(key string, value driver.Driver) bool { + virtualPath := utils.GetActualVirtualPath(value.GetStorage().VirtualPath) + if virtualPath == "/" { + virtualPath = "" + } + // not this + if path != virtualPath && !strings.HasPrefix(path, virtualPath+"/") { + return true + } + slashCount := strings.Count(virtualPath, "/") + // not the longest match + if slashCount < curSlashCount { + return true + } + if slashCount > curSlashCount { + storages = storages[:0] + curSlashCount = slashCount + } + storages = append(storages, value) + return true + }) + // make sure the order is the same for same input + sort.Slice(storages, func(i, j int) bool { + return storages[i].GetStorage().VirtualPath < storages[j].GetStorage().VirtualPath + }) + return storages +} + +// GetStorageVirtualFilesByPath Obtain the virtual file generated by the storage according to the path +// for example, there are: /a/b,/a/c,/a/d/e,/a/b.balance1,/av +// GetStorageVirtualFilesByPath(/a) => b,c,d +func GetStorageVirtualFilesByPath(prefix string) []model.Obj { + files := make([]model.Obj, 0) + storages := storagesMap.Values() + sort.Slice(storages, func(i, j int) bool { + if storages[i].GetStorage().Index == storages[j].GetStorage().Index { + return storages[i].GetStorage().VirtualPath < storages[j].GetStorage().VirtualPath + } + return storages[i].GetStorage().Index < storages[j].GetStorage().Index + }) + prefix = utils.StandardizePath(prefix) + if prefix == "/" { + prefix = "" + } + set := make(map[string]interface{}) + for _, v := range storages { + // TODO should save a balanced storage + // balance storage + if utils.IsBalance(v.GetStorage().VirtualPath) { + continue + } + virtualPath := v.GetStorage().VirtualPath + if len(virtualPath) <= len(prefix) { + continue + } + // not prefixed with `prefix` + if !strings.HasPrefix(virtualPath, prefix+"/") { + continue + } + name := strings.Split(strings.TrimPrefix(virtualPath, prefix), "/")[1] + if _, ok := set[name]; ok { + continue + } + files = append(files, model.Object{ + Name: name, + Size: 0, + Modified: v.GetStorage().Modified, + }) + set[name] = nil + } + return files +} + +var balanceMap generic_sync.MapOf[string, int] + +// GetBalancedStorage get storage by path +func GetBalancedStorage(path string) driver.Driver { + path = utils.StandardizePath(path) + storages := getStoragesByPath(path) + storageNum := len(storages) + switch storageNum { + case 0: + return nil + case 1: + return storages[0] + default: + virtualPath := utils.GetActualVirtualPath(storages[0].GetStorage().VirtualPath) + cur, ok := balanceMap.Load(virtualPath) + i := 0 + if ok { + i = cur + i = (i + 1) % storageNum + balanceMap.Store(virtualPath, i) + } else { + balanceMap.Store(virtualPath, i) + } + return storages[i] + } +} diff --git a/internal/operations/storage_test.go b/internal/operations/storage_test.go new file mode 100644 index 00000000..33200727 --- /dev/null +++ b/internal/operations/storage_test.go @@ -0,0 +1,85 @@ +package operations_test + +import ( + "context" + "github.com/alist-org/alist/v3/internal/db" + "testing" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/operations" + "github.com/alist-org/alist/v3/pkg/utils" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +func init() { + dB, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{}) + if err != nil { + panic("failed to connect database") + } + db.Init(dB) +} + +func TestCreateStorage(t *testing.T) { + var storages = []struct { + storage model.Storage + isErr bool + }{ + {storage: model.Storage{Driver: "Local", VirtualPath: "/local", Addition: `{"root_folder":"."}`}, isErr: false}, + {storage: model.Storage{Driver: "Local", VirtualPath: "/local", Addition: `{"root_folder":"."}`}, isErr: true}, + {storage: model.Storage{Driver: "None", VirtualPath: "/none", Addition: `{"root_folder":"."}`}, isErr: true}, + } + for _, storage := range storages { + err := operations.CreateStorage(context.Background(), storage.storage) + if err != nil { + if !storage.isErr { + t.Errorf("failed to create storage: %+v", err) + } else { + t.Logf("expect failed to create storage: %+v", err) + } + } + } +} + +func TestGetStorageVirtualFilesByPath(t *testing.T) { + setupStorages(t) + virtualFiles := operations.GetStorageVirtualFilesByPath("/a") + var names []string + for _, virtualFile := range virtualFiles { + names = append(names, virtualFile.GetName()) + } + var expectedNames = []string{"b", "c", "d"} + if utils.SliceEqual(names, expectedNames) { + t.Logf("passed") + } else { + t.Errorf("expected: %+v, got: %+v", expectedNames, names) + } +} + +func TestGetBalancedStorage(t *testing.T) { + setupStorages(t) + storage := operations.GetBalancedStorage("/a/d/e") + if storage.GetStorage().VirtualPath != "/a/d/e" { + t.Errorf("expected: /a/d/e, got: %+v", storage.GetStorage().VirtualPath) + } + storage = operations.GetBalancedStorage("/a/d/e") + if storage.GetStorage().VirtualPath != "/a/d/e.balance" { + t.Errorf("expected: /a/d/e.balance, got: %+v", storage.GetStorage().VirtualPath) + } +} + +func setupStorages(t *testing.T) { + var storages = []model.Storage{ + {Driver: "Local", VirtualPath: "/a/b", Index: 0, Addition: `{"root_folder":"."}`}, + {Driver: "Local", VirtualPath: "/a/c", Index: 1, Addition: `{"root_folder":"."}`}, + {Driver: "Local", VirtualPath: "/a/d", Index: 2, Addition: `{"root_folder":"."}`}, + {Driver: "Local", VirtualPath: "/a/d/e", Index: 3, Addition: `{"root_folder":"."}`}, + {Driver: "Local", VirtualPath: "/a/d/e.balance", Index: 4, Addition: `{"root_folder":"."}`}, + } + for _, storage := range storages { + err := operations.CreateStorage(context.Background(), storage) + if err != nil { + t.Fatalf("failed to create storage: %+v", err) + } + } +} diff --git a/pkg/task/task.go b/pkg/task/task.go index 585338b3..fcee3db7 100644 --- a/pkg/task/task.go +++ b/pkg/task/task.go @@ -1,4 +1,4 @@ -// Package task manage task, such as file upload, file copy between accounts, offline download, etc. +// Package task manage task, such as file upload, file copy between storages, offline download, etc. package task import ( diff --git a/server/controllers/account.go b/server/controllers/account.go index ee74cd32..e9e096c1 100644 --- a/server/controllers/account.go +++ b/server/controllers/account.go @@ -11,58 +11,58 @@ import ( log "github.com/sirupsen/logrus" ) -func ListAccounts(c *gin.Context) { +func ListStorages(c *gin.Context) { var req common.PageReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } log.Debugf("%+v", req) - accounts, total, err := db.GetAccounts(req.PageIndex, req.PageSize) + storages, total, err := db.GetStorages(req.PageIndex, req.PageSize) if err != nil { common.ErrorResp(c, err, 500) return } common.SuccessResp(c, common.PageResp{ - Content: accounts, + Content: storages, Total: total, }) } -func CreateAccount(c *gin.Context) { - var req model.Account +func CreateStorage(c *gin.Context) { + var req model.Storage if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } - if err := operations.CreateAccount(c, req); err != nil { + if err := operations.CreateStorage(c, req); err != nil { common.ErrorResp(c, err, 500, true) } else { common.SuccessResp(c) } } -func UpdateAccount(c *gin.Context) { - var req model.Account +func UpdateStorage(c *gin.Context) { + var req model.Storage if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } - if err := operations.UpdateAccount(c, req); err != nil { + if err := operations.UpdateStorage(c, req); err != nil { common.ErrorResp(c, err, 500, true) } else { common.SuccessResp(c) } } -func DeleteAccount(c *gin.Context) { +func DeleteStorage(c *gin.Context) { idStr := c.Query("id") id, err := strconv.Atoi(idStr) if err != nil { common.ErrorResp(c, err, 400) return } - if err := operations.DeleteAccountById(c, uint(id)); err != nil { + if err := operations.DeleteStorageById(c, uint(id)); err != nil { common.ErrorResp(c, err, 500, true) return } diff --git a/server/controllers/down.go b/server/controllers/down.go index 3ef0bc15..35fa98d8 100644 --- a/server/controllers/down.go +++ b/server/controllers/down.go @@ -19,12 +19,12 @@ import ( func Down(c *gin.Context) { rawPath := c.MustGet("path").(string) filename := stdpath.Base(rawPath) - account, err := fs.GetAccount(rawPath) + storage, err := fs.GetStorage(rawPath) if err != nil { common.ErrorResp(c, err, 500) return } - if shouldProxy(account, filename) { + if shouldProxy(storage, filename) { Proxy(c) return } else { @@ -43,13 +43,13 @@ func Down(c *gin.Context) { func Proxy(c *gin.Context) { rawPath := c.MustGet("path").(string) filename := stdpath.Base(rawPath) - account, err := fs.GetAccount(rawPath) + storage, err := fs.GetStorage(rawPath) if err != nil { common.ErrorResp(c, err, 500) return } - if canProxy(account, filename) { - downProxyUrl := account.GetAccount().DownProxyUrl + if canProxy(storage, filename) { + downProxyUrl := storage.GetStorage().DownProxyUrl if downProxyUrl != "" { _, ok := c.GetQuery("d") if ok { @@ -79,10 +79,10 @@ func Proxy(c *gin.Context) { // TODO need optimize // when should be proxy? // 1. config.MustProxy() -// 2. account.WebProxy +// 2. storage.WebProxy // 3. proxy_types -func shouldProxy(account driver.Driver, filename string) bool { - if account.Config().MustProxy() || account.GetAccount().WebProxy { +func shouldProxy(storage driver.Driver, filename string) bool { + if storage.Config().MustProxy() || storage.GetStorage().WebProxy { return true } proxyTypes := setting.GetByKey(conf.ProxyTypes) @@ -96,11 +96,11 @@ func shouldProxy(account driver.Driver, filename string) bool { // when can be proxy? // 1. text file // 2. config.MustProxy() -// 3. account.WebProxy +// 3. storage.WebProxy // 4. proxy_types // solution: text_file + shouldProxy() -func canProxy(account driver.Driver, filename string) bool { - if account.Config().MustProxy() || account.GetAccount().WebProxy { +func canProxy(storage driver.Driver, filename string) bool { + if storage.Config().MustProxy() || storage.GetStorage().WebProxy { return true } proxyTypes := setting.GetByKey(conf.ProxyTypes) diff --git a/server/controllers/fsmanage.go b/server/controllers/fsmanage.go index 297086cd..e5c9b01f 100644 --- a/server/controllers/fsmanage.go +++ b/server/controllers/fsmanage.go @@ -237,12 +237,12 @@ func Link(c *gin.Context) { } user := c.MustGet("user").(*model.User) rawPath := stdpath.Join(user.BasePath, req.Path) - account, err := fs.GetAccount(rawPath) + storage, err := fs.GetStorage(rawPath) if err != nil { common.ErrorResp(c, err, 500) return } - if account.Config().OnlyLocal { + if storage.Config().OnlyLocal { common.SuccessResp(c, model.Link{ URL: fmt.Sprintf("%s/p%s?d&sign=%s", common.GetBaseUrl(c.Request), req.Path, sign.Sign(stdpath.Base(rawPath))), }) diff --git a/server/controllers/fsread.go b/server/controllers/fsread.go index 51baf981..e4fe9d6a 100644 --- a/server/controllers/fsread.go +++ b/server/controllers/fsread.go @@ -218,16 +218,16 @@ func FsGet(c *gin.Context) { if u, ok := obj.(model.URL); ok { rawURL = u.URL() } else { - account, _ := fs.GetAccount(req.Path) - if account.Config().MustProxy() || account.GetAccount().WebProxy { - if account.GetAccount().DownProxyUrl != "" { - rawURL = fmt.Sprintf("%s%s?sign=%s", strings.Split(account.GetAccount().DownProxyUrl, "\n")[0], req.Path, sign.Sign(obj.GetName())) + storage, _ := fs.GetStorage(req.Path) + if storage.Config().MustProxy() || storage.GetStorage().WebProxy { + if storage.GetStorage().DownProxyUrl != "" { + rawURL = fmt.Sprintf("%s%s?sign=%s", strings.Split(storage.GetStorage().DownProxyUrl, "\n")[0], req.Path, sign.Sign(obj.GetName())) } else { rawURL = fmt.Sprintf("%s/p%s?sign=%s", common.GetBaseUrl(c.Request), req.Path, sign.Sign(obj.GetName())) } } else { - // if account is not proxy, use raw url by fs.Link - link, _, err := fs.Link(c, req.Path, model.LinkArgs{}) + // if storage is not proxy, use raw url by fs.Link + link, _, err := fs.Link(c, req.Path, model.LinkArgs{IP: c.ClientIP()}) if err != nil { common.ErrorResp(c, err, 500) return diff --git a/server/router.go b/server/router.go index 6afe26c7..8b20b580 100644 --- a/server/router.go +++ b/server/router.go @@ -37,11 +37,11 @@ func Init(r *gin.Engine) { user.POST("/update", controllers.UpdateUser) user.POST("/delete", controllers.DeleteUser) - account := admin.Group("/account") - account.GET("/list", controllers.ListAccounts) - account.POST("/create", controllers.CreateAccount) - account.POST("/update", controllers.UpdateAccount) - account.POST("/delete", controllers.DeleteAccount) + storage := admin.Group("/storage") + storage.GET("/list", controllers.ListStorages) + storage.POST("/create", controllers.CreateStorage) + storage.POST("/update", controllers.UpdateStorage) + storage.POST("/delete", controllers.DeleteStorage) driver := admin.Group("/driver") driver.GET("/list", controllers.ListDriverItems) diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index 1336ff6c..7beec768 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -216,8 +216,8 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta } w.Header().Set("ETag", etag) // Let ServeContent determine the Content-Type header. - account, _ := fs.GetAccount(reqPath) - if account.GetAccount().WebdavNative() { + storage, _ := fs.GetStorage(reqPath) + if storage.GetStorage().WebdavNative() { link, _, err := fs.Link(ctx, reqPath, model.LinkArgs{Header: r.Header}) if err != nil { return http.StatusInternalServerError, err @@ -226,7 +226,7 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta if err != nil { return http.StatusInternalServerError, err } - } else if account.Config().MustProxy() || account.GetAccount().WebdavProxy() { + } else if storage.Config().MustProxy() || storage.GetStorage().WebdavProxy() { u := fmt.Sprintf("%s/p%s?sign=%s", common.GetBaseUrl(r), reqPath, sign.Sign(path.Base(reqPath))) http.Redirect(w, r, u, 302) } else {