github
/
alist
mirror of https://github.com/Xhofe/alist
1
0
Fork 0
Code Issues Releases Wiki Activity

chore: rename account to storage

refactor/fs
Noah Hsu 2022-07-10 14:45:39 +08:00
parent efa20cc7bd
commit fc1204c914
31 changed files with 548 additions and 548 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
drivers/local/driver.go
View File

@ -16,7 +16,7 @@ import (
)
type Driver struct {
model.Account
model.Storage
Addition
}
@ -24,9 +24,9 @@ func (d Driver) Config() driver.Config {
return config
}
func (d *Driver) Init(ctx context.Context, account model.Account) error {
d.Account = account
err := utils.Json.UnmarshalFromString(d.Account.Addition, &d.Addition)
func (d *Driver) Init(ctx context.Context, storage model.Storage) error {
d.Storage = storage
err := utils.Json.UnmarshalFromString(d.Storage.Addition, &d.Addition)
if err != nil {
return errors.Wrap(err, "error while unmarshal addition")
}
@ -42,7 +42,7 @@ func (d *Driver) Init(ctx context.Context, account model.Account) error {
}
d.SetStatus("OK")
}
operations.MustSaveDriverAccount(d)
operations.MustSaveDriverStorage(d)
return err
}

12
internal/aria2/add.go
View File

@ -13,17 +13,17 @@ import (
)
func AddURI(ctx context.Context, uri string, dstDirPath string) error {
// check account
account, dstDirActualPath, err := operations.GetAccountAndActualPath(dstDirPath)
// check storage
storage, dstDirActualPath, err := operations.GetStorageAndActualPath(dstDirPath)
if err != nil {
return errors.WithMessage(err, "failed get account")
return errors.WithMessage(err, "failed get storage")
}
// check is it could upload
if account.Config().NoUpload {
if storage.Config().NoUpload {
return errors.WithStack(errs.UploadNotSupported)
}
// check path is valid
obj, err := operations.Get(ctx, account, dstDirActualPath)
obj, err := operations.Get(ctx, storage, dstDirActualPath)
if err != nil {
if !errs.IsObjectNotFound(err) {
return errors.WithMessage(err, "failed get object")
@ -45,7 +45,7 @@ func AddURI(ctx context.Context, uri string, dstDirPath string) error {
}
DownTaskManager.Submit(task.WithCancelCtx(&task.Task[string]{
ID: gid,
Name: fmt.Sprintf("download %s to [%s](%s)", uri, account.GetAccount().VirtualPath, dstDirActualPath),
Name: fmt.Sprintf("download %s to [%s](%s)", uri, storage.GetStorage().VirtualPath, dstDirActualPath),
Func: func(tsk *task.Task[string]) error {
m := &Monitor{
tsk: tsk,

4
internal/aria2/aria2_test.go
View File

@ -37,7 +37,7 @@ func TestConnect(t *testing.T) {
func TestDown(t *testing.T) {
TestConnect(t)
err := operations.CreateAccount(context.Background(), model.Account{
err := operations.CreateStorage(context.Background(), model.Storage{
ID: 0,
VirtualPath: "/",
Index: 0,
@ -47,7 +47,7 @@ func TestDown(t *testing.T) {
Remark: "",
})
if err != nil {
t.Fatalf("failed to create account: %+v", err)
t.Fatalf("failed to create storage: %+v", err)
}
err = AddURI(context.Background(), "https://nodejs.org/dist/index.json", "/test")
if err != nil {

8
internal/aria2/monitor.go
View File

@ -113,9 +113,9 @@ var TransferTaskManager = task.NewTaskManager[uint64](3, func(k *uint64) {
func (m *Monitor) Complete() error {
// check dstDir again
account, dstDirActualPath, err := operations.GetAccountAndActualPath(m.dstDirPath)
storage, dstDirActualPath, err := operations.GetStorageAndActualPath(m.dstDirPath)
if err != nil {
return errors.WithMessage(err, "failed get account")
return errors.WithMessage(err, "failed get storage")
}
// get files
files, err := client.GetFiles(m.tsk.ID)
@ -135,7 +135,7 @@ func (m *Monitor) Complete() error {
}()
for _, file := range files {
TransferTaskManager.Submit(task.WithCancelCtx[uint64](&task.Task[uint64]{
Name: fmt.Sprintf("transfer %s to [%s](%s)", file.Path, account.GetAccount().VirtualPath, dstDirActualPath),
Name: fmt.Sprintf("transfer %s to [%s](%s)", file.Path, storage.GetStorage().VirtualPath, dstDirActualPath),
Func: func(tsk *task.Task[uint64]) error {
defer wg.Done()
size, _ := strconv.ParseInt(file.Length, 10, 64)
@ -157,7 +157,7 @@ func (m *Monitor) Complete() error {
ReadCloser: f,
Mimetype: mimetype,
}
return operations.Put(tsk.Ctx, account, dstDirActualPath, stream, tsk.SetProgress)
return operations.Put(tsk.Ctx, storage, dstDirActualPath, stream, tsk.SetProgress)
},
}))
}

4
internal/bootstrap/data/dev.go
View File

@ -11,7 +11,7 @@ import (
)
func initDevData() {
err := operations.CreateAccount(context.Background(), model.Account{
err := operations.CreateStorage(context.Background(), model.Storage{
VirtualPath: "/",
Index: 0,
Driver: "Local",
@ -19,7 +19,7 @@ func initDevData() {
Addition: `{"root_folder":"."}`,
})
if err != nil {
log.Fatalf("failed to create account: %+v", err)
log.Fatalf("failed to create storage: %+v", err)
}
err = db.CreateUser(&model.User{
Username: "Noah",

52
internal/db/account.go
View File

@ -5,46 +5,46 @@ import (
"github.com/pkg/errors"
)
// why don't need `cache` for account?
// because all account store in `operations.accountsMap`
// the most of the read operation is from `operations.accountsMap`
// why don't need `cache` for storage?
// because all storage store in `operations.storagesMap`
// the most of the read operation is from `operations.storagesMap`
// just for persistence in database
// CreateAccount just insert account to database
func CreateAccount(account *model.Account) error {
return errors.WithStack(db.Create(account).Error)
// CreateStorage just insert storage to database
func CreateStorage(storage *model.Storage) error {
return errors.WithStack(db.Create(storage).Error)
}
// UpdateAccount just update account in database
func UpdateAccount(account *model.Account) error {
return errors.WithStack(db.Save(account).Error)
// UpdateStorage just update storage in database
func UpdateStorage(storage *model.Storage) error {
return errors.WithStack(db.Save(storage).Error)
}
// DeleteAccountById just delete account from database by id
func DeleteAccountById(id uint) error {
return errors.WithStack(db.Delete(&model.Account{}, id).Error)
// DeleteStorageById just delete storage from database by id
func DeleteStorageById(id uint) error {
return errors.WithStack(db.Delete(&model.Storage{}, id).Error)
}
// GetAccounts Get all accounts from database order by index
func GetAccounts(pageIndex, pageSize int) ([]model.Account, int64, error) {
accountDB := db.Model(&model.Account{})
// GetStorages Get all storages from database order by index
func GetStorages(pageIndex, pageSize int) ([]model.Storage, int64, error) {
storageDB := db.Model(&model.Storage{})
var count int64
if err := accountDB.Count(&count).Error; err != nil {
return nil, 0, errors.Wrapf(err, "failed get accounts count")
if err := storageDB.Count(&count).Error; err != nil {
return nil, 0, errors.Wrapf(err, "failed get storages count")
}
var accounts []model.Account
if err := accountDB.Order(columnName("index")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&accounts).Error; err != nil {
var storages []model.Storage
if err := storageDB.Order(columnName("index")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&storages).Error; err != nil {
return nil, 0, errors.WithStack(err)
}
return accounts, count, nil
return storages, count, nil
}
// GetAccountById Get Account by id, used to update account usually
func GetAccountById(id uint) (*model.Account, error) {
var account model.Account
account.ID = id
if err := db.First(&account).Error; err != nil {
// GetStorageById Get Storage by id, used to update storage usually
func GetStorageById(id uint) (*model.Storage, error) {
var storage model.Storage
storage.ID = id
if err := db.First(&storage).Error; err != nil {
return nil, errors.WithStack(err)
}
return &account, nil
return &storage, nil
}

2
internal/db/db.go
View File

@ -10,7 +10,7 @@ var db gorm.DB
func Init(d *gorm.DB) {
db = *d
err := db.AutoMigrate(new(model.Account), new(model.User), new(model.Meta), new(model.SettingItem))
err := db.AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem))
if err != nil {
log.Fatalf("failed migrate database: %s", err.Error())
}

6
internal/driver/driver.go
View File

@ -17,10 +17,10 @@ type Meta interface {
Config() Config
// Init If already initialized, drop first
// need to unmarshal string to addition first
Init(ctx context.Context, account model.Account) error
Init(ctx context.Context, storage model.Storage) error
Drop(ctx context.Context) error
// GetAccount just get raw account
GetAccount() model.Account
// GetStorage just get raw storage
GetStorage() model.Storage
GetAddition() Additional
}

2
internal/errs/errors.go
View File

@ -9,7 +9,7 @@ var (
NotSupport = errors.New("not support")
RelativePath = errors.New("access using relative path is not allowed")
MoveBetweenTwoAccounts = errors.New("can't move files between two account, try to copy")
MoveBetweenTwoStorages = errors.New("can't move files between two storages, try to copy")
UploadNotSupported = errors.New("upload not supported")
MetaNotFound = errors.New("meta not found")

44
internal/fs/copy.go
View File

@ -19,40 +19,40 @@ var CopyTaskManager = task.NewTaskManager(3, func(tid *uint64) {
atomic.AddUint64(tid, 1)
})
// Copy if in an account, call move method
// Copy if in the same storage, call move method
// if not, add copy task
func _copy(ctx context.Context, srcObjPath, dstDirPath string) (bool, error) {
srcAccount, srcObjActualPath, err := operations.GetAccountAndActualPath(srcObjPath)
srcStorage, srcObjActualPath, err := operations.GetStorageAndActualPath(srcObjPath)
if err != nil {
return false, errors.WithMessage(err, "failed get src account")
return false, errors.WithMessage(err, "failed get src storage")
}
dstAccount, dstDirActualPath, err := operations.GetAccountAndActualPath(dstDirPath)
dstStorage, dstDirActualPath, err := operations.GetStorageAndActualPath(dstDirPath)
if err != nil {
return false, errors.WithMessage(err, "failed get dst account")
return false, errors.WithMessage(err, "failed get dst storage")
}
// copy if in an account, just call driver.Copy
if srcAccount.GetAccount() == dstAccount.GetAccount() {
return false, operations.Copy(ctx, srcAccount, srcObjActualPath, dstDirActualPath)
// copy if in the same storage, just call driver.Copy
if srcStorage.GetStorage() == dstStorage.GetStorage() {
return false, operations.Copy(ctx, srcStorage, srcObjActualPath, dstDirActualPath)
}
// not in an account
// not in the same storage
CopyTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{
Name: fmt.Sprintf("copy [%s](%s) to [%s](%s)", srcAccount.GetAccount().VirtualPath, srcObjActualPath, dstAccount.GetAccount().VirtualPath, dstDirActualPath),
Name: fmt.Sprintf("copy [%s](%s) to [%s](%s)", srcStorage.GetStorage().VirtualPath, srcObjActualPath, dstStorage.GetStorage().VirtualPath, dstDirActualPath),
Func: func(task *task.Task[uint64]) error {
return copyBetween2Accounts(task, srcAccount, dstAccount, srcObjActualPath, dstDirActualPath)
return copyBetween2Storages(task, srcStorage, dstStorage, srcObjActualPath, dstDirActualPath)
},
}))
return true, nil
}
func copyBetween2Accounts(t *task.Task[uint64], srcAccount, dstAccount driver.Driver, srcObjPath, dstDirPath string) error {
func copyBetween2Storages(t *task.Task[uint64], srcStorage, dstStorage driver.Driver, srcObjPath, dstDirPath string) error {
t.SetStatus("getting src object")
srcObj, err := operations.Get(t.Ctx, srcAccount, srcObjPath)
srcObj, err := operations.Get(t.Ctx, srcStorage, srcObjPath)
if err != nil {
return errors.WithMessagef(err, "failed get src [%s] file", srcObjPath)
}
if srcObj.IsDir() {
t.SetStatus("src object is dir, listing objs")
objs, err := operations.List(t.Ctx, srcAccount, srcObjPath)
objs, err := operations.List(t.Ctx, srcStorage, srcObjPath)
if err != nil {
return errors.WithMessagef(err, "failed list src [%s] objs", srcObjPath)
}
@ -63,29 +63,29 @@ func copyBetween2Accounts(t *task.Task[uint64], srcAccount, dstAccount driver.Dr
srcObjPath := stdpath.Join(srcObjPath, obj.GetName())
dstObjPath := stdpath.Join(dstDirPath, obj.GetName())
CopyTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{
Name: fmt.Sprintf("copy [%s](%s) to [%s](%s)", srcAccount.GetAccount().VirtualPath, srcObjPath, dstAccount.GetAccount().VirtualPath, dstObjPath),
Name: fmt.Sprintf("copy [%s](%s) to [%s](%s)", srcStorage.GetStorage().VirtualPath, srcObjPath, dstStorage.GetStorage().VirtualPath, dstObjPath),
Func: func(t *task.Task[uint64]) error {
return copyBetween2Accounts(t, srcAccount, dstAccount, srcObjPath, dstObjPath)
return copyBetween2Storages(t, srcStorage, dstStorage, srcObjPath, dstObjPath)
},
}))
}
} else {
CopyTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{
Name: fmt.Sprintf("copy [%s](%s) to [%s](%s)", srcAccount.GetAccount().VirtualPath, srcObjPath, dstAccount.GetAccount().VirtualPath, dstDirPath),
Name: fmt.Sprintf("copy [%s](%s) to [%s](%s)", srcStorage.GetStorage().VirtualPath, srcObjPath, dstStorage.GetStorage().VirtualPath, dstDirPath),
Func: func(t *task.Task[uint64]) error {
return copyFileBetween2Accounts(t, srcAccount, dstAccount, srcObjPath, dstDirPath)
return copyFileBetween2Storages(t, srcStorage, dstStorage, srcObjPath, dstDirPath)
},
}))
}
return nil
}
func copyFileBetween2Accounts(tsk *task.Task[uint64], srcAccount, dstAccount driver.Driver, srcFilePath, dstDirPath string) error {
srcFile, err := operations.Get(tsk.Ctx, srcAccount, srcFilePath)
func copyFileBetween2Storages(tsk *task.Task[uint64], srcStorage, dstStorage driver.Driver, srcFilePath, dstDirPath string) error {
srcFile, err := operations.Get(tsk.Ctx, srcStorage, srcFilePath)
if err != nil {
return errors.WithMessagef(err, "failed get src [%s] file", srcFilePath)
}
link, _, err := operations.Link(tsk.Ctx, srcAccount, srcFilePath, model.LinkArgs{})
link, _, err := operations.Link(tsk.Ctx, srcStorage, srcFilePath, model.LinkArgs{})
if err != nil {
return errors.WithMessagef(err, "failed get [%s] link", srcFilePath)
}
@ -93,5 +93,5 @@ func copyFileBetween2Accounts(tsk *task.Task[uint64], srcAccount, dstAccount dri
if err != nil {
return errors.WithMessagef(err, "failed get [%s] stream", srcFilePath)
}
return operations.Put(tsk.Ctx, dstAccount, dstDirPath, stream, tsk.SetProgress)
return operations.Put(tsk.Ctx, dstStorage, dstDirPath, stream, tsk.SetProgress)
}

6
internal/fs/fs.go
View File

@ -95,10 +95,10 @@ func PutAsTask(dstDirPath string, file model.FileStreamer) error {
return err
}
func GetAccount(path string) (driver.Driver, error) {
accountDriver, _, err := operations.GetAccountAndActualPath(path)
func GetStorage(path string) (driver.Driver, error) {
storageDriver, _, err := operations.GetStorageAndActualPath(path)
if err != nil {
return nil, err
}
return accountDriver, nil
return storageDriver, nil
}

10
internal/fs/get.go
View File

@ -14,16 +14,16 @@ func get(ctx context.Context, path string) (model.Obj, error) {
path = utils.StandardizePath(path)
// maybe a virtual file
if path != "/" {
virtualFiles := operations.GetAccountVirtualFilesByPath(stdpath.Dir(path))
virtualFiles := operations.GetStorageVirtualFilesByPath(stdpath.Dir(path))
for _, f := range virtualFiles {
if f.GetName() == stdpath.Base(path) {
return f, nil
}
}
}
account, actualPath, err := operations.GetAccountAndActualPath(path)
storage, actualPath, err := operations.GetStorageAndActualPath(path)
if err != nil {
// if there are no account prefix with path, maybe root folder
// if there are no storage prefix with path, maybe root folder
if path == "/" {
return model.Object{
Name: "root",
@ -32,7 +32,7 @@ func get(ctx context.Context, path string) (model.Obj, error) {
IsFolder: true,
}, nil
}
return nil, errors.WithMessage(err, "failed get account")
return nil, errors.WithMessage(err, "failed get storage")
}
return operations.Get(ctx, account, actualPath)
return operations.Get(ctx, storage, actualPath)
}

6
internal/fs/link.go
View File

@ -8,9 +8,9 @@ import (
)
func link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, model.Obj, error) {
account, actualPath, err := operations.GetAccountAndActualPath(path)
storage, actualPath, err := operations.GetStorageAndActualPath(path)
if err != nil {
return nil, nil, errors.WithMessage(err, "failed get account")
return nil, nil, errors.WithMessage(err, "failed get storage")
}
return operations.Link(ctx, account, actualPath, args)
return operations.Link(ctx, storage, actualPath, args)
}

20
internal/fs/list.go
View File

@ -15,15 +15,15 @@ import (
func list(ctx context.Context, path string) ([]model.Obj, error) {
meta := ctx.Value("meta").(*model.Meta)
user := ctx.Value("user").(*model.User)
account, actualPath, err := operations.GetAccountAndActualPath(path)
virtualFiles := operations.GetAccountVirtualFilesByPath(path)
storage, actualPath, err := operations.GetStorageAndActualPath(path)
virtualFiles := operations.GetStorageVirtualFilesByPath(path)
if err != nil {
if len(virtualFiles) != 0 {
return virtualFiles, nil
}
return nil, errors.WithMessage(err, "failed get account")
return nil, errors.WithMessage(err, "failed get storage")
}
objs, err := operations.List(ctx, account, actualPath)
objs, err := operations.List(ctx, storage, actualPath)
if err != nil {
log.Errorf("%+v", err)
if len(virtualFiles) != 0 {
@ -31,19 +31,19 @@ func list(ctx context.Context, path string) ([]model.Obj, error) {
}
return nil, errors.WithMessage(err, "failed get objs")
}
for _, accountFile := range virtualFiles {
if !containsByName(objs, accountFile) {
objs = append(objs, accountFile)
for _, storageFile := range virtualFiles {
if !containsByName(objs, storageFile) {
objs = append(objs, storageFile)
}
}
if whetherHide(user, meta, path) {
objs = hide(objs, meta)
}
// sort objs
if account.Config().LocalSort {
model.SortFiles(objs, account.GetAccount().OrderBy, account.GetAccount().OrderDirection)
if storage.Config().LocalSort {
model.SortFiles(objs, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection)
}
model.ExtractFolder(objs, account.GetAccount().ExtractFolder)
model.ExtractFolder(objs, storage.GetStorage().ExtractFolder)
return objs, nil
}

18
internal/fs/put.go
View File

@ -18,12 +18,12 @@ var UploadTaskManager = task.NewTaskManager[uint64](3, func(tid *uint64) {
// putAsTask add as a put task and return immediately
func putAsTask(dstDirPath string, file model.FileStreamer) error {
account, dstDirActualPath, err := operations.GetAccountAndActualPath(dstDirPath)
if account.Config().NoUpload {
storage, dstDirActualPath, err := operations.GetStorageAndActualPath(dstDirPath)
if storage.Config().NoUpload {
return errors.WithStack(errs.UploadNotSupported)
}
if err != nil {
return errors.WithMessage(err, "failed get account")
return errors.WithMessage(err, "failed get storage")
}
if file.NeedStore() {
tempFile, err := utils.CreateTempFile(file)
@ -33,9 +33,9 @@ func putAsTask(dstDirPath string, file model.FileStreamer) error {
file.SetReadCloser(tempFile)
}
UploadTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{
Name: fmt.Sprintf("upload %s to [%s](%s)", file.GetName(), account.GetAccount().VirtualPath, dstDirActualPath),
Name: fmt.Sprintf("upload %s to [%s](%s)", file.GetName(), storage.GetStorage().VirtualPath, dstDirActualPath),
Func: func(task *task.Task[uint64]) error {
return operations.Put(task.Ctx, account, dstDirActualPath, file, nil)
return operations.Put(task.Ctx, storage, dstDirActualPath, file, nil)
},
}))
return nil
@ -43,12 +43,12 @@ func putAsTask(dstDirPath string, file model.FileStreamer) error {
// putDirect put the file and return after finish
func putDirectly(ctx context.Context, dstDirPath string, file model.FileStreamer) error {
account, dstDirActualPath, err := operations.GetAccountAndActualPath(dstDirPath)
if account.Config().NoUpload {
storage, dstDirActualPath, err := operations.GetStorageAndActualPath(dstDirPath)
if storage.Config().NoUpload {
return errors.WithStack(errs.UploadNotSupported)
}
if err != nil {
return errors.WithMessage(err, "failed get account")
return errors.WithMessage(err, "failed get storage")
}
return operations.Put(ctx, account, dstDirActualPath, file, nil)
return operations.Put(ctx, storage, dstDirActualPath, file, nil)
}

4
internal/fs/util.go
View File

@ -14,11 +14,11 @@ import (
)
func ClearCache(path string) {
account, actualPath, err := operations.GetAccountAndActualPath(path)
storage, actualPath, err := operations.GetStorageAndActualPath(path)
if err != nil {
return
}
operations.ClearCache(account, actualPath)
operations.ClearCache(storage, actualPath)
}
func containsByName(files []model.Obj, file model.Obj) bool {

32
internal/fs/write.go
View File

@ -8,40 +8,40 @@ import (
)
func makeDir(ctx context.Context, path string) error {
account, actualPath, err := operations.GetAccountAndActualPath(path)
storage, actualPath, err := operations.GetStorageAndActualPath(path)
if err != nil {
return errors.WithMessage(err, "failed get account")
return errors.WithMessage(err, "failed get storage")
}
return operations.MakeDir(ctx, account, actualPath)
return operations.MakeDir(ctx, storage, actualPath)
}
func move(ctx context.Context, srcPath, dstDirPath string) error {
srcAccount, srcActualPath, err := operations.GetAccountAndActualPath(srcPath)
srcStorage, srcActualPath, err := operations.GetStorageAndActualPath(srcPath)
if err != nil {
return errors.WithMessage(err, "failed get src account")
return errors.WithMessage(err, "failed get src storage")
}
dstAccount, dstDirActualPath, err := operations.GetAccountAndActualPath(dstDirPath)
dstStorage, dstDirActualPath, err := operations.GetStorageAndActualPath(dstDirPath)
if err != nil {
return errors.WithMessage(err, "failed get dst account")
return errors.WithMessage(err, "failed get dst storage")
}
if srcAccount.GetAccount() != dstAccount.GetAccount() {
return errors.WithStack(errs.MoveBetweenTwoAccounts)
if srcStorage.GetStorage() != dstStorage.GetStorage() {
return errors.WithStack(errs.MoveBetweenTwoStorages)
}
return operations.Move(ctx, srcAccount, srcActualPath, dstDirActualPath)
return operations.Move(ctx, srcStorage, srcActualPath, dstDirActualPath)
}
func rename(ctx context.Context, srcPath, dstName string) error {
account, srcActualPath, err := operations.GetAccountAndActualPath(srcPath)
storage, srcActualPath, err := operations.GetStorageAndActualPath(srcPath)
if err != nil {
return errors.WithMessage(err, "failed get account")
return errors.WithMessage(err, "failed get storage")
}
return operations.Rename(ctx, account, srcActualPath, dstName)
return operations.Rename(ctx, storage, srcActualPath, dstName)
}
func remove(ctx context.Context, path string) error {
account, actualPath, err := operations.GetAccountAndActualPath(path)
storage, actualPath, err := operations.GetStorageAndActualPath(path)
if err != nil {
return errors.WithMessage(err, "failed get account")
return errors.WithMessage(err, "failed get storage")
}
return operations.Remove(ctx, account, actualPath)
return operations.Remove(ctx, storage, actualPath)
}

6
internal/model/account.go → internal/model/storage.go
View File

@ -2,7 +2,7 @@ package model
import "time"
type Account struct {
type Storage struct {
ID uint `json:"id" gorm:"primaryKey"` // unique key
VirtualPath string `json:"virtual_path" gorm:"unique" binding:"required"` // must be standardized
Index int `json:"index"` // use to sort
@ -27,11 +27,11 @@ type Proxy struct {
DownProxyUrl string `json:"down_proxy_url"`
}
func (a Account) GetAccount() Account {
func (a Storage) GetStorage() Storage {
return a
}
func (a *Account) SetStatus(status string) {
func (a *Storage) SetStatus(status string) {
a.Status = status
}

245
internal/operations/account.go
View File

@ -1,245 +0,0 @@
package operations
import (
"context"
log "github.com/sirupsen/logrus"
"sort"
"strings"
"time"
"github.com/alist-org/alist/v3/internal/db"
"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/pkg/generic_sync"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/pkg/errors"
)
// Although the driver type is stored,
// there is an account in each driver,
// so it should actually be an account, just wrapped by the driver
var accountsMap generic_sync.MapOf[string, driver.Driver]
func GetAccountByVirtualPath(virtualPath string) (driver.Driver, error) {
accountDriver, ok := accountsMap.Load(virtualPath)
if !ok {
return nil, errors.Errorf("no virtual path for an account is: %s", virtualPath)
}
return accountDriver, nil
}
// CreateAccount Save the account to database so account can get an id
// then instantiate corresponding driver and save it in memory
func CreateAccount(ctx context.Context, account model.Account) error {
account.Modified = time.Now()
account.VirtualPath = utils.StandardizePath(account.VirtualPath)
var err error
// check driver first
driverName := account.Driver
driverNew, err := GetDriverNew(driverName)
if err != nil {
return errors.WithMessage(err, "failed get driver new")
}
accountDriver := driverNew()
// insert account to database
err = db.CreateAccount(&account)
if err != nil {
return errors.WithMessage(err, "failed create account in database")
}
// already has an id
err = accountDriver.Init(ctx, account)
if err != nil {
return errors.WithMessage(err, "failed init account but account is already created")
}
log.Debugf("account %+v is created", accountDriver)
accountsMap.Store(account.VirtualPath, accountDriver)
return nil
}
// UpdateAccount update account
// get old account first
// drop the account then reinitialize
func UpdateAccount(ctx context.Context, account model.Account) error {
oldAccount, err := db.GetAccountById(account.ID)
if err != nil {
return errors.WithMessage(err, "failed get old account")
}
if oldAccount.Driver != account.Driver {
return errors.Errorf("driver cannot be changed")
}
account.Modified = time.Now()
account.VirtualPath = utils.StandardizePath(account.VirtualPath)
err = db.UpdateAccount(&account)
if err != nil {
return errors.WithMessage(err, "failed update account in database")
}
accountDriver, err := GetAccountByVirtualPath(oldAccount.VirtualPath)
if oldAccount.VirtualPath != account.VirtualPath {
// virtual path renamed, need to drop the account
accountsMap.Delete(oldAccount.VirtualPath)
}
if err != nil {
return errors.WithMessage(err, "failed get account driver")
}
err = accountDriver.Drop(ctx)
if err != nil {
return errors.WithMessage(err, "failed drop account")
}
err = accountDriver.Init(ctx, account)
if err != nil {
return errors.WithMessage(err, "failed init account")
}
accountsMap.Store(account.VirtualPath, accountDriver)
return nil
}
func DeleteAccountById(ctx context.Context, id uint) error {
account, err := db.GetAccountById(id)
if err != nil {
return errors.WithMessage(err, "failed get account")
}
accountDriver, err := GetAccountByVirtualPath(account.VirtualPath)
if err != nil {
return errors.WithMessage(err, "failed get account driver")
}
// drop the account in the driver
if err := accountDriver.Drop(ctx); err != nil {
return errors.WithMessage(err, "failed drop account")
}
// delete the account in the database
if err := db.DeleteAccountById(id); err != nil {
return errors.WithMessage(err, "failed delete account in database")
}
// delete the account in the memory
accountsMap.Delete(account.VirtualPath)
return nil
}
// MustSaveDriverAccount call from specific driver
func MustSaveDriverAccount(driver driver.Driver) {
err := saveDriverAccount(driver)
if err != nil {
log.Errorf("failed save driver account: %s", err)
}
}
func saveDriverAccount(driver driver.Driver) error {
account := driver.GetAccount()
addition := driver.GetAddition()
bytes, err := utils.Json.Marshal(addition)
if err != nil {
return errors.Wrap(err, "error while marshal addition")
}
account.Addition = string(bytes)
err = db.UpdateAccount(&account)
if err != nil {
return errors.WithMessage(err, "failed update account in database")
}
return nil
}
// GetAccountsByPath get account by longest match path, contains balance account.
// for example, there is /a/b,/a/c,/a/d/e,/a/d/e.balance
// GetAccountsByPath(/a/d/e/f) => /a/d/e,/a/d/e.balance
func getAccountsByPath(path string) []driver.Driver {
accounts := make([]driver.Driver, 0)
curSlashCount := 0
accountsMap.Range(func(key string, value driver.Driver) bool {
virtualPath := utils.GetActualVirtualPath(value.GetAccount().VirtualPath)
if virtualPath == "/" {
virtualPath = ""
}
// not this
if path != virtualPath && !strings.HasPrefix(path, virtualPath+"/") {
return true
}
slashCount := strings.Count(virtualPath, "/")
// not the longest match
if slashCount < curSlashCount {
return true
}
if slashCount > curSlashCount {
accounts = accounts[:0]
curSlashCount = slashCount
}
accounts = append(accounts, value)
return true
})
// make sure the order is the same for same input
sort.Slice(accounts, func(i, j int) bool {
return accounts[i].GetAccount().VirtualPath < accounts[j].GetAccount().VirtualPath
})
return accounts
}
// GetAccountVirtualFilesByPath Obtain the virtual file generated by the account according to the path
// for example, there are: /a/b,/a/c,/a/d/e,/a/b.balance1,/av
// GetAccountVirtualFilesByPath(/a) => b,c,d
func GetAccountVirtualFilesByPath(prefix string) []model.Obj {
files := make([]model.Obj, 0)
accounts := accountsMap.Values()
sort.Slice(accounts, func(i, j int) bool {
if accounts[i].GetAccount().Index == accounts[j].GetAccount().Index {
return accounts[i].GetAccount().VirtualPath < accounts[j].GetAccount().VirtualPath
}
return accounts[i].GetAccount().Index < accounts[j].GetAccount().Index
})
prefix = utils.StandardizePath(prefix)
if prefix == "/" {
prefix = ""
}
set := make(map[string]interface{})
for _, v := range accounts {
// TODO should save a balanced account
// balance account
if utils.IsBalance(v.GetAccount().VirtualPath) {
continue
}
virtualPath := v.GetAccount().VirtualPath
if len(virtualPath) <= len(prefix) {
continue
}
// not prefixed with `prefix`
if !strings.HasPrefix(virtualPath, prefix+"/") {
continue
}
name := strings.Split(strings.TrimPrefix(virtualPath, prefix), "/")[1]
if _, ok := set[name]; ok {
continue
}
files = append(files, model.Object{
Name: name,
Size: 0,
Modified: v.GetAccount().Modified,
})
set[name] = nil
}
return files
}
var balanceMap generic_sync.MapOf[string, int]
// GetBalancedAccount get account by path
func GetBalancedAccount(path string) driver.Driver {
path = utils.StandardizePath(path)
accounts := getAccountsByPath(path)
accountNum := len(accounts)
switch accountNum {
case 0:
return nil
case 1:
return accounts[0]
default:
virtualPath := utils.GetActualVirtualPath(accounts[0].GetAccount().VirtualPath)
cur, ok := balanceMap.Load(virtualPath)
i := 0
if ok {
i = cur
i = (i + 1) % accountNum
balanceMap.Store(virtualPath, i)
} else {
balanceMap.Store(virtualPath, i)
}
return accounts[i]
}
}

85
internal/operations/account_test.go
View File

@ -1,85 +0,0 @@
package operations_test
import (
"context"
"github.com/alist-org/alist/v3/internal/db"
"testing"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/operations"
"github.com/alist-org/alist/v3/pkg/utils"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
func init() {
dB, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{})
if err != nil {
panic("failed to connect database")
}
db.Init(dB)
}
func TestCreateAccount(t *testing.T) {
var accounts = []struct {
account model.Account
iserr bool
}{
{account: model.Account{Driver: "Local", VirtualPath: "/local", Addition: `{"root_folder":"."}`}, iserr: false},
{account: model.Account{Driver: "Local", VirtualPath: "/local", Addition: `{"root_folder":"."}`}, iserr: true},
{account: model.Account{Driver: "None", VirtualPath: "/none", Addition: `{"root_folder":"."}`}, iserr: true},
}
for _, account := range accounts {
err := operations.CreateAccount(context.Background(), account.account)
if err != nil {
if !account.iserr {
t.Errorf("failed to create account: %+v", err)
} else {
t.Logf("expect failed to create account: %+v", err)
}
}
}
}
func TestGetAccountVirtualFilesByPath(t *testing.T) {
setupAccounts(t)
virtualFiles := operations.GetAccountVirtualFilesByPath("/a")
var names []string
for _, virtualFile := range virtualFiles {
names = append(names, virtualFile.GetName())
}
var expectedNames = []string{"b", "c", "d"}
if utils.SliceEqual(names, expectedNames) {
t.Logf("passed")
} else {
t.Errorf("expected: %+v, got: %+v", expectedNames, names)
}
}
func TestGetBalancedAccount(t *testing.T) {
setupAccounts(t)
account := operations.GetBalancedAccount("/a/d/e")
if account.GetAccount().VirtualPath != "/a/d/e" {
t.Errorf("expected: /a/d/e, got: %+v", account.GetAccount().VirtualPath)
}
account = operations.GetBalancedAccount("/a/d/e")
if account.GetAccount().VirtualPath != "/a/d/e.balance" {
t.Errorf("expected: /a/d/e.balance, got: %+v", account.GetAccount().VirtualPath)
}
}
func setupAccounts(t *testing.T) {
var accounts = []model.Account{
{Driver: "Local", VirtualPath: "/a/b", Index: 0, Addition: `{"root_folder":"."}`},
{Driver: "Local", VirtualPath: "/a/c", Index: 1, Addition: `{"root_folder":"."}`},
{Driver: "Local", VirtualPath: "/a/d", Index: 2, Addition: `{"root_folder":"."}`},
{Driver: "Local", VirtualPath: "/a/d/e", Index: 3, Addition: `{"root_folder":"."}`},
{Driver: "Local", VirtualPath: "/a/d/e.balance", Index: 4, Addition: `{"root_folder":"."}`},
}
for _, account := range accounts {
err := operations.CreateAccount(context.Background(), account)
if err != nil {
t.Fatalf("failed to create account: %+v", err)
}
}
}

90
internal/operations/fs.go
View File

@ -23,37 +23,37 @@ import (
var filesCache = cache.NewMemCache(cache.WithShards[[]model.Obj](64))
var filesG singleflight.Group[[]model.Obj]
func ClearCache(account driver.Driver, path string) {
key := stdpath.Join(account.GetAccount().VirtualPath, path)
func ClearCache(storage driver.Driver, path string) {
key := stdpath.Join(storage.GetStorage().VirtualPath, path)
filesCache.Del(key)
}
// List files in storage, not contains virtual file
func List(ctx context.Context, account driver.Driver, path string, refresh ...bool) ([]model.Obj, error) {
func List(ctx context.Context, storage driver.Driver, path string, refresh ...bool) ([]model.Obj, error) {
path = utils.StandardizePath(path)
log.Debugf("operations.List %s", path)
dir, err := Get(ctx, account, path)
dir, err := Get(ctx, storage, path)
if err != nil {
return nil, errors.WithMessage(err, "failed get dir")
}
if !dir.IsDir() {
return nil, errors.WithStack(errs.NotFolder)
}
if account.Config().NoCache {
return account.List(ctx, dir)
if storage.Config().NoCache {
return storage.List(ctx, dir)
}
key := stdpath.Join(account.GetAccount().VirtualPath, path)
key := stdpath.Join(storage.GetStorage().VirtualPath, path)
if len(refresh) == 0 || !refresh[0] {
if files, ok := filesCache.Get(key); ok {
return files, nil
}
}
files, err, _ := filesG.Do(key, func() ([]model.Obj, error) {
files, err := account.List(ctx, dir)
files, err := storage.List(ctx, dir)
if err != nil {
return nil, errors.WithMessage(err, "failed to list files")
}
// TODO: maybe can get duration from account's config
// TODO: maybe can get duration from storage's config
filesCache.Set(key, files, cache.WithEx[[]model.Obj](time.Minute*time.Duration(conf.Conf.CaCheExpiration)))
return files, nil
})
@ -74,34 +74,34 @@ func isRoot(path, rootFolderPath string) bool {
}
// Get object from list of files
func Get(ctx context.Context, account driver.Driver, path string) (model.Obj, error) {
func Get(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) {
path = utils.StandardizePath(path)
log.Debugf("operations.Get %s", path)
if g, ok := account.(driver.Getter); ok {
if g, ok := storage.(driver.Getter); ok {
return g.Get(ctx, path)
}
// is root folder
if r, ok := account.GetAddition().(driver.IRootFolderId); ok && utils.PathEqual(path, "/") {
if r, ok := storage.GetAddition().(driver.IRootFolderId); ok && utils.PathEqual(path, "/") {
return model.Object{
ID: r.GetRootFolderId(),
Name: "root",
Size: 0,
Modified: account.GetAccount().Modified,
Modified: storage.GetStorage().Modified,
IsFolder: true,
}, nil
}
if r, ok := account.GetAddition().(driver.IRootFolderPath); ok && isRoot(path, r.GetRootFolderPath()) {
if r, ok := storage.GetAddition().(driver.IRootFolderPath); ok && isRoot(path, r.GetRootFolderPath()) {
return model.Object{
ID: r.GetRootFolderPath(),
Name: "root",
Size: 0,
Modified: account.GetAccount().Modified,
Modified: storage.GetStorage().Modified,
IsFolder: true,
}, nil
}
// not root folder
dir, name := stdpath.Split(path)
files, err := List(ctx, account, dir)
files, err := List(ctx, storage, dir)
if err != nil {
return nil, errors.WithMessage(err, "failed get parent list")
}
@ -124,20 +124,20 @@ var linkCache = cache.NewMemCache(cache.WithShards[*model.Link](16))
var linkG singleflight.Group[*model.Link]
// Link get link, if is an url. should have an expiry time
func Link(ctx context.Context, account driver.Driver, path string, args model.LinkArgs) (*model.Link, model.Obj, error) {
file, err := Get(ctx, account, path)
func Link(ctx context.Context, storage driver.Driver, path string, args model.LinkArgs) (*model.Link, model.Obj, error) {
file, err := Get(ctx, storage, path)
if err != nil {
return nil, nil, errors.WithMessage(err, "failed to get file")
}
if file.IsDir() {
return nil, nil, errors.WithStack(errs.NotFile)
}
key := stdpath.Join(account.GetAccount().VirtualPath, path)
key := stdpath.Join(storage.GetStorage().VirtualPath, path)
if link, ok := linkCache.Get(key); ok {
return link, file, nil
}
fn := func() (*model.Link, error) {
link, err := account.Link(ctx, file, args)
link, err := storage.Link(ctx, file, args)
if err != nil {
return nil, errors.WithMessage(err, "failed get link")
}
@ -150,22 +150,22 @@ func Link(ctx context.Context, account driver.Driver, path string, args model.Li
return link, file, err
}
func MakeDir(ctx context.Context, account driver.Driver, path string) error {
func MakeDir(ctx context.Context, storage driver.Driver, path string) error {
// check if dir exists
f, err := Get(ctx, account, path)
f, err := Get(ctx, storage, path)
if err != nil {
if errs.IsObjectNotFound(err) {
parentPath, dirName := stdpath.Split(path)
err = MakeDir(ctx, account, parentPath)
err = MakeDir(ctx, storage, parentPath)
if err != nil {
return errors.WithMessagef(err, "failed to make parent dir [%s]", parentPath)
}
parentDir, err := Get(ctx, account, parentPath)
parentDir, err := Get(ctx, storage, parentPath)
// this should not happen
if err != nil {
return errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath)
}
return account.MakeDir(ctx, parentDir, dirName)
return storage.MakeDir(ctx, parentDir, dirName)
} else {
return errors.WithMessage(err, "failed to check if dir exists")
}
@ -180,38 +180,38 @@ func MakeDir(ctx context.Context, account driver.Driver, path string) error {
}
}
func Move(ctx context.Context, account driver.Driver, srcPath, dstDirPath string) error {
srcObj, err := Get(ctx, account, srcPath)
func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string) error {
srcObj, err := Get(ctx, storage, srcPath)
if err != nil {
return errors.WithMessage(err, "failed to get src object")
}
dstDir, err := Get(ctx, account, dstDirPath)
dstDir, err := Get(ctx, storage, dstDirPath)
if err != nil {
return errors.WithMessage(err, "failed to get dst dir")
}
return account.Move(ctx, srcObj, dstDir)
return storage.Move(ctx, srcObj, dstDir)
}
func Rename(ctx context.Context, account driver.Driver, srcPath, dstName string) error {
srcObj, err := Get(ctx, account, srcPath)
func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string) error {
srcObj, err := Get(ctx, storage, srcPath)
if err != nil {
return errors.WithMessage(err, "failed to get src object")
}
return account.Rename(ctx, srcObj, dstName)
return storage.Rename(ctx, srcObj, dstName)
}
// Copy Just copy file[s] in an account
func Copy(ctx context.Context, account driver.Driver, srcPath, dstDirPath string) error {
srcObj, err := Get(ctx, account, srcPath)
// Copy Just copy file[s] in a storage
func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string) error {
srcObj, err := Get(ctx, storage, srcPath)
if err != nil {
return errors.WithMessage(err, "failed to get src object")
}
dstDir, err := Get(ctx, account, dstDirPath)
return account.Copy(ctx, srcObj, dstDir)
dstDir, err := Get(ctx, storage, dstDirPath)
return storage.Copy(ctx, srcObj, dstDir)
}
func Remove(ctx context.Context, account driver.Driver, path string) error {
obj, err := Get(ctx, account, path)
func Remove(ctx context.Context, storage driver.Driver, path string) error {
obj, err := Get(ctx, storage, path)
if err != nil {
// if object not found, it's ok
if errs.IsObjectNotFound(err) {
@ -219,10 +219,10 @@ func Remove(ctx context.Context, account driver.Driver, path string) error {
}
return errors.WithMessage(err, "failed to get object")
}
return account.Remove(ctx, obj)
return storage.Remove(ctx, obj)
}
func Put(ctx context.Context, account driver.Driver, dstDirPath string, file model.FileStreamer, up driver.UpdateProgress) error {
func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file model.FileStreamer, up driver.UpdateProgress) error {
defer func() {
if f, ok := file.GetReadCloser().(*os.File); ok {
err := os.RemoveAll(f.Name())
@ -236,11 +236,11 @@ func Put(ctx context.Context, account driver.Driver, dstDirPath string, file mod
log.Errorf("failed to close file streamer, %v", err)
}
}()
err := MakeDir(ctx, account, dstDirPath)
err := MakeDir(ctx, storage, dstDirPath)
if err != nil {
return errors.WithMessagef(err, "failed to make dir [%s]", dstDirPath)
}
parentDir, err := Get(ctx, account, dstDirPath)
parentDir, err := Get(ctx, storage, dstDirPath)
// this should not happen
if err != nil {
return errors.WithMessagef(err, "failed to get dir [%s]", dstDirPath)
@ -249,11 +249,11 @@ func Put(ctx context.Context, account driver.Driver, dstDirPath string, file mod
if up == nil {
up = func(p int) {}
}
err = account.Put(ctx, parentDir, file, up)
err = storage.Put(ctx, parentDir, file, up)
log.Debugf("put file [%s] done", file.GetName())
if err == nil {
// clear cache
key := stdpath.Join(account.GetAccount().VirtualPath, dstDirPath)
key := stdpath.Join(storage.GetStorage().VirtualPath, dstDirPath)
filesCache.Del(key)
}
return err

22
internal/operations/path.go
View File

@ -13,27 +13,27 @@ import (
// ActualPath Get the actual path
// !!! maybe and \ in the path when use windows local
func ActualPath(account driver.Additional, rawPath string) string {
if i, ok := account.(driver.IRootFolderPath); ok {
func ActualPath(storage driver.Additional, rawPath string) string {
if i, ok := storage.(driver.IRootFolderPath); ok {
rawPath = stdpath.Join(i.GetRootFolderPath(), rawPath)
}
return utils.StandardizePath(rawPath)
}
// GetAccountAndActualPath Get the corresponding account
// GetStorageAndActualPath Get the corresponding storage and actual path
// for path: remove the virtual path prefix and join the actual root folder if exists
func GetAccountAndActualPath(rawPath string) (driver.Driver, string, error) {
func GetStorageAndActualPath(rawPath string) (driver.Driver, string, error) {
rawPath = utils.StandardizePath(rawPath)
if strings.Contains(rawPath, "..") {
return nil, "", errors.WithStack(errs.RelativePath)
}
account := GetBalancedAccount(rawPath)
if account == nil {
return nil, "", errors.Errorf("can't find account with rawPath: %s", rawPath)
storage := GetBalancedStorage(rawPath)
if storage == nil {
return nil, "", errors.Errorf("can't find storage with rawPath: %s", rawPath)
}
log.Debugln("use account: ", account.GetAccount().VirtualPath)
virtualPath := utils.GetActualVirtualPath(account.GetAccount().VirtualPath)
log.Debugln("use storage: ", storage.GetStorage().VirtualPath)
virtualPath := utils.GetActualVirtualPath(storage.GetStorage().VirtualPath)
actualPath := strings.TrimPrefix(rawPath, virtualPath)
actualPath = ActualPath(account.GetAddition(), actualPath)
return account, actualPath, nil
actualPath = ActualPath(storage.GetAddition(), actualPath)
return storage, actualPath, nil
}

245
internal/operations/storage.go Normal file
View File

@ -0,0 +1,245 @@
package operations
import (
"context"
log "github.com/sirupsen/logrus"
"sort"
"strings"
"time"
"github.com/alist-org/alist/v3/internal/db"
"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/pkg/generic_sync"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/pkg/errors"
)
// Although the driver type is stored,
// there is a storage in each driver,
// so it should actually be a storage, just wrapped by the driver
var storagesMap generic_sync.MapOf[string, driver.Driver]
func GetStorageByVirtualPath(virtualPath string) (driver.Driver, error) {
storageDriver, ok := storagesMap.Load(virtualPath)
if !ok {
return nil, errors.Errorf("no virtual path for an storage is: %s", virtualPath)
}
return storageDriver, nil
}
// CreateStorage Save the storage to database so storage can get an id
// then instantiate corresponding driver and save it in memory
func CreateStorage(ctx context.Context, storage model.Storage) error {
storage.Modified = time.Now()
storage.VirtualPath = utils.StandardizePath(storage.VirtualPath)
var err error
// check driver first
driverName := storage.Driver
driverNew, err := GetDriverNew(driverName)
if err != nil {
return errors.WithMessage(err, "failed get driver new")
}
storageDriver := driverNew()
// insert storage to database
err = db.CreateStorage(&storage)
if err != nil {
return errors.WithMessage(err, "failed create storage in database")
}
// already has an id
err = storageDriver.Init(ctx, storage)
if err != nil {
return errors.WithMessage(err, "failed init storage but storage is already created")
}
log.Debugf("storage %+v is created", storageDriver)
storagesMap.Store(storage.VirtualPath, storageDriver)
return nil
}
// UpdateStorage update storage
// get old storage first
// drop the storage then reinitialize
func UpdateStorage(ctx context.Context, storage model.Storage) error {
oldStorage, err := db.GetStorageById(storage.ID)
if err != nil {
return errors.WithMessage(err, "failed get old storage")
}
if oldStorage.Driver != storage.Driver {
return errors.Errorf("driver cannot be changed")
}
storage.Modified = time.Now()
storage.VirtualPath = utils.StandardizePath(storage.VirtualPath)
err = db.UpdateStorage(&storage)
if err != nil {
return errors.WithMessage(err, "failed update storage in database")
}
storageDriver, err := GetStorageByVirtualPath(oldStorage.VirtualPath)
if oldStorage.VirtualPath != storage.VirtualPath {
// virtual path renamed, need to drop the storage
storagesMap.Delete(oldStorage.VirtualPath)
}
if err != nil {
return errors.WithMessage(err, "failed get storage driver")
}
err = storageDriver.Drop(ctx)
if err != nil {
return errors.WithMessage(err, "failed drop storage")
}
err = storageDriver.Init(ctx, storage)
if err != nil {
return errors.WithMessage(err, "failed init storage")
}
storagesMap.Store(storage.VirtualPath, storageDriver)
return nil
}
func DeleteStorageById(ctx context.Context, id uint) error {
storage, err := db.GetStorageById(id)
if err != nil {
return errors.WithMessage(err, "failed get storage")
}
storageDriver, err := GetStorageByVirtualPath(storage.VirtualPath)
if err != nil {
return errors.WithMessage(err, "failed get storage driver")
}
// drop the storage in the driver
if err := storageDriver.Drop(ctx); err != nil {
return errors.WithMessage(err, "failed drop storage")
}
// delete the storage in the database
if err := db.DeleteStorageById(id); err != nil {
return errors.WithMessage(err, "failed delete storage in database")
}
// delete the storage in the memory
storagesMap.Delete(storage.VirtualPath)
return nil
}
// MustSaveDriverStorage call from specific driver
func MustSaveDriverStorage(driver driver.Driver) {
err := saveDriverStorage(driver)
if err != nil {
log.Errorf("failed save driver storage: %s", err)
}
}
func saveDriverStorage(driver driver.Driver) error {
storage := driver.GetStorage()
addition := driver.GetAddition()
bytes, err := utils.Json.Marshal(addition)
if err != nil {
return errors.Wrap(err, "error while marshal addition")
}
storage.Addition = string(bytes)
err = db.UpdateStorage(&storage)
if err != nil {
return errors.WithMessage(err, "failed update storage in database")
}
return nil
}
// getStoragesByPath get storage by longest match path, contains balance storage.
// for example, there is /a/b,/a/c,/a/d/e,/a/d/e.balance
// getStoragesByPath(/a/d/e/f) => /a/d/e,/a/d/e.balance
func getStoragesByPath(path string) []driver.Driver {
storages := make([]driver.Driver, 0)
curSlashCount := 0
storagesMap.Range(func(key string, value driver.Driver) bool {
virtualPath := utils.GetActualVirtualPath(value.GetStorage().VirtualPath)
if virtualPath == "/" {
virtualPath = ""
}
// not this
if path != virtualPath && !strings.HasPrefix(path, virtualPath+"/") {
return true
}
slashCount := strings.Count(virtualPath, "/")
// not the longest match
if slashCount < curSlashCount {
return true
}
if slashCount > curSlashCount {
storages = storages[:0]
curSlashCount = slashCount
}
storages = append(storages, value)
return true
})
// make sure the order is the same for same input
sort.Slice(storages, func(i, j int) bool {
return storages[i].GetStorage().VirtualPath < storages[j].GetStorage().VirtualPath
})
return storages
}
// GetStorageVirtualFilesByPath Obtain the virtual file generated by the storage according to the path
// for example, there are: /a/b,/a/c,/a/d/e,/a/b.balance1,/av
// GetStorageVirtualFilesByPath(/a) => b,c,d
func GetStorageVirtualFilesByPath(prefix string) []model.Obj {
files := make([]model.Obj, 0)
storages := storagesMap.Values()
sort.Slice(storages, func(i, j int) bool {
if storages[i].GetStorage().Index == storages[j].GetStorage().Index {
return storages[i].GetStorage().VirtualPath < storages[j].GetStorage().VirtualPath
}
return storages[i].GetStorage().Index < storages[j].GetStorage().Index
})
prefix = utils.StandardizePath(prefix)
if prefix == "/" {
prefix = ""
}
set := make(map[string]interface{})
for _, v := range storages {
// TODO should save a balanced storage
// balance storage
if utils.IsBalance(v.GetStorage().VirtualPath) {
continue
}
virtualPath := v.GetStorage().VirtualPath
if len(virtualPath) <= len(prefix) {
continue
}
// not prefixed with `prefix`
if !strings.HasPrefix(virtualPath, prefix+"/") {
continue
}
name := strings.Split(strings.TrimPrefix(virtualPath, prefix), "/")[1]
if _, ok := set[name]; ok {
continue
}
files = append(files, model.Object{
Name: name,
Size: 0,
Modified: v.GetStorage().Modified,
})
set[name] = nil
}
return files
}
var balanceMap generic_sync.MapOf[string, int]
// GetBalancedStorage get storage by path
func GetBalancedStorage(path string) driver.Driver {
path = utils.StandardizePath(path)
storages := getStoragesByPath(path)
storageNum := len(storages)
switch storageNum {
case 0:
return nil
case 1:
return storages[0]
default:
virtualPath := utils.GetActualVirtualPath(storages[0].GetStorage().VirtualPath)
cur, ok := balanceMap.Load(virtualPath)
i := 0
if ok {
i = cur
i = (i + 1) % storageNum
balanceMap.Store(virtualPath, i)
} else {
balanceMap.Store(virtualPath, i)
}
return storages[i]
}
}

85
internal/operations/storage_test.go Normal file
View File

@ -0,0 +1,85 @@
package operations_test
import (
"context"
"github.com/alist-org/alist/v3/internal/db"
"testing"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/operations"
"github.com/alist-org/alist/v3/pkg/utils"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
func init() {
dB, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{})
if err != nil {
panic("failed to connect database")
}
db.Init(dB)
}
func TestCreateStorage(t *testing.T) {
var storages = []struct {
storage model.Storage
isErr bool
}{
{storage: model.Storage{Driver: "Local", VirtualPath: "/local", Addition: `{"root_folder":"."}`}, isErr: false},
{storage: model.Storage{Driver: "Local", VirtualPath: "/local", Addition: `{"root_folder":"."}`}, isErr: true},
{storage: model.Storage{Driver: "None", VirtualPath: "/none", Addition: `{"root_folder":"."}`}, isErr: true},
}
for _, storage := range storages {
err := operations.CreateStorage(context.Background(), storage.storage)
if err != nil {
if !storage.isErr {
t.Errorf("failed to create storage: %+v", err)
} else {
t.Logf("expect failed to create storage: %+v", err)
}
}
}
}
func TestGetStorageVirtualFilesByPath(t *testing.T) {
setupStorages(t)
virtualFiles := operations.GetStorageVirtualFilesByPath("/a")
var names []string
for _, virtualFile := range virtualFiles {
names = append(names, virtualFile.GetName())
}
var expectedNames = []string{"b", "c", "d"}
if utils.SliceEqual(names, expectedNames) {
t.Logf("passed")
} else {
t.Errorf("expected: %+v, got: %+v", expectedNames, names)
}
}
func TestGetBalancedStorage(t *testing.T) {
setupStorages(t)
storage := operations.GetBalancedStorage("/a/d/e")
if storage.GetStorage().VirtualPath != "/a/d/e" {
t.Errorf("expected: /a/d/e, got: %+v", storage.GetStorage().VirtualPath)
}
storage = operations.GetBalancedStorage("/a/d/e")
if storage.GetStorage().VirtualPath != "/a/d/e.balance" {
t.Errorf("expected: /a/d/e.balance, got: %+v", storage.GetStorage().VirtualPath)
}
}
func setupStorages(t *testing.T) {
var storages = []model.Storage{
{Driver: "Local", VirtualPath: "/a/b", Index: 0, Addition: `{"root_folder":"."}`},
{Driver: "Local", VirtualPath: "/a/c", Index: 1, Addition: `{"root_folder":"."}`},
{Driver: "Local", VirtualPath: "/a/d", Index: 2, Addition: `{"root_folder":"."}`},
{Driver: "Local", VirtualPath: "/a/d/e", Index: 3, Addition: `{"root_folder":"."}`},
{Driver: "Local", VirtualPath: "/a/d/e.balance", Index: 4, Addition: `{"root_folder":"."}`},
}
for _, storage := range storages {
err := operations.CreateStorage(context.Background(), storage)
if err != nil {
t.Fatalf("failed to create storage: %+v", err)
}
}
}

2
pkg/task/task.go
View File

@ -1,4 +1,4 @@
// Package task manage task, such as file upload, file copy between accounts, offline download, etc.
// Package task manage task, such as file upload, file copy between storages, offline download, etc.
package task
import (

22
server/controllers/account.go
View File

@ -11,58 +11,58 @@ import (
log "github.com/sirupsen/logrus"
)
func ListAccounts(c *gin.Context) {
func ListStorages(c *gin.Context) {
var req common.PageReq
if err := c.ShouldBind(&req); err != nil {
common.ErrorResp(c, err, 400)
return
}
log.Debugf("%+v", req)
accounts, total, err := db.GetAccounts(req.PageIndex, req.PageSize)
storages, total, err := db.GetStorages(req.PageIndex, req.PageSize)
if err != nil {
common.ErrorResp(c, err, 500)
return
}
common.SuccessResp(c, common.PageResp{
Content: accounts,
Content: storages,
Total: total,
})
}
func CreateAccount(c *gin.Context) {
var req model.Account
func CreateStorage(c *gin.Context) {
var req model.Storage
if err := c.ShouldBind(&req); err != nil {
common.ErrorResp(c, err, 400)
return
}
if err := operations.CreateAccount(c, req); err != nil {
if err := operations.CreateStorage(c, req); err != nil {
common.ErrorResp(c, err, 500, true)
} else {
common.SuccessResp(c)
}
}
func UpdateAccount(c *gin.Context) {
var req model.Account
func UpdateStorage(c *gin.Context) {
var req model.Storage
if err := c.ShouldBind(&req); err != nil {
common.ErrorResp(c, err, 400)
return
}
if err := operations.UpdateAccount(c, req); err != nil {
if err := operations.UpdateStorage(c, req); err != nil {
common.ErrorResp(c, err, 500, true)
} else {
common.SuccessResp(c)
}
}
func DeleteAccount(c *gin.Context) {
func DeleteStorage(c *gin.Context) {
idStr := c.Query("id")
id, err := strconv.Atoi(idStr)
if err != nil {
common.ErrorResp(c, err, 400)
return
}
if err := operations.DeleteAccountById(c, uint(id)); err != nil {
if err := operations.DeleteStorageById(c, uint(id)); err != nil {
common.ErrorResp(c, err, 500, true)
return
}

22
server/controllers/down.go
View File

@ -19,12 +19,12 @@ import (
func Down(c *gin.Context) {
rawPath := c.MustGet("path").(string)
filename := stdpath.Base(rawPath)
account, err := fs.GetAccount(rawPath)
storage, err := fs.GetStorage(rawPath)
if err != nil {
common.ErrorResp(c, err, 500)
return
}
if shouldProxy(account, filename) {
if shouldProxy(storage, filename) {
Proxy(c)
return
} else {
@ -43,13 +43,13 @@ func Down(c *gin.Context) {
func Proxy(c *gin.Context) {
rawPath := c.MustGet("path").(string)
filename := stdpath.Base(rawPath)
account, err := fs.GetAccount(rawPath)
storage, err := fs.GetStorage(rawPath)
if err != nil {
common.ErrorResp(c, err, 500)
return
}
if canProxy(account, filename) {
downProxyUrl := account.GetAccount().DownProxyUrl
if canProxy(storage, filename) {
downProxyUrl := storage.GetStorage().DownProxyUrl
if downProxyUrl != "" {
_, ok := c.GetQuery("d")
if ok {
@ -79,10 +79,10 @@ func Proxy(c *gin.Context) {
// TODO need optimize
// when should be proxy?
// 1. config.MustProxy()
// 2. account.WebProxy
// 2. storage.WebProxy
// 3. proxy_types
func shouldProxy(account driver.Driver, filename string) bool {
if account.Config().MustProxy() || account.GetAccount().WebProxy {
func shouldProxy(storage driver.Driver, filename string) bool {
if storage.Config().MustProxy() || storage.GetStorage().WebProxy {
return true
}
proxyTypes := setting.GetByKey(conf.ProxyTypes)
@ -96,11 +96,11 @@ func shouldProxy(account driver.Driver, filename string) bool {
// when can be proxy?
// 1. text file
// 2. config.MustProxy()
// 3. account.WebProxy
// 3. storage.WebProxy
// 4. proxy_types
// solution: text_file + shouldProxy()
func canProxy(account driver.Driver, filename string) bool {
if account.Config().MustProxy() || account.GetAccount().WebProxy {
func canProxy(storage driver.Driver, filename string) bool {
if storage.Config().MustProxy() || storage.GetStorage().WebProxy {
return true
}
proxyTypes := setting.GetByKey(conf.ProxyTypes)

4
server/controllers/fsmanage.go
View File

@ -237,12 +237,12 @@ func Link(c *gin.Context) {
}
user := c.MustGet("user").(*model.User)
rawPath := stdpath.Join(user.BasePath, req.Path)
account, err := fs.GetAccount(rawPath)
storage, err := fs.GetStorage(rawPath)
if err != nil {
common.ErrorResp(c, err, 500)
return
}
if account.Config().OnlyLocal {
if storage.Config().OnlyLocal {
common.SuccessResp(c, model.Link{
URL: fmt.Sprintf("%s/p%s?d&sign=%s", common.GetBaseUrl(c.Request), req.Path, sign.Sign(stdpath.Base(rawPath))),
})

12
server/controllers/fsread.go
View File

@ -218,16 +218,16 @@ func FsGet(c *gin.Context) {
if u, ok := obj.(model.URL); ok {
rawURL = u.URL()
} else {
account, _ := fs.GetAccount(req.Path)
if account.Config().MustProxy() || account.GetAccount().WebProxy {
if account.GetAccount().DownProxyUrl != "" {
rawURL = fmt.Sprintf("%s%s?sign=%s", strings.Split(account.GetAccount().DownProxyUrl, "\n")[0], req.Path, sign.Sign(obj.GetName()))
storage, _ := fs.GetStorage(req.Path)
if storage.Config().MustProxy() || storage.GetStorage().WebProxy {
if storage.GetStorage().DownProxyUrl != "" {
rawURL = fmt.Sprintf("%s%s?sign=%s", strings.Split(storage.GetStorage().DownProxyUrl, "\n")[0], req.Path, sign.Sign(obj.GetName()))
} else {
rawURL = fmt.Sprintf("%s/p%s?sign=%s", common.GetBaseUrl(c.Request), req.Path, sign.Sign(obj.GetName()))
}
} else {
// if account is not proxy, use raw url by fs.Link
link, _, err := fs.Link(c, req.Path, model.LinkArgs{})
// if storage is not proxy, use raw url by fs.Link
link, _, err := fs.Link(c, req.Path, model.LinkArgs{IP: c.ClientIP()})
if err != nil {
common.ErrorResp(c, err, 500)
return

10
server/router.go
View File

@ -37,11 +37,11 @@ func Init(r *gin.Engine) {
user.POST("/update", controllers.UpdateUser)
user.POST("/delete", controllers.DeleteUser)
account := admin.Group("/account")
account.GET("/list", controllers.ListAccounts)
account.POST("/create", controllers.CreateAccount)
account.POST("/update", controllers.UpdateAccount)
account.POST("/delete", controllers.DeleteAccount)
storage := admin.Group("/storage")
storage.GET("/list", controllers.ListStorages)
storage.POST("/create", controllers.CreateStorage)
storage.POST("/update", controllers.UpdateStorage)
storage.POST("/delete", controllers.DeleteStorage)
driver := admin.Group("/driver")
driver.GET("/list", controllers.ListDriverItems)

6
server/webdav/webdav.go
View File

@ -216,8 +216,8 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta
}
w.Header().Set("ETag", etag)
// Let ServeContent determine the Content-Type header.
account, _ := fs.GetAccount(reqPath)
if account.GetAccount().WebdavNative() {
storage, _ := fs.GetStorage(reqPath)
if storage.GetStorage().WebdavNative() {
link, _, err := fs.Link(ctx, reqPath, model.LinkArgs{Header: r.Header})
if err != nil {
return http.StatusInternalServerError, err
@ -226,7 +226,7 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta
if err != nil {
return http.StatusInternalServerError, err
}
} else if account.Config().MustProxy() || account.GetAccount().WebdavProxy() {
} else if storage.Config().MustProxy() || storage.GetStorage().WebdavProxy() {
u := fmt.Sprintf("%s/p%s?sign=%s", common.GetBaseUrl(r), reqPath, sign.Sign(path.Base(reqPath)))
http.Redirect(w, r, u, 302)
} else {