github
/
alist
mirror of https://github.com/Xhofe/alist
1
0
Fork 0
Code Issues Releases Wiki Activity

refactor: change driver interface

refactor/fs
Noah Hsu 2022-06-15 20:31:23 +08:00
parent d9eb188b7a
commit 09ef7c7106
12 changed files with 181 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
conf/config.go
View File

@ -18,11 +18,6 @@ type Scheme struct {
KeyFile string `json:"key_file" env:"KEY_FILE"`
}
type CacheConfig struct {
Expiration int64 `json:"expiration" env:"CACHE_EXPIRATION"`
CleanupInterval int64 `json:"cleanup_interval" env:"CLEANUP_INTERVAL"`
}
type LogConfig struct {
Enable bool `json:"enable" env:"log_enable"`
Path string `json:"path" env:"LOG_PATH"`
@ -32,15 +27,15 @@ type LogConfig struct {
}
type Config struct {
Force bool `json:"force"`
Address string `json:"address" env:"ADDR"`
Port int `json:"port" env:"PORT"`
Assets string `json:"assets" env:"ASSETS"`
Database Database `json:"database"`
Scheme Scheme `json:"scheme"`
Cache CacheConfig `json:"cache"`
TempDir string `json:"temp_dir" env:"TEMP_DIR"`
Log LogConfig `json:"log"`
Force bool `json:"force"`
Address string `json:"address" env:"ADDR"`
Port int `json:"port" env:"PORT"`
CaCheExpiration int `json:"cache_expiration" env:"CACHE_EXPIRATION"`
Assets string `json:"assets" env:"ASSETS"`
Database Database `json:"database"`
Scheme Scheme `json:"scheme"`
TempDir string `json:"temp_dir" env:"TEMP_DIR"`
Log LogConfig `json:"log"`
}
func DefaultConfig() *Config {
@ -55,10 +50,7 @@ func DefaultConfig() *Config {
TablePrefix: "x_",
DBFile: "data/data.db",
},
Cache: CacheConfig{
Expiration: 60,
CleanupInterval: 120,
},
CaCheExpiration: 30,
Log: LogConfig{
Enable: true,
Path: "log/%Y-%m-%d-%H:%M.log",

64
drivers/local/driver.go
View File

@ -44,12 +44,42 @@ func (d *Driver) GetAddition() driver.Additional {
return d.Addition
}
func (d *Driver) List(ctx context.Context, path string) ([]model.FileInfo, error) {
func (d *Driver) List(ctx context.Context, dir model.Object) ([]model.Object, error) {
//TODO implement me
panic("implement me")
}
func (d *Driver) Link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, error) {
func (d *Driver) Link(ctx context.Context, file model.Object, args model.LinkArgs) (*model.Link, error) {
//TODO implement me
panic("implement me")
}
func (d *Driver) MakeDir(ctx context.Context, parentDir model.Object, dirName string) error {
//TODO implement me
panic("implement me")
}
func (d *Driver) Move(ctx context.Context, srcObject, dstDir model.Object) error {
//TODO implement me
panic("implement me")
}
func (d *Driver) Rename(ctx context.Context, srcObject model.Object, newName string) error {
//TODO implement me
panic("implement me")
}
func (d *Driver) Copy(ctx context.Context, srcObject, dstDir model.Object) error {
//TODO implement me
panic("implement me")
}
func (d *Driver) Remove(ctx context.Context, object model.Object) error {
//TODO implement me
panic("implement me")
}
func (d *Driver) Put(ctx context.Context, parentDir model.Object, stream model.FileStreamer) error {
//TODO implement me
panic("implement me")
}
@ -59,34 +89,4 @@ func (d Driver) Other(ctx context.Context, data interface{}) (interface{}, error
panic("implement me")
}
func (d *Driver) MakeDir(ctx context.Context, path string) error {
//TODO implement me
panic("implement me")
}
func (d *Driver) Move(ctx context.Context, src, dst string) error {
//TODO implement me
panic("implement me")
}
func (d *Driver) Rename(ctx context.Context, src, dst string) error {
//TODO implement me
panic("implement me")
}
func (d *Driver) Copy(ctx context.Context, src, dst string) error {
//TODO implement me
panic("implement me")
}
func (d *Driver) Remove(ctx context.Context, path string) error {
//TODO implement me
panic("implement me")
}
func (d *Driver) Put(ctx context.Context, parentPath string, stream model.FileStreamer) error {
//TODO implement me
panic("implement me")
}
var _ driver.Driver = (*Driver)(nil)

22
internal/driver/driver.go
View File

@ -29,16 +29,22 @@ type Other interface {
}
type Reader interface {
List(ctx context.Context, path string) ([]model.FileInfo, error)
Link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, error)
List(ctx context.Context, dir model.Object) ([]model.Object, error)
Link(ctx context.Context, file model.Object, args model.LinkArgs) (*model.Link, error)
//Get(ctx context.Context, path string) (FileInfo, error) // maybe not need
}
type Writer interface {
MakeDir(ctx context.Context, path string) error
Move(ctx context.Context, srcPath, dstPath string) error
Rename(ctx context.Context, srcPath, dstName string) error
Copy(ctx context.Context, srcPath, dstPath string) error
Remove(ctx context.Context, path string) error
Put(ctx context.Context, parentPath string, stream model.FileStreamer) error
// MakeDir make a folder named `dirName` in `parentDir`
MakeDir(ctx context.Context, parentDir model.Object, dirName string) error
// Move move `srcObject` to `dstDir`
Move(ctx context.Context, srcObject, dstDir model.Object) error
// Rename rename `srcObject` to `newName`
Rename(ctx context.Context, srcObject model.Object, newName string) error
// Copy copy `srcObject` to `dstDir`
Copy(ctx context.Context, srcObject, dstDir model.Object) error
// Remove remove `object`
Remove(ctx context.Context, object model.Object) error
// Put put `stream` to `parentDir`
Put(ctx context.Context, parentDir model.Object, stream model.FileStreamer) error
}

5
internal/fs/fs.go Normal file
View File

@ -0,0 +1,5 @@
package fs
// the param named path of functions in this package is a virtual path
// So, the purpose of this package is to convert virtual path to actual path
// then pass the actual path to the operations package

4
internal/fs/read.go
View File

@ -15,7 +15,7 @@ import (
// List files
// TODO: hide
// TODO: sort
func List(ctx context.Context, path string) ([]model.FileInfo, error) {
func List(ctx context.Context, path string) ([]model.Object, error) {
account, actualPath, err := operations.GetAccountAndActualPath(path)
virtualFiles := operations.GetAccountVirtualFilesByPath(path)
if err != nil {
@ -40,7 +40,7 @@ func List(ctx context.Context, path string) ([]model.FileInfo, error) {
return files, nil
}
func Get(ctx context.Context, path string) (model.FileInfo, error) {
func Get(ctx context.Context, path string) (model.Object, error) {
path = utils.StandardizationPath(path)
// maybe a virtual file
if path != "/" {

6
internal/fs/util.go
View File

@ -12,7 +12,7 @@ import (
"github.com/pkg/errors"
)
func containsByName(files []model.FileInfo, file model.FileInfo) bool {
func containsByName(files []model.Object, file model.Object) bool {
for _, f := range files {
if f.GetName() == file.GetName() {
return true
@ -23,7 +23,7 @@ func containsByName(files []model.FileInfo, file model.FileInfo) bool {
var httpClient = &http.Client{}
func getFileStreamFromLink(file model.FileInfo, link *model.Link) (model.FileStreamer, error) {
func getFileStreamFromLink(file model.Object, link *model.Link) (model.FileStreamer, error) {
var rc io.ReadCloser
mimetype := mime.TypeByExtension(stdpath.Ext(file.GetName()))
if link.Data != nil {
@ -57,7 +57,7 @@ func getFileStreamFromLink(file model.FileInfo, link *model.Link) (model.FileStr
mimetype = "application/octet-stream"
}
stream := model.FileStream{
FileInfo: file,
Object: file,
ReadCloser: rc,
Mimetype: mimetype,
}

6
internal/model/file.go
View File

@ -3,6 +3,7 @@ package model
import "time"
type File struct {
ID string
Name string
Size uint64
Modified time.Time
@ -25,7 +26,6 @@ func (f File) IsDir() bool {
return f.IsFolder
}
type FileWithId struct {
Id string
File
func (f File) GetID() string {
return f.ID
}

5
internal/model/ifile.go → internal/model/object.go
View File

@ -5,16 +5,17 @@ import (
"time"
)
type FileInfo interface {
type Object interface {
GetSize() uint64
GetName() string
ModTime() time.Time
IsDir() bool
GetID() string
}
type FileStreamer interface {
io.ReadCloser
FileInfo
Object
GetMimetype() string
}

2
internal/model/stream.go
View File

@ -5,7 +5,7 @@ import (
)
type FileStream struct {
FileInfo
Object
io.ReadCloser
Mimetype string
}

4
internal/operations/account.go
View File

@ -136,8 +136,8 @@ func getAccountsByPath(path string) []driver.Driver {
// GetAccountVirtualFilesByPath Obtain the virtual file generated by the account according to the path
// for example, there are: /a/b,/a/c,/a/d/e,/a/b.balance1,/av
// GetAccountVirtualFilesByPath(/a) => b,c,d
func GetAccountVirtualFilesByPath(prefix string) []model.FileInfo {
files := make([]model.FileInfo, 0)
func GetAccountVirtualFilesByPath(prefix string) []model.Object {
files := make([]model.Object, 0)
accounts := accountsMap.Values()
sort.Slice(accounts, func(i, j int) bool {
if accounts[i].GetAccount().Index == accounts[j].GetAccount().Index {

161
internal/operations/fs.go
View File

@ -6,6 +6,7 @@ import (
"time"
"github.com/Xhofe/go-cache"
"github.com/alist-org/alist/v3/conf"
"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/pkg/singleflight"
@ -15,50 +16,57 @@ import (
// In order to facilitate adding some other things before and after file operations
var filesCache = cache.NewMemCache(cache.WithShards[[]model.FileInfo](64))
var filesG singleflight.Group[[]model.FileInfo]
var filesCache = cache.NewMemCache(cache.WithShards[[]model.Object](64))
var filesG singleflight.Group[[]model.Object]
// List files in storage, not contains virtual file
func List(ctx context.Context, account driver.Driver, path string) ([]model.FileInfo, error) {
func List(ctx context.Context, account driver.Driver, path string) ([]model.Object, error) {
dir, err := Get(ctx, account, path)
if err != nil {
return nil, errors.WithMessage(err, "failed get dir")
}
if account.Config().NoCache {
return account.List(ctx, path)
return account.List(ctx, dir)
}
key := stdpath.Join(account.GetAccount().VirtualPath, path)
if files, ok := filesCache.Get(key); ok {
return files, nil
}
files, err, _ := filesG.Do(key, func() ([]model.FileInfo, error) {
files, err := account.List(ctx, path)
files, err, _ := filesG.Do(key, func() ([]model.Object, error) {
files, err := account.List(ctx, dir)
if err != nil {
return nil, errors.WithMessage(err, "failed to list files")
}
// TODO: get duration from global config or account's config
filesCache.Set(key, files, cache.WithEx[[]model.FileInfo](time.Minute*30))
// TODO: maybe can get duration from account's config
filesCache.Set(key, files, cache.WithEx[[]model.Object](time.Minute*time.Duration(conf.Conf.CaCheExpiration)))
return files, nil
})
return files, err
}
func Get(ctx context.Context, account driver.Driver, path string) (model.FileInfo, error) {
if r, ok := account.GetAddition().(driver.RootFolderId); ok && utils.PathEqual(path, "/") {
return model.FileWithId{
Id: r.GetRootFolderId(),
File: model.File{
Name: "root",
Size: 0,
Modified: account.GetAccount().Modified,
IsFolder: true,
},
}, nil
}
if r, ok := account.GetAddition().(driver.IRootFolderPath); ok && utils.PathEqual(path, r.GetRootFolderPath()) {
// Get get object from list of files
// TODO: maybe should set object ID with path here
func Get(ctx context.Context, account driver.Driver, path string) (model.Object, error) {
// is root folder
if r, ok := account.GetAddition().(driver.IRootFolderId); ok && utils.PathEqual(path, "/") {
return model.File{
ID: r.GetRootFolderId(),
Name: "root",
Size: 0,
Modified: account.GetAccount().Modified,
IsFolder: true,
}, nil
}
if r, ok := account.GetAddition().(driver.IRootFolderPath); ok && utils.PathEqual(path, r.GetRootFolderPath()) {
return model.File{
ID: r.GetRootFolderPath(),
Name: "root",
Size: 0,
Modified: account.GetAccount().Modified,
IsFolder: true,
}, nil
}
// not root folder
dir, name := stdpath.Split(path)
files, err := List(ctx, account, dir)
if err != nil {
@ -82,7 +90,11 @@ func Link(ctx context.Context, account driver.Driver, path string, args model.Li
return link, nil
}
fn := func() (*model.Link, error) {
link, err := account.Link(ctx, path, args)
file, err := Get(ctx, account, path)
if err != nil {
return nil, errors.WithMessage(err, "failed to get file")
}
link, err := account.Link(ctx, file, args)
if err != nil {
return nil, errors.WithMessage(err, "failed get link")
}
@ -98,54 +110,81 @@ func Link(ctx context.Context, account driver.Driver, path string, args model.Li
func MakeDir(ctx context.Context, account driver.Driver, path string) error {
// check if dir exists
f, err := Get(ctx, account, path)
if f != nil && f.IsDir() {
return nil
}
if err != nil && !driver.IsErrObjectNotFound(err) {
return errors.WithMessage(err, "failed to check if dir exists")
}
parentPath := stdpath.Dir(path)
err = MakeDir(ctx, account, parentPath)
if err != nil {
return errors.WithMessagef(err, "failed to make parent dir [%s]", parentPath)
}
return account.MakeDir(ctx, path)
}
func Move(ctx context.Context, account driver.Driver, srcPath, dstPath string) error {
return account.Move(ctx, srcPath, dstPath)
}
func Rename(ctx context.Context, account driver.Driver, srcPath, dstName string) error {
return account.Rename(ctx, srcPath, dstName)
}
// Copy Just copy file[s] in an account
func Copy(ctx context.Context, account driver.Driver, srcPath, dstPath string) error {
return account.Copy(ctx, srcPath, dstPath)
}
func Remove(ctx context.Context, account driver.Driver, path string) error {
return account.Remove(ctx, path)
}
func Put(ctx context.Context, account driver.Driver, parentPath string, file model.FileStreamer) error {
f, err := Get(ctx, account, parentPath)
if err != nil {
// if parent dir not exists, create it
if driver.IsErrObjectNotFound(err) {
parentPath, dirName := stdpath.Split(path)
err = MakeDir(ctx, account, parentPath)
if err != nil {
return errors.WithMessagef(err, "failed to make parent dir [%s]", parentPath)
}
parentDir, err := Get(ctx, account, parentPath)
// this should not happen
if err != nil {
return errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath)
}
return account.MakeDir(ctx, parentDir, dirName)
} else {
return errors.WithMessage(err, "failed to get parent dir")
return errors.WithMessage(err, "failed to check if dir exists")
}
} else {
// object exists, check if it is a dir
if !f.IsDir() {
return errors.Errorf("object [%s] is not a dir", parentPath)
// dir exists
if f.IsDir() {
return nil
} else {
// dir to make is a file
return errors.New("file exists")
}
}
return account.Put(ctx, parentPath, file)
}
func Move(ctx context.Context, account driver.Driver, srcPath, dstPath string) error {
srcObject, err := Get(ctx, account, srcPath)
if err != nil {
return errors.WithMessage(err, "failed to get src object")
}
dstDir, err := Get(ctx, account, stdpath.Dir(dstPath))
return account.Move(ctx, srcObject, dstDir)
}
func Rename(ctx context.Context, account driver.Driver, srcPath, dstName string) error {
srcObject, err := Get(ctx, account, srcPath)
if err != nil {
return errors.WithMessage(err, "failed to get src object")
}
return account.Rename(ctx, srcObject, dstName)
}
// Copy Just copy file[s] in an account
func Copy(ctx context.Context, account driver.Driver, srcPath, dstPath string) error {
srcObject, err := Get(ctx, account, srcPath)
if err != nil {
return errors.WithMessage(err, "failed to get src object")
}
dstDir, err := Get(ctx, account, stdpath.Dir(dstPath))
return account.Copy(ctx, srcObject, dstDir)
}
func Remove(ctx context.Context, account driver.Driver, path string) error {
object, err := Get(ctx, account, path)
if err != nil {
// if object not found, it's ok
if driver.IsErrObjectNotFound(err) {
return nil
}
return errors.WithMessage(err, "failed to get object")
}
return account.Remove(ctx, object)
}
func Put(ctx context.Context, account driver.Driver, parentPath string, file model.FileStreamer) error {
err := MakeDir(ctx, account, parentPath)
if err != nil {
return errors.WithMessagef(err, "failed to make dir [%s]", parentPath)
}
parentDir, err := Get(ctx, account, parentPath)
// this should not happen
if err != nil {
return errors.WithMessagef(err, "failed to get dir [%s]", parentPath)
}
return account.Put(ctx, parentDir, file)
}

10
internal/operations/path.go
View File

@ -1,22 +1,24 @@
package operations
import (
stdpath "path"
"strings"
"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"path"
"strings"
)
func ActualPath(account driver.Additional, rawPath string) string {
if i, ok := account.(driver.IRootFolderPath); ok {
rawPath = path.Join(i.GetRootFolderPath(), rawPath)
rawPath = stdpath.Join(i.GetRootFolderPath(), rawPath)
}
return utils.StandardizationPath(rawPath)
}
// GetAccountAndActualPath Get the corresponding account, and remove the virtual path prefix in path
// GetAccountAndActualPath Get the corresponding account
// for path: remove the virtual path prefix and join the actual root folder if exists
func GetAccountAndActualPath(rawPath string) (driver.Driver, string, error) {
rawPath = utils.StandardizationPath(rawPath)
account := GetBalancedAccount(rawPath)