From ffba5e0aec34d290ed3bc3241ea9b43acfec9f07 Mon Sep 17 00:00:00 2001 From: Noah Hsu Date: Sun, 4 Sep 2022 12:43:52 +0800 Subject: [PATCH] feat: add sftp driver (close #1466) --- drivers/all.go | 1 + drivers/ftp/meta.go | 10 ++-- drivers/sftp/driver.go | 106 +++++++++++++++++++++++++++++++++++++++++ drivers/sftp/meta.go | 30 ++++++++++++ drivers/sftp/types.go | 16 +++++++ drivers/sftp/util.go | 72 ++++++++++++++++++++++++++++ go.mod | 6 ++- go.sum | 8 ++++ internal/op/fs.go | 16 +++---- 9 files changed, 248 insertions(+), 17 deletions(-) create mode 100644 drivers/sftp/driver.go create mode 100644 drivers/sftp/meta.go create mode 100644 drivers/sftp/types.go create mode 100644 drivers/sftp/util.go diff --git a/drivers/all.go b/drivers/all.go index 64d666a8..4d4a39e3 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -11,6 +11,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/pikpak" _ "github.com/alist-org/alist/v3/drivers/quark" _ "github.com/alist-org/alist/v3/drivers/s3" + _ "github.com/alist-org/alist/v3/drivers/sftp" _ "github.com/alist-org/alist/v3/drivers/teambition" _ "github.com/alist-org/alist/v3/drivers/virtual" ) diff --git a/drivers/ftp/meta.go b/drivers/ftp/meta.go index f235ec5b..bb21267f 100644 --- a/drivers/ftp/meta.go +++ b/drivers/ftp/meta.go @@ -14,13 +14,9 @@ type Addition struct { var config = driver.Config{ Name: "FTP", - LocalSort: false, - OnlyLocal: false, - OnlyProxy: false, - NoCache: false, - NoUpload: false, - NeedMs: false, - DefaultRoot: "root, / or other", + LocalSort: true, + OnlyLocal: true, + DefaultRoot: "/", } func New() driver.Driver { diff --git a/drivers/sftp/driver.go b/drivers/sftp/driver.go new file mode 100644 index 00000000..c68e99bd --- /dev/null +++ b/drivers/sftp/driver.go @@ -0,0 +1,106 @@ +package sftp + +import ( + "context" + "os" + "path" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/pkg/sftp" +) + +type SFTP struct { + model.Storage + Addition + client *sftp.Client +} + +func (d *SFTP) Config() driver.Config { + return config +} + +func (d *SFTP) GetAddition() driver.Additional { + return d.Addition +} + +func (d *SFTP) Init(ctx context.Context, storage model.Storage) error { + d.Storage = storage + err := utils.Json.UnmarshalFromString(d.Storage.Addition, &d.Addition) + if err != nil { + return err + } + return d.initClient() +} + +func (d *SFTP) Drop(ctx context.Context) error { + if d.client != nil { + _ = d.client.Close() + } + return nil +} + +func (d *SFTP) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files, err := d.client.ReadDir(dir.GetPath()) + if err != nil { + return nil, err + } + return utils.SliceConvert(files, func(src os.FileInfo) (model.Obj, error) { + return fileToObj(src), nil + }) +} + +//func (d *SFTP) Get(ctx context.Context, path string) (model.Obj, error) { +// // this is optional +// return nil, errs.NotImplement +//} + +func (d *SFTP) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + remoteFile, err := d.client.Open(file.GetPath()) + if err != nil { + return nil, err + } + return &model.Link{ + Data: remoteFile, + }, nil +} + +func (d *SFTP) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + return d.client.MkdirAll(path.Join(parentDir.GetPath(), dirName)) +} + +func (d *SFTP) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + return d.client.Rename(srcObj.GetPath(), path.Join(dstDir.GetPath(), srcObj.GetName())) +} + +func (d *SFTP) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + return d.client.Rename(srcObj.GetPath(), path.Join(path.Dir(srcObj.GetPath()), newName)) +} + +func (d *SFTP) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + return errs.NotSupport +} + +func (d *SFTP) Remove(ctx context.Context, obj model.Obj) error { + return d.remove(obj.GetPath()) +} + +func (d *SFTP) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + dstFile, err := d.client.Create(path.Join(dstDir.GetPath(), stream.GetName())) + if err != nil { + return err + } + defer func() { + _ = dstFile.Close() + }() + err = utils.CopyWithCtx(ctx, dstFile, stream, stream.GetSize(), up) + return err +} + +func (d *SFTP) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { + return nil, errs.NotSupport +} + +var _ driver.Driver = (*SFTP)(nil) diff --git a/drivers/sftp/meta.go b/drivers/sftp/meta.go new file mode 100644 index 00000000..e5c056a8 --- /dev/null +++ b/drivers/sftp/meta.go @@ -0,0 +1,30 @@ +package sftp + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + Address string `json:"address" required:"true"` + Username string `json:"username" required:"true"` + PrivateKey string `json:"private_key" type:"text"` + Password string `json:"password"` + driver.RootFolderPath +} + +var config = driver.Config{ + Name: "SFTP", + LocalSort: true, + OnlyLocal: true, + DefaultRoot: "/", + CheckStatus: true, +} + +func New() driver.Driver { + return &SFTP{} +} + +func init() { + op.RegisterDriver(config, New) +} diff --git a/drivers/sftp/types.go b/drivers/sftp/types.go new file mode 100644 index 00000000..819ab168 --- /dev/null +++ b/drivers/sftp/types.go @@ -0,0 +1,16 @@ +package sftp + +import ( + "os" + + "github.com/alist-org/alist/v3/internal/model" +) + +func fileToObj(f os.FileInfo) model.Obj { + return &model.Object{ + Name: f.Name(), + Size: f.Size(), + Modified: f.ModTime(), + IsFolder: f.IsDir(), + } +} diff --git a/drivers/sftp/util.go b/drivers/sftp/util.go new file mode 100644 index 00000000..3deb8dcf --- /dev/null +++ b/drivers/sftp/util.go @@ -0,0 +1,72 @@ +package sftp + +import ( + "path" + + "github.com/pkg/sftp" + "golang.org/x/crypto/ssh" +) + +// do others that not defined in Driver interface + +func (d *SFTP) initClient() error { + var auth ssh.AuthMethod + if d.PrivateKey != "" { + signer, err := ssh.ParsePrivateKey([]byte(d.PrivateKey)) + if err != nil { + return err + } + auth = ssh.PublicKeys(signer) + } else { + auth = ssh.Password(d.Password) + } + config := &ssh.ClientConfig{ + User: d.Username, + Auth: []ssh.AuthMethod{auth}, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } + conn, err := ssh.Dial("tcp", d.Address, config) + if err != nil { + return err + } + d.client, err = sftp.NewClient(conn) + return err +} + +func (d *SFTP) remove(remotePath string) error { + f, err := d.client.Stat(remotePath) + if err != nil { + return nil + } + if f.IsDir() { + return d.removeDirectory(remotePath) + } else { + return d.removeFile(remotePath) + } +} + +func (d *SFTP) removeDirectory(remotePath string) error { + remoteFiles, err := d.client.ReadDir(remotePath) + if err != nil { + return err + } + for _, backupDir := range remoteFiles { + remoteFilePath := path.Join(remotePath, backupDir.Name()) + if backupDir.IsDir() { + err := d.removeDirectory(remoteFilePath) + if err != nil { + return err + } + } else { + err := d.removeFile(remoteFilePath) + if err != nil { + return err + } + } + } + return d.client.RemoveDirectory(remotePath) +} + +func (d *SFTP) removeFile(remotePath string) error { + return d.client.Remove(path.Join(remotePath)) +} diff --git a/go.mod b/go.mod index 922f7d24..8abb7577 100644 --- a/go.mod +++ b/go.mod @@ -49,18 +49,20 @@ require ( github.com/jinzhu/now v1.1.5 // indirect github.com/jlaffaye/ftp v0.0.0-20220829015825-b85cf1edccd4 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-sqlite3 v1.14.13 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.0.1 // indirect + github.com/pkg/sftp v1.13.5 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/ugorji/go/codec v1.2.7 // indirect - golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect + golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect golang.org/x/image v0.0.0-20220722155232-062f8c9fd539 // indirect golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b // indirect - golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect + golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 // indirect golang.org/x/text v0.3.7 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index 2f8970ee..810bc507 100644 --- a/go.sum +++ b/go.sum @@ -132,6 +132,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= @@ -174,6 +176,8 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.13.5 h1:a3RLUqkyjYRtBTZJZ1VRrKbN3zhuPLlUc3sphVz81go= +github.com/pkg/sftp v1.13.5/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/otp v1.3.0 h1:oJV/SkzR33anKXwQU3Of42rL4wbrffP4uvUf1SvS5Xs= @@ -242,6 +246,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM= +golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20220722155232-062f8c9fd539 h1:/eM0PCrQI2xd471rI+snWuu251/+/jpBpZqir2mPdnU= golang.org/x/image v0.0.0-20220722155232-062f8c9fd539/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY= @@ -281,6 +287,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbuf golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 h1:v6hYoSR9T5oet+pMXwUWkbiVqx/63mlHjefrHmxwfeY= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/internal/op/fs.go b/internal/op/fs.go index ebb36b04..c32728ea 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -29,7 +29,7 @@ func ClearCache(storage driver.Driver, path string) { // List files in storage, not contains virtual file func List(ctx context.Context, storage driver.Driver, path string, args model.ListArgs, refresh ...bool) ([]model.Obj, error) { - if storage.Config().CheckStatus && storage.GetStorage().Status != "ok" { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) } path = utils.StandardizePath(path) @@ -132,7 +132,7 @@ var linkG singleflight.Group[*model.Link] // Link get link, if is an url. should have an expiry time func Link(ctx context.Context, storage driver.Driver, path string, args model.LinkArgs) (*model.Link, model.Obj, error) { - if storage.Config().CheckStatus && storage.GetStorage().Status != "ok" { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return nil, nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) } file, err := Get(ctx, storage, path) @@ -174,7 +174,7 @@ func Other(ctx context.Context, storage driver.Driver, args model.FsOtherArgs) ( } func MakeDir(ctx context.Context, storage driver.Driver, path string) error { - if storage.Config().CheckStatus && storage.GetStorage().Status != "ok" { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return errors.Errorf("storage not init: %s", storage.GetStorage().Status) } // check if dir exists @@ -207,7 +207,7 @@ func MakeDir(ctx context.Context, storage driver.Driver, path string) error { } func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string) error { - if storage.Config().CheckStatus && storage.GetStorage().Status != "ok" { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return errors.Errorf("storage not init: %s", storage.GetStorage().Status) } srcObj, err := Get(ctx, storage, srcPath) @@ -222,7 +222,7 @@ func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string } func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string) error { - if storage.Config().CheckStatus && storage.GetStorage().Status != "ok" { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return errors.Errorf("storage not init: %s", storage.GetStorage().Status) } srcObj, err := Get(ctx, storage, srcPath) @@ -234,7 +234,7 @@ func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string) // Copy Just copy file[s] in a storage func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string) error { - if storage.Config().CheckStatus && storage.GetStorage().Status != "ok" { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return errors.Errorf("storage not init: %s", storage.GetStorage().Status) } srcObj, err := Get(ctx, storage, srcPath) @@ -246,7 +246,7 @@ func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string } func Remove(ctx context.Context, storage driver.Driver, path string) error { - if storage.Config().CheckStatus && storage.GetStorage().Status != "ok" { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return errors.Errorf("storage not init: %s", storage.GetStorage().Status) } obj, err := Get(ctx, storage, path) @@ -261,7 +261,7 @@ func Remove(ctx context.Context, storage driver.Driver, path string) error { } func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file model.FileStreamer, up driver.UpdateProgress) error { - if storage.Config().CheckStatus && storage.GetStorage().Status != "ok" { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return errors.Errorf("storage not init: %s", storage.GetStorage().Status) } defer func() {