diff --git a/drivers/all.go b/drivers/all.go index e953ad86..4b875781 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -11,5 +11,6 @@ import ( _ "github.com/Xhofe/alist/drivers/native" _ "github.com/Xhofe/alist/drivers/onedrive" _ "github.com/Xhofe/alist/drivers/pikpak" + _ "github.com/Xhofe/alist/drivers/s3" _ "github.com/Xhofe/alist/drivers/shandian" ) diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go new file mode 100644 index 00000000..15f682fd --- /dev/null +++ b/drivers/s3/driver.go @@ -0,0 +1,253 @@ +package s3 + +import ( + "fmt" + "github.com/Xhofe/alist/conf" + "github.com/Xhofe/alist/drivers/base" + "github.com/Xhofe/alist/model" + "github.com/Xhofe/alist/utils" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/gin-gonic/gin" + log "github.com/sirupsen/logrus" + "net/url" + "path/filepath" + "strings" + "time" +) + +type S3 struct { +} + +func (driver S3) Config() base.DriverConfig { + return base.DriverConfig{ + Name: "S3", + LocalSort: true, + } +} + +func (driver S3) Items() []base.Item { + return []base.Item{ + { + Name: "bucket", + Label: "Bucket", + Type: base.TypeString, + Required: true, + }, + { + Name: "endpoint", + Label: "Endpoint", + Type: base.TypeString, + Required: true, + }, + { + Name: "region", + Label: "Region", + Type: base.TypeString, + Required: true, + }, + { + Name: "access_key", + Label: "Access Key", + Type: base.TypeString, + Required: true, + }, + { + Name: "access_secret", + Label: "Access Secret", + Type: base.TypeString, + Required: true, + }, + { + Name: "custom_host", + Label: "Custom Host", + Type: base.TypeString, + }, + { + Name: "limit", + Label: "url expire time(hours)", + Type: base.TypeNumber, + Description: "default 4 hours", + }, + } +} + +func (driver S3) Save(account *model.Account, old *model.Account) error { + if account.Limit == 0 { + account.Limit = 4 + } + client, err := driver.NewSession(account) + if err != nil { + account.Status = err.Error() + } else { + sessionsMap[account.Name] = client + account.Status = "work" + } + _ = model.SaveAccount(account) + return err +} + +func (driver S3) File(path string, account *model.Account) (*model.File, error) { + path = utils.ParsePath(path) + if path == "/" { + return &model.File{ + Id: account.RootFolder, + Name: account.Name, + Size: 0, + Type: conf.FOLDER, + Driver: driver.Config().Name, + UpdatedAt: account.UpdatedAt, + }, nil + } + dir, name := filepath.Split(path) + files, err := driver.Files(dir, account) + if err != nil { + return nil, err + } + for _, file := range files { + if file.Name == name { + return &file, nil + } + } + return nil, base.ErrPathNotFound +} + +func (driver S3) Files(path string, account *model.Account) ([]model.File, error) { + path = utils.ParsePath(path) + var files []model.File + cache, err := base.GetCache(path, account) + if err == nil { + files, _ = cache.([]model.File) + } else { + files, err = driver.List(path, account) + if err == nil && len(files) > 0 { + _ = base.SetCache(path, files, account) + } + } + return files, err +} + +func (driver S3) Link(args base.Args, account *model.Account) (*base.Link, error) { + client, err := driver.GetClient(account) + if err != nil { + return nil, err + } + path := strings.TrimPrefix(args.Path, "/") + disposition := fmt.Sprintf(`attachment;filename="%s"`, url.QueryEscape(utils.Base(path))) + input := &s3.GetObjectInput{ + Bucket: &account.Bucket, + Key: &path, + ResponseContentDisposition: &disposition, + } + req, _ := client.GetObjectRequest(input) + link, err := req.Presign(time.Hour * time.Duration(account.Limit)) + if err != nil { + return nil, err + } + return &base.Link{ + Url: link, + }, nil +} + +func (driver S3) Path(path string, account *model.Account) (*model.File, []model.File, error) { + path = utils.ParsePath(path) + log.Debugf("s3 path: %s", path) + file, err := driver.File(path, account) + if err != nil { + return nil, nil, err + } + if !file.IsDir() { + return file, nil, nil + } + files, err := driver.Files(path, account) + if err != nil { + return nil, nil, err + } + return nil, files, nil +} + +func (driver S3) Proxy(c *gin.Context, account *model.Account) { + +} + +func (driver S3) Preview(path string, account *model.Account) (interface{}, error) { + return nil, base.ErrNotSupport +} + +func (driver S3) MakeDir(path string, account *model.Account) error { + // not support, default as success + return nil +} + +func (driver S3) Move(src string, dst string, account *model.Account) error { + err := driver.Copy(src, dst, account) + if err != nil { + return err + } + return driver.Delete(src, account) +} + +func (driver S3) Copy(src string, dst string, account *model.Account) error { + client, err := driver.GetClient(account) + if err != nil { + return err + } + srcFile, err := driver.File(src, account) + if err != nil { + return err + } + srcKey := driver.GetKey(src, account, srcFile.IsDir()) + dstKey := driver.GetKey(dst, account, srcFile.IsDir()) + input := &s3.CopyObjectInput{ + Bucket: &account.Bucket, + CopySource: &srcKey, + Key: &dstKey, + } + _, err = client.CopyObject(input) + if err == nil { + _ = base.DeleteCache(dst, account) + } + return err +} + +func (driver S3) Delete(path string, account *model.Account) error { + client, err := driver.GetClient(account) + if err != nil { + return err + } + file, err := driver.File(path, account) + if err != nil { + return err + } + key := driver.GetKey(path, account, file.IsDir()) + input := &s3.DeleteObjectInput{ + Bucket: &account.Bucket, + Key: &key, + } + _, err = client.DeleteObject(input) + if err == nil { + _ = base.DeleteCache(utils.Dir(path), account) + } + return err +} + +func (driver S3) Upload(file *model.FileStream, account *model.Account) error { + s, ok := sessionsMap[account.Name] + if !ok { + return fmt.Errorf("can't find [%s] session", account.Name) + } + uploader := s3manager.NewUploader(s) + key := driver.GetKey(utils.Join(file.ParentPath, file.GetFileName()), account, false) + input := &s3manager.UploadInput{ + Bucket: &account.Bucket, + Key: &key, + Body: file, + } + _, err := uploader.Upload(input) + if err == nil { + _ = base.DeleteCache(file.ParentPath, account) + } + return err +} + +var _ base.Driver = (*S3)(nil) diff --git a/drivers/s3/s3.go b/drivers/s3/s3.go new file mode 100644 index 00000000..bfccb01f --- /dev/null +++ b/drivers/s3/s3.go @@ -0,0 +1,118 @@ +package s3 + +import ( + "fmt" + "github.com/Xhofe/alist/conf" + "github.com/Xhofe/alist/drivers/base" + "github.com/Xhofe/alist/model" + "github.com/Xhofe/alist/utils" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + log "github.com/sirupsen/logrus" + "net/http" + "net/url" + "path" + "strings" +) + +var sessionsMap map[string]*session.Session + +func (driver S3) NewSession(account *model.Account) (*session.Session, error) { + cfg := &aws.Config{ + Credentials: credentials.NewStaticCredentials(account.AccessKey, account.AccessSecret, ""), + Region: &account.Region, + Endpoint: &account.Endpoint, + } + return session.NewSession(cfg) +} + +func (driver S3) GetClient(account *model.Account) (*s3.S3, error) { + s, ok := sessionsMap[account.Name] + if !ok { + return nil, fmt.Errorf("can't find [%s] session", account.Name) + } + client := s3.New(s) + if account.CustomHost != "" { + cURL, err := url.Parse(account.CustomHost) + if err != nil { + return nil, err + } + client.Handlers.Build.PushBack(func(r *request.Request) { + if r.HTTPRequest.Method != http.MethodGet { + return + } + r.HTTPRequest.URL.Scheme = cURL.Scheme + r.HTTPRequest.URL.Host = cURL.Host + }) + } + return client, nil +} + +func (driver S3) List(prefix string, account *model.Account) ([]model.File, error) { + prefix = driver.GetKey(prefix, account, true) + log.Debugf("list: %s", prefix) + client, err := driver.GetClient(account) + if err != nil { + return nil, err + } + files := make([]model.File, 0) + marker := "" + for { + input := &s3.ListObjectsInput{ + Bucket: &account.Bucket, + Marker: &marker, + Prefix: &prefix, + Delimiter: aws.String("/"), + } + listObjectsResult, err := client.ListObjects(input) + + if err != nil { + return nil, err + } + for _, object := range listObjectsResult.CommonPrefixes { + file := model.File{ + //Id: *object.Key, + Name: utils.Base(strings.Trim(*object.Prefix, "/")), + Driver: driver.Config().Name, + UpdatedAt: account.UpdatedAt, + TimeStr: "-", + Type: conf.FOLDER, + } + files = append(files, file) + } + for _, object := range listObjectsResult.Contents { + file := model.File{ + //Id: *object.Key, + Name: utils.Base(*object.Key), + Size: *object.Size, + Driver: driver.Config().Name, + UpdatedAt: object.LastModified, + Type: utils.GetFileType(path.Ext(*object.Key)), + } + files = append(files, file) + } + if *listObjectsResult.IsTruncated { + marker = *listObjectsResult.NextMarker + } else { + break + } + } + return files, nil +} + +func (driver S3) GetKey(path string, account *model.Account, dir bool) string { + path = utils.Join(account.RootFolder, path) + path = strings.TrimPrefix(path, "/") + if path != "" && dir { + path += "/" + } + return path +} + +func init() { + sessionsMap = make(map[string]*session.Session, 0) + base.RegisterDriver(&S3{}) +} diff --git a/go.mod b/go.mod index 151fdc23..28b6ee9d 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,17 @@ module github.com/Xhofe/alist go 1.17 require ( + github.com/aws/aws-sdk-go v1.27.0 github.com/eko/gocache/v2 v2.1.0 + github.com/gin-contrib/cors v1.3.1 github.com/gin-gonic/gin v1.7.4 - github.com/go-playground/validator/v10 v10.9.0 github.com/go-resty/resty/v2 v2.6.0 + github.com/jlaffaye/ftp v0.0.0-20211117213618-11820403398b + github.com/json-iterator/go v1.1.12 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/robfig/cron/v3 v3.0.0 github.com/sirupsen/logrus v1.8.1 + golang.org/x/text v0.3.7 gorm.io/driver/mysql v1.1.2 gorm.io/driver/postgres v1.1.2 gorm.io/driver/sqlite v1.1.6 @@ -23,10 +27,10 @@ require ( github.com/cenkalti/backoff/v4 v4.1.0 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/gin-contrib/cors v1.3.1 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-playground/locales v0.14.0 // indirect github.com/go-playground/universal-translator v0.18.0 // indirect + github.com/go-playground/validator/v10 v10.9.0 // indirect github.com/go-redis/redis/v8 v8.9.0 // indirect github.com/go-sql-driver/mysql v1.6.0 // indirect github.com/golang/protobuf v1.5.2 // indirect @@ -40,8 +44,7 @@ require ( github.com/jackc/pgx/v4 v4.13.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.2 // indirect - github.com/jlaffaye/ftp v0.0.0-20211117213618-11820403398b // indirect - github.com/json-iterator/go v1.1.12 // indirect + github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-sqlite3 v1.14.9 // indirect @@ -59,9 +62,8 @@ require ( go.opentelemetry.io/otel/metric v0.20.0 // indirect go.opentelemetry.io/otel/trace v0.20.0 // indirect golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect - golang.org/x/net v0.0.0-20210510120150-4163338589ed // indirect + golang.org/x/net v0.0.0-20211209124913-491a49abca63 // indirect golang.org/x/sys v0.0.0-20211023085530-d6a326fbbf70 // indirect - golang.org/x/text v0.3.7 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 // indirect diff --git a/go.sum b/go.sum index 9df5f657..4642b1cf 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,7 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= +github.com/aws/aws-sdk-go v1.27.0 h1:0xphMHGMLBrPMfxR2AmVjZKcMEESEgWF8Kru94BNByk= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -268,6 +269,7 @@ github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI= github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jlaffaye/ftp v0.0.0-20211117213618-11820403398b h1:Ur6QAxsHCK99Quj9PaWafoV4unb0DO/HWiKExD+TN5g= github.com/jlaffaye/ftp v0.0.0-20211117213618-11820403398b/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -569,8 +571,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210510120150-4163338589ed h1:p9UgmWI9wKpfYmgaV/IZKGdXc5qEK45tDwwwDyjS26I= -golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211209124913-491a49abca63 h1:iocB37TsdFuN6IBRZ+ry36wrkoV51/tl5vOWqkcPGvY= +golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/model/account.go b/model/account.go index 3050b65e..0bd22388 100644 --- a/model/account.go +++ b/model/account.go @@ -36,6 +36,13 @@ type Account struct { //AllowProxy bool `json:"allow_proxy"` // 是否允许中转下载 DownProxyUrl string `json:"down_proxy_url"` // 用于中转下载服务的URL 两处 1. path请求中返回的链接 2. down下载时进行302 APIProxyUrl string `json:"api_proxy_url"` // 用于中转api的地址 + // for s3 + Bucket string `json:"bucket"` + Endpoint string `json:"endpoint"` + Region string `json:"region"` + AccessKey string `json:"access_key"` + AccessSecret string `json:"access_secret"` + CustomHost string `json:"custom_host"` } var accountsMap = map[string]Account{}