github
/
alist
mirror of https://github.com/Xhofe/alist
1
0
Fork 0
Code Issues Releases Wiki Activity

🎇 s3 support

pull/548/head
微凉 2021-12-30 20:31:36 +08:00
parent db06b627cc
commit cf2506901f
6 changed files with 391 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
drivers/all.go
View File

@ -11,5 +11,6 @@ import (
_ "github.com/Xhofe/alist/drivers/native"
_ "github.com/Xhofe/alist/drivers/onedrive"
_ "github.com/Xhofe/alist/drivers/pikpak"
_ "github.com/Xhofe/alist/drivers/s3"
_ "github.com/Xhofe/alist/drivers/shandian"
)

253
drivers/s3/driver.go Normal file
View File

@ -0,0 +1,253 @@
package s3
import (
"fmt"
"github.com/Xhofe/alist/conf"
"github.com/Xhofe/alist/drivers/base"
"github.com/Xhofe/alist/model"
"github.com/Xhofe/alist/utils"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/gin-gonic/gin"
log "github.com/sirupsen/logrus"
"net/url"
"path/filepath"
"strings"
"time"
)
type S3 struct {
}
func (driver S3) Config() base.DriverConfig {
return base.DriverConfig{
Name: "S3",
LocalSort: true,
}
}
func (driver S3) Items() []base.Item {
return []base.Item{
{
Name: "bucket",
Label: "Bucket",
Type: base.TypeString,
Required: true,
},
{
Name: "endpoint",
Label: "Endpoint",
Type: base.TypeString,
Required: true,
},
{
Name: "region",
Label: "Region",
Type: base.TypeString,
Required: true,
},
{
Name: "access_key",
Label: "Access Key",
Type: base.TypeString,
Required: true,
},
{
Name: "access_secret",
Label: "Access Secret",
Type: base.TypeString,
Required: true,
},
{
Name: "custom_host",
Label: "Custom Host",
Type: base.TypeString,
},
{
Name: "limit",
Label: "url expire time(hours)",
Type: base.TypeNumber,
Description: "default 4 hours",
},
}
}
func (driver S3) Save(account *model.Account, old *model.Account) error {
if account.Limit == 0 {
account.Limit = 4
}
client, err := driver.NewSession(account)
if err != nil {
account.Status = err.Error()
} else {
sessionsMap[account.Name] = client
account.Status = "work"
}
_ = model.SaveAccount(account)
return err
}
func (driver S3) File(path string, account *model.Account) (*model.File, error) {
path = utils.ParsePath(path)
if path == "/" {
return &model.File{
Id: account.RootFolder,
Name: account.Name,
Size: 0,
Type: conf.FOLDER,
Driver: driver.Config().Name,
UpdatedAt: account.UpdatedAt,
}, nil
}
dir, name := filepath.Split(path)
files, err := driver.Files(dir, account)
if err != nil {
return nil, err
}
for _, file := range files {
if file.Name == name {
return &file, nil
}
}
return nil, base.ErrPathNotFound
}
func (driver S3) Files(path string, account *model.Account) ([]model.File, error) {
path = utils.ParsePath(path)
var files []model.File
cache, err := base.GetCache(path, account)
if err == nil {
files, _ = cache.([]model.File)
} else {
files, err = driver.List(path, account)
if err == nil && len(files) > 0 {
_ = base.SetCache(path, files, account)
}
}
return files, err
}
func (driver S3) Link(args base.Args, account *model.Account) (*base.Link, error) {
client, err := driver.GetClient(account)
if err != nil {
return nil, err
}
path := strings.TrimPrefix(args.Path, "/")
disposition := fmt.Sprintf(`attachment;filename="%s"`, url.QueryEscape(utils.Base(path)))
input := &s3.GetObjectInput{
Bucket: &account.Bucket,
Key: &path,
ResponseContentDisposition: &disposition,
}
req, _ := client.GetObjectRequest(input)
link, err := req.Presign(time.Hour * time.Duration(account.Limit))
if err != nil {
return nil, err
}
return &base.Link{
Url: link,
}, nil
}
func (driver S3) Path(path string, account *model.Account) (*model.File, []model.File, error) {
path = utils.ParsePath(path)
log.Debugf("s3 path: %s", path)
file, err := driver.File(path, account)
if err != nil {
return nil, nil, err
}
if !file.IsDir() {
return file, nil, nil
}
files, err := driver.Files(path, account)
if err != nil {
return nil, nil, err
}
return nil, files, nil
}
func (driver S3) Proxy(c *gin.Context, account *model.Account) {
}
func (driver S3) Preview(path string, account *model.Account) (interface{}, error) {
return nil, base.ErrNotSupport
}
func (driver S3) MakeDir(path string, account *model.Account) error {
// not support, default as success
return nil
}
func (driver S3) Move(src string, dst string, account *model.Account) error {
err := driver.Copy(src, dst, account)
if err != nil {
return err
}
return driver.Delete(src, account)
}
func (driver S3) Copy(src string, dst string, account *model.Account) error {
client, err := driver.GetClient(account)
if err != nil {
return err
}
srcFile, err := driver.File(src, account)
if err != nil {
return err
}
srcKey := driver.GetKey(src, account, srcFile.IsDir())
dstKey := driver.GetKey(dst, account, srcFile.IsDir())
input := &s3.CopyObjectInput{
Bucket: &account.Bucket,
CopySource: &srcKey,
Key: &dstKey,
}
_, err = client.CopyObject(input)
if err == nil {
_ = base.DeleteCache(dst, account)
}
return err
}
func (driver S3) Delete(path string, account *model.Account) error {
client, err := driver.GetClient(account)
if err != nil {
return err
}
file, err := driver.File(path, account)
if err != nil {
return err
}
key := driver.GetKey(path, account, file.IsDir())
input := &s3.DeleteObjectInput{
Bucket: &account.Bucket,
Key: &key,
}
_, err = client.DeleteObject(input)
if err == nil {
_ = base.DeleteCache(utils.Dir(path), account)
}
return err
}
func (driver S3) Upload(file *model.FileStream, account *model.Account) error {
s, ok := sessionsMap[account.Name]
if !ok {
return fmt.Errorf("can't find [%s] session", account.Name)
}
uploader := s3manager.NewUploader(s)
key := driver.GetKey(utils.Join(file.ParentPath, file.GetFileName()), account, false)
input := &s3manager.UploadInput{
Bucket: &account.Bucket,
Key: &key,
Body: file,
}
_, err := uploader.Upload(input)
if err == nil {
_ = base.DeleteCache(file.ParentPath, account)
}
return err
}
var _ base.Driver = (*S3)(nil)

118
drivers/s3/s3.go Normal file
View File

@ -0,0 +1,118 @@
package s3
import (
"fmt"
"github.com/Xhofe/alist/conf"
"github.com/Xhofe/alist/drivers/base"
"github.com/Xhofe/alist/model"
"github.com/Xhofe/alist/utils"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
log "github.com/sirupsen/logrus"
"net/http"
"net/url"
"path"
"strings"
)
var sessionsMap map[string]*session.Session
func (driver S3) NewSession(account *model.Account) (*session.Session, error) {
cfg := &aws.Config{
Credentials: credentials.NewStaticCredentials(account.AccessKey, account.AccessSecret, ""),
Region: &account.Region,
Endpoint: &account.Endpoint,
}
return session.NewSession(cfg)
}
func (driver S3) GetClient(account *model.Account) (*s3.S3, error) {
s, ok := sessionsMap[account.Name]
if !ok {
return nil, fmt.Errorf("can't find [%s] session", account.Name)
}
client := s3.New(s)
if account.CustomHost != "" {
cURL, err := url.Parse(account.CustomHost)
if err != nil {
return nil, err
}
client.Handlers.Build.PushBack(func(r *request.Request) {
if r.HTTPRequest.Method != http.MethodGet {
return
}
r.HTTPRequest.URL.Scheme = cURL.Scheme
r.HTTPRequest.URL.Host = cURL.Host
})
}
return client, nil
}
func (driver S3) List(prefix string, account *model.Account) ([]model.File, error) {
prefix = driver.GetKey(prefix, account, true)
log.Debugf("list: %s", prefix)
client, err := driver.GetClient(account)
if err != nil {
return nil, err
}
files := make([]model.File, 0)
marker := ""
for {
input := &s3.ListObjectsInput{
Bucket: &account.Bucket,
Marker: &marker,
Prefix: &prefix,
Delimiter: aws.String("/"),
}
listObjectsResult, err := client.ListObjects(input)
if err != nil {
return nil, err
}
for _, object := range listObjectsResult.CommonPrefixes {
file := model.File{
//Id: *object.Key,
Name: utils.Base(strings.Trim(*object.Prefix, "/")),
Driver: driver.Config().Name,
UpdatedAt: account.UpdatedAt,
TimeStr: "-",
Type: conf.FOLDER,
}
files = append(files, file)
}
for _, object := range listObjectsResult.Contents {
file := model.File{
//Id: *object.Key,
Name: utils.Base(*object.Key),
Size: *object.Size,
Driver: driver.Config().Name,
UpdatedAt: object.LastModified,
Type: utils.GetFileType(path.Ext(*object.Key)),
}
files = append(files, file)
}
if *listObjectsResult.IsTruncated {
marker = *listObjectsResult.NextMarker
} else {
break
}
}
return files, nil
}
func (driver S3) GetKey(path string, account *model.Account, dir bool) string {
path = utils.Join(account.RootFolder, path)
path = strings.TrimPrefix(path, "/")
if path != "" && dir {
path += "/"
}
return path
}
func init() {
sessionsMap = make(map[string]*session.Session, 0)
base.RegisterDriver(&S3{})
}

14
go.mod
View File

@ -3,13 +3,17 @@ module github.com/Xhofe/alist
go 1.17
require (
github.com/aws/aws-sdk-go v1.27.0
github.com/eko/gocache/v2 v2.1.0
github.com/gin-contrib/cors v1.3.1
github.com/gin-gonic/gin v1.7.4
github.com/go-playground/validator/v10 v10.9.0
github.com/go-resty/resty/v2 v2.6.0
github.com/jlaffaye/ftp v0.0.0-20211117213618-11820403398b
github.com/json-iterator/go v1.1.12
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/robfig/cron/v3 v3.0.0
github.com/sirupsen/logrus v1.8.1
golang.org/x/text v0.3.7
gorm.io/driver/mysql v1.1.2
gorm.io/driver/postgres v1.1.2
gorm.io/driver/sqlite v1.1.6
@ -23,10 +27,10 @@ require (
github.com/cenkalti/backoff/v4 v4.1.0 // indirect
github.com/cespare/xxhash/v2 v2.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/gin-contrib/cors v1.3.1 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-playground/locales v0.14.0 // indirect
github.com/go-playground/universal-translator v0.18.0 // indirect
github.com/go-playground/validator/v10 v10.9.0 // indirect
github.com/go-redis/redis/v8 v8.9.0 // indirect
github.com/go-sql-driver/mysql v1.6.0 // indirect
github.com/golang/protobuf v1.5.2 // indirect
@ -40,8 +44,7 @@ require (
github.com/jackc/pgx/v4 v4.13.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.2 // indirect
github.com/jlaffaye/ftp v0.0.0-20211117213618-11820403398b // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mattn/go-sqlite3 v1.14.9 // indirect
@ -59,9 +62,8 @@ require (
go.opentelemetry.io/otel/metric v0.20.0 // indirect
go.opentelemetry.io/otel/trace v0.20.0 // indirect
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
golang.org/x/net v0.0.0-20210510120150-4163338589ed // indirect
golang.org/x/net v0.0.0-20211209124913-491a49abca63 // indirect
golang.org/x/sys v0.0.0-20211023085530-d6a326fbbf70 // indirect
golang.org/x/text v0.3.7 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 // indirect

6
go.sum
View File

@ -29,6 +29,7 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A=
github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
github.com/aws/aws-sdk-go v1.27.0 h1:0xphMHGMLBrPMfxR2AmVjZKcMEESEgWF8Kru94BNByk=
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@ -268,6 +269,7 @@ github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI=
github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jlaffaye/ftp v0.0.0-20211117213618-11820403398b h1:Ur6QAxsHCK99Quj9PaWafoV4unb0DO/HWiKExD+TN5g=
github.com/jlaffaye/ftp v0.0.0-20211117213618-11820403398b/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
@ -569,8 +571,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210510120150-4163338589ed h1:p9UgmWI9wKpfYmgaV/IZKGdXc5qEK45tDwwwDyjS26I=
golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211209124913-491a49abca63 h1:iocB37TsdFuN6IBRZ+ry36wrkoV51/tl5vOWqkcPGvY=
golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

7
model/account.go
View File

@ -36,6 +36,13 @@ type Account struct {
//AllowProxy bool `json:"allow_proxy"` // 是否允许中转下载
DownProxyUrl string `json:"down_proxy_url"` // 用于中转下载服务的URL 两处 1. path请求中返回的链接 2. down下载时进行302
APIProxyUrl string `json:"api_proxy_url"` // 用于中转api的地址
// for s3
Bucket string `json:"bucket"`
Endpoint string `json:"endpoint"`
Region string `json:"region"`
AccessKey string `json:"access_key"`
AccessSecret string `json:"access_secret"`
CustomHost string `json:"custom_host"`
}
var accountsMap = map[string]Account{}