feat: add Dropbox driver (#4639 close #4590)

Co-authored-by: Andy Hsu <i@nn.ci>

README.md

@@ -73,6 +73,7 @@ English | [中文](./README_cn.md) | [Contributing](./CONTRIBUTING.md) | [CODE_O
     - [x] SMB
     - [x] [115](https://115.com/)
     - [X] Cloudreve
+    - [x] [Dropbox](https://www.dropbox.com/)
 - [x] Easy to deploy and out-of-the-box
 - [x] File preview (PDF, markdown, code, plain text, ...)
 - [x] Image preview in gallery mode

README_cn.md

@@ -72,6 +72,7 @@
     - [x] SMB
     - [x] [115](https://115.com/)
     - [X] Cloudreve
+    - [x] [Dropbox](https://www.dropbox.com/)
 - [x] 部署方便，开箱即用
 - [x] 文件预览（PDF、markdown、代码、纯文本……）
 - [x] 画廊模式下的图像预览

drivers/all.go

@@ -16,6 +16,7 @@ import (
 	_ "github.com/alist-org/alist/v3/drivers/baidu_photo"
 	_ "github.com/alist-org/alist/v3/drivers/baidu_share"
 	_ "github.com/alist-org/alist/v3/drivers/cloudreve"
+	_ "github.com/alist-org/alist/v3/drivers/dropbox"
 	_ "github.com/alist-org/alist/v3/drivers/ftp"
 	_ "github.com/alist-org/alist/v3/drivers/google_drive"
 	_ "github.com/alist-org/alist/v3/drivers/google_photo"

drivers/dropbox/driver.go

@@ -0,0 +1,224 @@
+package dropbox
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"time"
+
+	"github.com/alist-org/alist/v3/drivers/base"
+	"github.com/alist-org/alist/v3/internal/driver"
+	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/pkg/utils"
+	"github.com/go-resty/resty/v2"
+	log "github.com/sirupsen/logrus"
+)
+
+type Dropbox struct {
+	model.Storage
+	Addition
+	base        string
+	contentBase string
+}
+
+func (d *Dropbox) Config() driver.Config {
+	return config
+}
+
+func (d *Dropbox) GetAddition() driver.Additional {
+	return &d.Addition
+}
+
+func (d *Dropbox) Init(ctx context.Context) error {
+	query := "foo"
+	res, err := d.request("/2/check/user", http.MethodPost, func(req *resty.Request) {
+		req.SetBody(base.Json{
+			"query": query,
+		})
+	})
+	if err != nil {
+		return err
+	}
+	result := utils.Json.Get(res, "result").ToString()
+	if result != query {
+		return fmt.Errorf("failed to check user: %s", string(res))
+	}
+	return nil
+}
+
+func (d *Dropbox) Drop(ctx context.Context) error {
+	return nil
+}
+
+func (d *Dropbox) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
+	files, err := d.getFiles(ctx, dir.GetPath())
+	if err != nil {
+		return nil, err
+	}
+	return utils.SliceConvert(files, func(src File) (model.Obj, error) {
+		return fileToObj(src), nil
+	})
+}
+
+func (d *Dropbox) link(ctx context.Context, file model.Obj) (*model.Link, error) {
+	res, err := d.request("/2/files/get_temporary_link", http.MethodPost, func(req *resty.Request) {
+		req.SetBody(base.Json{
+			"path": file.GetID(),
+		})
+	})
+	if err != nil {
+		return nil, err
+	}
+	url := utils.Json.Get(res, "link").ToString()
+	exp := time.Hour
+	return &model.Link{
+		URL:        url,
+		Expiration: &exp,
+	}, nil
+}
+
+func (d *Dropbox) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
+	return d.link(ctx, file)
+}
+
+func (d *Dropbox) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error {
+	_, err := d.request("/2/files/create_folder_v2", http.MethodPost, func(req *resty.Request) {
+		req.SetBody(base.Json{
+			"autorename": false,
+			"path":       parentDir.GetPath() + "/" + dirName,
+		})
+	})
+	return err
+}
+
+func (d *Dropbox) Move(ctx context.Context, srcObj, dstDir model.Obj) error {
+	toPath := dstDir.GetPath() + "/" + srcObj.GetName()
+
+	_, err := d.request("/2/files/move_v2", http.MethodPost, func(req *resty.Request) {
+		req.SetBody(base.Json{
+			"allow_ownership_transfer": false,
+			"allow_shared_folder":      false,
+			"autorename":               false,
+			"from_path":                srcObj.GetID(),
+			"to_path":                  toPath,
+		})
+	})
+	return err
+}
+
+func (d *Dropbox) Rename(ctx context.Context, srcObj model.Obj, newName string) error {
+	path := srcObj.GetPath()
+	fileName := srcObj.GetName()
+	toPath := path[:len(path)-len(fileName)] + newName
+
+	_, err := d.request("/2/files/move_v2", http.MethodPost, func(req *resty.Request) {
+		req.SetBody(base.Json{
+			"allow_ownership_transfer": false,
+			"allow_shared_folder":      false,
+			"autorename":               false,
+			"from_path":                srcObj.GetID(),
+			"to_path":                  toPath,
+		})
+	})
+	return err
+}
+
+func (d *Dropbox) Copy(ctx context.Context, srcObj, dstDir model.Obj) error {
+	toPath := dstDir.GetPath() + "/" + srcObj.GetName()
+	_, err := d.request("/2/files/copy_v2", http.MethodPost, func(req *resty.Request) {
+		req.SetBody(base.Json{
+			"allow_ownership_transfer": false,
+			"allow_shared_folder":      false,
+			"autorename":               false,
+			"from_path":                srcObj.GetID(),
+			"to_path":                  toPath,
+		})
+	})
+	return err
+}
+
+func (d *Dropbox) Remove(ctx context.Context, obj model.Obj) error {
+	uri := "/2/files/delete_v2"
+	_, err := d.request(uri, http.MethodPost, func(req *resty.Request) {
+		req.SetBody(base.Json{
+			"path": obj.GetID(),
+		})
+	})
+	return err
+}
+
+func (d *Dropbox) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error {
+	// 1. start
+	sessionId, err := d.startUploadSession(ctx)
+	if err != nil {
+		return err
+	}
+
+	// 2.append
+	// A single request should not upload more than 150 MB, and each call must be multiple of 4MB  (except for last call)
+	const PartSize = 20971520
+	count := 1
+	if stream.GetSize() > PartSize {
+		count = int(math.Ceil(float64(stream.GetSize()) / float64(PartSize)))
+	}
+	offset := int64(0)
+
+	for i := 0; i < count; i++ {
+		if utils.IsCanceled(ctx) {
+			return ctx.Err()
+		}
+
+		start := i * PartSize
+		byteSize := stream.GetSize() - int64(start)
+		if byteSize > PartSize {
+			byteSize = PartSize
+		}
+
+		url := d.contentBase + "/2/files/upload_session/append_v2"
+		reader := io.LimitReader(stream, PartSize)
+		req, err := http.NewRequest(http.MethodPost, url, reader)
+		if err != nil {
+			log.Errorf("failed to update file when append to upload session, err: %+v", err)
+			return err
+		}
+		req = req.WithContext(ctx)
+		req.Header.Set("Content-Type", "application/octet-stream")
+		req.Header.Set("Authorization", "Bearer "+d.AccessToken)
+
+		args := UploadAppendArgs{
+			Close: false,
+			Cursor: UploadCursor{
+				Offset:    offset,
+				SessionID: sessionId,
+			},
+		}
+		argsJson, _ := utils.Json.MarshalToString(args)
+		req.Header.Set("Dropbox-API-Arg", argsJson)
+
+		res, err := base.HttpClient.Do(req)
+		_ = res.Body.Close()
+
+		if err != nil {
+			return err
+		}
+
+		if count > 0 {
+			up((i + 1) * 100 / count)
+		}
+
+		offset += byteSize
+
+	}
+	// 3.finish
+	toPath := dstDir.GetPath() + "/" + stream.GetName()
+	err2 := d.finishUploadSession(ctx, toPath, offset, sessionId)
+	if err2 != nil {
+		return err2
+	}
+
+	return err
+}
+
+var _ driver.Driver = (*Dropbox)(nil)

drivers/dropbox/meta.go

@@ -0,0 +1,42 @@
+package dropbox
+
+import (
+	"github.com/alist-org/alist/v3/internal/driver"
+	"github.com/alist-org/alist/v3/internal/op"
+)
+
+const (
+	DefaultClientID = "76lrwrklhdn1icb"
+)
+
+type Addition struct {
+	RefreshToken string `json:"refresh_token" required:"true"`
+	driver.RootPath
+
+	OauthTokenURL string `json:"oauth_token_url" default:"https://api.xhofe.top/alist/dropbox/token"`
+	ClientID      string `json:"client_id" required:"false" help:"Keep it empty if you don't have one"`
+	ClientSecret  string `json:"client_secret" required:"false" help:"Keep it empty if you don't have one"`
+
+	AccessToken string
+}
+
+var config = driver.Config{
+	Name:              "Dropbox",
+	LocalSort:         false,
+	OnlyLocal:         false,
+	OnlyProxy:         false,
+	NoCache:           false,
+	NoUpload:          false,
+	NeedMs:            false,
+	DefaultRoot:       "",
+	NoOverwriteUpload: true,
+}
+
+func init() {
+	op.RegisterDriver(func() driver.Driver {
+		return &Dropbox{
+			base:        "https://api.dropboxapi.com",
+			contentBase: "https://content.dropboxapi.com",
+		}
+	})
+}

drivers/dropbox/types.go

@@ -0,0 +1,79 @@
+package dropbox
+
+import (
+	"github.com/alist-org/alist/v3/internal/model"
+	"time"
+)
+
+type TokenResp struct {
+	AccessToken string `json:"access_token"`
+	TokenType   string `json:"token_type"`
+	ExpiresIn   int    `json:"expires_in"`
+}
+
+type ErrorResp struct {
+	Error struct {
+		Tag string `json:".tag"`
+	} `json:"error"`
+	ErrorSummary string `json:"error_summary"`
+}
+
+type RefreshTokenErrorResp struct {
+	Error            string `json:"error"`
+	ErrorDescription string `json:"error_description"`
+}
+
+type File struct {
+	Tag            string    `json:".tag"`
+	Name           string    `json:"name"`
+	PathLower      string    `json:"path_lower"`
+	PathDisplay    string    `json:"path_display"`
+	ID             string    `json:"id"`
+	ClientModified time.Time `json:"client_modified"`
+	ServerModified time.Time `json:"server_modified"`
+	Rev            string    `json:"rev"`
+	Size           int       `json:"size"`
+	IsDownloadable bool      `json:"is_downloadable"`
+	ContentHash    string    `json:"content_hash"`
+}
+
+type ListResp struct {
+	Entries []File `json:"entries"`
+	Cursor  string `json:"cursor"`
+	HasMore bool   `json:"has_more"`
+}
+
+type UploadCursor struct {
+	Offset    int64  `json:"offset"`
+	SessionID string `json:"session_id"`
+}
+
+type UploadAppendArgs struct {
+	Close  bool         `json:"close"`
+	Cursor UploadCursor `json:"cursor"`
+}
+
+type UploadFinishArgs struct {
+	Commit struct {
+		Autorename     bool   `json:"autorename"`
+		Mode           string `json:"mode"`
+		Mute           bool   `json:"mute"`
+		Path           string `json:"path"`
+		StrictConflict bool   `json:"strict_conflict"`
+	} `json:"commit"`
+	Cursor UploadCursor `json:"cursor"`
+}
+
+func fileToObj(f File) *model.ObjThumb {
+	return &model.ObjThumb{
+		Object: model.Object{
+			ID:       f.ID,
+			Path:     f.PathDisplay,
+			Name:     f.Name,
+			Size:     int64(f.Size),
+			Modified: f.ServerModified,
+			IsFolder: f.Tag == "folder",
+		},
+		Thumbnail: model.Thumbnail{},
+	}
+}

drivers/dropbox/util.go

@@ -0,0 +1,199 @@
+package dropbox
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/alist-org/alist/v3/drivers/base"
+	"github.com/alist-org/alist/v3/internal/op"
+	"github.com/alist-org/alist/v3/pkg/utils"
+	"github.com/go-resty/resty/v2"
+	log "github.com/sirupsen/logrus"
+)
+
+func (d *Dropbox) refreshToken() error {
+	url := d.base + "/oauth2/token"
+	if utils.SliceContains([]string{"", DefaultClientID}, d.ClientID) {
+		url = d.OauthTokenURL
+	}
+	var tokenResp TokenResp
+	resp, err := base.RestyClient.R().
+		//ForceContentType("application/x-www-form-urlencoded").
+		//SetBasicAuth(d.ClientID, d.ClientSecret).
+		SetFormData(map[string]string{
+			"grant_type":    "refresh_token",
+			"refresh_token": d.RefreshToken,
+			"client_id":     d.ClientID,
+			"client_secret": d.ClientSecret,
+		}).
+		Post(url)
+	if err != nil {
+		return err
+	}
+	log.Debugf("[dropbox] refresh token response: %s", resp.String())
+	if resp.StatusCode() != 200 {
+		return fmt.Errorf("failed to refresh token: %s", resp.String())
+	}
+	_ = utils.Json.UnmarshalFromString(resp.String(), &tokenResp)
+	d.AccessToken = tokenResp.AccessToken
+	op.MustSaveDriverStorage(d)
+	return nil
+}
+
+func (d *Dropbox) request(uri, method string, callback base.ReqCallback, retry ...bool) ([]byte, error) {
+	req := base.RestyClient.R()
+	req.SetHeader("Authorization", "Bearer "+d.AccessToken)
+	if method == http.MethodPost {
+		req.SetHeader("Content-Type", "application/json")
+	}
+	if callback != nil {
+		callback(req)
+	}
+	var e ErrorResp
+	req.SetError(&e)
+	res, err := req.Execute(method, d.base+uri)
+	if err != nil {
+		return nil, err
+	}
+	log.Debugf("[dropbox] request (%s) response: %s", uri, res.String())
+	isRetry := len(retry) > 0 && retry[0]
+	if res.StatusCode() != 200 {
+		body := res.String()
+		if !isRetry && (utils.SliceMeet([]string{"expired_access_token", "invalid_access_token", "authorization"}, body,
+			func(item string, v string) bool {
+				return strings.Contains(v, item)
+			}) || d.AccessToken == "") {
+			err = d.refreshToken()
+			if err != nil {
+				return nil, err
+			}
+			return d.request(uri, method, callback, true)
+		}
+		return nil, fmt.Errorf("%s:%s", e.Error, e.ErrorSummary)
+	}
+	return res.Body(), nil
+}
+
+func (d *Dropbox) list(ctx context.Context, data base.Json, isContinue bool) (*ListResp, error) {
+	var resp ListResp
+	uri := "/2/files/list_folder"
+	if isContinue {
+		uri += "/continue"
+	}
+	_, err := d.request(uri, http.MethodPost, func(req *resty.Request) {
+		req.SetBody(data).SetResult(&resp)
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+
+func (d *Dropbox) getFiles(ctx context.Context, path string) ([]File, error) {
+	hasMore := true
+	var marker string
+	res := make([]File, 0)
+
+	data := base.Json{
+		"include_deleted":                     false,
+		"include_has_explicit_shared_members": false,
+		"include_mounted_folders":             false,
+		"include_non_downloadable_files":      false,
+		"limit":                               2000,
+		"path":                                path,
+		"recursive":                           false,
+	}
+	resp, err := d.list(ctx, data, false)
+	if err != nil {
+		return nil, err
+	}
+	marker = resp.Cursor
+	hasMore = resp.HasMore
+	res = append(res, resp.Entries...)
+
+	for hasMore {
+		data := base.Json{
+			"cursor": marker,
+		}
+		resp, err := d.list(ctx, data, true)
+		if err != nil {
+			return nil, err
+		}
+		marker = resp.Cursor
+		hasMore = resp.HasMore
+		res = append(res, resp.Entries...)
+	}
+	return res, nil
+}
+
+func (d *Dropbox) finishUploadSession(ctx context.Context, toPath string, offset int64, sessionId string) error {
+	url := d.contentBase + "/2/files/upload_session/finish"
+	req, err := http.NewRequest(http.MethodPost, url, nil)
+	if err != nil {
+		return err
+	}
+	req = req.WithContext(ctx)
+	req.Header.Set("Content-Type", "application/octet-stream")
+	req.Header.Set("Authorization", "Bearer "+d.AccessToken)
+
+	uploadFinishArgs := UploadFinishArgs{
+		Commit: struct {
+			Autorename     bool   `json:"autorename"`
+			Mode           string `json:"mode"`
+			Mute           bool   `json:"mute"`
+			Path           string `json:"path"`
+			StrictConflict bool   `json:"strict_conflict"`
+		}{
+			Autorename:     true,
+			Mode:           "add",
+			Mute:           false,
+			Path:           toPath,
+			StrictConflict: false,
+		},
+		Cursor: UploadCursor{
+			Offset:    offset,
+			SessionID: sessionId,
+		},
+	}
+
+	argsJson, err := utils.Json.MarshalToString(uploadFinishArgs)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Dropbox-API-Arg", argsJson)
+
+	res, err := base.HttpClient.Do(req)
+	if err != nil {
+		log.Errorf("failed to update file when finish session, err: %+v", err)
+		return err
+	}
+	_ = res.Body.Close()
+	return nil
+}
+
+func (d *Dropbox) startUploadSession(ctx context.Context) (string, error) {
+	url := d.contentBase + "/2/files/upload_session/start"
+	req, err := http.NewRequest(http.MethodPost, url, nil)
+	if err != nil {
+		return "", err
+	}
+	req = req.WithContext(ctx)
+	req.Header.Set("Content-Type", "application/octet-stream")
+	req.Header.Set("Authorization", "Bearer "+d.AccessToken)
+	req.Header.Set("Dropbox-API-Arg", "{\"close\":false}")
+
+	res, err := base.HttpClient.Do(req)
+	if err != nil {
+		log.Errorf("failed to update file when start session, err: %+v", err)
+		return "", err
+	}
+
+	body, err := io.ReadAll(res.Body)
+	sessionId := utils.Json.Get(body, "session_id").ToString()
+
+	_ = res.Body.Close()
+	return sessionId, nil
+}

pkg/utils/slice.go

@@ -60,3 +60,12 @@ func MergeErrors(errs ...error) error {
 	}
 	return nil
 }
+
+func SliceMeet[T1, T2 any](arr []T1, v T2, meet func(item T1, v T2) bool) bool {
+	for _, item := range arr {
+		if meet(item, v) {
+			return true
+		}
+	}
+	return false
+}