feat(url-tree): support url tree driver writing (#7779 close #5166)

* feat: support url tree writing

* fix: meta writable

* feat: disable writable via addition

drivers/url_tree/driver.go

@@ -2,7 +2,11 @@ package url_tree
 
 import (
 	"context"
+	"errors"
+	"github.com/alist-org/alist/v3/internal/op"
 	stdpath "path"
+	"strings"
+	"sync"
 
 	"github.com/alist-org/alist/v3/internal/driver"
 	"github.com/alist-org/alist/v3/internal/errs"
@@ -14,7 +18,8 @@ import (
 type Urls struct {
 	model.Storage
 	Addition
-	root *Node
+	root  *Node
+	mutex sync.RWMutex
 }
 
 func (d *Urls) Config() driver.Config {
@@ -40,11 +45,15 @@ func (d *Urls) Drop(ctx context.Context) error {
 }
 
 func (d *Urls) Get(ctx context.Context, path string) (model.Obj, error) {
+	d.mutex.RLock()
+	defer d.mutex.RUnlock()
 	node := GetNodeFromRootByPath(d.root, path)
 	return nodeToObj(node, path)
 }
 
 func (d *Urls) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
+	d.mutex.RLock()
+	defer d.mutex.RUnlock()
 	node := GetNodeFromRootByPath(d.root, dir.GetPath())
 	log.Debugf("path: %s, node: %+v", dir.GetPath(), node)
 	if node == nil {
@@ -59,6 +68,8 @@ func (d *Urls) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]
 }
 
 func (d *Urls) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
+	d.mutex.RLock()
+	defer d.mutex.RUnlock()
 	node := GetNodeFromRootByPath(d.root, file.GetPath())
 	log.Debugf("path: %s, node: %+v", file.GetPath(), node)
 	if node == nil {
@@ -72,6 +83,174 @@ func (d *Urls) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*
 	return nil, errs.NotFile
 }
 
+func (d *Urls) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) {
+	if !d.Writable {
+		return nil, errs.PermissionDenied
+	}
+	d.mutex.Lock()
+	defer d.mutex.Unlock()
+	node := GetNodeFromRootByPath(d.root, parentDir.GetPath())
+	if node == nil {
+		return nil, errs.ObjectNotFound
+	}
+	if node.isFile() {
+		return nil, errs.NotFolder
+	}
+	dir := &Node{
+		Name:  dirName,
+		Level: node.Level + 1,
+	}
+	node.Children = append(node.Children, dir)
+	d.updateStorage()
+	return nodeToObj(dir, stdpath.Join(parentDir.GetPath(), dirName))
+}
+
+func (d *Urls) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) {
+	if !d.Writable {
+		return nil, errs.PermissionDenied
+	}
+	if strings.HasPrefix(dstDir.GetPath(), srcObj.GetPath()) {
+		return nil, errors.New("cannot move parent dir to child")
+	}
+	d.mutex.Lock()
+	defer d.mutex.Unlock()
+	dstNode := GetNodeFromRootByPath(d.root, dstDir.GetPath())
+	if dstNode == nil || dstNode.isFile() {
+		return nil, errs.NotFolder
+	}
+	srcDir, srcName := stdpath.Split(srcObj.GetPath())
+	srcParentNode := GetNodeFromRootByPath(d.root, srcDir)
+	if srcParentNode == nil {
+		return nil, errs.ObjectNotFound
+	}
+	newChildren := make([]*Node, 0, len(srcParentNode.Children))
+	var srcNode *Node
+	for _, child := range srcParentNode.Children {
+		if child.Name == srcName {
+			srcNode = child
+		} else {
+			newChildren = append(newChildren, child)
+		}
+	}
+	if srcNode == nil {
+		return nil, errs.ObjectNotFound
+	}
+	srcParentNode.Children = newChildren
+	srcNode.setLevel(dstNode.Level + 1)
+	dstNode.Children = append(dstNode.Children, srcNode)
+	d.root.calSize()
+	d.updateStorage()
+	return nodeToObj(srcNode, stdpath.Join(dstDir.GetPath(), srcName))
+}
+
+func (d *Urls) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) {
+	if !d.Writable {
+		return nil, errs.PermissionDenied
+	}
+	d.mutex.Lock()
+	defer d.mutex.Unlock()
+	srcNode := GetNodeFromRootByPath(d.root, srcObj.GetPath())
+	if srcNode == nil {
+		return nil, errs.ObjectNotFound
+	}
+	srcNode.Name = newName
+	d.updateStorage()
+	return nodeToObj(srcNode, stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName))
+}
+
+func (d *Urls) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) {
+	if !d.Writable {
+		return nil, errs.PermissionDenied
+	}
+	if strings.HasPrefix(dstDir.GetPath(), srcObj.GetPath()) {
+		return nil, errors.New("cannot copy parent dir to child")
+	}
+	d.mutex.Lock()
+	defer d.mutex.Unlock()
+	dstNode := GetNodeFromRootByPath(d.root, dstDir.GetPath())
+	if dstNode == nil || dstNode.isFile() {
+		return nil, errs.NotFolder
+	}
+	srcNode := GetNodeFromRootByPath(d.root, srcObj.GetPath())
+	if srcNode == nil {
+		return nil, errs.ObjectNotFound
+	}
+	newNode := srcNode.deepCopy(dstNode.Level + 1)
+	dstNode.Children = append(dstNode.Children, newNode)
+	d.root.calSize()
+	d.updateStorage()
+	return nodeToObj(newNode, stdpath.Join(dstDir.GetPath(), stdpath.Base(srcObj.GetPath())))
+}
+
+func (d *Urls) Remove(ctx context.Context, obj model.Obj) error {
+	if !d.Writable {
+		return errs.PermissionDenied
+	}
+	d.mutex.Lock()
+	defer d.mutex.Unlock()
+	objDir, objName := stdpath.Split(obj.GetPath())
+	nodeParent := GetNodeFromRootByPath(d.root, objDir)
+	if nodeParent == nil {
+		return errs.ObjectNotFound
+	}
+	newChildren := make([]*Node, 0, len(nodeParent.Children))
+	var deletedObj *Node
+	for _, child := range nodeParent.Children {
+		if child.Name != objName {
+			newChildren = append(newChildren, child)
+		} else {
+			deletedObj = child
+		}
+	}
+	if deletedObj == nil {
+		return errs.ObjectNotFound
+	}
+	nodeParent.Children = newChildren
+	if deletedObj.Size > 0 {
+		d.root.calSize()
+	}
+	d.updateStorage()
+	return nil
+}
+
+func (d *Urls) PutURL(ctx context.Context, dstDir model.Obj, name, url string) (model.Obj, error) {
+	if !d.Writable {
+		return nil, errs.PermissionDenied
+	}
+	d.mutex.Lock()
+	defer d.mutex.Unlock()
+	dirNode := GetNodeFromRootByPath(d.root, dstDir.GetPath())
+	if dirNode == nil || dirNode.isFile() {
+		return nil, errs.NotFolder
+	}
+	newNode := &Node{
+		Name:  name,
+		Level: dirNode.Level + 1,
+		Url:   url,
+	}
+	dirNode.Children = append(dirNode.Children, newNode)
+	if d.HeadSize {
+		size, err := getSizeFromUrl(url)
+		if err != nil {
+			log.Errorf("get size from url error: %s", err)
+		} else {
+			newNode.Size = size
+			d.root.calSize()
+		}
+	}
+	d.updateStorage()
+	return nodeToObj(newNode, stdpath.Join(dstDir.GetPath(), name))
+}
+
+func (d *Urls) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error {
+	return errs.UploadNotSupported
+}
+
+func (d *Urls) updateStorage() {
+	d.UrlStructure = StringifyTree(d.root)
+	op.MustSaveDriverStorage(d)
+}
+
 //func (d *Template) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) {
 //	return nil, errs.NotSupport
 //}

drivers/url_tree/meta.go

@@ -12,6 +12,7 @@ type Addition struct {
 	// define other
 	UrlStructure string `json:"url_structure" type:"text" required:"true" default:"https://jsd.nn.ci/gh/alist-org/alist/README.md\nhttps://jsd.nn.ci/gh/alist-org/alist/README_cn.md\nfolder:\n  CONTRIBUTING.md:1635:https://jsd.nn.ci/gh/alist-org/alist/CONTRIBUTING.md\n  CODE_OF_CONDUCT.md:2093:https://jsd.nn.ci/gh/alist-org/alist/CODE_OF_CONDUCT.md" help:"structure:FolderName:\n  [FileName:][FileSize:][Modified:]Url"`
 	HeadSize     bool   `json:"head_size" type:"bool" default:"false" help:"Use head method to get file size, but it may be failed."`
+	Writable     bool   `json:"writable" type:"bool" default:"false"`
 }
 
 var config = driver.Config{
@@ -20,7 +21,7 @@ var config = driver.Config{
 	OnlyLocal:         false,
 	OnlyProxy:         false,
 	NoCache:           true,
-	NoUpload:          true,
+	NoUpload:          false,
 	NeedMs:            false,
 	DefaultRoot:       "",
 	CheckStatus:       true,

drivers/url_tree/types.go

@@ -1,5 +1,7 @@
 package url_tree
 
+import "github.com/alist-org/alist/v3/pkg/utils"
+
 // Node is a node in the folder tree
 type Node struct {
 	Url      string
@@ -44,3 +46,19 @@ func (node *Node) calSize() int64 {
 	node.Size = size
 	return size
 }
+
+func (node *Node) setLevel(level int) {
+	node.Level = level
+	for _, child := range node.Children {
+		child.setLevel(level + 1)
+	}
+}
+
+func (node *Node) deepCopy(level int) *Node {
+	ret := *node
+	ret.Level = level
+	ret.Children, _ = utils.SliceConvert(ret.Children, func(child *Node) (*Node, error) {
+		return child.deepCopy(level + 1), nil
+	})
+	return &ret
+}

drivers/url_tree/util.go

@@ -153,6 +153,9 @@ func splitPath(path string) []string {
 	if path == "/" {
 		return []string{"root"}
 	}
+	if strings.HasSuffix(path, "/") {
+		path = path[:len(path)-1]
+	}
 	parts := strings.Split(path, "/")
 	parts[0] = "root"
 	return parts
@@ -190,3 +193,46 @@ func getSizeFromUrl(url string) (int64, error) {
 	}
 	return size, nil
 }
+
+func StringifyTree(node *Node) string {
+	sb := strings.Builder{}
+	if node.Level == -1 {
+		for i, child := range node.Children {
+			sb.WriteString(StringifyTree(child))
+			if i < len(node.Children)-1 {
+				sb.WriteString("\n")
+			}
+		}
+		return sb.String()
+	}
+	for i := 0; i < node.Level; i++ {
+		sb.WriteString("  ")
+	}
+	if node.Url == "" {
+		sb.WriteString(node.Name)
+		sb.WriteString(":")
+		for _, child := range node.Children {
+			sb.WriteString("\n")
+			sb.WriteString(StringifyTree(child))
+		}
+	} else if node.Size == 0 && node.Modified == 0 {
+		if stdpath.Base(node.Url) == node.Name {
+			sb.WriteString(node.Url)
+		} else {
+			sb.WriteString(fmt.Sprintf("%s:%s", node.Name, node.Url))
+		}
+	} else {
+		sb.WriteString(node.Name)
+		sb.WriteString(":")
+		if node.Size != 0 || node.Modified != 0 {
+			sb.WriteString(strconv.FormatInt(node.Size, 10))
+			sb.WriteString(":")
+		}
+		if node.Modified != 0 {
+			sb.WriteString(strconv.FormatInt(node.Modified, 10))
+			sb.WriteString(":")
+		}
+		sb.WriteString(node.Url)
+	}
+	return sb.String()
+}

internal/driver/driver.go

@@ -80,6 +80,13 @@ type Put interface {
 	Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up UpdateProgress) error
 }
 
+type PutURL interface {
+	// PutURL directly put a URL into the storage
+	// Applicable to index-based drivers like URL-Tree or drivers that support uploading files as URLs
+	// Called when using SimpleHttp for offline downloading, skipping creating a download task
+	PutURL(ctx context.Context, dstDir model.Obj, name, url string) error
+}
+
 //type WriteResult interface {
 //	MkdirResult
 //	MoveResult
@@ -109,6 +116,13 @@ type PutResult interface {
 	Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up UpdateProgress) (model.Obj, error)
 }
 
+type PutURLResult interface {
+	// PutURL directly put a URL into the storage
+	// Applicable to index-based drivers like URL-Tree or drivers that support uploading files as URLs
+	// Called when using SimpleHttp for offline downloading, skipping creating a download task
+	PutURL(ctx context.Context, dstDir model.Obj, name, url string) (model.Obj, error)
+}
+
 type UpdateProgress func(percentage float64)
 
 type Progress struct {

internal/offline_download/tool/add.go

@@ -2,8 +2,11 @@ package tool
 
 import (
 	"context"
+	"github.com/alist-org/alist/v3/internal/driver"
 	"github.com/alist-org/alist/v3/internal/model"
 	"github.com/alist-org/alist/v3/internal/task"
+	"net/url"
+	"path"
 	"path/filepath"
 
 	"github.com/alist-org/alist/v3/internal/conf"
@@ -30,18 +33,6 @@ type AddURLArgs struct {
 }
 
 func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, error) {
-	// get tool
-	tool, err := Tools.Get(args.Tool)
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed get tool")
-	}
-	// check tool is ready
-	if !tool.IsReady() {
-		// try to init tool
-		if _, err := tool.Init(); err != nil {
-			return nil, errors.Wrapf(err, "failed init tool %s", args.Tool)
-		}
-	}
 	// check storage
 	storage, dstDirActualPath, err := op.GetStorageAndActualPath(args.DstDirPath)
 	if err != nil {
@@ -63,6 +54,23 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro
 			return nil, errors.WithStack(errs.NotFolder)
 		}
 	}
+	// try putting url
+	if args.Tool == "SimpleHttp" && tryPutUrl(ctx, storage, dstDirActualPath, args.URL) {
+		return nil, nil
+	}
+
+	// get tool
+	tool, err := Tools.Get(args.Tool)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed get tool")
+	}
+	// check tool is ready
+	if !tool.IsReady() {
+		// try to init tool
+		if _, err := tool.Init(); err != nil {
+			return nil, errors.Wrapf(err, "failed init tool %s", args.Tool)
+		}
+	}
 
 	uid := uuid.NewString()
 	tempDir := filepath.Join(conf.Conf.TempDir, args.Tool, uid)
@@ -98,3 +106,18 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro
 	DownloadTaskManager.Add(t)
 	return t, nil
 }
+
+func tryPutUrl(ctx context.Context, storage driver.Driver, dstDirActualPath, urlStr string) bool {
+	_, ok := storage.(driver.PutURL)
+	_, okResult := storage.(driver.PutURLResult)
+	if !ok && !okResult {
+		return false
+	}
+	u, err := url.Parse(urlStr)
+	if err != nil {
+		return false
+	}
+	dstName := path.Base(u.Path)
+	err = op.PutURL(ctx, storage, dstDirActualPath, dstName, urlStr)
+	return err == nil
+}

internal/op/fs.go

@@ -586,3 +586,43 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod
 	}
 	return errors.WithStack(err)
 }
+
+func PutURL(ctx context.Context, storage driver.Driver, dstDirPath, dstName, url string, lazyCache ...bool) error {
+	if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
+		return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
+	}
+	dstDirPath = utils.FixAndCleanPath(dstDirPath)
+	_, err := GetUnwrap(ctx, storage, stdpath.Join(dstDirPath, dstName))
+	if err == nil {
+		return errors.New("obj already exists")
+	}
+	err = MakeDir(ctx, storage, dstDirPath)
+	if err != nil {
+		return errors.WithMessagef(err, "failed to put url")
+	}
+	dstDir, err := GetUnwrap(ctx, storage, dstDirPath)
+	if err != nil {
+		return errors.WithMessagef(err, "failed to put url")
+	}
+	switch s := storage.(type) {
+	case driver.PutURLResult:
+		var newObj model.Obj
+		newObj, err = s.PutURL(ctx, dstDir, dstName, url)
+		if err == nil {
+			if newObj != nil {
+				addCacheObj(storage, dstDirPath, model.WrapObjName(newObj))
+			} else if !utils.IsBool(lazyCache...) {
+				ClearCache(storage, dstDirPath)
+			}
+		}
+	case driver.PutURL:
+		err = s.PutURL(ctx, dstDir, dstName, url)
+		if err == nil && !utils.IsBool(lazyCache...) {
+			ClearCache(storage, dstDirPath)
+		}
+	default:
+		return errs.NotImplement
+	}
+	log.Debugf("put url [%s](%s) done", dstName, url)
+	return errors.WithStack(err)
+}

server/handles/offline_download.go

@@ -145,7 +145,9 @@ func AddOfflineDownload(c *gin.Context) {
 			common.ErrorResp(c, err, 500)
 			return
 		}
-		tasks = append(tasks, t)
+		if t != nil {
+			tasks = append(tasks, t)
+		}
 	}
 	common.SuccessResp(c, gin.H{
 		"tasks": getTaskInfos(tasks),