feat(drivers): new driver `UrlTree` (close #3268 in #3933)

* feat(drivers): new driver `urls` (close #3268)

* chore: rename

* support customize basic info or get from url

* dfs tree to calculate folder size

* go mod tidy

* add help message

drivers/all.go

@@ -36,6 +36,7 @@ import (
 	_ "github.com/alist-org/alist/v3/drivers/terabox"
 	_ "github.com/alist-org/alist/v3/drivers/thunder"
 	_ "github.com/alist-org/alist/v3/drivers/trainbit"
+	_ "github.com/alist-org/alist/v3/drivers/url_tree"
 	_ "github.com/alist-org/alist/v3/drivers/uss"
 	_ "github.com/alist-org/alist/v3/drivers/virtual"
 	_ "github.com/alist-org/alist/v3/drivers/webdav"

drivers/url_tree/driver.go

@@ -0,0 +1,79 @@
+package url_tree
+
+import (
+	"context"
+	stdpath "path"
+
+	"github.com/alist-org/alist/v3/internal/driver"
+	"github.com/alist-org/alist/v3/internal/errs"
+	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/pkg/utils"
+	log "github.com/sirupsen/logrus"
+)
+
+type Urls struct {
+	model.Storage
+	Addition
+	root *Node
+}
+
+func (d *Urls) Config() driver.Config {
+	return config
+}
+
+func (d *Urls) GetAddition() driver.Additional {
+	return &d.Addition
+}
+
+func (d *Urls) Init(ctx context.Context) error {
+	node, err := BuildTree(d.UrlStructure, d.HeadSize)
+	if err != nil {
+		return err
+	}
+	node.calSize()
+	d.root = node
+	return nil
+}
+
+func (d *Urls) Drop(ctx context.Context) error {
+	return nil
+}
+
+func (d *Urls) Get(ctx context.Context, path string) (model.Obj, error) {
+	node := GetNodeFromRootByPath(d.root, path)
+	return nodeToObj(node, path)
+}
+
+func (d *Urls) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
+	node := GetNodeFromRootByPath(d.root, dir.GetPath())
+	log.Debugf("path: %s, node: %+v", dir.GetPath(), node)
+	if node == nil {
+		return nil, errs.ObjectNotFound
+	}
+	if node.isFile() {
+		return nil, errs.NotFolder
+	}
+	return utils.SliceConvert(node.Children, func(node *Node) (model.Obj, error) {
+		return nodeToObj(node, stdpath.Join(dir.GetPath(), node.Name))
+	})
+}
+
+func (d *Urls) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
+	node := GetNodeFromRootByPath(d.root, file.GetPath())
+	log.Debugf("path: %s, node: %+v", file.GetPath(), node)
+	if node == nil {
+		return nil, errs.ObjectNotFound
+	}
+	if node.isFile() {
+		return &model.Link{
+			URL: node.Url,
+		}, nil
+	}
+	return nil, errs.NotFile
+}
+
+//func (d *Template) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) {
+//	return nil, errs.NotSupport
+//}
+
+var _ driver.Driver = (*Urls)(nil)

drivers/url_tree/meta.go

@@ -0,0 +1,35 @@
+package url_tree
+
+import (
+	"github.com/alist-org/alist/v3/internal/driver"
+	"github.com/alist-org/alist/v3/internal/op"
+)
+
+type Addition struct {
+	// Usually one of two
+	// driver.RootPath
+	// driver.RootID
+	// define other
+	UrlStructure string `json:"url_structure" type:"text" required:"true" default:"https://jsd.nn.ci/gh/alist-org/alist/README.md\nhttps://jsd.nn.ci/gh/alist-org/alist/README_cn.md\nfolder:\n  CONTRIBUTING.md:1635:https://jsd.nn.ci/gh/alist-org/alist/CONTRIBUTING.md\n  CODE_OF_CONDUCT.md:2093:https://jsd.nn.ci/gh/alist-org/alist/CODE_OF_CONDUCT.md" help:"structure:FolderName:\n  [FileName:][FileSize:][Modified:]Url"`
+	HeadSize     bool   `json:"head_size" type:"bool" default:"false" help:"Use head method to get file size, but it may be failed."`
+}
+
+var config = driver.Config{
+	Name:              "UrlTree",
+	LocalSort:         true,
+	OnlyLocal:         false,
+	OnlyProxy:         false,
+	NoCache:           true,
+	NoUpload:          true,
+	NeedMs:            false,
+	DefaultRoot:       "",
+	CheckStatus:       true,
+	Alert:             "",
+	NoOverwriteUpload: false,
+}
+
+func init() {
+	op.RegisterDriver(func() driver.Driver {
+		return &Urls{}
+	})
+}

drivers/url_tree/types.go

@@ -0,0 +1,46 @@
+package url_tree
+
+// Node is a node in the folder tree
+type Node struct {
+	Url      string
+	Name     string
+	Level    int
+	Modified int64
+	Size     int64
+	Children []*Node
+}
+
+func (node *Node) getByPath(paths []string) *Node {
+	if len(paths) == 0 || node == nil {
+		return nil
+	}
+	if node.Name != paths[0] {
+		return nil
+	}
+	if len(paths) == 1 {
+		return node
+	}
+	for _, child := range node.Children {
+		tmp := child.getByPath(paths[1:])
+		if tmp != nil {
+			return tmp
+		}
+	}
+	return nil
+}
+
+func (node *Node) isFile() bool {
+	return node.Url != ""
+}
+
+func (node *Node) calSize() int64 {
+	if node.isFile() {
+		return node.Size
+	}
+	var size int64 = 0
+	for _, child := range node.Children {
+		size += child.calSize()
+	}
+	node.Size = size
+	return size
+}

drivers/url_tree/urls_test.go

@@ -0,0 +1,47 @@
+package url_tree_test
+
+import (
+	"testing"
+
+	"github.com/alist-org/alist/v3/drivers/url_tree"
+)
+
+func testTree() (*url_tree.Node, error) {
+	text := `folder1:
+  name1:url1
+  url2
+  folder2:
+    url3
+    url4
+  url5
+folder3:
+  url6
+  url7
+url8`
+	return url_tree.BuildTree(text, false)
+}
+
+func TestBuildTree(t *testing.T) {
+	node, err := testTree()
+	if err != nil {
+		t.Errorf("failed to build tree: %+v", err)
+	} else {
+		t.Logf("tree: %+v", node)
+	}
+}
+
+func TestGetNode(t *testing.T) {
+	root, err := testTree()
+	if err != nil {
+		t.Errorf("failed to build tree: %+v", err)
+		return
+	}
+	node := url_tree.GetNodeFromRootByPath(root, "/")
+	if node != root {
+		t.Errorf("got wrong node: %+v", node)
+	}
+	url3 := url_tree.GetNodeFromRootByPath(root, "/folder1/folder2/url3")
+	if url3 != root.Children[0].Children[2].Children[0] {
+		t.Errorf("got wrong node: %+v", url3)
+	}
+}

drivers/url_tree/util.go

@@ -0,0 +1,192 @@
+package url_tree
+
+import (
+	"fmt"
+	stdpath "path"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/alist-org/alist/v3/drivers/base"
+	"github.com/alist-org/alist/v3/internal/errs"
+	"github.com/alist-org/alist/v3/internal/model"
+	log "github.com/sirupsen/logrus"
+)
+
+// build tree from text, text structure definition:
+/**
+ * FolderName:
+ *   [FileName:][FileSize:][Modified:]Url
+ */
+/**
+ * For example:
+ * folder1:
+ *   name1:url1
+ *   url2
+ *   folder2:
+ *     url3
+ *     url4
+ *   url5
+ * folder3:
+ *   url6
+ *   url7
+ * url8
+ */
+// if there are no name, use the last segment of url as name
+func BuildTree(text string, headSize bool) (*Node, error) {
+	lines := strings.Split(text, "\n")
+	var root = &Node{Level: -1, Name: "root"}
+	stack := []*Node{root}
+	for _, line := range lines {
+		// calculate indent
+		indent := 0
+		for i := 0; i < len(line); i++ {
+			if line[i] != ' ' {
+				break
+			}
+			indent++
+		}
+		// if indent is not a multiple of 2, it is an error
+		if indent%2 != 0 {
+			return nil, fmt.Errorf("the line '%s' is not a multiple of 2", line)
+		}
+		// calculate level
+		level := indent / 2
+		line = strings.TrimSpace(line[indent:])
+		// if the line is empty, skip
+		if line == "" {
+			continue
+		}
+		// if level isn't greater than the level of the top of the stack
+		// it is not the child of the top of the stack
+		if level <= stack[len(stack)-1].Level {
+			// pop the top of the stack
+			stack = stack[:len(stack)-1]
+		}
+		// if the line is a folder
+		if isFolder(line) {
+			// create a new node
+			node := &Node{
+				Level: level,
+				Name:  strings.TrimSuffix(line, ":"),
+			}
+			// add the node to the top of the stack
+			stack[len(stack)-1].Children = append(stack[len(stack)-1].Children, node)
+			// push the node to the stack
+			stack = append(stack, node)
+		} else {
+			// if the line is a file
+			// create a new node
+			node, err := parseFileLine(line, headSize)
+			if err != nil {
+				return nil, err
+			}
+			node.Level = level
+			// add the node to the top of the stack
+			stack[len(stack)-1].Children = append(stack[len(stack)-1].Children, node)
+		}
+	}
+	return root, nil
+}
+
+func isFolder(line string) bool {
+	return strings.HasSuffix(line, ":")
+}
+
+// line definition:
+// [FileName:][FileSize:][Modified:]Url
+func parseFileLine(line string, headSize bool) (*Node, error) {
+	// if there is no url, it is an error
+	if !strings.Contains(line, "http://") && !strings.Contains(line, "https://") {
+		return nil, fmt.Errorf("invalid line: %s, because url is required for file", line)
+	}
+	index := strings.Index(line, "http://")
+	if index == -1 {
+		index = strings.Index(line, "https://")
+	}
+	url := line[index:]
+	info := line[:index]
+	node := &Node{
+		Url: url,
+	}
+	haveSize := false
+	if index > 0 {
+		if !strings.HasSuffix(info, ":") {
+			return nil, fmt.Errorf("invalid line: %s, because file info must end with ':'", line)
+		}
+		info = info[:len(info)-1]
+		if info == "" {
+			return nil, fmt.Errorf("invalid line: %s, because file name can't be empty", line)
+		}
+		infoParts := strings.Split(info, ":")
+		node.Name = infoParts[0]
+		if len(infoParts) > 1 {
+			size, err := strconv.ParseInt(infoParts[1], 10, 64)
+			if err != nil {
+				return nil, fmt.Errorf("invalid line: %s, because file size must be an integer", line)
+			}
+			node.Size = size
+			haveSize = true
+			if len(infoParts) > 2 {
+				modified, err := strconv.ParseInt(infoParts[2], 10, 64)
+				if err != nil {
+					return nil, fmt.Errorf("invalid line: %s, because file modified must be an unix timestamp", line)
+				}
+				node.Modified = modified
+			}
+		}
+	} else {
+		node.Name = stdpath.Base(url)
+	}
+	if !haveSize && headSize {
+		size, err := getSizeFromUrl(url)
+		if err != nil {
+			log.Errorf("get size from url error: %s", err)
+		} else {
+			node.Size = size
+		}
+	}
+	return node, nil
+}
+
+func splitPath(path string) []string {
+	if path == "/" {
+		return []string{"root"}
+	}
+	parts := strings.Split(path, "/")
+	parts[0] = "root"
+	return parts
+}
+
+func GetNodeFromRootByPath(root *Node, path string) *Node {
+	return root.getByPath(splitPath(path))
+}
+
+func nodeToObj(node *Node, path string) (model.Obj, error) {
+	if node == nil {
+		return nil, errs.ObjectNotFound
+	}
+	return &model.Object{
+		Name:     node.Name,
+		Size:     node.Size,
+		Modified: time.Unix(node.Modified, 0),
+		IsFolder: !node.isFile(),
+		Path:     path,
+	}, nil
+}
+
+func getSizeFromUrl(url string) (int64, error) {
+	res, err := base.RestyClient.R().SetDoNotParseResponse(true).Head(url)
+	if err != nil {
+		return 0, err
+	}
+	defer res.RawResponse.Body.Close()
+	if res.StatusCode() >= 300 {
+		return 0, fmt.Errorf("get size from url %s failed, status code: %d", url, res.StatusCode())
+	}
+	size, err := strconv.ParseInt(res.Header().Get("Content-Length"), 10, 64)
+	if err != nil {
+		return 0, err
+	}
+	return size, nil
+}

go.sum

@@ -17,14 +17,10 @@ github.com/aws/aws-sdk-go v1.44.194 h1:1ZDK+QDcc5oRbZGgRZSz561eR8XVizXCeGpoZKo33
 github.com/aws/aws-sdk-go v1.44.194/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA=
 github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
-github.com/blevesearch/bleve/v2 v2.3.6 h1:NlntUHcV5CSWIhpugx4d/BRMGCiaoI8ZZXrXlahzNq4=
-github.com/blevesearch/bleve/v2 v2.3.6/go.mod h1:JM2legf1cKVkdV8Ehu7msKIOKC0McSw0Q16Fmv9vsW4=
 github.com/blevesearch/bleve/v2 v2.3.7 h1:nIfIrhv28tvgBpbVF8Dq7/U1zW/YiwSqg/PBgE3x8bo=
 github.com/blevesearch/bleve/v2 v2.3.7/go.mod h1:2tToYD6mDeseIA13jcZiEEqYrVLg6xdk0v6+F7dWquU=
 github.com/blevesearch/bleve_index_api v1.0.5 h1:Lc986kpC4Z0/n1g3gg8ul7H+lxgOQPcXb9SxvQGu+tw=
 github.com/blevesearch/bleve_index_api v1.0.5/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms=
-github.com/blevesearch/geo v0.1.16 h1:unVaqUmlwprk56596OQRkGjtq1VZ8XFWSARj+h2cIBY=
-github.com/blevesearch/geo v0.1.16/go.mod h1:a1OlySNE+oDQ5qY0vJGYNoLIsMpbKbx8dnmuRP8D7H0=
 github.com/blevesearch/geo v0.1.17 h1:AguzI6/5mHXapzB0gE9IKWo+wWPHZmXZoscHcjFgAFA=
 github.com/blevesearch/geo v0.1.17/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM=
 github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
@@ -35,14 +31,10 @@ github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCD
 github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs=
 github.com/blevesearch/scorch_segment_api/v2 v2.1.4 h1:LmGmo5twU3gV+natJbKmOktS9eMhokPGKWuR+jX84vk=
 github.com/blevesearch/scorch_segment_api/v2 v2.1.4/go.mod h1:PgVnbbg/t1UkgezPDu8EHLi1BHQ17xUwsFdU6NnOYS0=
-github.com/blevesearch/segment v0.9.0 h1:5lG7yBCx98or7gK2cHMKPukPZ/31Kag7nONpoBt22Ac=
-github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ=
 github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU=
 github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw=
 github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s=
 github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs=
-github.com/blevesearch/upsidedown_store_api v1.0.1 h1:1SYRwyoFLwG3sj0ed89RLtM15amfX2pXlYbFOnF8zNU=
-github.com/blevesearch/upsidedown_store_api v1.0.1/go.mod h1:MQDVGpHZrpe3Uy26zJBf/a8h0FZY6xJbthIMm8myH2Q=
 github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMGZzVrdmaozG2MfoB+A=
 github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ=
 github.com/blevesearch/vellum v1.0.9 h1:PL+NWVk3dDGPCV0hoDu9XLLJgqU4E5s/dOeEJByQ2uQ=
@@ -55,8 +47,6 @@ github.com/blevesearch/zapx/v13 v13.3.7 h1:igIQg5eKmjw168I7av0Vtwedf7kHnQro/M+ub
 github.com/blevesearch/zapx/v13 v13.3.7/go.mod h1:yyrB4kJ0OT75UPZwT/zS+Ru0/jYKorCOOSY5dBzAy+s=
 github.com/blevesearch/zapx/v14 v14.3.7 h1:gfe+fbWslDWP/evHLtp/GOvmNM3sw1BbqD7LhycBX20=
 github.com/blevesearch/zapx/v14 v14.3.7/go.mod h1:9J/RbOkqZ1KSjmkOes03AkETX7hrXT0sFMpWH4ewC4w=
-github.com/blevesearch/zapx/v15 v15.3.8 h1:q4uMngBHzL1IIhRc8AJUEkj6dGOE3u1l3phLu7hq8uk=
-github.com/blevesearch/zapx/v15 v15.3.8/go.mod h1:m7Y6m8soYUvS7MjN9eKlz1xrLCcmqfFadmu7GhWIrLY=
 github.com/blevesearch/zapx/v15 v15.3.9 h1:/s9zqKxFaZKQTTcMO2b/Tup0ch5MSztlvw+frVDfIBk=
 github.com/blevesearch/zapx/v15 v15.3.9/go.mod h1:m7Y6m8soYUvS7MjN9eKlz1xrLCcmqfFadmu7GhWIrLY=
 github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
@@ -76,8 +66,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/deckarep/golang-set/v2 v2.2.0 h1:2pMQd3Soi6qfw7E5MMKaEh5W5ES18bW3AbFFnGl6LgQ=
-github.com/deckarep/golang-set/v2 v2.2.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
 github.com/deckarep/golang-set/v2 v2.3.0 h1:qs18EKUfHm2X9fA50Mr/M5hccg2tNnVqsiBImnyDs0g=
 github.com/deckarep/golang-set/v2 v2.3.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
 github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
@@ -128,7 +116,6 @@ github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
 github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -162,7 +149,6 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
-github.com/json-iterator/go v0.0.0-20171115153421-f7279a603ede/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=