github
/
alist
mirror of https://github.com/Xhofe/alist
1
0
Fork 0
Code Issues Releases Wiki Activity

feat(url-tree): support url tree driver writing (#7779 close #5166) 

* feat: support url tree writing

* fix: meta writable

* feat: disable writable via addition
pull/7716/head^2
KirCute_ECT 2025-01-10 20:50:56 +08:00 committed by GitHub
parent 25b4b55ee1
commit 51bcf83511
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 338 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

181
drivers/url_tree/driver.go
View File

@ -2,7 +2,11 @@ package url_tree
import (
"context"
"errors"
"github.com/alist-org/alist/v3/internal/op"
stdpath "path"
"strings"
"sync"
"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/errs"
@ -14,7 +18,8 @@ import (
type Urls struct {
model.Storage
Addition
root *Node
root *Node
mutex sync.RWMutex
}
func (d *Urls) Config() driver.Config {
@ -40,11 +45,15 @@ func (d *Urls) Drop(ctx context.Context) error {
}
func (d *Urls) Get(ctx context.Context, path string) (model.Obj, error) {
d.mutex.RLock()
defer d.mutex.RUnlock()
node := GetNodeFromRootByPath(d.root, path)
return nodeToObj(node, path)
}
func (d *Urls) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
d.mutex.RLock()
defer d.mutex.RUnlock()
node := GetNodeFromRootByPath(d.root, dir.GetPath())
log.Debugf("path: %s, node: %+v", dir.GetPath(), node)
if node == nil {
@ -59,6 +68,8 @@ func (d *Urls) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]
}
func (d *Urls) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
d.mutex.RLock()
defer d.mutex.RUnlock()
node := GetNodeFromRootByPath(d.root, file.GetPath())
log.Debugf("path: %s, node: %+v", file.GetPath(), node)
if node == nil {
@ -72,6 +83,174 @@ func (d *Urls) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*
return nil, errs.NotFile
}
func (d *Urls) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) {
if !d.Writable {
return nil, errs.PermissionDenied
}
d.mutex.Lock()
defer d.mutex.Unlock()
node := GetNodeFromRootByPath(d.root, parentDir.GetPath())
if node == nil {
return nil, errs.ObjectNotFound
}
if node.isFile() {
return nil, errs.NotFolder
}
dir := &Node{
Name: dirName,
Level: node.Level + 1,
}
node.Children = append(node.Children, dir)
d.updateStorage()
return nodeToObj(dir, stdpath.Join(parentDir.GetPath(), dirName))
}
func (d *Urls) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) {
if !d.Writable {
return nil, errs.PermissionDenied
}
if strings.HasPrefix(dstDir.GetPath(), srcObj.GetPath()) {
return nil, errors.New("cannot move parent dir to child")
}
d.mutex.Lock()
defer d.mutex.Unlock()
dstNode := GetNodeFromRootByPath(d.root, dstDir.GetPath())
if dstNode == nil || dstNode.isFile() {
return nil, errs.NotFolder
}
srcDir, srcName := stdpath.Split(srcObj.GetPath())
srcParentNode := GetNodeFromRootByPath(d.root, srcDir)
if srcParentNode == nil {
return nil, errs.ObjectNotFound
}
newChildren := make([]*Node, 0, len(srcParentNode.Children))
var srcNode *Node
for _, child := range srcParentNode.Children {
if child.Name == srcName {
srcNode = child
} else {
newChildren = append(newChildren, child)
}
}
if srcNode == nil {
return nil, errs.ObjectNotFound
}
srcParentNode.Children = newChildren
srcNode.setLevel(dstNode.Level + 1)
dstNode.Children = append(dstNode.Children, srcNode)
d.root.calSize()
d.updateStorage()
return nodeToObj(srcNode, stdpath.Join(dstDir.GetPath(), srcName))
}
func (d *Urls) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) {
if !d.Writable {
return nil, errs.PermissionDenied
}
d.mutex.Lock()
defer d.mutex.Unlock()
srcNode := GetNodeFromRootByPath(d.root, srcObj.GetPath())
if srcNode == nil {
return nil, errs.ObjectNotFound
}
srcNode.Name = newName
d.updateStorage()
return nodeToObj(srcNode, stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName))
}
func (d *Urls) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) {
if !d.Writable {
return nil, errs.PermissionDenied
}
if strings.HasPrefix(dstDir.GetPath(), srcObj.GetPath()) {
return nil, errors.New("cannot copy parent dir to child")
}
d.mutex.Lock()
defer d.mutex.Unlock()
dstNode := GetNodeFromRootByPath(d.root, dstDir.GetPath())
if dstNode == nil || dstNode.isFile() {
return nil, errs.NotFolder
}
srcNode := GetNodeFromRootByPath(d.root, srcObj.GetPath())
if srcNode == nil {
return nil, errs.ObjectNotFound
}
newNode := srcNode.deepCopy(dstNode.Level + 1)
dstNode.Children = append(dstNode.Children, newNode)
d.root.calSize()
d.updateStorage()
return nodeToObj(newNode, stdpath.Join(dstDir.GetPath(), stdpath.Base(srcObj.GetPath())))
}
func (d *Urls) Remove(ctx context.Context, obj model.Obj) error {
if !d.Writable {
return errs.PermissionDenied
}
d.mutex.Lock()
defer d.mutex.Unlock()
objDir, objName := stdpath.Split(obj.GetPath())
nodeParent := GetNodeFromRootByPath(d.root, objDir)
if nodeParent == nil {
return errs.ObjectNotFound
}
newChildren := make([]*Node, 0, len(nodeParent.Children))
var deletedObj *Node
for _, child := range nodeParent.Children {
if child.Name != objName {
newChildren = append(newChildren, child)
} else {
deletedObj = child
}
}
if deletedObj == nil {
return errs.ObjectNotFound
}
nodeParent.Children = newChildren
if deletedObj.Size > 0 {
d.root.calSize()
}
d.updateStorage()
return nil
}
func (d *Urls) PutURL(ctx context.Context, dstDir model.Obj, name, url string) (model.Obj, error) {
if !d.Writable {
return nil, errs.PermissionDenied
}
d.mutex.Lock()
defer d.mutex.Unlock()
dirNode := GetNodeFromRootByPath(d.root, dstDir.GetPath())
if dirNode == nil || dirNode.isFile() {
return nil, errs.NotFolder
}
newNode := &Node{
Name: name,
Level: dirNode.Level + 1,
Url: url,
}
dirNode.Children = append(dirNode.Children, newNode)
if d.HeadSize {
size, err := getSizeFromUrl(url)
if err != nil {
log.Errorf("get size from url error: %s", err)
} else {
newNode.Size = size
d.root.calSize()
}
}
d.updateStorage()
return nodeToObj(newNode, stdpath.Join(dstDir.GetPath(), name))
}
func (d *Urls) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error {
return errs.UploadNotSupported
}
func (d *Urls) updateStorage() {
d.UrlStructure = StringifyTree(d.root)
op.MustSaveDriverStorage(d)
}
//func (d *Template) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) {
// return nil, errs.NotSupport
//}

3
drivers/url_tree/meta.go
View File

@ -12,6 +12,7 @@ type Addition struct {
// define other
UrlStructure string `json:"url_structure" type:"text" required:"true" default:"https://jsd.nn.ci/gh/alist-org/alist/README.md\nhttps://jsd.nn.ci/gh/alist-org/alist/README_cn.md\nfolder:\n CONTRIBUTING.md:1635:https://jsd.nn.ci/gh/alist-org/alist/CONTRIBUTING.md\n CODE_OF_CONDUCT.md:2093:https://jsd.nn.ci/gh/alist-org/alist/CODE_OF_CONDUCT.md" help:"structure:FolderName:\n [FileName:][FileSize:][Modified:]Url"`
HeadSize bool `json:"head_size" type:"bool" default:"false" help:"Use head method to get file size, but it may be failed."`
Writable bool `json:"writable" type:"bool" default:"false"`
}
var config = driver.Config{
@ -20,7 +21,7 @@ var config = driver.Config{
OnlyLocal: false,
OnlyProxy: false,
NoCache: true,
NoUpload: true,
NoUpload: false,
NeedMs: false,
DefaultRoot: "",
CheckStatus: true,

18
drivers/url_tree/types.go
View File

@ -1,5 +1,7 @@
package url_tree
import "github.com/alist-org/alist/v3/pkg/utils"
// Node is a node in the folder tree
type Node struct {
Url string
@ -44,3 +46,19 @@ func (node *Node) calSize() int64 {
node.Size = size
return size
}
func (node *Node) setLevel(level int) {
node.Level = level
for _, child := range node.Children {
child.setLevel(level + 1)
}
}
func (node *Node) deepCopy(level int) *Node {
ret := *node
ret.Level = level
ret.Children, _ = utils.SliceConvert(ret.Children, func(child *Node) (*Node, error) {
return child.deepCopy(level + 1), nil
})
return &ret
}

46
drivers/url_tree/util.go
View File

@ -153,6 +153,9 @@ func splitPath(path string) []string {
if path == "/" {
return []string{"root"}
}
if strings.HasSuffix(path, "/") {
path = path[:len(path)-1]
}
parts := strings.Split(path, "/")
parts[0] = "root"
return parts
@ -190,3 +193,46 @@ func getSizeFromUrl(url string) (int64, error) {
}
return size, nil
}
func StringifyTree(node *Node) string {
sb := strings.Builder{}
if node.Level == -1 {
for i, child := range node.Children {
sb.WriteString(StringifyTree(child))
if i < len(node.Children)-1 {
sb.WriteString("\n")
}
}
return sb.String()
}
for i := 0; i < node.Level; i++ {
sb.WriteString(" ")
}
if node.Url == "" {
sb.WriteString(node.Name)
sb.WriteString(":")
for _, child := range node.Children {
sb.WriteString("\n")
sb.WriteString(StringifyTree(child))
}
} else if node.Size == 0 && node.Modified == 0 {
if stdpath.Base(node.Url) == node.Name {
sb.WriteString(node.Url)
} else {
sb.WriteString(fmt.Sprintf("%s:%s", node.Name, node.Url))
}
} else {
sb.WriteString(node.Name)
sb.WriteString(":")
if node.Size != 0 || node.Modified != 0 {
sb.WriteString(strconv.FormatInt(node.Size, 10))
sb.WriteString(":")
}
if node.Modified != 0 {
sb.WriteString(strconv.FormatInt(node.Modified, 10))
sb.WriteString(":")
}
sb.WriteString(node.Url)
}
return sb.String()
}

14
internal/driver/driver.go
View File

@ -80,6 +80,13 @@ type Put interface {
Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up UpdateProgress) error
}
type PutURL interface {
// PutURL directly put a URL into the storage
// Applicable to index-based drivers like URL-Tree or drivers that support uploading files as URLs
// Called when using SimpleHttp for offline downloading, skipping creating a download task
PutURL(ctx context.Context, dstDir model.Obj, name, url string) error
}
//type WriteResult interface {
// MkdirResult
// MoveResult
@ -109,6 +116,13 @@ type PutResult interface {
Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up UpdateProgress) (model.Obj, error)
}
type PutURLResult interface {
// PutURL directly put a URL into the storage
// Applicable to index-based drivers like URL-Tree or drivers that support uploading files as URLs
// Called when using SimpleHttp for offline downloading, skipping creating a download task
PutURL(ctx context.Context, dstDir model.Obj, name, url string) (model.Obj, error)
}
type UpdateProgress func(percentage float64)
type Progress struct {

47
internal/offline_download/tool/add.go
View File

@ -2,8 +2,11 @@ package tool
import (
"context"
"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/task"
"net/url"
"path"
"path/filepath"
"github.com/alist-org/alist/v3/internal/conf"
@ -30,18 +33,6 @@ type AddURLArgs struct {
}
func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, error) {
// get tool
tool, err := Tools.Get(args.Tool)
if err != nil {
return nil, errors.Wrapf(err, "failed get tool")
}
// check tool is ready
if !tool.IsReady() {
// try to init tool
if _, err := tool.Init(); err != nil {
return nil, errors.Wrapf(err, "failed init tool %s", args.Tool)
}
}
// check storage
storage, dstDirActualPath, err := op.GetStorageAndActualPath(args.DstDirPath)
if err != nil {
@ -63,6 +54,23 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro
return nil, errors.WithStack(errs.NotFolder)
}
}
// try putting url
if args.Tool == "SimpleHttp" && tryPutUrl(ctx, storage, dstDirActualPath, args.URL) {
return nil, nil
}
// get tool
tool, err := Tools.Get(args.Tool)
if err != nil {
return nil, errors.Wrapf(err, "failed get tool")
}
// check tool is ready
if !tool.IsReady() {
// try to init tool
if _, err := tool.Init(); err != nil {
return nil, errors.Wrapf(err, "failed init tool %s", args.Tool)
}
}
uid := uuid.NewString()
tempDir := filepath.Join(conf.Conf.TempDir, args.Tool, uid)
@ -98,3 +106,18 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro
DownloadTaskManager.Add(t)
return t, nil
}
func tryPutUrl(ctx context.Context, storage driver.Driver, dstDirActualPath, urlStr string) bool {
_, ok := storage.(driver.PutURL)
_, okResult := storage.(driver.PutURLResult)
if !ok && !okResult {
return false
}
u, err := url.Parse(urlStr)
if err != nil {
return false
}
dstName := path.Base(u.Path)
err = op.PutURL(ctx, storage, dstDirActualPath, dstName, urlStr)
return err == nil
}

40
internal/op/fs.go
View File

@ -586,3 +586,43 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod
}
return errors.WithStack(err)
}
func PutURL(ctx context.Context, storage driver.Driver, dstDirPath, dstName, url string, lazyCache ...bool) error {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
}
dstDirPath = utils.FixAndCleanPath(dstDirPath)
_, err := GetUnwrap(ctx, storage, stdpath.Join(dstDirPath, dstName))
if err == nil {
return errors.New("obj already exists")
}
err = MakeDir(ctx, storage, dstDirPath)
if err != nil {
return errors.WithMessagef(err, "failed to put url")
}
dstDir, err := GetUnwrap(ctx, storage, dstDirPath)
if err != nil {
return errors.WithMessagef(err, "failed to put url")
}
switch s := storage.(type) {
case driver.PutURLResult:
var newObj model.Obj
newObj, err = s.PutURL(ctx, dstDir, dstName, url)
if err == nil {
if newObj != nil {
addCacheObj(storage, dstDirPath, model.WrapObjName(newObj))
} else if !utils.IsBool(lazyCache...) {
ClearCache(storage, dstDirPath)
}
}
case driver.PutURL:
err = s.PutURL(ctx, dstDir, dstName, url)
if err == nil && !utils.IsBool(lazyCache...) {
ClearCache(storage, dstDirPath)
}
default:
return errs.NotImplement
}
log.Debugf("put url [%s](%s) done", dstName, url)
return errors.WithStack(err)
}

4
server/handles/offline_download.go
View File

@ -145,7 +145,9 @@ func AddOfflineDownload(c *gin.Context) {
common.ErrorResp(c, err, 500)
return
}
tasks = append(tasks, t)
if t != nil {
tasks = append(tasks, t)
}
}
common.SuccessResp(c, gin.H{
"tasks": getTaskInfos(tasks),