From ddcba93eea2cae4ae2c177064b965f45be2cb015 Mon Sep 17 00:00:00 2001
From: Noah Hsu <xhofe@qq.com>
Date: Mon, 28 Nov 2022 13:45:25 +0800
Subject: [PATCH] feat: multiple search indexes (#2514)
* refactor: abstract search interface
* wip: ~
* fix cycle import
* objs update hook
* wip: ~
* Delete search/none
* auto update index while cache changed
* db searcher
TODO: bleve init issue
cannot open index, metadata missing
* fix size type
why float64??
* fix typo
* fix nil pointer using
* api adapt ui
* bleve: fix clear & change struct
---
drivers/alist_v3/driver.go | 2 +-
drivers/alist_v3/types.go | 4 +-
go.mod | 1 +
go.sum | 2 +
internal/bootstrap/data/setting.go | 15 ++--
internal/bootstrap/index.go | 7 +-
internal/conf/config.go | 6 +-
internal/conf/const.go | 4 +-
internal/db/db.go | 17 ++--
internal/db/searchnode.go | 49 +++++++++++
internal/db/settinghooks.go | 101 ++++++++++------------
internal/db/settingitem.go | 14 +++-
internal/errs/search.go | 7 ++
internal/fs/walk.go | 45 ++++++++++
internal/index/build.go | 102 -----------------------
internal/index/index.go | 47 -----------
internal/index/search.go | 19 -----
internal/index/util.go | 46 ----------
{server/common => internal/model}/req.go | 2 +-
internal/model/search.go | 36 ++++++++
internal/op/fs.go | 6 ++
internal/op/hook.go | 13 +++
internal/search/bleve/init.go | 38 +++++++++
internal/search/bleve/search.go | 90 ++++++++++++++++++++
internal/search/build.go | 100 ++++++++++++++++++++++
internal/search/db/init.go | 16 ++++
internal/search/db/search.go | 46 ++++++++++
internal/search/import.go | 6 ++
internal/search/progress.go | 36 ++++++++
internal/search/search.go | 54 ++++++++++++
internal/search/searcher/manage.go | 9 ++
internal/search/searcher/searcher.go | 29 +++++++
internal/search/update.go | 73 ++++++++++++++++
pkg/utils/file.go | 7 ++
pkg/utils/slice.go | 9 ++
server/handles/fsread.go | 16 ++--
server/handles/index.go | 58 ++++++-------
server/handles/meta.go | 2 +-
server/handles/search.go | 42 ++++++++++
server/handles/storage.go | 2 +-
server/handles/user.go | 2 +-
server/middlewares/search.go | 19 +++++
server/router.go | 6 +-
43 files changed, 855 insertions(+), 350 deletions(-)
create mode 100644 internal/db/searchnode.go
create mode 100644 internal/errs/search.go
create mode 100644 internal/fs/walk.go
delete mode 100644 internal/index/build.go
delete mode 100644 internal/index/index.go
delete mode 100644 internal/index/search.go
delete mode 100644 internal/index/util.go
rename {server/common => internal/model}/req.go (95%)
create mode 100644 internal/model/search.go
create mode 100644 internal/op/hook.go
create mode 100644 internal/search/bleve/init.go
create mode 100644 internal/search/bleve/search.go
create mode 100644 internal/search/build.go
create mode 100644 internal/search/db/init.go
create mode 100644 internal/search/db/search.go
create mode 100644 internal/search/import.go
create mode 100644 internal/search/progress.go
create mode 100644 internal/search/search.go
create mode 100644 internal/search/searcher/manage.go
create mode 100644 internal/search/searcher/searcher.go
create mode 100644 internal/search/update.go
create mode 100644 server/handles/search.go
create mode 100644 server/middlewares/search.go
diff --git a/drivers/alist_v3/driver.go b/drivers/alist_v3/driver.go
index 6b8bc55e..3f633c73 100644
--- a/drivers/alist_v3/driver.go
+++ b/drivers/alist_v3/driver.go
@@ -46,7 +46,7 @@ func (d *AListV3) List(ctx context.Context, dir model.Obj, args model.ListArgs)
SetResult(&resp).
SetHeader("Authorization", d.AccessToken).
SetBody(ListReq{
- PageReq: common.PageReq{
+ PageReq: model.PageReq{
Page: 1,
PerPage: 0,
},
diff --git a/drivers/alist_v3/types.go b/drivers/alist_v3/types.go
index cf5006a6..f2ff32a0 100644
--- a/drivers/alist_v3/types.go
+++ b/drivers/alist_v3/types.go
@@ -3,11 +3,11 @@ package alist_v3
import (
"time"
- "github.com/alist-org/alist/v3/server/common"
+ "github.com/alist-org/alist/v3/internal/model"
)
type ListReq struct {
- common.PageReq
+ model.PageReq
Path string `json:"path" form:"path"`
Password string `json:"password" form:"password"`
Refresh bool `json:"refresh"`
diff --git a/go.mod b/go.mod
index 631953d5..9f7f145f 100644
--- a/go.mod
+++ b/go.mod
@@ -58,6 +58,7 @@ require (
github.com/blevesearch/zapx/v15 v15.3.6 // indirect
github.com/bluele/gcache v0.0.2 // indirect
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
+ github.com/deckarep/golang-set/v2 v2.1.0 // indirect
github.com/gaoyb7/115drive-webdav v0.1.8 // indirect
github.com/geoffgarside/ber v1.1.0 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
diff --git a/go.sum b/go.sum
index 13bf079a..0e111af6 100644
--- a/go.sum
+++ b/go.sum
@@ -67,6 +67,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/deckarep/golang-set/v2 v2.1.0 h1:g47V4Or+DUdzbs8FxCCmgb6VYd+ptPAngjM6dtGktsI=
+github.com/deckarep/golang-set/v2 v2.1.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
github.com/gaoyb7/115drive-webdav v0.1.8 h1:EJt4PSmcbvBY4KUh2zSo5p6fN9LZFNkIzuKejipubVw=
diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go
index 636af4a2..a4c09c0a 100644
--- a/internal/bootstrap/data/setting.go
+++ b/internal/bootstrap/data/setting.go
@@ -25,12 +25,13 @@ func initSettings() {
settings[i].Flag = model.DEPRECATED
}
}
- if settings != nil && len(settings) > 0 {
- err = db.SaveSettingItems(settings)
- if err != nil {
- log.Fatalf("failed save settings: %+v", err)
- }
- }
+ // what's going on here???
+ //if settings != nil && len(settings) > 0 {
+ // err = db.SaveSettingItems(settings)
+ // if err != nil {
+ // log.Fatalf("failed save settings: %+v", err)
+ // }
+ //}
// insert new items
for i := range initialSettingItems {
v := initialSettingItems[i]
@@ -122,11 +123,13 @@ func InitialSettings() []model.SettingItem {
Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE},
{Key: conf.OcrApi, Value: "https://api.nn.ci/ocr/file/json", Type: conf.TypeString, Group: model.GLOBAL},
{Key: conf.FilenameCharMapping, Value: `{"/": "|"}`, Type: conf.TypeText, Group: model.GLOBAL},
+ {Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,bleve,none", Group: model.GLOBAL},
// aria2 settings
{Key: conf.Aria2Uri, Value: "http://localhost:6800/jsonrpc", Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE},
{Key: conf.Aria2Secret, Value: "", Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE},
// single settings
{Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE},
+ {Key: conf.IndexProgress, Value: "{}", Type: conf.TypeText, Group: model.SINGLE, Flag: model.PRIVATE},
}
if flags.Dev {
initialSettingItems = append(initialSettingItems, []model.SettingItem{
diff --git a/internal/bootstrap/index.go b/internal/bootstrap/index.go
index bee273a6..02a73318 100644
--- a/internal/bootstrap/index.go
+++ b/internal/bootstrap/index.go
@@ -1,10 +1,5 @@
package bootstrap
-import (
- "github.com/alist-org/alist/v3/internal/conf"
- "github.com/alist-org/alist/v3/internal/index"
-)
-
func InitIndex() {
- index.Init(&conf.Conf.IndexDir)
+ // TODO init ? Probably not.
}
diff --git a/internal/conf/config.go b/internal/conf/config.go
index 122209ee..735e870f 100644
--- a/internal/conf/config.go
+++ b/internal/conf/config.go
@@ -45,13 +45,13 @@ type Config struct {
Database Database `json:"database"`
Scheme Scheme `json:"scheme"`
TempDir string `json:"temp_dir" env:"TEMP_DIR"`
- IndexDir string `json:"index_dir" env:"INDEX_DIR"`
+ BleveDir string `json:"bleve_dir" env:"BLEVE_DIR"`
Log LogConfig `json:"log"`
}
func DefaultConfig() *Config {
tempDir := filepath.Join(flags.DataDir, "temp")
- indexDir := filepath.Join(flags.DataDir, "index")
+ indexDir := filepath.Join(flags.DataDir, "bleve")
logPath := filepath.Join(flags.DataDir, "log/log.log")
dbPath := filepath.Join(flags.DataDir, "data.db")
return &Config{
@@ -66,7 +66,7 @@ func DefaultConfig() *Config {
TablePrefix: "x_",
DBFile: dbPath,
},
- IndexDir: indexDir,
+ BleveDir: indexDir,
Log: LogConfig{
Enable: true,
Name: logPath,
diff --git a/internal/conf/const.go b/internal/conf/const.go
index 400928c5..c4a2e546 100644
--- a/internal/conf/const.go
+++ b/internal/conf/const.go
@@ -40,13 +40,15 @@ const (
PrivacyRegs = "privacy_regs"
OcrApi = "ocr_api"
FilenameCharMapping = "filename_char_mapping"
+ SearchIndex = "search_index"
// aria2
Aria2Uri = "aria2_uri"
Aria2Secret = "aria2_secret"
// single
- Token = "token"
+ Token = "token"
+ IndexProgress = "index_progress"
)
const (
diff --git a/internal/db/db.go b/internal/db/db.go
index 1b7acc92..c7480a64 100644
--- a/internal/db/db.go
+++ b/internal/db/db.go
@@ -12,13 +12,18 @@ var db *gorm.DB
func Init(d *gorm.DB) {
db = d
- var err error
- if conf.Conf.Database.Type == "mysql" {
- err = db.Set("gorm:table_options", "ENGINE=InnoDB CHARSET=utf8mb4").AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem))
- } else {
- err = db.AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem))
- }
+ err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode))
if err != nil {
log.Fatalf("failed migrate database: %s", err.Error())
}
}
+
+func AutoMigrate(dst ...interface{}) error {
+ var err error
+ if conf.Conf.Database.Type == "mysql" {
+ err = db.Set("gorm:table_options", "ENGINE=InnoDB CHARSET=utf8mb4").AutoMigrate(dst...)
+ } else {
+ err = db.AutoMigrate(dst...)
+ }
+ return err
+}
diff --git a/internal/db/searchnode.go b/internal/db/searchnode.go
new file mode 100644
index 00000000..a37fce9c
--- /dev/null
+++ b/internal/db/searchnode.go
@@ -0,0 +1,49 @@
+package db
+
+import (
+ "fmt"
+
+ "github.com/alist-org/alist/v3/internal/model"
+ "github.com/pkg/errors"
+)
+
+func CreateSearchNode(node *model.SearchNode) error {
+ return db.Create(node).Error
+}
+
+func DeleteSearchNodesByParent(parent string) error {
+ return db.Where(fmt.Sprintf("%s LIKE ?",
+ columnName("path")), fmt.Sprintf("%s%%", parent)).
+ Delete(&model.SearchNode{}).Error
+}
+
+func ClearSearchNodes() error {
+ return db.Where("1 = 1").Delete(&model.SearchNode{}).Error
+}
+
+func GetSearchNodesByParent(parent string) ([]model.SearchNode, error) {
+ var nodes []model.SearchNode
+ if err := db.Where(fmt.Sprintf("%s = ?",
+ columnName("parent")), parent).Find(&nodes).Error; err != nil {
+ return nil, err
+ }
+ return nodes, nil
+}
+
+func SearchNode(req model.SearchReq) ([]model.SearchNode, int64, error) {
+ searchDB := db.Model(&model.SearchNode{}).Where(
+ fmt.Sprintf("%s LIKE ? AND %s LIKE ?",
+ columnName("parent"),
+ columnName("name")),
+ fmt.Sprintf("%s%%", req.Parent),
+ fmt.Sprintf("%%%s%%", req.Keywords))
+ var count int64
+ if err := searchDB.Count(&count).Error; err != nil {
+ return nil, 0, errors.Wrapf(err, "failed get users count")
+ }
+ var files []model.SearchNode
+ if err := searchDB.Offset((req.Page - 1) * req.PerPage).Limit(req.PerPage).Find(&files).Error; err != nil {
+ return nil, 0, err
+ }
+ return files, count, nil
+}
diff --git a/internal/db/settinghooks.go b/internal/db/settinghooks.go
index 6d968729..f7d85a07 100644
--- a/internal/db/settinghooks.go
+++ b/internal/db/settinghooks.go
@@ -11,81 +11,64 @@ import (
log "github.com/sirupsen/logrus"
)
-type SettingItemHook struct {
- Hook func(item *model.SettingItem) error
-}
+type SettingItemHook func(item *model.SettingItem) error
-var SettingItemHooks = map[string]SettingItemHook{
- conf.VideoTypes: {
- Hook: func(item *model.SettingItem) error {
- conf.TypesMap[conf.VideoTypes] = strings.Split(item.Value, ",")
- return nil
- },
+var settingItemHooks = map[string]SettingItemHook{
+ conf.VideoTypes: func(item *model.SettingItem) error {
+ conf.TypesMap[conf.VideoTypes] = strings.Split(item.Value, ",")
+ return nil
},
- conf.AudioTypes: {
- Hook: func(item *model.SettingItem) error {
- conf.TypesMap[conf.AudioTypes] = strings.Split(item.Value, ",")
- return nil
- },
+ conf.AudioTypes: func(item *model.SettingItem) error {
+ conf.TypesMap[conf.AudioTypes] = strings.Split(item.Value, ",")
+ return nil
},
- conf.ImageTypes: {
- Hook: func(item *model.SettingItem) error {
- conf.TypesMap[conf.ImageTypes] = strings.Split(item.Value, ",")
- return nil
- },
+ conf.ImageTypes: func(item *model.SettingItem) error {
+ conf.TypesMap[conf.ImageTypes] = strings.Split(item.Value, ",")
+ return nil
},
- conf.TextTypes: {
- Hook: func(item *model.SettingItem) error {
- conf.TypesMap[conf.TextTypes] = strings.Split(item.Value, ",")
- return nil
- },
+ conf.TextTypes: func(item *model.SettingItem) error {
+ conf.TypesMap[conf.TextTypes] = strings.Split(item.Value, ",")
+ return nil
},
- //conf.OfficeTypes: {
- // Hook: func(item *model.SettingItem) error {
- // conf.TypesMap[conf.OfficeTypes] = strings.Split(item.Value, ",")
- // return nil
- // },
- //},
- conf.ProxyTypes: {
- func(item *model.SettingItem) error {
- conf.TypesMap[conf.ProxyTypes] = strings.Split(item.Value, ",")
- return nil
- },
+ conf.ProxyTypes: func(item *model.SettingItem) error {
+ conf.TypesMap[conf.ProxyTypes] = strings.Split(item.Value, ",")
+ return nil
},
- conf.PrivacyRegs: {
- Hook: func(item *model.SettingItem) error {
- regStrs := strings.Split(item.Value, "\n")
- regs := make([]*regexp.Regexp, 0, len(regStrs))
- for _, regStr := range regStrs {
- reg, err := regexp.Compile(regStr)
- if err != nil {
- return errors.WithStack(err)
- }
- regs = append(regs, reg)
- }
- conf.PrivacyReg = regs
- return nil
- },
- },
- conf.FilenameCharMapping: {
- Hook: func(item *model.SettingItem) error {
- err := utils.Json.UnmarshalFromString(item.Value, &conf.FilenameCharMap)
+
+ conf.PrivacyRegs: func(item *model.SettingItem) error {
+ regStrs := strings.Split(item.Value, "\n")
+ regs := make([]*regexp.Regexp, 0, len(regStrs))
+ for _, regStr := range regStrs {
+ reg, err := regexp.Compile(regStr)
if err != nil {
- return err
+ return errors.WithStack(err)
}
- log.Debugf("filename char mapping: %+v", conf.FilenameCharMap)
- return nil
- },
+ regs = append(regs, reg)
+ }
+ conf.PrivacyReg = regs
+ return nil
+ },
+ conf.FilenameCharMapping: func(item *model.SettingItem) error {
+ err := utils.Json.UnmarshalFromString(item.Value, &conf.FilenameCharMap)
+ if err != nil {
+ return err
+ }
+ log.Debugf("filename char mapping: %+v", conf.FilenameCharMap)
+ return nil
},
}
func HandleSettingItem(item *model.SettingItem) (bool, error) {
- if hook, ok := SettingItemHooks[item.Key]; ok {
- return true, hook.Hook(item)
+ if hook, ok := settingItemHooks[item.Key]; ok {
+ return true, hook(item)
}
return false, nil
}
+func RegisterSettingItemHook(key string, hook SettingItemHook) {
+ settingItemHooks[key] = hook
+}
+
// func HandleSettingItems(items []model.SettingItem) error {
// for i := range items {
// if err := HandleSettingItem(&items[i]); err != nil {
diff --git a/internal/db/settingitem.go b/internal/db/settingitem.go
index 9b8d4d7c..da8b102b 100644
--- a/internal/db/settingitem.go
+++ b/internal/db/settingitem.go
@@ -108,11 +108,17 @@ func SaveSettingItems(items []model.SettingItem) error {
others = append(others, items[i])
}
}
- err := db.Save(others).Error
- if err == nil {
- settingsUpdate()
+ if len(others) > 0 {
+ err := db.Save(others).Error
+ if err != nil {
+ if len(others) < len(items) {
+ settingsUpdate()
+ }
+ return err
+ }
}
- return err
+ settingsUpdate()
+ return nil
}
func SaveSettingItem(item model.SettingItem) error {
diff --git a/internal/errs/search.go b/internal/errs/search.go
new file mode 100644
index 00000000..9c864f4d
--- /dev/null
+++ b/internal/errs/search.go
@@ -0,0 +1,7 @@
+package errs
+
+import "fmt"
+
+var (
+ SearchNotAvailable = fmt.Errorf("search not available")
+)
diff --git a/internal/fs/walk.go b/internal/fs/walk.go
new file mode 100644
index 00000000..1404c34a
--- /dev/null
+++ b/internal/fs/walk.go
@@ -0,0 +1,45 @@
+package fs
+
+import (
+ "context"
+ "path"
+ "path/filepath"
+
+ "github.com/alist-org/alist/v3/internal/db"
+ "github.com/alist-org/alist/v3/internal/model"
+)
+
+// WalkFS traverses filesystem fs starting at name up to depth levels.
+//
+// WalkFS will stop when current depth > `depth`. For each visited node,
+// WalkFS calls walkFn. If a visited file system node is a directory and
+// walkFn returns path.SkipDir, walkFS will skip traversal of this node.
+func WalkFS(ctx context.Context, depth int, name string, info model.Obj, walkFn func(reqPath string, info model.Obj, err error) error) error {
+ // This implementation is based on Walk's code in the standard path/path package.
+ walkFnErr := walkFn(name, info, nil)
+ if walkFnErr != nil {
+ if info.IsDir() && walkFnErr == filepath.SkipDir {
+ return nil
+ }
+ return walkFnErr
+ }
+ if !info.IsDir() || depth == 0 {
+ return nil
+ }
+ meta, _ := db.GetNearestMeta(name)
+ // Read directory names.
+ objs, err := List(context.WithValue(ctx, "meta", meta), name)
+ if err != nil {
+ return walkFnErr
+ }
+ for _, fileInfo := range objs {
+ filename := path.Join(name, fileInfo.GetName())
+ if err := WalkFS(ctx, depth-1, filename, fileInfo, walkFn); err != nil {
+ if err == filepath.SkipDir {
+ break
+ }
+ return err
+ }
+ }
+ return nil
+}
diff --git a/internal/index/build.go b/internal/index/build.go
deleted file mode 100644
index 1835774a..00000000
--- a/internal/index/build.go
+++ /dev/null
@@ -1,102 +0,0 @@
-package index
-
-import (
- "context"
- "path"
- "path/filepath"
- "time"
-
- "github.com/alist-org/alist/v3/internal/db"
- "github.com/alist-org/alist/v3/internal/fs"
- "github.com/alist-org/alist/v3/internal/model"
- "github.com/blevesearch/bleve/v2"
- "github.com/google/uuid"
-)
-
-// walkFS traverses filesystem fs starting at name up to depth levels.
-//
-// walkFS will stop when current depth > `depth`. For each visited node,
-// walkFS calls walkFn. If a visited file system node is a directory and
-// walkFn returns path.SkipDir, walkFS will skip traversal of this node.
-func walkFS(ctx context.Context, depth int, name string, info model.Obj, walkFn func(reqPath string, info model.Obj, err error) error) error {
- // This implementation is based on Walk's code in the standard path/path package.
- walkFnErr := walkFn(name, info, nil)
- if walkFnErr != nil {
- if info.IsDir() && walkFnErr == filepath.SkipDir {
- return nil
- }
- return walkFnErr
- }
- if !info.IsDir() || depth == 0 {
- return nil
- }
- meta, _ := db.GetNearestMeta(name)
- // Read directory names.
- objs, err := fs.List(context.WithValue(ctx, "meta", meta), name)
- if err != nil {
- return walkFnErr
- }
- for _, fileInfo := range objs {
- filename := path.Join(name, fileInfo.GetName())
- if err := walkFS(ctx, depth-1, filename, fileInfo, walkFn); err != nil {
- if err == filepath.SkipDir {
- break
- }
- return err
- }
- }
- return nil
-}
-
-type Data struct {
- Path string
-}
-
-func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth int) {
- // TODO: partial remove indices
- Reset()
- var batchs []*bleve.Batch
- var fileCount uint64 = 0
- for _, indexPath := range indexPaths {
- batch := func() *bleve.Batch {
- batch := index.NewBatch()
- // TODO: cache unchanged part
- walkFn := func(indexPath string, info model.Obj, err error) error {
- for _, avoidPath := range ignorePaths {
- if indexPath == avoidPath {
- return filepath.SkipDir
- }
- }
- if !info.IsDir() {
- batch.Index(uuid.NewString(), Data{Path: indexPath})
- fileCount += 1
- if fileCount%100 == 0 {
- WriteProgress(&Progress{
- FileCount: fileCount,
- IsDone: false,
- LastDoneTime: nil,
- })
- }
- }
- return nil
- }
- fi, err := fs.Get(ctx, indexPath)
- if err != nil {
- return batch
- }
- // TODO: run walkFS concurrently
- walkFS(ctx, maxDepth, indexPath, fi, walkFn)
- return batch
- }()
- batchs = append(batchs, batch)
- }
- for _, batch := range batchs {
- index.Batch(batch)
- }
- now := time.Now()
- WriteProgress(&Progress{
- FileCount: fileCount,
- IsDone: true,
- LastDoneTime: &now,
- })
-}
diff --git a/internal/index/index.go b/internal/index/index.go
deleted file mode 100644
index 393c54a9..00000000
--- a/internal/index/index.go
+++ /dev/null
@@ -1,47 +0,0 @@
-package index
-
-import (
- "os"
-
- "github.com/alist-org/alist/v3/internal/conf"
- "github.com/blevesearch/bleve/v2"
- log "github.com/sirupsen/logrus"
-)
-
-var index bleve.Index
-
-func Init(indexPath *string) {
- fileIndex, err := bleve.Open(*indexPath)
- if err == bleve.ErrorIndexPathDoesNotExist {
- log.Infof("Creating new index...")
- indexMapping := bleve.NewIndexMapping()
- fileIndex, err = bleve.New(*indexPath, indexMapping)
- if err != nil {
- log.Fatal(err)
- }
- }
- index = fileIndex
- progress := ReadProgress()
- if !progress.IsDone {
- log.Warnf("Last index build does not succeed!")
- WriteProgress(&Progress{
- FileCount: progress.FileCount,
- IsDone: false,
- LastDoneTime: nil,
- })
- }
-}
-
-func Reset() {
- log.Infof("Removing old index...")
- err := os.RemoveAll(conf.Conf.IndexDir)
- if err != nil {
- log.Fatal(err)
- }
- Init(&conf.Conf.IndexDir)
- WriteProgress(&Progress{
- FileCount: 0,
- IsDone: false,
- LastDoneTime: nil,
- })
-}
diff --git a/internal/index/search.go b/internal/index/search.go
deleted file mode 100644
index c0577044..00000000
--- a/internal/index/search.go
+++ /dev/null
@@ -1,19 +0,0 @@
-package index
-
-import (
- "github.com/blevesearch/bleve/v2"
- log "github.com/sirupsen/logrus"
-)
-
-func Search(queryString string, size int) (*bleve.SearchResult, error) {
- query := bleve.NewMatchQuery(queryString)
- search := bleve.NewSearchRequest(query)
- search.Size = size
- search.Fields = []string{"Path"}
- searchResults, err := index.Search(search)
- if err != nil {
- log.Errorf("search error: %+v", err)
- return nil, err
- }
- return searchResults, nil
-}
diff --git a/internal/index/util.go b/internal/index/util.go
deleted file mode 100644
index 2e890634..00000000
--- a/internal/index/util.go
+++ /dev/null
@@ -1,46 +0,0 @@
-package index
-
-import (
- "errors"
- "os"
- "path/filepath"
- "time"
-
- "github.com/alist-org/alist/v3/internal/conf"
- "github.com/alist-org/alist/v3/pkg/utils"
- log "github.com/sirupsen/logrus"
-)
-
-type Progress struct {
- FileCount uint64 `json:"file_count"`
- IsDone bool `json:"is_done"`
- LastDoneTime *time.Time `json:"last_done_time"`
-}
-
-func ReadProgress() Progress {
- progressFilePath := filepath.Join(conf.Conf.IndexDir, "progress.json")
- _, err := os.Stat(progressFilePath)
- progress := Progress{0, false, nil}
- if errors.Is(err, os.ErrNotExist) {
- if !utils.WriteJsonToFile(progressFilePath, progress) {
- log.Fatalf("failed to create index progress file")
- }
- }
- progressBytes, err := os.ReadFile(progressFilePath)
- if err != nil {
- log.Fatalf("reading index progress file error: %+v", err)
- }
- err = utils.Json.Unmarshal(progressBytes, &progress)
- if err != nil {
- log.Fatalf("load index progress error: %+v", err)
- }
- return progress
-}
-
-func WriteProgress(progress *Progress) {
- progressFilePath := filepath.Join(conf.Conf.IndexDir, "progress.json")
- log.Infof("write index progress: %v", progress)
- if !utils.WriteJsonToFile(progressFilePath, progress) {
- log.Fatalf("failed to write to index progress file")
- }
-}
diff --git a/server/common/req.go b/internal/model/req.go
similarity index 95%
rename from server/common/req.go
rename to internal/model/req.go
index 62476a86..fe3a08bd 100644
--- a/server/common/req.go
+++ b/internal/model/req.go
@@ -1,4 +1,4 @@
-package common
+package model
type PageReq struct {
Page int `json:"page" form:"page"`
diff --git a/internal/model/search.go b/internal/model/search.go
new file mode 100644
index 00000000..dd467251
--- /dev/null
+++ b/internal/model/search.go
@@ -0,0 +1,36 @@
+package model
+
+import (
+ "fmt"
+ "time"
+)
+
+type IndexProgress struct {
+ ObjCount uint64 `json:"obj_count"`
+ IsDone bool `json:"is_done"`
+ LastDoneTime *time.Time `json:"last_done_time"`
+ Error string `json:"error"`
+}
+
+type SearchReq struct {
+ Parent string `json:"parent"`
+ Keywords string `json:"keywords"`
+ PageReq
+}
+
+type SearchNode struct {
+ Parent string `json:"parent" gorm:"index"`
+ Name string `json:"name" gorm:"index"`
+ IsDir bool `json:"is_dir"`
+ Size int64 `json:"size"`
+}
+
+func (p *SearchReq) Validate() error {
+ if p.Page < 1 {
+ return fmt.Errorf("page can't < 1")
+ }
+ if p.PerPage < 1 {
+ return fmt.Errorf("per_page can't < 1")
+ }
+ return nil
+}
diff --git a/internal/op/fs.go b/internal/op/fs.go
index 4bf45038..0da4e1be 100644
--- a/internal/op/fs.go
+++ b/internal/op/fs.go
@@ -59,6 +59,12 @@ func List(ctx context.Context, storage driver.Driver, path string, args model.Li
if err != nil {
return nil, errors.Wrapf(err, "failed to list objs")
}
+ // call hooks
+ go func() {
+ for _, hook := range objsUpdateHooks {
+ hook(args.ReqPath, files)
+ }
+ }()
if !storage.Config().NoCache {
if len(files) > 0 {
log.Debugf("set cache: %s => %+v", key, files)
diff --git a/internal/op/hook.go b/internal/op/hook.go
new file mode 100644
index 00000000..7c624e9e
--- /dev/null
+++ b/internal/op/hook.go
@@ -0,0 +1,13 @@
+package op
+
+import "github.com/alist-org/alist/v3/internal/model"
+
+type ObjsUpdateHook = func(parent string, objs []model.Obj)
+
+var (
+ objsUpdateHooks = make([]ObjsUpdateHook, 0)
+)
+
+func RegisterObjsUpdateHook(hook ObjsUpdateHook) {
+ objsUpdateHooks = append(objsUpdateHooks, hook)
+}
diff --git a/internal/search/bleve/init.go b/internal/search/bleve/init.go
new file mode 100644
index 00000000..07203685
--- /dev/null
+++ b/internal/search/bleve/init.go
@@ -0,0 +1,38 @@
+package bleve
+
+import (
+ "github.com/alist-org/alist/v3/internal/conf"
+ "github.com/alist-org/alist/v3/internal/search/searcher"
+ "github.com/blevesearch/bleve/v2"
+ log "github.com/sirupsen/logrus"
+)
+
+var config = searcher.Config{
+ Name: "bleve",
+}
+
+func Init(indexPath *string) (bleve.Index, error) {
+ log.Debugf("bleve path: %s", *indexPath)
+ fileIndex, err := bleve.Open(*indexPath)
+ if err == bleve.ErrorIndexPathDoesNotExist {
+ log.Infof("Creating new index...")
+ indexMapping := bleve.NewIndexMapping()
+ fileIndex, err = bleve.New(*indexPath, indexMapping)
+ if err != nil {
+ return nil, err
+ }
+ } else if err != nil {
+ return nil, err
+ }
+ return fileIndex, nil
+}
+
+func init() {
+ searcher.RegisterSearcher(config, func() (searcher.Searcher, error) {
+ b, err := Init(&conf.Conf.BleveDir)
+ if err != nil {
+ return nil, err
+ }
+ return &Bleve{BIndex: b}, nil
+ })
+}
diff --git a/internal/search/bleve/search.go b/internal/search/bleve/search.go
new file mode 100644
index 00000000..d629c7a1
--- /dev/null
+++ b/internal/search/bleve/search.go
@@ -0,0 +1,90 @@
+package bleve
+
+import (
+ "context"
+ "os"
+
+ "github.com/alist-org/alist/v3/internal/conf"
+ "github.com/alist-org/alist/v3/internal/errs"
+ "github.com/alist-org/alist/v3/internal/model"
+ "github.com/alist-org/alist/v3/internal/search/searcher"
+ "github.com/alist-org/alist/v3/pkg/utils"
+ "github.com/blevesearch/bleve/v2"
+ search2 "github.com/blevesearch/bleve/v2/search"
+ "github.com/google/uuid"
+ log "github.com/sirupsen/logrus"
+)
+
+type Bleve struct {
+ BIndex bleve.Index
+}
+
+func (b *Bleve) Config() searcher.Config {
+ return config
+}
+
+func (b *Bleve) Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) {
+ query := bleve.NewMatchQuery(req.Keywords)
+ query.SetField("name")
+ search := bleve.NewSearchRequest(query)
+ search.Size = req.PerPage
+ search.Fields = []string{"*"}
+ searchResults, err := b.BIndex.Search(search)
+ if err != nil {
+ log.Errorf("search error: %+v", err)
+ return nil, 0, err
+ }
+ res, err := utils.SliceConvert(searchResults.Hits, func(src *search2.DocumentMatch) (model.SearchNode, error) {
+ return model.SearchNode{
+ Parent: src.Fields["parent"].(string),
+ Name: src.Fields["name"].(string),
+ IsDir: src.Fields["is_dir"].(bool),
+ Size: int64(src.Fields["size"].(float64)),
+ }, nil
+ })
+ return res, int64(len(res)), nil
+}
+
+func (b *Bleve) Index(ctx context.Context, parent string, obj model.Obj) error {
+ return b.BIndex.Index(uuid.NewString(), model.SearchNode{
+ Parent: parent,
+ Name: obj.GetName(),
+ IsDir: obj.IsDir(),
+ Size: obj.GetSize(),
+ })
+}
+
+func (b *Bleve) Get(ctx context.Context, parent string) ([]model.SearchNode, error) {
+ return nil, errs.NotSupport
+}
+
+func (b *Bleve) Del(ctx context.Context, prefix string) error {
+ return errs.NotSupport
+}
+
+func (b *Bleve) Release(ctx context.Context) error {
+ if b.BIndex != nil {
+ return b.BIndex.Close()
+ }
+ return nil
+}
+
+func (b *Bleve) Clear(ctx context.Context) error {
+ err := b.Release(ctx)
+ if err != nil {
+ return err
+ }
+ log.Infof("Removing old index...")
+ err = os.RemoveAll(conf.Conf.BleveDir)
+ if err != nil {
+ log.Errorf("clear bleve error: %+v", err)
+ }
+ bIndex, err := Init(&conf.Conf.BleveDir)
+ if err != nil {
+ return err
+ }
+ b.BIndex = bIndex
+ return nil
+}
+
+var _ searcher.Searcher = (*Bleve)(nil)
diff --git a/internal/search/build.go b/internal/search/build.go
new file mode 100644
index 00000000..59bdfa42
--- /dev/null
+++ b/internal/search/build.go
@@ -0,0 +1,100 @@
+package search
+
+import (
+ "context"
+ "path"
+ "path/filepath"
+ "time"
+
+ "github.com/alist-org/alist/v3/internal/db"
+ "github.com/alist-org/alist/v3/internal/fs"
+ "github.com/alist-org/alist/v3/internal/model"
+ log "github.com/sirupsen/logrus"
+)
+
+var (
+ Running = false
+)
+
+func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth int, count bool) error {
+ var objCount uint64 = 0
+ Running = true
+ var (
+ err error
+ fi model.Obj
+ )
+ defer func() {
+ Running = false
+ now := time.Now()
+ eMsg := ""
+ if err != nil {
+ log.Errorf("build index error: %+v", err)
+ eMsg = err.Error()
+ } else {
+ log.Infof("success build index, count: %d", objCount)
+ }
+ if count {
+ WriteProgress(&model.IndexProgress{
+ ObjCount: objCount,
+ IsDone: err == nil,
+ LastDoneTime: &now,
+ Error: eMsg,
+ })
+ }
+ }()
+ admin, err := db.GetAdmin()
+ if err != nil {
+ return err
+ }
+ if count {
+ WriteProgress(&model.IndexProgress{
+ ObjCount: 0,
+ IsDone: false,
+ })
+ }
+ for _, indexPath := range indexPaths {
+ walkFn := func(indexPath string, info model.Obj, err error) error {
+ for _, avoidPath := range ignorePaths {
+ if indexPath == avoidPath {
+ return filepath.SkipDir
+ }
+ }
+ // ignore root
+ if indexPath == "/" {
+ return nil
+ }
+ err = instance.Index(ctx, path.Dir(indexPath), info)
+ if err != nil {
+ return err
+ } else {
+ objCount++
+ }
+ if objCount%100 == 0 {
+ log.Infof("index obj count: %d", objCount)
+ log.Debugf("current success index: %s", indexPath)
+ if count {
+ WriteProgress(&model.IndexProgress{
+ ObjCount: objCount,
+ IsDone: false,
+ LastDoneTime: nil,
+ })
+ }
+ }
+ return nil
+ }
+ fi, err = fs.Get(ctx, indexPath)
+ if err != nil {
+ return err
+ }
+ // TODO: run walkFS concurrently
+ err = fs.WalkFS(context.WithValue(ctx, "user", admin), maxDepth, indexPath, fi, walkFn)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func Clear(ctx context.Context) error {
+ return instance.Clear(ctx)
+}
diff --git a/internal/search/db/init.go b/internal/search/db/init.go
new file mode 100644
index 00000000..bd83ffe6
--- /dev/null
+++ b/internal/search/db/init.go
@@ -0,0 +1,16 @@
+package db
+
+import (
+ "github.com/alist-org/alist/v3/internal/search/searcher"
+)
+
+var config = searcher.Config{
+ Name: "database",
+ AutoUpdate: true,
+}
+
+func init() {
+ searcher.RegisterSearcher(config, func() (searcher.Searcher, error) {
+ return &DB{}, nil
+ })
+}
diff --git a/internal/search/db/search.go b/internal/search/db/search.go
new file mode 100644
index 00000000..92bb1645
--- /dev/null
+++ b/internal/search/db/search.go
@@ -0,0 +1,46 @@
+package db
+
+import (
+ "context"
+
+ "github.com/alist-org/alist/v3/internal/db"
+ "github.com/alist-org/alist/v3/internal/model"
+ "github.com/alist-org/alist/v3/internal/search/searcher"
+)
+
+type DB struct{}
+
+func (D DB) Config() searcher.Config {
+ return config
+}
+
+func (D DB) Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) {
+ return db.SearchNode(req)
+}
+
+func (D DB) Index(ctx context.Context, parent string, obj model.Obj) error {
+ return db.CreateSearchNode(&model.SearchNode{
+ Parent: parent,
+ Name: obj.GetName(),
+ IsDir: obj.IsDir(),
+ Size: obj.GetSize(),
+ })
+}
+
+func (D DB) Get(ctx context.Context, parent string) ([]model.SearchNode, error) {
+ return db.GetSearchNodesByParent(parent)
+}
+
+func (D DB) Del(ctx context.Context, prefix string) error {
+ return db.DeleteSearchNodesByParent(prefix)
+}
+
+func (D DB) Release(ctx context.Context) error {
+ return nil
+}
+
+func (D DB) Clear(ctx context.Context) error {
+ return db.ClearSearchNodes()
+}
+
+var _ searcher.Searcher = (*DB)(nil)
diff --git a/internal/search/import.go b/internal/search/import.go
new file mode 100644
index 00000000..929b1583
--- /dev/null
+++ b/internal/search/import.go
@@ -0,0 +1,6 @@
+package search
+
+import (
+ _ "github.com/alist-org/alist/v3/internal/search/bleve"
+ _ "github.com/alist-org/alist/v3/internal/search/db"
+)
diff --git a/internal/search/progress.go b/internal/search/progress.go
new file mode 100644
index 00000000..85cc4f37
--- /dev/null
+++ b/internal/search/progress.go
@@ -0,0 +1,36 @@
+package search
+
+import (
+ "context"
+
+ "github.com/alist-org/alist/v3/internal/conf"
+ "github.com/alist-org/alist/v3/internal/db"
+ "github.com/alist-org/alist/v3/internal/model"
+ "github.com/alist-org/alist/v3/internal/setting"
+ "github.com/alist-org/alist/v3/pkg/utils"
+ log "github.com/sirupsen/logrus"
+)
+
+func Progress(ctx context.Context) (*model.IndexProgress, error) {
+ p := setting.GetStr(conf.IndexProgress)
+ var progress model.IndexProgress
+ err := utils.Json.UnmarshalFromString(p, &progress)
+ return &progress, err
+}
+
+func WriteProgress(progress *model.IndexProgress) {
+ p, err := utils.Json.MarshalToString(progress)
+ if err != nil {
+ log.Errorf("marshal progress error: %+v", err)
+ }
+ err = db.SaveSettingItem(model.SettingItem{
+ Key: conf.IndexProgress,
+ Value: p,
+ Type: conf.TypeText,
+ Group: model.SINGLE,
+ Flag: model.PRIVATE,
+ })
+ if err != nil {
+ log.Errorf("save progress error: %+v", err)
+ }
+}
diff --git a/internal/search/search.go b/internal/search/search.go
new file mode 100644
index 00000000..1c836ebe
--- /dev/null
+++ b/internal/search/search.go
@@ -0,0 +1,54 @@
+package search
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/alist-org/alist/v3/internal/conf"
+ "github.com/alist-org/alist/v3/internal/db"
+ "github.com/alist-org/alist/v3/internal/model"
+ "github.com/alist-org/alist/v3/internal/search/searcher"
+ log "github.com/sirupsen/logrus"
+)
+
+var instance searcher.Searcher = nil
+
+// Init or reset index
+func Init(mode string) error {
+ if instance != nil {
+ err := instance.Release(context.Background())
+ if err != nil {
+ log.Errorf("release instance err: %+v", err)
+ }
+ instance = nil
+ }
+ if Running {
+ return fmt.Errorf("index is running")
+ }
+ if mode == "none" {
+ log.Warnf("not enable search")
+ return nil
+ }
+ s, ok := searcher.NewMap[mode]
+ if !ok {
+ return fmt.Errorf("not support index: %s", mode)
+ }
+ i, err := s()
+ if err != nil {
+ log.Errorf("init searcher error: %+v", err)
+ } else {
+ instance = i
+ }
+ return err
+}
+
+func Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) {
+ return instance.Search(ctx, req)
+}
+
+func init() {
+ db.RegisterSettingItemHook(conf.SearchIndex, func(item *model.SettingItem) error {
+ log.Debugf("searcher init, mode: %s", item.Value)
+ return Init(item.Value)
+ })
+}
diff --git a/internal/search/searcher/manage.go b/internal/search/searcher/manage.go
new file mode 100644
index 00000000..92bdd883
--- /dev/null
+++ b/internal/search/searcher/manage.go
@@ -0,0 +1,9 @@
+package searcher
+
+type New func() (Searcher, error)
+
+var NewMap = map[string]New{}
+
+func RegisterSearcher(config Config, searcher New) {
+ NewMap[config.Name] = searcher
+}
diff --git a/internal/search/searcher/searcher.go b/internal/search/searcher/searcher.go
new file mode 100644
index 00000000..77378f32
--- /dev/null
+++ b/internal/search/searcher/searcher.go
@@ -0,0 +1,29 @@
+package searcher
+
+import (
+ "context"
+
+ "github.com/alist-org/alist/v3/internal/model"
+)
+
+type Config struct {
+ Name string
+ AutoUpdate bool
+}
+
+type Searcher interface {
+ // Config of the searcher
+ Config() Config
+ // Search specific keywords in specific path
+ Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error)
+ // Index obj with parent
+ Index(ctx context.Context, parent string, obj model.Obj) error
+ // Get by parent
+ Get(ctx context.Context, parent string) ([]model.SearchNode, error)
+ // Del with prefix
+ Del(ctx context.Context, prefix string) error
+ // Release resource
+ Release(ctx context.Context) error
+ // Clear all index
+ Clear(ctx context.Context) error
+}
diff --git a/internal/search/update.go b/internal/search/update.go
new file mode 100644
index 00000000..2da44283
--- /dev/null
+++ b/internal/search/update.go
@@ -0,0 +1,73 @@
+package search
+
+import (
+ "context"
+ "path"
+
+ "github.com/alist-org/alist/v3/internal/model"
+ "github.com/alist-org/alist/v3/internal/op"
+ mapset "github.com/deckarep/golang-set/v2"
+ log "github.com/sirupsen/logrus"
+)
+
+func Update(parent string, objs []model.Obj) {
+ if instance != nil && !instance.Config().AutoUpdate {
+ return
+ }
+ ctx := context.Background()
+ // only update when index have built
+ progress, err := Progress(ctx)
+ if err != nil {
+ log.Errorf("update search index error while get progress: %+v", err)
+ return
+ }
+ if !progress.IsDone {
+ return
+ }
+ nodes, err := instance.Get(ctx, parent)
+ if err != nil {
+ log.Errorf("update search index error while get nodes: %+v", err)
+ return
+ }
+ now := mapset.NewSet[string]()
+ for i := range objs {
+ now.Add(objs[i].GetName())
+ }
+ old := mapset.NewSet[string]()
+ for i := range nodes {
+ old.Add(nodes[i].Name)
+ }
+ // delete data that no longer exists
+ toDelete := old.Difference(now)
+ toAdd := now.Difference(old)
+ for i := range nodes {
+ if toDelete.Contains(nodes[i].Name) {
+ err = instance.Del(ctx, path.Join(parent, nodes[i].Name))
+ if err != nil {
+ log.Errorf("update search index error while del old node: %+v", err)
+ return
+ }
+ }
+ }
+ for i := range objs {
+ if toAdd.Contains(objs[i].GetName()) {
+ err = instance.Index(ctx, parent, objs[i])
+ if err != nil {
+ log.Errorf("update search index error while index new node: %+v", err)
+ return
+ }
+ // build index if it's a folder
+ if objs[i].IsDir() {
+ err = BuildIndex(ctx, []string{path.Join(parent, objs[i].GetName())}, nil, -1, false)
+ if err != nil {
+ log.Errorf("update search index error while build index: %+v", err)
+ return
+ }
+ }
+ }
+ }
+}
+
+func init() {
+ op.RegisterObjsUpdateHook(Update)
+}
diff --git a/pkg/utils/file.go b/pkg/utils/file.go
index cb580ba7..985be6da 100644
--- a/pkg/utils/file.go
+++ b/pkg/utils/file.go
@@ -138,6 +138,13 @@ func GetFileType(filename string) int {
return conf.UNKNOWN
}
+func GetObjType(filename string, isDir bool) int {
+ if isDir {
+ return conf.FOLDER
+ }
+ return GetFileType(filename)
+}
+
func GetMimeType(name string) string {
ext := path.Ext(name)
m := mime.TypeByExtension(ext)
diff --git a/pkg/utils/slice.go b/pkg/utils/slice.go
index 5ab5b018..c1d1584c 100644
--- a/pkg/utils/slice.go
+++ b/pkg/utils/slice.go
@@ -35,3 +35,12 @@ func SliceConvert[S any, D any](srcS []S, convert func(src S) (D, error)) ([]D,
}
return res, nil
}
+
+func MustSliceConvert[S any, D any](srcS []S, convert func(src S) D) []D {
+ var res []D
+ for i := range srcS {
+ dst := convert(srcS[i])
+ res = append(res, dst)
+ }
+ return res
+}
diff --git a/server/handles/fsread.go b/server/handles/fsread.go
index 106c87ae..b13b8402 100644
--- a/server/handles/fsread.go
+++ b/server/handles/fsread.go
@@ -19,7 +19,7 @@ import (
)
type ListReq struct {
- common.PageReq
+ model.PageReq
Path string `json:"path" form:"path"`
Password string `json:"password" form:"password"`
Refresh bool `json:"refresh"`
@@ -86,7 +86,7 @@ func FsList(c *gin.Context) {
provider = storage.GetStorage().Driver
}
common.SuccessResp(c, FsListResp{
- Content: toObjResp(objs, req.Path, isEncrypt(meta, req.Path)),
+ Content: toObjsResp(objs, req.Path, isEncrypt(meta, req.Path)),
Total: int64(total),
Readme: getReadme(meta, req.Path),
Write: user.CanWrite() || common.CanWrite(meta, req.Path),
@@ -165,7 +165,7 @@ func isEncrypt(meta *model.Meta, path string) bool {
return true
}
-func pagination(objs []model.Obj, req *common.PageReq) (int, []model.Obj) {
+func pagination(objs []model.Obj, req *model.PageReq) (int, []model.Obj) {
pageIndex, pageSize := req.Page, req.PerPage
total := len(objs)
start := (pageIndex - 1) * pageSize
@@ -179,17 +179,13 @@ func pagination(objs []model.Obj, req *common.PageReq) (int, []model.Obj) {
return total, objs[start:end]
}
-func toObjResp(objs []model.Obj, parent string, encrypt bool) []ObjResp {
+func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp {
var resp []ObjResp
for _, obj := range objs {
thumb := ""
if t, ok := obj.(model.Thumb); ok {
thumb = t.Thumb()
}
- tp := conf.FOLDER
- if !obj.IsDir() {
- tp = utils.GetFileType(obj.GetName())
- }
resp = append(resp, ObjResp{
Name: utils.MappingName(obj.GetName(), conf.FilenameCharMap),
Size: obj.GetSize(),
@@ -197,7 +193,7 @@ func toObjResp(objs []model.Obj, parent string, encrypt bool) []ObjResp {
Modified: obj.ModTime(),
Sign: common.Sign(obj, parent, encrypt),
Thumb: thumb,
- Type: tp,
+ Type: utils.GetObjType(obj.GetName(), obj.IsDir()),
})
}
return resp
@@ -299,7 +295,7 @@ func FsGet(c *gin.Context) {
RawURL: rawURL,
Readme: getReadme(meta, req.Path),
Provider: provider,
- Related: toObjResp(related, parentPath, isEncrypt(parentMeta, parentPath)),
+ Related: toObjsResp(related, parentPath, isEncrypt(parentMeta, parentPath)),
})
}
diff --git a/server/handles/index.go b/server/handles/index.go
index d5f033e6..1c924b31 100644
--- a/server/handles/index.go
+++ b/server/handles/index.go
@@ -2,53 +2,49 @@ package handles
import (
"context"
- "strconv"
- "github.com/alist-org/alist/v3/internal/db"
- "github.com/alist-org/alist/v3/internal/index"
+ "github.com/alist-org/alist/v3/internal/search"
"github.com/alist-org/alist/v3/server/common"
"github.com/gin-gonic/gin"
+ log "github.com/sirupsen/logrus"
)
+type BuildIndexReq struct {
+ Paths []string `json:"paths"`
+ MaxDepth int `json:"max_depth"`
+ IgnorePaths []string `json:"ignore_paths"`
+}
+
func BuildIndex(c *gin.Context) {
+ var req BuildIndexReq
+ if err := c.ShouldBind(&req); err != nil {
+ common.ErrorResp(c, err, 400)
+ return
+ }
+ if search.Running {
+ common.ErrorStrResp(c, "index is running", 400)
+ return
+ }
go func() {
- // TODO: consider run build index as non-admin
- user, _ := db.GetAdmin()
- ctx := context.WithValue(c.Request.Context(), "user", user)
- maxDepth, err := strconv.Atoi(c.PostForm("max_depth"))
+ ctx := context.Background()
+ err := search.Clear(ctx)
if err != nil {
- maxDepth = -1
+ log.Errorf("clear index error: %+v", err)
+ return
+ }
+ err = search.BuildIndex(context.Background(), req.Paths, req.IgnorePaths, req.MaxDepth, true)
+ if err != nil {
+ log.Errorf("build index error: %+v", err)
}
- indexPaths := []string{"/"}
- ignorePaths := c.PostFormArray("ignore_paths")
- index.BuildIndex(ctx, indexPaths, ignorePaths, maxDepth)
}()
common.SuccessResp(c)
}
func GetProgress(c *gin.Context) {
- progress := index.ReadProgress()
- common.SuccessResp(c, progress)
-}
-
-func Search(c *gin.Context) {
- results := []string{}
- query, exists := c.GetQuery("query")
- if !exists {
- common.SuccessResp(c, results)
- }
- sizeStr, _ := c.GetQuery("size")
- size, err := strconv.Atoi(sizeStr)
- if err != nil {
- size = 10
- }
- searchResults, err := index.Search(query, size)
+ progress, err := search.Progress(c)
if err != nil {
common.ErrorResp(c, err, 500)
return
}
- for _, documentMatch := range searchResults.Hits {
- results = append(results, documentMatch.Fields["Path"].(string))
- }
- common.SuccessResp(c, results)
+ common.SuccessResp(c, progress)
}
diff --git a/server/handles/meta.go b/server/handles/meta.go
index 07498374..16019ed2 100644
--- a/server/handles/meta.go
+++ b/server/handles/meta.go
@@ -15,7 +15,7 @@ import (
)
func ListMetas(c *gin.Context) {
- var req common.PageReq
+ var req model.PageReq
if err := c.ShouldBind(&req); err != nil {
common.ErrorResp(c, err, 400)
return
diff --git a/server/handles/search.go b/server/handles/search.go
new file mode 100644
index 00000000..24742f08
--- /dev/null
+++ b/server/handles/search.go
@@ -0,0 +1,42 @@
+package handles
+
+import (
+ "github.com/alist-org/alist/v3/internal/model"
+ "github.com/alist-org/alist/v3/internal/search"
+ "github.com/alist-org/alist/v3/pkg/utils"
+ "github.com/alist-org/alist/v3/server/common"
+ "github.com/gin-gonic/gin"
+)
+
+type SearchResp struct {
+ model.SearchNode
+ Type int `json:"type"`
+}
+
+func Search(c *gin.Context) {
+ var req model.SearchReq
+ if err := c.ShouldBind(&req); err != nil {
+ common.ErrorResp(c, err, 400)
+ return
+ }
+ if err := req.Validate(); err != nil {
+ common.ErrorResp(c, err, 400)
+ return
+ }
+ nodes, total, err := search.Search(c, req)
+ if err != nil {
+ common.ErrorResp(c, err, 500)
+ return
+ }
+ common.SuccessResp(c, common.PageResp{
+ Content: utils.MustSliceConvert(nodes, nodeToSearchResp),
+ Total: total,
+ })
+}
+
+func nodeToSearchResp(node model.SearchNode) SearchResp {
+ return SearchResp{
+ SearchNode: node,
+ Type: utils.GetObjType(node.Name, node.IsDir),
+ }
+}
diff --git a/server/handles/storage.go b/server/handles/storage.go
index 76a387de..d523d148 100644
--- a/server/handles/storage.go
+++ b/server/handles/storage.go
@@ -12,7 +12,7 @@ import (
)
func ListStorages(c *gin.Context) {
- var req common.PageReq
+ var req model.PageReq
if err := c.ShouldBind(&req); err != nil {
common.ErrorResp(c, err, 400)
return
diff --git a/server/handles/user.go b/server/handles/user.go
index 18175750..cbee8f33 100644
--- a/server/handles/user.go
+++ b/server/handles/user.go
@@ -11,7 +11,7 @@ import (
)
func ListUsers(c *gin.Context) {
- var req common.PageReq
+ var req model.PageReq
if err := c.ShouldBind(&req); err != nil {
common.ErrorResp(c, err, 400)
return
diff --git a/server/middlewares/search.go b/server/middlewares/search.go
new file mode 100644
index 00000000..5d84aade
--- /dev/null
+++ b/server/middlewares/search.go
@@ -0,0 +1,19 @@
+package middlewares
+
+import (
+ "github.com/alist-org/alist/v3/internal/conf"
+ "github.com/alist-org/alist/v3/internal/errs"
+ "github.com/alist-org/alist/v3/internal/setting"
+ "github.com/alist-org/alist/v3/server/common"
+ "github.com/gin-gonic/gin"
+)
+
+func SearchIndex(c *gin.Context) {
+ mode := setting.GetStr(conf.SearchIndex)
+ if mode == "none" {
+ common.ErrorResp(c, errs.SearchNotAvailable, 500)
+ c.Abort()
+ } else {
+ c.Next()
+ }
+}
diff --git a/server/router.go b/server/router.go
index 7205a7be..4158387b 100644
--- a/server/router.go
+++ b/server/router.go
@@ -109,13 +109,13 @@ func admin(g *gin.RouterGroup) {
ms.POST("/send", message.HttpInstance.SendHandle)
index := g.Group("/index")
- index.POST("/build", handles.BuildIndex)
- index.GET("/progress", handles.GetProgress)
- index.GET("/search", handles.Search)
+ index.POST("/build", middlewares.SearchIndex, handles.BuildIndex)
+ index.GET("/progress", middlewares.SearchIndex, handles.GetProgress)
}
func _fs(g *gin.RouterGroup) {
g.Any("/list", handles.FsList)
+ g.Any("/search", middlewares.SearchIndex, handles.Search)
g.Any("/get", handles.FsGet)
g.Any("/other", handles.FsOther)
g.Any("/dirs", handles.FsDirs)