From ddcba93eea2cae4ae2c177064b965f45be2cb015 Mon Sep 17 00:00:00 2001 From: Noah Hsu <xhofe@qq.com> Date: Mon, 28 Nov 2022 13:45:25 +0800 Subject: [PATCH] feat: multiple search indexes (#2514) * refactor: abstract search interface * wip: ~ * fix cycle import * objs update hook * wip: ~ * Delete search/none * auto update index while cache changed * db searcher TODO: bleve init issue cannot open index, metadata missing * fix size type why float64?? * fix typo * fix nil pointer using * api adapt ui * bleve: fix clear & change struct --- drivers/alist_v3/driver.go | 2 +- drivers/alist_v3/types.go | 4 +- go.mod | 1 + go.sum | 2 + internal/bootstrap/data/setting.go | 15 ++-- internal/bootstrap/index.go | 7 +- internal/conf/config.go | 6 +- internal/conf/const.go | 4 +- internal/db/db.go | 17 ++-- internal/db/searchnode.go | 49 +++++++++++ internal/db/settinghooks.go | 101 ++++++++++------------ internal/db/settingitem.go | 14 +++- internal/errs/search.go | 7 ++ internal/fs/walk.go | 45 ++++++++++ internal/index/build.go | 102 ----------------------- internal/index/index.go | 47 ----------- internal/index/search.go | 19 ----- internal/index/util.go | 46 ---------- {server/common => internal/model}/req.go | 2 +- internal/model/search.go | 36 ++++++++ internal/op/fs.go | 6 ++ internal/op/hook.go | 13 +++ internal/search/bleve/init.go | 38 +++++++++ internal/search/bleve/search.go | 90 ++++++++++++++++++++ internal/search/build.go | 100 ++++++++++++++++++++++ internal/search/db/init.go | 16 ++++ internal/search/db/search.go | 46 ++++++++++ internal/search/import.go | 6 ++ internal/search/progress.go | 36 ++++++++ internal/search/search.go | 54 ++++++++++++ internal/search/searcher/manage.go | 9 ++ internal/search/searcher/searcher.go | 29 +++++++ internal/search/update.go | 73 ++++++++++++++++ pkg/utils/file.go | 7 ++ pkg/utils/slice.go | 9 ++ server/handles/fsread.go | 16 ++-- server/handles/index.go | 58 ++++++------- server/handles/meta.go | 2 +- server/handles/search.go | 42 ++++++++++ server/handles/storage.go | 2 +- server/handles/user.go | 2 +- server/middlewares/search.go | 19 +++++ server/router.go | 6 +- 43 files changed, 855 insertions(+), 350 deletions(-) create mode 100644 internal/db/searchnode.go create mode 100644 internal/errs/search.go create mode 100644 internal/fs/walk.go delete mode 100644 internal/index/build.go delete mode 100644 internal/index/index.go delete mode 100644 internal/index/search.go delete mode 100644 internal/index/util.go rename {server/common => internal/model}/req.go (95%) create mode 100644 internal/model/search.go create mode 100644 internal/op/hook.go create mode 100644 internal/search/bleve/init.go create mode 100644 internal/search/bleve/search.go create mode 100644 internal/search/build.go create mode 100644 internal/search/db/init.go create mode 100644 internal/search/db/search.go create mode 100644 internal/search/import.go create mode 100644 internal/search/progress.go create mode 100644 internal/search/search.go create mode 100644 internal/search/searcher/manage.go create mode 100644 internal/search/searcher/searcher.go create mode 100644 internal/search/update.go create mode 100644 server/handles/search.go create mode 100644 server/middlewares/search.go diff --git a/drivers/alist_v3/driver.go b/drivers/alist_v3/driver.go index 6b8bc55e..3f633c73 100644 --- a/drivers/alist_v3/driver.go +++ b/drivers/alist_v3/driver.go @@ -46,7 +46,7 @@ func (d *AListV3) List(ctx context.Context, dir model.Obj, args model.ListArgs) SetResult(&resp). SetHeader("Authorization", d.AccessToken). SetBody(ListReq{ - PageReq: common.PageReq{ + PageReq: model.PageReq{ Page: 1, PerPage: 0, }, diff --git a/drivers/alist_v3/types.go b/drivers/alist_v3/types.go index cf5006a6..f2ff32a0 100644 --- a/drivers/alist_v3/types.go +++ b/drivers/alist_v3/types.go @@ -3,11 +3,11 @@ package alist_v3 import ( "time" - "github.com/alist-org/alist/v3/server/common" + "github.com/alist-org/alist/v3/internal/model" ) type ListReq struct { - common.PageReq + model.PageReq Path string `json:"path" form:"path"` Password string `json:"password" form:"password"` Refresh bool `json:"refresh"` diff --git a/go.mod b/go.mod index 631953d5..9f7f145f 100644 --- a/go.mod +++ b/go.mod @@ -58,6 +58,7 @@ require ( github.com/blevesearch/zapx/v15 v15.3.6 // indirect github.com/bluele/gcache v0.0.2 // indirect github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect + github.com/deckarep/golang-set/v2 v2.1.0 // indirect github.com/gaoyb7/115drive-webdav v0.1.8 // indirect github.com/geoffgarside/ber v1.1.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect diff --git a/go.sum b/go.sum index 13bf079a..0e111af6 100644 --- a/go.sum +++ b/go.sum @@ -67,6 +67,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckarep/golang-set/v2 v2.1.0 h1:g47V4Or+DUdzbs8FxCCmgb6VYd+ptPAngjM6dtGktsI= +github.com/deckarep/golang-set/v2 v2.1.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/gaoyb7/115drive-webdav v0.1.8 h1:EJt4PSmcbvBY4KUh2zSo5p6fN9LZFNkIzuKejipubVw= diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 636af4a2..a4c09c0a 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -25,12 +25,13 @@ func initSettings() { settings[i].Flag = model.DEPRECATED } } - if settings != nil && len(settings) > 0 { - err = db.SaveSettingItems(settings) - if err != nil { - log.Fatalf("failed save settings: %+v", err) - } - } + // what's going on here??? + //if settings != nil && len(settings) > 0 { + // err = db.SaveSettingItems(settings) + // if err != nil { + // log.Fatalf("failed save settings: %+v", err) + // } + //} // insert new items for i := range initialSettingItems { v := initialSettingItems[i] @@ -122,11 +123,13 @@ func InitialSettings() []model.SettingItem { Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.OcrApi, Value: "https://api.nn.ci/ocr/file/json", Type: conf.TypeString, Group: model.GLOBAL}, {Key: conf.FilenameCharMapping, Value: `{"/": "|"}`, Type: conf.TypeText, Group: model.GLOBAL}, + {Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,bleve,none", Group: model.GLOBAL}, // aria2 settings {Key: conf.Aria2Uri, Value: "http://localhost:6800/jsonrpc", Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE}, {Key: conf.Aria2Secret, Value: "", Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE}, // single settings {Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}, + {Key: conf.IndexProgress, Value: "{}", Type: conf.TypeText, Group: model.SINGLE, Flag: model.PRIVATE}, } if flags.Dev { initialSettingItems = append(initialSettingItems, []model.SettingItem{ diff --git a/internal/bootstrap/index.go b/internal/bootstrap/index.go index bee273a6..02a73318 100644 --- a/internal/bootstrap/index.go +++ b/internal/bootstrap/index.go @@ -1,10 +1,5 @@ package bootstrap -import ( - "github.com/alist-org/alist/v3/internal/conf" - "github.com/alist-org/alist/v3/internal/index" -) - func InitIndex() { - index.Init(&conf.Conf.IndexDir) + // TODO init ? Probably not. } diff --git a/internal/conf/config.go b/internal/conf/config.go index 122209ee..735e870f 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -45,13 +45,13 @@ type Config struct { Database Database `json:"database"` Scheme Scheme `json:"scheme"` TempDir string `json:"temp_dir" env:"TEMP_DIR"` - IndexDir string `json:"index_dir" env:"INDEX_DIR"` + BleveDir string `json:"bleve_dir" env:"BLEVE_DIR"` Log LogConfig `json:"log"` } func DefaultConfig() *Config { tempDir := filepath.Join(flags.DataDir, "temp") - indexDir := filepath.Join(flags.DataDir, "index") + indexDir := filepath.Join(flags.DataDir, "bleve") logPath := filepath.Join(flags.DataDir, "log/log.log") dbPath := filepath.Join(flags.DataDir, "data.db") return &Config{ @@ -66,7 +66,7 @@ func DefaultConfig() *Config { TablePrefix: "x_", DBFile: dbPath, }, - IndexDir: indexDir, + BleveDir: indexDir, Log: LogConfig{ Enable: true, Name: logPath, diff --git a/internal/conf/const.go b/internal/conf/const.go index 400928c5..c4a2e546 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -40,13 +40,15 @@ const ( PrivacyRegs = "privacy_regs" OcrApi = "ocr_api" FilenameCharMapping = "filename_char_mapping" + SearchIndex = "search_index" // aria2 Aria2Uri = "aria2_uri" Aria2Secret = "aria2_secret" // single - Token = "token" + Token = "token" + IndexProgress = "index_progress" ) const ( diff --git a/internal/db/db.go b/internal/db/db.go index 1b7acc92..c7480a64 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -12,13 +12,18 @@ var db *gorm.DB func Init(d *gorm.DB) { db = d - var err error - if conf.Conf.Database.Type == "mysql" { - err = db.Set("gorm:table_options", "ENGINE=InnoDB CHARSET=utf8mb4").AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem)) - } else { - err = db.AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem)) - } + err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode)) if err != nil { log.Fatalf("failed migrate database: %s", err.Error()) } } + +func AutoMigrate(dst ...interface{}) error { + var err error + if conf.Conf.Database.Type == "mysql" { + err = db.Set("gorm:table_options", "ENGINE=InnoDB CHARSET=utf8mb4").AutoMigrate(dst...) + } else { + err = db.AutoMigrate(dst...) + } + return err +} diff --git a/internal/db/searchnode.go b/internal/db/searchnode.go new file mode 100644 index 00000000..a37fce9c --- /dev/null +++ b/internal/db/searchnode.go @@ -0,0 +1,49 @@ +package db + +import ( + "fmt" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" +) + +func CreateSearchNode(node *model.SearchNode) error { + return db.Create(node).Error +} + +func DeleteSearchNodesByParent(parent string) error { + return db.Where(fmt.Sprintf("%s LIKE ?", + columnName("path")), fmt.Sprintf("%s%%", parent)). + Delete(&model.SearchNode{}).Error +} + +func ClearSearchNodes() error { + return db.Where("1 = 1").Delete(&model.SearchNode{}).Error +} + +func GetSearchNodesByParent(parent string) ([]model.SearchNode, error) { + var nodes []model.SearchNode + if err := db.Where(fmt.Sprintf("%s = ?", + columnName("parent")), parent).Find(&nodes).Error; err != nil { + return nil, err + } + return nodes, nil +} + +func SearchNode(req model.SearchReq) ([]model.SearchNode, int64, error) { + searchDB := db.Model(&model.SearchNode{}).Where( + fmt.Sprintf("%s LIKE ? AND %s LIKE ?", + columnName("parent"), + columnName("name")), + fmt.Sprintf("%s%%", req.Parent), + fmt.Sprintf("%%%s%%", req.Keywords)) + var count int64 + if err := searchDB.Count(&count).Error; err != nil { + return nil, 0, errors.Wrapf(err, "failed get users count") + } + var files []model.SearchNode + if err := searchDB.Offset((req.Page - 1) * req.PerPage).Limit(req.PerPage).Find(&files).Error; err != nil { + return nil, 0, err + } + return files, count, nil +} diff --git a/internal/db/settinghooks.go b/internal/db/settinghooks.go index 6d968729..f7d85a07 100644 --- a/internal/db/settinghooks.go +++ b/internal/db/settinghooks.go @@ -11,81 +11,64 @@ import ( log "github.com/sirupsen/logrus" ) -type SettingItemHook struct { - Hook func(item *model.SettingItem) error -} +type SettingItemHook func(item *model.SettingItem) error -var SettingItemHooks = map[string]SettingItemHook{ - conf.VideoTypes: { - Hook: func(item *model.SettingItem) error { - conf.TypesMap[conf.VideoTypes] = strings.Split(item.Value, ",") - return nil - }, +var settingItemHooks = map[string]SettingItemHook{ + conf.VideoTypes: func(item *model.SettingItem) error { + conf.TypesMap[conf.VideoTypes] = strings.Split(item.Value, ",") + return nil }, - conf.AudioTypes: { - Hook: func(item *model.SettingItem) error { - conf.TypesMap[conf.AudioTypes] = strings.Split(item.Value, ",") - return nil - }, + conf.AudioTypes: func(item *model.SettingItem) error { + conf.TypesMap[conf.AudioTypes] = strings.Split(item.Value, ",") + return nil }, - conf.ImageTypes: { - Hook: func(item *model.SettingItem) error { - conf.TypesMap[conf.ImageTypes] = strings.Split(item.Value, ",") - return nil - }, + conf.ImageTypes: func(item *model.SettingItem) error { + conf.TypesMap[conf.ImageTypes] = strings.Split(item.Value, ",") + return nil }, - conf.TextTypes: { - Hook: func(item *model.SettingItem) error { - conf.TypesMap[conf.TextTypes] = strings.Split(item.Value, ",") - return nil - }, + conf.TextTypes: func(item *model.SettingItem) error { + conf.TypesMap[conf.TextTypes] = strings.Split(item.Value, ",") + return nil }, - //conf.OfficeTypes: { - // Hook: func(item *model.SettingItem) error { - // conf.TypesMap[conf.OfficeTypes] = strings.Split(item.Value, ",") - // return nil - // }, - //}, - conf.ProxyTypes: { - func(item *model.SettingItem) error { - conf.TypesMap[conf.ProxyTypes] = strings.Split(item.Value, ",") - return nil - }, + conf.ProxyTypes: func(item *model.SettingItem) error { + conf.TypesMap[conf.ProxyTypes] = strings.Split(item.Value, ",") + return nil }, - conf.PrivacyRegs: { - Hook: func(item *model.SettingItem) error { - regStrs := strings.Split(item.Value, "\n") - regs := make([]*regexp.Regexp, 0, len(regStrs)) - for _, regStr := range regStrs { - reg, err := regexp.Compile(regStr) - if err != nil { - return errors.WithStack(err) - } - regs = append(regs, reg) - } - conf.PrivacyReg = regs - return nil - }, - }, - conf.FilenameCharMapping: { - Hook: func(item *model.SettingItem) error { - err := utils.Json.UnmarshalFromString(item.Value, &conf.FilenameCharMap) + + conf.PrivacyRegs: func(item *model.SettingItem) error { + regStrs := strings.Split(item.Value, "\n") + regs := make([]*regexp.Regexp, 0, len(regStrs)) + for _, regStr := range regStrs { + reg, err := regexp.Compile(regStr) if err != nil { - return err + return errors.WithStack(err) } - log.Debugf("filename char mapping: %+v", conf.FilenameCharMap) - return nil - }, + regs = append(regs, reg) + } + conf.PrivacyReg = regs + return nil + }, + conf.FilenameCharMapping: func(item *model.SettingItem) error { + err := utils.Json.UnmarshalFromString(item.Value, &conf.FilenameCharMap) + if err != nil { + return err + } + log.Debugf("filename char mapping: %+v", conf.FilenameCharMap) + return nil }, } func HandleSettingItem(item *model.SettingItem) (bool, error) { - if hook, ok := SettingItemHooks[item.Key]; ok { - return true, hook.Hook(item) + if hook, ok := settingItemHooks[item.Key]; ok { + return true, hook(item) } return false, nil } +func RegisterSettingItemHook(key string, hook SettingItemHook) { + settingItemHooks[key] = hook +} + // func HandleSettingItems(items []model.SettingItem) error { // for i := range items { // if err := HandleSettingItem(&items[i]); err != nil { diff --git a/internal/db/settingitem.go b/internal/db/settingitem.go index 9b8d4d7c..da8b102b 100644 --- a/internal/db/settingitem.go +++ b/internal/db/settingitem.go @@ -108,11 +108,17 @@ func SaveSettingItems(items []model.SettingItem) error { others = append(others, items[i]) } } - err := db.Save(others).Error - if err == nil { - settingsUpdate() + if len(others) > 0 { + err := db.Save(others).Error + if err != nil { + if len(others) < len(items) { + settingsUpdate() + } + return err + } } - return err + settingsUpdate() + return nil } func SaveSettingItem(item model.SettingItem) error { diff --git a/internal/errs/search.go b/internal/errs/search.go new file mode 100644 index 00000000..9c864f4d --- /dev/null +++ b/internal/errs/search.go @@ -0,0 +1,7 @@ +package errs + +import "fmt" + +var ( + SearchNotAvailable = fmt.Errorf("search not available") +) diff --git a/internal/fs/walk.go b/internal/fs/walk.go new file mode 100644 index 00000000..1404c34a --- /dev/null +++ b/internal/fs/walk.go @@ -0,0 +1,45 @@ +package fs + +import ( + "context" + "path" + "path/filepath" + + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" +) + +// WalkFS traverses filesystem fs starting at name up to depth levels. +// +// WalkFS will stop when current depth > `depth`. For each visited node, +// WalkFS calls walkFn. If a visited file system node is a directory and +// walkFn returns path.SkipDir, walkFS will skip traversal of this node. +func WalkFS(ctx context.Context, depth int, name string, info model.Obj, walkFn func(reqPath string, info model.Obj, err error) error) error { + // This implementation is based on Walk's code in the standard path/path package. + walkFnErr := walkFn(name, info, nil) + if walkFnErr != nil { + if info.IsDir() && walkFnErr == filepath.SkipDir { + return nil + } + return walkFnErr + } + if !info.IsDir() || depth == 0 { + return nil + } + meta, _ := db.GetNearestMeta(name) + // Read directory names. + objs, err := List(context.WithValue(ctx, "meta", meta), name) + if err != nil { + return walkFnErr + } + for _, fileInfo := range objs { + filename := path.Join(name, fileInfo.GetName()) + if err := WalkFS(ctx, depth-1, filename, fileInfo, walkFn); err != nil { + if err == filepath.SkipDir { + break + } + return err + } + } + return nil +} diff --git a/internal/index/build.go b/internal/index/build.go deleted file mode 100644 index 1835774a..00000000 --- a/internal/index/build.go +++ /dev/null @@ -1,102 +0,0 @@ -package index - -import ( - "context" - "path" - "path/filepath" - "time" - - "github.com/alist-org/alist/v3/internal/db" - "github.com/alist-org/alist/v3/internal/fs" - "github.com/alist-org/alist/v3/internal/model" - "github.com/blevesearch/bleve/v2" - "github.com/google/uuid" -) - -// walkFS traverses filesystem fs starting at name up to depth levels. -// -// walkFS will stop when current depth > `depth`. For each visited node, -// walkFS calls walkFn. If a visited file system node is a directory and -// walkFn returns path.SkipDir, walkFS will skip traversal of this node. -func walkFS(ctx context.Context, depth int, name string, info model.Obj, walkFn func(reqPath string, info model.Obj, err error) error) error { - // This implementation is based on Walk's code in the standard path/path package. - walkFnErr := walkFn(name, info, nil) - if walkFnErr != nil { - if info.IsDir() && walkFnErr == filepath.SkipDir { - return nil - } - return walkFnErr - } - if !info.IsDir() || depth == 0 { - return nil - } - meta, _ := db.GetNearestMeta(name) - // Read directory names. - objs, err := fs.List(context.WithValue(ctx, "meta", meta), name) - if err != nil { - return walkFnErr - } - for _, fileInfo := range objs { - filename := path.Join(name, fileInfo.GetName()) - if err := walkFS(ctx, depth-1, filename, fileInfo, walkFn); err != nil { - if err == filepath.SkipDir { - break - } - return err - } - } - return nil -} - -type Data struct { - Path string -} - -func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth int) { - // TODO: partial remove indices - Reset() - var batchs []*bleve.Batch - var fileCount uint64 = 0 - for _, indexPath := range indexPaths { - batch := func() *bleve.Batch { - batch := index.NewBatch() - // TODO: cache unchanged part - walkFn := func(indexPath string, info model.Obj, err error) error { - for _, avoidPath := range ignorePaths { - if indexPath == avoidPath { - return filepath.SkipDir - } - } - if !info.IsDir() { - batch.Index(uuid.NewString(), Data{Path: indexPath}) - fileCount += 1 - if fileCount%100 == 0 { - WriteProgress(&Progress{ - FileCount: fileCount, - IsDone: false, - LastDoneTime: nil, - }) - } - } - return nil - } - fi, err := fs.Get(ctx, indexPath) - if err != nil { - return batch - } - // TODO: run walkFS concurrently - walkFS(ctx, maxDepth, indexPath, fi, walkFn) - return batch - }() - batchs = append(batchs, batch) - } - for _, batch := range batchs { - index.Batch(batch) - } - now := time.Now() - WriteProgress(&Progress{ - FileCount: fileCount, - IsDone: true, - LastDoneTime: &now, - }) -} diff --git a/internal/index/index.go b/internal/index/index.go deleted file mode 100644 index 393c54a9..00000000 --- a/internal/index/index.go +++ /dev/null @@ -1,47 +0,0 @@ -package index - -import ( - "os" - - "github.com/alist-org/alist/v3/internal/conf" - "github.com/blevesearch/bleve/v2" - log "github.com/sirupsen/logrus" -) - -var index bleve.Index - -func Init(indexPath *string) { - fileIndex, err := bleve.Open(*indexPath) - if err == bleve.ErrorIndexPathDoesNotExist { - log.Infof("Creating new index...") - indexMapping := bleve.NewIndexMapping() - fileIndex, err = bleve.New(*indexPath, indexMapping) - if err != nil { - log.Fatal(err) - } - } - index = fileIndex - progress := ReadProgress() - if !progress.IsDone { - log.Warnf("Last index build does not succeed!") - WriteProgress(&Progress{ - FileCount: progress.FileCount, - IsDone: false, - LastDoneTime: nil, - }) - } -} - -func Reset() { - log.Infof("Removing old index...") - err := os.RemoveAll(conf.Conf.IndexDir) - if err != nil { - log.Fatal(err) - } - Init(&conf.Conf.IndexDir) - WriteProgress(&Progress{ - FileCount: 0, - IsDone: false, - LastDoneTime: nil, - }) -} diff --git a/internal/index/search.go b/internal/index/search.go deleted file mode 100644 index c0577044..00000000 --- a/internal/index/search.go +++ /dev/null @@ -1,19 +0,0 @@ -package index - -import ( - "github.com/blevesearch/bleve/v2" - log "github.com/sirupsen/logrus" -) - -func Search(queryString string, size int) (*bleve.SearchResult, error) { - query := bleve.NewMatchQuery(queryString) - search := bleve.NewSearchRequest(query) - search.Size = size - search.Fields = []string{"Path"} - searchResults, err := index.Search(search) - if err != nil { - log.Errorf("search error: %+v", err) - return nil, err - } - return searchResults, nil -} diff --git a/internal/index/util.go b/internal/index/util.go deleted file mode 100644 index 2e890634..00000000 --- a/internal/index/util.go +++ /dev/null @@ -1,46 +0,0 @@ -package index - -import ( - "errors" - "os" - "path/filepath" - "time" - - "github.com/alist-org/alist/v3/internal/conf" - "github.com/alist-org/alist/v3/pkg/utils" - log "github.com/sirupsen/logrus" -) - -type Progress struct { - FileCount uint64 `json:"file_count"` - IsDone bool `json:"is_done"` - LastDoneTime *time.Time `json:"last_done_time"` -} - -func ReadProgress() Progress { - progressFilePath := filepath.Join(conf.Conf.IndexDir, "progress.json") - _, err := os.Stat(progressFilePath) - progress := Progress{0, false, nil} - if errors.Is(err, os.ErrNotExist) { - if !utils.WriteJsonToFile(progressFilePath, progress) { - log.Fatalf("failed to create index progress file") - } - } - progressBytes, err := os.ReadFile(progressFilePath) - if err != nil { - log.Fatalf("reading index progress file error: %+v", err) - } - err = utils.Json.Unmarshal(progressBytes, &progress) - if err != nil { - log.Fatalf("load index progress error: %+v", err) - } - return progress -} - -func WriteProgress(progress *Progress) { - progressFilePath := filepath.Join(conf.Conf.IndexDir, "progress.json") - log.Infof("write index progress: %v", progress) - if !utils.WriteJsonToFile(progressFilePath, progress) { - log.Fatalf("failed to write to index progress file") - } -} diff --git a/server/common/req.go b/internal/model/req.go similarity index 95% rename from server/common/req.go rename to internal/model/req.go index 62476a86..fe3a08bd 100644 --- a/server/common/req.go +++ b/internal/model/req.go @@ -1,4 +1,4 @@ -package common +package model type PageReq struct { Page int `json:"page" form:"page"` diff --git a/internal/model/search.go b/internal/model/search.go new file mode 100644 index 00000000..dd467251 --- /dev/null +++ b/internal/model/search.go @@ -0,0 +1,36 @@ +package model + +import ( + "fmt" + "time" +) + +type IndexProgress struct { + ObjCount uint64 `json:"obj_count"` + IsDone bool `json:"is_done"` + LastDoneTime *time.Time `json:"last_done_time"` + Error string `json:"error"` +} + +type SearchReq struct { + Parent string `json:"parent"` + Keywords string `json:"keywords"` + PageReq +} + +type SearchNode struct { + Parent string `json:"parent" gorm:"index"` + Name string `json:"name" gorm:"index"` + IsDir bool `json:"is_dir"` + Size int64 `json:"size"` +} + +func (p *SearchReq) Validate() error { + if p.Page < 1 { + return fmt.Errorf("page can't < 1") + } + if p.PerPage < 1 { + return fmt.Errorf("per_page can't < 1") + } + return nil +} diff --git a/internal/op/fs.go b/internal/op/fs.go index 4bf45038..0da4e1be 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -59,6 +59,12 @@ func List(ctx context.Context, storage driver.Driver, path string, args model.Li if err != nil { return nil, errors.Wrapf(err, "failed to list objs") } + // call hooks + go func() { + for _, hook := range objsUpdateHooks { + hook(args.ReqPath, files) + } + }() if !storage.Config().NoCache { if len(files) > 0 { log.Debugf("set cache: %s => %+v", key, files) diff --git a/internal/op/hook.go b/internal/op/hook.go new file mode 100644 index 00000000..7c624e9e --- /dev/null +++ b/internal/op/hook.go @@ -0,0 +1,13 @@ +package op + +import "github.com/alist-org/alist/v3/internal/model" + +type ObjsUpdateHook = func(parent string, objs []model.Obj) + +var ( + objsUpdateHooks = make([]ObjsUpdateHook, 0) +) + +func RegisterObjsUpdateHook(hook ObjsUpdateHook) { + objsUpdateHooks = append(objsUpdateHooks, hook) +} diff --git a/internal/search/bleve/init.go b/internal/search/bleve/init.go new file mode 100644 index 00000000..07203685 --- /dev/null +++ b/internal/search/bleve/init.go @@ -0,0 +1,38 @@ +package bleve + +import ( + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/search/searcher" + "github.com/blevesearch/bleve/v2" + log "github.com/sirupsen/logrus" +) + +var config = searcher.Config{ + Name: "bleve", +} + +func Init(indexPath *string) (bleve.Index, error) { + log.Debugf("bleve path: %s", *indexPath) + fileIndex, err := bleve.Open(*indexPath) + if err == bleve.ErrorIndexPathDoesNotExist { + log.Infof("Creating new index...") + indexMapping := bleve.NewIndexMapping() + fileIndex, err = bleve.New(*indexPath, indexMapping) + if err != nil { + return nil, err + } + } else if err != nil { + return nil, err + } + return fileIndex, nil +} + +func init() { + searcher.RegisterSearcher(config, func() (searcher.Searcher, error) { + b, err := Init(&conf.Conf.BleveDir) + if err != nil { + return nil, err + } + return &Bleve{BIndex: b}, nil + }) +} diff --git a/internal/search/bleve/search.go b/internal/search/bleve/search.go new file mode 100644 index 00000000..d629c7a1 --- /dev/null +++ b/internal/search/bleve/search.go @@ -0,0 +1,90 @@ +package bleve + +import ( + "context" + "os" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/search/searcher" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/blevesearch/bleve/v2" + search2 "github.com/blevesearch/bleve/v2/search" + "github.com/google/uuid" + log "github.com/sirupsen/logrus" +) + +type Bleve struct { + BIndex bleve.Index +} + +func (b *Bleve) Config() searcher.Config { + return config +} + +func (b *Bleve) Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) { + query := bleve.NewMatchQuery(req.Keywords) + query.SetField("name") + search := bleve.NewSearchRequest(query) + search.Size = req.PerPage + search.Fields = []string{"*"} + searchResults, err := b.BIndex.Search(search) + if err != nil { + log.Errorf("search error: %+v", err) + return nil, 0, err + } + res, err := utils.SliceConvert(searchResults.Hits, func(src *search2.DocumentMatch) (model.SearchNode, error) { + return model.SearchNode{ + Parent: src.Fields["parent"].(string), + Name: src.Fields["name"].(string), + IsDir: src.Fields["is_dir"].(bool), + Size: int64(src.Fields["size"].(float64)), + }, nil + }) + return res, int64(len(res)), nil +} + +func (b *Bleve) Index(ctx context.Context, parent string, obj model.Obj) error { + return b.BIndex.Index(uuid.NewString(), model.SearchNode{ + Parent: parent, + Name: obj.GetName(), + IsDir: obj.IsDir(), + Size: obj.GetSize(), + }) +} + +func (b *Bleve) Get(ctx context.Context, parent string) ([]model.SearchNode, error) { + return nil, errs.NotSupport +} + +func (b *Bleve) Del(ctx context.Context, prefix string) error { + return errs.NotSupport +} + +func (b *Bleve) Release(ctx context.Context) error { + if b.BIndex != nil { + return b.BIndex.Close() + } + return nil +} + +func (b *Bleve) Clear(ctx context.Context) error { + err := b.Release(ctx) + if err != nil { + return err + } + log.Infof("Removing old index...") + err = os.RemoveAll(conf.Conf.BleveDir) + if err != nil { + log.Errorf("clear bleve error: %+v", err) + } + bIndex, err := Init(&conf.Conf.BleveDir) + if err != nil { + return err + } + b.BIndex = bIndex + return nil +} + +var _ searcher.Searcher = (*Bleve)(nil) diff --git a/internal/search/build.go b/internal/search/build.go new file mode 100644 index 00000000..59bdfa42 --- /dev/null +++ b/internal/search/build.go @@ -0,0 +1,100 @@ +package search + +import ( + "context" + "path" + "path/filepath" + "time" + + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + log "github.com/sirupsen/logrus" +) + +var ( + Running = false +) + +func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth int, count bool) error { + var objCount uint64 = 0 + Running = true + var ( + err error + fi model.Obj + ) + defer func() { + Running = false + now := time.Now() + eMsg := "" + if err != nil { + log.Errorf("build index error: %+v", err) + eMsg = err.Error() + } else { + log.Infof("success build index, count: %d", objCount) + } + if count { + WriteProgress(&model.IndexProgress{ + ObjCount: objCount, + IsDone: err == nil, + LastDoneTime: &now, + Error: eMsg, + }) + } + }() + admin, err := db.GetAdmin() + if err != nil { + return err + } + if count { + WriteProgress(&model.IndexProgress{ + ObjCount: 0, + IsDone: false, + }) + } + for _, indexPath := range indexPaths { + walkFn := func(indexPath string, info model.Obj, err error) error { + for _, avoidPath := range ignorePaths { + if indexPath == avoidPath { + return filepath.SkipDir + } + } + // ignore root + if indexPath == "/" { + return nil + } + err = instance.Index(ctx, path.Dir(indexPath), info) + if err != nil { + return err + } else { + objCount++ + } + if objCount%100 == 0 { + log.Infof("index obj count: %d", objCount) + log.Debugf("current success index: %s", indexPath) + if count { + WriteProgress(&model.IndexProgress{ + ObjCount: objCount, + IsDone: false, + LastDoneTime: nil, + }) + } + } + return nil + } + fi, err = fs.Get(ctx, indexPath) + if err != nil { + return err + } + // TODO: run walkFS concurrently + err = fs.WalkFS(context.WithValue(ctx, "user", admin), maxDepth, indexPath, fi, walkFn) + if err != nil { + return err + } + } + return nil +} + +func Clear(ctx context.Context) error { + return instance.Clear(ctx) +} diff --git a/internal/search/db/init.go b/internal/search/db/init.go new file mode 100644 index 00000000..bd83ffe6 --- /dev/null +++ b/internal/search/db/init.go @@ -0,0 +1,16 @@ +package db + +import ( + "github.com/alist-org/alist/v3/internal/search/searcher" +) + +var config = searcher.Config{ + Name: "database", + AutoUpdate: true, +} + +func init() { + searcher.RegisterSearcher(config, func() (searcher.Searcher, error) { + return &DB{}, nil + }) +} diff --git a/internal/search/db/search.go b/internal/search/db/search.go new file mode 100644 index 00000000..92bb1645 --- /dev/null +++ b/internal/search/db/search.go @@ -0,0 +1,46 @@ +package db + +import ( + "context" + + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/search/searcher" +) + +type DB struct{} + +func (D DB) Config() searcher.Config { + return config +} + +func (D DB) Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) { + return db.SearchNode(req) +} + +func (D DB) Index(ctx context.Context, parent string, obj model.Obj) error { + return db.CreateSearchNode(&model.SearchNode{ + Parent: parent, + Name: obj.GetName(), + IsDir: obj.IsDir(), + Size: obj.GetSize(), + }) +} + +func (D DB) Get(ctx context.Context, parent string) ([]model.SearchNode, error) { + return db.GetSearchNodesByParent(parent) +} + +func (D DB) Del(ctx context.Context, prefix string) error { + return db.DeleteSearchNodesByParent(prefix) +} + +func (D DB) Release(ctx context.Context) error { + return nil +} + +func (D DB) Clear(ctx context.Context) error { + return db.ClearSearchNodes() +} + +var _ searcher.Searcher = (*DB)(nil) diff --git a/internal/search/import.go b/internal/search/import.go new file mode 100644 index 00000000..929b1583 --- /dev/null +++ b/internal/search/import.go @@ -0,0 +1,6 @@ +package search + +import ( + _ "github.com/alist-org/alist/v3/internal/search/bleve" + _ "github.com/alist-org/alist/v3/internal/search/db" +) diff --git a/internal/search/progress.go b/internal/search/progress.go new file mode 100644 index 00000000..85cc4f37 --- /dev/null +++ b/internal/search/progress.go @@ -0,0 +1,36 @@ +package search + +import ( + "context" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" + log "github.com/sirupsen/logrus" +) + +func Progress(ctx context.Context) (*model.IndexProgress, error) { + p := setting.GetStr(conf.IndexProgress) + var progress model.IndexProgress + err := utils.Json.UnmarshalFromString(p, &progress) + return &progress, err +} + +func WriteProgress(progress *model.IndexProgress) { + p, err := utils.Json.MarshalToString(progress) + if err != nil { + log.Errorf("marshal progress error: %+v", err) + } + err = db.SaveSettingItem(model.SettingItem{ + Key: conf.IndexProgress, + Value: p, + Type: conf.TypeText, + Group: model.SINGLE, + Flag: model.PRIVATE, + }) + if err != nil { + log.Errorf("save progress error: %+v", err) + } +} diff --git a/internal/search/search.go b/internal/search/search.go new file mode 100644 index 00000000..1c836ebe --- /dev/null +++ b/internal/search/search.go @@ -0,0 +1,54 @@ +package search + +import ( + "context" + "fmt" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/search/searcher" + log "github.com/sirupsen/logrus" +) + +var instance searcher.Searcher = nil + +// Init or reset index +func Init(mode string) error { + if instance != nil { + err := instance.Release(context.Background()) + if err != nil { + log.Errorf("release instance err: %+v", err) + } + instance = nil + } + if Running { + return fmt.Errorf("index is running") + } + if mode == "none" { + log.Warnf("not enable search") + return nil + } + s, ok := searcher.NewMap[mode] + if !ok { + return fmt.Errorf("not support index: %s", mode) + } + i, err := s() + if err != nil { + log.Errorf("init searcher error: %+v", err) + } else { + instance = i + } + return err +} + +func Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) { + return instance.Search(ctx, req) +} + +func init() { + db.RegisterSettingItemHook(conf.SearchIndex, func(item *model.SettingItem) error { + log.Debugf("searcher init, mode: %s", item.Value) + return Init(item.Value) + }) +} diff --git a/internal/search/searcher/manage.go b/internal/search/searcher/manage.go new file mode 100644 index 00000000..92bdd883 --- /dev/null +++ b/internal/search/searcher/manage.go @@ -0,0 +1,9 @@ +package searcher + +type New func() (Searcher, error) + +var NewMap = map[string]New{} + +func RegisterSearcher(config Config, searcher New) { + NewMap[config.Name] = searcher +} diff --git a/internal/search/searcher/searcher.go b/internal/search/searcher/searcher.go new file mode 100644 index 00000000..77378f32 --- /dev/null +++ b/internal/search/searcher/searcher.go @@ -0,0 +1,29 @@ +package searcher + +import ( + "context" + + "github.com/alist-org/alist/v3/internal/model" +) + +type Config struct { + Name string + AutoUpdate bool +} + +type Searcher interface { + // Config of the searcher + Config() Config + // Search specific keywords in specific path + Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) + // Index obj with parent + Index(ctx context.Context, parent string, obj model.Obj) error + // Get by parent + Get(ctx context.Context, parent string) ([]model.SearchNode, error) + // Del with prefix + Del(ctx context.Context, prefix string) error + // Release resource + Release(ctx context.Context) error + // Clear all index + Clear(ctx context.Context) error +} diff --git a/internal/search/update.go b/internal/search/update.go new file mode 100644 index 00000000..2da44283 --- /dev/null +++ b/internal/search/update.go @@ -0,0 +1,73 @@ +package search + +import ( + "context" + "path" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + mapset "github.com/deckarep/golang-set/v2" + log "github.com/sirupsen/logrus" +) + +func Update(parent string, objs []model.Obj) { + if instance != nil && !instance.Config().AutoUpdate { + return + } + ctx := context.Background() + // only update when index have built + progress, err := Progress(ctx) + if err != nil { + log.Errorf("update search index error while get progress: %+v", err) + return + } + if !progress.IsDone { + return + } + nodes, err := instance.Get(ctx, parent) + if err != nil { + log.Errorf("update search index error while get nodes: %+v", err) + return + } + now := mapset.NewSet[string]() + for i := range objs { + now.Add(objs[i].GetName()) + } + old := mapset.NewSet[string]() + for i := range nodes { + old.Add(nodes[i].Name) + } + // delete data that no longer exists + toDelete := old.Difference(now) + toAdd := now.Difference(old) + for i := range nodes { + if toDelete.Contains(nodes[i].Name) { + err = instance.Del(ctx, path.Join(parent, nodes[i].Name)) + if err != nil { + log.Errorf("update search index error while del old node: %+v", err) + return + } + } + } + for i := range objs { + if toAdd.Contains(objs[i].GetName()) { + err = instance.Index(ctx, parent, objs[i]) + if err != nil { + log.Errorf("update search index error while index new node: %+v", err) + return + } + // build index if it's a folder + if objs[i].IsDir() { + err = BuildIndex(ctx, []string{path.Join(parent, objs[i].GetName())}, nil, -1, false) + if err != nil { + log.Errorf("update search index error while build index: %+v", err) + return + } + } + } + } +} + +func init() { + op.RegisterObjsUpdateHook(Update) +} diff --git a/pkg/utils/file.go b/pkg/utils/file.go index cb580ba7..985be6da 100644 --- a/pkg/utils/file.go +++ b/pkg/utils/file.go @@ -138,6 +138,13 @@ func GetFileType(filename string) int { return conf.UNKNOWN } +func GetObjType(filename string, isDir bool) int { + if isDir { + return conf.FOLDER + } + return GetFileType(filename) +} + func GetMimeType(name string) string { ext := path.Ext(name) m := mime.TypeByExtension(ext) diff --git a/pkg/utils/slice.go b/pkg/utils/slice.go index 5ab5b018..c1d1584c 100644 --- a/pkg/utils/slice.go +++ b/pkg/utils/slice.go @@ -35,3 +35,12 @@ func SliceConvert[S any, D any](srcS []S, convert func(src S) (D, error)) ([]D, } return res, nil } + +func MustSliceConvert[S any, D any](srcS []S, convert func(src S) D) []D { + var res []D + for i := range srcS { + dst := convert(srcS[i]) + res = append(res, dst) + } + return res +} diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 106c87ae..b13b8402 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -19,7 +19,7 @@ import ( ) type ListReq struct { - common.PageReq + model.PageReq Path string `json:"path" form:"path"` Password string `json:"password" form:"password"` Refresh bool `json:"refresh"` @@ -86,7 +86,7 @@ func FsList(c *gin.Context) { provider = storage.GetStorage().Driver } common.SuccessResp(c, FsListResp{ - Content: toObjResp(objs, req.Path, isEncrypt(meta, req.Path)), + Content: toObjsResp(objs, req.Path, isEncrypt(meta, req.Path)), Total: int64(total), Readme: getReadme(meta, req.Path), Write: user.CanWrite() || common.CanWrite(meta, req.Path), @@ -165,7 +165,7 @@ func isEncrypt(meta *model.Meta, path string) bool { return true } -func pagination(objs []model.Obj, req *common.PageReq) (int, []model.Obj) { +func pagination(objs []model.Obj, req *model.PageReq) (int, []model.Obj) { pageIndex, pageSize := req.Page, req.PerPage total := len(objs) start := (pageIndex - 1) * pageSize @@ -179,17 +179,13 @@ func pagination(objs []model.Obj, req *common.PageReq) (int, []model.Obj) { return total, objs[start:end] } -func toObjResp(objs []model.Obj, parent string, encrypt bool) []ObjResp { +func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp { var resp []ObjResp for _, obj := range objs { thumb := "" if t, ok := obj.(model.Thumb); ok { thumb = t.Thumb() } - tp := conf.FOLDER - if !obj.IsDir() { - tp = utils.GetFileType(obj.GetName()) - } resp = append(resp, ObjResp{ Name: utils.MappingName(obj.GetName(), conf.FilenameCharMap), Size: obj.GetSize(), @@ -197,7 +193,7 @@ func toObjResp(objs []model.Obj, parent string, encrypt bool) []ObjResp { Modified: obj.ModTime(), Sign: common.Sign(obj, parent, encrypt), Thumb: thumb, - Type: tp, + Type: utils.GetObjType(obj.GetName(), obj.IsDir()), }) } return resp @@ -299,7 +295,7 @@ func FsGet(c *gin.Context) { RawURL: rawURL, Readme: getReadme(meta, req.Path), Provider: provider, - Related: toObjResp(related, parentPath, isEncrypt(parentMeta, parentPath)), + Related: toObjsResp(related, parentPath, isEncrypt(parentMeta, parentPath)), }) } diff --git a/server/handles/index.go b/server/handles/index.go index d5f033e6..1c924b31 100644 --- a/server/handles/index.go +++ b/server/handles/index.go @@ -2,53 +2,49 @@ package handles import ( "context" - "strconv" - "github.com/alist-org/alist/v3/internal/db" - "github.com/alist-org/alist/v3/internal/index" + "github.com/alist-org/alist/v3/internal/search" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" + log "github.com/sirupsen/logrus" ) +type BuildIndexReq struct { + Paths []string `json:"paths"` + MaxDepth int `json:"max_depth"` + IgnorePaths []string `json:"ignore_paths"` +} + func BuildIndex(c *gin.Context) { + var req BuildIndexReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if search.Running { + common.ErrorStrResp(c, "index is running", 400) + return + } go func() { - // TODO: consider run build index as non-admin - user, _ := db.GetAdmin() - ctx := context.WithValue(c.Request.Context(), "user", user) - maxDepth, err := strconv.Atoi(c.PostForm("max_depth")) + ctx := context.Background() + err := search.Clear(ctx) if err != nil { - maxDepth = -1 + log.Errorf("clear index error: %+v", err) + return + } + err = search.BuildIndex(context.Background(), req.Paths, req.IgnorePaths, req.MaxDepth, true) + if err != nil { + log.Errorf("build index error: %+v", err) } - indexPaths := []string{"/"} - ignorePaths := c.PostFormArray("ignore_paths") - index.BuildIndex(ctx, indexPaths, ignorePaths, maxDepth) }() common.SuccessResp(c) } func GetProgress(c *gin.Context) { - progress := index.ReadProgress() - common.SuccessResp(c, progress) -} - -func Search(c *gin.Context) { - results := []string{} - query, exists := c.GetQuery("query") - if !exists { - common.SuccessResp(c, results) - } - sizeStr, _ := c.GetQuery("size") - size, err := strconv.Atoi(sizeStr) - if err != nil { - size = 10 - } - searchResults, err := index.Search(query, size) + progress, err := search.Progress(c) if err != nil { common.ErrorResp(c, err, 500) return } - for _, documentMatch := range searchResults.Hits { - results = append(results, documentMatch.Fields["Path"].(string)) - } - common.SuccessResp(c, results) + common.SuccessResp(c, progress) } diff --git a/server/handles/meta.go b/server/handles/meta.go index 07498374..16019ed2 100644 --- a/server/handles/meta.go +++ b/server/handles/meta.go @@ -15,7 +15,7 @@ import ( ) func ListMetas(c *gin.Context) { - var req common.PageReq + var req model.PageReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return diff --git a/server/handles/search.go b/server/handles/search.go new file mode 100644 index 00000000..24742f08 --- /dev/null +++ b/server/handles/search.go @@ -0,0 +1,42 @@ +package handles + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/search" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" +) + +type SearchResp struct { + model.SearchNode + Type int `json:"type"` +} + +func Search(c *gin.Context) { + var req model.SearchReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := req.Validate(); err != nil { + common.ErrorResp(c, err, 400) + return + } + nodes, total, err := search.Search(c, req) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, common.PageResp{ + Content: utils.MustSliceConvert(nodes, nodeToSearchResp), + Total: total, + }) +} + +func nodeToSearchResp(node model.SearchNode) SearchResp { + return SearchResp{ + SearchNode: node, + Type: utils.GetObjType(node.Name, node.IsDir), + } +} diff --git a/server/handles/storage.go b/server/handles/storage.go index 76a387de..d523d148 100644 --- a/server/handles/storage.go +++ b/server/handles/storage.go @@ -12,7 +12,7 @@ import ( ) func ListStorages(c *gin.Context) { - var req common.PageReq + var req model.PageReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return diff --git a/server/handles/user.go b/server/handles/user.go index 18175750..cbee8f33 100644 --- a/server/handles/user.go +++ b/server/handles/user.go @@ -11,7 +11,7 @@ import ( ) func ListUsers(c *gin.Context) { - var req common.PageReq + var req model.PageReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return diff --git a/server/middlewares/search.go b/server/middlewares/search.go new file mode 100644 index 00000000..5d84aade --- /dev/null +++ b/server/middlewares/search.go @@ -0,0 +1,19 @@ +package middlewares + +import ( + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" +) + +func SearchIndex(c *gin.Context) { + mode := setting.GetStr(conf.SearchIndex) + if mode == "none" { + common.ErrorResp(c, errs.SearchNotAvailable, 500) + c.Abort() + } else { + c.Next() + } +} diff --git a/server/router.go b/server/router.go index 7205a7be..4158387b 100644 --- a/server/router.go +++ b/server/router.go @@ -109,13 +109,13 @@ func admin(g *gin.RouterGroup) { ms.POST("/send", message.HttpInstance.SendHandle) index := g.Group("/index") - index.POST("/build", handles.BuildIndex) - index.GET("/progress", handles.GetProgress) - index.GET("/search", handles.Search) + index.POST("/build", middlewares.SearchIndex, handles.BuildIndex) + index.GET("/progress", middlewares.SearchIndex, handles.GetProgress) } func _fs(g *gin.RouterGroup) { g.Any("/list", handles.FsList) + g.Any("/search", middlewares.SearchIndex, handles.Search) g.Any("/get", handles.FsGet) g.Any("/other", handles.FsOther) g.Any("/dirs", handles.FsDirs)