github
/
alist
mirror of https://github.com/Xhofe/alist
1
0
Fork 0
Code Issues Releases Wiki Activity

feat: multiple search indexes (#2514) 

* refactor: abstract search interface

* wip: ~

* fix cycle import

* objs update hook

* wip: ~

* Delete search/none

* auto update index while cache changed

* db searcher

TODO: bleve init issue

cannot open index, metadata missing

* fix size type

why float64??

* fix typo

* fix nil pointer using

* api adapt ui

* bleve: fix clear & change struct
pull/2520/head
Noah Hsu 2022-11-28 13:45:25 +08:00 committed by GitHub
parent bb969d8dc6
commit ddcba93eea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
43 changed files with 855 additions and 350 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
drivers/alist_v3/driver.go
View File

@ -46,7 +46,7 @@ func (d *AListV3) List(ctx context.Context, dir model.Obj, args model.ListArgs)
SetResult(&resp).
SetHeader("Authorization", d.AccessToken).
SetBody(ListReq{
PageReq: common.PageReq{
PageReq: model.PageReq{
Page: 1,
PerPage: 0,
},

4
drivers/alist_v3/types.go
View File

@ -3,11 +3,11 @@ package alist_v3
import (
"time"
"github.com/alist-org/alist/v3/server/common"
"github.com/alist-org/alist/v3/internal/model"
)
type ListReq struct {
common.PageReq
model.PageReq
Path string `json:"path" form:"path"`
Password string `json:"password" form:"password"`
Refresh bool `json:"refresh"`

1
go.mod
View File

@ -58,6 +58,7 @@ require (
github.com/blevesearch/zapx/v15 v15.3.6 // indirect
github.com/bluele/gcache v0.0.2 // indirect
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
github.com/deckarep/golang-set/v2 v2.1.0 // indirect
github.com/gaoyb7/115drive-webdav v0.1.8 // indirect
github.com/geoffgarside/ber v1.1.0 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect

2
go.sum
View File

@ -67,6 +67,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/deckarep/golang-set/v2 v2.1.0 h1:g47V4Or+DUdzbs8FxCCmgb6VYd+ptPAngjM6dtGktsI=
github.com/deckarep/golang-set/v2 v2.1.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
github.com/gaoyb7/115drive-webdav v0.1.8 h1:EJt4PSmcbvBY4KUh2zSo5p6fN9LZFNkIzuKejipubVw=

15
internal/bootstrap/data/setting.go
View File

@ -25,12 +25,13 @@ func initSettings() {
settings[i].Flag = model.DEPRECATED
}
}
if settings != nil && len(settings) > 0 {
err = db.SaveSettingItems(settings)
if err != nil {
log.Fatalf("failed save settings: %+v", err)
}
}
// what's going on here???
//if settings != nil && len(settings) > 0 {
// err = db.SaveSettingItems(settings)
// if err != nil {
// log.Fatalf("failed save settings: %+v", err)
// }
//}
// insert new items
for i := range initialSettingItems {
v := initialSettingItems[i]
@ -122,11 +123,13 @@ func InitialSettings() []model.SettingItem {
Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE},
{Key: conf.OcrApi, Value: "https://api.nn.ci/ocr/file/json", Type: conf.TypeString, Group: model.GLOBAL},
{Key: conf.FilenameCharMapping, Value: `{"/": "|"}`, Type: conf.TypeText, Group: model.GLOBAL},
{Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,bleve,none", Group: model.GLOBAL},
// aria2 settings
{Key: conf.Aria2Uri, Value: "http://localhost:6800/jsonrpc", Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE},
{Key: conf.Aria2Secret, Value: "", Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE},
// single settings
{Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE},
{Key: conf.IndexProgress, Value: "{}", Type: conf.TypeText, Group: model.SINGLE, Flag: model.PRIVATE},
}
if flags.Dev {
initialSettingItems = append(initialSettingItems, []model.SettingItem{

7
internal/bootstrap/index.go
View File

@ -1,10 +1,5 @@
package bootstrap
import (
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/index"
)
func InitIndex() {
index.Init(&conf.Conf.IndexDir)
// TODO init ? Probably not.
}

6
internal/conf/config.go
View File

@ -45,13 +45,13 @@ type Config struct {
Database Database `json:"database"`
Scheme Scheme `json:"scheme"`
TempDir string `json:"temp_dir" env:"TEMP_DIR"`
IndexDir string `json:"index_dir" env:"INDEX_DIR"`
BleveDir string `json:"bleve_dir" env:"BLEVE_DIR"`
Log LogConfig `json:"log"`
}
func DefaultConfig() *Config {
tempDir := filepath.Join(flags.DataDir, "temp")
indexDir := filepath.Join(flags.DataDir, "index")
indexDir := filepath.Join(flags.DataDir, "bleve")
logPath := filepath.Join(flags.DataDir, "log/log.log")
dbPath := filepath.Join(flags.DataDir, "data.db")
return &Config{
@ -66,7 +66,7 @@ func DefaultConfig() *Config {
TablePrefix: "x_",
DBFile: dbPath,
},
IndexDir: indexDir,
BleveDir: indexDir,
Log: LogConfig{
Enable: true,
Name: logPath,

4
internal/conf/const.go
View File

@ -40,13 +40,15 @@ const (
PrivacyRegs = "privacy_regs"
OcrApi = "ocr_api"
FilenameCharMapping = "filename_char_mapping"
SearchIndex = "search_index"
// aria2
Aria2Uri = "aria2_uri"
Aria2Secret = "aria2_secret"
// single
Token = "token"
Token = "token"
IndexProgress = "index_progress"
)
const (

17
internal/db/db.go
View File

@ -12,13 +12,18 @@ var db *gorm.DB
func Init(d *gorm.DB) {
db = d
var err error
if conf.Conf.Database.Type == "mysql" {
err = db.Set("gorm:table_options", "ENGINE=InnoDB CHARSET=utf8mb4").AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem))
} else {
err = db.AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem))
}
err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode))
if err != nil {
log.Fatalf("failed migrate database: %s", err.Error())
}
}
func AutoMigrate(dst ...interface{}) error {
var err error
if conf.Conf.Database.Type == "mysql" {
err = db.Set("gorm:table_options", "ENGINE=InnoDB CHARSET=utf8mb4").AutoMigrate(dst...)
} else {
err = db.AutoMigrate(dst...)
}
return err
}

49
internal/db/searchnode.go Normal file
View File

@ -0,0 +1,49 @@
package db
import (
"fmt"
"github.com/alist-org/alist/v3/internal/model"
"github.com/pkg/errors"
)
func CreateSearchNode(node *model.SearchNode) error {
return db.Create(node).Error
}
func DeleteSearchNodesByParent(parent string) error {
return db.Where(fmt.Sprintf("%s LIKE ?",
columnName("path")), fmt.Sprintf("%s%%", parent)).
Delete(&model.SearchNode{}).Error
}
func ClearSearchNodes() error {
return db.Where("1 = 1").Delete(&model.SearchNode{}).Error
}
func GetSearchNodesByParent(parent string) ([]model.SearchNode, error) {
var nodes []model.SearchNode
if err := db.Where(fmt.Sprintf("%s = ?",
columnName("parent")), parent).Find(&nodes).Error; err != nil {
return nil, err
}
return nodes, nil
}
func SearchNode(req model.SearchReq) ([]model.SearchNode, int64, error) {
searchDB := db.Model(&model.SearchNode{}).Where(
fmt.Sprintf("%s LIKE ? AND %s LIKE ?",
columnName("parent"),
columnName("name")),
fmt.Sprintf("%s%%", req.Parent),
fmt.Sprintf("%%%s%%", req.Keywords))
var count int64
if err := searchDB.Count(&count).Error; err != nil {
return nil, 0, errors.Wrapf(err, "failed get users count")
}
var files []model.SearchNode
if err := searchDB.Offset((req.Page - 1) * req.PerPage).Limit(req.PerPage).Find(&files).Error; err != nil {
return nil, 0, err
}
return files, count, nil
}

101
internal/db/settinghooks.go
View File

@ -11,81 +11,64 @@ import (
log "github.com/sirupsen/logrus"
)
type SettingItemHook struct {
Hook func(item *model.SettingItem) error
}
type SettingItemHook func(item *model.SettingItem) error
var SettingItemHooks = map[string]SettingItemHook{
conf.VideoTypes: {
Hook: func(item *model.SettingItem) error {
conf.TypesMap[conf.VideoTypes] = strings.Split(item.Value, ",")
return nil
},
var settingItemHooks = map[string]SettingItemHook{
conf.VideoTypes: func(item *model.SettingItem) error {
conf.TypesMap[conf.VideoTypes] = strings.Split(item.Value, ",")
return nil
},
conf.AudioTypes: {
Hook: func(item *model.SettingItem) error {
conf.TypesMap[conf.AudioTypes] = strings.Split(item.Value, ",")
return nil
},
conf.AudioTypes: func(item *model.SettingItem) error {
conf.TypesMap[conf.AudioTypes] = strings.Split(item.Value, ",")
return nil
},
conf.ImageTypes: {
Hook: func(item *model.SettingItem) error {
conf.TypesMap[conf.ImageTypes] = strings.Split(item.Value, ",")
return nil
},
conf.ImageTypes: func(item *model.SettingItem) error {
conf.TypesMap[conf.ImageTypes] = strings.Split(item.Value, ",")
return nil
},
conf.TextTypes: {
Hook: func(item *model.SettingItem) error {
conf.TypesMap[conf.TextTypes] = strings.Split(item.Value, ",")
return nil
},
conf.TextTypes: func(item *model.SettingItem) error {
conf.TypesMap[conf.TextTypes] = strings.Split(item.Value, ",")
return nil
},
//conf.OfficeTypes: {
// Hook: func(item *model.SettingItem) error {
// conf.TypesMap[conf.OfficeTypes] = strings.Split(item.Value, ",")
// return nil
// },
//},
conf.ProxyTypes: {
func(item *model.SettingItem) error {
conf.TypesMap[conf.ProxyTypes] = strings.Split(item.Value, ",")
return nil
},
conf.ProxyTypes: func(item *model.SettingItem) error {
conf.TypesMap[conf.ProxyTypes] = strings.Split(item.Value, ",")
return nil
},
conf.PrivacyRegs: {
Hook: func(item *model.SettingItem) error {
regStrs := strings.Split(item.Value, "\n")
regs := make([]*regexp.Regexp, 0, len(regStrs))
for _, regStr := range regStrs {
reg, err := regexp.Compile(regStr)
if err != nil {
return errors.WithStack(err)
}
regs = append(regs, reg)
}
conf.PrivacyReg = regs
return nil
},
},
conf.FilenameCharMapping: {
Hook: func(item *model.SettingItem) error {
err := utils.Json.UnmarshalFromString(item.Value, &conf.FilenameCharMap)
conf.PrivacyRegs: func(item *model.SettingItem) error {
regStrs := strings.Split(item.Value, "\n")
regs := make([]*regexp.Regexp, 0, len(regStrs))
for _, regStr := range regStrs {
reg, err := regexp.Compile(regStr)
if err != nil {
return err
return errors.WithStack(err)
}
log.Debugf("filename char mapping: %+v", conf.FilenameCharMap)
return nil
},
regs = append(regs, reg)
}
conf.PrivacyReg = regs
return nil
},
conf.FilenameCharMapping: func(item *model.SettingItem) error {
err := utils.Json.UnmarshalFromString(item.Value, &conf.FilenameCharMap)
if err != nil {
return err
}
log.Debugf("filename char mapping: %+v", conf.FilenameCharMap)
return nil
},
}
func HandleSettingItem(item *model.SettingItem) (bool, error) {
if hook, ok := SettingItemHooks[item.Key]; ok {
return true, hook.Hook(item)
if hook, ok := settingItemHooks[item.Key]; ok {
return true, hook(item)
}
return false, nil
}
func RegisterSettingItemHook(key string, hook SettingItemHook) {
settingItemHooks[key] = hook
}
// func HandleSettingItems(items []model.SettingItem) error {
// for i := range items {
// if err := HandleSettingItem(&items[i]); err != nil {

14
internal/db/settingitem.go
View File

@ -108,11 +108,17 @@ func SaveSettingItems(items []model.SettingItem) error {
others = append(others, items[i])
}
}
err := db.Save(others).Error
if err == nil {
settingsUpdate()
if len(others) > 0 {
err := db.Save(others).Error
if err != nil {
if len(others) < len(items) {
settingsUpdate()
}
return err
}
}
return err
settingsUpdate()
return nil
}
func SaveSettingItem(item model.SettingItem) error {

7
internal/errs/search.go Normal file
View File

@ -0,0 +1,7 @@
package errs
import "fmt"
var (
SearchNotAvailable = fmt.Errorf("search not available")
)

45
internal/fs/walk.go Normal file
View File

@ -0,0 +1,45 @@
package fs
import (
"context"
"path"
"path/filepath"
"github.com/alist-org/alist/v3/internal/db"
"github.com/alist-org/alist/v3/internal/model"
)
// WalkFS traverses filesystem fs starting at name up to depth levels.
//
// WalkFS will stop when current depth > `depth`. For each visited node,
// WalkFS calls walkFn. If a visited file system node is a directory and
// walkFn returns path.SkipDir, walkFS will skip traversal of this node.
func WalkFS(ctx context.Context, depth int, name string, info model.Obj, walkFn func(reqPath string, info model.Obj, err error) error) error {
// This implementation is based on Walk's code in the standard path/path package.
walkFnErr := walkFn(name, info, nil)
if walkFnErr != nil {
if info.IsDir() && walkFnErr == filepath.SkipDir {
return nil
}
return walkFnErr
}
if !info.IsDir() || depth == 0 {
return nil
}
meta, _ := db.GetNearestMeta(name)
// Read directory names.
objs, err := List(context.WithValue(ctx, "meta", meta), name)
if err != nil {
return walkFnErr
}
for _, fileInfo := range objs {
filename := path.Join(name, fileInfo.GetName())
if err := WalkFS(ctx, depth-1, filename, fileInfo, walkFn); err != nil {
if err == filepath.SkipDir {
break
}
return err
}
}
return nil
}

102
internal/index/build.go
View File

@ -1,102 +0,0 @@
package index
import (
"context"
"path"
"path/filepath"
"time"
"github.com/alist-org/alist/v3/internal/db"
"github.com/alist-org/alist/v3/internal/fs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/blevesearch/bleve/v2"
"github.com/google/uuid"
)
// walkFS traverses filesystem fs starting at name up to depth levels.
//
// walkFS will stop when current depth > `depth`. For each visited node,
// walkFS calls walkFn. If a visited file system node is a directory and
// walkFn returns path.SkipDir, walkFS will skip traversal of this node.
func walkFS(ctx context.Context, depth int, name string, info model.Obj, walkFn func(reqPath string, info model.Obj, err error) error) error {
// This implementation is based on Walk's code in the standard path/path package.
walkFnErr := walkFn(name, info, nil)
if walkFnErr != nil {
if info.IsDir() && walkFnErr == filepath.SkipDir {
return nil
}
return walkFnErr
}
if !info.IsDir() || depth == 0 {
return nil
}
meta, _ := db.GetNearestMeta(name)
// Read directory names.
objs, err := fs.List(context.WithValue(ctx, "meta", meta), name)
if err != nil {
return walkFnErr
}
for _, fileInfo := range objs {
filename := path.Join(name, fileInfo.GetName())
if err := walkFS(ctx, depth-1, filename, fileInfo, walkFn); err != nil {
if err == filepath.SkipDir {
break
}
return err
}
}
return nil
}
type Data struct {
Path string
}
func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth int) {
// TODO: partial remove indices
Reset()
var batchs []*bleve.Batch
var fileCount uint64 = 0
for _, indexPath := range indexPaths {
batch := func() *bleve.Batch {
batch := index.NewBatch()
// TODO: cache unchanged part
walkFn := func(indexPath string, info model.Obj, err error) error {
for _, avoidPath := range ignorePaths {
if indexPath == avoidPath {
return filepath.SkipDir
}
}
if !info.IsDir() {
batch.Index(uuid.NewString(), Data{Path: indexPath})
fileCount += 1
if fileCount%100 == 0 {
WriteProgress(&Progress{
FileCount: fileCount,
IsDone: false,
LastDoneTime: nil,
})
}
}
return nil
}
fi, err := fs.Get(ctx, indexPath)
if err != nil {
return batch
}
// TODO: run walkFS concurrently
walkFS(ctx, maxDepth, indexPath, fi, walkFn)
return batch
}()
batchs = append(batchs, batch)
}
for _, batch := range batchs {
index.Batch(batch)
}
now := time.Now()
WriteProgress(&Progress{
FileCount: fileCount,
IsDone: true,
LastDoneTime: &now,
})
}

47
internal/index/index.go
View File

@ -1,47 +0,0 @@
package index
import (
"os"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/blevesearch/bleve/v2"
log "github.com/sirupsen/logrus"
)
var index bleve.Index
func Init(indexPath *string) {
fileIndex, err := bleve.Open(*indexPath)
if err == bleve.ErrorIndexPathDoesNotExist {
log.Infof("Creating new index...")
indexMapping := bleve.NewIndexMapping()
fileIndex, err = bleve.New(*indexPath, indexMapping)
if err != nil {
log.Fatal(err)
}
}
index = fileIndex
progress := ReadProgress()
if !progress.IsDone {
log.Warnf("Last index build does not succeed!")
WriteProgress(&Progress{
FileCount: progress.FileCount,
IsDone: false,
LastDoneTime: nil,
})
}
}
func Reset() {
log.Infof("Removing old index...")
err := os.RemoveAll(conf.Conf.IndexDir)
if err != nil {
log.Fatal(err)
}
Init(&conf.Conf.IndexDir)
WriteProgress(&Progress{
FileCount: 0,
IsDone: false,
LastDoneTime: nil,
})
}

19
internal/index/search.go
View File

@ -1,19 +0,0 @@
package index
import (
"github.com/blevesearch/bleve/v2"
log "github.com/sirupsen/logrus"
)
func Search(queryString string, size int) (*bleve.SearchResult, error) {
query := bleve.NewMatchQuery(queryString)
search := bleve.NewSearchRequest(query)
search.Size = size
search.Fields = []string{"Path"}
searchResults, err := index.Search(search)
if err != nil {
log.Errorf("search error: %+v", err)
return nil, err
}
return searchResults, nil
}

46
internal/index/util.go
View File

@ -1,46 +0,0 @@
package index
import (
"errors"
"os"
"path/filepath"
"time"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/pkg/utils"
log "github.com/sirupsen/logrus"
)
type Progress struct {
FileCount uint64 `json:"file_count"`
IsDone bool `json:"is_done"`
LastDoneTime *time.Time `json:"last_done_time"`
}
func ReadProgress() Progress {
progressFilePath := filepath.Join(conf.Conf.IndexDir, "progress.json")
_, err := os.Stat(progressFilePath)
progress := Progress{0, false, nil}
if errors.Is(err, os.ErrNotExist) {
if !utils.WriteJsonToFile(progressFilePath, progress) {
log.Fatalf("failed to create index progress file")
}
}
progressBytes, err := os.ReadFile(progressFilePath)
if err != nil {
log.Fatalf("reading index progress file error: %+v", err)
}
err = utils.Json.Unmarshal(progressBytes, &progress)
if err != nil {
log.Fatalf("load index progress error: %+v", err)
}
return progress
}
func WriteProgress(progress *Progress) {
progressFilePath := filepath.Join(conf.Conf.IndexDir, "progress.json")
log.Infof("write index progress: %v", progress)
if !utils.WriteJsonToFile(progressFilePath, progress) {
log.Fatalf("failed to write to index progress file")
}
}

2
server/common/req.go → internal/model/req.go
View File

@ -1,4 +1,4 @@
package common
package model
type PageReq struct {
Page int `json:"page" form:"page"`

36
internal/model/search.go Normal file
View File

@ -0,0 +1,36 @@
package model
import (
"fmt"
"time"
)
type IndexProgress struct {
ObjCount uint64 `json:"obj_count"`
IsDone bool `json:"is_done"`
LastDoneTime *time.Time `json:"last_done_time"`
Error string `json:"error"`
}
type SearchReq struct {
Parent string `json:"parent"`
Keywords string `json:"keywords"`
PageReq
}
type SearchNode struct {
Parent string `json:"parent" gorm:"index"`
Name string `json:"name" gorm:"index"`
IsDir bool `json:"is_dir"`
Size int64 `json:"size"`
}
func (p *SearchReq) Validate() error {
if p.Page < 1 {
return fmt.Errorf("page can't < 1")
}
if p.PerPage < 1 {
return fmt.Errorf("per_page can't < 1")
}
return nil
}

6
internal/op/fs.go
View File

@ -59,6 +59,12 @@ func List(ctx context.Context, storage driver.Driver, path string, args model.Li
if err != nil {
return nil, errors.Wrapf(err, "failed to list objs")
}
// call hooks
go func() {
for _, hook := range objsUpdateHooks {
hook(args.ReqPath, files)
}
}()
if !storage.Config().NoCache {
if len(files) > 0 {
log.Debugf("set cache: %s => %+v", key, files)

13
internal/op/hook.go Normal file
View File

@ -0,0 +1,13 @@
package op
import "github.com/alist-org/alist/v3/internal/model"
type ObjsUpdateHook = func(parent string, objs []model.Obj)
var (
objsUpdateHooks = make([]ObjsUpdateHook, 0)
)
func RegisterObjsUpdateHook(hook ObjsUpdateHook) {
objsUpdateHooks = append(objsUpdateHooks, hook)
}

38
internal/search/bleve/init.go Normal file
View File

@ -0,0 +1,38 @@
package bleve
import (
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/search/searcher"
"github.com/blevesearch/bleve/v2"
log "github.com/sirupsen/logrus"
)
var config = searcher.Config{
Name: "bleve",
}
func Init(indexPath *string) (bleve.Index, error) {
log.Debugf("bleve path: %s", *indexPath)
fileIndex, err := bleve.Open(*indexPath)
if err == bleve.ErrorIndexPathDoesNotExist {
log.Infof("Creating new index...")
indexMapping := bleve.NewIndexMapping()
fileIndex, err = bleve.New(*indexPath, indexMapping)
if err != nil {
return nil, err
}
} else if err != nil {
return nil, err
}
return fileIndex, nil
}
func init() {
searcher.RegisterSearcher(config, func() (searcher.Searcher, error) {
b, err := Init(&conf.Conf.BleveDir)
if err != nil {
return nil, err
}
return &Bleve{BIndex: b}, nil
})
}

90
internal/search/bleve/search.go Normal file
View File

@ -0,0 +1,90 @@
package bleve
import (
"context"
"os"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/search/searcher"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/blevesearch/bleve/v2"
search2 "github.com/blevesearch/bleve/v2/search"
"github.com/google/uuid"
log "github.com/sirupsen/logrus"
)
type Bleve struct {
BIndex bleve.Index
}
func (b *Bleve) Config() searcher.Config {
return config
}
func (b *Bleve) Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) {
query := bleve.NewMatchQuery(req.Keywords)
query.SetField("name")
search := bleve.NewSearchRequest(query)
search.Size = req.PerPage
search.Fields = []string{"*"}
searchResults, err := b.BIndex.Search(search)
if err != nil {
log.Errorf("search error: %+v", err)
return nil, 0, err
}
res, err := utils.SliceConvert(searchResults.Hits, func(src *search2.DocumentMatch) (model.SearchNode, error) {
return model.SearchNode{
Parent: src.Fields["parent"].(string),
Name: src.Fields["name"].(string),
IsDir: src.Fields["is_dir"].(bool),
Size: int64(src.Fields["size"].(float64)),
}, nil
})
return res, int64(len(res)), nil
}
func (b *Bleve) Index(ctx context.Context, parent string, obj model.Obj) error {
return b.BIndex.Index(uuid.NewString(), model.SearchNode{
Parent: parent,
Name: obj.GetName(),
IsDir: obj.IsDir(),
Size: obj.GetSize(),
})
}
func (b *Bleve) Get(ctx context.Context, parent string) ([]model.SearchNode, error) {
return nil, errs.NotSupport
}
func (b *Bleve) Del(ctx context.Context, prefix string) error {
return errs.NotSupport
}
func (b *Bleve) Release(ctx context.Context) error {
if b.BIndex != nil {
return b.BIndex.Close()
}
return nil
}
func (b *Bleve) Clear(ctx context.Context) error {
err := b.Release(ctx)
if err != nil {
return err
}
log.Infof("Removing old index...")
err = os.RemoveAll(conf.Conf.BleveDir)
if err != nil {
log.Errorf("clear bleve error: %+v", err)
}
bIndex, err := Init(&conf.Conf.BleveDir)
if err != nil {
return err
}
b.BIndex = bIndex
return nil
}
var _ searcher.Searcher = (*Bleve)(nil)

100
internal/search/build.go Normal file
View File

@ -0,0 +1,100 @@
package search
import (
"context"
"path"
"path/filepath"
"time"
"github.com/alist-org/alist/v3/internal/db"
"github.com/alist-org/alist/v3/internal/fs"
"github.com/alist-org/alist/v3/internal/model"
log "github.com/sirupsen/logrus"
)
var (
Running = false
)
func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth int, count bool) error {
var objCount uint64 = 0
Running = true
var (
err error
fi model.Obj
)
defer func() {
Running = false
now := time.Now()
eMsg := ""
if err != nil {
log.Errorf("build index error: %+v", err)
eMsg = err.Error()
} else {
log.Infof("success build index, count: %d", objCount)
}
if count {
WriteProgress(&model.IndexProgress{
ObjCount: objCount,
IsDone: err == nil,
LastDoneTime: &now,
Error: eMsg,
})
}
}()
admin, err := db.GetAdmin()
if err != nil {
return err
}
if count {
WriteProgress(&model.IndexProgress{
ObjCount: 0,
IsDone: false,
})
}
for _, indexPath := range indexPaths {
walkFn := func(indexPath string, info model.Obj, err error) error {
for _, avoidPath := range ignorePaths {
if indexPath == avoidPath {
return filepath.SkipDir
}
}
// ignore root
if indexPath == "/" {
return nil
}
err = instance.Index(ctx, path.Dir(indexPath), info)
if err != nil {
return err
} else {
objCount++
}
if objCount%100 == 0 {
log.Infof("index obj count: %d", objCount)
log.Debugf("current success index: %s", indexPath)
if count {
WriteProgress(&model.IndexProgress{
ObjCount: objCount,
IsDone: false,
LastDoneTime: nil,
})
}
}
return nil
}
fi, err = fs.Get(ctx, indexPath)
if err != nil {
return err
}
// TODO: run walkFS concurrently
err = fs.WalkFS(context.WithValue(ctx, "user", admin), maxDepth, indexPath, fi, walkFn)
if err != nil {
return err
}
}
return nil
}
func Clear(ctx context.Context) error {
return instance.Clear(ctx)
}

16
internal/search/db/init.go Normal file
View File

@ -0,0 +1,16 @@
package db
import (
"github.com/alist-org/alist/v3/internal/search/searcher"
)
var config = searcher.Config{
Name: "database",
AutoUpdate: true,
}
func init() {
searcher.RegisterSearcher(config, func() (searcher.Searcher, error) {
return &DB{}, nil
})
}

46
internal/search/db/search.go Normal file
View File

@ -0,0 +1,46 @@
package db
import (
"context"
"github.com/alist-org/alist/v3/internal/db"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/search/searcher"
)
type DB struct{}
func (D DB) Config() searcher.Config {
return config
}
func (D DB) Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) {
return db.SearchNode(req)
}
func (D DB) Index(ctx context.Context, parent string, obj model.Obj) error {
return db.CreateSearchNode(&model.SearchNode{
Parent: parent,
Name: obj.GetName(),
IsDir: obj.IsDir(),
Size: obj.GetSize(),
})
}
func (D DB) Get(ctx context.Context, parent string) ([]model.SearchNode, error) {
return db.GetSearchNodesByParent(parent)
}
func (D DB) Del(ctx context.Context, prefix string) error {
return db.DeleteSearchNodesByParent(prefix)
}
func (D DB) Release(ctx context.Context) error {
return nil
}
func (D DB) Clear(ctx context.Context) error {
return db.ClearSearchNodes()
}
var _ searcher.Searcher = (*DB)(nil)

6
internal/search/import.go Normal file
View File

@ -0,0 +1,6 @@
package search
import (
_ "github.com/alist-org/alist/v3/internal/search/bleve"
_ "github.com/alist-org/alist/v3/internal/search/db"
)

36
internal/search/progress.go Normal file
View File

@ -0,0 +1,36 @@
package search
import (
"context"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/db"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/setting"
"github.com/alist-org/alist/v3/pkg/utils"
log "github.com/sirupsen/logrus"
)
func Progress(ctx context.Context) (*model.IndexProgress, error) {
p := setting.GetStr(conf.IndexProgress)
var progress model.IndexProgress
err := utils.Json.UnmarshalFromString(p, &progress)
return &progress, err
}
func WriteProgress(progress *model.IndexProgress) {
p, err := utils.Json.MarshalToString(progress)
if err != nil {
log.Errorf("marshal progress error: %+v", err)
}
err = db.SaveSettingItem(model.SettingItem{
Key: conf.IndexProgress,
Value: p,
Type: conf.TypeText,
Group: model.SINGLE,
Flag: model.PRIVATE,
})
if err != nil {
log.Errorf("save progress error: %+v", err)
}
}

54
internal/search/search.go Normal file
View File

@ -0,0 +1,54 @@
package search
import (
"context"
"fmt"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/db"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/search/searcher"
log "github.com/sirupsen/logrus"
)
var instance searcher.Searcher = nil
// Init or reset index
func Init(mode string) error {
if instance != nil {
err := instance.Release(context.Background())
if err != nil {
log.Errorf("release instance err: %+v", err)
}
instance = nil
}
if Running {
return fmt.Errorf("index is running")
}
if mode == "none" {
log.Warnf("not enable search")
return nil
}
s, ok := searcher.NewMap[mode]
if !ok {
return fmt.Errorf("not support index: %s", mode)
}
i, err := s()
if err != nil {
log.Errorf("init searcher error: %+v", err)
} else {
instance = i
}
return err
}
func Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) {
return instance.Search(ctx, req)
}
func init() {
db.RegisterSettingItemHook(conf.SearchIndex, func(item *model.SettingItem) error {
log.Debugf("searcher init, mode: %s", item.Value)
return Init(item.Value)
})
}

9
internal/search/searcher/manage.go Normal file
View File

@ -0,0 +1,9 @@
package searcher
type New func() (Searcher, error)
var NewMap = map[string]New{}
func RegisterSearcher(config Config, searcher New) {
NewMap[config.Name] = searcher
}

29
internal/search/searcher/searcher.go Normal file
View File

@ -0,0 +1,29 @@
package searcher
import (
"context"
"github.com/alist-org/alist/v3/internal/model"
)
type Config struct {
Name string
AutoUpdate bool
}
type Searcher interface {
// Config of the searcher
Config() Config
// Search specific keywords in specific path
Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error)
// Index obj with parent
Index(ctx context.Context, parent string, obj model.Obj) error
// Get by parent
Get(ctx context.Context, parent string) ([]model.SearchNode, error)
// Del with prefix
Del(ctx context.Context, prefix string) error
// Release resource
Release(ctx context.Context) error
// Clear all index
Clear(ctx context.Context) error
}

73
internal/search/update.go Normal file
View File

@ -0,0 +1,73 @@
package search
import (
"context"
"path"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
mapset "github.com/deckarep/golang-set/v2"
log "github.com/sirupsen/logrus"
)
func Update(parent string, objs []model.Obj) {
if instance != nil && !instance.Config().AutoUpdate {
return
}
ctx := context.Background()
// only update when index have built
progress, err := Progress(ctx)
if err != nil {
log.Errorf("update search index error while get progress: %+v", err)
return
}
if !progress.IsDone {
return
}
nodes, err := instance.Get(ctx, parent)
if err != nil {
log.Errorf("update search index error while get nodes: %+v", err)
return
}
now := mapset.NewSet[string]()
for i := range objs {
now.Add(objs[i].GetName())
}
old := mapset.NewSet[string]()
for i := range nodes {
old.Add(nodes[i].Name)
}
// delete data that no longer exists
toDelete := old.Difference(now)
toAdd := now.Difference(old)
for i := range nodes {
if toDelete.Contains(nodes[i].Name) {
err = instance.Del(ctx, path.Join(parent, nodes[i].Name))
if err != nil {
log.Errorf("update search index error while del old node: %+v", err)
return
}
}
}
for i := range objs {
if toAdd.Contains(objs[i].GetName()) {
err = instance.Index(ctx, parent, objs[i])
if err != nil {
log.Errorf("update search index error while index new node: %+v", err)
return
}
// build index if it's a folder
if objs[i].IsDir() {
err = BuildIndex(ctx, []string{path.Join(parent, objs[i].GetName())}, nil, -1, false)
if err != nil {
log.Errorf("update search index error while build index: %+v", err)
return
}
}
}
}
}
func init() {
op.RegisterObjsUpdateHook(Update)
}

7
pkg/utils/file.go
View File

@ -138,6 +138,13 @@ func GetFileType(filename string) int {
return conf.UNKNOWN
}
func GetObjType(filename string, isDir bool) int {
if isDir {
return conf.FOLDER
}
return GetFileType(filename)
}
func GetMimeType(name string) string {
ext := path.Ext(name)
m := mime.TypeByExtension(ext)

9
pkg/utils/slice.go
View File

@ -35,3 +35,12 @@ func SliceConvert[S any, D any](srcS []S, convert func(src S) (D, error)) ([]D,
}
return res, nil
}
func MustSliceConvert[S any, D any](srcS []S, convert func(src S) D) []D {
var res []D
for i := range srcS {
dst := convert(srcS[i])
res = append(res, dst)
}
return res
}

16
server/handles/fsread.go
View File

@ -19,7 +19,7 @@ import (
)
type ListReq struct {
common.PageReq
model.PageReq
Path string `json:"path" form:"path"`
Password string `json:"password" form:"password"`
Refresh bool `json:"refresh"`
@ -86,7 +86,7 @@ func FsList(c *gin.Context) {
provider = storage.GetStorage().Driver
}
common.SuccessResp(c, FsListResp{
Content: toObjResp(objs, req.Path, isEncrypt(meta, req.Path)),
Content: toObjsResp(objs, req.Path, isEncrypt(meta, req.Path)),
Total: int64(total),
Readme: getReadme(meta, req.Path),
Write: user.CanWrite() || common.CanWrite(meta, req.Path),
@ -165,7 +165,7 @@ func isEncrypt(meta *model.Meta, path string) bool {
return true
}
func pagination(objs []model.Obj, req *common.PageReq) (int, []model.Obj) {
func pagination(objs []model.Obj, req *model.PageReq) (int, []model.Obj) {
pageIndex, pageSize := req.Page, req.PerPage
total := len(objs)
start := (pageIndex - 1) * pageSize
@ -179,17 +179,13 @@ func pagination(objs []model.Obj, req *common.PageReq) (int, []model.Obj) {
return total, objs[start:end]
}
func toObjResp(objs []model.Obj, parent string, encrypt bool) []ObjResp {
func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp {
var resp []ObjResp
for _, obj := range objs {
thumb := ""
if t, ok := obj.(model.Thumb); ok {
thumb = t.Thumb()
}
tp := conf.FOLDER
if !obj.IsDir() {
tp = utils.GetFileType(obj.GetName())
}
resp = append(resp, ObjResp{
Name: utils.MappingName(obj.GetName(), conf.FilenameCharMap),
Size: obj.GetSize(),
@ -197,7 +193,7 @@ func toObjResp(objs []model.Obj, parent string, encrypt bool) []ObjResp {
Modified: obj.ModTime(),
Sign: common.Sign(obj, parent, encrypt),
Thumb: thumb,
Type: tp,
Type: utils.GetObjType(obj.GetName(), obj.IsDir()),
})
}
return resp
@ -299,7 +295,7 @@ func FsGet(c *gin.Context) {
RawURL: rawURL,
Readme: getReadme(meta, req.Path),
Provider: provider,
Related: toObjResp(related, parentPath, isEncrypt(parentMeta, parentPath)),
Related: toObjsResp(related, parentPath, isEncrypt(parentMeta, parentPath)),
})
}

58
server/handles/index.go
View File

@ -2,53 +2,49 @@ package handles
import (
"context"
"strconv"
"github.com/alist-org/alist/v3/internal/db"
"github.com/alist-org/alist/v3/internal/index"
"github.com/alist-org/alist/v3/internal/search"
"github.com/alist-org/alist/v3/server/common"
"github.com/gin-gonic/gin"
log "github.com/sirupsen/logrus"
)
type BuildIndexReq struct {
Paths []string `json:"paths"`
MaxDepth int `json:"max_depth"`
IgnorePaths []string `json:"ignore_paths"`
}
func BuildIndex(c *gin.Context) {
var req BuildIndexReq
if err := c.ShouldBind(&req); err != nil {
common.ErrorResp(c, err, 400)
return
}
if search.Running {
common.ErrorStrResp(c, "index is running", 400)
return
}
go func() {
// TODO: consider run build index as non-admin
user, _ := db.GetAdmin()
ctx := context.WithValue(c.Request.Context(), "user", user)
maxDepth, err := strconv.Atoi(c.PostForm("max_depth"))
ctx := context.Background()
err := search.Clear(ctx)
if err != nil {
maxDepth = -1
log.Errorf("clear index error: %+v", err)
return
}
err = search.BuildIndex(context.Background(), req.Paths, req.IgnorePaths, req.MaxDepth, true)
if err != nil {
log.Errorf("build index error: %+v", err)
}
indexPaths := []string{"/"}
ignorePaths := c.PostFormArray("ignore_paths")
index.BuildIndex(ctx, indexPaths, ignorePaths, maxDepth)
}()
common.SuccessResp(c)
}
func GetProgress(c *gin.Context) {
progress := index.ReadProgress()
common.SuccessResp(c, progress)
}
func Search(c *gin.Context) {
results := []string{}
query, exists := c.GetQuery("query")
if !exists {
common.SuccessResp(c, results)
}
sizeStr, _ := c.GetQuery("size")
size, err := strconv.Atoi(sizeStr)
if err != nil {
size = 10
}
searchResults, err := index.Search(query, size)
progress, err := search.Progress(c)
if err != nil {
common.ErrorResp(c, err, 500)
return
}
for _, documentMatch := range searchResults.Hits {
results = append(results, documentMatch.Fields["Path"].(string))
}
common.SuccessResp(c, results)
common.SuccessResp(c, progress)
}

2
server/handles/meta.go
View File

@ -15,7 +15,7 @@ import (
)
func ListMetas(c *gin.Context) {
var req common.PageReq
var req model.PageReq
if err := c.ShouldBind(&req); err != nil {
common.ErrorResp(c, err, 400)
return

42
server/handles/search.go Normal file
View File

@ -0,0 +1,42 @@
package handles
import (
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/search"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/alist-org/alist/v3/server/common"
"github.com/gin-gonic/gin"
)
type SearchResp struct {
model.SearchNode
Type int `json:"type"`
}
func Search(c *gin.Context) {
var req model.SearchReq
if err := c.ShouldBind(&req); err != nil {
common.ErrorResp(c, err, 400)
return
}
if err := req.Validate(); err != nil {
common.ErrorResp(c, err, 400)
return
}
nodes, total, err := search.Search(c, req)
if err != nil {
common.ErrorResp(c, err, 500)
return
}
common.SuccessResp(c, common.PageResp{
Content: utils.MustSliceConvert(nodes, nodeToSearchResp),
Total: total,
})
}
func nodeToSearchResp(node model.SearchNode) SearchResp {
return SearchResp{
SearchNode: node,
Type: utils.GetObjType(node.Name, node.IsDir),
}
}

2
server/handles/storage.go
View File

@ -12,7 +12,7 @@ import (
)
func ListStorages(c *gin.Context) {
var req common.PageReq
var req model.PageReq
if err := c.ShouldBind(&req); err != nil {
common.ErrorResp(c, err, 400)
return

2
server/handles/user.go
View File

@ -11,7 +11,7 @@ import (
)
func ListUsers(c *gin.Context) {
var req common.PageReq
var req model.PageReq
if err := c.ShouldBind(&req); err != nil {
common.ErrorResp(c, err, 400)
return

19
server/middlewares/search.go Normal file
View File

@ -0,0 +1,19 @@
package middlewares
import (
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/setting"
"github.com/alist-org/alist/v3/server/common"
"github.com/gin-gonic/gin"
)
func SearchIndex(c *gin.Context) {
mode := setting.GetStr(conf.SearchIndex)
if mode == "none" {
common.ErrorResp(c, errs.SearchNotAvailable, 500)
c.Abort()
} else {
c.Next()
}
}

6
server/router.go
View File

@ -109,13 +109,13 @@ func admin(g *gin.RouterGroup) {
ms.POST("/send", message.HttpInstance.SendHandle)
index := g.Group("/index")
index.POST("/build", handles.BuildIndex)
index.GET("/progress", handles.GetProgress)
index.GET("/search", handles.Search)
index.POST("/build", middlewares.SearchIndex, handles.BuildIndex)
index.GET("/progress", middlewares.SearchIndex, handles.GetProgress)
}
func _fs(g *gin.RouterGroup) {
g.Any("/list", handles.FsList)
g.Any("/search", middlewares.SearchIndex, handles.Search)
g.Any("/get", handles.FsGet)
g.Any("/other", handles.FsOther)
g.Any("/dirs", handles.FsDirs)