diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index dcf8d76c..20269a21 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -131,7 +131,8 @@ func InitialSettings() []model.SettingItem { // single settings {Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}, - {Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,bleve,none", Group: model.SINGLE}, + {Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,bleve,none", Group: model.INDEX}, + {Key: conf.IgnorePaths, Value: "", Type: conf.TypeText, Group: model.INDEX, Flag: model.PRIVATE, Help: `one path per line`}, {Key: conf.IndexProgress, Value: "{}", Type: conf.TypeText, Group: model.SINGLE, Flag: model.PRIVATE}, } if flags.Dev { diff --git a/internal/conf/const.go b/internal/conf/const.go index 495ac2bd..5da67c98 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -41,7 +41,10 @@ const ( PrivacyRegs = "privacy_regs" OcrApi = "ocr_api" FilenameCharMapping = "filename_char_mapping" - SearchIndex = "search_index" + + // index + SearchIndex = "search_index" + IgnorePaths = "ignore_paths" // aria2 Aria2Uri = "aria2_uri" diff --git a/internal/model/setting.go b/internal/model/setting.go index 0452bf4a..10771da5 100644 --- a/internal/model/setting.go +++ b/internal/model/setting.go @@ -1,12 +1,13 @@ package model const ( - SITE = iota + SINGLE = iota + SITE STYLE PREVIEW GLOBAL - SINGLE ARIA2 + INDEX ) const ( diff --git a/internal/search/build.go b/internal/search/build.go index 868c6ae3..88028002 100644 --- a/internal/search/build.go +++ b/internal/search/build.go @@ -11,8 +11,10 @@ import ( "github.com/alist-org/alist/v3/internal/db" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/mq" "github.com/alist-org/alist/v3/pkg/utils" + mapset "github.com/deckarep/golang-set/v2" log "github.com/sirupsen/logrus" ) @@ -22,18 +24,8 @@ var ( ) func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth int, count bool) error { - storages, err := db.GetEnabledStorages() - if err != nil { - return err - } - var skipDrivers = []string{"AList V2", "AList V3"} - for _, storage := range storages { - if utils.SliceContains(skipDrivers, storage.Driver) { - // TODO: request for indexing permission - ignorePaths = append(ignorePaths, storage.MountPath) - } - } var ( + err error objCount uint64 = 0 fi model.Obj ) @@ -154,3 +146,73 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth func Clear(ctx context.Context) error { return instance.Clear(ctx) } + +func Update(parent string, objs []model.Obj) { + if instance == nil || !instance.Config().AutoUpdate || Running.Load() { + return + } + ignorePaths, err := GetIgnorePaths() + if err != nil { + log.Errorf("update search index error while get ignore paths: %+v", err) + return + } + if isIgnorePath(parent, ignorePaths) { + return + } + ctx := context.Background() + // only update when index have built + progress, err := Progress() + if err != nil { + log.Errorf("update search index error while get progress: %+v", err) + return + } + if !progress.IsDone { + return + } + nodes, err := instance.Get(ctx, parent) + if err != nil { + log.Errorf("update search index error while get nodes: %+v", err) + return + } + now := mapset.NewSet[string]() + for i := range objs { + now.Add(objs[i].GetName()) + } + old := mapset.NewSet[string]() + for i := range nodes { + old.Add(nodes[i].Name) + } + // delete data that no longer exists + toDelete := old.Difference(now) + toAdd := now.Difference(old) + for i := range nodes { + if toDelete.Contains(nodes[i].Name) { + err = instance.Del(ctx, path.Join(parent, nodes[i].Name)) + if err != nil { + log.Errorf("update search index error while del old node: %+v", err) + return + } + } + } + for i := range objs { + if toAdd.Contains(objs[i].GetName()) { + err = Index(ctx, parent, objs[i]) + if err != nil { + log.Errorf("update search index error while index new node: %+v", err) + return + } + // build index if it's a folder + if objs[i].IsDir() { + err = BuildIndex(ctx, []string{path.Join(parent, objs[i].GetName())}, ignorePaths, -1, false) + if err != nil { + log.Errorf("update search index error while build index: %+v", err) + return + } + } + } + } +} + +func init() { + op.RegisterObjsUpdateHook(Update) +} diff --git a/internal/search/update.go b/internal/search/update.go deleted file mode 100644 index 47f982ca..00000000 --- a/internal/search/update.go +++ /dev/null @@ -1,73 +0,0 @@ -package search - -import ( - "context" - "path" - - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/op" - mapset "github.com/deckarep/golang-set/v2" - log "github.com/sirupsen/logrus" -) - -func Update(parent string, objs []model.Obj) { - if instance == nil || !instance.Config().AutoUpdate || Running.Load() { - return - } - ctx := context.Background() - // only update when index have built - progress, err := Progress() - if err != nil { - log.Errorf("update search index error while get progress: %+v", err) - return - } - if !progress.IsDone { - return - } - nodes, err := instance.Get(ctx, parent) - if err != nil { - log.Errorf("update search index error while get nodes: %+v", err) - return - } - now := mapset.NewSet[string]() - for i := range objs { - now.Add(objs[i].GetName()) - } - old := mapset.NewSet[string]() - for i := range nodes { - old.Add(nodes[i].Name) - } - // delete data that no longer exists - toDelete := old.Difference(now) - toAdd := now.Difference(old) - for i := range nodes { - if toDelete.Contains(nodes[i].Name) { - err = instance.Del(ctx, path.Join(parent, nodes[i].Name)) - if err != nil { - log.Errorf("update search index error while del old node: %+v", err) - return - } - } - } - for i := range objs { - if toAdd.Contains(objs[i].GetName()) { - err = Index(ctx, parent, objs[i]) - if err != nil { - log.Errorf("update search index error while index new node: %+v", err) - return - } - // build index if it's a folder - if objs[i].IsDir() { - err = BuildIndex(ctx, []string{path.Join(parent, objs[i].GetName())}, nil, -1, false) - if err != nil { - log.Errorf("update search index error while build index: %+v", err) - return - } - } - } - } -} - -func init() { - op.RegisterObjsUpdateHook(Update) -} diff --git a/internal/search/progress.go b/internal/search/util.go similarity index 52% rename from internal/search/progress.go rename to internal/search/util.go index 65d1b8be..b17963d9 100644 --- a/internal/search/progress.go +++ b/internal/search/util.go @@ -1,6 +1,8 @@ package search import ( + "strings" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/db" "github.com/alist-org/alist/v3/internal/model" @@ -32,3 +34,32 @@ func WriteProgress(progress *model.IndexProgress) { log.Errorf("save progress error: %+v", err) } } + +func GetIgnorePaths() ([]string, error) { + storages, err := db.GetEnabledStorages() + if err != nil { + return nil, err + } + ignorePaths := make([]string, 0) + var skipDrivers = []string{"AList V2", "AList V3"} + for _, storage := range storages { + if utils.SliceContains(skipDrivers, storage.Driver) { + // TODO: request for indexing permission + ignorePaths = append(ignorePaths, storage.MountPath) + } + } + customIgnorePaths := setting.GetStr(conf.IgnorePaths) + if customIgnorePaths != "" { + ignorePaths = append(ignorePaths, strings.Split(customIgnorePaths, "\n")...) + } + return ignorePaths, nil +} + +func isIgnorePath(path string, ignorePaths []string) bool { + for _, ignorePath := range ignorePaths { + if strings.HasPrefix(path, ignorePath) { + return true + } + } + return false +} diff --git a/server/handles/index.go b/server/handles/index.go index 3739da7e..afd5e2c8 100644 --- a/server/handles/index.go +++ b/server/handles/index.go @@ -25,6 +25,12 @@ func BuildIndex(c *gin.Context) { common.ErrorStrResp(c, "index is running", 400) return } + ignorePaths, err := search.GetIgnorePaths() + if err != nil { + common.ErrorResp(c, err, 500) + return + } + ignorePaths = append(ignorePaths, req.IgnorePaths...) go func() { ctx := context.Background() err := search.Clear(ctx) @@ -32,7 +38,7 @@ func BuildIndex(c *gin.Context) { log.Errorf("clear index error: %+v", err) return } - err = search.BuildIndex(context.Background(), req.Paths, req.IgnorePaths, req.MaxDepth, true) + err = search.BuildIndex(context.Background(), req.Paths, ignorePaths, req.MaxDepth, true) if err != nil { log.Errorf("build index error: %+v", err) }