github
/
alist
mirror of https://github.com/Xhofe/alist
1
0
Fork 0
Code Issues Releases Wiki Activity

refactor:separate the setting method from the db package to the op package and add the cache

pull/2747/head
foxxorcat 2022-12-18 00:04:44 +08:00
parent fb64f00640
commit efec8eb6ee
10 changed files with 336 additions and 218 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
internal/bootstrap/data/setting.go
View File

@ -3,8 +3,8 @@ package data
import (
"github.com/alist-org/alist/v3/cmd/flags"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/db"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/pkg/utils/random"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
@ -16,40 +16,44 @@ var initialSettingItems []model.SettingItem
func initSettings() {
InitialSettings()
// check deprecated
settings, err := db.GetSettingItems()
settings, err := op.GetSettingItems()
if err != nil {
log.Fatalf("failed get settings: %+v", err)
}
for i := range settings {
if !isActive(settings[i].Key) {
if !isActive(settings[i].Key) && settings[i].Flag != model.DEPRECATED {
settings[i].Flag = model.DEPRECATED
err = op.SaveSettingItem(&settings[i])
if err != nil {
log.Fatalf("failed save setting: %+v", err)
}
}
}
// what's going on here???
//if settings != nil && len(settings) > 0 {
// err = db.SaveSettingItems(settings)
// if err != nil {
// log.Fatalf("failed save settings: %+v", err)
// }
//}
// insert new items
for i := range initialSettingItems {
v := initialSettingItems[i]
stored, err := db.GetSettingItemByKey(v.Key)
if errors.Is(err, gorm.ErrRecordNotFound) || v.Key == conf.VERSION {
err = db.SaveSettingItem(v)
if err != nil {
log.Fatalf("failed create setting: %+v", err)
}
} else if err != nil {
// create or save setting
for i, item := range initialSettingItems {
// err
stored, err := op.GetSettingItemByKey(item.Key)
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
log.Fatalf("failed get setting: %+v", err)
} else {
v.Value = stored.Value
err = db.SaveSettingItem(v)
if err != nil {
log.Fatalf("failed resave setting: %+v", err)
}
continue
}
// save
if stored != nil {
item.Value = stored.Value
}
if stored == nil || item != *stored {
err = op.SaveSettingItem(&initialSettingItems[i])
if err != nil {
log.Fatalf("failed save setting: %+v", err)
}
continue
}
// Not save so needs to execute hook
op.HandleSettingItemHook(&initialSettingItems[i])
}
}

79
internal/db/settinghooks.go
View File

@ -1,79 +0,0 @@
package db
import (
"regexp"
"strings"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
type SettingItemHook func(item *model.SettingItem) error
var settingItemHooks = map[string]SettingItemHook{
conf.VideoTypes: func(item *model.SettingItem) error {
conf.TypesMap[conf.VideoTypes] = strings.Split(item.Value, ",")
return nil
},
conf.AudioTypes: func(item *model.SettingItem) error {
conf.TypesMap[conf.AudioTypes] = strings.Split(item.Value, ",")
return nil
},
conf.ImageTypes: func(item *model.SettingItem) error {
conf.TypesMap[conf.ImageTypes] = strings.Split(item.Value, ",")
return nil
},
conf.TextTypes: func(item *model.SettingItem) error {
conf.TypesMap[conf.TextTypes] = strings.Split(item.Value, ",")
return nil
},
conf.ProxyTypes: func(item *model.SettingItem) error {
conf.TypesMap[conf.ProxyTypes] = strings.Split(item.Value, ",")
return nil
},
conf.PrivacyRegs: func(item *model.SettingItem) error {
regStrs := strings.Split(item.Value, "\n")
regs := make([]*regexp.Regexp, 0, len(regStrs))
for _, regStr := range regStrs {
reg, err := regexp.Compile(regStr)
if err != nil {
return errors.WithStack(err)
}
regs = append(regs, reg)
}
conf.PrivacyReg = regs
return nil
},
conf.FilenameCharMapping: func(item *model.SettingItem) error {
err := utils.Json.UnmarshalFromString(item.Value, &conf.FilenameCharMap)
if err != nil {
return err
}
log.Debugf("filename char mapping: %+v", conf.FilenameCharMap)
return nil
},
}
func HandleSettingItem(item *model.SettingItem) (bool, error) {
if hook, ok := settingItemHooks[item.Key]; ok {
return true, hook(item)
}
return false, nil
}
func RegisterSettingItemHook(key string, hook SettingItemHook) {
settingItemHooks[key] = hook
}
// func HandleSettingItems(items []model.SettingItem) error {
// for i := range items {
// if err := HandleSettingItem(&items[i]); err != nil {
// return err
// }
// }
// return nil
// }

106
internal/db/settingitem.go
View File

@ -4,45 +4,9 @@ import (
"fmt"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/pkg/generic_sync"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
var settingsMap generic_sync.MapOf[string, string]
var publicSettingsMap generic_sync.MapOf[string, string]
func settingsUpdate() {
settingsMap.Clear()
publicSettingsMap.Clear()
}
func GetPublicSettingsMap() map[string]string {
if publicSettingsMap.Empty() {
settingItems, err := GetPublicSettingItems()
if err != nil {
log.Errorf("failed to get settingItems: %+v", err)
}
for _, settingItem := range settingItems {
publicSettingsMap.Store(settingItem.Key, settingItem.Value)
}
}
return publicSettingsMap.ToMap()
}
func GetSettingsMap() *generic_sync.MapOf[string, string] {
if settingsMap.Empty() {
settingItems, err := GetSettingItems()
if err != nil {
log.Errorf("failed to get settingItems: %+v", err)
}
for _, settingItem := range settingItems {
settingsMap.Store(settingItem.Key, settingItem.Value)
}
}
return &settingsMap
}
func GetSettingItems() ([]model.SettingItem, error) {
var settingItems []model.SettingItem
if err := db.Find(&settingItems).Error; err != nil {
@ -59,18 +23,18 @@ func GetSettingItemByKey(key string) (*model.SettingItem, error) {
return &settingItem, nil
}
func GetSettingItemInKeys(keys []string) ([]model.SettingItem, error) {
var settingItem []model.SettingItem
if err := db.Where(fmt.Sprintf("%s in ?", columnName("key")), keys).Find(&settingItem).Error; err != nil {
return nil, errors.WithStack(err)
}
return settingItem, nil
}
// func GetSettingItemInKeys(keys []string) ([]model.SettingItem, error) {
// var settingItem []model.SettingItem
// if err := db.Where(fmt.Sprintf("%s in ?", columnName("key")), keys).Find(&settingItem).Error; err != nil {
// return nil, errors.WithStack(err)
// }
// return settingItem, nil
// }
func GetPublicSettingItems() ([]model.SettingItem, error) {
var settingItems []model.SettingItem
if err := db.Where(fmt.Sprintf("%s in ?", columnName("flag")), []int{model.PUBLIC, model.READONLY}).Find(&settingItems).Error; err != nil {
return nil, err
return nil, errors.WithStack(err)
}
return settingItems, nil
}
@ -91,58 +55,14 @@ func GetSettingItemsInGroups(groups []int) ([]model.SettingItem, error) {
return settingItems, nil
}
func SaveSettingItems(items []model.SettingItem) error {
others := make([]model.SettingItem, 0)
for i := range items {
if ok, err := HandleSettingItem(&items[i]); ok {
if err != nil {
return err
} else {
err = db.Save(items[i]).Error
if err != nil {
return errors.WithStack(err)
}
}
} else {
others = append(others, items[i])
}
}
if len(others) > 0 {
err := db.Save(others).Error
if err != nil {
if len(others) < len(items) {
settingsUpdate()
}
return err
}
}
settingsUpdate()
return nil
func SaveSettingItems(items []model.SettingItem) (err error) {
return errors.WithStack(db.Save(items).Error)
}
func SaveSettingItem(item model.SettingItem) error {
_, err := HandleSettingItem(&item)
if err != nil {
return err
}
err = db.Save(item).Error
if err == nil {
settingsUpdate()
}
return errors.WithStack(err)
func SaveSettingItem(item *model.SettingItem) error {
return errors.WithStack(db.Save(item).Error)
}
func DeleteSettingItemByKey(key string) error {
settingItem := model.SettingItem{
Key: key,
}
old, err := GetSettingItemByKey(key)
if err != nil {
return errors.WithMessage(err, "failed to get settingItem")
}
if !old.IsDeprecated() {
return errors.Errorf("setting [%s] is not deprecated", key)
}
settingsUpdate()
return errors.WithStack(db.Delete(&settingItem).Error)
return errors.WithStack(db.Delete(&model.SettingItem{Key: key}).Error)
}

87
internal/op/hook.go
View File

@ -1,7 +1,17 @@
package op
import "github.com/alist-org/alist/v3/internal/model"
import (
"regexp"
"strings"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
// Obj
type ObjsUpdateHook = func(parent string, objs []model.Obj)
var (
@ -11,3 +21,78 @@ var (
func RegisterObjsUpdateHook(hook ObjsUpdateHook) {
objsUpdateHooks = append(objsUpdateHooks, hook)
}
func HandleObjsUpdateHook(parent string, objs []model.Obj) {
for _, hook := range objsUpdateHooks {
hook(parent, objs)
}
}
// Setting
type SettingItemHook func(item *model.SettingItem) error
var settingItemHooks = map[string]SettingItemHook{
conf.VideoTypes: func(item *model.SettingItem) error {
conf.TypesMap[conf.VideoTypes] = strings.Split(item.Value, ",")
return nil
},
conf.AudioTypes: func(item *model.SettingItem) error {
conf.TypesMap[conf.AudioTypes] = strings.Split(item.Value, ",")
return nil
},
conf.ImageTypes: func(item *model.SettingItem) error {
conf.TypesMap[conf.ImageTypes] = strings.Split(item.Value, ",")
return nil
},
conf.TextTypes: func(item *model.SettingItem) error {
conf.TypesMap[conf.TextTypes] = strings.Split(item.Value, ",")
return nil
},
conf.ProxyTypes: func(item *model.SettingItem) error {
conf.TypesMap[conf.ProxyTypes] = strings.Split(item.Value, ",")
return nil
},
conf.PrivacyRegs: func(item *model.SettingItem) error {
regStrs := strings.Split(item.Value, "\n")
regs := make([]*regexp.Regexp, 0, len(regStrs))
for _, regStr := range regStrs {
reg, err := regexp.Compile(regStr)
if err != nil {
return errors.WithStack(err)
}
regs = append(regs, reg)
}
conf.PrivacyReg = regs
return nil
},
conf.FilenameCharMapping: func(item *model.SettingItem) error {
err := utils.Json.UnmarshalFromString(item.Value, &conf.FilenameCharMap)
if err != nil {
return err
}
log.Debugf("filename char mapping: %+v", conf.FilenameCharMap)
return nil
},
}
func RegisterSettingItemHook(key string, hook SettingItemHook) {
settingItemHooks[key] = hook
}
func HandleSettingItemHook(item *model.SettingItem) (hashook bool, err error) {
if hook, ok := settingItemHooks[item.Key]; ok {
return true, hook(item)
}
return false, nil
}
func HandleSettingItemsHook(items []model.SettingItem) (err error) {
for i := 0; i < len(items); i++ {
_, err = HandleSettingItemHook(&items[i])
if err != nil {
return err
}
}
return nil
}

189
internal/op/setting.go Normal file
View File

@ -0,0 +1,189 @@
package op
import (
"sort"
"strconv"
"time"
"github.com/Xhofe/go-cache"
"github.com/alist-org/alist/v3/internal/db"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/pkg/singleflight"
"github.com/pkg/errors"
)
var settingCache = cache.NewMemCache(cache.WithShards[*model.SettingItem](4))
var settingG singleflight.Group[*model.SettingItem]
var settingCacheF = func(item *model.SettingItem) {
settingCache.Set(item.Key, item, cache.WithEx[*model.SettingItem](time.Hour))
}
var settingGroupCache = cache.NewMemCache(cache.WithShards[[]model.SettingItem](4))
var settingGroupG singleflight.Group[[]model.SettingItem]
var settingGroupCacheF = func(key string, item []model.SettingItem) {
settingGroupCache.Set(key, item, cache.WithEx[[]model.SettingItem](time.Hour))
}
func settingCacheUpdate() {
settingCache.Clear()
settingGroupCache.Clear()
}
func GetPublicSettingsMap() map[string]string {
items, _ := GetPublicSettingItems()
pSettings := make(map[string]string)
for _, item := range items {
pSettings[item.Key] = item.Value
}
return pSettings
}
func GetSettingsMap() map[string]string {
items, _ := GetSettingItems()
settings := make(map[string]string)
for _, item := range items {
settings[item.Key] = item.Value
}
return settings
}
func GetSettingItems() ([]model.SettingItem, error) {
if items, ok := settingGroupCache.Get("ALL_SETTING_ITEMS"); ok {
return items, nil
}
items, err, _ := settingGroupG.Do("ALL_SETTING_ITEMS", func() ([]model.SettingItem, error) {
_items, err := db.GetSettingItems()
if err != nil {
return nil, err
}
settingGroupCacheF("ALL_SETTING_ITEMS", _items)
return _items, nil
})
return items, err
}
func GetPublicSettingItems() ([]model.SettingItem, error) {
if items, ok := settingGroupCache.Get("ALL_PUBLIC_SETTING_ITEMS"); ok {
return items, nil
}
items, err, _ := settingGroupG.Do("ALL_PUBLIC_SETTING_ITEMS", func() ([]model.SettingItem, error) {
_items, err := db.GetPublicSettingItems()
if err != nil {
return nil, err
}
settingGroupCacheF("ALL_PUBLIC_SETTING_ITEMS", _items)
return _items, nil
})
return items, err
}
func GetSettingItemByKey(key string) (*model.SettingItem, error) {
if item, ok := settingCache.Get(key); ok {
return item, nil
}
item, err, _ := settingG.Do(key, func() (*model.SettingItem, error) {
_item, err := db.GetSettingItemByKey(key)
if err != nil {
return nil, err
}
settingCacheF(_item)
return _item, nil
})
return item, err
}
func GetSettingItemInKeys(keys []string) ([]model.SettingItem, error) {
var items []model.SettingItem
for _, key := range keys {
item, err := GetSettingItemByKey(key)
if err != nil {
return nil, err
}
items = append(items, *item)
}
return items, nil
}
func GetSettingItemsByGroup(group int) ([]model.SettingItem, error) {
key := strconv.Itoa(group)
if items, ok := settingGroupCache.Get(key); ok {
return items, nil
}
items, err, _ := settingGroupG.Do(key, func() ([]model.SettingItem, error) {
_items, err := db.GetSettingItemsByGroup(group)
if err != nil {
return nil, err
}
settingGroupCacheF(key, _items)
return _items, nil
})
return items, err
}
func GetSettingItemsInGroups(groups []int) ([]model.SettingItem, error) {
sort.Ints(groups)
var key string
for _, group := range groups {
key += strconv.Itoa(group)
}
if items, ok := settingGroupCache.Get(key); ok {
return items, nil
}
items, err, _ := settingGroupG.Do(key, func() ([]model.SettingItem, error) {
_items, err := db.GetSettingItemsInGroups(groups)
if err != nil {
return nil, err
}
settingGroupCacheF(key, _items)
return _items, nil
})
return items, err
}
func SaveSettingItems(items []model.SettingItem) (err error) {
// save
if err = db.SaveSettingItems(items); err != nil {
return err
}
// hook
if err = HandleSettingItemsHook(items); err != nil {
return err
}
// update
if err = db.SaveSettingItems(items); err != nil {
return err
}
settingCacheUpdate()
return nil
}
func SaveSettingItem(item *model.SettingItem) (err error) {
// save
if err = db.SaveSettingItem(item); err != nil {
return err
}
// hook
if _, err := HandleSettingItemHook(item); err != nil {
return err
}
// update
if err = db.SaveSettingItem(item); err != nil {
return err
}
settingCacheUpdate()
return nil
}
func DeleteSettingItemByKey(key string) error {
old, err := GetSettingItemByKey(key)
if err != nil {
return errors.WithMessage(err, "failed to get settingItem")
}
if !old.IsDeprecated() {
return errors.Errorf("setting [%s] is not deprecated", key)
}
settingCacheUpdate()
return db.DeleteSettingItemByKey(key)
}

4
internal/search/search.go
View File

@ -5,9 +5,9 @@ import (
"fmt"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/db"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/internal/search/searcher"
log "github.com/sirupsen/logrus"
)
@ -88,7 +88,7 @@ func BatchIndex(ctx context.Context, objs []ObjWithParent) error {
}
func init() {
db.RegisterSettingItemHook(conf.SearchIndex, func(item *model.SettingItem) error {
op.RegisterSettingItemHook(conf.SearchIndex, func(item *model.SettingItem) error {
log.Debugf("searcher init, mode: %s", item.Value)
return Init(item.Value)
})

3
internal/search/util.go
View File

@ -6,7 +6,6 @@ import (
"github.com/alist-org/alist/v3/drivers/alist_v3"
"github.com/alist-org/alist/v3/drivers/base"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/db"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/internal/setting"
@ -26,7 +25,7 @@ func WriteProgress(progress *model.IndexProgress) {
if err != nil {
log.Errorf("marshal progress error: %+v", err)
}
err = db.SaveSettingItem(model.SettingItem{
err = op.SaveSettingItem(&model.SettingItem{
Key: conf.IndexProgress,
Value: p,
Type: conf.TypeText,

8
internal/setting/setting.go
View File

@ -3,18 +3,18 @@ package setting
import (
"strconv"
"github.com/alist-org/alist/v3/internal/db"
"github.com/alist-org/alist/v3/internal/op"
)
func GetStr(key string, defaultValue ...string) string {
val, ok := db.GetSettingsMap().Load(key)
if !ok {
val, _ := op.GetSettingItemByKey(key)
if val == nil {
if len(defaultValue) > 0 {
return defaultValue[0]
}
return ""
}
return val
return val.Value
}
func GetInt(key string, defaultVal int) int {

4
server/handles/aria2.go
View File

@ -3,8 +3,8 @@ package handles
import (
"github.com/alist-org/alist/v3/internal/aria2"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/db"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/server/common"
"github.com/gin-gonic/gin"
)
@ -24,7 +24,7 @@ func SetAria2(c *gin.Context) {
{Key: conf.Aria2Uri, Value: req.Uri, Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE},
{Key: conf.Aria2Secret, Value: req.Secret, Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE},
}
if err := db.SaveSettingItems(items); err != nil {
if err := op.SaveSettingItems(items); err != nil {
common.ErrorResp(c, err, 500)
return
}

18
server/handles/setting.go
View File

@ -5,8 +5,8 @@ import (
"strings"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/db"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/internal/sign"
"github.com/alist-org/alist/v3/pkg/utils/random"
"github.com/alist-org/alist/v3/server/common"
@ -17,7 +17,7 @@ import (
func ResetToken(c *gin.Context) {
token := random.Token()
item := model.SettingItem{Key: "token", Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}
if err := db.SaveSettingItem(item); err != nil {
if err := op.SaveSettingItem(&item); err != nil {
common.ErrorResp(c, err, 500)
return
}
@ -29,14 +29,14 @@ func GetSetting(c *gin.Context) {
key := c.Query("key")
keys := c.Query("keys")
if key != "" {
item, err := db.GetSettingItemByKey(key)
item, err := op.GetSettingItemByKey(key)
if err != nil {
common.ErrorResp(c, err, 400)
return
}
common.SuccessResp(c, item)
} else {
items, err := db.GetSettingItemInKeys(strings.Split(keys, ","))
items, err := op.GetSettingItemInKeys(strings.Split(keys, ","))
if err != nil {
common.ErrorResp(c, err, 400)
return
@ -51,7 +51,7 @@ func SaveSettings(c *gin.Context) {
common.ErrorResp(c, err, 400)
return
}
if err := db.SaveSettingItems(req); err != nil {
if err := op.SaveSettingItems(req); err != nil {
common.ErrorResp(c, err, 500)
} else {
common.SuccessResp(c)
@ -65,7 +65,7 @@ func ListSettings(c *gin.Context) {
var settings []model.SettingItem
var err error
if groupsStr == "" && groupStr == "" {
settings, err = db.GetSettingItems()
settings, err = op.GetSettingItems()
} else {
var groupStrings []string
if groupsStr != "" {
@ -82,7 +82,7 @@ func ListSettings(c *gin.Context) {
}
groups = append(groups, group)
}
settings, err = db.GetSettingItemsInGroups(groups)
settings, err = op.GetSettingItemsInGroups(groups)
}
if err != nil {
common.ErrorResp(c, err, 400)
@ -93,7 +93,7 @@ func ListSettings(c *gin.Context) {
func DeleteSetting(c *gin.Context) {
key := c.Query("key")
if err := db.DeleteSettingItemByKey(key); err != nil {
if err := op.DeleteSettingItemByKey(key); err != nil {
common.ErrorResp(c, err, 500)
return
}
@ -101,5 +101,5 @@ func DeleteSetting(c *gin.Context) {
}
func PublicSettings(c *gin.Context) {
common.SuccessResp(c, db.GetPublicSettingsMap())
common.SuccessResp(c, op.GetPublicSettingsMap())
}