mirror of https://github.com/Xhofe/alist
feat(search): search with `meilisearch` (#6060)
* feat(search): search with meilisearch. * feat(search): meilisearch supports auto update. * chores: remove utils.Log. * fix(search): the null pointer caused by deleting non-existing file/folder indexes. --------- Co-authored-by: Andy Hsu <i@nn.ci>pull/6081/head
不插电
GitHub
parent
1f835502ba
commit
f1979a8bbc
7
go.mod
7
go.mod
|
|
@ -36,6 +36,7 @@ require (
|
|||
github.com/jlaffaye/ftp v0.2.0
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/maruel/natural v1.1.1
|
||||
github.com/meilisearch/meilisearch-go v0.26.1
|
||||
github.com/minio/sio v0.3.0
|
||||
github.com/natefinch/lumberjack v2.0.0+incompatible
|
||||
github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831
|
||||
|
|
@ -73,6 +74,7 @@ require (
|
|||
github.com/abbot/go-http-auth v0.4.0 // indirect
|
||||
github.com/aead/ecdh v0.2.0 // indirect
|
||||
github.com/andreburgaud/crypt2go v1.2.0 // indirect
|
||||
github.com/andybalholm/brotli v1.0.4 // indirect
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/benbjohnson/clock v1.3.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
|
|
@ -133,7 +135,9 @@ require (
|
|||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/jinzhu/now v1.1.5 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
|
||||
github.com/klauspost/compress v1.16.5 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
|
||||
github.com/kr/fs v0.1.0 // indirect
|
||||
github.com/leodido/go-urn v1.2.4 // indirect
|
||||
|
|
@ -142,6 +146,7 @@ require (
|
|||
github.com/libp2p/go-libp2p v0.27.8 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.19 // indirect
|
||||
github.com/mattn/go-localereader v0.0.1 // indirect
|
||||
|
|
@ -189,6 +194,8 @@ require (
|
|||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/u2takey/go-utils v0.3.1 // indirect
|
||||
github.com/ugorji/go/codec v1.2.11 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d // indirect
|
||||
github.com/x448/float16 v0.8.4 // indirect
|
||||
github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.3 // indirect
|
||||
|
|
|
|||
21
go.sum
21
go.sum
|
|
@ -27,6 +27,8 @@ github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F
|
|||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
|
||||
github.com/andreburgaud/crypt2go v1.2.0 h1:oly/ENAodeqTYpUafgd4r3v+VKLQnmOKUyfpj+TxHbE=
|
||||
github.com/andreburgaud/crypt2go v1.2.0/go.mod h1:kKRqlrX/3Q9Ki7HdUsoh0cX1Urq14/Hcta4l4VrIXrI=
|
||||
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
|
||||
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
|
||||
github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
|
||||
github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
|
||||
|
|
@ -247,6 +249,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
|
|||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
|
||||
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
|
||||
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
|
|
@ -254,6 +258,10 @@ github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 h1:G+9t9cEtnC
|
|||
github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004/go.mod h1:KmHnJWQrgEvbuy0vcvj00gtMqbvNn1L+3YUZLK/B92c=
|
||||
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
|
||||
github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
|
||||
github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
|
||||
|
|
@ -284,6 +292,8 @@ github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i
|
|||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
|
||||
github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a h1:N9zuLhTvBSRt0gWSiJswwQ2HqDmtX/ZCDJURnKUt1Ik=
|
||||
github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a/go.mod h1:JKx41uQRwqlTZabZc+kILPrO/3jlKnQ2Z8b7YiVw5cE=
|
||||
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
|
||||
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||
github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo=
|
||||
github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
|
|
@ -301,6 +311,8 @@ github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOj
|
|||
github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
|
||||
github.com/meilisearch/meilisearch-go v0.26.1 h1:3bmo2uLijX7kvBmiZ9LupVfC95TFcRJDgrRTzbOoE4A=
|
||||
github.com/meilisearch/meilisearch-go v0.26.1/go.mod h1:SxuSqDcPBIykjWz1PX+KzsYzArNLSCadQodWs8extS0=
|
||||
github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
|
||||
github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
|
||||
github.com/minio/sio v0.3.0 h1:syEFBewzOMOYVzSTFpp1MqpSZk8rUNbz8VIIc+PNzus=
|
||||
|
|
@ -449,8 +461,13 @@ github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4d
|
|||
github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
|
||||
github.com/upyun/go-sdk/v3 v3.0.4 h1:2DCJa/Yi7/3ZybT9UCPATSzvU3wpPPxhXinNlb1Hi8Q=
|
||||
github.com/upyun/go-sdk/v3 v3.0.4/go.mod h1:P/SnuuwhrIgAVRd/ZpzDWqCsBAf/oHg7UggbAxyZa0E=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d h1:xS9QTPgKl9ewGsAOPc+xW7DeStJDqYPfisDmeSCcbco=
|
||||
github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I=
|
||||
github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ=
|
||||
github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
|
||||
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
|
||||
github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 h1:jxZvjx8Ve5sOXorZG0KzTxbp0Cr1n3FEegfmyd9br1k=
|
||||
github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I=
|
||||
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
|
||||
|
|
@ -478,6 +495,7 @@ golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPh
|
|||
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
|
||||
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
|
||||
|
|
@ -497,6 +515,7 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY
|
|||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
|
|
@ -524,6 +543,8 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220702020025-31831981b65f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
|
|
|
|||
|
|
@ -145,7 +145,7 @@ func InitialSettings() []model.SettingItem {
|
|||
|
||||
// single settings
|
||||
{Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE},
|
||||
{Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,database_non_full_text,bleve,none", Group: model.INDEX},
|
||||
{Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,database_non_full_text,bleve,meilisearch,none", Group: model.INDEX},
|
||||
{Key: conf.AutoUpdateIndex, Value: "false", Type: conf.TypeBool, Group: model.INDEX},
|
||||
{Key: conf.IgnorePaths, Value: "", Type: conf.TypeText, Group: model.INDEX, Flag: model.PRIVATE, Help: `one path per line`},
|
||||
{Key: conf.MaxIndexDepth, Value: "20", Type: conf.TypeNumber, Group: model.INDEX, Flag: model.PRIVATE, Help: `max depth of index`},
|
||||
|
|
|
|||
|
|
@ -1,10 +1,9 @@
|
|||
package conf
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
|
||||
"github.com/alist-org/alist/v3/cmd/flags"
|
||||
"github.com/alist-org/alist/v3/pkg/utils/random"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
type Database struct {
|
||||
|
|
@ -20,6 +19,12 @@ type Database struct {
|
|||
DSN string `json:"dsn" env:"DSN"`
|
||||
}
|
||||
|
||||
type Meilisearch struct {
|
||||
Host string `json:"host" env:"HOST"`
|
||||
APIKey string `json:"api_key" env:"API_KEY"`
|
||||
IndexPrefix string `json:"index_prefix" env:"INDEX_PREFIX"`
|
||||
}
|
||||
|
||||
type Scheme struct {
|
||||
Address string `json:"address" env:"ADDR"`
|
||||
HttpPort int `json:"http_port" env:"HTTP_PORT"`
|
||||
|
|
@ -65,6 +70,7 @@ type Config struct {
|
|||
JwtSecret string `json:"jwt_secret" env:"JWT_SECRET"`
|
||||
TokenExpiresIn int `json:"token_expires_in" env:"TOKEN_EXPIRES_IN"`
|
||||
Database Database `json:"database" envPrefix:"DB_"`
|
||||
Meilisearch Meilisearch `json:"meilisearch" env:"MEILISEARCH"`
|
||||
Scheme Scheme `json:"scheme"`
|
||||
TempDir string `json:"temp_dir" env:"TEMP_DIR"`
|
||||
BleveDir string `json:"bleve_dir" env:"BLEVE_DIR"`
|
||||
|
|
@ -101,6 +107,9 @@ func DefaultConfig() *Config {
|
|||
TablePrefix: "x_",
|
||||
DBFile: dbPath,
|
||||
},
|
||||
Meilisearch: Meilisearch{
|
||||
Host: "http://localhost:7700",
|
||||
},
|
||||
BleveDir: indexDir,
|
||||
Log: LogConfig{
|
||||
Enable: true,
|
||||
|
|
|
|||
|
|
@ -4,4 +4,5 @@ import (
|
|||
_ "github.com/alist-org/alist/v3/internal/search/bleve"
|
||||
_ "github.com/alist-org/alist/v3/internal/search/db"
|
||||
_ "github.com/alist-org/alist/v3/internal/search/db_non_full_text"
|
||||
_ "github.com/alist-org/alist/v3/internal/search/meilisearch"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,89 @@
|
|||
package meilisearch
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/alist-org/alist/v3/internal/conf"
|
||||
"github.com/alist-org/alist/v3/internal/model"
|
||||
"github.com/alist-org/alist/v3/internal/search/searcher"
|
||||
"github.com/alist-org/alist/v3/pkg/utils"
|
||||
"github.com/meilisearch/meilisearch-go"
|
||||
)
|
||||
|
||||
var config = searcher.Config{
|
||||
Name: "meilisearch",
|
||||
AutoUpdate: true,
|
||||
}
|
||||
|
||||
func init() {
|
||||
searcher.RegisterSearcher(config, func() (searcher.Searcher, error) {
|
||||
m := Meilisearch{
|
||||
Client: meilisearch.NewClient(meilisearch.ClientConfig{
|
||||
Host: conf.Conf.Meilisearch.Host,
|
||||
APIKey: conf.Conf.Meilisearch.APIKey,
|
||||
}),
|
||||
IndexUid: conf.Conf.Meilisearch.IndexPrefix + "alist",
|
||||
FilterableAttributes: []string{"parent", "is_dir", "name"},
|
||||
SearchableAttributes: []string{"name"},
|
||||
}
|
||||
|
||||
_, err := m.Client.GetIndex(m.IndexUid)
|
||||
if err != nil {
|
||||
var mErr *meilisearch.Error
|
||||
ok := errors.As(err, &mErr)
|
||||
if ok && mErr.MeilisearchApiError.Code == "index_not_found" {
|
||||
task, err := m.Client.CreateIndex(&meilisearch.IndexConfig{
|
||||
Uid: m.IndexUid,
|
||||
PrimaryKey: "id",
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
forTask, err := m.Client.WaitForTask(task.TaskUID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if forTask.Status != meilisearch.TaskStatusSucceeded {
|
||||
return nil, fmt.Errorf("index creation failed, task status is %s", forTask.Status)
|
||||
}
|
||||
} else {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
attributes, err := m.Client.Index(m.IndexUid).GetFilterableAttributes()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if attributes == nil || !utils.SliceAllContains(*attributes, m.FilterableAttributes...) {
|
||||
_, err = m.Client.Index(m.IndexUid).UpdateFilterableAttributes(&m.FilterableAttributes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
attributes, err = m.Client.Index(m.IndexUid).GetSearchableAttributes()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if attributes == nil || !utils.SliceAllContains(*attributes, m.SearchableAttributes...) {
|
||||
_, err = m.Client.Index(m.IndexUid).UpdateSearchableAttributes(&m.SearchableAttributes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
pagination, err := m.Client.Index(m.IndexUid).GetPagination()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if pagination.MaxTotalHits != int64(model.MaxInt) {
|
||||
_, err := m.Client.Index(m.IndexUid).UpdatePagination(&meilisearch.Pagination{
|
||||
MaxTotalHits: int64(model.MaxInt),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return &m, nil
|
||||
})
|
||||
}
|
||||
|
|
@ -0,0 +1,227 @@
|
|||
package meilisearch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/alist-org/alist/v3/internal/model"
|
||||
"github.com/alist-org/alist/v3/internal/search/searcher"
|
||||
"github.com/alist-org/alist/v3/pkg/utils"
|
||||
"github.com/google/uuid"
|
||||
"github.com/meilisearch/meilisearch-go"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type searchDocument struct {
|
||||
ID string `json:"id"`
|
||||
model.SearchNode
|
||||
}
|
||||
|
||||
type Meilisearch struct {
|
||||
Client *meilisearch.Client
|
||||
IndexUid string
|
||||
FilterableAttributes []string
|
||||
SearchableAttributes []string
|
||||
}
|
||||
|
||||
func (m *Meilisearch) Config() searcher.Config {
|
||||
return config
|
||||
}
|
||||
|
||||
func (m *Meilisearch) Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) {
|
||||
mReq := &meilisearch.SearchRequest{
|
||||
AttributesToSearchOn: m.SearchableAttributes,
|
||||
Page: int64(req.Page),
|
||||
HitsPerPage: int64(req.PerPage),
|
||||
}
|
||||
if req.Scope != 0 {
|
||||
mReq.Filter = fmt.Sprintf("is_dir = %v", req.Scope == 1)
|
||||
}
|
||||
search, err := m.Client.Index(m.IndexUid).Search(req.Keywords, mReq)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
nodes, err := utils.SliceConvert(search.Hits, func(src any) (model.SearchNode, error) {
|
||||
srcMap := src.(map[string]any)
|
||||
return model.SearchNode{
|
||||
Parent: srcMap["parent"].(string),
|
||||
Name: srcMap["name"].(string),
|
||||
IsDir: srcMap["is_dir"].(bool),
|
||||
Size: int64(srcMap["size"].(float64)),
|
||||
}, nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return nodes, search.TotalHits, nil
|
||||
}
|
||||
|
||||
func (m *Meilisearch) Index(ctx context.Context, node model.SearchNode) error {
|
||||
return m.BatchIndex(ctx, []model.SearchNode{node})
|
||||
}
|
||||
|
||||
func (m *Meilisearch) BatchIndex(ctx context.Context, nodes []model.SearchNode) error {
|
||||
documents, _ := utils.SliceConvert(nodes, func(src model.SearchNode) (*searchDocument, error) {
|
||||
|
||||
return &searchDocument{
|
||||
ID: uuid.NewString(),
|
||||
SearchNode: src,
|
||||
}, nil
|
||||
})
|
||||
|
||||
_, err := m.Client.Index(m.IndexUid).AddDocuments(documents)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
//// Wait for the task to complete and check
|
||||
//forTask, err := m.Client.WaitForTask(task.TaskUID, meilisearch.WaitParams{
|
||||
// Context: ctx,
|
||||
// Interval: time.Millisecond * 50,
|
||||
//})
|
||||
//if err != nil {
|
||||
// return err
|
||||
//}
|
||||
//if forTask.Status != meilisearch.TaskStatusSucceeded {
|
||||
// return fmt.Errorf("BatchIndex failed, task status is %s", forTask.Status)
|
||||
//}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Meilisearch) getDocumentsByParent(ctx context.Context, parent string) ([]*searchDocument, error) {
|
||||
var result meilisearch.DocumentsResult
|
||||
err := m.Client.Index(m.IndexUid).GetDocuments(&meilisearch.DocumentsQuery{
|
||||
Filter: fmt.Sprintf("parent = '%s'", strings.ReplaceAll(parent, "'", "\\'")),
|
||||
Limit: int64(model.MaxInt),
|
||||
}, &result)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return utils.SliceConvert(result.Results, func(src map[string]any) (*searchDocument, error) {
|
||||
return &searchDocument{
|
||||
ID: src["id"].(string),
|
||||
SearchNode: model.SearchNode{
|
||||
Parent: src["parent"].(string),
|
||||
Name: src["name"].(string),
|
||||
IsDir: src["is_dir"].(bool),
|
||||
Size: int64(src["size"].(float64)),
|
||||
},
|
||||
}, nil
|
||||
})
|
||||
}
|
||||
|
||||
func (m *Meilisearch) Get(ctx context.Context, parent string) ([]model.SearchNode, error) {
|
||||
result, err := m.getDocumentsByParent(ctx, parent)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return utils.SliceConvert(result, func(src *searchDocument) (model.SearchNode, error) {
|
||||
return src.SearchNode, nil
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
func (m *Meilisearch) getParentsByPrefix(ctx context.Context, parent string) ([]string, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
parents := []string{parent}
|
||||
get, err := m.getDocumentsByParent(ctx, parent)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range get {
|
||||
if node.IsDir {
|
||||
arr, err := m.getParentsByPrefix(ctx, path.Join(node.Parent, node.Name))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
parents = append(parents, arr...)
|
||||
}
|
||||
}
|
||||
return parents, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Meilisearch) DelDirChild(ctx context.Context, prefix string) error {
|
||||
dfs, err := m.getParentsByPrefix(ctx, utils.FixAndCleanPath(prefix))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
utils.SliceReplace(dfs, func(src string) string {
|
||||
return "'" + strings.ReplaceAll(src, "'", "\\'") + "'"
|
||||
})
|
||||
s := fmt.Sprintf("parent IN [%s]", strings.Join(dfs, ","))
|
||||
task, err := m.Client.Index(m.IndexUid).DeleteDocumentsByFilter(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
taskStatus, err := m.getTaskStatus(ctx, task.TaskUID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if taskStatus != meilisearch.TaskStatusSucceeded {
|
||||
return fmt.Errorf("DelDir failed, task status is %s", taskStatus)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Meilisearch) Del(ctx context.Context, prefix string) error {
|
||||
prefix = utils.FixAndCleanPath(prefix)
|
||||
dir, name := path.Split(prefix)
|
||||
get, err := m.getDocumentsByParent(ctx, dir[:len(dir)-1])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var document *searchDocument
|
||||
for _, v := range get {
|
||||
if v.Name == name {
|
||||
document = v
|
||||
break
|
||||
}
|
||||
}
|
||||
if document == nil {
|
||||
// Defensive programming. Document may be the folder, try deleting Child
|
||||
return m.DelDirChild(ctx, prefix)
|
||||
}
|
||||
if document.IsDir {
|
||||
err = m.DelDirChild(ctx, prefix)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
task, err := m.Client.Index(m.IndexUid).DeleteDocument(document.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
taskStatus, err := m.getTaskStatus(ctx, task.TaskUID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if taskStatus != meilisearch.TaskStatusSucceeded {
|
||||
return fmt.Errorf("DelDir failed, task status is %s", taskStatus)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Meilisearch) Release(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Meilisearch) Clear(ctx context.Context) error {
|
||||
_, err := m.Client.Index(m.IndexUid).DeleteAllDocuments()
|
||||
return err
|
||||
}
|
||||
|
||||
func (m *Meilisearch) getTaskStatus(ctx context.Context, taskUID int64) (meilisearch.TaskStatus, error) {
|
||||
forTask, err := m.Client.WaitForTask(taskUID, meilisearch.WaitParams{
|
||||
Context: ctx,
|
||||
Interval: time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
return meilisearch.TaskStatusUnknown, err
|
||||
}
|
||||
return forTask.Status, nil
|
||||
}
|
||||
|
|
@ -29,6 +29,20 @@ func SliceContains[T comparable](arr []T, v T) bool {
|
|||
return false
|
||||
}
|
||||
|
||||
// SliceAllContains check if slice all contains elements
|
||||
func SliceAllContains[T comparable](arr []T, vs ...T) bool {
|
||||
vsMap := make(map[T]struct{})
|
||||
for _, v := range arr {
|
||||
vsMap[v] = struct{}{}
|
||||
}
|
||||
for _, v := range vs {
|
||||
if _, ok := vsMap[v]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// SliceConvert convert slice to another type slice
|
||||
func SliceConvert[S any, D any](srcS []S, convert func(src S) (D, error)) ([]D, error) {
|
||||
res := make([]D, 0, len(srcS))
|
||||
|
|
@ -79,3 +93,9 @@ func SliceFilter[T any](arr []T, filter func(src T) bool) []T {
|
|||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func SliceReplace[T any](arr []T, replace func(src T) T) {
|
||||
for i, src := range arr {
|
||||
arr[i] = replace(src)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue