mirror of https://github.com/Xhofe/alist
* feat: build index & search with bleve (#1740) * delete unused struct Co-authored-by: Noah Hsu <i@nn.ci>pull/2471/head
BoYanZh
GitHub
parent
2b902de6fd
commit
330a767fd7
|
|
@ -15,6 +15,7 @@ func Init() {
|
|||
bootstrap.InitConfig()
|
||||
bootstrap.Log()
|
||||
bootstrap.InitDB()
|
||||
bootstrap.InitIndex()
|
||||
data.InitData()
|
||||
}
|
||||
|
||||
|
|
|
|||
23
go.mod
23
go.mod
|
|
@ -6,6 +6,7 @@ require (
|
|||
github.com/SheltonZhu/115driver v1.0.12
|
||||
github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a
|
||||
github.com/aws/aws-sdk-go v1.44.142
|
||||
github.com/blevesearch/bleve/v2 v2.3.5
|
||||
github.com/caarlos0/env/v6 v6.10.1
|
||||
github.com/disintegration/imaging v1.6.2
|
||||
github.com/gin-contrib/cors v1.4.0
|
||||
|
|
@ -35,9 +36,26 @@ require (
|
|||
)
|
||||
|
||||
require (
|
||||
github.com/RoaringBitmap/roaring v0.9.4 // indirect
|
||||
github.com/aead/ecdh v0.2.0 // indirect
|
||||
github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible // indirect
|
||||
github.com/andreburgaud/crypt2go v1.1.0 // indirect
|
||||
github.com/bits-and-blooms/bitset v1.2.0 // indirect
|
||||
github.com/blevesearch/bleve_index_api v1.0.4 // indirect
|
||||
github.com/blevesearch/geo v0.1.15 // indirect
|
||||
github.com/blevesearch/go-porterstemmer v1.0.3 // indirect
|
||||
github.com/blevesearch/gtreap v0.1.1 // indirect
|
||||
github.com/blevesearch/mmap-go v1.0.4 // indirect
|
||||
github.com/blevesearch/scorch_segment_api/v2 v2.1.3 // indirect
|
||||
github.com/blevesearch/segment v0.9.0 // indirect
|
||||
github.com/blevesearch/snowballstem v0.9.0 // indirect
|
||||
github.com/blevesearch/upsidedown_store_api v1.0.1 // indirect
|
||||
github.com/blevesearch/vellum v1.0.9 // indirect
|
||||
github.com/blevesearch/zapx/v11 v11.3.6 // indirect
|
||||
github.com/blevesearch/zapx/v12 v12.3.6 // indirect
|
||||
github.com/blevesearch/zapx/v13 v13.3.6 // indirect
|
||||
github.com/blevesearch/zapx/v14 v14.3.6 // indirect
|
||||
github.com/blevesearch/zapx/v15 v15.3.6 // indirect
|
||||
github.com/bluele/gcache v0.0.2 // indirect
|
||||
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
|
||||
github.com/gaoyb7/115drive-webdav v0.1.8 // indirect
|
||||
|
|
@ -48,6 +66,9 @@ require (
|
|||
github.com/go-playground/validator/v10 v10.11.0 // indirect
|
||||
github.com/go-sql-driver/mysql v1.6.0 // indirect
|
||||
github.com/goccy/go-json v0.9.7 // indirect
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
|
||||
github.com/golang/protobuf v1.5.0 // indirect
|
||||
github.com/golang/snappy v0.0.1 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.0.1 // indirect
|
||||
|
|
@ -68,12 +89,14 @@ require (
|
|||
github.com/mattn/go-sqlite3 v1.14.15 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/mschoch/smat v0.2.0 // indirect
|
||||
github.com/orzogc/fake115uploader v0.3.3-0.20221009101310-08b764073b77 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.0.1 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.17 // indirect
|
||||
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/ugorji/go/codec v1.2.7 // indirect
|
||||
go.etcd.io/bbolt v1.3.5 // indirect
|
||||
golang.org/x/image v0.0.0-20220722155232-062f8c9fd539 // indirect
|
||||
golang.org/x/sys v0.2.0 // indirect
|
||||
golang.org/x/text v0.4.0 // indirect
|
||||
|
|
|
|||
49
go.sum
49
go.sum
|
|
@ -2,6 +2,8 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
|
|||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
|
||||
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
|
||||
github.com/RoaringBitmap/roaring v0.9.4 h1:ckvZSX5gwCRaJYBNe7syNawCU5oruY9gQmjXlp4riwo=
|
||||
github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA=
|
||||
github.com/SheltonZhu/115driver v1.0.12 h1:+GlIM5h8tzuec6MzK0wFwb7bY77nav7JhY8lTljzls4=
|
||||
github.com/SheltonZhu/115driver v1.0.12/go.mod h1:00ixivHH5HqDj4S7kAWbkuUrjtsJTxc7cGv5RMw3RVs=
|
||||
github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a h1:RenIAa2q4H8UcS/cqmwdT1WCWIAH5aumP8m8RpbqVsE=
|
||||
|
|
@ -14,6 +16,41 @@ github.com/andreburgaud/crypt2go v1.1.0 h1:eitZxTPY1krUsxinsng3Qvt/Ud7q/aQmmYRh8
|
|||
github.com/andreburgaud/crypt2go v1.1.0/go.mod h1:4qhZPzarj1dCIRmCkpdgCklwp+hBq9yEt0zPe9Ayuhc=
|
||||
github.com/aws/aws-sdk-go v1.44.142 h1:KZ1/FDwCSft1DuNllFaBtWpcG0CW2NgQjvOrE1TdlXE=
|
||||
github.com/aws/aws-sdk-go v1.44.142/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
|
||||
github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA=
|
||||
github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
|
||||
github.com/blevesearch/bleve/v2 v2.3.5 h1:1wuR7eB8Fk9UaCaBUfnQt5V7zIpi4VDok9ExN7Rl+/8=
|
||||
github.com/blevesearch/bleve/v2 v2.3.5/go.mod h1:FneKGHMRrCLrp4X9+iy3wlBqgM2ALucg7bp8jUuAi/s=
|
||||
github.com/blevesearch/bleve_index_api v1.0.3/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4=
|
||||
github.com/blevesearch/bleve_index_api v1.0.4 h1:mtlzsyJjMIlDngqqB1mq8kPryUMIuEVVbRbJHOWEexU=
|
||||
github.com/blevesearch/bleve_index_api v1.0.4/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms=
|
||||
github.com/blevesearch/geo v0.1.15 h1:0NybEduqE5fduFRYiUKF0uqybAIFKXYjkBdXKYn7oA4=
|
||||
github.com/blevesearch/geo v0.1.15/go.mod h1:cRIvqCdk3cgMhGeHNNe6yPzb+w56otxbfo1FBJfR2Pc=
|
||||
github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
|
||||
github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M=
|
||||
github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y=
|
||||
github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk=
|
||||
github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc=
|
||||
github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs=
|
||||
github.com/blevesearch/scorch_segment_api/v2 v2.1.3 h1:2UzpR2dR5DvSZk8tVJkcQ7D5xhoK/UBelYw8ttBHrRQ=
|
||||
github.com/blevesearch/scorch_segment_api/v2 v2.1.3/go.mod h1:eZrfp1y+lUh+DzFjUcTBUSnKGuunyFIpBIvqYVzJfvc=
|
||||
github.com/blevesearch/segment v0.9.0 h1:5lG7yBCx98or7gK2cHMKPukPZ/31Kag7nONpoBt22Ac=
|
||||
github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ=
|
||||
github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s=
|
||||
github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs=
|
||||
github.com/blevesearch/upsidedown_store_api v1.0.1 h1:1SYRwyoFLwG3sj0ed89RLtM15amfX2pXlYbFOnF8zNU=
|
||||
github.com/blevesearch/upsidedown_store_api v1.0.1/go.mod h1:MQDVGpHZrpe3Uy26zJBf/a8h0FZY6xJbthIMm8myH2Q=
|
||||
github.com/blevesearch/vellum v1.0.9 h1:PL+NWVk3dDGPCV0hoDu9XLLJgqU4E5s/dOeEJByQ2uQ=
|
||||
github.com/blevesearch/vellum v1.0.9/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k=
|
||||
github.com/blevesearch/zapx/v11 v11.3.6 h1:50jET4HUJ6eCqGxdhUt+mjybMvEX2MWyqLGtCx3yUgc=
|
||||
github.com/blevesearch/zapx/v11 v11.3.6/go.mod h1:B0CzJRj/pS7hJIroflRtFsa9mRHpMSucSgre0FVINns=
|
||||
github.com/blevesearch/zapx/v12 v12.3.6 h1:G304NHBLgQeZ+IHK/XRCM0nhHqAts8MEvHI6LhoDNM4=
|
||||
github.com/blevesearch/zapx/v12 v12.3.6/go.mod h1:iYi7tIKpauwU5os5wTxJITixr5Km21Hl365otMwdaP0=
|
||||
github.com/blevesearch/zapx/v13 v13.3.6 h1:vavltQHNdjQezhLZs5nIakf+w/uOa1oqZxB58Jy/3Ig=
|
||||
github.com/blevesearch/zapx/v13 v13.3.6/go.mod h1:X+FsTwCU8qOHtK0d/ArvbOH7qiIgViSQ1GQvcR6LSkI=
|
||||
github.com/blevesearch/zapx/v14 v14.3.6 h1:b9lub7TvcwUyJxK/cQtnN79abngKxsI7zMZnICU0WhE=
|
||||
github.com/blevesearch/zapx/v14 v14.3.6/go.mod h1:9X8W3XoikagU0rwcTqwZho7p9cC7m7zhPZO94S4wUvM=
|
||||
github.com/blevesearch/zapx/v15 v15.3.6 h1:VSswg/ysDxHgitcNkpUNtaTYS4j3uItpXWLAASphl6k=
|
||||
github.com/blevesearch/zapx/v15 v15.3.6/go.mod h1:5DbhhDTGtuQSns1tS2aJxJLPc91boXCvjOMeCLD1saM=
|
||||
github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
|
||||
github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0=
|
||||
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI=
|
||||
|
|
@ -64,10 +101,16 @@ github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPh
|
|||
github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs=
|
||||
github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo=
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
|
||||
github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
|
||||
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
|
||||
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
|
|
@ -140,6 +183,7 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
|
|||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||
github.com/json-iterator/go v0.0.0-20171115153421-f7279a603ede/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
|
|
@ -178,6 +222,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
|
|||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
|
||||
github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
|
||||
github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM=
|
||||
github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk=
|
||||
github.com/orzogc/fake115uploader v0.3.3-0.20221009101310-08b764073b77 h1:dg/EaaJLPIg4xn2kaZil7Ax3wfoxcFXaBwyOTlcz5AI=
|
||||
|
|
@ -245,6 +291,8 @@ github.com/winfsp/cgofuse v1.5.0 h1:MsBP7Mi/LiJf/7/F3O/7HjjR009ds6KCdqXzKpZSWxI=
|
|||
github.com/winfsp/cgofuse v1.5.0/go.mod h1:h3awhoUOcn2VYVKCwDaYxSLlZwnyK+A8KaDoLUp2lbU=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
|
||||
go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
|
||||
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
|
||||
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
|
||||
|
|
@ -303,6 +351,7 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
|
|
|
|||
|
|
@ -0,0 +1,10 @@
|
|||
package bootstrap
|
||||
|
||||
import (
|
||||
"github.com/alist-org/alist/v3/internal/conf"
|
||||
"github.com/alist-org/alist/v3/internal/index"
|
||||
)
|
||||
|
||||
func InitIndex() {
|
||||
index.Init(&conf.Conf.IndexDir)
|
||||
}
|
||||
|
|
@ -45,11 +45,13 @@ type Config struct {
|
|||
Database Database `json:"database"`
|
||||
Scheme Scheme `json:"scheme"`
|
||||
TempDir string `json:"temp_dir" env:"TEMP_DIR"`
|
||||
IndexDir string `json:"index_dir" env:"INDEX_DIR"`
|
||||
Log LogConfig `json:"log"`
|
||||
}
|
||||
|
||||
func DefaultConfig() *Config {
|
||||
tempDir := filepath.Join(flags.DataDir, "temp")
|
||||
indexDir := filepath.Join(flags.DataDir, "index")
|
||||
logPath := filepath.Join(flags.DataDir, "log/log.log")
|
||||
dbPath := filepath.Join(flags.DataDir, "data.db")
|
||||
return &Config{
|
||||
|
|
@ -64,6 +66,7 @@ func DefaultConfig() *Config {
|
|||
TablePrefix: "x_",
|
||||
DBFile: dbPath,
|
||||
},
|
||||
IndexDir: indexDir,
|
||||
Log: LogConfig{
|
||||
Enable: true,
|
||||
Name: logPath,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,106 @@
|
|||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/alist-org/alist/v3/internal/db"
|
||||
"github.com/alist-org/alist/v3/internal/fs"
|
||||
"github.com/alist-org/alist/v3/internal/model"
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// walkFS traverses filesystem fs starting at name up to depth levels.
|
||||
//
|
||||
// walkFS will stop when current depth > `depth`. For each visited node,
|
||||
// walkFS calls walkFn. If a visited file system node is a directory and
|
||||
// walkFn returns path.SkipDir, walkFS will skip traversal of this node.
|
||||
func walkFS(ctx context.Context, depth int, name string, info model.Obj, walkFn func(reqPath string, info model.Obj, err error) error) error {
|
||||
// This implementation is based on Walk's code in the standard path/path package.
|
||||
walkFnErr := walkFn(name, info, nil)
|
||||
if walkFnErr != nil {
|
||||
if info.IsDir() && walkFnErr == filepath.SkipDir {
|
||||
return nil
|
||||
}
|
||||
return walkFnErr
|
||||
}
|
||||
if !info.IsDir() || depth == 0 {
|
||||
return nil
|
||||
}
|
||||
meta, _ := db.GetNearestMeta(name)
|
||||
// Read directory names.
|
||||
objs, err := fs.List(context.WithValue(ctx, "meta", meta), name)
|
||||
if err != nil {
|
||||
return walkFnErr
|
||||
}
|
||||
for _, fileInfo := range objs {
|
||||
filename := path.Join(name, fileInfo.GetName())
|
||||
if err := walkFS(ctx, depth-1, filename, fileInfo, walkFn); err != nil {
|
||||
if err == filepath.SkipDir {
|
||||
break
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type Data struct {
|
||||
Path string
|
||||
}
|
||||
|
||||
func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth int) {
|
||||
WriteProgress(&Progress{
|
||||
FileCount: 0,
|
||||
IsDone: false,
|
||||
LastDoneTime: nil,
|
||||
})
|
||||
var batchs []*bleve.Batch
|
||||
var fileCount uint64 = 0
|
||||
for _, indexPath := range indexPaths {
|
||||
batch := func() *bleve.Batch {
|
||||
batch := index.NewBatch()
|
||||
// TODO: cache unchanged part
|
||||
// TODO: store current progress
|
||||
walkFn := func(indexPath string, info model.Obj, err error) error {
|
||||
for _, avoidPath := range ignorePaths {
|
||||
if indexPath == avoidPath {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
}
|
||||
if !info.IsDir() {
|
||||
batch.Index(uuid.NewString(), Data{Path: indexPath})
|
||||
fileCount += 1
|
||||
if fileCount%100 == 0 {
|
||||
WriteProgress(&Progress{
|
||||
FileCount: fileCount,
|
||||
IsDone: false,
|
||||
LastDoneTime: nil,
|
||||
})
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
fi, err := fs.Get(ctx, indexPath)
|
||||
if err != nil {
|
||||
return batch
|
||||
}
|
||||
// TODO: run walkFS concurrently
|
||||
walkFS(ctx, maxDepth, indexPath, fi, walkFn)
|
||||
return batch
|
||||
}()
|
||||
batchs = append(batchs, batch)
|
||||
}
|
||||
for _, batch := range batchs {
|
||||
index.Batch(batch)
|
||||
}
|
||||
now := time.Now()
|
||||
WriteProgress(&Progress{
|
||||
FileCount: fileCount,
|
||||
IsDone: true,
|
||||
LastDoneTime: &now,
|
||||
})
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
package index
|
||||
|
||||
import (
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
var index bleve.Index
|
||||
|
||||
func Init(indexPath *string) {
|
||||
fileIndex, err := bleve.Open(*indexPath)
|
||||
if err == bleve.ErrorIndexPathDoesNotExist {
|
||||
log.Infof("Creating new index...")
|
||||
indexMapping := bleve.NewIndexMapping()
|
||||
fileIndex, err = bleve.New(*indexPath, indexMapping)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
index = fileIndex
|
||||
progress := ReadProgress()
|
||||
if !progress.IsDone {
|
||||
log.Warnf("Last index build does not succeed!")
|
||||
WriteProgress(&Progress{
|
||||
FileCount: progress.FileCount,
|
||||
IsDone: false,
|
||||
LastDoneTime: nil,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
package index
|
||||
|
||||
import (
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func Search(queryString string, size int) (*bleve.SearchResult, error) {
|
||||
query := bleve.NewMatchQuery(queryString)
|
||||
search := bleve.NewSearchRequest(query)
|
||||
search.Size = size
|
||||
search.Fields = []string{"Path"}
|
||||
searchResults, err := index.Search(search)
|
||||
if err != nil {
|
||||
log.Errorf("search error: %+v", err)
|
||||
return nil, err
|
||||
}
|
||||
return searchResults, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
package index
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/alist-org/alist/v3/internal/conf"
|
||||
"github.com/alist-org/alist/v3/pkg/utils"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type Progress struct {
|
||||
FileCount uint64 `json:"file_count"`
|
||||
IsDone bool `json:"is_done"`
|
||||
LastDoneTime *time.Time `json:"last_done_time"`
|
||||
}
|
||||
|
||||
func ReadProgress() Progress {
|
||||
progressFilePath := filepath.Join(conf.Conf.IndexDir, "progress.json")
|
||||
_, err := os.Stat(progressFilePath)
|
||||
progress := Progress{0, false, nil}
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
if !utils.WriteJsonToFile(progressFilePath, progress) {
|
||||
log.Fatalf("failed to create index progress file")
|
||||
}
|
||||
}
|
||||
progressBytes, err := os.ReadFile(progressFilePath)
|
||||
if err != nil {
|
||||
log.Fatalf("reading index progress file error: %+v", err)
|
||||
}
|
||||
err = utils.Json.Unmarshal(progressBytes, &progress)
|
||||
if err != nil {
|
||||
log.Fatalf("load index progress error: %+v", err)
|
||||
}
|
||||
return progress
|
||||
}
|
||||
|
||||
func WriteProgress(progress *Progress) {
|
||||
progressFilePath := filepath.Join(conf.Conf.IndexDir, "progress.json")
|
||||
log.Infof("write index progress: %v", progress)
|
||||
if !utils.WriteJsonToFile(progressFilePath, progress) {
|
||||
log.Fatalf("failed to write to index progress file")
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
package handles
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
|
||||
"github.com/alist-org/alist/v3/internal/db"
|
||||
"github.com/alist-org/alist/v3/internal/index"
|
||||
"github.com/alist-org/alist/v3/server/common"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
func BuildIndex(c *gin.Context) {
|
||||
go func() {
|
||||
// TODO: consider run build index as non-admin
|
||||
user, _ := db.GetAdmin()
|
||||
ctx := context.WithValue(c.Request.Context(), "user", user)
|
||||
maxDepth, err := strconv.Atoi(c.PostForm("max_depth"))
|
||||
if err != nil {
|
||||
maxDepth = -1
|
||||
}
|
||||
indexPaths := []string{"/"}
|
||||
ignorePaths := c.PostFormArray("ignore_paths")
|
||||
index.BuildIndex(ctx, indexPaths, ignorePaths, maxDepth)
|
||||
}()
|
||||
common.SuccessResp(c)
|
||||
}
|
||||
|
||||
func GetProgress(c *gin.Context) {
|
||||
progress := index.ReadProgress()
|
||||
common.SuccessResp(c, progress)
|
||||
}
|
||||
|
||||
func Search(c *gin.Context) {
|
||||
results := []string{}
|
||||
query, exists := c.GetQuery("query")
|
||||
if !exists {
|
||||
common.SuccessResp(c, results)
|
||||
}
|
||||
sizeStr, _ := c.GetQuery("size")
|
||||
size, err := strconv.Atoi(sizeStr)
|
||||
if err != nil {
|
||||
size = 10
|
||||
}
|
||||
searchResults, err := index.Search(query, size)
|
||||
if err != nil {
|
||||
common.ErrorResp(c, err, 500)
|
||||
return
|
||||
}
|
||||
for _, documentMatch := range searchResults.Hits {
|
||||
results = append(results, documentMatch.Fields["Path"].(string))
|
||||
}
|
||||
common.SuccessResp(c, results)
|
||||
}
|
||||
|
|
@ -107,6 +107,11 @@ func admin(g *gin.RouterGroup) {
|
|||
ms := g.Group("/message")
|
||||
ms.POST("/get", message.HttpInstance.GetHandle)
|
||||
ms.POST("/send", message.HttpInstance.SendHandle)
|
||||
|
||||
index := g.Group("/index")
|
||||
index.POST("/build", handles.BuildIndex)
|
||||
index.GET("/progress", handles.GetProgress)
|
||||
index.GET("/search", handles.Search)
|
||||
}
|
||||
|
||||
func _fs(g *gin.RouterGroup) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue