mirror of https://github.com/Xhofe/alist
fix(aliyundrive_open): fail limit on concurrently call (#4851)
Andy Hsu
parent
cee6bc6b5d
commit
7e9cdd8b07
|
|
@ -6,6 +6,7 @@ import (
|
|||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/Xhofe/rateg"
|
||||
"github.com/alist-org/alist/v3/drivers/base"
|
||||
"github.com/alist-org/alist/v3/internal/driver"
|
||||
"github.com/alist-org/alist/v3/internal/errs"
|
||||
|
|
@ -39,8 +40,14 @@ func (d *AliyundriveOpen) Init(ctx context.Context) error {
|
|||
return err
|
||||
}
|
||||
d.DriveId = utils.Json.Get(res, "default_drive_id").ToString()
|
||||
d.limitList = utils.LimitRateCtx(d.list, time.Second/3)
|
||||
d.limitLink = utils.LimitRateCtx(d.link, time.Second+time.Millisecond*2)
|
||||
d.limitList = rateg.LimitFnCtx(d.list, rateg.LimitFnOption{
|
||||
Limit: 4,
|
||||
Bucket: 1,
|
||||
})
|
||||
d.limitLink = rateg.LimitFnCtx(d.link, rateg.LimitFnOption{
|
||||
Limit: 1,
|
||||
Bucket: 1,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import (
|
|||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/Xhofe/rateg"
|
||||
"github.com/alist-org/alist/v3/drivers/base"
|
||||
"github.com/alist-org/alist/v3/internal/driver"
|
||||
"github.com/alist-org/alist/v3/internal/errs"
|
||||
|
|
@ -52,8 +53,14 @@ func (d *AliyundriveShare) Init(ctx context.Context) error {
|
|||
log.Errorf("%+v", err)
|
||||
}
|
||||
})
|
||||
d.limitList = utils.LimitRateCtx(d.list, time.Second/4)
|
||||
d.limitLink = utils.LimitRateCtx(d.link, time.Second)
|
||||
d.limitList = rateg.LimitFnCtx(d.list, rateg.LimitFnOption{
|
||||
Limit: 4,
|
||||
Bucket: 1,
|
||||
})
|
||||
d.limitLink = rateg.LimitFnCtx(d.link, rateg.LimitFnOption{
|
||||
Limit: 1,
|
||||
Bucket: 1,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
3
go.mod
3
go.mod
|
|
@ -49,6 +49,7 @@ require (
|
|||
require (
|
||||
github.com/BurntSushi/toml v0.3.1 // indirect
|
||||
github.com/RoaringBitmap/roaring v1.2.3 // indirect
|
||||
github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 // indirect
|
||||
github.com/aead/ecdh v0.2.0 // indirect
|
||||
github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible // indirect
|
||||
github.com/andreburgaud/crypt2go v1.1.0 // indirect
|
||||
|
|
@ -135,7 +136,7 @@ require (
|
|||
golang.org/x/arch v0.3.0 // indirect
|
||||
golang.org/x/sys v0.10.0 // indirect
|
||||
golang.org/x/text v0.11.0 // indirect
|
||||
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect
|
||||
golang.org/x/time v0.3.0 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/protobuf v1.31.0 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
|
||||
|
|
|
|||
5
go.sum
5
go.sum
|
|
@ -6,6 +6,8 @@ github.com/SheltonZhu/115driver v1.0.14 h1:uW3dl8J9KDMw+3gPxQdhTysoGhw0/uI1484GT
|
|||
github.com/SheltonZhu/115driver v1.0.14/go.mod h1:00ixivHH5HqDj4S7kAWbkuUrjtsJTxc7cGv5RMw3RVs=
|
||||
github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a h1:RenIAa2q4H8UcS/cqmwdT1WCWIAH5aumP8m8RpbqVsE=
|
||||
github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a/go.mod h1:sSBbaOg90XwWKtpT56kVujF0bIeVITnPlssLclogS04=
|
||||
github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 h1:WnvifFgYyogPz2ZFvaVLk4gI/Co0paF92FmxSR6U1zY=
|
||||
github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4/go.mod h1:8pWlL2rpusvx7Xa6yYaIWOJ8bR3gPdFBUT7OystyGOY=
|
||||
github.com/Xhofe/wopan-sdk-go v0.1.1 h1:dSrTxNYclqNuo9libjtC+R6C4RCen/inh/dUXd12vpM=
|
||||
github.com/Xhofe/wopan-sdk-go v0.1.1/go.mod h1:xWcUS7PoFLDD9gy2BK2VQfilEsZngLMz2Vkx3oF2zJY=
|
||||
github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ=
|
||||
|
|
@ -210,6 +212,7 @@ github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APP
|
|||
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI=
|
||||
github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
|
||||
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
|
||||
github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
|
||||
github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
|
|
@ -394,6 +397,8 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb
|
|||
golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y=
|
||||
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
|
||||
golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
|
|
|
|||
|
|
@ -1,114 +0,0 @@
|
|||
package utils
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"time"
|
||||
)
|
||||
|
||||
func LimitRateReflect(f interface{}, interval time.Duration) func(...interface{}) []interface{} {
|
||||
// Use closures to save the time of the last function call
|
||||
var lastCall time.Time
|
||||
|
||||
fValue := reflect.ValueOf(f)
|
||||
fType := fValue.Type()
|
||||
|
||||
if fType.Kind() != reflect.Func {
|
||||
panic("f must be a function")
|
||||
}
|
||||
|
||||
//if fType.NumOut() == 0 {
|
||||
// panic("f must have at least one output parameter")
|
||||
//}
|
||||
|
||||
outCount := fType.NumOut()
|
||||
outTypes := make([]reflect.Type, outCount)
|
||||
|
||||
for i := 0; i < outCount; i++ {
|
||||
outTypes[i] = fType.Out(i)
|
||||
}
|
||||
|
||||
// Returns a new function, which is used to limit the function to be called only once at a specified time interval
|
||||
return func(args ...interface{}) []interface{} {
|
||||
// Calculate the time interval since the last function call
|
||||
elapsed := time.Since(lastCall)
|
||||
// If the interval is less than the specified time, wait for the remaining time
|
||||
if elapsed < interval {
|
||||
time.Sleep(interval - elapsed)
|
||||
}
|
||||
// Update the time of the last function call
|
||||
lastCall = time.Now()
|
||||
|
||||
inCount := fType.NumIn()
|
||||
in := make([]reflect.Value, inCount)
|
||||
|
||||
if len(args) != inCount {
|
||||
panic("wrong number of arguments")
|
||||
}
|
||||
|
||||
for i := 0; i < inCount; i++ {
|
||||
in[i] = reflect.ValueOf(args[i])
|
||||
}
|
||||
|
||||
out := fValue.Call(in)
|
||||
|
||||
if len(out) != outCount {
|
||||
panic("function returned wrong number of values")
|
||||
}
|
||||
|
||||
result := make([]interface{}, outCount)
|
||||
|
||||
for i := 0; i < outCount; i++ {
|
||||
result[i] = out[i].Interface()
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
type Fn[T any, R any] func(T) (R, error)
|
||||
type FnCtx[T any, R any] func(context.Context, T) (R, error)
|
||||
|
||||
func LimitRate[T any, R any](f Fn[T, R], interval time.Duration) Fn[T, R] {
|
||||
// Use closures to save the time of the last function call
|
||||
var lastCall time.Time
|
||||
// Returns a new function, which is used to limit the function to be called only once at a specified time interval
|
||||
return func(t T) (R, error) {
|
||||
// Calculate the time interval since the last function call
|
||||
elapsed := time.Since(lastCall)
|
||||
// If the interval is less than the specified time, wait for the remaining time
|
||||
if elapsed < interval {
|
||||
time.Sleep(interval - elapsed)
|
||||
}
|
||||
// Update the time of the last function call
|
||||
lastCall = time.Now()
|
||||
// Execute the function that needs to be limited
|
||||
return f(t)
|
||||
}
|
||||
}
|
||||
|
||||
func LimitRateCtx[T any, R any](f FnCtx[T, R], interval time.Duration) FnCtx[T, R] {
|
||||
// Use closures to save the time of the last function call
|
||||
var lastCall time.Time
|
||||
// Returns a new function, which is used to limit the function to be called only once at a specified time interval
|
||||
return func(ctx context.Context, t T) (R, error) {
|
||||
// Calculate the time interval since the last function call
|
||||
elapsed := time.Since(lastCall)
|
||||
// If the interval is less than the specified time, wait for the remaining time
|
||||
if elapsed < interval {
|
||||
t := time.NewTimer(interval - elapsed)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.Stop()
|
||||
var zero R
|
||||
return zero, ctx.Err()
|
||||
case <-t.C:
|
||||
|
||||
}
|
||||
}
|
||||
// Update the time of the last function call
|
||||
lastCall = time.Now()
|
||||
// Execute the function that needs to be limited
|
||||
return f(ctx, t)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,59 +0,0 @@
|
|||
package utils_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/alist-org/alist/v3/pkg/utils"
|
||||
)
|
||||
|
||||
func myFunction(a int) (int, error) {
|
||||
// do something
|
||||
return a + 1, nil
|
||||
}
|
||||
|
||||
func TestLimitRate(t *testing.T) {
|
||||
myLimitedFunction := utils.LimitRate(myFunction, time.Second)
|
||||
result, _ := myLimitedFunction(1)
|
||||
t.Log(result) // Output: 2
|
||||
result, _ = myLimitedFunction(2)
|
||||
t.Log(result) // Output: 3
|
||||
}
|
||||
|
||||
type Test struct {
|
||||
limitFn func(string) (string, error)
|
||||
}
|
||||
|
||||
func (t *Test) myFunction(a string) (string, error) {
|
||||
// do something
|
||||
return a + " world", nil
|
||||
}
|
||||
|
||||
func TestLimitRateStruct(t *testing.T) {
|
||||
test := &Test{}
|
||||
test.limitFn = utils.LimitRate(test.myFunction, time.Second)
|
||||
result, _ := test.limitFn("hello")
|
||||
t.Log(result) // Output: hello world
|
||||
result, _ = test.limitFn("hi")
|
||||
t.Log(result) // Output: hi world
|
||||
}
|
||||
|
||||
func myFunctionCtx(ctx context.Context, a int) (int, error) {
|
||||
// do something
|
||||
return a + 1, nil
|
||||
}
|
||||
func TestLimitRateCtx(t *testing.T) {
|
||||
myLimitedFunction := utils.LimitRateCtx(myFunctionCtx, time.Second)
|
||||
result, _ := myLimitedFunction(context.Background(), 1)
|
||||
t.Log(result) // Output: 2
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go func() {
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
cancel()
|
||||
}()
|
||||
result, err := myLimitedFunction(ctx, 2)
|
||||
t.Log(result, err) // Output: 0 context canceled
|
||||
result, _ = myLimitedFunction(context.Background(), 3)
|
||||
t.Log(result) // Output: 4
|
||||
}
|
||||
Loading…
Reference in New Issue