From 6c38c5972da06aa2f5b0badaafd3f5fe6683fca9 Mon Sep 17 00:00:00 2001 From: Jason-Fly <869914918@qq.com> Date: Sat, 16 Nov 2024 13:18:49 +0800 Subject: [PATCH] fix(terabox): big file upload issue (#7498 close #7490) --- drivers/terabox/driver.go | 122 ++++++++++++++++++++------------------ drivers/terabox/types.go | 4 ++ drivers/terabox/util.go | 21 +++++++ 3 files changed, 88 insertions(+), 59 deletions(-) diff --git a/drivers/terabox/driver.go b/drivers/terabox/driver.go index 11db351b..362de69e 100644 --- a/drivers/terabox/driver.go +++ b/drivers/terabox/driver.go @@ -10,8 +10,6 @@ import ( "math" stdpath "path" "strconv" - "strings" - "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/pkg/utils" @@ -24,9 +22,9 @@ import ( type Terabox struct { model.Storage Addition - JsToken string + JsToken string url_domain_prefix string - base_url string + base_url string } func (d *Terabox) Config() driver.Config { @@ -145,52 +143,24 @@ func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt } log.Debugln(locateupload_resp) - tempFile, err := stream.CacheFullInTempFile() - if err != nil { - return err - } - var Default int64 = 4 * 1024 * 1024 - defaultByteData := make([]byte, Default) - count := int(math.Ceil(float64(stream.GetSize()) / float64(Default))) - // cal md5 - h1 := md5.New() - h2 := md5.New() - block_list := make([]string, 0) - left := stream.GetSize() - for i := 0; i < count; i++ { - byteSize := Default - var byteData []byte - if left < Default { - byteSize = left - byteData = make([]byte, byteSize) - } else { - byteData = defaultByteData - } - left -= byteSize - _, err = io.ReadFull(tempFile, byteData) - if err != nil { - return err - } - h1.Write(byteData) - h2.Write(byteData) - block_list = append(block_list, fmt.Sprintf("\"%s\"", hex.EncodeToString(h2.Sum(nil)))) - h2.Reset() - } - - _, err = tempFile.Seek(0, io.SeekStart) - if err != nil { - return err - } - + // precreate file rawPath := stdpath.Join(dstDir.GetPath(), stream.GetName()) path := encodeURIComponent(rawPath) - block_list_str := fmt.Sprintf("[%s]", strings.Join(block_list, ",")) + + var precreateBlockListStr string + if stream.GetSize() > initialChunkSize { + precreateBlockListStr = `["5910a591dd8fc18c32a8f3df4fdc1761","a5fc157d78e6ad1c7e114b056c92821e"]` + } else { + precreateBlockListStr = `["5910a591dd8fc18c32a8f3df4fdc1761"]` + } + data := map[string]string{ - "path": rawPath, - "autoinit": "1", - "target_path": dstDir.GetPath(), - "block_list": block_list_str, - "local_mtime": strconv.FormatInt(time.Now().Unix(), 10), + "path": rawPath, + "autoinit": "1", + "target_path": dstDir.GetPath(), + "block_list": precreateBlockListStr, + "local_mtime": strconv.FormatInt(stream.ModTime().Unix(), 10), + "file_limit_switch_v34": "true", } var precreateResp PrecreateResp log.Debugln(data) @@ -206,6 +176,13 @@ func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt if precreateResp.ReturnType == 2 { return nil } + + // upload chunks + tempFile, err := stream.CacheFullInTempFile() + if err != nil { + return err + } + params := map[string]string{ "method": "upload", "path": path, @@ -215,24 +192,37 @@ func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt "channel": "dubox", "clienttype": "0", } - left = stream.GetSize() - for i, partseq := range precreateResp.BlockList { + + streamSize := stream.GetSize() + chunkSize := calculateChunkSize(streamSize) + chunkByteData := make([]byte, chunkSize) + count := int(math.Ceil(float64(streamSize) / float64(chunkSize))) + left := streamSize + uploadBlockList := make([]string, 0, count) + h := md5.New() + for partseq := 0; partseq < count; partseq++ { if utils.IsCanceled(ctx) { return ctx.Err() } - byteSize := Default + byteSize := chunkSize var byteData []byte - if left < Default { + if left >= chunkSize { + byteData = chunkByteData + } else { byteSize = left byteData = make([]byte, byteSize) - } else { - byteData = defaultByteData } left -= byteSize _, err = io.ReadFull(tempFile, byteData) if err != nil { return err } + + // calculate md5 + h.Write(byteData) + uploadBlockList = append(uploadBlockList, hex.EncodeToString(h.Sum(nil))) + h.Reset() + u := "https://" + locateupload_resp.Host + "/rest/2.0/pcs/superfile2" params["partseq"] = strconv.Itoa(partseq) res, err := base.RestyClient.R(). @@ -245,25 +235,39 @@ func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt return err } log.Debugln(res.String()) - if len(precreateResp.BlockList) > 0 { - up(float64(i) * 100 / float64(len(precreateResp.BlockList))) + if count > 0 { + up(float64(partseq) * 100 / float64(count)) } } + + // create file params = map[string]string{ "isdir": "0", "rtype": "1", } + + uploadBlockListStr, err := utils.Json.MarshalToString(uploadBlockList) + if err != nil { + return err + } data = map[string]string{ "path": rawPath, "size": strconv.FormatInt(stream.GetSize(), 10), "uploadid": precreateResp.Uploadid, "target_path": dstDir.GetPath(), - "block_list": block_list_str, - "local_mtime": strconv.FormatInt(time.Now().Unix(), 10), + "block_list": uploadBlockListStr, + "local_mtime": strconv.FormatInt(stream.ModTime().Unix(), 10), } - res, err = d.post_form("/api/create", params, data, nil) + var createResp CreateResp + res, err = d.post_form("/api/create", params, data, &createResp) log.Debugln(string(res)) - return err + if err != nil { + return err + } + if createResp.Errno != 0 { + return fmt.Errorf("[terabox] failed to create file, errno: %d", createResp.Errno) + } + return nil } var _ driver.Driver = (*Terabox)(nil) diff --git a/drivers/terabox/types.go b/drivers/terabox/types.go index 8bdbc6fc..f4d50dde 100644 --- a/drivers/terabox/types.go +++ b/drivers/terabox/types.go @@ -99,3 +99,7 @@ type CheckLoginResp struct { type LocateUploadResp struct { Host string `json:"host"` } + +type CreateResp struct { + Errno int `json:"errno"` +} diff --git a/drivers/terabox/util.go b/drivers/terabox/util.go index e0f3d74e..002f80b5 100644 --- a/drivers/terabox/util.go +++ b/drivers/terabox/util.go @@ -17,6 +17,11 @@ import ( log "github.com/sirupsen/logrus" ) +const ( + initialChunkSize int64 = 4 << 20 // 4MB + initialSizeThreshold int64 = 4 << 30 // 4GB +) + func getStrBetween(raw, start, end string) string { regexPattern := fmt.Sprintf(`%s(.*?)%s`, regexp.QuoteMeta(start), regexp.QuoteMeta(end)) regex := regexp.MustCompile(regexPattern) @@ -258,3 +263,19 @@ func encodeURIComponent(str string) string { r = strings.ReplaceAll(r, "+", "%20") return r } + +func calculateChunkSize(streamSize int64) int64 { + chunkSize := initialChunkSize + sizeThreshold := initialSizeThreshold + + if streamSize < chunkSize { + return streamSize + } + + for streamSize > sizeThreshold { + chunkSize <<= 1 + sizeThreshold <<= 1 + } + + return chunkSize +}