From 99a186d01beebac03756f4bddda92bdb34b2c120 Mon Sep 17 00:00:00 2001 From: aimuz Date: Sun, 8 Jan 2023 16:31:00 +0800 Subject: [PATCH] fix(139): upload failed (#2950) fix: The file size is exceeded and cannot be uploaded fix: File name has special characters, signature fails improve: optimize memory usage Signed-off-by: aimuz Signed-off-by: aimuz --- drivers/139/driver.go | 58 ++++++++++++++++++--------------------- drivers/139/util.go | 7 +++-- internal/driver/driver.go | 20 ++++++++++++++ 3 files changed, 51 insertions(+), 34 deletions(-) diff --git a/drivers/139/driver.go b/drivers/139/driver.go index 4f3650c8..b765b2ba 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -1,11 +1,9 @@ package _139 import ( - "bytes" "context" "fmt" "io" - "math" "net/http" "strconv" @@ -229,10 +227,10 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr "manualRename": 2, "operation": 0, "fileCount": 1, - "totalSize": stream.GetSize(), + "totalSize": 0, // 去除上传大小限制 "uploadContentList": []base.Json{{ "contentName": stream.GetName(), - "contentSize": stream.GetSize(), + "contentSize": 0, // 去除上传大小限制 // "digest": "5a3231986ce7a6b46e408612d385bafa" }}, "parentCatalogID": dstDir.GetID(), @@ -250,10 +248,10 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr "operation": 0, "path": "", "seqNo": "", - "totalSize": stream.GetSize(), + "totalSize": 0, "uploadContentList": []base.Json{{ "contentName": stream.GetName(), - "contentSize": stream.GetSize(), + "contentSize": 0, // "digest": "5a3231986ce7a6b46e408612d385bafa" }}, }) @@ -265,51 +263,47 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr if err != nil { return err } + + // Progress + p := driver.NewProgress(stream.GetSize(), up) + var Default int64 = 104857600 - part := int(math.Ceil(float64(stream.GetSize()) / float64(Default))) - var start int64 = 0 - for i := 0; i < part; i++ { + part := (stream.GetSize() + Default - 1) / Default + for i := int64(0); i < part; i++ { if utils.IsCanceled(ctx) { return ctx.Err() } + + start := i * Default byteSize := stream.GetSize() - start if byteSize > Default { byteSize = Default } - byteData := make([]byte, byteSize) - _, err = io.ReadFull(stream, byteData) - if err != nil { - return err - } - req, err := http.NewRequest("POST", resp.Data.UploadResult.RedirectionURL, bytes.NewBuffer(byteData)) + + limitReader := io.LimitReader(stream, byteSize) + // Update Progress + r := io.TeeReader(limitReader, p) + req, err := http.NewRequest("POST", resp.Data.UploadResult.RedirectionURL, r) if err != nil { return err } + req = req.WithContext(ctx) - headers := map[string]string{ - "Accept": "*/*", - "Content-Type": "text/plain;name=" + unicode(stream.GetName()), - "contentSize": strconv.FormatInt(stream.GetSize(), 10), - "range": fmt.Sprintf("bytes=%d-%d", start, start+byteSize-1), - "content-length": strconv.FormatInt(byteSize, 10), - "uploadtaskID": resp.Data.UploadResult.UploadTaskID, - "rangeType": "0", - "Referer": "https://yun.139.com/", - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36 Edg/95.0.1020.44", - "x-SvcType": "1", - } - for k, v := range headers { - req.Header.Set(k, v) - } + req.Header.Set("Content-Type", "text/plain;name="+unicode(stream.GetName())) + req.Header.Set("contentSize", strconv.FormatInt(stream.GetSize(), 10)) + req.Header.Set("range", fmt.Sprintf("bytes=%d-%d", start, start+byteSize-1)) + req.Header.Set("uploadtaskID", resp.Data.UploadResult.UploadTaskID) + req.Header.Set("rangeType", "0") + req.ContentLength = byteSize + res, err := base.HttpClient.Do(req) if err != nil { return err } log.Debugf("%+v", res) res.Body.Close() - start += byteSize - up(i * 100 / part) } + return nil } diff --git a/drivers/139/util.go b/drivers/139/util.go index 79622147..30e997b0 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -28,12 +28,15 @@ func (d *Yun139) isFamily() bool { func encodeURIComponent(str string) string { r := url.QueryEscape(str) r = strings.Replace(r, "+", "%20", -1) + r = strings.Replace(r, "%21", "!", -1) + r = strings.Replace(r, "%27", "'", -1) + r = strings.Replace(r, "%28", "(", -1) + r = strings.Replace(r, "%29", ")", -1) + r = strings.Replace(r, "%2A", "*", -1) return r } func calSign(body, ts, randStr string) string { - body = strings.ReplaceAll(body, "\n", "") - body = strings.ReplaceAll(body, " ", "") body = encodeURIComponent(body) strs := strings.Split(body, "") sort.Strings(strs) diff --git a/internal/driver/driver.go b/internal/driver/driver.go index c82ff3bc..dcac1205 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -105,3 +105,23 @@ type PutResult interface { } type UpdateProgress func(percentage int) + +type Progress struct { + Total int64 + Done int64 + up UpdateProgress +} + +func (p *Progress) Write(b []byte) (n int, err error) { + n = len(b) + p.Done += int64(n) + p.up(int(float64(p.Done) / float64(p.Total) * 100)) + return +} + +func NewProgress(total int64, up UpdateProgress) *Progress { + return &Progress{ + Total: total, + up: up, + } +}