From f2b2728be71ebb6658f5ed2a6493a37305929c7e Mon Sep 17 00:00:00 2001 From: foxxorcat Date: Mon, 16 May 2022 09:48:33 +0800 Subject: [PATCH 1/3] fix(123,189pc,alidriver,xunlei):tempfile remove --- drivers/123/driver.go | 6 ++++-- drivers/189pc/driver.go | 19 ++++++++++++------- drivers/alidrive/driver.go | 6 ++++-- drivers/xunlei/driver.go | 6 ++++-- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/drivers/123/driver.go b/drivers/123/driver.go index 43cf7cf7..50fc39c3 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -306,8 +306,10 @@ func (driver Pan123) Upload(file *model.FileStream, account *model.Account) erro if err != nil { return err } - defer tempFile.Close() - defer os.Remove(tempFile.Name()) + defer func() { + _ = tempFile.Close() + _ = os.Remove(tempFile.Name()) + }() h := md5.New() if _, err = io.Copy(io.MultiWriter(tempFile, h), file); err != nil { return err diff --git a/drivers/189pc/driver.go b/drivers/189pc/driver.go index e8ea2e1c..29a1f5ab 100644 --- a/drivers/189pc/driver.go +++ b/drivers/189pc/driver.go @@ -650,9 +650,10 @@ func (driver Cloud189) FastUpload(file *model.FileStream, parentFile *model.File if err != nil { return err } - defer tempFile.Close() - defer os.Remove(tempFile.Name()) - + defer func() { + _ = tempFile.Close() + _ = os.Remove(tempFile.Name()) + }() // 初始化上传 state := GetState(account) @@ -756,8 +757,10 @@ func (driver Cloud189) uploadFamily(file *model.FileStream, parentFile *model.Fi return err } - defer tempFile.Close() - defer os.Remove(tempFile.Name()) + defer func() { + _ = tempFile.Close() + _ = os.Remove(tempFile.Name()) + }() fileMd5 := md5.New() if _, err = io.Copy(io.MultiWriter(fileMd5, tempFile), file); err != nil { @@ -805,8 +808,10 @@ func (driver Cloud189) uploadPerson(file *model.FileStream, parentFile *model.Fi return err } - defer tempFile.Close() - defer os.Remove(tempFile.Name()) + defer func() { + _ = tempFile.Close() + _ = os.Remove(tempFile.Name()) + }() fileMd5 := md5.New() if _, err = io.Copy(io.MultiWriter(fileMd5, tempFile), file); err != nil { diff --git a/drivers/alidrive/driver.go b/drivers/alidrive/driver.go index 013d9a49..be964c90 100644 --- a/drivers/alidrive/driver.go +++ b/drivers/alidrive/driver.go @@ -467,8 +467,10 @@ func (driver AliDrive) Upload(file *model.FileStream, account *model.Account) er return err } - defer tempFile.Close() - defer os.Remove(tempFile.Name()) + defer func() { + _ = tempFile.Close() + _ = os.Remove(tempFile.Name()) + }() delete(reqBody, "pre_hash") h := sha1.New() diff --git a/drivers/xunlei/driver.go b/drivers/xunlei/driver.go index fa9db1cc..c2d461ed 100644 --- a/drivers/xunlei/driver.go +++ b/drivers/xunlei/driver.go @@ -363,8 +363,10 @@ func (driver XunLeiCloud) Upload(file *model.FileStream, account *model.Account) return err } - defer tempFile.Close() - defer os.Remove(tempFile.Name()) + defer func() { + _ = tempFile.Close() + _ = os.Remove(tempFile.Name()) + }() gcid, err := getGcid(io.TeeReader(file, tempFile), int64(file.Size)) if err != nil { From 6a66e39d5bd99f9d8cd90ba955e1de59f76167ed Mon Sep 17 00:00:00 2001 From: foxxorcat Date: Mon, 16 May 2022 21:03:00 +0800 Subject: [PATCH 2/3] feat(123):add io stream upload --- drivers/123/driver.go | 61 +++++++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/drivers/123/driver.go b/drivers/123/driver.go index 50fc39c3..47c8b21b 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -1,7 +1,9 @@ package _23 import ( + "bytes" "crypto/md5" + "encoding/binary" "encoding/hex" "fmt" "io" @@ -68,6 +70,12 @@ func (driver Pan123) Items() []base.Item { Required: true, Default: "asc", }, + { + Name: "bool_1", + Label: "stream upload", + Type: base.TypeBool, + Description: "io stream upload (test)", + }, } } @@ -302,25 +310,44 @@ func (driver Pan123) Upload(file *model.FileStream, account *model.Account) erro return base.ErrNotFolder } - tempFile, err := ioutil.TempFile(conf.Conf.TempDir, "file-*") - if err != nil { - return err - } - defer func() { - _ = tempFile.Close() - _ = os.Remove(tempFile.Name()) - }() + const DEFAULT int64 = 10485760 + var uploadFile io.Reader h := md5.New() - if _, err = io.Copy(io.MultiWriter(tempFile, h), file); err != nil { - return err + if account.Bool1 && file.GetSize() > uint64(DEFAULT) { + // 只计算前10MIB + buf := bytes.NewBuffer(make([]byte, 0, DEFAULT)) + if n, err := io.CopyN(io.MultiWriter(buf, h), file, DEFAULT); err != io.EOF && n == 0 { + return err + } + // 增加额外参数防止MD5碰撞 + h.Write([]byte(file.Name)) + num := make([]byte, 8) + binary.BigEndian.PutUint64(num, file.Size) + h.Write(num) + // 拼装 + uploadFile = io.MultiReader(buf, file) + } else { + // 计算完整文件MD5 + tempFile, err := ioutil.TempFile(conf.Conf.TempDir, "file-*") + if err != nil { + return err + } + defer func() { + _ = tempFile.Close() + _ = os.Remove(tempFile.Name()) + }() + + if _, err = io.Copy(io.MultiWriter(tempFile, h), file); err != nil { + return err + } + + _, err = tempFile.Seek(0, io.SeekStart) + if err != nil { + return err + } + uploadFile = tempFile } etag := hex.EncodeToString(h.Sum(nil)) - - _, err = tempFile.Seek(0, io.SeekStart) - if err != nil { - return err - } - data := base.Json{ "driveId": 0, "duplicate": 2, // 2->覆盖 1->重命名 0->默认 @@ -354,7 +381,7 @@ func (driver Pan123) Upload(file *model.FileStream, account *model.Account) erro input := &s3manager.UploadInput{ Bucket: &resp.Data.Bucket, Key: &resp.Data.Key, - Body: tempFile, + Body: uploadFile, } _, err = uploader.Upload(input) if err != nil { From 04e04a1aa61216f5aa7c55061906a3ec7b0dd24d Mon Sep 17 00:00:00 2001 From: foxxorcat Date: Mon, 16 May 2022 23:33:12 +0800 Subject: [PATCH 3/3] fix(189pc): delete user-agent for upload --- drivers/189pc/driver.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/189pc/driver.go b/drivers/189pc/driver.go index 29a1f5ab..5337f79d 100644 --- a/drivers/189pc/driver.go +++ b/drivers/189pc/driver.go @@ -607,7 +607,7 @@ func (driver Cloud189) CommonUpload(file *model.FileStream, parentFile *model.Fi uploadData := uploadUrl.UploadUrls[fmt.Sprint("partNumber_", i)] req, _ := http.NewRequest(http.MethodPut, uploadData.RequestURL, byteData) - req.Header.Del("User-Agent") + req.Header.Set("User-Agent", "") for k, v := range ParseHttpHeader(uploadData.RequestHeader) { req.Header.Set(k, v) } @@ -719,7 +719,7 @@ func (driver Cloud189) FastUpload(file *model.FileStream, parentFile *model.File for i := 1; i <= count; i++ { uploadData := uploadUrls.UploadUrls[fmt.Sprint("partNumber_", i)] req, _ := http.NewRequest(http.MethodPut, uploadData.RequestURL, io.NewSectionReader(tempFile, int64(i-1)*DEFAULT, DEFAULT)) - req.Header.Del("User-Agent") + req.Header.Set("User-Agent", "") for k, v := range ParseHttpHeader(uploadData.RequestHeader) { req.Header.Set(k, v) }