fix(123): remove stream upload method (close #4772)

pull/4798/head
Andy Hsu 2023-07-14 19:12:18 +08:00
parent fce872bc1b
commit 5dd73d80d8
2 changed files with 18 additions and 38 deletions

View File

@ -1,11 +1,9 @@
package _123 package _123
import ( import (
"bytes"
"context" "context"
"crypto/md5" "crypto/md5"
"encoding/base64" "encoding/base64"
"encoding/binary"
"encoding/hex" "encoding/hex"
"fmt" "fmt"
"io" "io"
@ -183,40 +181,23 @@ func (d *Pan123) Remove(ctx context.Context, obj model.Obj) error {
} }
func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error {
const DEFAULT int64 = 10485760 // const DEFAULT int64 = 10485760
var uploadFile io.Reader
h := md5.New() h := md5.New()
if d.StreamUpload && stream.GetSize() > DEFAULT { // need to calculate md5 of the full content
// 只计算前10MIB tempFile, err := utils.CreateTempFile(stream.GetReadCloser())
buf := bytes.NewBuffer(make([]byte, 0, DEFAULT)) if err != nil {
if n, err := io.CopyN(io.MultiWriter(buf, h), stream, DEFAULT); err != io.EOF && n == 0 { return err
return err }
} defer func() {
// 增加额外参数防止MD5碰撞 _ = tempFile.Close()
h.Write([]byte(stream.GetName())) _ = os.Remove(tempFile.Name())
num := make([]byte, 8) }()
binary.BigEndian.PutUint64(num, uint64(stream.GetSize())) if _, err = io.Copy(h, tempFile); err != nil {
h.Write(num) return err
// 拼装 }
uploadFile = io.MultiReader(buf, stream) _, err = tempFile.Seek(0, io.SeekStart)
} else { if err != nil {
// 计算完整文件MD5 return err
tempFile, err := utils.CreateTempFile(stream.GetReadCloser())
if err != nil {
return err
}
defer func() {
_ = tempFile.Close()
_ = os.Remove(tempFile.Name())
}()
if _, err = io.Copy(h, tempFile); err != nil {
return err
}
_, err = tempFile.Seek(0, io.SeekStart)
if err != nil {
return err
}
uploadFile = tempFile
} }
etag := hex.EncodeToString(h.Sum(nil)) etag := hex.EncodeToString(h.Sum(nil))
data := base.Json{ data := base.Json{
@ -240,7 +221,7 @@ func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
return nil return nil
} }
if resp.Data.AccessKeyId == "" || resp.Data.SecretAccessKey == "" || resp.Data.SessionToken == "" { if resp.Data.AccessKeyId == "" || resp.Data.SecretAccessKey == "" || resp.Data.SessionToken == "" {
err = d.newUpload(ctx, &resp, stream, uploadFile, up) err = d.newUpload(ctx, &resp, stream, tempFile, up)
return err return err
} else { } else {
cfg := &aws.Config{ cfg := &aws.Config{
@ -257,7 +238,7 @@ func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
input := &s3manager.UploadInput{ input := &s3manager.UploadInput{
Bucket: &resp.Data.Bucket, Bucket: &resp.Data.Bucket,
Key: &resp.Data.Key, Key: &resp.Data.Key,
Body: uploadFile, Body: tempFile,
} }
_, err = uploader.UploadWithContext(ctx, input) _, err = uploader.UploadWithContext(ctx, input)
} }

View File

@ -11,7 +11,6 @@ type Addition struct {
driver.RootID driver.RootID
OrderBy string `json:"order_by" type:"select" options:"file_name,size,update_at" default:"file_name"` OrderBy string `json:"order_by" type:"select" options:"file_name,size,update_at" default:"file_name"`
OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"`
StreamUpload bool `json:"stream_upload"`
AccessToken string AccessToken string
} }