From b797f4302c5fd264d8da4ac468cc02b39f7586bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=AE=E5=87=89?= Date: Thu, 27 Jan 2022 23:48:29 +0800 Subject: [PATCH] :sparkler: use tempFile to cal md5 --- bootstrap/conf.go | 5 +++++ drivers/123/driver.go | 39 ++++++++++++++++++++++++++++++++++-- drivers/mediatrack/driver.go | 29 ++++++++++++++++++++++----- drivers/native/driver.go | 10 +++++++++ drivers/operate/operate.go | 2 ++ 5 files changed, 78 insertions(+), 7 deletions(-) diff --git a/bootstrap/conf.go b/bootstrap/conf.go index ba6596a9..a254b22a 100644 --- a/bootstrap/conf.go +++ b/bootstrap/conf.go @@ -5,6 +5,7 @@ import ( "github.com/Xhofe/alist/utils" log "github.com/sirupsen/logrus" "io/ioutil" + "os" ) // InitConf init config @@ -41,4 +42,8 @@ func InitConf() { if err != nil { log.Fatalf("update config struct error: %s", err.Error()) } + err = os.MkdirAll("data/temp", 0700) + if err != nil { + log.Fatalf("create temp dir error: %s", err.Error()) + } } diff --git a/drivers/123/driver.go b/drivers/123/driver.go index 97b4dca0..dc6646a0 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -1,6 +1,8 @@ package _23 import ( + "crypto/md5" + "encoding/hex" "fmt" "github.com/Xhofe/alist/conf" "github.com/Xhofe/alist/drivers/base" @@ -12,7 +14,10 @@ import ( "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" + "io" + "io/ioutil" "net/url" + "os" "path/filepath" "strconv" ) @@ -294,10 +299,37 @@ func (driver Pan123) Upload(file *model.FileStream, account *model.Account) erro return base.ErrNotFolder } parentFileId, _ := strconv.Atoi(parentFile.Id) + tempFile, err := ioutil.TempFile("data/temp", "file-*") + if err != nil { + return err + } + defer func() { + _ = tempFile.Close() + _ = os.Remove(tempFile.Name()) + }() + _, err = io.Copy(tempFile, file) + if err != nil { + return err + } + _, err = tempFile.Seek(0, io.SeekStart) + if err != nil { + return err + } + h := md5.New() + _, err = io.Copy(h, tempFile) + if err != nil { + return err + } + etag := hex.EncodeToString(h.Sum(nil)) + log.Debugln("md5:", etag) + _, err = tempFile.Seek(0, io.SeekStart) + if err != nil { + return err + } data := base.Json{ "driveId": 0, "duplicate": true, - "etag": "836aae6cac845e17fce51919594737d0", //maybe file's md5 + "etag": etag, "fileName": file.GetFileName(), "parentFileId": parentFileId, "size": file.GetSize(), @@ -310,6 +342,9 @@ func (driver Pan123) Upload(file *model.FileStream, account *model.Account) erro if err != nil { return err } + if resp.Data.Key == "" { + return nil + } cfg := &aws.Config{ Credentials: credentials.NewStaticCredentials(resp.Data.AccessKeyId, resp.Data.SecretAccessKey, resp.Data.SessionToken), Region: aws.String("123pan"), @@ -324,7 +359,7 @@ func (driver Pan123) Upload(file *model.FileStream, account *model.Account) erro input := &s3manager.UploadInput{ Bucket: &resp.Data.Bucket, Key: &resp.Data.Key, - Body: file, + Body: tempFile, } _, err = uploader.Upload(input) if err != nil { diff --git a/drivers/mediatrack/driver.go b/drivers/mediatrack/driver.go index 94db8c55..8f3245db 100644 --- a/drivers/mediatrack/driver.go +++ b/drivers/mediatrack/driver.go @@ -1,7 +1,6 @@ package mediatrack import ( - "bytes" "crypto/md5" "encoding/hex" "fmt" @@ -18,6 +17,8 @@ import ( jsoniter "github.com/json-iterator/go" log "github.com/sirupsen/logrus" "io" + "io/ioutil" + "os" "path/filepath" ) @@ -265,21 +266,39 @@ func (driver MediaTrack) Upload(file *model.FileStream, account *model.Account) if err != nil { return err } - var buf bytes.Buffer - read := io.TeeReader(file, &buf) + tempFile, err := ioutil.TempFile("data/temp", "file-*") + if err != nil { + return err + } + defer func() { + _ = tempFile.Close() + _ = os.Remove(tempFile.Name()) + }() + _, err = io.Copy(tempFile, file) + if err != nil { + return err + } + _, err = tempFile.Seek(0, io.SeekStart) + if err != nil { + return err + } uploader := s3manager.NewUploader(s) input := &s3manager.UploadInput{ Bucket: &resp.Data.Bucket, Key: &resp.Data.Object, - Body: read, + Body: tempFile, } _, err = uploader.Upload(input) if err != nil { return err } url := fmt.Sprintf("https://jayce.api.mediatrack.cn/v3/assets/%s/children", parentFile.Id) + _, err = tempFile.Seek(0, io.SeekStart) + if err != nil { + return err + } h := md5.New() - _, err = io.Copy(h, &buf) + _, err = io.Copy(h, tempFile) if err != nil { return err } diff --git a/drivers/native/driver.go b/drivers/native/driver.go index d8ea1fd0..572bbbf9 100644 --- a/drivers/native/driver.go +++ b/drivers/native/driver.go @@ -249,6 +249,16 @@ func (driver Native) Upload(file *model.FileStream, account *model.Account) erro defer func() { _ = out.Close() }() + //var buf bytes.Buffer + //reader := io.TeeReader(file, &buf) + //h := md5.New() + //_, err = io.Copy(h, reader) + //if err != nil { + // return err + //} + //hash := hex.EncodeToString(h.Sum(nil)) + //log.Debugln("md5:", hash) + //_, err = io.Copy(out, &buf) _, err = io.Copy(out, file) return err } diff --git a/drivers/operate/operate.go b/drivers/operate/operate.go index 67bc93b8..36f2af1c 100644 --- a/drivers/operate/operate.go +++ b/drivers/operate/operate.go @@ -5,6 +5,7 @@ import ( "github.com/Xhofe/alist/model" "github.com/Xhofe/alist/utils" log "github.com/sirupsen/logrus" + "runtime/debug" ) func MakeDir(driver base.Driver, account *model.Account, path string, clearCache bool) error { @@ -78,5 +79,6 @@ func Upload(driver base.Driver, account *model.Account, file *model.FileStream, if err != nil { log.Errorf("upload error: %s", err.Error()) } + debug.FreeOSMemory() return err }