From bf2e5768d61f035607e21aa2be0cfee0953b725a Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Sun, 3 Apr 2022 17:56:21 +0800 Subject: [PATCH] feat: add rapid upload switch for 189pc and alidrive (#892) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 189PC增加快传开关 * alidrive增加快传开关 --- drivers/189pc/189.go | 69 +++--- drivers/189pc/driver.go | 415 ++++++++++++++++++++++--------------- drivers/189pc/type.go | 4 +- drivers/189pc/util.go | 51 ++++- drivers/alidrive/driver.go | 47 +++-- 5 files changed, 345 insertions(+), 241 deletions(-) diff --git a/drivers/189pc/189.go b/drivers/189pc/189.go index 9dcf1c68..a22f51be 100644 --- a/drivers/189pc/189.go +++ b/drivers/189pc/189.go @@ -34,7 +34,7 @@ func GetState(account *model.Account) *State { SetHeaders(map[string]string{ "Accept": "application/json;charset=UTF-8", "User-Agent": base.UserAgent, - }), + }).SetTimeout(base.DefaultTimeout), } userStateCache.States[account.Username] = state return state @@ -198,7 +198,7 @@ func (s *State) refreshSession(account *model.Account) error { "accessToken": s.AccessToken, }). SetHeader("X-Request-ID", uuid.NewString()). - Get("https://api.cloud.189.cn/getSessionForPC.action") + Get(API_URL + "/getSessionForPC.action") if err != nil { return err } @@ -223,10 +223,8 @@ func (s *State) refreshSession(account *model.Account) error { return nil } -func (s *State) IsLogin() bool { - _, err := s.Request("GET", API_URL+"/getUserInfo.action", nil, func(r *resty.Request) { - r.SetQueryParams(clientSuffix()) - }, nil) +func (s *State) IsLogin(account *model.Account) bool { + _, err := s.Request(http.MethodGet, API_URL+"/getUserInfo.action", nil, func(r *resty.Request) { r.SetQueryParams(clientSuffix()) }, account) return err == nil } @@ -242,12 +240,12 @@ func (s *State) RefreshSession(account *model.Account) error { return s.refreshSession(account) } -func (s *State) Request(method string, fullUrl string, params url.Values, callback func(*resty.Request), account *model.Account) (*resty.Response, error) { +func (s *State) Request(method string, fullUrl string, params Params, callback func(*resty.Request), account *model.Account) (*resty.Response, error) { s.Lock() dateOfGmt := getHttpDateStr() sessionKey := s.SessionKey sessionSecret := s.SessionSecret - if account != nil && isFamily(account) { + if isFamily(account) { sessionKey = s.FamilySessionKey sessionSecret = s.FamilySessionSecret } @@ -267,25 +265,12 @@ func (s *State) Request(method string, fullUrl string, params url.Values, callba } req.SetHeader("Signature", signatureOfHmac(sessionSecret, sessionKey, method, fullUrl, dateOfGmt, paramsData)) - callback(req) + if callback != nil { + callback(req) + } s.Unlock() - var err error - var res *resty.Response - switch method { - case "GET": - res, err = req.Get(fullUrl) - case "POST": - res, err = req.Post(fullUrl) - case "DELETE": - res, err = req.Delete(fullUrl) - case "PATCH": - res, err = req.Patch(fullUrl) - case "PUT": - res, err = req.Put(fullUrl) - default: - return nil, base.ErrNotSupport - } + res, err := req.Execute(method, fullUrl) if err != nil { return nil, err } @@ -298,6 +283,9 @@ func (s *State) Request(method string, fullUrl string, params url.Values, callba } if erron.Code != "" && erron.Code != "SUCCESS" { if erron.Msg == "" { + if erron.Message == "" { + return nil, fmt.Errorf(res.String()) + } return nil, fmt.Errorf(erron.Message) } return nil, fmt.Errorf(erron.Msg) @@ -306,25 +294,18 @@ func (s *State) Request(method string, fullUrl string, params url.Values, callba return nil, fmt.Errorf(erron.ErrorMsg) } - if account != nil { - switch utils.Json.Get(res.Body(), "res_code").ToInt64() { - case 11, 18: - if err := s.RefreshSession(account); err != nil { - return nil, err - } - return s.Request(method, fullUrl, params, callback, account) - case 0: - if res.StatusCode() == http.StatusOK { - return res, nil - } - fallthrough - default: - return nil, fmt.Errorf(res.String()) + switch utils.Json.Get(res.Body(), "res_code").ToInt64() { + case 11, 18: + if err := s.RefreshSession(account); err != nil { + return nil, err } + return s.Request(method, fullUrl, params, callback, account) + case 0: + if res.StatusCode() == http.StatusOK { + return res, nil + } + return nil, fmt.Errorf(res.String()) + default: + return nil, fmt.Errorf(utils.Json.Get(res.Body(), "res_message").ToString()) } - - if utils.Json.Get(res.Body(), "res_code").ToInt64() != 0 { - return res, fmt.Errorf(utils.Json.Get(res.Body(), "res_message").ToString()) - } - return res, nil } diff --git a/drivers/189pc/driver.go b/drivers/189pc/driver.go index 114fe7e9..c256267b 100644 --- a/drivers/189pc/driver.go +++ b/drivers/189pc/driver.go @@ -1,16 +1,19 @@ package _189 import ( + "bytes" "crypto/md5" + "encoding/base64" "encoding/hex" "fmt" "io" "io/ioutil" + "math" "net/http" + "net/url" "os" "path/filepath" "strings" - "time" "github.com/Xhofe/alist/conf" "github.com/Xhofe/alist/drivers/base" @@ -50,10 +53,9 @@ func (driver Cloud189) Items() []base.Item { Description: "account password", }, { - Name: "root_folder", - Label: "root folder file_id", - Type: base.TypeString, - Required: true, + Name: "root_folder", + Label: "root folder file_id", + Type: base.TypeString, }, { Name: "internal_type", @@ -63,10 +65,9 @@ func (driver Cloud189) Items() []base.Item { Values: "Personal,Family", }, { - Name: "site_id", - Label: "family id", - Type: base.TypeString, - Required: true, + Name: "site_id", + Label: "family id", + Type: base.TypeString, }, { Name: "order_by", @@ -82,6 +83,11 @@ func (driver Cloud189) Items() []base.Item { Values: "true,false", Required: true, }, + { + Name: "bool_1", + Label: "fast upload", + Type: base.TypeBool, + }, } } @@ -92,10 +98,14 @@ func (driver Cloud189) Save(account *model.Account, old *model.Account) error { if !isFamily(account) && account.RootFolder == "" { account.RootFolder = "-11" + account.SiteId = "" + } + if isFamily(account) && account.RootFolder == "-11" { + account.RootFolder = "" } state := GetState(account) - if !state.IsLogin() { + if !state.IsLogin(account) { if err := state.Login(account); err != nil { return err } @@ -121,7 +131,7 @@ func (driver Cloud189) Save(account *model.Account, old *model.Account) error { func (driver Cloud189) getFamilyInfoList(account *model.Account) ([]FamilyInfoResp, error) { var resp FamilyInfoListResp - _, err := GetState(account).Request("GET", API_URL+"/family/manage/getFamilyList.action", nil, func(r *resty.Request) { + _, err := GetState(account).Request(http.MethodGet, API_URL+"/family/manage/getFamilyList.action", nil, func(r *resty.Request) { r.SetQueryParams(clientSuffix()) r.SetResult(&resp) }, account) @@ -179,16 +189,16 @@ func (driver Cloud189) Files(path string, account *model.Account) ([]model.File, client := GetState(account) for pageNum := 1; ; pageNum++ { var resp Cloud189FilesResp - queryparam := map[string]string{ - "folderId": file.Id, - "fileType": "0", - "mediaAttr": "0", - "iconOption": "5", - "pageNum": fmt.Sprint(pageNum), - "pageSize": "130", - } - _, err = client.Request("GET", fullUrl, nil, func(r *resty.Request) { - r.SetQueryParams(clientSuffix()).SetQueryParams(queryparam) + _, err = client.Request(http.MethodGet, fullUrl, nil, func(r *resty.Request) { + r.SetQueryParams(clientSuffix()). + SetQueryParams(map[string]string{ + "folderId": file.Id, + "fileType": "0", + "mediaAttr": "0", + "iconOption": "5", + "pageNum": fmt.Sprint(pageNum), + "pageSize": "130", + }) if isFamily(account) { r.SetQueryParams(map[string]string{ "familyId": account.SiteId, @@ -212,10 +222,6 @@ func (driver Cloud189) Files(path string, account *model.Account) ([]model.File, break } - mustTime := func(str string) *time.Time { - time, _ := http.ParseTime(str) - return &time - } for _, folder := range resp.FileListAO.FolderList { files = append(files, model.File{ Id: fmt.Sprint(folder.ID), @@ -223,7 +229,7 @@ func (driver Cloud189) Files(path string, account *model.Account) ([]model.File, Size: 0, Type: conf.FOLDER, Driver: driver.Config().Name, - UpdatedAt: mustTime(folder.CreateDate), + UpdatedAt: MustParseTime(folder.CreateDate), }) } for _, file := range resp.FileListAO.FileList { @@ -233,7 +239,7 @@ func (driver Cloud189) Files(path string, account *model.Account) ([]model.File, Size: file.Size, Type: utils.GetFileType(filepath.Ext(file.Name)), Driver: driver.Config().Name, - UpdatedAt: mustTime(file.CreateDate), + UpdatedAt: MustParseTime(file.CreateDate), Thumbnail: file.Icon.SmallUrl, }) } @@ -279,7 +285,7 @@ func (driver Cloud189) Link(args base.Args, account *model.Account) (*base.Link, var downloadUrl struct { URL string `json:"fileDownloadUrl"` } - _, err = GetState(account).Request("GET", fullUrl, nil, func(r *resty.Request) { + _, err = GetState(account).Request(http.MethodGet, fullUrl, nil, func(r *resty.Request) { r.SetQueryParams(clientSuffix()).SetQueryParam("fileId", file.Id) if isFamily(account) { r.SetQueryParams(map[string]string{ @@ -324,7 +330,7 @@ func (driver Cloud189) MakeDir(path string, account *model.Account) error { } fullUrl += "/createFolder.action" - _, err = GetState(account).Request("POST", fullUrl, nil, func(r *resty.Request) { + _, err = GetState(account).Request(http.MethodPost, fullUrl, nil, func(r *resty.Request) { r.SetQueryParams(clientSuffix()).SetQueryParams(map[string]string{ "folderName": name, "relativePath": "", @@ -354,7 +360,7 @@ func (driver Cloud189) Move(src string, dst string, account *model.Account) erro return err } - _, err = GetState(account).Request("POST", API_URL+"/batch/createBatchTask.action", nil, func(r *resty.Request) { + _, err = GetState(account).Request(http.MethodPost, API_URL+"/batch/createBatchTask.action", nil, func(r *resty.Request) { r.SetFormData(clientSuffix()).SetFormData(map[string]string{ "type": "MOVE", "taskInfos": string(MustToBytes(utils.Json.Marshal( @@ -390,10 +396,10 @@ func (driver Cloud189) Move(src string, dst string, account *model.Account) erro var queryParam map[string]string fullUrl := API_URL - method := "POST" + method := http.MethodPost if isFamily(account) { fullUrl += "/family/file" - method = "GET" + method = http.MethodGet } if srcFile.IsDir() { fullUrl += "/moveFolder.action" @@ -431,10 +437,10 @@ func (driver Cloud189) Rename(src string, dst string, account *model.Account) er var queryParam map[string]string fullUrl := API_URL - method := "POST" + method := http.MethodPost if isFamily(account) { fullUrl += "/family/file" - method = "GET" + method = http.MethodGet } if srcFile.IsDir() { fullUrl += "/renameFolder.action" @@ -470,7 +476,7 @@ func (driver Cloud189) Copy(src string, dst string, account *model.Account) erro return err } - _, err = GetState(account).Request("POST", API_URL+"/batch/createBatchTask.action", nil, func(r *resty.Request) { + _, err = GetState(account).Request(http.MethodPost, API_URL+"/batch/createBatchTask.action", nil, func(r *resty.Request) { r.SetFormData(clientSuffix()).SetFormData(map[string]string{ "type": "COPY", "taskInfos": string(MustToBytes(utils.Json.Marshal( @@ -500,7 +506,7 @@ func (driver Cloud189) Delete(path string, account *model.Account) error { return err } - _, err = GetState(account).Request("POST", API_URL+"/batch/createBatchTask.action", nil, func(r *resty.Request) { + _, err = GetState(account).Request(http.MethodPost, API_URL+"/batch/createBatchTask.action", nil, func(r *resty.Request) { r.SetFormData(clientSuffix()).SetFormData(map[string]string{ "type": "DELETE", "taskInfos": string(MustToBytes(utils.Json.Marshal( @@ -535,12 +541,209 @@ func (driver Cloud189) Upload(file *model.FileStream, account *model.Account) er return base.ErrNotFolder } - if isFamily(account) { - return driver.uploadFamily(file, parentFile, account) + if account.Bool1 { + return driver.FastUpload(file, parentFile, account) } - return driver.uploadPerson(file, parentFile, account) + return driver.CommonUpload(file, parentFile, account) + /* + if isFamily(account) { + return driver.uploadFamily(file, parentFile, account) + } + return driver.uploadPerson(file, parentFile, account) + */ } +func (driver Cloud189) CommonUpload(file *model.FileStream, parentFile *model.File, account *model.Account) error { + // 初始化上传 + state := GetState(account) + const DEFAULT int64 = 10485760 + count := int(math.Ceil(float64(file.Size) / float64(DEFAULT))) + + params := Params{ + "parentFolderId": parentFile.Id, + "fileName": url.PathEscape(file.Name), + "fileSize": fmt.Sprint(file.Size), + "sliceSize": fmt.Sprint(DEFAULT), + "lazyCheck": "1", + } + + fullUrl := UPLOAD_URL + if isFamily(account) { + params.Set("familyId", account.SiteId) + fullUrl += "/family" + } else { + //params.Set("extend", `{"opScene":"1","relativepath":"","rootfolderid":""}`) + fullUrl += "/person" + } + + var initMultiUpload InitMultiUploadResp + _, err := state.Request(http.MethodGet, fullUrl+"/initMultiUpload", params, func(r *resty.Request) { r.SetQueryParams(clientSuffix()).SetResult(&initMultiUpload) }, account) + if err != nil { + return err + } + + fileMd5 := md5.New() + silceMd5 := md5.New() + silceMd5Hexs := make([]string, 0, count) + byteData := bytes.NewBuffer(make([]byte, DEFAULT)) + for i := 1; i <= count; i++ { + byteData.Reset() + silceMd5.Reset() + if n, err := io.CopyN(io.MultiWriter(fileMd5, silceMd5, byteData), file, DEFAULT); err != io.EOF && n == 0 { + return err + } + md5Bytes := silceMd5.Sum(nil) + silceMd5Hexs = append(silceMd5Hexs, strings.ToUpper(hex.EncodeToString(md5Bytes))) + silceMd5Base64 := base64.StdEncoding.EncodeToString(md5Bytes) + + var uploadUrl UploadUrlsResp + _, err = state.Request(http.MethodGet, fullUrl+"/getMultiUploadUrls", + Params{"partInfo": fmt.Sprintf("%d-%s", i, silceMd5Base64), "uploadFileId": initMultiUpload.Data.UploadFileID}, + func(r *resty.Request) { r.SetQueryParams(clientSuffix()).SetResult(&uploadUrl) }, + account) + if err != nil { + return err + } + + uploadData := uploadUrl.UploadUrls[fmt.Sprint("partNumber_", i)] + req, _ := http.NewRequest(http.MethodPut, uploadData.RequestURL, byteData) + for k, v := range ParseHttpHeader(uploadData.RequestHeader) { + req.Header.Set(k, v) + } + for k, v := range clientSuffix() { + req.URL.RawQuery += fmt.Sprintf("&%s=%s", k, v) + } + r, err := base.HttpClient.Do(req) + if err != nil { + return err + } + if r.StatusCode != http.StatusOK { + data, _ := io.ReadAll(r.Body) + return fmt.Errorf(string(data)) + } + } + + fileMd5Hex := strings.ToUpper(hex.EncodeToString(fileMd5.Sum(nil))) + sliceMd5Hex := fileMd5Hex + if int64(file.Size) > DEFAULT { + sliceMd5Hex = strings.ToUpper(utils.GetMD5Encode(strings.Join(silceMd5Hexs, "\n"))) + } + + _, err = state.Request(http.MethodGet, fullUrl+"/commitMultiUploadFile", + Params{ + "uploadFileId": initMultiUpload.Data.UploadFileID, + "fileMd5": fileMd5Hex, + "sliceMd5": sliceMd5Hex, + "lazyCheck": "1", + "isLog": "0", + "opertype": "3", + }, + func(r *resty.Request) { r.SetQueryParams(clientSuffix()) }, account) + return err +} + +func (driver Cloud189) FastUpload(file *model.FileStream, parentFile *model.File, account *model.Account) error { + tempFile, err := ioutil.TempFile(conf.Conf.TempDir, "file-*") + if err != nil { + return err + } + defer tempFile.Close() + defer os.Remove(tempFile.Name()) + + // 初始化上传 + state := GetState(account) + + const DEFAULT int64 = 10485760 + count := int(math.Ceil(float64(file.Size) / float64(DEFAULT))) + + // 优先计算所需信息 + fileMd5 := md5.New() + silceMd5 := md5.New() + silceMd5Hexs := make([]string, 0, count) + silceMd5Base64s := make([]string, 0, count) + for i := 1; i <= count; i++ { + silceMd5.Reset() + if n, err := io.CopyN(io.MultiWriter(fileMd5, silceMd5, tempFile), file, DEFAULT); err != nil && n == 0 { + return err + } + md5Byte := silceMd5.Sum(nil) + silceMd5Hexs = append(silceMd5Hexs, strings.ToUpper(hex.EncodeToString(md5Byte))) + silceMd5Base64s = append(silceMd5Base64s, fmt.Sprint(i, "-", base64.StdEncoding.EncodeToString(md5Byte))) + } + fileMd5Hex := strings.ToUpper(hex.EncodeToString(fileMd5.Sum(nil))) + sliceMd5Hex := fileMd5Hex + if int64(file.Size) > DEFAULT { + sliceMd5Hex = strings.ToUpper(utils.GetMD5Encode(strings.Join(silceMd5Hexs, "\n"))) + } + + params := Params{ + "parentFolderId": parentFile.Id, + "fileName": url.PathEscape(file.Name), + "fileSize": fmt.Sprint(file.Size), + "fileMd5": fileMd5Hex, + "sliceSize": fmt.Sprint(DEFAULT), + "sliceMd5": sliceMd5Hex, + } + + fullUrl := UPLOAD_URL + if isFamily(account) { + params.Set("familyId", account.SiteId) + fullUrl += "/family" + } else { + //params.Set("extend", `{"opScene":"1","relativepath":"","rootfolderid":""}`) + fullUrl += "/person" + } + + var uploadInfo InitMultiUploadResp + _, err = state.Request(http.MethodGet, fullUrl+"/initMultiUpload", params, func(r *resty.Request) { r.SetQueryParams(clientSuffix()).SetResult(&uploadInfo) }, account) + if err != nil { + return err + } + + if uploadInfo.Data.FileDataExists != 1 { + var uploadUrls UploadUrlsResp + _, err := state.Request(http.MethodGet, fullUrl+"/getMultiUploadUrls", + Params{ + "uploadFileId": uploadInfo.Data.UploadFileID, + "partInfo": strings.Join(silceMd5Base64s, ","), + }, + func(r *resty.Request) { r.SetQueryParams(clientSuffix()).SetResult(&uploadUrls) }, + account) + if err != nil { + return err + } + for i := 1; i <= count; i++ { + uploadData := uploadUrls.UploadUrls[fmt.Sprint("partNumber_", i)] + req, _ := http.NewRequest(http.MethodPut, uploadData.RequestURL, io.NewSectionReader(tempFile, int64(i-1)*DEFAULT, DEFAULT)) + for k, v := range ParseHttpHeader(uploadData.RequestHeader) { + req.Header.Set(k, v) + } + for k, v := range clientSuffix() { + req.URL.RawQuery += fmt.Sprintf("&%s=%s", k, v) + } + r, err := base.HttpClient.Do(req) + if err != nil { + return err + } + if r.StatusCode != http.StatusOK { + data, _ := io.ReadAll(r.Body) + return fmt.Errorf(string(data)) + } + } + } + + _, err = state.Request(http.MethodGet, fullUrl+"/commitMultiUploadFile", + Params{ + "uploadFileId": uploadInfo.Data.UploadFileID, + "isLog": "0", + "opertype": "3", + }, + func(r *resty.Request) { r.SetQueryParams(clientSuffix()) }, + account) + return err +} + +/* func (driver Cloud189) uploadFamily(file *model.FileStream, parentFile *model.File, account *model.Account) error { tempFile, err := ioutil.TempFile(conf.Conf.TempDir, "file-*") if err != nil { @@ -557,7 +760,7 @@ func (driver Cloud189) uploadFamily(file *model.FileStream, parentFile *model.Fi client := GetState(account) var createUpload CreateUploadFileResult - _, err = client.Request("GET", API_URL+"/family/file/createFamilyFile.action", nil, func(r *resty.Request) { + _, err = client.Request(http.MethodGet, API_URL+"/family/file/createFamilyFile.action", nil, func(r *resty.Request) { r.SetQueryParams(map[string]string{ "fileMd5": hex.EncodeToString(fileMd5.Sum(nil)), "fileName": file.Name, @@ -579,7 +782,7 @@ func (driver Cloud189) uploadFamily(file *model.FileStream, parentFile *model.Fi } } - _, err = client.Request("GET", createUpload.FileCommitUrl, nil, func(r *resty.Request) { + _, err = client.Request(http.MethodGet, createUpload.FileCommitUrl, nil, func(r *resty.Request) { r.SetQueryParams(clientSuffix()) r.SetHeaders(map[string]string{ "FamilyId": account.SiteId, @@ -606,7 +809,7 @@ func (driver Cloud189) uploadPerson(file *model.FileStream, parentFile *model.Fi client := GetState(account) var createUpload CreateUploadFileResult - _, err = client.Request("POST", API_URL+"/createUploadFile.action", nil, func(r *resty.Request) { + _, err = client.Request(http.MethodPost, API_URL+"/createUploadFile.action", nil, func(r *resty.Request) { r.SetQueryParams(clientSuffix()) r.SetFormData(clientSuffix()).SetFormData(map[string]string{ "parentFolderId": parentFile.Id, @@ -634,7 +837,7 @@ func (driver Cloud189) uploadPerson(file *model.FileStream, parentFile *model.Fi } } - _, err = client.Request("POST", createUpload.FileCommitUrl, nil, func(r *resty.Request) { + _, err = client.Request(http.MethodPost, createUpload.FileCommitUrl, nil, func(r *resty.Request) { r.SetQueryParams(clientSuffix()) r.SetFormData(map[string]string{ "uploadFileId": fmt.Sprint(createUpload.UploadFileId), @@ -689,7 +892,7 @@ func (driver Cloud189) getUploadFileState(uploadFileId int64, account *model.Acc fullUrl += "/getUploadFileStatus.action" } var uploadFileState UploadFileStatusResult - _, err := GetState(account).Request("GET", fullUrl, nil, func(r *resty.Request) { + _, err := GetState(account).Request(http.MethodGet, fullUrl, nil, func(r *resty.Request) { r.SetQueryParams(clientSuffix()) r.SetQueryParams(map[string]string{ "uploadFileId": fmt.Sprint(uploadFileId), @@ -704,128 +907,6 @@ func (driver Cloud189) getUploadFileState(uploadFileId int64, account *model.Acc return nil, err } return &uploadFileState, nil -} +}*/ -/* -暂时未解决 -func (driver Cloud189) Upload(file *model.FileStream, account *model.Account) error { - if file == nil { - return base.ErrEmptyFile - } - - parentFile, err := driver.File(file.ParentPath, account) - if err != nil { - return err - } - if !parentFile.IsDir() { - return base.ErrNotFolder - } - - fullUrl := UPLOAD_URL - if isFamily(account) { - fullUrl += "/family" - } else { - fullUrl += "/person" - } - - tempFile, err := ioutil.TempFile(conf.Conf.TempDir, "file-*") - if err != nil { - return err - } - - defer tempFile.Close() - defer os.Remove(tempFile.Name()) - - // 初始化上传 - const DEFAULT int64 = 10485760 - count := int64(math.Ceil(float64(file.Size) / float64(DEFAULT))) - fileMd5 := md5.New() - silceMd5 := md5.New() - silceMd5Hexs := make([]string, 0, count) - silceMd5Base64s := make([]string, 0, count) - for i := int64(1); i <= count; i++ { - if _, err := io.CopyN(io.MultiWriter(fileMd5, silceMd5, tempFile), file, DEFAULT); err != io.EOF { - return err - } - md5Byte := silceMd5.Sum(nil) - silceMd5Hexs = append(silceMd5Hexs, strings.ToUpper(hex.EncodeToString(md5Byte))) - silceMd5Base64s = append(silceMd5Base64s, fmt.Sprint(i, "-", base64.StdEncoding.EncodeToString(md5Byte))) - } - fileMd5Hex := strings.ToUpper(hex.EncodeToString(fileMd5.Sum(nil))) - sliceMd5Hex := fileMd5Hex - if int64(file.Size) > DEFAULT { - sliceMd5Hex = strings.ToUpper(utils.GetMD5Encode(strings.Join(silceMd5Hexs, "\n"))) - } - - qID := uuid.NewString() - client := GetState(account) - param := MapToUrlValues(map[string]interface{}{ - "parentFolderId": parentFile.Id, - "fileName": url.QueryEscape(file.Name), - "fileMd5": fileMd5Hex, - "fileSize": fmt.Sprint(file.Size), - "sliceMd5": sliceMd5Hex, - "sliceSize": fmt.Sprint(DEFAULT), - }) - if isFamily(account) { - param.Set("familyId", account.SiteId) - } - - var uploadInfo InitMultiUploadResp - _, err = client.Request("GET", fullUrl+"/initMultiUpload", param, func(r *resty.Request) { - r.SetQueryParams(clientSuffix()) - r.SetHeader("X-Request-ID", qID) - r.SetResult(&uploadInfo) - }, account) - if err != nil { - return err - } - - if uploadInfo.Data.FileDataExists != 1 { - param = MapToUrlValues(map[string]interface{}{ - "uploadFileId": uploadInfo.Data.UploadFileID, - "partInfo": strings.Join(silceMd5Base64s, ","), - }) - if isFamily(account) { - param.Set("familyId", account.SiteId) - } - var uploadUrls UploadUrlsResp - _, err := client.Request("GET", fullUrl+"/getMultiUploadUrls", param, func(r *resty.Request) { - r.SetQueryParams(clientSuffix()) - r.SetHeader("X-Request-ID", qID).SetHeader("content-type", "application/x-www-form-urlencoded") - r.SetResult(&uploadUrls) - - }, account) - if err != nil { - return err - } - var i int64 - for _, uploadurl := range uploadUrls.UploadUrls { - req := resty.New().SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}).SetProxy("http://192.168.0.30:8888").R() - for _, header := range strings.Split(decodeURIComponent(uploadurl.RequestHeader), "&") { - i := strings.Index(header, "=") - req.SetHeader(header[0:i], header[i+1:]) - } - _, err := req.SetBody(io.NewSectionReader(tempFile, i*DEFAULT, DEFAULT)).Put(uploadurl.RequestURL) - if err != nil { - return err - } - } - } - - param = MapToUrlValues(map[string]interface{}{ - "uploadFileId": uploadInfo.Data.UploadFileID, - "isLog": "0", - "opertype": "1", - }) - if isFamily(account) { - param.Set("familyId", account.SiteId) - } - _, err = client.Request("GET", fullUrl+"/commitMultiUploadFile", param, func(r *resty.Request) { - r.SetHeader("X-Request-ID", qID) - r.SetQueryParams(clientSuffix()) - }, account) - return err -} -*/ var _ base.Driver = (*Cloud189)(nil) diff --git a/drivers/189pc/type.go b/drivers/189pc/type.go index 12e54087..b8d2c0af 100644 --- a/drivers/189pc/type.go +++ b/drivers/189pc/type.go @@ -137,6 +137,7 @@ type BatchTaskInfo struct { //SrcParentId string `json:"srcParentId"` } +/* type CreateUploadFileResult struct { // UploadFileId 上传文件请求ID UploadFileId int64 `json:"uploadFileId"` @@ -157,8 +158,8 @@ type UploadFileStatusResult struct { FileCommitUrl string `json:"fileCommitUrl"` FileDataExists int `json:"fileDataExists"` } +*/ -/* type InitMultiUploadResp struct { //Code string `json:"code"` Data struct { @@ -177,4 +178,3 @@ type Part struct { RequestURL string `json:"requestURL"` RequestHeader string `json:"requestHeader"` } -*/ diff --git a/drivers/189pc/util.go b/drivers/189pc/util.go index f8153127..4f5d2baa 100644 --- a/drivers/189pc/util.go +++ b/drivers/189pc/util.go @@ -15,6 +15,7 @@ import ( rand2 "math/rand" "net/http" "net/url" + "sort" "strings" "time" @@ -118,19 +119,17 @@ func toFamilyOrderBy(o string) string { } } -func MapToUrlValues(m map[string]interface{}) url.Values { - url := make(url.Values, len(m)) - for k, v := range m { - url.Add(k, fmt.Sprint(v)) +func ParseHttpHeader(str string) map[string]string { + header := make(map[string]string) + for _, value := range strings.Split(str, "&") { + i := strings.Index(value, "=") + header[strings.TrimSpace(value[0:i])] = strings.TrimSpace(value[i+1:]) } - return url + return header } -func decodeURIComponent(str string) string { - r, _ := url.QueryUnescape(str) - //r, _ := url.PathUnescape(str) - //r = strings.ReplaceAll(r, " ", "+") - return r +func MustString(str string, err error) string { + return str } func MustToBytes(b []byte, err error) []byte { @@ -143,3 +142,35 @@ func BoolToNumber(b bool) int { } return 0 } + +func MustParseTime(str string) *time.Time { + time, _ := http.ParseTime(str) + return &time +} + +type Params map[string]string + +func (p Params) Set(k, v string) { + p[k] = v +} + +func (p Params) Encode() string { + if p == nil { + return "" + } + var buf strings.Builder + keys := make([]string, 0, len(p)) + for k := range p { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + if buf.Len() > 0 { + buf.WriteByte('&') + } + buf.WriteString(k) + buf.WriteByte('=') + buf.WriteString(p[k]) + } + return buf.String() +} diff --git a/drivers/alidrive/driver.go b/drivers/alidrive/driver.go index a3690840..9ad81566 100644 --- a/drivers/alidrive/driver.go +++ b/drivers/alidrive/driver.go @@ -66,6 +66,11 @@ func (driver AliDrive) Items() []base.Item { Required: false, Description: ">0 and <=200", }, + { + Name: "bool_1", + Label: "fast upload", + Type: base.TypeBool, + }, } } @@ -391,8 +396,7 @@ func (driver AliDrive) Upload(file *model.FileStream, account *model.Account) er if file == nil { return base.ErrEmptyFile } - const DEFAULT int64 = 10485760 - var count = int64(math.Ceil(float64(file.GetSize()) / float64(DEFAULT))) + parentFile, err := driver.File(file.ParentPath, account) if err != nil { return err @@ -401,16 +405,14 @@ func (driver AliDrive) Upload(file *model.FileStream, account *model.Account) er return base.ErrNotFolder } + const DEFAULT int64 = 10485760 + var count = int(math.Ceil(float64(file.GetSize()) / float64(DEFAULT))) + partInfoList := make([]base.Json, 0, count) - var i int64 - for i = 0; i < count; i++ { - partInfoList = append(partInfoList, base.Json{ - "part_number": i + 1, - }) + for i := 1; i <= count; i++ { + partInfoList = append(partInfoList, base.Json{"part_number": i}) } - buf := make([]byte, 1024) - n, _ := file.Read(buf[:]) reqBody := base.Json{ "check_name_mode": "overwrite", "drive_id": account.DriveId, @@ -419,9 +421,17 @@ func (driver AliDrive) Upload(file *model.FileStream, account *model.Account) er "part_info_list": partInfoList, "size": file.GetSize(), "type": "file", - "pre_hash": utils.GetSHA1Encode(string(buf[:n])), } - fileReader := io.MultiReader(bytes.NewReader(buf[:n]), file.File) + + if account.Bool1 { + buf := make([]byte, 1024) + n, _ := file.Read(buf[:]) + reqBody["pre_hash"] = utils.GetSHA1Encode(string(buf[:n])) + file.File = io.NopCloser(io.MultiReader(bytes.NewReader(buf[:n]), file.File)) + } else { + reqBody["content_hash_name"] = "none" + reqBody["proof_version"] = "v1" + } var resp UploadResp var e AliRespError @@ -444,7 +454,7 @@ func (driver AliDrive) Upload(file *model.FileStream, account *model.Account) er return fmt.Errorf("%s", e.Message) } - if e.Code == "PreHashMatched" { + if e.Code == "PreHashMatched" && account.Bool1 { tempFile, err := ioutil.TempFile(conf.Conf.TempDir, "file-*") if err != nil { return err @@ -455,7 +465,7 @@ func (driver AliDrive) Upload(file *model.FileStream, account *model.Account) er delete(reqBody, "pre_hash") h := sha1.New() - if _, err = io.Copy(tempFile, io.TeeReader(fileReader, h)); err != nil { + if _, err = io.Copy(io.MultiWriter(tempFile, h), file.File); err != nil { return err } reqBody["content_hash"] = hex.EncodeToString(h.Sum(nil)) @@ -470,10 +480,11 @@ func (driver AliDrive) Upload(file *model.FileStream, account *model.Account) er o = i ? r.mod(i) : new gt.BigNumber(0); (t.file.slice(o.toNumber(), Math.min(o.plus(8).toNumber(), t.file.size))) */ + buf := make([]byte, 8) r, _ := new(big.Int).SetString(utils.GetMD5Encode(account.AccessToken)[:16], 16) i := new(big.Int).SetUint64(file.Size) o := r.Mod(r, i) - n, _ = io.NewSectionReader(tempFile, o.Int64(), 8).Read(buf[:8]) + n, _ := io.NewSectionReader(tempFile, o.Int64(), 8).Read(buf[:8]) reqBody["proof_code"] = base64.StdEncoding.EncodeToString(buf[:n]) _, err = client.Post("https://api.aliyundrive.com/adrive/v2/file/createWithFolders") @@ -491,11 +502,11 @@ func (driver AliDrive) Upload(file *model.FileStream, account *model.Account) er if _, err = tempFile.Seek(0, io.SeekStart); err != nil { return err } - fileReader = tempFile + file.File = tempFile } - for i = 0; i < count; i++ { - req, err := http.NewRequest("PUT", resp.PartInfoList[i].UploadUrl, io.LimitReader(fileReader, DEFAULT)) + for _, partInfo := range resp.PartInfoList { + req, err := http.NewRequest("PUT", partInfo.UploadUrl, io.LimitReader(file.File, DEFAULT)) if err != nil { return err } @@ -523,7 +534,7 @@ func (driver AliDrive) Upload(file *model.FileStream, account *model.Account) er if err != nil { return err } - if e.Code != "" { + if e.Code != "" && e.Code != "PreHashMatched" { //if e.Code == "AccessTokenInvalid" { // err = driver.RefreshToken(account) // if err != nil {