feat: add rapid upload switch for 189pc and alidrive (#892)

* 189PC增加快传开关

* alidrive增加快传开关
pull/1073/head
foxxorcat 2022-04-03 17:56:21 +08:00 committed by GitHub
parent 18c82e79b5
commit bf2e5768d6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 345 additions and 241 deletions

View File

@ -34,7 +34,7 @@ func GetState(account *model.Account) *State {
SetHeaders(map[string]string{ SetHeaders(map[string]string{
"Accept": "application/json;charset=UTF-8", "Accept": "application/json;charset=UTF-8",
"User-Agent": base.UserAgent, "User-Agent": base.UserAgent,
}), }).SetTimeout(base.DefaultTimeout),
} }
userStateCache.States[account.Username] = state userStateCache.States[account.Username] = state
return state return state
@ -198,7 +198,7 @@ func (s *State) refreshSession(account *model.Account) error {
"accessToken": s.AccessToken, "accessToken": s.AccessToken,
}). }).
SetHeader("X-Request-ID", uuid.NewString()). SetHeader("X-Request-ID", uuid.NewString()).
Get("https://api.cloud.189.cn/getSessionForPC.action") Get(API_URL + "/getSessionForPC.action")
if err != nil { if err != nil {
return err return err
} }
@ -223,10 +223,8 @@ func (s *State) refreshSession(account *model.Account) error {
return nil return nil
} }
func (s *State) IsLogin() bool { func (s *State) IsLogin(account *model.Account) bool {
_, err := s.Request("GET", API_URL+"/getUserInfo.action", nil, func(r *resty.Request) { _, err := s.Request(http.MethodGet, API_URL+"/getUserInfo.action", nil, func(r *resty.Request) { r.SetQueryParams(clientSuffix()) }, account)
r.SetQueryParams(clientSuffix())
}, nil)
return err == nil return err == nil
} }
@ -242,12 +240,12 @@ func (s *State) RefreshSession(account *model.Account) error {
return s.refreshSession(account) return s.refreshSession(account)
} }
func (s *State) Request(method string, fullUrl string, params url.Values, callback func(*resty.Request), account *model.Account) (*resty.Response, error) { func (s *State) Request(method string, fullUrl string, params Params, callback func(*resty.Request), account *model.Account) (*resty.Response, error) {
s.Lock() s.Lock()
dateOfGmt := getHttpDateStr() dateOfGmt := getHttpDateStr()
sessionKey := s.SessionKey sessionKey := s.SessionKey
sessionSecret := s.SessionSecret sessionSecret := s.SessionSecret
if account != nil && isFamily(account) { if isFamily(account) {
sessionKey = s.FamilySessionKey sessionKey = s.FamilySessionKey
sessionSecret = s.FamilySessionSecret sessionSecret = s.FamilySessionSecret
} }
@ -267,25 +265,12 @@ func (s *State) Request(method string, fullUrl string, params url.Values, callba
} }
req.SetHeader("Signature", signatureOfHmac(sessionSecret, sessionKey, method, fullUrl, dateOfGmt, paramsData)) req.SetHeader("Signature", signatureOfHmac(sessionSecret, sessionKey, method, fullUrl, dateOfGmt, paramsData))
callback(req) if callback != nil {
callback(req)
}
s.Unlock() s.Unlock()
var err error res, err := req.Execute(method, fullUrl)
var res *resty.Response
switch method {
case "GET":
res, err = req.Get(fullUrl)
case "POST":
res, err = req.Post(fullUrl)
case "DELETE":
res, err = req.Delete(fullUrl)
case "PATCH":
res, err = req.Patch(fullUrl)
case "PUT":
res, err = req.Put(fullUrl)
default:
return nil, base.ErrNotSupport
}
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -298,6 +283,9 @@ func (s *State) Request(method string, fullUrl string, params url.Values, callba
} }
if erron.Code != "" && erron.Code != "SUCCESS" { if erron.Code != "" && erron.Code != "SUCCESS" {
if erron.Msg == "" { if erron.Msg == "" {
if erron.Message == "" {
return nil, fmt.Errorf(res.String())
}
return nil, fmt.Errorf(erron.Message) return nil, fmt.Errorf(erron.Message)
} }
return nil, fmt.Errorf(erron.Msg) return nil, fmt.Errorf(erron.Msg)
@ -306,25 +294,18 @@ func (s *State) Request(method string, fullUrl string, params url.Values, callba
return nil, fmt.Errorf(erron.ErrorMsg) return nil, fmt.Errorf(erron.ErrorMsg)
} }
if account != nil { switch utils.Json.Get(res.Body(), "res_code").ToInt64() {
switch utils.Json.Get(res.Body(), "res_code").ToInt64() { case 11, 18:
case 11, 18: if err := s.RefreshSession(account); err != nil {
if err := s.RefreshSession(account); err != nil { return nil, err
return nil, err
}
return s.Request(method, fullUrl, params, callback, account)
case 0:
if res.StatusCode() == http.StatusOK {
return res, nil
}
fallthrough
default:
return nil, fmt.Errorf(res.String())
} }
return s.Request(method, fullUrl, params, callback, account)
case 0:
if res.StatusCode() == http.StatusOK {
return res, nil
}
return nil, fmt.Errorf(res.String())
default:
return nil, fmt.Errorf(utils.Json.Get(res.Body(), "res_message").ToString())
} }
if utils.Json.Get(res.Body(), "res_code").ToInt64() != 0 {
return res, fmt.Errorf(utils.Json.Get(res.Body(), "res_message").ToString())
}
return res, nil
} }

View File

@ -1,16 +1,19 @@
package _189 package _189
import ( import (
"bytes"
"crypto/md5" "crypto/md5"
"encoding/base64"
"encoding/hex" "encoding/hex"
"fmt" "fmt"
"io" "io"
"io/ioutil" "io/ioutil"
"math"
"net/http" "net/http"
"net/url"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"time"
"github.com/Xhofe/alist/conf" "github.com/Xhofe/alist/conf"
"github.com/Xhofe/alist/drivers/base" "github.com/Xhofe/alist/drivers/base"
@ -50,10 +53,9 @@ func (driver Cloud189) Items() []base.Item {
Description: "account password", Description: "account password",
}, },
{ {
Name: "root_folder", Name: "root_folder",
Label: "root folder file_id", Label: "root folder file_id",
Type: base.TypeString, Type: base.TypeString,
Required: true,
}, },
{ {
Name: "internal_type", Name: "internal_type",
@ -63,10 +65,9 @@ func (driver Cloud189) Items() []base.Item {
Values: "Personal,Family", Values: "Personal,Family",
}, },
{ {
Name: "site_id", Name: "site_id",
Label: "family id", Label: "family id",
Type: base.TypeString, Type: base.TypeString,
Required: true,
}, },
{ {
Name: "order_by", Name: "order_by",
@ -82,6 +83,11 @@ func (driver Cloud189) Items() []base.Item {
Values: "true,false", Values: "true,false",
Required: true, Required: true,
}, },
{
Name: "bool_1",
Label: "fast upload",
Type: base.TypeBool,
},
} }
} }
@ -92,10 +98,14 @@ func (driver Cloud189) Save(account *model.Account, old *model.Account) error {
if !isFamily(account) && account.RootFolder == "" { if !isFamily(account) && account.RootFolder == "" {
account.RootFolder = "-11" account.RootFolder = "-11"
account.SiteId = ""
}
if isFamily(account) && account.RootFolder == "-11" {
account.RootFolder = ""
} }
state := GetState(account) state := GetState(account)
if !state.IsLogin() { if !state.IsLogin(account) {
if err := state.Login(account); err != nil { if err := state.Login(account); err != nil {
return err return err
} }
@ -121,7 +131,7 @@ func (driver Cloud189) Save(account *model.Account, old *model.Account) error {
func (driver Cloud189) getFamilyInfoList(account *model.Account) ([]FamilyInfoResp, error) { func (driver Cloud189) getFamilyInfoList(account *model.Account) ([]FamilyInfoResp, error) {
var resp FamilyInfoListResp var resp FamilyInfoListResp
_, err := GetState(account).Request("GET", API_URL+"/family/manage/getFamilyList.action", nil, func(r *resty.Request) { _, err := GetState(account).Request(http.MethodGet, API_URL+"/family/manage/getFamilyList.action", nil, func(r *resty.Request) {
r.SetQueryParams(clientSuffix()) r.SetQueryParams(clientSuffix())
r.SetResult(&resp) r.SetResult(&resp)
}, account) }, account)
@ -179,16 +189,16 @@ func (driver Cloud189) Files(path string, account *model.Account) ([]model.File,
client := GetState(account) client := GetState(account)
for pageNum := 1; ; pageNum++ { for pageNum := 1; ; pageNum++ {
var resp Cloud189FilesResp var resp Cloud189FilesResp
queryparam := map[string]string{ _, err = client.Request(http.MethodGet, fullUrl, nil, func(r *resty.Request) {
"folderId": file.Id, r.SetQueryParams(clientSuffix()).
"fileType": "0", SetQueryParams(map[string]string{
"mediaAttr": "0", "folderId": file.Id,
"iconOption": "5", "fileType": "0",
"pageNum": fmt.Sprint(pageNum), "mediaAttr": "0",
"pageSize": "130", "iconOption": "5",
} "pageNum": fmt.Sprint(pageNum),
_, err = client.Request("GET", fullUrl, nil, func(r *resty.Request) { "pageSize": "130",
r.SetQueryParams(clientSuffix()).SetQueryParams(queryparam) })
if isFamily(account) { if isFamily(account) {
r.SetQueryParams(map[string]string{ r.SetQueryParams(map[string]string{
"familyId": account.SiteId, "familyId": account.SiteId,
@ -212,10 +222,6 @@ func (driver Cloud189) Files(path string, account *model.Account) ([]model.File,
break break
} }
mustTime := func(str string) *time.Time {
time, _ := http.ParseTime(str)
return &time
}
for _, folder := range resp.FileListAO.FolderList { for _, folder := range resp.FileListAO.FolderList {
files = append(files, model.File{ files = append(files, model.File{
Id: fmt.Sprint(folder.ID), Id: fmt.Sprint(folder.ID),
@ -223,7 +229,7 @@ func (driver Cloud189) Files(path string, account *model.Account) ([]model.File,
Size: 0, Size: 0,
Type: conf.FOLDER, Type: conf.FOLDER,
Driver: driver.Config().Name, Driver: driver.Config().Name,
UpdatedAt: mustTime(folder.CreateDate), UpdatedAt: MustParseTime(folder.CreateDate),
}) })
} }
for _, file := range resp.FileListAO.FileList { for _, file := range resp.FileListAO.FileList {
@ -233,7 +239,7 @@ func (driver Cloud189) Files(path string, account *model.Account) ([]model.File,
Size: file.Size, Size: file.Size,
Type: utils.GetFileType(filepath.Ext(file.Name)), Type: utils.GetFileType(filepath.Ext(file.Name)),
Driver: driver.Config().Name, Driver: driver.Config().Name,
UpdatedAt: mustTime(file.CreateDate), UpdatedAt: MustParseTime(file.CreateDate),
Thumbnail: file.Icon.SmallUrl, Thumbnail: file.Icon.SmallUrl,
}) })
} }
@ -279,7 +285,7 @@ func (driver Cloud189) Link(args base.Args, account *model.Account) (*base.Link,
var downloadUrl struct { var downloadUrl struct {
URL string `json:"fileDownloadUrl"` URL string `json:"fileDownloadUrl"`
} }
_, err = GetState(account).Request("GET", fullUrl, nil, func(r *resty.Request) { _, err = GetState(account).Request(http.MethodGet, fullUrl, nil, func(r *resty.Request) {
r.SetQueryParams(clientSuffix()).SetQueryParam("fileId", file.Id) r.SetQueryParams(clientSuffix()).SetQueryParam("fileId", file.Id)
if isFamily(account) { if isFamily(account) {
r.SetQueryParams(map[string]string{ r.SetQueryParams(map[string]string{
@ -324,7 +330,7 @@ func (driver Cloud189) MakeDir(path string, account *model.Account) error {
} }
fullUrl += "/createFolder.action" fullUrl += "/createFolder.action"
_, err = GetState(account).Request("POST", fullUrl, nil, func(r *resty.Request) { _, err = GetState(account).Request(http.MethodPost, fullUrl, nil, func(r *resty.Request) {
r.SetQueryParams(clientSuffix()).SetQueryParams(map[string]string{ r.SetQueryParams(clientSuffix()).SetQueryParams(map[string]string{
"folderName": name, "folderName": name,
"relativePath": "", "relativePath": "",
@ -354,7 +360,7 @@ func (driver Cloud189) Move(src string, dst string, account *model.Account) erro
return err return err
} }
_, err = GetState(account).Request("POST", API_URL+"/batch/createBatchTask.action", nil, func(r *resty.Request) { _, err = GetState(account).Request(http.MethodPost, API_URL+"/batch/createBatchTask.action", nil, func(r *resty.Request) {
r.SetFormData(clientSuffix()).SetFormData(map[string]string{ r.SetFormData(clientSuffix()).SetFormData(map[string]string{
"type": "MOVE", "type": "MOVE",
"taskInfos": string(MustToBytes(utils.Json.Marshal( "taskInfos": string(MustToBytes(utils.Json.Marshal(
@ -390,10 +396,10 @@ func (driver Cloud189) Move(src string, dst string, account *model.Account) erro
var queryParam map[string]string var queryParam map[string]string
fullUrl := API_URL fullUrl := API_URL
method := "POST" method := http.MethodPost
if isFamily(account) { if isFamily(account) {
fullUrl += "/family/file" fullUrl += "/family/file"
method = "GET" method = http.MethodGet
} }
if srcFile.IsDir() { if srcFile.IsDir() {
fullUrl += "/moveFolder.action" fullUrl += "/moveFolder.action"
@ -431,10 +437,10 @@ func (driver Cloud189) Rename(src string, dst string, account *model.Account) er
var queryParam map[string]string var queryParam map[string]string
fullUrl := API_URL fullUrl := API_URL
method := "POST" method := http.MethodPost
if isFamily(account) { if isFamily(account) {
fullUrl += "/family/file" fullUrl += "/family/file"
method = "GET" method = http.MethodGet
} }
if srcFile.IsDir() { if srcFile.IsDir() {
fullUrl += "/renameFolder.action" fullUrl += "/renameFolder.action"
@ -470,7 +476,7 @@ func (driver Cloud189) Copy(src string, dst string, account *model.Account) erro
return err return err
} }
_, err = GetState(account).Request("POST", API_URL+"/batch/createBatchTask.action", nil, func(r *resty.Request) { _, err = GetState(account).Request(http.MethodPost, API_URL+"/batch/createBatchTask.action", nil, func(r *resty.Request) {
r.SetFormData(clientSuffix()).SetFormData(map[string]string{ r.SetFormData(clientSuffix()).SetFormData(map[string]string{
"type": "COPY", "type": "COPY",
"taskInfos": string(MustToBytes(utils.Json.Marshal( "taskInfos": string(MustToBytes(utils.Json.Marshal(
@ -500,7 +506,7 @@ func (driver Cloud189) Delete(path string, account *model.Account) error {
return err return err
} }
_, err = GetState(account).Request("POST", API_URL+"/batch/createBatchTask.action", nil, func(r *resty.Request) { _, err = GetState(account).Request(http.MethodPost, API_URL+"/batch/createBatchTask.action", nil, func(r *resty.Request) {
r.SetFormData(clientSuffix()).SetFormData(map[string]string{ r.SetFormData(clientSuffix()).SetFormData(map[string]string{
"type": "DELETE", "type": "DELETE",
"taskInfos": string(MustToBytes(utils.Json.Marshal( "taskInfos": string(MustToBytes(utils.Json.Marshal(
@ -535,12 +541,209 @@ func (driver Cloud189) Upload(file *model.FileStream, account *model.Account) er
return base.ErrNotFolder return base.ErrNotFolder
} }
if isFamily(account) { if account.Bool1 {
return driver.uploadFamily(file, parentFile, account) return driver.FastUpload(file, parentFile, account)
} }
return driver.uploadPerson(file, parentFile, account) return driver.CommonUpload(file, parentFile, account)
/*
if isFamily(account) {
return driver.uploadFamily(file, parentFile, account)
}
return driver.uploadPerson(file, parentFile, account)
*/
} }
func (driver Cloud189) CommonUpload(file *model.FileStream, parentFile *model.File, account *model.Account) error {
// 初始化上传
state := GetState(account)
const DEFAULT int64 = 10485760
count := int(math.Ceil(float64(file.Size) / float64(DEFAULT)))
params := Params{
"parentFolderId": parentFile.Id,
"fileName": url.PathEscape(file.Name),
"fileSize": fmt.Sprint(file.Size),
"sliceSize": fmt.Sprint(DEFAULT),
"lazyCheck": "1",
}
fullUrl := UPLOAD_URL
if isFamily(account) {
params.Set("familyId", account.SiteId)
fullUrl += "/family"
} else {
//params.Set("extend", `{"opScene":"1","relativepath":"","rootfolderid":""}`)
fullUrl += "/person"
}
var initMultiUpload InitMultiUploadResp
_, err := state.Request(http.MethodGet, fullUrl+"/initMultiUpload", params, func(r *resty.Request) { r.SetQueryParams(clientSuffix()).SetResult(&initMultiUpload) }, account)
if err != nil {
return err
}
fileMd5 := md5.New()
silceMd5 := md5.New()
silceMd5Hexs := make([]string, 0, count)
byteData := bytes.NewBuffer(make([]byte, DEFAULT))
for i := 1; i <= count; i++ {
byteData.Reset()
silceMd5.Reset()
if n, err := io.CopyN(io.MultiWriter(fileMd5, silceMd5, byteData), file, DEFAULT); err != io.EOF && n == 0 {
return err
}
md5Bytes := silceMd5.Sum(nil)
silceMd5Hexs = append(silceMd5Hexs, strings.ToUpper(hex.EncodeToString(md5Bytes)))
silceMd5Base64 := base64.StdEncoding.EncodeToString(md5Bytes)
var uploadUrl UploadUrlsResp
_, err = state.Request(http.MethodGet, fullUrl+"/getMultiUploadUrls",
Params{"partInfo": fmt.Sprintf("%d-%s", i, silceMd5Base64), "uploadFileId": initMultiUpload.Data.UploadFileID},
func(r *resty.Request) { r.SetQueryParams(clientSuffix()).SetResult(&uploadUrl) },
account)
if err != nil {
return err
}
uploadData := uploadUrl.UploadUrls[fmt.Sprint("partNumber_", i)]
req, _ := http.NewRequest(http.MethodPut, uploadData.RequestURL, byteData)
for k, v := range ParseHttpHeader(uploadData.RequestHeader) {
req.Header.Set(k, v)
}
for k, v := range clientSuffix() {
req.URL.RawQuery += fmt.Sprintf("&%s=%s", k, v)
}
r, err := base.HttpClient.Do(req)
if err != nil {
return err
}
if r.StatusCode != http.StatusOK {
data, _ := io.ReadAll(r.Body)
return fmt.Errorf(string(data))
}
}
fileMd5Hex := strings.ToUpper(hex.EncodeToString(fileMd5.Sum(nil)))
sliceMd5Hex := fileMd5Hex
if int64(file.Size) > DEFAULT {
sliceMd5Hex = strings.ToUpper(utils.GetMD5Encode(strings.Join(silceMd5Hexs, "\n")))
}
_, err = state.Request(http.MethodGet, fullUrl+"/commitMultiUploadFile",
Params{
"uploadFileId": initMultiUpload.Data.UploadFileID,
"fileMd5": fileMd5Hex,
"sliceMd5": sliceMd5Hex,
"lazyCheck": "1",
"isLog": "0",
"opertype": "3",
},
func(r *resty.Request) { r.SetQueryParams(clientSuffix()) }, account)
return err
}
func (driver Cloud189) FastUpload(file *model.FileStream, parentFile *model.File, account *model.Account) error {
tempFile, err := ioutil.TempFile(conf.Conf.TempDir, "file-*")
if err != nil {
return err
}
defer tempFile.Close()
defer os.Remove(tempFile.Name())
// 初始化上传
state := GetState(account)
const DEFAULT int64 = 10485760
count := int(math.Ceil(float64(file.Size) / float64(DEFAULT)))
// 优先计算所需信息
fileMd5 := md5.New()
silceMd5 := md5.New()
silceMd5Hexs := make([]string, 0, count)
silceMd5Base64s := make([]string, 0, count)
for i := 1; i <= count; i++ {
silceMd5.Reset()
if n, err := io.CopyN(io.MultiWriter(fileMd5, silceMd5, tempFile), file, DEFAULT); err != nil && n == 0 {
return err
}
md5Byte := silceMd5.Sum(nil)
silceMd5Hexs = append(silceMd5Hexs, strings.ToUpper(hex.EncodeToString(md5Byte)))
silceMd5Base64s = append(silceMd5Base64s, fmt.Sprint(i, "-", base64.StdEncoding.EncodeToString(md5Byte)))
}
fileMd5Hex := strings.ToUpper(hex.EncodeToString(fileMd5.Sum(nil)))
sliceMd5Hex := fileMd5Hex
if int64(file.Size) > DEFAULT {
sliceMd5Hex = strings.ToUpper(utils.GetMD5Encode(strings.Join(silceMd5Hexs, "\n")))
}
params := Params{
"parentFolderId": parentFile.Id,
"fileName": url.PathEscape(file.Name),
"fileSize": fmt.Sprint(file.Size),
"fileMd5": fileMd5Hex,
"sliceSize": fmt.Sprint(DEFAULT),
"sliceMd5": sliceMd5Hex,
}
fullUrl := UPLOAD_URL
if isFamily(account) {
params.Set("familyId", account.SiteId)
fullUrl += "/family"
} else {
//params.Set("extend", `{"opScene":"1","relativepath":"","rootfolderid":""}`)
fullUrl += "/person"
}
var uploadInfo InitMultiUploadResp
_, err = state.Request(http.MethodGet, fullUrl+"/initMultiUpload", params, func(r *resty.Request) { r.SetQueryParams(clientSuffix()).SetResult(&uploadInfo) }, account)
if err != nil {
return err
}
if uploadInfo.Data.FileDataExists != 1 {
var uploadUrls UploadUrlsResp
_, err := state.Request(http.MethodGet, fullUrl+"/getMultiUploadUrls",
Params{
"uploadFileId": uploadInfo.Data.UploadFileID,
"partInfo": strings.Join(silceMd5Base64s, ","),
},
func(r *resty.Request) { r.SetQueryParams(clientSuffix()).SetResult(&uploadUrls) },
account)
if err != nil {
return err
}
for i := 1; i <= count; i++ {
uploadData := uploadUrls.UploadUrls[fmt.Sprint("partNumber_", i)]
req, _ := http.NewRequest(http.MethodPut, uploadData.RequestURL, io.NewSectionReader(tempFile, int64(i-1)*DEFAULT, DEFAULT))
for k, v := range ParseHttpHeader(uploadData.RequestHeader) {
req.Header.Set(k, v)
}
for k, v := range clientSuffix() {
req.URL.RawQuery += fmt.Sprintf("&%s=%s", k, v)
}
r, err := base.HttpClient.Do(req)
if err != nil {
return err
}
if r.StatusCode != http.StatusOK {
data, _ := io.ReadAll(r.Body)
return fmt.Errorf(string(data))
}
}
}
_, err = state.Request(http.MethodGet, fullUrl+"/commitMultiUploadFile",
Params{
"uploadFileId": uploadInfo.Data.UploadFileID,
"isLog": "0",
"opertype": "3",
},
func(r *resty.Request) { r.SetQueryParams(clientSuffix()) },
account)
return err
}
/*
func (driver Cloud189) uploadFamily(file *model.FileStream, parentFile *model.File, account *model.Account) error { func (driver Cloud189) uploadFamily(file *model.FileStream, parentFile *model.File, account *model.Account) error {
tempFile, err := ioutil.TempFile(conf.Conf.TempDir, "file-*") tempFile, err := ioutil.TempFile(conf.Conf.TempDir, "file-*")
if err != nil { if err != nil {
@ -557,7 +760,7 @@ func (driver Cloud189) uploadFamily(file *model.FileStream, parentFile *model.Fi
client := GetState(account) client := GetState(account)
var createUpload CreateUploadFileResult var createUpload CreateUploadFileResult
_, err = client.Request("GET", API_URL+"/family/file/createFamilyFile.action", nil, func(r *resty.Request) { _, err = client.Request(http.MethodGet, API_URL+"/family/file/createFamilyFile.action", nil, func(r *resty.Request) {
r.SetQueryParams(map[string]string{ r.SetQueryParams(map[string]string{
"fileMd5": hex.EncodeToString(fileMd5.Sum(nil)), "fileMd5": hex.EncodeToString(fileMd5.Sum(nil)),
"fileName": file.Name, "fileName": file.Name,
@ -579,7 +782,7 @@ func (driver Cloud189) uploadFamily(file *model.FileStream, parentFile *model.Fi
} }
} }
_, err = client.Request("GET", createUpload.FileCommitUrl, nil, func(r *resty.Request) { _, err = client.Request(http.MethodGet, createUpload.FileCommitUrl, nil, func(r *resty.Request) {
r.SetQueryParams(clientSuffix()) r.SetQueryParams(clientSuffix())
r.SetHeaders(map[string]string{ r.SetHeaders(map[string]string{
"FamilyId": account.SiteId, "FamilyId": account.SiteId,
@ -606,7 +809,7 @@ func (driver Cloud189) uploadPerson(file *model.FileStream, parentFile *model.Fi
client := GetState(account) client := GetState(account)
var createUpload CreateUploadFileResult var createUpload CreateUploadFileResult
_, err = client.Request("POST", API_URL+"/createUploadFile.action", nil, func(r *resty.Request) { _, err = client.Request(http.MethodPost, API_URL+"/createUploadFile.action", nil, func(r *resty.Request) {
r.SetQueryParams(clientSuffix()) r.SetQueryParams(clientSuffix())
r.SetFormData(clientSuffix()).SetFormData(map[string]string{ r.SetFormData(clientSuffix()).SetFormData(map[string]string{
"parentFolderId": parentFile.Id, "parentFolderId": parentFile.Id,
@ -634,7 +837,7 @@ func (driver Cloud189) uploadPerson(file *model.FileStream, parentFile *model.Fi
} }
} }
_, err = client.Request("POST", createUpload.FileCommitUrl, nil, func(r *resty.Request) { _, err = client.Request(http.MethodPost, createUpload.FileCommitUrl, nil, func(r *resty.Request) {
r.SetQueryParams(clientSuffix()) r.SetQueryParams(clientSuffix())
r.SetFormData(map[string]string{ r.SetFormData(map[string]string{
"uploadFileId": fmt.Sprint(createUpload.UploadFileId), "uploadFileId": fmt.Sprint(createUpload.UploadFileId),
@ -689,7 +892,7 @@ func (driver Cloud189) getUploadFileState(uploadFileId int64, account *model.Acc
fullUrl += "/getUploadFileStatus.action" fullUrl += "/getUploadFileStatus.action"
} }
var uploadFileState UploadFileStatusResult var uploadFileState UploadFileStatusResult
_, err := GetState(account).Request("GET", fullUrl, nil, func(r *resty.Request) { _, err := GetState(account).Request(http.MethodGet, fullUrl, nil, func(r *resty.Request) {
r.SetQueryParams(clientSuffix()) r.SetQueryParams(clientSuffix())
r.SetQueryParams(map[string]string{ r.SetQueryParams(map[string]string{
"uploadFileId": fmt.Sprint(uploadFileId), "uploadFileId": fmt.Sprint(uploadFileId),
@ -704,128 +907,6 @@ func (driver Cloud189) getUploadFileState(uploadFileId int64, account *model.Acc
return nil, err return nil, err
} }
return &uploadFileState, nil return &uploadFileState, nil
} }*/
/*
func (driver Cloud189) Upload(file *model.FileStream, account *model.Account) error {
if file == nil {
return base.ErrEmptyFile
}
parentFile, err := driver.File(file.ParentPath, account)
if err != nil {
return err
}
if !parentFile.IsDir() {
return base.ErrNotFolder
}
fullUrl := UPLOAD_URL
if isFamily(account) {
fullUrl += "/family"
} else {
fullUrl += "/person"
}
tempFile, err := ioutil.TempFile(conf.Conf.TempDir, "file-*")
if err != nil {
return err
}
defer tempFile.Close()
defer os.Remove(tempFile.Name())
// 初始化上传
const DEFAULT int64 = 10485760
count := int64(math.Ceil(float64(file.Size) / float64(DEFAULT)))
fileMd5 := md5.New()
silceMd5 := md5.New()
silceMd5Hexs := make([]string, 0, count)
silceMd5Base64s := make([]string, 0, count)
for i := int64(1); i <= count; i++ {
if _, err := io.CopyN(io.MultiWriter(fileMd5, silceMd5, tempFile), file, DEFAULT); err != io.EOF {
return err
}
md5Byte := silceMd5.Sum(nil)
silceMd5Hexs = append(silceMd5Hexs, strings.ToUpper(hex.EncodeToString(md5Byte)))
silceMd5Base64s = append(silceMd5Base64s, fmt.Sprint(i, "-", base64.StdEncoding.EncodeToString(md5Byte)))
}
fileMd5Hex := strings.ToUpper(hex.EncodeToString(fileMd5.Sum(nil)))
sliceMd5Hex := fileMd5Hex
if int64(file.Size) > DEFAULT {
sliceMd5Hex = strings.ToUpper(utils.GetMD5Encode(strings.Join(silceMd5Hexs, "\n")))
}
qID := uuid.NewString()
client := GetState(account)
param := MapToUrlValues(map[string]interface{}{
"parentFolderId": parentFile.Id,
"fileName": url.QueryEscape(file.Name),
"fileMd5": fileMd5Hex,
"fileSize": fmt.Sprint(file.Size),
"sliceMd5": sliceMd5Hex,
"sliceSize": fmt.Sprint(DEFAULT),
})
if isFamily(account) {
param.Set("familyId", account.SiteId)
}
var uploadInfo InitMultiUploadResp
_, err = client.Request("GET", fullUrl+"/initMultiUpload", param, func(r *resty.Request) {
r.SetQueryParams(clientSuffix())
r.SetHeader("X-Request-ID", qID)
r.SetResult(&uploadInfo)
}, account)
if err != nil {
return err
}
if uploadInfo.Data.FileDataExists != 1 {
param = MapToUrlValues(map[string]interface{}{
"uploadFileId": uploadInfo.Data.UploadFileID,
"partInfo": strings.Join(silceMd5Base64s, ","),
})
if isFamily(account) {
param.Set("familyId", account.SiteId)
}
var uploadUrls UploadUrlsResp
_, err := client.Request("GET", fullUrl+"/getMultiUploadUrls", param, func(r *resty.Request) {
r.SetQueryParams(clientSuffix())
r.SetHeader("X-Request-ID", qID).SetHeader("content-type", "application/x-www-form-urlencoded")
r.SetResult(&uploadUrls)
}, account)
if err != nil {
return err
}
var i int64
for _, uploadurl := range uploadUrls.UploadUrls {
req := resty.New().SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}).SetProxy("http://192.168.0.30:8888").R()
for _, header := range strings.Split(decodeURIComponent(uploadurl.RequestHeader), "&") {
i := strings.Index(header, "=")
req.SetHeader(header[0:i], header[i+1:])
}
_, err := req.SetBody(io.NewSectionReader(tempFile, i*DEFAULT, DEFAULT)).Put(uploadurl.RequestURL)
if err != nil {
return err
}
}
}
param = MapToUrlValues(map[string]interface{}{
"uploadFileId": uploadInfo.Data.UploadFileID,
"isLog": "0",
"opertype": "1",
})
if isFamily(account) {
param.Set("familyId", account.SiteId)
}
_, err = client.Request("GET", fullUrl+"/commitMultiUploadFile", param, func(r *resty.Request) {
r.SetHeader("X-Request-ID", qID)
r.SetQueryParams(clientSuffix())
}, account)
return err
}
*/
var _ base.Driver = (*Cloud189)(nil) var _ base.Driver = (*Cloud189)(nil)

View File

@ -137,6 +137,7 @@ type BatchTaskInfo struct {
//SrcParentId string `json:"srcParentId"` //SrcParentId string `json:"srcParentId"`
} }
/*
type CreateUploadFileResult struct { type CreateUploadFileResult struct {
// UploadFileId 上传文件请求ID // UploadFileId 上传文件请求ID
UploadFileId int64 `json:"uploadFileId"` UploadFileId int64 `json:"uploadFileId"`
@ -157,8 +158,8 @@ type UploadFileStatusResult struct {
FileCommitUrl string `json:"fileCommitUrl"` FileCommitUrl string `json:"fileCommitUrl"`
FileDataExists int `json:"fileDataExists"` FileDataExists int `json:"fileDataExists"`
} }
*/
/*
type InitMultiUploadResp struct { type InitMultiUploadResp struct {
//Code string `json:"code"` //Code string `json:"code"`
Data struct { Data struct {
@ -177,4 +178,3 @@ type Part struct {
RequestURL string `json:"requestURL"` RequestURL string `json:"requestURL"`
RequestHeader string `json:"requestHeader"` RequestHeader string `json:"requestHeader"`
} }
*/

View File

@ -15,6 +15,7 @@ import (
rand2 "math/rand" rand2 "math/rand"
"net/http" "net/http"
"net/url" "net/url"
"sort"
"strings" "strings"
"time" "time"
@ -118,19 +119,17 @@ func toFamilyOrderBy(o string) string {
} }
} }
func MapToUrlValues(m map[string]interface{}) url.Values { func ParseHttpHeader(str string) map[string]string {
url := make(url.Values, len(m)) header := make(map[string]string)
for k, v := range m { for _, value := range strings.Split(str, "&") {
url.Add(k, fmt.Sprint(v)) i := strings.Index(value, "=")
header[strings.TrimSpace(value[0:i])] = strings.TrimSpace(value[i+1:])
} }
return url return header
} }
func decodeURIComponent(str string) string { func MustString(str string, err error) string {
r, _ := url.QueryUnescape(str) return str
//r, _ := url.PathUnescape(str)
//r = strings.ReplaceAll(r, " ", "+")
return r
} }
func MustToBytes(b []byte, err error) []byte { func MustToBytes(b []byte, err error) []byte {
@ -143,3 +142,35 @@ func BoolToNumber(b bool) int {
} }
return 0 return 0
} }
func MustParseTime(str string) *time.Time {
time, _ := http.ParseTime(str)
return &time
}
type Params map[string]string
func (p Params) Set(k, v string) {
p[k] = v
}
func (p Params) Encode() string {
if p == nil {
return ""
}
var buf strings.Builder
keys := make([]string, 0, len(p))
for k := range p {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
if buf.Len() > 0 {
buf.WriteByte('&')
}
buf.WriteString(k)
buf.WriteByte('=')
buf.WriteString(p[k])
}
return buf.String()
}

View File

@ -66,6 +66,11 @@ func (driver AliDrive) Items() []base.Item {
Required: false, Required: false,
Description: ">0 and <=200", Description: ">0 and <=200",
}, },
{
Name: "bool_1",
Label: "fast upload",
Type: base.TypeBool,
},
} }
} }
@ -391,8 +396,7 @@ func (driver AliDrive) Upload(file *model.FileStream, account *model.Account) er
if file == nil { if file == nil {
return base.ErrEmptyFile return base.ErrEmptyFile
} }
const DEFAULT int64 = 10485760
var count = int64(math.Ceil(float64(file.GetSize()) / float64(DEFAULT)))
parentFile, err := driver.File(file.ParentPath, account) parentFile, err := driver.File(file.ParentPath, account)
if err != nil { if err != nil {
return err return err
@ -401,16 +405,14 @@ func (driver AliDrive) Upload(file *model.FileStream, account *model.Account) er
return base.ErrNotFolder return base.ErrNotFolder
} }
const DEFAULT int64 = 10485760
var count = int(math.Ceil(float64(file.GetSize()) / float64(DEFAULT)))
partInfoList := make([]base.Json, 0, count) partInfoList := make([]base.Json, 0, count)
var i int64 for i := 1; i <= count; i++ {
for i = 0; i < count; i++ { partInfoList = append(partInfoList, base.Json{"part_number": i})
partInfoList = append(partInfoList, base.Json{
"part_number": i + 1,
})
} }
buf := make([]byte, 1024)
n, _ := file.Read(buf[:])
reqBody := base.Json{ reqBody := base.Json{
"check_name_mode": "overwrite", "check_name_mode": "overwrite",
"drive_id": account.DriveId, "drive_id": account.DriveId,
@ -419,9 +421,17 @@ func (driver AliDrive) Upload(file *model.FileStream, account *model.Account) er
"part_info_list": partInfoList, "part_info_list": partInfoList,
"size": file.GetSize(), "size": file.GetSize(),
"type": "file", "type": "file",
"pre_hash": utils.GetSHA1Encode(string(buf[:n])),
} }
fileReader := io.MultiReader(bytes.NewReader(buf[:n]), file.File)
if account.Bool1 {
buf := make([]byte, 1024)
n, _ := file.Read(buf[:])
reqBody["pre_hash"] = utils.GetSHA1Encode(string(buf[:n]))
file.File = io.NopCloser(io.MultiReader(bytes.NewReader(buf[:n]), file.File))
} else {
reqBody["content_hash_name"] = "none"
reqBody["proof_version"] = "v1"
}
var resp UploadResp var resp UploadResp
var e AliRespError var e AliRespError
@ -444,7 +454,7 @@ func (driver AliDrive) Upload(file *model.FileStream, account *model.Account) er
return fmt.Errorf("%s", e.Message) return fmt.Errorf("%s", e.Message)
} }
if e.Code == "PreHashMatched" { if e.Code == "PreHashMatched" && account.Bool1 {
tempFile, err := ioutil.TempFile(conf.Conf.TempDir, "file-*") tempFile, err := ioutil.TempFile(conf.Conf.TempDir, "file-*")
if err != nil { if err != nil {
return err return err
@ -455,7 +465,7 @@ func (driver AliDrive) Upload(file *model.FileStream, account *model.Account) er
delete(reqBody, "pre_hash") delete(reqBody, "pre_hash")
h := sha1.New() h := sha1.New()
if _, err = io.Copy(tempFile, io.TeeReader(fileReader, h)); err != nil { if _, err = io.Copy(io.MultiWriter(tempFile, h), file.File); err != nil {
return err return err
} }
reqBody["content_hash"] = hex.EncodeToString(h.Sum(nil)) reqBody["content_hash"] = hex.EncodeToString(h.Sum(nil))
@ -470,10 +480,11 @@ func (driver AliDrive) Upload(file *model.FileStream, account *model.Account) er
o = i ? r.mod(i) : new gt.BigNumber(0); o = i ? r.mod(i) : new gt.BigNumber(0);
(t.file.slice(o.toNumber(), Math.min(o.plus(8).toNumber(), t.file.size))) (t.file.slice(o.toNumber(), Math.min(o.plus(8).toNumber(), t.file.size)))
*/ */
buf := make([]byte, 8)
r, _ := new(big.Int).SetString(utils.GetMD5Encode(account.AccessToken)[:16], 16) r, _ := new(big.Int).SetString(utils.GetMD5Encode(account.AccessToken)[:16], 16)
i := new(big.Int).SetUint64(file.Size) i := new(big.Int).SetUint64(file.Size)
o := r.Mod(r, i) o := r.Mod(r, i)
n, _ = io.NewSectionReader(tempFile, o.Int64(), 8).Read(buf[:8]) n, _ := io.NewSectionReader(tempFile, o.Int64(), 8).Read(buf[:8])
reqBody["proof_code"] = base64.StdEncoding.EncodeToString(buf[:n]) reqBody["proof_code"] = base64.StdEncoding.EncodeToString(buf[:n])
_, err = client.Post("https://api.aliyundrive.com/adrive/v2/file/createWithFolders") _, err = client.Post("https://api.aliyundrive.com/adrive/v2/file/createWithFolders")
@ -491,11 +502,11 @@ func (driver AliDrive) Upload(file *model.FileStream, account *model.Account) er
if _, err = tempFile.Seek(0, io.SeekStart); err != nil { if _, err = tempFile.Seek(0, io.SeekStart); err != nil {
return err return err
} }
fileReader = tempFile file.File = tempFile
} }
for i = 0; i < count; i++ { for _, partInfo := range resp.PartInfoList {
req, err := http.NewRequest("PUT", resp.PartInfoList[i].UploadUrl, io.LimitReader(fileReader, DEFAULT)) req, err := http.NewRequest("PUT", partInfo.UploadUrl, io.LimitReader(file.File, DEFAULT))
if err != nil { if err != nil {
return err return err
} }
@ -523,7 +534,7 @@ func (driver AliDrive) Upload(file *model.FileStream, account *model.Account) er
if err != nil { if err != nil {
return err return err
} }
if e.Code != "" { if e.Code != "" && e.Code != "PreHashMatched" {
//if e.Code == "AccessTokenInvalid" { //if e.Code == "AccessTokenInvalid" {
// err = driver.RefreshToken(account) // err = driver.RefreshToken(account)
// if err != nil { // if err != nil {