mirror of https://github.com/Xhofe/alist
fix(115): 20GB file upload restriction (#7452)
* fix(115): multipart upload error * feat(115): Modify default page size * fix(115): Replace temporary repair schemepull/7453/head v3.39.1
foxxorcat
GitHub
parent
64ceb5afb6
commit
b803b0070e
|
|
@ -9,7 +9,7 @@ type Addition struct {
|
|||
Cookie string `json:"cookie" type:"text" help:"one of QR code token and cookie required"`
|
||||
QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"`
|
||||
QRCodeSource string `json:"qrcode_source" type:"select" options:"web,android,ios,tv,alipaymini,wechatmini,qandroid" default:"linux" help:"select the QR code device, default linux"`
|
||||
PageSize int64 `json:"page_size" type:"number" default:"56" help:"list api per page size of 115 driver"`
|
||||
PageSize int64 `json:"page_size" type:"number" default:"1000" help:"list api per page size of 115 driver"`
|
||||
LimitRate float64 `json:"limit_rate" type:"number" default:"2" help:"limit all api request rate (1r/[limit_rate]s)"`
|
||||
driver.RootID
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
package _115
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/SheltonZhu/115driver/pkg/driver"
|
||||
"github.com/alist-org/alist/v3/internal/model"
|
||||
"github.com/alist-org/alist/v3/pkg/utils"
|
||||
"time"
|
||||
)
|
||||
|
||||
var _ model.Obj = (*FileObj)(nil)
|
||||
|
|
@ -20,3 +21,18 @@ func (f *FileObj) CreateTime() time.Time {
|
|||
func (f *FileObj) GetHash() utils.HashInfo {
|
||||
return utils.NewHashInfo(utils.SHA1, f.Sha1)
|
||||
}
|
||||
|
||||
type UploadResult struct {
|
||||
driver.BasicResp
|
||||
Data struct {
|
||||
PickCode string `json:"pick_code"`
|
||||
FileSize int `json:"file_size"`
|
||||
FileID string `json:"file_id"`
|
||||
ThumbURL string `json:"thumb_url"`
|
||||
Sha1 string `json:"sha1"`
|
||||
Aid int `json:"aid"`
|
||||
FileName string `json:"file_name"`
|
||||
Cid string `json:"cid"`
|
||||
IsVideo int `json:"is_video"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ import (
|
|||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
|
@ -254,6 +253,7 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i
|
|||
ossClient *oss.Client
|
||||
bucket *oss.Bucket
|
||||
ossToken *driver115.UploadOSSTokenResp
|
||||
bodyBytes []byte
|
||||
err error
|
||||
)
|
||||
|
||||
|
|
@ -268,12 +268,14 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i
|
|||
f(options)
|
||||
}
|
||||
}
|
||||
// oss 启用Sequential必须按顺序上传
|
||||
options.ThreadsNum = 1
|
||||
|
||||
if ossToken, err = d.client.GetOSSToken(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if ossClient, err = oss.New(driver115.OSSEndpoint, ossToken.AccessKeyID, ossToken.AccessKeySecret); err != nil {
|
||||
if ossClient, err = oss.New(driver115.OSSEndpoint, ossToken.AccessKeyID, ossToken.AccessKeySecret, oss.EnableMD5(true), oss.EnableCRC(true)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
@ -294,6 +296,7 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i
|
|||
if imur, err = bucket.InitiateMultipartUpload(params.Object,
|
||||
oss.SetHeader(driver115.OssSecurityTokenHeaderName, ossToken.SecurityToken),
|
||||
oss.UserAgentHeader(driver115.OSSUserAgent),
|
||||
oss.EnableSha1(), oss.Sequential(),
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -337,8 +340,7 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i
|
|||
continue
|
||||
}
|
||||
|
||||
b := bytes.NewBuffer(buf)
|
||||
if part, err = bucket.UploadPart(imur, b, chunk.Size, chunk.Number, driver115.OssOption(params, ossToken)...); err == nil {
|
||||
if part, err = bucket.UploadPart(imur, bytes.NewBuffer(buf), chunk.Size, chunk.Number, driver115.OssOption(params, ossToken)...); err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
|
@ -373,14 +375,20 @@ LOOP:
|
|||
}
|
||||
}
|
||||
|
||||
// EOF错误是xml的Unmarshal导致的，响应其实是json格式，所以实际上上传是成功的
|
||||
if _, err = bucket.CompleteMultipartUpload(imur, parts, driver115.OssOption(params, ossToken)...); err != nil && !errors.Is(err, io.EOF) {
|
||||
// 当文件名含有 &< 这两个字符之一时响应的xml解析会出现错误，实际上上传是成功的
|
||||
if filename := filepath.Base(stream.GetName()); !strings.ContainsAny(filename, "&<") {
|
||||
return err
|
||||
}
|
||||
// 不知道啥原因，oss那边分片上传不计算sha1，导致115服务器校验错误
|
||||
// params.Callback.Callback = strings.ReplaceAll(params.Callback.Callback, "${sha1}", params.SHA1)
|
||||
if _, err := bucket.CompleteMultipartUpload(imur, parts, append(
|
||||
driver115.OssOption(params, ossToken),
|
||||
oss.CallbackResult(&bodyBytes),
|
||||
)...); err != nil {
|
||||
return err
|
||||
}
|
||||
return d.checkUploadStatus(dirID, params.SHA1)
|
||||
|
||||
var uploadResult UploadResult
|
||||
if err = json.Unmarshal(bodyBytes, &uploadResult); err != nil {
|
||||
return err
|
||||
}
|
||||
return uploadResult.Err(string(bodyBytes))
|
||||
}
|
||||
|
||||
func chunksProducer(ch chan oss.FileChunk, chunks []oss.FileChunk) {
|
||||
|
|
@ -389,27 +397,6 @@ func chunksProducer(ch chan oss.FileChunk, chunks []oss.FileChunk) {
|
|||
}
|
||||
}
|
||||
|
||||
func (d *Pan115) checkUploadStatus(dirID, sha1 string) error {
|
||||
// 验证上传是否成功
|
||||
req := d.client.NewRequest().ForceContentType("application/json;charset=UTF-8")
|
||||
opts := []driver115.GetFileOptions{
|
||||
driver115.WithOrder(driver115.FileOrderByTime),
|
||||
driver115.WithShowDirEnable(false),
|
||||
driver115.WithAsc(false),
|
||||
driver115.WithLimit(500),
|
||||
}
|
||||
fResp, err := driver115.GetFiles(req, dirID, opts...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, fileInfo := range fResp.Files {
|
||||
if fileInfo.Sha1 == sha1 {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return driver115.ErrUploadFailed
|
||||
}
|
||||
|
||||
func SplitFile(fileSize int64) (chunks []oss.FileChunk, err error) {
|
||||
for i := int64(1); i < 10; i++ {
|
||||
if fileSize < i*utils.GB { // 文件大小小于iGB时分为i*1000片
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ type Addition struct {
|
|||
Cookie string `json:"cookie" type:"text" help:"one of QR code token and cookie required"`
|
||||
QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"`
|
||||
QRCodeSource string `json:"qrcode_source" type:"select" options:"web,android,ios,tv,alipaymini,wechatmini,qandroid" default:"linux" help:"select the QR code device, default linux"`
|
||||
PageSize int64 `json:"page_size" type:"number" default:"20" help:"list api per page size of 115 driver"`
|
||||
PageSize int64 `json:"page_size" type:"number" default:"1000" help:"list api per page size of 115 driver"`
|
||||
LimitRate float64 `json:"limit_rate" type:"number" default:"2" help:"limit all api request rate (1r/[limit_rate]s)"`
|
||||
ShareCode string `json:"share_code" type:"text" required:"true" help:"share code of 115 share link"`
|
||||
ReceiveCode string `json:"receive_code" type:"text" required:"true" help:"receive code of 115 share link"`
|
||||
|
|
|
|||
Loading…
Reference in New Issue