github
/
Cloudreve
mirror of https://github.com/cloudreve/Cloudreve
1
0
Fork 0
Code Issues Releases Wiki Activity

Feat: process upload callback sent from slave node

pull/1107/head
HFO4 2022-03-03 19:17:25 +08:00
parent 4925a356e3
commit e0714fdd53
11 changed files with 222 additions and 235 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

287
middleware/auth.go
View File

@ -1,24 +1,19 @@
package middleware package middleware
import ( import (
"bytes" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem"
"context"
"crypto/md5"
"fmt"
"io/ioutil"
"net/http" "net/http"
model "github.com/cloudreve/Cloudreve/v3/models" model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/auth" "github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/cache" "github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/onedrive"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/oss"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/upyun"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer" "github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gin-contrib/sessions" "github.com/gin-contrib/sessions"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
"github.com/qiniu/api.v7/v7/auth/qbox" )
const (
CallbackFailedStatusCode = http.StatusUnauthorized
) )
// SignRequired 验证请求签名 // SignRequired 验证请求签名
@ -117,48 +112,60 @@ func WebDAVAuth() gin.HandlerFunc {
} }
} }
// 对上传会话进行验证
func UseUploadSession(policyType string) gin.HandlerFunc {
return func(c *gin.Context) {
// 验证key并查找用户
resp := uploadCallbackCheck(c, policyType)
if resp.Code != 0 {
c.JSON(CallbackFailedStatusCode, resp)
c.Abort()
return
}
c.Next()
}
}
// uploadCallbackCheck 对上传回调请求的 callback key 进行验证，如果成功则返回上传用户 // uploadCallbackCheck 对上传回调请求的 callback key 进行验证，如果成功则返回上传用户
func uploadCallbackCheck(c *gin.Context) (serializer.Response, *model.User) { func uploadCallbackCheck(c *gin.Context, policyType string) serializer.Response {
// 验证 Callback Key // 验证 Callback Key
callbackKey := c.Param("key") sessionID := c.Param("sessionID")
if callbackKey == "" { if sessionID == "" {
return serializer.ParamErr("Callback Key 不能为空", nil), nil return serializer.ParamErr("Session ID 不能为空", nil)
} }
callbackSessionRaw, exist := cache.Get("callback_" + callbackKey)
callbackSessionRaw, exist := cache.Get(filesystem.UploadSessionCachePrefix + sessionID)
if !exist { if !exist {
return serializer.ParamErr("回调会话不存在或已过期", nil), nil return serializer.ParamErr("上传会话不存在或已过期", nil)
} }
callbackSession := callbackSessionRaw.(serializer.UploadSession) callbackSession := callbackSessionRaw.(serializer.UploadSession)
c.Set("callbackSession", &callbackSession) c.Set(filesystem.UploadSessionCtx, &callbackSession)
if callbackSession.Policy.Type != policyType {
return serializer.Err(serializer.CodePolicyNotAllowed, "Policy not supported", nil)
}
// 清理回调会话 // 清理回调会话
_ = cache.Deletes([]string{callbackKey}, "callback_") _ = cache.Deletes([]string{sessionID}, filesystem.UploadSessionCachePrefix)
// 查找用户 // 查找用户
user, err := model.GetActiveUserByID(callbackSession.UID) user, err := model.GetActiveUserByID(callbackSession.UID)
if err != nil { if err != nil {
return serializer.Err(serializer.CodeCheckLogin, "找不到用户", err), nil return serializer.Err(serializer.CodeCheckLogin, "找不到用户", err)
} }
c.Set("user", &user) c.Set(filesystem.UserCtx, &user)
return serializer.Response{}
return serializer.Response{}, &user
} }
// RemoteCallbackAuth 远程回调签名验证 // RemoteCallbackAuth 远程回调签名验证
func RemoteCallbackAuth() gin.HandlerFunc { func RemoteCallbackAuth() gin.HandlerFunc {
return func(c *gin.Context) { return func(c *gin.Context) {
// 验证key并查找用户
resp, user := uploadCallbackCheck(c)
if resp.Code != 0 {
c.JSON(200, resp)
c.Abort()
return
}
// 验证签名 // 验证签名
authInstance := auth.HMACAuth{SecretKey: []byte(user.Policy.SecretKey)} session := c.MustGet(filesystem.UploadSessionCtx).(*serializer.UploadSession)
authInstance := auth.HMACAuth{SecretKey: []byte(session.Policy.SecretKey)}
if err := auth.CheckRequest(authInstance, c.Request); err != nil { if err := auth.CheckRequest(authInstance, c.Request); err != nil {
c.JSON(200, serializer.Err(serializer.CodeCheckLogin, err.Error(), err)) c.JSON(CallbackFailedStatusCode, serializer.Err(serializer.CodeCredentialInvalid, err.Error(), err))
c.Abort() c.Abort()
return return
} }
@ -171,28 +178,28 @@ func RemoteCallbackAuth() gin.HandlerFunc {
// QiniuCallbackAuth 七牛回调签名验证 // QiniuCallbackAuth 七牛回调签名验证
func QiniuCallbackAuth() gin.HandlerFunc { func QiniuCallbackAuth() gin.HandlerFunc {
return func(c *gin.Context) { return func(c *gin.Context) {
// 验证key并查找用户 //// 验证key并查找用户
resp, user := uploadCallbackCheck(c) //resp, user := uploadCallbackCheck(c)
if resp.Code != 0 { //if resp.Code != 0 {
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg}) // c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg})
c.Abort() // c.Abort()
return // return
} //}
//
// 验证回调是否来自qiniu //// 验证回调是否来自qiniu
mac := qbox.NewMac(user.Policy.AccessKey, user.Policy.SecretKey) //mac := qbox.NewMac(user.Policy.AccessKey, user.Policy.SecretKey)
ok, err := mac.VerifyCallback(c.Request) //ok, err := mac.VerifyCallback(c.Request)
if err != nil { //if err != nil {
util.Log().Debug("无法验证回调请求，%s", err) // util.Log().Debug("无法验证回调请求，%s", err)
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "无法验证回调请求"}) // c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "无法验证回调请求"})
c.Abort() // c.Abort()
return // return
} //}
if !ok { //if !ok {
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "回调签名无效"}) // c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "回调签名无效"})
c.Abort() // c.Abort()
return // return
} //}
c.Next() c.Next()
} }
@ -201,21 +208,21 @@ func QiniuCallbackAuth() gin.HandlerFunc {
// OSSCallbackAuth 阿里云OSS回调签名验证 // OSSCallbackAuth 阿里云OSS回调签名验证
func OSSCallbackAuth() gin.HandlerFunc { func OSSCallbackAuth() gin.HandlerFunc {
return func(c *gin.Context) { return func(c *gin.Context) {
// 验证key并查找用户 //// 验证key并查找用户
resp, _ := uploadCallbackCheck(c) //resp, _ := uploadCallbackCheck(c)
if resp.Code != 0 { //if resp.Code != 0 {
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg}) // c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg})
c.Abort() // c.Abort()
return // return
} //}
//
err := oss.VerifyCallbackSignature(c.Request) //err := oss.VerifyCallbackSignature(c.Request)
if err != nil { //if err != nil {
util.Log().Debug("回调签名验证失败，%s", err) // util.Log().Debug("回调签名验证失败，%s", err)
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "回调签名验证失败"}) // c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "回调签名验证失败"})
c.Abort() // c.Abort()
return // return
} //}
c.Next() c.Next()
} }
@ -224,53 +231,53 @@ func OSSCallbackAuth() gin.HandlerFunc {
// UpyunCallbackAuth 又拍云回调签名验证 // UpyunCallbackAuth 又拍云回调签名验证
func UpyunCallbackAuth() gin.HandlerFunc { func UpyunCallbackAuth() gin.HandlerFunc {
return func(c *gin.Context) { return func(c *gin.Context) {
// 验证key并查找用户 //// 验证key并查找用户
resp, user := uploadCallbackCheck(c) //resp, user := uploadCallbackCheck(c)
if resp.Code != 0 { //if resp.Code != 0 {
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg}) // c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg})
c.Abort() // c.Abort()
return // return
} //}
//
// 获取请求正文 //// 获取请求正文
body, err := ioutil.ReadAll(c.Request.Body) //body, err := ioutil.ReadAll(c.Request.Body)
c.Request.Body.Close() //c.Request.Body.Close()
if err != nil { //if err != nil {
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: err.Error()}) // c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: err.Error()})
c.Abort() // c.Abort()
return // return
} //}
//
c.Request.Body = ioutil.NopCloser(bytes.NewReader(body)) //c.Request.Body = ioutil.NopCloser(bytes.NewReader(body))
//
// 准备验证Upyun回调签名 //// 准备验证Upyun回调签名
handler := upyun.Driver{Policy: &user.Policy} //handler := upyun.Driver{Policy: &user.Policy}
contentMD5 := c.Request.Header.Get("Content-Md5") //contentMD5 := c.Request.Header.Get("Content-Md5")
date := c.Request.Header.Get("Date") //date := c.Request.Header.Get("Date")
actualSignature := c.Request.Header.Get("Authorization") //actualSignature := c.Request.Header.Get("Authorization")
//
// 计算正文MD5 //// 计算正文MD5
actualContentMD5 := fmt.Sprintf("%x", md5.Sum(body)) //actualContentMD5 := fmt.Sprintf("%x", md5.Sum(body))
if actualContentMD5 != contentMD5 { //if actualContentMD5 != contentMD5 {
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "MD5不一致"}) // c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "MD5不一致"})
c.Abort() // c.Abort()
return // return
} //}
//
// 计算理论签名 //// 计算理论签名
signature := handler.Sign(context.Background(), []string{ //signature := handler.Sign(context.Background(), []string{
"POST", // "POST",
c.Request.URL.Path, // c.Request.URL.Path,
date, // date,
contentMD5, // contentMD5,
}) //})
//
// 对比签名 //// 对比签名
if signature != actualSignature { //if signature != actualSignature {
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "鉴权失败"}) // c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "鉴权失败"})
c.Abort() // c.Abort()
return // return
} //}
c.Next() c.Next()
} }
@ -280,16 +287,16 @@ func UpyunCallbackAuth() gin.HandlerFunc {
// TODO 解耦 // TODO 解耦
func OneDriveCallbackAuth() gin.HandlerFunc { func OneDriveCallbackAuth() gin.HandlerFunc {
return func(c *gin.Context) { return func(c *gin.Context) {
// 验证key并查找用户 //// 验证key并查找用户
resp, _ := uploadCallbackCheck(c) //resp, _ := uploadCallbackCheck(c)
if resp.Code != 0 { //if resp.Code != 0 {
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg}) // c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg})
c.Abort() // c.Abort()
return // return
} //}
//
// 发送回调结束信号 //// 发送回调结束信号
onedrive.FinishCallback(c.Param("key")) //onedrive.FinishCallback(c.Param("key"))
c.Next() c.Next()
} }
@ -299,13 +306,13 @@ func OneDriveCallbackAuth() gin.HandlerFunc {
// TODO 解耦 测试 // TODO 解耦 测试
func COSCallbackAuth() gin.HandlerFunc { func COSCallbackAuth() gin.HandlerFunc {
return func(c *gin.Context) { return func(c *gin.Context) {
// 验证key并查找用户 //// 验证key并查找用户
resp, _ := uploadCallbackCheck(c) //resp, _ := uploadCallbackCheck(c)
if resp.Code != 0 { //if resp.Code != 0 {
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg}) // c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg})
c.Abort() // c.Abort()
return // return
} //}
c.Next() c.Next()
} }
@ -314,13 +321,13 @@ func COSCallbackAuth() gin.HandlerFunc {
// S3CallbackAuth Amazon S3回调签名验证 // S3CallbackAuth Amazon S3回调签名验证
func S3CallbackAuth() gin.HandlerFunc { func S3CallbackAuth() gin.HandlerFunc {
return func(c *gin.Context) { return func(c *gin.Context) {
// 验证key并查找用户 //// 验证key并查找用户
resp, _ := uploadCallbackCheck(c) //resp, _ := uploadCallbackCheck(c)
if resp.Code != 0 { //if resp.Code != 0 {
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg}) // c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg})
c.Abort() // c.Abort()
return // return
} //}
c.Next() c.Next()
} }

3
models/file.go
View File

@ -299,7 +299,7 @@ func (file *File) UpdateSourceName(value string) error {
return DB.Model(&file).Set("gorm:association_autoupdate", false).Update("source_name", value).Error return DB.Model(&file).Set("gorm:association_autoupdate", false).Update("source_name", value).Error
} }
func (file *File) PopChunkToFile(lastModified *time.Time) error { func (file *File) PopChunkToFile(lastModified *time.Time, picInfo string) error {
file.UploadSessionID = nil file.UploadSessionID = nil
if lastModified != nil { if lastModified != nil {
file.UpdatedAt = *lastModified file.UpdatedAt = *lastModified
@ -308,6 +308,7 @@ func (file *File) PopChunkToFile(lastModified *time.Time) error {
return DB.Model(file).UpdateColumns(map[string]interface{}{ return DB.Model(file).UpdateColumns(map[string]interface{}{
"upload_session_id": file.UploadSessionID, "upload_session_id": file.UploadSessionID,
"updated_at": file.UpdatedAt, "updated_at": file.UpdatedAt,
"pic_info": picInfo,
}).Error }).Error
} }

9
pkg/cluster/slave.go
View File

@ -4,6 +4,7 @@ import (
"bytes" "bytes"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt"
model "github.com/cloudreve/Cloudreve/v3/models" model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/aria2/common" "github.com/cloudreve/Cloudreve/v3/pkg/aria2/common"
"github.com/cloudreve/Cloudreve/v3/pkg/aria2/rpc" "github.com/cloudreve/Cloudreve/v3/pkg/aria2/rpc"
@ -437,14 +438,12 @@ func RemoteCallback(url string, body serializer.UploadCallback) error {
} }
// 解析回调服务端响应 // 解析回调服务端响应
resp = resp.CheckHTTPResponse(200)
if resp.Err != nil {
return serializer.NewError(serializer.CodeCallbackError, "主机服务器返回异常响应", resp.Err)
}
response, err := resp.DecodeResponse() response, err := resp.DecodeResponse()
if err != nil { if err != nil {
return serializer.NewError(serializer.CodeCallbackError, "从机无法解析主机返回的响应", err) msg := fmt.Sprintf("从机无法解析主机返回的响应 (StatusCode=%d)", resp.Response.StatusCode)
return serializer.NewError(serializer.CodeCallbackError, msg, err)
} }
if response.Code != 0 { if response.Code != 0 {
return serializer.NewError(response.Code, response.Msg, errors.New(response.Error)) return serializer.NewError(response.Code, response.Msg, errors.New(response.Error))
} }

22
pkg/filesystem/driver/remote/handler.go
View File

@ -45,7 +45,7 @@ func NewDriver(policy *model.Policy) (*Driver, error) {
} }
// List 列取文件 // List 列取文件
func (handler Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) { func (handler *Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
var res []response.Object var res []response.Object
reqBody := serializer.ListRequest{ reqBody := serializer.ListRequest{
@ -87,7 +87,7 @@ func (handler Driver) List(ctx context.Context, path string, recursive bool) ([]
} }
// getAPIUrl 获取接口请求地址 // getAPIUrl 获取接口请求地址
func (handler Driver) getAPIUrl(scope string, routes ...string) string { func (handler *Driver) getAPIUrl(scope string, routes ...string) string {
serverURL, err := url.Parse(handler.Policy.Server) serverURL, err := url.Parse(handler.Policy.Server)
if err != nil { if err != nil {
return "" return ""
@ -113,7 +113,7 @@ func (handler Driver) getAPIUrl(scope string, routes ...string) string {
} }
// Get 获取文件内容 // Get 获取文件内容
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) { func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 尝试获取速度限制 // 尝试获取速度限制
speedLimit := 0 speedLimit := 0
if user, ok := ctx.Value(fsctx.UserCtx).(model.User); ok { if user, ok := ctx.Value(fsctx.UserCtx).(model.User); ok {
@ -150,7 +150,7 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
} }
// Put 将文件流保存到指定目录 // Put 将文件流保存到指定目录
func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error { func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
defer file.Close() defer file.Close()
// 凭证有效期 // 凭证有效期
@ -206,7 +206,7 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
// Delete 删除一个或多个文件， // Delete 删除一个或多个文件，
// 返回未删除的文件，及遇到的最后一个错误 // 返回未删除的文件，及遇到的最后一个错误
func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) { func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
// 封装接口请求正文 // 封装接口请求正文
reqBody := serializer.RemoteDeleteRequest{ reqBody := serializer.RemoteDeleteRequest{
Files: files, Files: files,
@ -252,7 +252,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
} }
// Thumb 获取文件缩略图 // Thumb 获取文件缩略图
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) { func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
sourcePath := base64.RawURLEncoding.EncodeToString([]byte(path)) sourcePath := base64.RawURLEncoding.EncodeToString([]byte(path))
thumbURL := handler.getAPIUrl("thumb") + "/" + sourcePath thumbURL := handler.getAPIUrl("thumb") + "/" + sourcePath
ttl := model.GetIntSetting("preview_timeout", 60) ttl := model.GetIntSetting("preview_timeout", 60)
@ -268,7 +268,7 @@ func (handler Driver) Thumb(ctx context.Context, path string) (*response.Content
} }
// Source 获取外链URL // Source 获取外链URL
func (handler Driver) Source( func (handler *Driver) Source(
ctx context.Context, ctx context.Context,
path string, path string,
baseURL url.URL, baseURL url.URL,
@ -322,9 +322,9 @@ func (handler Driver) Source(
} }
// Token 获取上传策略和认证Token // Token 获取上传策略和认证Token
func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) { func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
siteURL := model.GetSiteURL() siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse(path.Join("/api/v3/callback/remote" + uploadSession.Key + uploadSession.CallbackSecret)) apiBaseURI, _ := url.Parse(path.Join("/api/v3/callback/remote", uploadSession.Key, uploadSession.CallbackSecret))
apiURL := siteURL.ResolveReference(apiBaseURI) apiURL := siteURL.ResolveReference(apiBaseURI)
// 在从机端创建上传会话 // 在从机端创建上传会话
@ -347,7 +347,7 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
}, nil }, nil
} }
func (handler Driver) getUploadCredential(ctx context.Context, policy serializer.UploadPolicy, TTL int64) (serializer.UploadCredential, error) { func (handler *Driver) getUploadCredential(ctx context.Context, policy serializer.UploadPolicy, TTL int64) (serializer.UploadCredential, error) {
policyEncoded, err := policy.EncodeUploadPolicy() policyEncoded, err := policy.EncodeUploadPolicy()
if err != nil { if err != nil {
return serializer.UploadCredential{}, err return serializer.UploadCredential{}, err
@ -371,6 +371,6 @@ func (handler Driver) getUploadCredential(ctx context.Context, policy serializer
} }
// 取消上传凭证 // 取消上传凭证
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error { func (handler *Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
return nil return nil
} }

2
pkg/filesystem/filesystem.go
View File

@ -207,7 +207,7 @@ func NewFileSystemFromCallback(c *gin.Context) (*FileSystem, error) {
} }
// 获取回调会话 // 获取回调会话
callbackSessionRaw, ok := c.Get("callbackSession") callbackSessionRaw, ok := c.Get(UploadSessionCtx)
if !ok { if !ok {
return nil, errors.New("找不到回调会话") return nil, errors.New("找不到回调会话")
} }

11
pkg/filesystem/hooks.go
View File

@ -194,9 +194,7 @@ func SlaveAfterUpload(session *serializer.UploadSession) Hook {
// 发送回调请求 // 发送回调请求
callbackBody := serializer.UploadCallback{ callbackBody := serializer.UploadCallback{
SourceName: file.SourceName,
PicInfo: file.PicInfo, PicInfo: file.PicInfo,
Size: fileInfo.Size,
} }
return cluster.RemoteCallback(session.Callback, callbackBody) return cluster.RemoteCallback(session.Callback, callbackBody)
@ -287,12 +285,13 @@ func HookChunkUploadFailed(ctx context.Context, fs *FileSystem, fileHeader fsctx
return fileInfo.Model.(*model.File).UpdateSize(fileInfo.AppendStart) return fileInfo.Model.(*model.File).UpdateSize(fileInfo.AppendStart)
} }
// HookChunkUploadFinished 分片上传结束后处理文件 // HookPopPlaceholderToFile 将占位文件提升为正式文件
func HookChunkUploadFinished(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error { func HookPopPlaceholderToFile(picInfo string) Hook {
return func(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
fileInfo := fileHeader.Info() fileInfo := fileHeader.Info()
fileModel := fileInfo.Model.(*model.File) fileModel := fileInfo.Model.(*model.File)
return fileModel.PopChunkToFile(fileInfo.LastModified, picInfo)
return fileModel.PopChunkToFile(fileInfo.LastModified) }
} }
// HookChunkUploadFinished 分片上传结束后处理文件 // HookChunkUploadFinished 分片上传结束后处理文件

8
pkg/filesystem/upload.go
View File

@ -23,6 +23,8 @@ import (
const ( const (
UploadSessionMetaKey = "upload_session" UploadSessionMetaKey = "upload_session"
UploadSessionCtx = "uploadSession"
UserCtx = "user"
UploadSessionCachePrefix = "callback_" UploadSessionCachePrefix = "callback_"
) )
@ -47,11 +49,11 @@ func (fs *FileSystem) Upload(ctx context.Context, file *fsctx.FileStream) (err e
file.SavePath = savePath file.SavePath = savePath
} }
// 保存文件
if file.Mode&fsctx.Nop != fsctx.Nop {
// 处理客户端未完成上传时，关闭连接 // 处理客户端未完成上传时，关闭连接
go fs.CancelUpload(ctx, savePath, file) go fs.CancelUpload(ctx, savePath, file)
// 保存文件
if file.Mode&fsctx.Nop != fsctx.Nop {
err = fs.Handler.Put(ctx, file) err = fs.Handler.Put(ctx, file)
if err != nil { if err != nil {
fs.Trigger(ctx, "AfterUploadFailed", file) fs.Trigger(ctx, "AfterUploadFailed", file)
@ -202,7 +204,7 @@ func (fs *FileSystem) CreateUploadSession(ctx context.Context, file *fsctx.FileS
// 创建回调会话 // 创建回调会话
err = cache.Set( err = cache.Set(
UploadSessionCachePrefix+callbackKey, UploadSessionCachePrefix+callbackKey,
uploadSession, *uploadSession,
callBackSessionTTL, callBackSessionTTL,
) )
if err != nil { if err != nil {

2
pkg/serializer/upload.go
View File

@ -51,9 +51,7 @@ type UploadSession struct {
// UploadCallback 上传回调正文 // UploadCallback 上传回调正文
type UploadCallback struct { type UploadCallback struct {
SourceName string `json:"source_name"`
PicInfo string `json:"pic_info"` PicInfo string `json:"pic_info"`
Size uint64 `json:"size"`
} }
// GeneralUploadCallbackFailed 存储策略上传回调失败响应 // GeneralUploadCallbackFailed 存储策略上传回调失败响应

3
routers/router.go
View File

@ -223,7 +223,8 @@ func InitMasterRouter() *gin.Engine {
{ {
// 远程策略上传回调 // 远程策略上传回调
callback.POST( callback.POST(
"remote/:key", "remote/:sessionID/:key",
middleware.UseUploadSession("remote"),
middleware.RemoteCallbackAuth(), middleware.RemoteCallbackAuth(),
controllers.RemoteCallback, controllers.RemoteCallback,
) )

87
service/callback/upload.go
View File

@ -3,6 +3,7 @@ package callback
import ( import (
"context" "context"
"fmt" "fmt"
model "github.com/cloudreve/Cloudreve/v3/models"
"strings" "strings"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem"
@ -11,13 +12,12 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/s3" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/s3"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer" "github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
) )
// CallbackProcessService 上传请求回调正文接口 // CallbackProcessService 上传请求回调正文接口
type CallbackProcessService interface { type CallbackProcessService interface {
GetBody(*serializer.UploadSession) serializer.UploadCallback GetBody() serializer.UploadCallback
} }
// RemoteUploadCallbackService 远程存储上传回调请求服务 // RemoteUploadCallbackService 远程存储上传回调请求服务
@ -26,7 +26,7 @@ type RemoteUploadCallbackService struct {
} }
// GetBody 返回回调正文 // GetBody 返回回调正文
func (service RemoteUploadCallbackService) GetBody(session *serializer.UploadSession) serializer.UploadCallback { func (service RemoteUploadCallbackService) GetBody() serializer.UploadCallback {
return service.Data return service.Data
} }
@ -68,11 +68,8 @@ type S3Callback struct {
} }
// GetBody 返回回调正文 // GetBody 返回回调正文
func (service UpyunCallbackService) GetBody(session *serializer.UploadSession) serializer.UploadCallback { func (service UpyunCallbackService) GetBody() serializer.UploadCallback {
res := serializer.UploadCallback{ res := serializer.UploadCallback{}
SourceName: service.SourceName,
Size: service.Size,
}
if service.Width != "" { if service.Width != "" {
res.PicInfo = service.Width + "," + service.Height res.PicInfo = service.Width + "," + service.Height
} }
@ -81,47 +78,41 @@ func (service UpyunCallbackService) GetBody(session *serializer.UploadSession) s
} }
// GetBody 返回回调正文 // GetBody 返回回调正文
func (service UploadCallbackService) GetBody(session *serializer.UploadSession) serializer.UploadCallback { func (service UploadCallbackService) GetBody() serializer.UploadCallback {
return serializer.UploadCallback{ return serializer.UploadCallback{
SourceName: service.SourceName,
PicInfo: service.PicInfo, PicInfo: service.PicInfo,
Size: service.Size,
} }
} }
// GetBody 返回回调正文 // GetBody 返回回调正文
func (service OneDriveCallback) GetBody(session *serializer.UploadSession) serializer.UploadCallback { func (service OneDriveCallback) GetBody() serializer.UploadCallback {
var picInfo = "0,0" var picInfo = "0,0"
if service.Meta.Image.Width != 0 { if service.Meta.Image.Width != 0 {
picInfo = fmt.Sprintf("%d,%d", service.Meta.Image.Width, service.Meta.Image.Height) picInfo = fmt.Sprintf("%d,%d", service.Meta.Image.Width, service.Meta.Image.Height)
} }
return serializer.UploadCallback{ return serializer.UploadCallback{
SourceName: session.SavePath,
PicInfo: picInfo, PicInfo: picInfo,
Size: session.Size,
} }
} }
// GetBody 返回回调正文 // GetBody 返回回调正文
func (service COSCallback) GetBody(session *serializer.UploadSession) serializer.UploadCallback { func (service COSCallback) GetBody() serializer.UploadCallback {
return serializer.UploadCallback{ return serializer.UploadCallback{
SourceName: session.SavePath,
PicInfo: "", PicInfo: "",
Size: session.Size,
} }
} }
// GetBody 返回回调正文 // GetBody 返回回调正文
func (service S3Callback) GetBody(session *serializer.UploadSession) serializer.UploadCallback { func (service S3Callback) GetBody() serializer.UploadCallback {
return serializer.UploadCallback{ return serializer.UploadCallback{
SourceName: session.SavePath,
PicInfo: "", PicInfo: "",
Size: session.Size,
} }
} }
// ProcessCallback 处理上传结果回调 // ProcessCallback 处理上传结果回调
func ProcessCallback(service CallbackProcessService, c *gin.Context) serializer.Response { func ProcessCallback(service CallbackProcessService, c *gin.Context) serializer.Response {
callbackBody := service.GetBody()
// 创建文件系统 // 创建文件系统
fs, err := filesystem.NewFileSystemFromCallback(c) fs, err := filesystem.NewFileSystemFromCallback(c)
if err != nil { if err != nil {
@ -129,51 +120,39 @@ func ProcessCallback(service CallbackProcessService, c *gin.Context) serializer.
} }
defer fs.Recycle() defer fs.Recycle()
// 获取回调会话 // 获取上传会话
callbackSessionRaw, _ := c.Get("callbackSession") uploadSession := c.MustGet(filesystem.UploadSessionCtx).(*serializer.UploadSession)
callbackSession := callbackSessionRaw.(*serializer.UploadSession)
callbackBody := service.GetBody(callbackSession)
// 获取父目录 // 查找上传会话创建的占位文件
exist, parentFolder := fs.IsPathExist(callbackSession.VirtualPath) file, err := model.GetFilesByUploadSession(uploadSession.Key, fs.User.ID)
if !exist {
newFolder, err := fs.CreateDirectory(context.Background(), callbackSession.VirtualPath)
if err != nil { if err != nil {
return serializer.Err(serializer.CodeParamErr, "指定目录不存在", err) return serializer.Err(serializer.CodeUploadSessionExpired, "LocalUpload session file placeholder not exist", err)
}
parentFolder = newFolder
} }
// 创建文件头 fileData := fsctx.FileStream{
fileHeader := fsctx.FileStream{ Size: uploadSession.Size,
Size: callbackBody.Size, Name: uploadSession.Name,
VirtualPath: callbackSession.VirtualPath, VirtualPath: uploadSession.VirtualPath,
Name: callbackSession.Name, SavePath: uploadSession.SavePath,
SavePath: callbackBody.SourceName, Mode: fsctx.Nop,
Model: file,
LastModified: uploadSession.LastModified,
} }
// 添加钩子 // 占位符未扣除容量需要校验和扣除
fs.Use("BeforeAddFile", filesystem.HookValidateFile) if !fs.Policy.IsUploadPlaceholderWithSize() {
fs.Use("BeforeAddFile", filesystem.HookValidateCapacity) fs.Use("AfterUpload", filesystem.HookValidateCapacity)
fs.Use("AfterUpload", filesystem.HookChunkUploaded)
}
fs.Use("AfterUpload", filesystem.HookPopPlaceholderToFile(callbackBody.PicInfo))
fs.Use("AfterValidateFailed", filesystem.HookDeleteTempFile) fs.Use("AfterValidateFailed", filesystem.HookDeleteTempFile)
fs.Use("BeforeAddFileFailed", filesystem.HookDeleteTempFile) err = fs.Upload(context.Background(), &fileData)
// 向数据库中添加文件
file, err := fs.AddFile(context.Background(), parentFolder, &fileHeader)
if err != nil { if err != nil {
return serializer.Err(serializer.CodeUploadFailed, err.Error(), err) return serializer.Err(serializer.CodeUploadFailed, err.Error(), err)
} }
// 如果是图片，则更新图片信息 return serializer.Response{}
if callbackBody.PicInfo != "" {
if err := file.UpdatePicInfo(callbackBody.PicInfo); err != nil {
util.Log().Debug("无法更新回调文件的图片信息：%s", err)
}
}
return serializer.Response{
Code: 0,
}
} }
// PreProcess 对OneDrive客户端回调进行预处理验证 // PreProcess 对OneDrive客户端回调进行预处理验证

3
service/explorer/upload.go
View File

@ -192,13 +192,14 @@ func processChunkUpload(ctx context.Context, c *gin.Context, fs *filesystem.File
fs.Use("AfterUpload", filesystem.HookChunkUploaded) fs.Use("AfterUpload", filesystem.HookChunkUploaded)
fs.Use("AfterValidateFailed", filesystem.HookChunkUploadFailed) fs.Use("AfterValidateFailed", filesystem.HookChunkUploadFailed)
if isLastChunk { if isLastChunk {
fs.Use("AfterUpload", filesystem.HookChunkUploadFinished) fs.Use("AfterUpload", filesystem.HookPopPlaceholderToFile(""))
fs.Use("AfterUpload", filesystem.HookGenerateThumb) fs.Use("AfterUpload", filesystem.HookGenerateThumb)
fs.Use("AfterUpload", filesystem.HookDeleteUploadSession(session.Key)) fs.Use("AfterUpload", filesystem.HookDeleteUploadSession(session.Key))
} }
} else { } else {
if isLastChunk { if isLastChunk {
fs.Use("AfterUpload", filesystem.SlaveAfterUpload(session)) fs.Use("AfterUpload", filesystem.SlaveAfterUpload(session))
fs.Use("AfterUpload", filesystem.HookDeleteUploadSession(session.Key))
} }
} }