package op import ( "context" stderrors "errors" "io" stdpath "path" "strings" "time" "github.com/alist-org/alist/v3/internal/archive/tool" "github.com/alist-org/alist/v3/internal/stream" "github.com/Xhofe/go-cache" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/singleflight" "github.com/alist-org/alist/v3/pkg/utils" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) var archiveMetaCache = cache.NewMemCache(cache.WithShards[*model.ArchiveMetaProvider](64)) var archiveMetaG singleflight.Group[*model.ArchiveMetaProvider] func GetArchiveMeta(ctx context.Context, storage driver.Driver, path string, args model.ArchiveMetaArgs) (*model.ArchiveMetaProvider, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) } path = utils.FixAndCleanPath(path) key := Key(storage, path) if !args.Refresh { if meta, ok := archiveMetaCache.Get(key); ok { log.Debugf("use cache when get %s archive meta", path) return meta, nil } } fn := func() (*model.ArchiveMetaProvider, error) { _, m, err := getArchiveMeta(ctx, storage, path, args) if err != nil { return nil, errors.Wrapf(err, "failed to get %s archive met: %+v", path, err) } if m.Expiration != nil { archiveMetaCache.Set(key, m, cache.WithEx[*model.ArchiveMetaProvider](*m.Expiration)) } return m, nil } if storage.Config().OnlyLocal { meta, err := fn() return meta, err } meta, err, _ := archiveMetaG.Do(key, fn) return meta, err } func getArchiveToolAndStream(ctx context.Context, storage driver.Driver, path string, args model.LinkArgs) (model.Obj, tool.Tool, *stream.SeekableStream, error) { l, obj, err := Link(ctx, storage, path, args) if err != nil { return nil, nil, nil, errors.WithMessagef(err, "failed get [%s] link", path) } ext := stdpath.Ext(obj.GetName()) t, err := tool.GetArchiveTool(ext) if err != nil { return nil, nil, nil, errors.WithMessagef(err, "failed get [%s] archive tool", ext) } ss, err := stream.NewSeekableStream(stream.FileStream{Ctx: ctx, Obj: obj}, l) if err != nil { return nil, nil, nil, errors.WithMessagef(err, "failed get [%s] stream", path) } return obj, t, ss, nil } func getArchiveMeta(ctx context.Context, storage driver.Driver, path string, args model.ArchiveMetaArgs) (model.Obj, *model.ArchiveMetaProvider, error) { storageAr, ok := storage.(driver.ArchiveReader) if ok { obj, err := GetUnwrap(ctx, storage, path) if err != nil { return nil, nil, errors.WithMessage(err, "failed to get file") } if obj.IsDir() { return nil, nil, errors.WithStack(errs.NotFile) } meta, err := storageAr.GetArchiveMeta(ctx, obj, args.ArchiveArgs) if !errors.Is(err, errs.NotImplement) { archiveMetaProvider := &model.ArchiveMetaProvider{ArchiveMeta: meta, DriverProviding: true} if meta.GetTree() != nil { archiveMetaProvider.Sort = &storage.GetStorage().Sort } if !storage.Config().NoCache { Expiration := time.Minute * time.Duration(storage.GetStorage().CacheExpiration) archiveMetaProvider.Expiration = &Expiration } return obj, archiveMetaProvider, err } } obj, t, ss, err := getArchiveToolAndStream(ctx, storage, path, args.LinkArgs) if err != nil { return nil, nil, err } defer func() { if err := ss.Close(); err != nil { log.Errorf("failed to close file streamer, %v", err) } }() meta, err := t.GetMeta(ss, args.ArchiveArgs) if err != nil { return nil, nil, err } archiveMetaProvider := &model.ArchiveMetaProvider{ArchiveMeta: meta, DriverProviding: false} if meta.GetTree() != nil { archiveMetaProvider.Sort = &storage.GetStorage().Sort } if !storage.Config().NoCache { Expiration := time.Minute * time.Duration(storage.GetStorage().CacheExpiration) archiveMetaProvider.Expiration = &Expiration } else if ss.Link.MFile == nil { // alias、crypt 驱动 archiveMetaProvider.Expiration = ss.Link.Expiration } return obj, archiveMetaProvider, err } var archiveListCache = cache.NewMemCache(cache.WithShards[[]model.Obj](64)) var archiveListG singleflight.Group[[]model.Obj] func ListArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) ([]model.Obj, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) } path = utils.FixAndCleanPath(path) metaKey := Key(storage, path) key := stdpath.Join(metaKey, args.InnerPath) if !args.Refresh { if files, ok := archiveListCache.Get(key); ok { log.Debugf("use cache when list archive [%s]%s", path, args.InnerPath) return files, nil } // if meta, ok := archiveMetaCache.Get(metaKey); ok { // log.Debugf("use meta cache when list archive [%s]%s", path, args.InnerPath) // return getChildrenFromArchiveMeta(meta, args.InnerPath) // } } objs, err, _ := archiveListG.Do(key, func() ([]model.Obj, error) { obj, files, err := listArchive(ctx, storage, path, args) if err != nil { return nil, errors.Wrapf(err, "failed to list archive [%s]%s: %+v", path, args.InnerPath, err) } // set path for _, f := range files { if s, ok := f.(model.SetPath); ok && f.GetPath() == "" && obj.GetPath() != "" { s.SetPath(stdpath.Join(obj.GetPath(), args.InnerPath, f.GetName())) } } // warp obj name model.WrapObjsName(files) // sort objs if storage.Config().LocalSort { model.SortFiles(files, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection) } model.ExtractFolder(files, storage.GetStorage().ExtractFolder) if !storage.Config().NoCache { if len(files) > 0 { log.Debugf("set cache: %s => %+v", key, files) archiveListCache.Set(key, files, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) } else { log.Debugf("del cache: %s", key) archiveListCache.Del(key) } } return files, nil }) return objs, err } func _listArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) (model.Obj, []model.Obj, error) { storageAr, ok := storage.(driver.ArchiveReader) if ok { obj, err := GetUnwrap(ctx, storage, path) if err != nil { return nil, nil, errors.WithMessage(err, "failed to get file") } if obj.IsDir() { return nil, nil, errors.WithStack(errs.NotFile) } files, err := storageAr.ListArchive(ctx, obj, args.ArchiveInnerArgs) if !errors.Is(err, errs.NotImplement) { return obj, files, err } } obj, t, ss, err := getArchiveToolAndStream(ctx, storage, path, args.LinkArgs) if err != nil { return nil, nil, err } defer func() { if err := ss.Close(); err != nil { log.Errorf("failed to close file streamer, %v", err) } }() files, err := t.List(ss, args.ArchiveInnerArgs) return obj, files, err } func listArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) (model.Obj, []model.Obj, error) { obj, files, err := _listArchive(ctx, storage, path, args) if errors.Is(err, errs.NotSupport) { var meta model.ArchiveMeta meta, err = GetArchiveMeta(ctx, storage, path, model.ArchiveMetaArgs{ ArchiveArgs: args.ArchiveArgs, Refresh: args.Refresh, }) if err != nil { return nil, nil, err } files, err = getChildrenFromArchiveMeta(meta, args.InnerPath) if err != nil { return nil, nil, err } } if err == nil && obj == nil { obj, err = GetUnwrap(ctx, storage, path) } if err != nil { return nil, nil, err } return obj, files, err } func getChildrenFromArchiveMeta(meta model.ArchiveMeta, innerPath string) ([]model.Obj, error) { obj := meta.GetTree() if obj == nil { return nil, errors.WithStack(errs.NotImplement) } dirs := splitPath(innerPath) for _, dir := range dirs { var next model.ObjTree for _, c := range obj { if c.GetName() == dir { next = c break } } if next == nil { return nil, errors.WithStack(errs.ObjectNotFound) } if !next.IsDir() || next.GetChildren() == nil { return nil, errors.WithStack(errs.NotFolder) } obj = next.GetChildren() } return utils.SliceConvert(obj, func(src model.ObjTree) (model.Obj, error) { return src, nil }) } func splitPath(path string) []string { var parts []string for { dir, file := stdpath.Split(path) if file == "" { break } parts = append([]string{file}, parts...) path = strings.TrimSuffix(dir, "/") } return parts } func ArchiveGet(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) (model.Obj, model.Obj, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return nil, nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) } path = utils.FixAndCleanPath(path) af, err := GetUnwrap(ctx, storage, path) if err != nil { return nil, nil, errors.WithMessage(err, "failed to get file") } if af.IsDir() { return nil, nil, errors.WithStack(errs.NotFile) } if g, ok := storage.(driver.ArchiveGetter); ok { obj, err := g.ArchiveGet(ctx, af, args.ArchiveInnerArgs) if err == nil { return af, model.WrapObjName(obj), nil } } if utils.PathEqual(args.InnerPath, "/") { return af, &model.ObjWrapName{ Name: RootName, Obj: &model.Object{ Name: af.GetName(), Path: af.GetPath(), ID: af.GetID(), Size: af.GetSize(), Modified: af.ModTime(), IsFolder: true, }, }, nil } innerDir, name := stdpath.Split(args.InnerPath) args.InnerPath = strings.TrimSuffix(innerDir, "/") files, err := ListArchive(ctx, storage, path, args) if err != nil { return nil, nil, errors.WithMessage(err, "failed get parent list") } for _, f := range files { if f.GetName() == name { return af, f, nil } } return nil, nil, errors.WithStack(errs.ObjectNotFound) } type extractLink struct { Link *model.Link Obj model.Obj } var extractCache = cache.NewMemCache(cache.WithShards[*extractLink](16)) var extractG singleflight.Group[*extractLink] func DriverExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (*model.Link, model.Obj, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return nil, nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) } key := stdpath.Join(Key(storage, path), args.InnerPath) if link, ok := extractCache.Get(key); ok { return link.Link, link.Obj, nil } else if link, ok := extractCache.Get(key + ":" + args.IP); ok { return link.Link, link.Obj, nil } fn := func() (*extractLink, error) { link, err := driverExtract(ctx, storage, path, args) if err != nil { return nil, errors.Wrapf(err, "failed extract archive") } if link.Link.Expiration != nil { if link.Link.IPCacheKey { key = key + ":" + args.IP } extractCache.Set(key, link, cache.WithEx[*extractLink](*link.Link.Expiration)) } return link, nil } if storage.Config().OnlyLocal { link, err := fn() if err != nil { return nil, nil, err } return link.Link, link.Obj, nil } link, err, _ := extractG.Do(key, fn) if err != nil { return nil, nil, err } return link.Link, link.Obj, err } func driverExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (*extractLink, error) { storageAr, ok := storage.(driver.ArchiveReader) if !ok { return nil, errs.DriverExtractNotSupported } archiveFile, extracted, err := ArchiveGet(ctx, storage, path, model.ArchiveListArgs{ ArchiveInnerArgs: args, Refresh: false, }) if err != nil { return nil, errors.WithMessage(err, "failed to get file") } if extracted.IsDir() { return nil, errors.WithStack(errs.NotFile) } link, err := storageAr.Extract(ctx, archiveFile, args) return &extractLink{Link: link, Obj: extracted}, err } type streamWithParent struct { rc io.ReadCloser parent *stream.SeekableStream } func (s *streamWithParent) Read(p []byte) (int, error) { return s.rc.Read(p) } func (s *streamWithParent) Close() error { err1 := s.rc.Close() err2 := s.parent.Close() return stderrors.Join(err1, err2) } func InternalExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { _, t, ss, err := getArchiveToolAndStream(ctx, storage, path, args.LinkArgs) if err != nil { return nil, 0, err } rc, size, err := t.Extract(ss, args) if err != nil { if e := ss.Close(); e != nil { log.Errorf("failed to close file streamer, %v", e) } return nil, 0, err } return &streamWithParent{rc: rc, parent: ss}, size, nil } func ArchiveDecompress(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, args model.ArchiveDecompressArgs, lazyCache ...bool) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return errors.Errorf("storage not init: %s", storage.GetStorage().Status) } srcPath = utils.FixAndCleanPath(srcPath) dstDirPath = utils.FixAndCleanPath(dstDirPath) srcObj, err := GetUnwrap(ctx, storage, srcPath) if err != nil { return errors.WithMessage(err, "failed to get src object") } dstDir, err := GetUnwrap(ctx, storage, dstDirPath) if err != nil { return errors.WithMessage(err, "failed to get dst dir") } switch s := storage.(type) { case driver.ArchiveDecompressResult: var newObjs []model.Obj newObjs, err = s.ArchiveDecompress(ctx, srcObj, dstDir, args) if err == nil { if newObjs != nil && len(newObjs) > 0 { for _, newObj := range newObjs { addCacheObj(storage, dstDirPath, model.WrapObjName(newObj)) } } else if !utils.IsBool(lazyCache...) { ClearCache(storage, dstDirPath) } } case driver.ArchiveDecompress: err = s.ArchiveDecompress(ctx, srcObj, dstDir, args) if err == nil && !utils.IsBool(lazyCache...) { ClearCache(storage, dstDirPath) } default: return errs.NotImplement } return errors.WithStack(err) }