From 91717b7c4985853687ce8a16e742c4a54cddfdd2 Mon Sep 17 00:00:00 2001 From: Aaron Liu Date: Thu, 21 Aug 2025 10:20:13 +0800 Subject: [PATCH] feat(archive): add support for 7z and bz2 / extract rar and 7zip files protected with password (#2668) --- assets | 2 +- go.mod | 29 ++-- go.sum | 61 ++++++--- .../manager/entitysource/entitysource.go | 127 +++++++++++++----- pkg/filemanager/workflows/extract.go | 113 +++++++++++++--- service/explorer/workflows.go | 3 +- 6 files changed, 248 insertions(+), 87 deletions(-) diff --git a/assets b/assets index 2c5b89c..5a1665a 160000 --- a/assets +++ b/assets @@ -1 +1 @@ -Subproject commit 2c5b89c59cd69a7690938f4a980468006b49b547 +Subproject commit 5a1665a96a96234fb7ea5fd5131b15ecebe127be diff --git a/go.mod b/go.mod index d12dcdd..c1834d6 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/juju/ratelimit v1.0.1 github.com/ks3sdklib/aws-sdk-go v1.6.2 github.com/lib/pq v1.10.9 - github.com/mholt/archiver/v4 v4.0.0-alpha.6 + github.com/mholt/archives v0.1.3 github.com/mojocn/base64Captcha v0.0.0-20190801020520-752b1cd608b2 github.com/pquerna/otp v1.2.0 github.com/qiniu/go-sdk/v7 v7.19.0 @@ -65,9 +65,13 @@ require ( require ( ariga.io/atlas v0.19.1-0.20240203083654-5948b60a8e43 // indirect cloud.google.com/go v0.81.0 // indirect + github.com/STARRY-S/zip v0.2.1 // indirect github.com/agext/levenshtein v1.2.1 // indirect - github.com/andybalholm/brotli v1.0.4 // indirect + github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3 // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect + github.com/bodgit/plumbing v1.3.0 // indirect + github.com/bodgit/sevenzip v1.6.0 // indirect + github.com/bodgit/windows v1.0.1 // indirect github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect github.com/bytedance/sonic v1.11.6 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect @@ -76,7 +80,7 @@ require ( github.com/cloudwego/iasm v0.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3 // indirect - github.com/dsnet/compress v0.0.1 // indirect + github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect github.com/dsoprea/go-exif/v2 v2.0.0-20200604193436-ca8584a0e1c4 // indirect github.com/dsoprea/go-iptc v0.0.0-20200609062250-162ae6b44feb // indirect github.com/dsoprea/go-logging v0.0.0-20200710184922-b02d349568dd // indirect @@ -95,10 +99,11 @@ require ( github.com/goccy/go-json v0.10.2 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect - github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-tpm v0.9.1 // indirect github.com/gorilla/context v1.1.2 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl/v2 v2.13.0 // indirect @@ -106,32 +111,34 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jmespath/go-jmespath v0.3.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.7 // indirect + github.com/klauspost/compress v1.17.11 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect - github.com/klauspost/pgzip v1.2.5 // indirect + github.com/klauspost/pgzip v1.2.6 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mikelolasagasti/xz v1.0.1 // indirect + github.com/minio/minlz v1.0.0 // indirect github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mozillazg/go-httpheader v0.4.0 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect - github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect + github.com/nwaples/rardecode/v2 v2.1.0 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect - github.com/pierrec/lz4/v4 v4.1.14 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/sorairolake/lzip-go v0.3.5 // indirect github.com/stretchr/objx v0.5.2 // indirect - github.com/therootcompany/xz v1.0.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect - github.com/ulikunitz/xz v0.5.10 // indirect + github.com/ulikunitz/xz v0.5.12 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/zclconf/go-cty v1.8.0 // indirect - go4.org v0.0.0-20200411211856-f5505b9728dd // indirect + go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/arch v0.8.0 // indirect golang.org/x/crypto v0.36.0 // indirect golang.org/x/mod v0.20.0 // indirect diff --git a/go.sum b/go.sum index 6f70e0b..ed10902 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,8 @@ github.com/Masterminds/sprig v2.15.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuN github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= +github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= +github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= @@ -100,8 +102,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= -github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= -github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3 h1:8PmGpDEZl9yDpcdEr6Odf23feCxK3LNUNMxjXg41pZQ= +github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= @@ -138,6 +140,12 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI= +github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= +github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs= +github.com/bodgit/sevenzip v1.6.0 h1:a4R0Wu6/P1o1pP/3VV++aEOcyeBxeO/xE2Y9NSTrr6A= +github.com/bodgit/sevenzip v1.6.0/go.mod h1:zOBh9nJUof7tcrlqJFv1koWRrhz3LbDbUNngkuZxLMc= +github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= +github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= @@ -213,8 +221,8 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/dhowden/tag v0.0.0-20230630033851-978a0926ee25 h1:simG0vMYFvNriGhaaat7QVVkaVkXzvqcohaBoLZl9Hg= github.com/dhowden/tag v0.0.0-20230630033851-978a0926ee25/go.mod h1:Z3Lomva4pyMWYezjMAU5QWRh0p1VvO4199OHlFnyKkM= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= -github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= -github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= +github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= +github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dsoprea/go-exif/v2 v2.0.0-20200321225314-640175a69fe4/go.mod h1:Lm2lMM2zx8p4a34ZemkaUV95AnMl4ZvLbCUbwOvLC2E= github.com/dsoprea/go-exif/v2 v2.0.0-20200520183328-015129a9efd5/go.mod h1:9EXlPeHfblFFnwu5UOqmP2eoZfJyAZ2Ri/Vki33ajO0= @@ -414,8 +422,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v1.9.2 h1:HrutZBLhSIU8abiSfW8pj8mPhOyMYjZT/wcA4/L9L9s= github.com/gomodule/redigo v1.9.2/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -519,11 +525,15 @@ github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoP github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.6.4/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= @@ -610,14 +620,14 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= github.com/kisom/goutils v1.4.3/go.mod h1:Lp5qrquG7yhYnWzZCI/68Pa/GpFynw//od6EkGnWpac= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= -github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= -github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= +github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -686,11 +696,15 @@ github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/mholt/archiver/v4 v4.0.0-alpha.6 h1:3wvos9Kn1GpKNBz+MpozinGREPslLo1ds1W16vTkErQ= -github.com/mholt/archiver/v4 v4.0.0-alpha.6/go.mod h1:9PTygYq90FQBWPspdwAng6dNjYiBuTYKqmA6c15KuCo= +github.com/mholt/archives v0.1.3 h1:aEAaOtNra78G+TvV5ohmXrJOAzf++dIlYeDW3N9q458= +github.com/mholt/archives v0.1.3/go.mod h1:LUCGp++/IbV/I0Xq4SzcIR6uwgeh2yjnQWamjRQfLTU= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/pkcs11 v1.0.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/mikelolasagasti/xz v1.0.1 h1:Q2F2jX0RYJUG3+WsM+FJknv+6eVjsjXNDV0KJXZzkD0= +github.com/mikelolasagasti/xz v1.0.1/go.mod h1:muAirjiOUxPRXwm9HdDtB3uoRPrGnL85XHtokL9Hcgc= +github.com/minio/minlz v1.0.0 h1:Kj7aJZ1//LlTP1DM8Jm7lNKvvJS2m74gyyXXn3+uJWQ= +github.com/minio/minlz v1.0.0/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -736,8 +750,8 @@ github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdh github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/nishanths/predeclared v0.0.0-20200524104333-86fad755b4d3/go.mod h1:nt3d53pc1VYcphSCIaYAJtnPYnr3Zyn8fMq2wvPGPso= github.com/nkovacs/streamquote v1.0.0/go.mod h1:BN+NaZ2CmdKqUuTUXUEm9j95B2TRbpOWpxbJYzzgUsc= -github.com/nwaples/rardecode/v2 v2.0.0-beta.2 h1:e3mzJFJs4k83GXBEiTaQ5HgSc/kOK8q0rDaRO0MPaOk= -github.com/nwaples/rardecode/v2 v2.0.0-beta.2/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= +github.com/nwaples/rardecode/v2 v2.1.0 h1:JQl9ZoBPDy+nIZGb1mx8+anfHp/LV3NE2MjMiv0ct/U= +github.com/nwaples/rardecode/v2 v2.1.0/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -774,8 +788,8 @@ github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE= -github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -877,6 +891,8 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k github.com/soheilhy/cmux v0.1.5-0.20210205191134-5ec6847320e5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/sorairolake/lzip-go v0.3.5 h1:ms5Xri9o1JBIWvOFAorYtUNik6HI3HgBTkISiqu0Cwg= +github.com/sorairolake/lzip-go v0.3.5/go.mod h1:N0KYq5iWrMXI0ZEXKXaS9hCyOjZUQdBDEIbXfoUwbdk= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/speps/go-hashids v2.0.0+incompatible h1:kSfxGfESueJKTx0mpER9Y/1XHl+FVQjtCqRyYcviFbw= github.com/speps/go-hashids v2.0.0+incompatible/go.mod h1:P7hqPzMdnZOfyIk+xrlG1QaSMw+gCBdHKsBDnhpaZvc= @@ -928,8 +944,6 @@ github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.563/go.mod github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.563/go.mod h1:uom4Nvi9W+Qkom0exYiJ9VWJjXwyxtPYTkKkaLMlfE0= github.com/tencentyun/cos-go-sdk-v5 v0.7.54 h1:FRamEhNBbSeggyYfWfzFejTLftgbICocSYFk4PKTSV4= github.com/tencentyun/cos-go-sdk-v5 v0.7.54/go.mod h1:UN+VdbCl1hg+kKi5RXqZgaP+Boqfmk+D04GRc4XFk70= -github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= -github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= @@ -951,8 +965,9 @@ github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65E github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= -github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= +github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/upyun/go-sdk v2.1.0+incompatible h1:OdjXghQ/TVetWV16Pz3C1/SUpjhGBVPr+cLiqZLLyq0= github.com/upyun/go-sdk v2.1.0+incompatible/go.mod h1:eu3F5Uz4b9ZE5bE5QsCL6mgSNWRwfj0zpJ9J626HEqs= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -974,6 +989,8 @@ github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0B github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1023,8 +1040,9 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9E go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= -go4.org v0.0.0-20200411211856-f5505b9728dd h1:BNJlw5kRTzdmyfh5U8F93HA2OwkP7ZGwA51eJ/0wKOU= go4.org v0.0.0-20200411211856-f5505b9728dd/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= +go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= +go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= gocloud.dev v0.19.0/go.mod h1:SmKwiR8YwIMMJvQBKLsC3fHNyMwXLw3PMDO+VVteJMI= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= @@ -1160,6 +1178,7 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= diff --git a/pkg/filemanager/manager/entitysource/entitysource.go b/pkg/filemanager/manager/entitysource/entitysource.go index f664c19..a763236 100644 --- a/pkg/filemanager/manager/entitysource/entitysource.go +++ b/pkg/filemanager/manager/entitysource/entitysource.go @@ -163,6 +163,10 @@ type ( rsc io.ReadCloser pos int64 o *EntitySourceOptions + + // Cache for resetRequest URL and expiry + cachedUrl string + cachedExpiry time.Time } ) @@ -215,6 +219,10 @@ func NewEntitySource( } func (f *entitySource) Apply(opts ...EntitySourceOption) { + if len(opts) > 0 { + // Clear cache when options are applied as they might affect URL generation + f.clearUrlCache() + } for _, opt := range opts { opt.Apply(f.o) } @@ -247,6 +255,10 @@ func (f *entitySource) LocalPath(ctx context.Context) string { } func (f *entitySource) Serve(w http.ResponseWriter, r *http.Request, opts ...EntitySourceOption) { + if len(opts) > 0 { + // Clear cache when options are applied as they might affect URL generation + f.clearUrlCache() + } for _, opt := range opts { opt.Apply(f.o) } @@ -478,16 +490,22 @@ func (f *entitySource) Read(p []byte) (n int, err error) { } func (f *entitySource) ReadAt(p []byte, off int64) (n int, err error) { - if f.IsLocal() { - if f.rsc == nil { - err = f.resetRequest() - } - if readAt, ok := f.rsc.(io.ReaderAt); ok { - return readAt.ReadAt(p, off) + if f.rsc == nil { + err = f.resetRequest() + if err != nil { + return 0, err } } + if readAt, ok := f.rsc.(io.ReaderAt); ok { + return readAt.ReadAt(p, off) + } - return 0, errors.New("source does not support ReadAt") + // For non-local sources, use HTTP range request to read at specific offset + rsc, err := f.getRsc(off) + if err != nil { + return 0, err + } + return io.ReadFull(rsc, p) } func (f *entitySource) Seek(offset int64, whence int) (int64, error) { @@ -524,6 +542,12 @@ func (f *entitySource) Close() error { return nil } +// clearUrlCache clears the cached URL and expiry +func (f *entitySource) clearUrlCache() { + f.cachedUrl = "" + f.cachedExpiry = time.Time{} +} + func (f *entitySource) ShouldInternalProxy(opts ...EntitySourceOption) bool { for _, opt := range opts { opt.Apply(f.o) @@ -534,6 +558,10 @@ func (f *entitySource) ShouldInternalProxy(opts ...EntitySourceOption) bool { } func (f *entitySource) Url(ctx context.Context, opts ...EntitySourceOption) (*EntityUrl, error) { + if len(opts) > 0 { + // Clear cache when options are applied as they might affect URL generation + f.clearUrlCache() + } for _, opt := range opts { opt.Apply(f.o) } @@ -613,50 +641,75 @@ func (f *entitySource) Url(ctx context.Context, opts ...EntitySourceOption) (*En func (f *entitySource) resetRequest() error { // For inbound files, we can use the handler to open the file directly - if f.IsLocal() { - if f.rsc == nil { - file, err := f.handler.Open(f.o.Ctx, f.e.Source()) - if err != nil { - return fmt.Errorf("failed to open inbound file: %w", err) - } - - if f.pos > 0 { - _, err = file.Seek(f.pos, io.SeekStart) - if err != nil { - return fmt.Errorf("failed to seek inbound file: %w", err) - } - } - - f.rsc = file - - if f.o.SpeedLimit > 0 { - bucket := ratelimit.NewBucketWithRate(float64(f.o.SpeedLimit), f.o.SpeedLimit) - f.rsc = lrs{f.rsc, ratelimit.Reader(f.rsc, bucket)} - } - } - + if f.IsLocal() && f.rsc != nil { return nil } - expire := time.Now().Add(defaultUrlExpire) - u, err := f.Url(driver.WithForcePublicEndpoint(f.o.Ctx, false), WithNoInternalProxy(), WithExpire(&expire)) + rsc, err := f.getRsc(f.pos) if err != nil { - return fmt.Errorf("failed to generate download url: %w", err) + return fmt.Errorf("failed to get rsc: %w", err) + } + f.rsc = rsc + return nil +} + +func (f *entitySource) getRsc(pos int64) (io.ReadCloser, error) { + // For inbound files, we can use the handler to open the file directly + if f.IsLocal() { + file, err := f.handler.Open(f.o.Ctx, f.e.Source()) + if err != nil { + return nil, fmt.Errorf("failed to open inbound file: %w", err) + } + + if pos > 0 { + _, err = file.Seek(pos, io.SeekStart) + if err != nil { + return nil, fmt.Errorf("failed to seek inbound file: %w", err) + } + } + + if f.o.SpeedLimit > 0 { + bucket := ratelimit.NewBucketWithRate(float64(f.o.SpeedLimit), f.o.SpeedLimit) + return lrs{f.rsc, ratelimit.Reader(f.rsc, bucket)}, nil + } else { + return file, nil + } + + } + + var urlStr string + now := time.Now() + + // Check if we have a valid cached URL and expiry + if f.cachedUrl != "" && now.Before(f.cachedExpiry.Add(-time.Minute)) { + // Use cached URL if it's still valid (with 1 minute buffer before expiry) + urlStr = f.cachedUrl + } else { + // Generate new URL and cache it + expire := now.Add(defaultUrlExpire) + u, err := f.Url(driver.WithForcePublicEndpoint(f.o.Ctx, false), WithNoInternalProxy(), WithExpire(&expire)) + if err != nil { + return nil, fmt.Errorf("failed to generate download url: %w", err) + } + + // Cache the URL and expiry + f.cachedUrl = u.Url + f.cachedExpiry = expire + urlStr = u.Url } h := http.Header{} - h.Set("Range", fmt.Sprintf("bytes=%d-", f.pos)) - resp := f.c.Request(http.MethodGet, u.Url, nil, + h.Set("Range", fmt.Sprintf("bytes=%d-", pos)) + resp := f.c.Request(http.MethodGet, urlStr, nil, request.WithContext(f.o.Ctx), request.WithLogger(f.l), request.WithHeader(h), ).CheckHTTPResponse(http.StatusOK, http.StatusPartialContent) if resp.Err != nil { - return fmt.Errorf("failed to request download url: %w", resp.Err) + return nil, fmt.Errorf("failed to request download url: %w", resp.Err) } - f.rsc = resp.Response.Body - return nil + return resp.Response.Body, nil } // capExpireTime make sure expire time is not too long or too short (if min or max is set) diff --git a/pkg/filemanager/workflows/extract.go b/pkg/filemanager/workflows/extract.go index 7126e60..00616f7 100644 --- a/pkg/filemanager/workflows/extract.go +++ b/pkg/filemanager/workflows/extract.go @@ -26,7 +26,14 @@ import ( "github.com/cloudreve/Cloudreve/v4/pkg/queue" "github.com/cloudreve/Cloudreve/v4/pkg/util" "github.com/gofrs/uuid" - "github.com/mholt/archiver/v4" + "github.com/mholt/archives" + "golang.org/x/text/encoding" + "golang.org/x/text/encoding/charmap" + "golang.org/x/text/encoding/japanese" + "golang.org/x/text/encoding/korean" + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/encoding/traditionalchinese" + "golang.org/x/text/encoding/unicode" ) type ( @@ -47,6 +54,7 @@ type ( TempZipFilePath string `json:"temp_zip_file_path,omitempty"` ProcessedCursor string `json:"processed_cursor,omitempty"` SlaveTaskID int `json:"slave_task_id,omitempty"` + Password string `json:"password,omitempty"` NodeState `json:",inline"` Phase ExtractArchiveTaskPhase `json:"phase,omitempty"` } @@ -70,13 +78,54 @@ func init() { queue.RegisterResumableTaskFactory(queue.ExtractArchiveTaskType, NewExtractArchiveTaskFromModel) } +var encodings = map[string]encoding.Encoding{ + "ibm866": charmap.CodePage866, + "iso8859_2": charmap.ISO8859_2, + "iso8859_3": charmap.ISO8859_3, + "iso8859_4": charmap.ISO8859_4, + "iso8859_5": charmap.ISO8859_5, + "iso8859_6": charmap.ISO8859_6, + "iso8859_7": charmap.ISO8859_7, + "iso8859_8": charmap.ISO8859_8, + "iso8859_8I": charmap.ISO8859_8I, + "iso8859_10": charmap.ISO8859_10, + "iso8859_13": charmap.ISO8859_13, + "iso8859_14": charmap.ISO8859_14, + "iso8859_15": charmap.ISO8859_15, + "iso8859_16": charmap.ISO8859_16, + "koi8r": charmap.KOI8R, + "koi8u": charmap.KOI8U, + "macintosh": charmap.Macintosh, + "windows874": charmap.Windows874, + "windows1250": charmap.Windows1250, + "windows1251": charmap.Windows1251, + "windows1252": charmap.Windows1252, + "windows1253": charmap.Windows1253, + "windows1254": charmap.Windows1254, + "windows1255": charmap.Windows1255, + "windows1256": charmap.Windows1256, + "windows1257": charmap.Windows1257, + "windows1258": charmap.Windows1258, + "macintoshcyrillic": charmap.MacintoshCyrillic, + "gbk": simplifiedchinese.GBK, + "gb18030": simplifiedchinese.GB18030, + "big5": traditionalchinese.Big5, + "eucjp": japanese.EUCJP, + "iso2022jp": japanese.ISO2022JP, + "shiftjis": japanese.ShiftJIS, + "euckr": korean.EUCKR, + "utf16be": unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM), + "utf16le": unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM), +} + // NewExtractArchiveTask creates a new ExtractArchiveTask -func NewExtractArchiveTask(ctx context.Context, src, dst, encoding string) (queue.Task, error) { +func NewExtractArchiveTask(ctx context.Context, src, dst, encoding, password string) (queue.Task, error) { state := &ExtractArchiveTaskState{ Uri: src, Dst: dst, Encoding: encoding, NodeState: NodeState{}, + Password: password, } stateBytes, err := json.Marshal(state) if err != nil { @@ -197,6 +246,7 @@ func (m *ExtractArchiveTask) createSlaveExtractTask(ctx context.Context, dep dep Encoding: m.state.Encoding, Dst: m.state.Dst, UserID: user.ID, + Password: m.state.Password, } payloadStr, err := json.Marshal(payload) @@ -277,20 +327,21 @@ func (m *ExtractArchiveTask) masterExtractArchive(ctx context.Context, dep depen m.l.Info("Extracting archive %q to %q", uri, m.state.Dst) // Identify file format - format, readStream, err := archiver.Identify(archiveFile.DisplayName(), es) + format, readStream, err := archives.Identify(ctx, archiveFile.DisplayName(), es) if err != nil { return task.StatusError, fmt.Errorf("failed to identify archive format: %w", err) } - m.l.Info("Archive file %q format identified as %q", uri, format.Name()) + m.l.Info("Archive file %q format identified as %q", uri, format.Extension()) - extractor, ok := format.(archiver.Extractor) + extractor, ok := format.(archives.Extractor) if !ok { return task.StatusError, fmt.Errorf("format not an extractor %s") } - if format.Name() == ".zip" { - // Zip extractor requires a Seeker+ReadAt + formatExt := format.Extension() + if formatExt == ".zip" || formatExt == ".7z" { + // Zip/7Z extractor requires a Seeker+ReadAt if m.state.TempZipFilePath == "" && !es.IsLocal() { m.state.Phase = ExtractArchivePhaseDownloadZip m.ResumeAfter(0) @@ -315,11 +366,25 @@ func (m *ExtractArchiveTask) masterExtractArchive(ctx context.Context, dep depen readStream = es } + } + if zipExtractor, ok := extractor.(archives.Zip); ok { if m.state.Encoding != "" { m.l.Info("Using encoding %q for zip archive", m.state.Encoding) - extractor = archiver.Zip{TextEncoding: m.state.Encoding} + encoding, ok := encodings[strings.ToLower(m.state.Encoding)] + if !ok { + m.l.Warning("Unknown encoding %q, fallback to default encoding", m.state.Encoding) + } else { + zipExtractor.TextEncoding = encoding + extractor = zipExtractor + } } + } else if rarExtractor, ok := extractor.(archives.Rar); ok && m.state.Password != "" { + rarExtractor.Password = m.state.Password + extractor = rarExtractor + } else if sevenZipExtractor, ok := extractor.(archives.SevenZip); ok && m.state.Password != "" { + sevenZipExtractor.Password = m.state.Password + extractor = sevenZipExtractor } needSkipToCursor := false @@ -332,7 +397,7 @@ func (m *ExtractArchiveTask) masterExtractArchive(ctx context.Context, dep depen m.Unlock() // extract and upload - err = extractor.Extract(ctx, readStream, nil, func(ctx context.Context, f archiver.File) error { + err = extractor.Extract(ctx, readStream, func(ctx context.Context, f archives.FileInfo) error { if needSkipToCursor && f.NameInArchive != m.state.ProcessedCursor { atomic.AddInt64(&m.progress[ProgressTypeExtractCount].Current, 1) atomic.AddInt64(&m.progress[ProgressTypeExtractSize].Current, f.Size()) @@ -533,6 +598,7 @@ type ( TempPath string `json:"temp_path,omitempty"` TempZipFilePath string `json:"temp_zip_file_path,omitempty"` ProcessedCursor string `json:"processed_cursor,omitempty"` + Password string `json:"password,omitempty"` } ) @@ -602,18 +668,19 @@ func (m *SlaveExtractArchiveTask) Do(ctx context.Context) (task.Status, error) { defer es.Close() // 2. Identify file format - format, readStream, err := archiver.Identify(m.state.FileName, es) + format, readStream, err := archives.Identify(ctx, m.state.FileName, es) if err != nil { return task.StatusError, fmt.Errorf("failed to identify archive format: %w", err) } - m.l.Info("Archive file %q format identified as %q", m.state.FileName, format.Name()) + m.l.Info("Archive file %q format identified as %q", m.state.FileName, format.Extension()) - extractor, ok := format.(archiver.Extractor) + extractor, ok := format.(archives.Extractor) if !ok { - return task.StatusError, fmt.Errorf("format not an extractor %s") + return task.StatusError, fmt.Errorf("format not an extractor %q", format.Extension()) } - if format.Name() == ".zip" { + formatExt := format.Extension() + if formatExt == ".zip" || formatExt == ".7z" { if _, err = es.Seek(0, 0); err != nil { return task.StatusError, fmt.Errorf("failed to seek entity source: %w", err) } @@ -666,11 +733,25 @@ func (m *SlaveExtractArchiveTask) Do(ctx context.Context) (task.Status, error) { if es.IsLocal() { readStream = es } + } + if zipExtractor, ok := extractor.(archives.Zip); ok { if m.state.Encoding != "" { m.l.Info("Using encoding %q for zip archive", m.state.Encoding) - extractor = archiver.Zip{TextEncoding: m.state.Encoding} + encoding, ok := encodings[strings.ToLower(m.state.Encoding)] + if !ok { + m.l.Warning("Unknown encoding %q, fallback to default encoding", m.state.Encoding) + } else { + zipExtractor.TextEncoding = encoding + extractor = zipExtractor + } } + } else if rarExtractor, ok := extractor.(archives.Rar); ok && m.state.Password != "" { + rarExtractor.Password = m.state.Password + extractor = rarExtractor + } else if sevenZipExtractor, ok := extractor.(archives.SevenZip); ok && m.state.Password != "" { + sevenZipExtractor.Password = m.state.Password + extractor = sevenZipExtractor } needSkipToCursor := false @@ -679,7 +760,7 @@ func (m *SlaveExtractArchiveTask) Do(ctx context.Context) (task.Status, error) { } // 3. Extract and upload - err = extractor.Extract(ctx, readStream, nil, func(ctx context.Context, f archiver.File) error { + err = extractor.Extract(ctx, readStream, func(ctx context.Context, f archives.FileInfo) error { if needSkipToCursor && f.NameInArchive != m.state.ProcessedCursor { atomic.AddInt64(&m.progress[ProgressTypeExtractCount].Current, 1) atomic.AddInt64(&m.progress[ProgressTypeExtractSize].Current, f.Size()) diff --git a/service/explorer/workflows.go b/service/explorer/workflows.go index 7d54bfc..138b292 100644 --- a/service/explorer/workflows.go +++ b/service/explorer/workflows.go @@ -173,6 +173,7 @@ type ( Src []string `json:"src" binding:"required"` Dst string `json:"dst" binding:"required"` Encoding string `json:"encoding"` + Password string `json:"password"` } CreateArchiveParamCtx struct{} ) @@ -203,7 +204,7 @@ func (service *ArchiveWorkflowService) CreateExtractTask(c *gin.Context) (*TaskR } // Create task - t, err := workflows.NewExtractArchiveTask(c, service.Src[0], service.Dst, service.Encoding) + t, err := workflows.NewExtractArchiveTask(c, service.Src[0], service.Dst, service.Encoding, service.Password) if err != nil { return nil, serializer.NewError(serializer.CodeCreateTaskError, "Failed to create task", err) }