feats: (Un)archiving & dir size (#1)

* feat: file archiving (#1)

* feat: file archiving

* resource: use name query for archive name

* feat: file unarchiving

* resource: Return bad param error on unarchive fail

* fix: adjust style according to lint

* feat: directory size calculation (#2)
pull/3756/head
Laurynas Gadliauskas 2021-05-27 16:52:05 +03:00 committed by GitHub
parent 46ee595389
commit 11092eed3c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 575 additions and 1 deletions

View File

@ -39,6 +39,8 @@ type FileInfo struct {
Content string `json:"content,omitempty"`
Checksums map[string]string `json:"checksums,omitempty"`
Token string `json:"token,omitempty"`
DiskUsage int64 `json:"diskUsage,omitempty"`
Inodes int64 `json:"inodes,omitempty"`
}
// FileOptions are the options when getting a file info.

View File

@ -2,6 +2,7 @@ package fileutils
import (
"errors"
"path/filepath"
"github.com/spf13/afero"
)
@ -60,3 +61,46 @@ func CopyDir(fs afero.Fs, source, dest string) error {
return nil
}
func DiskUsage(fs afero.Fs, path string, maxDepth int) (size, inodes int64, err error) {
info, err := fs.Stat(path)
if err != nil {
return 0, 0, err
}
size = info.Size()
inodes = int64(1)
if !info.IsDir() {
return size, inodes, err
}
if maxDepth < 1 {
return size, inodes, err
}
dir, err := fs.Open(path)
if err != nil {
return size, inodes, err
}
defer dir.Close()
fis, err := dir.Readdir(-1)
if err != nil {
return size, inodes, err
}
for _, fi := range fis {
if fi.Name() == "." || fi.Name() == ".." {
continue
}
s, i, e := DiskUsage(fs, filepath.Join(path, fi.Name()), maxDepth-1)
if e != nil {
return size, inodes, e
}
size += s
inodes += i
}
return size, inodes, err
}

120
fileutils/dir_test.go Normal file
View File

@ -0,0 +1,120 @@
package fileutils
import (
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/spf13/afero"
"github.com/stretchr/testify/require"
)
const RootTestDir string = "testing_dir_func"
func createFileStructure() error {
childDir := filepath.Join(RootTestDir, "child_dir")
err := os.MkdirAll(childDir, 0755)
if err != nil {
return err
}
data := []byte("test_data")
err = ioutil.WriteFile(filepath.Join(childDir, "test_file"), data, 0600)
if err != nil {
return err
}
err = ioutil.WriteFile(filepath.Join(RootTestDir, "test_file"), data, 0600)
if err != nil {
return err
}
return nil
}
func cleanupFileStructure() {
os.RemoveAll(RootTestDir)
}
func TestDiskUsageOnFile(t *testing.T) {
err := createFileStructure()
if err != nil {
t.Errorf("createFileStructure() failed: %s", err)
}
defer cleanupFileStructure()
cwd, err := os.Getwd()
if err != nil {
t.Errorf("Getwd() failed: %s", err)
}
fs := afero.NewBasePathFs(afero.NewOsFs(), cwd)
size, inodes, err := DiskUsage(fs, filepath.Join(RootTestDir, "test_file"), 100)
require.NoError(t, err)
require.Equal(t, int64(9), size)
require.Equal(t, int64(1), inodes)
}
func TestDiskUsageOnNestedDir(t *testing.T) {
err := createFileStructure()
if err != nil {
t.Errorf("createFileStructure() failed: %s", err)
}
defer cleanupFileStructure()
cwd, err := os.Getwd()
if err != nil {
t.Errorf("Getwd() failed: %s", err)
}
fs := afero.NewBasePathFs(afero.NewOsFs(), cwd)
size, inodes, err := DiskUsage(fs, filepath.Join(RootTestDir, "child_dir"), 100)
require.NoError(t, err)
require.Equal(t, int64(105), size)
require.Equal(t, int64(2), inodes)
}
func TestDiskUsageOnRootDir(t *testing.T) {
err := createFileStructure()
if err != nil {
t.Errorf("createFileStructure() failed: %s", err)
}
defer cleanupFileStructure()
cwd, err := os.Getwd()
if err != nil {
t.Errorf("Getwd() failed: %s", err)
}
fs := afero.NewBasePathFs(afero.NewOsFs(), cwd)
size, inodes, err := DiskUsage(fs, RootTestDir, 100)
require.NoError(t, err)
require.Equal(t, int64(242), size)
require.Equal(t, int64(4), inodes)
}
func TestDiskUsageOnRootDirStopsAtDepthLimit(t *testing.T) {
err := createFileStructure()
if err != nil {
t.Errorf("createFileStructure() failed: %s", err)
}
defer cleanupFileStructure()
cwd, err := os.Getwd()
if err != nil {
t.Errorf("Getwd() failed: %s", err)
}
fs := afero.NewBasePathFs(afero.NewOsFs(), cwd)
size, inodes, err := DiskUsage(fs, RootTestDir, 1)
require.NoError(t, err)
require.Equal(t, int64(233), size)
require.Equal(t, int64(3), inodes)
}

View File

@ -154,3 +154,34 @@ export async function checksum(url, algo) {
const data = await resourceAction(`${url}?checksum=${algo}`, "GET");
return (await data.json()).checksums[algo];
}
export async function diskUsage(url) {
const data = await resourceAction(`${url}?disk_usage=true`, "GET");
return await data.json();
}
export async function archive(url, name, format, ...files) {
let arg = "";
for (let file of files) {
arg += file + ",";
}
arg = arg.substring(0, arg.length - 1);
arg = encodeURIComponent(arg);
url += `?files=${arg}&`;
url += `name=${encodeURIComponent(name)}&`;
if (format) {
url += `algo=${format}&`;
}
return post(url);
}
export async function unarchive(path, name, override) {
const to = encodeURIComponent(removePrefix(name));
const action = `unarchive`;
const url = `${path}?action=${action}&destination=${to}&override=${override}`;
return resourceAction(url, "PATCH");
}

View File

@ -0,0 +1,88 @@
<template>
<div class="card floating">
<div class="card-title">
<h2>{{ $t("prompts.archive") }}</h2>
</div>
<div class="card-content">
<p>{{ $t("prompts.archiveMessage") }}</p>
<input
class="input input--block"
v-focus
type="text"
@keyup.enter="submit"
v-model.trim="name"
required
/>
<button
v-for="(ext, format) in formats"
:key="format"
class="button button--block"
@click="archive(format)"
v-focus
>
{{ ext }}
</button>
</div>
</div>
</template>
<script>
import { mapState, mapGetters } from "vuex";
import { files as api } from "@/api";
import url from "@/utils/url";
export default {
name: "archive",
data: function () {
return {
name: "",
formats: {
zip: "zip",
tar: "tar",
targz: "tar.gz",
tarbz2: "tar.bz2",
tarxz: "tar.xz",
tarlz4: "tar.lz4",
tarsz: "tar.sz",
},
};
},
computed: {
...mapState(["req", "selected"]),
...mapGetters(["isFiles", "isListing"]),
},
methods: {
cancel: function () {
this.$store.commit("closeHovers");
},
archive: async function (format) {
let items = [];
for (let i of this.selected) {
items.push(this.req.items[i].name);
}
let uri = this.isFiles ? this.$route.path : "/";
if (!this.isListing) {
uri = url.removeLastDir(uri);
}
uri += "/archive";
uri = uri.replace("//", "/");
try {
await api.archive(uri, this.name, format, ...items);
this.$store.commit("setReload", true);
} catch (e) {
this.$showError(e);
}
this.$store.commit("closeHovers");
},
},
};
</script>

View File

@ -12,10 +12,16 @@
<p class="break-word" v-if="selected.length < 2">
<strong>{{ $t("prompts.displayName") }}</strong> {{ name }}
</p>
<p v-if="!dir || selected.length > 1">
<p v-if="!dir">
<strong>{{ $t("prompts.size") }}:</strong>
<span id="content_length"></span> {{ humanSize }}
</p>
<p v-if="dir">
<strong>{{ $t("prompts.size") }}: </strong>
<code>
<a @click="diskUsage($event)">{{ $t("prompts.show") }}</a>
</code>
</p>
<p v-if="selected.length < 2" :title="modTime">
<strong>{{ $t("prompts.lastModified") }}:</strong> {{ humanTime }}
</p>
@ -147,6 +153,38 @@ export default {
this.$showError(e);
}
},
diskUsage: async function (event) {
event.preventDefault();
// eslint-disable-next-line
event.target.innerHTML = this.$t('files.loading');
let links = [];
if (this.selectedCount === 0 || !this.isListing) {
links.push(this.$route.path);
} else {
for (let selected of this.selected) {
links.push(this.req.items[selected].url);
}
}
let size = 0;
let inodes = 0;
for (let link of links) {
try {
let data = await api.diskUsage(link);
size += data.diskUsage;
inodes += data.inodes;
} catch (e) {
this.$showError(e);
}
}
// eslint-disable-next-line
event.target.innerHTML = filesize(size) + " " + this.$t("prompts.inodeCount", { count: inodes })
},
},
};
</script>

View File

@ -12,6 +12,8 @@ import Delete from "./Delete";
import Rename from "./Rename";
import Download from "./Download";
import Move from "./Move";
import Archive from "./Archive";
import Unarchive from "./Unarchive";
import Copy from "./Copy";
import NewFile from "./NewFile";
import NewDir from "./NewDir";
@ -31,6 +33,8 @@ export default {
Rename,
Download,
Move,
Archive,
Unarchive,
Copy,
Share,
NewFile,
@ -91,6 +95,8 @@ export default {
"delete",
"rename",
"move",
"archive",
"unarchive",
"copy",
"newFile",
"newDir",

View File

@ -0,0 +1,77 @@
<template>
<div class="card floating">
<div class="card-title">
<h2>{{ $t("prompts.unarchive") }}</h2>
</div>
<div class="card-content">
<p>{{ $t("prompts.unarchiveMessage") }}</p>
<input
class="input input--block"
v-focus
type="text"
@keyup.enter="submit"
v-model.trim="name"
/>
</div>
<div class="card-action">
<button
class="button button--flat button--grey"
@click="$store.commit('closeHovers')"
:aria-label="$t('buttons.cancel')"
:title="$t('buttons.cancel')"
>
{{ $t("buttons.cancel") }}
</button>
<button
@click="submit"
class="button button--flat"
type="submit"
:aria-label="$t('buttons.unarchive')"
:title="$t('buttons.unarchive')"
>
{{ $t("buttons.unarchive") }}
</button>
</div>
</div>
</template>
<script>
import { mapState, mapGetters } from "vuex";
import { files as api } from "@/api";
export default {
name: "rename",
data: function () {
return {
name: "",
};
},
computed: {
...mapState(["req", "selected", "selectedCount"]),
...mapGetters(["isListing", "isFiles"]),
},
methods: {
cancel: function () {
this.$store.commit("closeHovers");
},
submit: async function () {
let item = this.req.items[this.selected[0]];
let uri = this.isFiles ? this.$route.path + "/" : "/";
let dst = uri + this.name;
dst = dst.replace("//", "/");
try {
await api.unarchive(item.url, dst, false);
this.$store.commit("setReload", true);
} catch (e) {
this.$showError(e);
}
this.$store.commit("closeHovers");
},
},
};
</script>

View File

@ -1,5 +1,7 @@
{
"buttons": {
"archive": "Archive",
"unarchive": "Unarchive",
"cancel": "Cancel",
"close": "Close",
"copy": "Copy",
@ -113,6 +115,10 @@
},
"permanent": "Permanent",
"prompts": {
"archive": "Archive",
"archiveMessage": "Choose archive name and format:",
"unarchive": "Unarchive",
"unarchiveMessage": "Choose the destination folder name:",
"copy": "Copy",
"copyMessage": "Choose the place to copy your files:",
"currentlyNavigating": "Currently navigating on:",
@ -144,6 +150,7 @@
"scheduleMessage": "Pick a date and time to schedule the publication of this post.",
"show": "Show",
"size": "Size",
"inodeCount": "({count} inodes)",
"upload": "Upload",
"uploadMessage": "Select an option to upload.",
"optionalPassword": "Optional password"

View File

@ -37,6 +37,20 @@
:label="$t('buttons.moveFile')"
show="move"
/>
<action
v-if="headerButtons.archive"
id="archive-button"
icon="archive"
:label="$t('buttons.archive')"
show="archive"
/>
<action
v-if="headerButtons.unarchive"
id="unarchive-button"
icon="unarchive"
:label="$t('buttons.unarchive')"
show="unarchive"
/>
<action
v-if="headerButtons.delete"
id="delete-button"
@ -366,11 +380,29 @@ export default {
share: this.selectedCount === 1 && this.user.perm.share,
move: this.selectedCount > 0 && this.user.perm.rename,
copy: this.selectedCount > 0 && this.user.perm.create,
archive: this.selectedCount > 0 && this.user.perm.create,
unarchive: this.selectedCount === 1 && this.onlyArchivesSelected,
};
},
isMobile() {
return this.width <= 736;
},
onlyArchivesSelected() {
let extensions = [".zip", ".tar", ".gz", ".bz2", ".xz", ".lz4", ".sz"];
if (this.selectedCount < 1) {
return false;
}
for (const i of this.selected) {
let item = this.req.items[i];
if (item.isDir || !extensions.includes(item.extension)) {
return false;
}
}
return true;
},
},
watch: {
req: function () {

View File

@ -12,6 +12,7 @@ import (
"path/filepath"
"strings"
"github.com/mholt/archiver"
"github.com/spf13/afero"
"github.com/filebrowser/filebrowser/v2/errors"
@ -33,6 +34,17 @@ var resourceGetHandler = withUser(func(w http.ResponseWriter, r *http.Request, d
return errToStatus(err), err
}
if r.URL.Query().Get("disk_usage") == "true" {
du, inodes, err := fileutils.DiskUsage(file.Fs, file.Path, 100)
if err != nil {
return http.StatusInternalServerError, err
}
file.DiskUsage = du
file.Inodes = inodes
file.Content = ""
return renderJSON(w, r, file)
}
if file.IsDir {
file.Listing.Sorting = d.user.Sorting
file.Listing.ApplySort()
@ -106,6 +118,15 @@ func resourcePostHandler(fileCache FileCache) handleFunc {
return errToStatus(err), err
}
// Archive creation on POST.
if strings.HasSuffix(r.URL.Path, "/archive") {
if !d.user.Perm.Create {
return http.StatusForbidden, nil
}
return archiveHandler(r, d)
}
file, err := files.NewFileInfo(files.FileOptions{
Fs: d.user.Fs,
Path: r.URL.Path,
@ -305,6 +326,20 @@ func patchAction(ctx context.Context, action, src, dst string, d *data, fileCach
}
return fileutils.Copy(d.user.Fs, src, dst)
case "unarchive":
if !d.user.Perm.Create {
return errors.ErrPermissionDenied
}
src = d.user.FullPath(path.Clean("/" + src))
dst = d.user.FullPath(path.Clean("/" + dst))
// THIS COULD BE VUNERABLE TO https://github.com/snyk/zip-slip-vulnerability
err := archiver.Unarchive(src, dst)
if err != nil {
return errors.ErrInvalidRequestParams
}
return nil
case "rename":
if !d.user.Perm.Rename {
return errors.ErrPermissionDenied
@ -335,3 +370,97 @@ func patchAction(ctx context.Context, action, src, dst string, d *data, fileCach
return fmt.Errorf("unsupported action %s: %w", action, errors.ErrInvalidRequestParams)
}
}
func archiveHandler(r *http.Request, d *data) (int, error) {
dir := strings.TrimSuffix(r.URL.Path, "/archive")
destDir, err := files.NewFileInfo(files.FileOptions{
Fs: d.user.Fs,
Path: dir,
Modify: d.user.Perm.Modify,
Expand: false,
ReadHeader: false,
Checker: d,
})
if err != nil {
return errToStatus(err), err
}
filenames, err := parseQueryFiles(r, destDir, d.user)
if err != nil {
return http.StatusInternalServerError, err
}
archFile, err := parseQueryFilename(r, destDir)
if err != nil {
return http.StatusBadRequest, err
}
extension, ar, err := parseArchiver(r.URL.Query().Get("algo"))
if err != nil {
return http.StatusInternalServerError, err
}
archFile += extension
_, err = d.user.Fs.Stat(archFile)
if err == nil {
return http.StatusConflict, nil
}
dir, _ = path.Split(archFile)
err = d.user.Fs.MkdirAll(dir, 0775)
if err != nil {
return errToStatus(err), err
}
for i, path := range filenames {
_, err = d.user.Fs.Stat(path)
if err != nil {
return errToStatus(err), err
}
filenames[i] = d.user.FullPath(path)
}
dst := d.user.FullPath(archFile)
err = ar.Archive(filenames, dst)
if err != nil {
return http.StatusInternalServerError, err
}
return errToStatus(err), err
}
func parseQueryFilename(r *http.Request, f *files.FileInfo) (string, error) {
name := r.URL.Query().Get("name")
name, err := url.QueryUnescape(strings.Replace(name, "+", "%2B", -1))
if err != nil {
return "", err
}
name = strings.TrimSpace(name)
if name == "" {
return "", fmt.Errorf("empty name provided")
}
return filepath.Join(f.Path, slashClean(name)), nil
}
func parseArchiver(algo string) (string, archiver.Archiver, error) {
switch algo {
case "zip", "true", "":
return ".zip", archiver.NewZip(), nil
case "tar":
return ".tar", archiver.NewTar(), nil
case "targz":
return ".tar.gz", archiver.NewTarGz(), nil
case "tarbz2":
return ".tar.bz2", archiver.NewTarBz2(), nil
case "tarxz":
return ".tar.xz", archiver.NewTarXz(), nil
case "tarlz4":
return ".tar.lz4", archiver.NewTarLz4(), nil
case "tarsz":
return ".tar.sz", archiver.NewTarSz(), nil
default:
return "", nil, fmt.Errorf("format not implemented")
}
}