feat: add file size and type restriction for local file uploads (#6390)

#### What type of PR is this? /kind feature /area core #### What this PR does / why we need it: 本次 PR 为本地附件存储策略增加了对上传单文件大小和文件类型限制的功能，具体包括： 1. 单文件大小限制：实现了对单个文件上传大小的验证功能，确保上传文件不超过设定的最大值。 2. 文件类型限制：添加了文件类型限制功能，使用 Apache Tika 读取上传文件的 magic numbers 得到文件 mime type 并根据用户配置来决定是否允许上传参考链接： - [List of file signatures](https://en.wikipedia.org/wiki/List_of_file_signatures) - [File Magic Numbers: The Easy way to Identify File Extensions](https://library.mosse-institute.com/articles/2022/04/file-magic-numbers-the-easy-way-to-identify-file-extensions/file-magic-numbers-the-easy-way-to-identify-file-extensions.html) #### Which issue(s) this PR fixes: Fixes #6385 #### Does this PR introduce a user-facing change? ```release-note 为本地附件存储策略增加了对上传单文件大小和文件类型限制的功能 ```
2024-08-01 09:58:12 +08:00 · 2024-08-01 09:58:12 +08:00 · 58fe872844
parent 39ff455178
commit 58fe872844
11 changed files with 326 additions and 2 deletions
--- a/api/build.gradle
+++ b/api/build.gradle
@ -65,6 +65,7 @@ dependencies {
    api "org.springframework.integration:spring-integration-core"
    api "com.github.java-json-tools:json-patch"
    api "org.thymeleaf.extras:thymeleaf-extras-springsecurity6"
+    api 'org.apache.tika:tika-core'

    api "io.github.resilience4j:resilience4j-spring-boot3"
    api "io.github.resilience4j:resilience4j-reactor"
--- a/api/src/main/java/run/halo/app/infra/FileCategoryMatcher.java
+++ b/api/src/main/java/run/halo/app/infra/FileCategoryMatcher.java
@ -0,0 +1,107 @@
+package run.halo.app.infra;
+
+import java.util.Set;
+
+/**
+ * <p>Classifies files based on their MIME types.</p>
+ * <p>It provides different categories such as IMAGE, SVG, AUDIO, VIDEO, ARCHIVE, and DOCUMENT.
+ * Each category has a <code>match</code> method that checks if a given MIME type belongs to that
+ * category.</p>
+ * <p>The categories are defined as follows:</p>
+ * <pre>
+ * - IMAGE: Matches all image MIME types except for SVG.
+ * - SVG: Specifically matches the SVG image MIME type.
+ * - AUDIO: Matches all audio MIME types.
+ * - VIDEO: Matches all video MIME types.
+ * - ARCHIVE: Matches common archive MIME types like zip, rar, tar, etc.
+ * - DOCUMENT: Matches common document MIME types like plain text, PDF, Word, Excel, etc.
+ * </pre>
+ *
+ * @author guqing
+ * @since 2.18.0
+ */
+public enum FileCategoryMatcher {
+    ALL {
+        @Override
+        public boolean match(String mimeType) {
+            return true;
+        }
+    },
+    IMAGE {
+        @Override
+        public boolean match(String mimeType) {
+            return mimeType.startsWith("image/") && !mimeType.equals("image/svg+xml");
+        }
+    },
+    SVG {
+        @Override
+        public boolean match(String mimeType) {
+            return mimeType.equals("image/svg+xml");
+        }
+    },
+    AUDIO {
+        @Override
+        public boolean match(String mimeType) {
+            return mimeType.startsWith("audio/");
+        }
+    },
+    VIDEO {
+        @Override
+        public boolean match(String mimeType) {
+            return mimeType.startsWith("video/");
+        }
+    },
+    ARCHIVE {
+        static final Set<String> ARCHIVE_MIME_TYPES = Set.of(
+            "application/zip",
+            "application/x-rar-compressed",
+            "application/x-tar",
+            "application/gzip",
+            "application/x-bzip2",
+            "application/x-xz",
+            "application/x-7z-compressed"
+        );
+
+        @Override
+        public boolean match(String mimeType) {
+            return ARCHIVE_MIME_TYPES.contains(mimeType);
+        }
+    },
+    DOCUMENT {
+        static final Set<String> DOCUMENT_MIME_TYPES = Set.of(
+            "text/plain",
+            "application/rtf",
+            "text/csv",
+            "text/xml",
+            "application/pdf",
+            "application/msword",
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+            "application/vnd.ms-excel",
+            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+            "application/vnd.ms-powerpoint",
+            "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+            "application/vnd.oasis.opendocument.text",
+            "application/vnd.oasis.opendocument.spreadsheet",
+            "application/vnd.oasis.opendocument.presentation"
+        );
+
+        @Override
+        public boolean match(String mimeType) {
+            return DOCUMENT_MIME_TYPES.contains(mimeType);
+        }
+    };
+
+    public abstract boolean match(String mimeType);
+
+    /**
+     * Get the file category matcher by name.
+     */
+    public static FileCategoryMatcher of(String name) {
+        for (var matcher : values()) {
+            if (matcher.name().equalsIgnoreCase(name)) {
+                return matcher;
+            }
+        }
+        throw new IllegalArgumentException("Unsupported file category matcher for name: " + name);
+    }
+}
--- a/api/src/main/java/run/halo/app/infra/utils/FileTypeDetectUtils.java
+++ b/api/src/main/java/run/halo/app/infra/utils/FileTypeDetectUtils.java
@ -0,0 +1,34 @@
+package run.halo.app.infra.utils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import lombok.experimental.UtilityClass;
+import org.apache.tika.Tika;
+import org.apache.tika.mime.MimeTypeException;
+import org.apache.tika.mime.MimeTypes;
+
+@UtilityClass
+public class FileTypeDetectUtils {
+
+    private static final Tika tika = new Tika();
+
+    /**
+     * Detect mime type.
+     *
+     * @param inputStream input stream will be closed after detection.
+     */
+    public static String detectMimeType(InputStream inputStream) throws IOException {
+        try {
+            return tika.detect(inputStream);
+        } finally {
+            if (inputStream != null) {
+                inputStream.close();
+            }
+        }
+    }
+
+    public static String detectFileExtension(String mimeType) throws MimeTypeException {
+        MimeTypes mimeTypes = MimeTypes.getDefaultMimeTypes();
+        return mimeTypes.forName(mimeType).getExtension();
+    }
+}
--- a/application/src/main/java/run/halo/app/core/extension/attachment/endpoint/LocalAttachmentUploadHandler.java
+++ b/application/src/main/java/run/halo/app/core/extension/attachment/endpoint/LocalAttachmentUploadHandler.java
@ -15,15 +15,20 @@ import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicReference;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
+import org.reactivestreams.Publisher;
 import org.springframework.core.io.buffer.DataBuffer;
 import org.springframework.core.io.buffer.DataBufferUtils;
 import org.springframework.http.MediaType;
+import org.springframework.http.codec.multipart.FilePart;
 import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
 import org.springframework.util.StringUtils;
+import org.springframework.util.unit.DataSize;
 import org.springframework.web.util.UriComponentsBuilder;
 import org.springframework.web.util.UriUtils;
 import reactor.core.Exceptions;
@ -38,8 +43,12 @@ import run.halo.app.core.extension.attachment.Policy;
 import run.halo.app.extension.ConfigMap;
 import run.halo.app.extension.Metadata;
 import run.halo.app.infra.ExternalUrlSupplier;
+import run.halo.app.infra.FileCategoryMatcher;
 import run.halo.app.infra.exception.AttachmentAlreadyExistsException;
+import run.halo.app.infra.exception.FileSizeExceededException;
+import run.halo.app.infra.exception.FileTypeNotAllowedException;
 import run.halo.app.infra.properties.HaloProperties;
+import run.halo.app.infra.utils.FileTypeDetectUtils;
 import run.halo.app.infra.utils.JsonUtils;

@Slf4j
@ -81,7 +90,7 @@ class LocalAttachmentUploadHandler implements AttachmentHandler {
                }
                checkDirectoryTraversal(uploadRoot, attachmentPath);

-                return Mono.fromRunnable(
+                return validateFile(file, setting).then(Mono.fromRunnable(
                        () -> {
                            try {
                                // init parent folders
@ -125,10 +134,55 @@ class LocalAttachmentUploadHandler implements AttachmentHandler {
                        return attachment;
                    })
                    .onErrorMap(FileAlreadyExistsException.class,
-                        e -> new AttachmentAlreadyExistsException(e.getFile()));
+                        e -> new AttachmentAlreadyExistsException(e.getFile()))
+                );
            });
    }

+    private Mono<Void> validateFile(FilePart file, PolicySetting setting) {
+        var validations = new ArrayList<Publisher<?>>(2);
+        var maxSize = setting.getMaxFileSize();
+        if (maxSize != null && maxSize.toBytes() > 0) {
+            validations.add(
+                file.content()
+                    .map(DataBuffer::readableByteCount)
+                    .reduce(0L, Long::sum)
+                    .filter(size -> size <= setting.getMaxFileSize().toBytes())
+                    .switchIfEmpty(Mono.error(new FileSizeExceededException(
+                        "File size exceeds the maximum limit",
+                        "problemDetail.attachment.upload.fileSizeExceeded",
+                        new Object[] {setting.getMaxFileSize().toKilobytes() + "KB"})
+                    ))
+            );
+        }
+        if (!CollectionUtils.isEmpty(setting.getAllowedFileTypes())) {
+            var typeValidator = file.content()
+                .next()
+                .handle((dataBuffer, sink) -> {
+                    var mimeType = "Unknown";
+                    try {
+                        mimeType = FileTypeDetectUtils.detectMimeType(dataBuffer.asInputStream());
+                        var isAllow = setting.getAllowedFileTypes()
+                            .stream()
+                            .map(FileCategoryMatcher::of)
+                            .anyMatch(matcher -> matcher.match(file.filename()));
+                        if (isAllow) {
+                            sink.next(dataBuffer);
+                            return;
+                        }
+                    } catch (IOException e) {
+                        log.warn("Failed to detect file type", e);
+                    }
+                    sink.error(new FileTypeNotAllowedException("File type is not allowed",
+                        "problemDetail.attachment.upload.fileTypeNotSupported",
+                        new Object[] {mimeType})
+                    );
+                });
+            validations.add(typeValidator);
+        }
+        return Mono.when(validations);
+    }
+
    @Override
    public Mono<Attachment> delete(DeleteContext deleteContext) {
        return Mono.just(deleteContext)
@ -206,6 +260,16 @@ class LocalAttachmentUploadHandler implements AttachmentHandler {

        private String location;

+        private DataSize maxFileSize;
+
+        private Set<String> allowedFileTypes;
+
+        public void setMaxFileSize(String maxFileSize) {
+            if (!StringUtils.hasText(maxFileSize)) {
+                return;
+            }
+            this.maxFileSize = DataSize.parse(maxFileSize);
+        }
    }

    /**
--- a/application/src/main/java/run/halo/app/infra/exception/FileSizeExceededException.java
+++ b/application/src/main/java/run/halo/app/infra/exception/FileSizeExceededException.java
@ -0,0 +1,18 @@
+package run.halo.app.infra.exception;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.web.server.ResponseStatusException;
+
+public class FileSizeExceededException extends ResponseStatusException {
+
+    public FileSizeExceededException(String reason, String messageDetailCode,
+        Object[] messageDetailArguments) {
+        this(reason, null, messageDetailCode, messageDetailArguments);
+    }
+
+    public FileSizeExceededException(String reason, Throwable cause,
+        String messageDetailCode, Object[] messageDetailArguments) {
+        super(HttpStatus.PAYLOAD_TOO_LARGE, reason, cause, messageDetailCode,
+            messageDetailArguments);
+    }
+}
--- a/application/src/main/java/run/halo/app/infra/exception/FileTypeNotAllowedException.java
+++ b/application/src/main/java/run/halo/app/infra/exception/FileTypeNotAllowedException.java
@ -0,0 +1,18 @@
+package run.halo.app.infra.exception;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.web.server.ResponseStatusException;
+
+public class FileTypeNotAllowedException extends ResponseStatusException {
+
+    public FileTypeNotAllowedException(String reason, String messageDetailCode,
+        Object[] messageDetailArguments) {
+        this(reason, null, messageDetailCode, messageDetailArguments);
+    }
+
+    public FileTypeNotAllowedException(String reason, Throwable cause,
+        String messageDetailCode, Object[] messageDetailArguments) {
+        super(HttpStatus.UNSUPPORTED_MEDIA_TYPE, reason, cause, messageDetailCode,
+            messageDetailArguments);
+    }
+}
--- a/application/src/main/resources/config/i18n/messages.properties
+++ b/application/src/main/resources/config/i18n/messages.properties
@ -11,6 +11,8 @@ problemDetail.title.org.springframework.web.server.MethodNotAllowedException=Met
 problemDetail.title.org.springframework.security.authentication.BadCredentialsException=Bad Credentials
 problemDetail.title.run.halo.app.extension.exception.SchemaViolationException=Schema Violation
 problemDetail.title.run.halo.app.infra.exception.AttachmentAlreadyExistsException=Attachment Already Exists
+problemDetail.title.run.halo.app.infra.exception.FileTypeNotAllowedException=File Type Not Allowed
+problemDetail.title.run.halo.app.infra.exception.FileSizeExceededException=File Size Exceeded
 problemDetail.title.run.halo.app.infra.exception.AccessDeniedException=Access Denied
 problemDetail.title.reactor.core.Exceptions.RetryExhaustedException=Retry Exhausted
 problemDetail.title.run.halo.app.infra.exception.ThemeInstallationException=Theme Install Error
@ -75,5 +77,7 @@ problemDetail.plugin.missingManifest=Missing plugin manifest file "plugin.yaml"
 problemDetail.internalServerError=Something went wrong, please try again later.
 problemDetail.conflict=Conflict detected, please check the data and retry.
 problemDetail.migration.backup.notFound=The backup file does not exist or has been deleted.
+problemDetail.attachment.upload.fileSizeExceeded=Make sure the file size is less than {0}.
+problemDetail.attachment.upload.fileTypeNotSupported=Unsupported upload of {0} type files.

 title.visibility.identification.private=(Private)
--- a/application/src/main/resources/config/i18n/messages_zh.properties
+++ b/application/src/main/resources/config/i18n/messages_zh.properties
@ -3,6 +3,8 @@ problemDetail.title.org.springframework.security.authentication.BadCredentialsEx
 problemDetail.title.run.halo.app.infra.exception.UnsatisfiedAttributeValueException=请求参数属性值不满足要求
 problemDetail.title.run.halo.app.infra.exception.PluginInstallationException=插件安装失败
 problemDetail.title.run.halo.app.infra.exception.AttachmentAlreadyExistsException=附件已存在
+problemDetail.title.run.halo.app.infra.exception.FileTypeNotAllowedException=文件类型不允许
+problemDetail.title.run.halo.app.infra.exception.FileSizeExceededException=文件大小超出限制
 problemDetail.title.run.halo.app.infra.exception.DuplicateNameException=名称重复
 problemDetail.title.run.halo.app.infra.exception.PluginAlreadyExistsException=插件已存在
 problemDetail.title.run.halo.app.infra.exception.ThemeInstallationException=主题安装失败
@ -47,5 +49,7 @@ problemDetail.theme.install.alreadyExists=主题 {0} 已存在。
 problemDetail.internalServerError=服务器内部发生错误，请稍候再试。
 problemDetail.conflict=检测到冲突，请检查数据后重试。
 problemDetail.migration.backup.notFound=备份文件不存在或已删除。
+problemDetail.attachment.upload.fileSizeExceeded=最大支持上传 {0} 大小的文件。
+problemDetail.attachment.upload.fileTypeNotSupported=不支持上传 {0} 类型的文件。

 title.visibility.identification.private=（私有）
--- a/application/src/main/resources/extensions/attachment-local-policy.yaml
+++ b/application/src/main/resources/extensions/attachment-local-policy.yaml
@ -35,6 +35,33 @@ spec:
          name: location
          label: 存储位置
          help: ~/.halo2/attachments/upload 下的子目录
+        - $formkit: text
+          name: maxFileSize
+          label: 最大单文件大小
+          validation: [['matches', '/^(0|[1-9]\d*)(?:[KMG]B)?$/']]
+          validation-visibility: "live"
+          validation-messages:
+            matches: "输入格式错误，遵循：整数 + 大写的单位（KB, MB, GB）"
+          help: "0 表示不限制，示例：5KB、10MB、1GB"
+        - $formkit: checkbox
+          name: allowedFileTypes
+          label: 文件类型限制
+          help: 限制允许上传的文件类型
+          options:
+            - label: 无限制
+              value: ALL
+            - label: 图片
+              value: IMAGE
+            - label: SVG
+              value: SVG
+            - label: 视频
+              value: VIDEO
+            - label: 音频
+              value: AUDIO
+            - label: 文档
+              value: DOCUMENT
+            - label: 压缩包
+              value: ARCHIVE
 ---
 apiVersion: storage.halo.run/v1alpha1
 kind: Group
--- a/application/src/test/java/run/halo/app/infra/utils/FileTypeDetectUtilsTest.java
+++ b/application/src/test/java/run/halo/app/infra/utils/FileTypeDetectUtilsTest.java
@ -0,0 +1,45 @@
+package run.halo.app.infra.utils;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import org.apache.tika.mime.MimeTypeException;
+import org.junit.jupiter.api.Test;
+import org.springframework.util.ResourceUtils;
+
+/**
+ * Test for {@link FileTypeDetectUtils}.
+ *
+ * @author guqing
+ * @since 2.18.0
+ */
+class FileTypeDetectUtilsTest {
+
+    @Test
+    void detectMimeTypeTest() throws IOException {
+        var file = ResourceUtils.getFile("classpath:app.key");
+        String mimeType = FileTypeDetectUtils.detectMimeType(Files.newInputStream(file.toPath()));
+        assertThat(mimeType).isEqualTo("application/x-x509-key; format=pem");
+
+        file = ResourceUtils.getFile("classpath:console/index.html");
+        mimeType = FileTypeDetectUtils.detectMimeType(Files.newInputStream(file.toPath()));
+        assertThat(mimeType).isEqualTo("text/plain");
+
+        file = ResourceUtils.getFile("classpath:themes/test-theme.zip");
+        mimeType = FileTypeDetectUtils.detectMimeType(Files.newInputStream(file.toPath()));
+        assertThat(mimeType).isEqualTo("application/zip");
+    }
+
+    @Test
+    void detectFileExtensionTest() throws MimeTypeException {
+        var ext = FileTypeDetectUtils.detectFileExtension("application/x-x509-key; format=pem");
+        assertThat(ext).isEqualTo("");
+
+        ext = FileTypeDetectUtils.detectFileExtension("text/plain");
+        assertThat(ext).isEqualTo(".txt");
+
+        ext = FileTypeDetectUtils.detectFileExtension("application/zip");
+        assertThat(ext).isEqualTo(".zip");
+    }
+}
--- a/platform/application/build.gradle
+++ b/platform/application/build.gradle
@ -22,6 +22,7 @@ ext {
    lucene = "9.11.1"
    resilience4jVersion = "2.2.0"
    twoFactorAuth = "1.3"
+    tika = "2.9.2"
 }

 javaPlatform {
@ -54,6 +55,7 @@ dependencies {
        api "io.github.resilience4j:resilience4j-spring-boot3:$resilience4jVersion"
        api "io.github.resilience4j:resilience4j-reactor:$resilience4jVersion"
        api "com.j256.two-factor-auth:two-factor-auth:$twoFactorAuth"
+        api "org.apache.tika:tika-core:$tika"
    }

 }