From 540cafcdbf91bd223e23eec7f4470e62de58b523 Mon Sep 17 00:00:00 2001 From: John Niang Date: Thu, 1 Dec 2022 01:03:51 +0800 Subject: [PATCH] Allow migrating attachments from other places (#2807) #### What type of PR is this? /kind improvement /area core /milestone 2.0.0 #### What this PR does / why we need it: This PR provides an ability to migrate attachments from other places, like Halo 1.x or Wordpress. We could simply configure resource mappings to support attachments migration: ```yaml halo: attachment: resource-mappings: - pathPattern: /upload/** locations: - upload - migrate-from-1.x - pathPattern: /wp-content/uploads/** locations: - migrate-from-wp ``` Meanwhile, I refactored LocalAttachmentUploadHandler for managing attachments from migration in the future. #### Which issue(s) this PR fixes: Fixes https://github.com/halo-dev/halo/issues/2585 #### Special notes for your reviewer: **Steps to test:** 1. Try to configure the resource mappings 2. Put some static resources into the corresponding location 3. Access it from Browser At last, please make sure the functionalities of attachment are ok as before. #### Does this PR introduce a user-facing change? ```release-note None ``` --- .../run/halo/app/config/WebFluxConfig.java | 30 ++++++++++++++- .../core/extension/attachment/Constant.java | 8 ++++ .../LocalAttachmentUploadHandler.java | 37 ++++++++++++++----- .../attachment/AttachmentReconciler.java | 17 ++++----- .../properties/AttachmentProperties.java | 26 +++++++++++++ .../app/infra/properties/HaloProperties.java | 3 ++ src/main/resources/application.yaml | 5 +++ .../extensions/attachment-local-policy.yaml | 18 ++++++++- 8 files changed, 123 insertions(+), 21 deletions(-) create mode 100644 src/main/java/run/halo/app/infra/properties/AttachmentProperties.java diff --git a/src/main/java/run/halo/app/config/WebFluxConfig.java b/src/main/java/run/halo/app/config/WebFluxConfig.java index 622079803..c07806035 100644 --- a/src/main/java/run/halo/app/config/WebFluxConfig.java +++ b/src/main/java/run/halo/app/config/WebFluxConfig.java @@ -3,21 +3,26 @@ package run.halo.app.config; import static org.springframework.util.ResourceUtils.FILE_URL_PREFIX; import static org.springframework.web.reactive.function.server.RequestPredicates.GET; import static org.springframework.web.reactive.function.server.RouterFunctions.route; +import static run.halo.app.infra.utils.FileUtils.checkDirectoryTraversal; import com.fasterxml.jackson.databind.ObjectMapper; import java.net.URI; +import java.time.Duration; import java.time.Instant; import java.util.List; +import java.util.Objects; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.http.CacheControl; import org.springframework.http.codec.CodecConfigurer; import org.springframework.http.codec.HttpMessageWriter; import org.springframework.http.codec.ServerCodecConfigurer; import org.springframework.http.codec.json.Jackson2JsonDecoder; import org.springframework.http.codec.json.Jackson2JsonEncoder; import org.springframework.lang.NonNull; +import org.springframework.web.reactive.config.ResourceHandlerRegistration; import org.springframework.web.reactive.config.ResourceHandlerRegistry; import org.springframework.web.reactive.config.WebFluxConfigurer; import org.springframework.web.reactive.function.BodyInserters; @@ -114,8 +119,12 @@ public class WebFluxConfig implements WebFluxConfigurer { @Override public void addResourceHandlers(ResourceHandlerRegistry registry) { var attachmentsRoot = haloProp.getWorkDir().resolve("attachments"); - registry.addResourceHandler("/upload/**") - .addResourceLocations(FILE_URL_PREFIX + attachmentsRoot + "/"); + + // Mandatory resource mapping + var uploadRegistration = registry.addResourceHandler("/upload/**") + .addResourceLocations(FILE_URL_PREFIX + attachmentsRoot.resolve("upload") + "/") + .setUseLastModified(true) + .setCacheControl(CacheControl.maxAge(Duration.ofDays(365))); // For console project registry.addResourceHandler("/console/**") @@ -123,6 +132,23 @@ public class WebFluxConfig implements WebFluxConfigurer { .resourceChain(true) .addResolver(new EncodedResourceResolver()) .addResolver(new PathResourceResolver()); + + // Additional resource mappings + var staticResources = haloProp.getAttachment().getResourceMappings(); + staticResources.forEach(staticResource -> { + ResourceHandlerRegistration registration; + if (Objects.equals(staticResource.getPathPattern(), "/upload/**")) { + registration = uploadRegistration; + } else { + registration = registry.addResourceHandler(staticResource.getPathPattern()); + } + staticResource.getLocations().forEach(location -> { + var path = attachmentsRoot.resolve(location); + checkDirectoryTraversal(attachmentsRoot, path); + registration.addResourceLocations(FILE_URL_PREFIX + path + "/"); + }); + }); + } diff --git a/src/main/java/run/halo/app/core/extension/attachment/Constant.java b/src/main/java/run/halo/app/core/extension/attachment/Constant.java index 25bf72bf2..0c32d3cb8 100644 --- a/src/main/java/run/halo/app/core/extension/attachment/Constant.java +++ b/src/main/java/run/halo/app/core/extension/attachment/Constant.java @@ -5,7 +5,15 @@ public enum Constant { public static final String GROUP = "storage.halo.run"; public static final String VERSION = "v1alpha1"; + /** + * The relative path starting from attachments folder is for deletion. + */ public static final String LOCAL_REL_PATH_ANNO_KEY = GROUP + "/local-relative-path"; + /** + * The encoded URI is for building external url. + */ + public static final String URI_ANNO_KEY = GROUP + "/uri"; + public static final String EXTERNAL_LINK_ANNO_KEY = GROUP + "/external-link"; public static final String FINALIZER_NAME = "attachment-manager"; diff --git a/src/main/java/run/halo/app/core/extension/attachment/endpoint/LocalAttachmentUploadHandler.java b/src/main/java/run/halo/app/core/extension/attachment/endpoint/LocalAttachmentUploadHandler.java index 312063cbe..002c56261 100644 --- a/src/main/java/run/halo/app/core/extension/attachment/endpoint/LocalAttachmentUploadHandler.java +++ b/src/main/java/run/halo/app/core/extension/attachment/endpoint/LocalAttachmentUploadHandler.java @@ -4,8 +4,10 @@ import static java.nio.file.StandardOpenOption.CREATE_NEW; import static run.halo.app.infra.utils.FileUtils.checkDirectoryTraversal; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Map; import java.util.Optional; import java.util.UUID; @@ -15,6 +17,7 @@ import org.springframework.core.io.buffer.DataBufferUtils; import org.springframework.http.MediaType; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; +import org.springframework.web.util.UriComponentsBuilder; import reactor.core.Exceptions; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; @@ -49,15 +52,17 @@ class LocalAttachmentUploadHandler implements AttachmentHandler { var settingJson = configMap.getData().getOrDefault("default", "{}"); var setting = JsonUtils.jsonToObject(settingJson, PolicySetting.class); - var attachmentsRoot = getAttachmentsRoot(); - var attachmentRoot = attachmentsRoot; + final var attachmentsRoot = getAttachmentsRoot(); + final var uploadRoot = attachmentsRoot.resolve("upload"); + final var file = option.file(); + final Path attachmentPath; if (StringUtils.hasText(setting.getLocation())) { - attachmentRoot = attachmentsRoot.resolve(setting.getLocation()); + attachmentPath = + uploadRoot.resolve(setting.getLocation()).resolve(file.filename()); + } else { + attachmentPath = uploadRoot.resolve(file.filename()); } - var file = option.file(); - var attachmentPath = attachmentRoot.resolve(file.filename()); - // check the directory traversal before saving - checkDirectoryTraversal(attachmentsRoot, attachmentPath); + checkDirectoryTraversal(uploadRoot, attachmentPath); return Mono.fromRunnable( () -> { @@ -76,8 +81,22 @@ class LocalAttachmentUploadHandler implements AttachmentHandler { // TODO check the file extension var metadata = new Metadata(); metadata.setName(UUID.randomUUID().toString()); - metadata.setAnnotations(Map.of(Constant.LOCAL_REL_PATH_ANNO_KEY, - attachmentsRoot.relativize(attachmentPath).toString())); + var relativePath = attachmentsRoot.relativize(attachmentPath).toString(); + + var pathSegments = new ArrayList(); + pathSegments.add("upload"); + for (Path p : uploadRoot.relativize(attachmentPath)) { + pathSegments.add(p.toString()); + } + + var uri = UriComponentsBuilder.newInstance() + .pathSegment(pathSegments.toArray(String[]::new)) + .encode(StandardCharsets.UTF_8) + .build() + .toString(); + metadata.setAnnotations(Map.of( + Constant.LOCAL_REL_PATH_ANNO_KEY, relativePath, + Constant.URI_ANNO_KEY, uri)); var spec = new AttachmentSpec(); spec.setSize(attachmentPath.toFile().length()); file.headers().getContentType(); diff --git a/src/main/java/run/halo/app/core/extension/reconciler/attachment/AttachmentReconciler.java b/src/main/java/run/halo/app/core/extension/reconciler/attachment/AttachmentReconciler.java index 560d741e3..d9aa34c59 100644 --- a/src/main/java/run/halo/app/core/extension/reconciler/attachment/AttachmentReconciler.java +++ b/src/main/java/run/halo/app/core/extension/reconciler/attachment/AttachmentReconciler.java @@ -1,13 +1,11 @@ package run.halo.app.core.extension.reconciler.attachment; -import static java.nio.charset.StandardCharsets.UTF_8; - import java.util.HashSet; import java.util.Objects; import java.util.Set; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; -import org.springframework.web.util.UriUtils; +import org.springframework.web.util.UriComponentsBuilder; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import run.halo.app.core.extension.attachment.Attachment; @@ -67,15 +65,16 @@ public class AttachmentReconciler implements Reconciler { var annotations = attachment.getMetadata().getAnnotations(); if (annotations != null) { String permalink = null; - var localRelativePath = annotations.get(Constant.LOCAL_REL_PATH_ANNO_KEY); - if (localRelativePath != null) { - // TODO Add router function here. - var encodedPath = UriUtils.encodePath("/upload/" + localRelativePath, UTF_8); - permalink = externalUrl.get().resolve(encodedPath).normalize().toString(); + var uri = annotations.get(Constant.URI_ANNO_KEY); + if (uri != null) { + permalink = UriComponentsBuilder.fromUri(externalUrl.get()) + // The URI has been encoded before, so there is no need to encode it again. + .path(uri) + .build() + .toString(); } else { var externalLink = annotations.get(Constant.EXTERNAL_LINK_ANNO_KEY); if (externalLink != null) { - // TODO Set the external link into status permalink = externalLink; } } diff --git a/src/main/java/run/halo/app/infra/properties/AttachmentProperties.java b/src/main/java/run/halo/app/infra/properties/AttachmentProperties.java new file mode 100644 index 000000000..2315c69ed --- /dev/null +++ b/src/main/java/run/halo/app/infra/properties/AttachmentProperties.java @@ -0,0 +1,26 @@ +package run.halo.app.infra.properties; + +import java.util.LinkedList; +import java.util.List; +import lombok.Data; + +@Data +public class AttachmentProperties { + + private List resourceMappings = new LinkedList<>(); + + @Data + public static class ResourceMapping { + + /** + * Like: {@code /upload/**}. + */ + private String pathPattern; + + /** + * The location is a relative path to attachments folder in working directory. + */ + private List locations; + + } +} diff --git a/src/main/java/run/halo/app/infra/properties/HaloProperties.java b/src/main/java/run/halo/app/infra/properties/HaloProperties.java index 9816cfc81..8bfc1c7dc 100644 --- a/src/main/java/run/halo/app/infra/properties/HaloProperties.java +++ b/src/main/java/run/halo/app/infra/properties/HaloProperties.java @@ -45,4 +45,7 @@ public class HaloProperties { @Valid private final ThemeProperties theme = new ThemeProperties(); + + @Valid + private final AttachmentProperties attachment = new AttachmentProperties(); } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 65549b28c..991a13951 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -20,6 +20,11 @@ halo: work-dir: ${user.home}/.halo2 plugin: plugins-root: ${halo.work-dir}/plugins + attachment: + resource-mappings: + - pathPattern: /upload/** + locations: + - migrate-from-1.x springdoc: api-docs: diff --git a/src/main/resources/extensions/attachment-local-policy.yaml b/src/main/resources/extensions/attachment-local-policy.yaml index 68d035553..d9d6232f8 100644 --- a/src/main/resources/extensions/attachment-local-policy.yaml +++ b/src/main/resources/extensions/attachment-local-policy.yaml @@ -6,6 +6,22 @@ spec: displayName: Local Storage settingName: local-policy-template-setting --- +apiVersion: storage.halo.run/v1alpha1 +kind: Policy +metadata: + name: default-policy +spec: + displayName: 本地存储 + templateName: local + configMapName: default-policy-config +--- +apiVersion: v1alpha1 +kind: ConfigMap +metadata: + name: default-policy-config +data: + default: "{\"location\":\"\"}" +--- apiVersion: v1alpha1 kind: Setting metadata: @@ -18,4 +34,4 @@ spec: - $formkit: text name: location label: 存储位置 - help: ~/.halo2/attachments 下的子目录 + help: ~/.halo2/attachments/upload 下的子目录