From 540cafcdbf91bd223e23eec7f4470e62de58b523 Mon Sep 17 00:00:00 2001 From: John Niang Date: Thu, 1 Dec 2022 01:03:51 +0800 Subject: [PATCH] Allow migrating attachments from other places (#2807) #### What type of PR is this? /kind improvement /area core /milestone 2.0.0 #### What this PR does / why we need it: This PR provides an ability to migrate attachments from other places, like Halo 1.x or Wordpress. We could simply configure resource mappings to support attachments migration: ```yaml halo: attachment: resource-mappings: - pathPattern: /upload/** locations: - upload - migrate-from-1.x - pathPattern: /wp-content/uploads/** locations: - migrate-from-wp ``` Meanwhile, I refactored LocalAttachmentUploadHandler for managing attachments from migration in the future. #### Which issue(s) this PR fixes: Fixes https://github.com/halo-dev/halo/issues/2585 #### Special notes for your reviewer: **Steps to test:** 1. Try to configure the resource mappings 2. Put some static resources into the corresponding location 3. Access it from Browser At last, please make sure the functionalities of attachment are ok as before. #### Does this PR introduce a user-facing change? ```release-note None ``` --- .../run/halo/app/config/WebFluxConfig.java | 30 ++++++++++++++- .../core/extension/attachment/Constant.java | 8 ++++ .../LocalAttachmentUploadHandler.java | 37 ++++++++++++++----- .../attachment/AttachmentReconciler.java | 17 ++++----- .../properties/AttachmentProperties.java | 26 +++++++++++++ .../app/infra/properties/HaloProperties.java | 3 ++ src/main/resources/application.yaml | 5 +++ .../extensions/attachment-local-policy.yaml | 18 ++++++++- 8 files changed, 123 insertions(+), 21 deletions(-) create mode 100644 src/main/java/run/halo/app/infra/properties/AttachmentProperties.java diff --git a/src/main/java/run/halo/app/config/WebFluxConfig.java b/src/main/java/run/halo/app/config/WebFluxConfig.java index 6220798039..c07806035f 100644 --- a/src/main/java/run/halo/app/config/WebFluxConfig.java +++ b/src/main/java/run/halo/app/config/WebFluxConfig.java @@ -3,21 +3,26 @@ import static org.springframework.util.ResourceUtils.FILE_URL_PREFIX; import static org.springframework.web.reactive.function.server.RequestPredicates.GET; import static org.springframework.web.reactive.function.server.RouterFunctions.route; +import static run.halo.app.infra.utils.FileUtils.checkDirectoryTraversal; import com.fasterxml.jackson.databind.ObjectMapper; import java.net.URI; +import java.time.Duration; import java.time.Instant; import java.util.List; +import java.util.Objects; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.http.CacheControl; import org.springframework.http.codec.CodecConfigurer; import org.springframework.http.codec.HttpMessageWriter; import org.springframework.http.codec.ServerCodecConfigurer; import org.springframework.http.codec.json.Jackson2JsonDecoder; import org.springframework.http.codec.json.Jackson2JsonEncoder; import org.springframework.lang.NonNull; +import org.springframework.web.reactive.config.ResourceHandlerRegistration; import org.springframework.web.reactive.config.ResourceHandlerRegistry; import org.springframework.web.reactive.config.WebFluxConfigurer; import org.springframework.web.reactive.function.BodyInserters; @@ -114,8 +119,12 @@ private Mono redirectConsole(ServerRequest request) { @Override public void addResourceHandlers(ResourceHandlerRegistry registry) { var attachmentsRoot = haloProp.getWorkDir().resolve("attachments"); - registry.addResourceHandler("/upload/**") - .addResourceLocations(FILE_URL_PREFIX + attachmentsRoot + "/"); + + // Mandatory resource mapping + var uploadRegistration = registry.addResourceHandler("/upload/**") + .addResourceLocations(FILE_URL_PREFIX + attachmentsRoot.resolve("upload") + "/") + .setUseLastModified(true) + .setCacheControl(CacheControl.maxAge(Duration.ofDays(365))); // For console project registry.addResourceHandler("/console/**") @@ -123,6 +132,23 @@ public void addResourceHandlers(ResourceHandlerRegistry registry) { .resourceChain(true) .addResolver(new EncodedResourceResolver()) .addResolver(new PathResourceResolver()); + + // Additional resource mappings + var staticResources = haloProp.getAttachment().getResourceMappings(); + staticResources.forEach(staticResource -> { + ResourceHandlerRegistration registration; + if (Objects.equals(staticResource.getPathPattern(), "/upload/**")) { + registration = uploadRegistration; + } else { + registration = registry.addResourceHandler(staticResource.getPathPattern()); + } + staticResource.getLocations().forEach(location -> { + var path = attachmentsRoot.resolve(location); + checkDirectoryTraversal(attachmentsRoot, path); + registration.addResourceLocations(FILE_URL_PREFIX + path + "/"); + }); + }); + } diff --git a/src/main/java/run/halo/app/core/extension/attachment/Constant.java b/src/main/java/run/halo/app/core/extension/attachment/Constant.java index 25bf72bf2b..0c32d3cb8b 100644 --- a/src/main/java/run/halo/app/core/extension/attachment/Constant.java +++ b/src/main/java/run/halo/app/core/extension/attachment/Constant.java @@ -5,7 +5,15 @@ public enum Constant { public static final String GROUP = "storage.halo.run"; public static final String VERSION = "v1alpha1"; + /** + * The relative path starting from attachments folder is for deletion. + */ public static final String LOCAL_REL_PATH_ANNO_KEY = GROUP + "/local-relative-path"; + /** + * The encoded URI is for building external url. + */ + public static final String URI_ANNO_KEY = GROUP + "/uri"; + public static final String EXTERNAL_LINK_ANNO_KEY = GROUP + "/external-link"; public static final String FINALIZER_NAME = "attachment-manager"; diff --git a/src/main/java/run/halo/app/core/extension/attachment/endpoint/LocalAttachmentUploadHandler.java b/src/main/java/run/halo/app/core/extension/attachment/endpoint/LocalAttachmentUploadHandler.java index 312063cbe8..002c562614 100644 --- a/src/main/java/run/halo/app/core/extension/attachment/endpoint/LocalAttachmentUploadHandler.java +++ b/src/main/java/run/halo/app/core/extension/attachment/endpoint/LocalAttachmentUploadHandler.java @@ -4,8 +4,10 @@ import static run.halo.app.infra.utils.FileUtils.checkDirectoryTraversal; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Map; import java.util.Optional; import java.util.UUID; @@ -15,6 +17,7 @@ import org.springframework.http.MediaType; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; +import org.springframework.web.util.UriComponentsBuilder; import reactor.core.Exceptions; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; @@ -49,15 +52,17 @@ public Mono upload(UploadContext uploadOption) { var settingJson = configMap.getData().getOrDefault("default", "{}"); var setting = JsonUtils.jsonToObject(settingJson, PolicySetting.class); - var attachmentsRoot = getAttachmentsRoot(); - var attachmentRoot = attachmentsRoot; + final var attachmentsRoot = getAttachmentsRoot(); + final var uploadRoot = attachmentsRoot.resolve("upload"); + final var file = option.file(); + final Path attachmentPath; if (StringUtils.hasText(setting.getLocation())) { - attachmentRoot = attachmentsRoot.resolve(setting.getLocation()); + attachmentPath = + uploadRoot.resolve(setting.getLocation()).resolve(file.filename()); + } else { + attachmentPath = uploadRoot.resolve(file.filename()); } - var file = option.file(); - var attachmentPath = attachmentRoot.resolve(file.filename()); - // check the directory traversal before saving - checkDirectoryTraversal(attachmentsRoot, attachmentPath); + checkDirectoryTraversal(uploadRoot, attachmentPath); return Mono.fromRunnable( () -> { @@ -76,8 +81,22 @@ public Mono upload(UploadContext uploadOption) { // TODO check the file extension var metadata = new Metadata(); metadata.setName(UUID.randomUUID().toString()); - metadata.setAnnotations(Map.of(Constant.LOCAL_REL_PATH_ANNO_KEY, - attachmentsRoot.relativize(attachmentPath).toString())); + var relativePath = attachmentsRoot.relativize(attachmentPath).toString(); + + var pathSegments = new ArrayList(); + pathSegments.add("upload"); + for (Path p : uploadRoot.relativize(attachmentPath)) { + pathSegments.add(p.toString()); + } + + var uri = UriComponentsBuilder.newInstance() + .pathSegment(pathSegments.toArray(String[]::new)) + .encode(StandardCharsets.UTF_8) + .build() + .toString(); + metadata.setAnnotations(Map.of( + Constant.LOCAL_REL_PATH_ANNO_KEY, relativePath, + Constant.URI_ANNO_KEY, uri)); var spec = new AttachmentSpec(); spec.setSize(attachmentPath.toFile().length()); file.headers().getContentType(); diff --git a/src/main/java/run/halo/app/core/extension/reconciler/attachment/AttachmentReconciler.java b/src/main/java/run/halo/app/core/extension/reconciler/attachment/AttachmentReconciler.java index 560d741e38..d9aa34c598 100644 --- a/src/main/java/run/halo/app/core/extension/reconciler/attachment/AttachmentReconciler.java +++ b/src/main/java/run/halo/app/core/extension/reconciler/attachment/AttachmentReconciler.java @@ -1,13 +1,11 @@ package run.halo.app.core.extension.reconciler.attachment; -import static java.nio.charset.StandardCharsets.UTF_8; - import java.util.HashSet; import java.util.Objects; import java.util.Set; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; -import org.springframework.web.util.UriUtils; +import org.springframework.web.util.UriComponentsBuilder; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import run.halo.app.core.extension.attachment.Attachment; @@ -67,15 +65,16 @@ public Result reconcile(Request request) { var annotations = attachment.getMetadata().getAnnotations(); if (annotations != null) { String permalink = null; - var localRelativePath = annotations.get(Constant.LOCAL_REL_PATH_ANNO_KEY); - if (localRelativePath != null) { - // TODO Add router function here. - var encodedPath = UriUtils.encodePath("/upload/" + localRelativePath, UTF_8); - permalink = externalUrl.get().resolve(encodedPath).normalize().toString(); + var uri = annotations.get(Constant.URI_ANNO_KEY); + if (uri != null) { + permalink = UriComponentsBuilder.fromUri(externalUrl.get()) + // The URI has been encoded before, so there is no need to encode it again. + .path(uri) + .build() + .toString(); } else { var externalLink = annotations.get(Constant.EXTERNAL_LINK_ANNO_KEY); if (externalLink != null) { - // TODO Set the external link into status permalink = externalLink; } } diff --git a/src/main/java/run/halo/app/infra/properties/AttachmentProperties.java b/src/main/java/run/halo/app/infra/properties/AttachmentProperties.java new file mode 100644 index 0000000000..2315c69ed5 --- /dev/null +++ b/src/main/java/run/halo/app/infra/properties/AttachmentProperties.java @@ -0,0 +1,26 @@ +package run.halo.app.infra.properties; + +import java.util.LinkedList; +import java.util.List; +import lombok.Data; + +@Data +public class AttachmentProperties { + + private List resourceMappings = new LinkedList<>(); + + @Data + public static class ResourceMapping { + + /** + * Like: {@code /upload/**}. + */ + private String pathPattern; + + /** + * The location is a relative path to attachments folder in working directory. + */ + private List locations; + + } +} diff --git a/src/main/java/run/halo/app/infra/properties/HaloProperties.java b/src/main/java/run/halo/app/infra/properties/HaloProperties.java index 9816cfc811..8bfc1c7dc9 100644 --- a/src/main/java/run/halo/app/infra/properties/HaloProperties.java +++ b/src/main/java/run/halo/app/infra/properties/HaloProperties.java @@ -45,4 +45,7 @@ public class HaloProperties { @Valid private final ThemeProperties theme = new ThemeProperties(); + + @Valid + private final AttachmentProperties attachment = new AttachmentProperties(); } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 65549b28c0..991a139511 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -20,6 +20,11 @@ halo: work-dir: ${user.home}/.halo2 plugin: plugins-root: ${halo.work-dir}/plugins + attachment: + resource-mappings: + - pathPattern: /upload/** + locations: + - migrate-from-1.x springdoc: api-docs: diff --git a/src/main/resources/extensions/attachment-local-policy.yaml b/src/main/resources/extensions/attachment-local-policy.yaml index 68d035553e..d9d6232f8a 100644 --- a/src/main/resources/extensions/attachment-local-policy.yaml +++ b/src/main/resources/extensions/attachment-local-policy.yaml @@ -6,6 +6,22 @@ spec: displayName: Local Storage settingName: local-policy-template-setting --- +apiVersion: storage.halo.run/v1alpha1 +kind: Policy +metadata: + name: default-policy +spec: + displayName: 本地存储 + templateName: local + configMapName: default-policy-config +--- +apiVersion: v1alpha1 +kind: ConfigMap +metadata: + name: default-policy-config +data: + default: "{\"location\":\"\"}" +--- apiVersion: v1alpha1 kind: Setting metadata: @@ -18,4 +34,4 @@ spec: - $formkit: text name: location label: 存储位置 - help: ~/.halo2/attachments 下的子目录 + help: ~/.halo2/attachments/upload 下的子目录