From 36d926709748885e3214818a58e916b5c651b886 Mon Sep 17 00:00:00 2001 From: guqing Date: Mon, 12 Aug 2024 16:35:50 +0800 Subject: [PATCH 1/4] feat: ability to configure and use image thumbnails --- build.gradle | 2 +- .../s3os/AttachmentThumbnailReconciler.java | 74 ++++++++++ .../java/run/halo/s3os/S3LinkServiceImpl.java | 4 +- .../run/halo/s3os/S3OsAttachmentHandler.java | 49 ++----- src/main/java/run/halo/s3os/S3OsPlugin.java | 1 + .../java/run/halo/s3os/S3OsProperties.java | 42 +++++- .../run/halo/s3os/S3ThumbnailProvider.java | 128 ++++++++++++++++++ .../resources/extensions/ext-definitions.yaml | 9 ++ .../extensions/policy-template-s3os.yaml | 43 +++++- .../AttachmentThumbnailReconcilerTest.java | 25 ++++ 10 files changed, 326 insertions(+), 51 deletions(-) create mode 100644 src/main/java/run/halo/s3os/AttachmentThumbnailReconciler.java create mode 100644 src/main/java/run/halo/s3os/S3ThumbnailProvider.java create mode 100644 src/main/resources/extensions/ext-definitions.yaml create mode 100644 src/test/java/run/halo/s3os/AttachmentThumbnailReconcilerTest.java diff --git a/build.gradle b/build.gradle index 92bcde0..af92100 100644 --- a/build.gradle +++ b/build.gradle @@ -20,7 +20,7 @@ repositories { } dependencies { - implementation platform('run.halo.tools.platform:plugin:2.17.0-SNAPSHOT') + implementation platform('run.halo.tools.platform:plugin:2.19.0-SNAPSHOT') compileOnly 'run.halo.app:api' implementation platform('software.amazon.awssdk:bom:2.19.8') diff --git a/src/main/java/run/halo/s3os/AttachmentThumbnailReconciler.java b/src/main/java/run/halo/s3os/AttachmentThumbnailReconciler.java new file mode 100644 index 0000000..d662174 --- /dev/null +++ b/src/main/java/run/halo/s3os/AttachmentThumbnailReconciler.java @@ -0,0 +1,74 @@ +package run.halo.s3os; + +import static run.halo.app.infra.FileCategoryMatcher.IMAGE; + +import java.time.Duration; +import java.time.Instant; +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Component; +import org.springframework.util.Assert; +import run.halo.app.core.extension.attachment.Attachment; +import run.halo.app.extension.ExtensionClient; +import run.halo.app.extension.MetadataUtil; +import run.halo.app.extension.controller.Controller; +import run.halo.app.extension.controller.ControllerBuilder; +import run.halo.app.extension.controller.Reconciler; + +/** + *

This {@link Reconciler} used to check thumbnail status are generated + * if not, update the attachment to trigger thumbnail generation by halo

+ */ +@Component +@RequiredArgsConstructor +public class AttachmentThumbnailReconciler implements Reconciler { + static final String REQUEST_GEN_THUMBNAIL = "s3os.halo.run/request-gen-thumbnail"; + private final ExtensionClient client; + + @Override + public Result reconcile(Request request) { + client.fetch(Attachment.class, request.name()) + .ifPresent(attachment -> { + var annotations = MetadataUtil.nullSafeAnnotations(attachment); + var requestTime = annotations.get(REQUEST_GEN_THUMBNAIL); + if (isMadeWithIn1Day(requestTime)) { + // skip if request is made within 1 day + return; + } + + if (!isImage(attachment)) { + // skip non-image attachments + return; + } + + var status = attachment.getStatus(); + if (status == null || status.getThumbnails() == null) { + // update to trigger attachment thumbnail generation + annotations.put(REQUEST_GEN_THUMBNAIL, Instant.now().toString()); + client.update(attachment); + } + }); + return Result.doNotRetry(); + } + + static boolean isMadeWithIn1Day(String requestTimeStr) { + if (StringUtils.isBlank(requestTimeStr)) { + return false; + } + var requestTime = Instant.parse(requestTimeStr); + return Duration.between(requestTime, Instant.now()).minusDays(1).isNegative(); + } + + public static boolean isImage(Attachment attachment) { + Assert.notNull(attachment, "Attachment must not be null"); + var mediaType = attachment.getSpec().getMediaType(); + return mediaType != null && IMAGE.match(mediaType); + } + + @Override + public Controller setupWith(ControllerBuilder builder) { + return builder + .extension(new Attachment()) + .build(); + } +} diff --git a/src/main/java/run/halo/s3os/S3LinkServiceImpl.java b/src/main/java/run/halo/s3os/S3LinkServiceImpl.java index ba6b6d1..8e81459 100644 --- a/src/main/java/run/halo/s3os/S3LinkServiceImpl.java +++ b/src/main/java/run/halo/s3os/S3LinkServiceImpl.java @@ -67,7 +67,7 @@ public Mono listObjects(String policyName, String continuationToke return client.fetch(ConfigMap.class, configMapName); }) .flatMap((configMap) -> { - var properties = handler.getProperties(configMap); + var properties = S3OsProperties.convertFrom(configMap); var finalLocation = FilePathUtils.getFilePathByPlaceholder(properties.getLocation()); return Mono.using(() -> handler.buildS3Client(properties), // 执行 listObjects @@ -231,7 +231,7 @@ public Mono addAttachmentRecord(String policyName, return client.fetch(ConfigMap.class, configMapName); }) .flatMap(configMap -> { - var properties = handler.getProperties(configMap); + var properties = S3OsProperties.convertFrom(configMap); return Mono.using(() -> handler.buildS3Client(properties), (s3Client) -> Mono.fromCallable( () -> s3Client.headObject( diff --git a/src/main/java/run/halo/s3os/S3OsAttachmentHandler.java b/src/main/java/run/halo/s3os/S3OsAttachmentHandler.java index c2db9c3..ca57fb1 100644 --- a/src/main/java/run/halo/s3os/S3OsAttachmentHandler.java +++ b/src/main/java/run/halo/s3os/S3OsAttachmentHandler.java @@ -3,7 +3,6 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.nio.file.FileAlreadyExistsException; import java.time.Duration; import java.util.HashMap; @@ -23,7 +22,6 @@ import org.springframework.lang.Nullable; import org.springframework.web.server.ServerErrorException; import org.springframework.web.server.ServerWebInputException; -import org.springframework.web.util.UriUtils; import reactor.core.Exceptions; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -38,7 +36,6 @@ import run.halo.app.extension.ConfigMap; import run.halo.app.extension.Metadata; import run.halo.app.extension.MetadataUtil; -import run.halo.app.infra.utils.JsonUtils; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.awscore.presigner.SdkPresigner; import software.amazon.awssdk.core.SdkResponse; @@ -47,16 +44,7 @@ import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.S3Configuration; -import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; -import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload; -import software.amazon.awssdk.services.s3.model.CompletedPart; -import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; -import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; -import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.HeadObjectRequest; -import software.amazon.awssdk.services.s3.model.HeadObjectResponse; -import software.amazon.awssdk.services.s3.model.NoSuchKeyException; -import software.amazon.awssdk.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.services.s3.model.*; import software.amazon.awssdk.services.s3.presigner.S3Presigner; import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; import software.amazon.awssdk.utils.SdkAutoCloseable; @@ -79,7 +67,7 @@ public class S3OsAttachmentHandler implements AttachmentHandler { public Mono upload(UploadContext uploadContext) { return Mono.just(uploadContext).filter(context -> this.shouldHandle(context.policy())) .flatMap(context -> { - final var properties = getProperties(context.configMap()); + final var properties = S3OsProperties.convertFrom(context.configMap()); return upload(context, properties) .subscribeOn(Schedulers.boundedElastic()) .map(objectDetail -> this.buildAttachment(properties, objectDetail)) @@ -102,7 +90,7 @@ public Mono delete(DeleteContext deleteContext) { log.info("Skip deleting object {} from S3.", objectKey); return Mono.just(context); } - var properties = getProperties(deleteContext.configMap()); + var properties = S3OsProperties.convertFrom(deleteContext.configMap()); return Mono.using(() -> buildS3Client(properties), client -> Mono.fromCallable( () -> client.deleteObject(DeleteObjectRequest.builder() @@ -123,7 +111,7 @@ public Mono delete(DeleteContext deleteContext) { @Override public Mono getSharedURL(Attachment attachment, Policy policy, ConfigMap configMap, - Duration ttl) { + Duration ttl) { if (!this.shouldHandle(policy)) { return Mono.empty(); } @@ -132,7 +120,7 @@ public Mono getSharedURL(Attachment attachment, Policy policy, ConfigMap co return Mono.error(new IllegalArgumentException( "Cannot obtain object key from attachment " + attachment.getMetadata().getName())); } - var properties = getProperties(configMap); + var properties = S3OsProperties.convertFrom(configMap); return Mono.using(() -> buildS3Presigner(properties), s3Presigner -> { @@ -168,8 +156,8 @@ public Mono getPermalink(Attachment attachment, Policy policy, ConfigMap co // fallback to default handler for backward compatibility return Mono.empty(); } - var properties = getProperties(configMap); - var objectURL = getObjectURL(properties, objectKey); + var properties = S3OsProperties.convertFrom(configMap); + var objectURL = properties.toObjectURL(objectKey); var urlSuffix = getUrlSuffixAnnotation(attachment); if (StringUtils.isNotBlank(urlSuffix)) { objectURL += urlSuffix; @@ -195,13 +183,8 @@ private String getUrlSuffixAnnotation(Attachment attachment) { return annotations.get(URL_SUFFIX_ANNO_KEY); } - S3OsProperties getProperties(ConfigMap configMap) { - var settingJson = configMap.getData().getOrDefault("default", "{}"); - return JsonUtils.jsonToObject(settingJson, S3OsProperties.class); - } - Attachment buildAttachment(S3OsProperties properties, ObjectDetail objectDetail) { - String externalLink = getObjectURL(properties, objectDetail.uploadState.objectKey); + String externalLink = properties.toObjectURL(objectDetail.uploadState.objectKey); var urlSuffix = UrlUtils.findUrlSuffix(properties.getUrlSuffixes(), objectDetail.uploadState.fileName); @@ -229,22 +212,6 @@ Attachment buildAttachment(S3OsProperties properties, ObjectDetail objectDetail) return attachment; } - String getObjectURL(S3OsProperties properties, String objectKey) { - String objectURL; - if (StringUtils.isBlank(properties.getDomain())) { - String host; - if (properties.getEnablePathStyleAccess()) { - host = properties.getEndpoint() + "/" + properties.getBucket(); - } else { - host = properties.getBucket() + "." + properties.getEndpoint(); - } - objectURL = properties.getProtocol() + "://" + host + "/" + objectKey; - } else { - objectURL = properties.getProtocol() + "://" + properties.getDomain() + "/" + objectKey; - } - return UriUtils.encodePath(objectURL, StandardCharsets.UTF_8); - } - S3Client buildS3Client(S3OsProperties properties) { return S3Client.builder() .region(Region.of(properties.getRegion())) diff --git a/src/main/java/run/halo/s3os/S3OsPlugin.java b/src/main/java/run/halo/s3os/S3OsPlugin.java index 7851027..5c3cdbd 100644 --- a/src/main/java/run/halo/s3os/S3OsPlugin.java +++ b/src/main/java/run/halo/s3os/S3OsPlugin.java @@ -10,6 +10,7 @@ */ @Component public class S3OsPlugin extends BasePlugin { + public static final String POLICY_SETTING_NAME = "s3os-policy-template-setting"; public S3OsPlugin(PluginContext pluginContext) { super(pluginContext); diff --git a/src/main/java/run/halo/s3os/S3OsProperties.java b/src/main/java/run/halo/s3os/S3OsProperties.java index 634eea2..b87c7da 100644 --- a/src/main/java/run/halo/s3os/S3OsProperties.java +++ b/src/main/java/run/halo/s3os/S3OsProperties.java @@ -1,16 +1,17 @@ package run.halo.s3os; +import java.nio.charset.StandardCharsets; import java.util.List; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import lombok.RequiredArgsConstructor; import org.springframework.util.StringUtils; - -import java.time.LocalDate; +import org.springframework.web.util.UriUtils; +import run.halo.app.extension.ConfigMap; +import run.halo.app.infra.utils.JsonUtils; @Data -class S3OsProperties { +public class S3OsProperties { private String bucket; @@ -49,6 +50,14 @@ class S3OsProperties { private List urlSuffixes; + private ThumbnailParam thumbnailParam; + + public record ThumbnailParam(String type, String pattern) { + public boolean hasPattern() { + return StringUtils.hasText(type) && StringUtils.hasText(pattern); + } + } + @Data @AllArgsConstructor @NoArgsConstructor @@ -103,14 +112,14 @@ public void setRandomStringLength(String randomStringLength) { // if you use In if (length >= 4 && length <= 16) { this.randomStringLength = length; } + } catch (NumberFormatException ignored) { } - catch (NumberFormatException ignored) { } } public void setRegion(String region) { if (!StringUtils.hasText(region)) { this.region = "Auto"; - }else { + } else { this.region = region; } } @@ -118,4 +127,25 @@ public void setRegion(String region) { public void setEndpoint(String endpoint) { this.endpoint = UrlUtils.removeHttpPrefix(endpoint); } + + public String toObjectURL(String objectKey) { + String objectURL; + if (!StringUtils.hasText(this.getDomain())) { + String host; + if (this.getEnablePathStyleAccess()) { + host = this.getEndpoint() + "/" + this.getBucket(); + } else { + host = this.getBucket() + "." + this.getEndpoint(); + } + objectURL = this.getProtocol() + "://" + host + "/" + objectKey; + } else { + objectURL = this.getProtocol() + "://" + this.getDomain() + "/" + objectKey; + } + return UriUtils.encodePath(objectURL, StandardCharsets.UTF_8); + } + + public static S3OsProperties convertFrom(ConfigMap configMap) { + var settingJson = configMap.getData().getOrDefault("default", "{}"); + return JsonUtils.jsonToObject(settingJson, S3OsProperties.class); + } } diff --git a/src/main/java/run/halo/s3os/S3ThumbnailProvider.java b/src/main/java/run/halo/s3os/S3ThumbnailProvider.java new file mode 100644 index 0000000..0e6ac8c --- /dev/null +++ b/src/main/java/run/halo/s3os/S3ThumbnailProvider.java @@ -0,0 +1,128 @@ +package run.halo.s3os; + +import static run.halo.s3os.S3OsPlugin.POLICY_SETTING_NAME; + +import java.net.URI; +import java.net.URL; +import java.util.List; +import java.util.Map; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import lombok.Builder; +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.springframework.data.domain.Sort; +import org.springframework.stereotype.Component; +import org.springframework.util.Assert; +import org.springframework.web.util.UriComponentsBuilder; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import run.halo.app.core.attachment.ThumbnailProvider; +import run.halo.app.core.attachment.ThumbnailSize; +import run.halo.app.core.extension.attachment.Policy; +import run.halo.app.core.extension.attachment.PolicyTemplate; +import run.halo.app.extension.ConfigMap; +import run.halo.app.extension.ListOptions; +import run.halo.app.extension.ReactiveExtensionClient; + +@Component +@RequiredArgsConstructor +public class S3ThumbnailProvider implements ThumbnailProvider { + static final String WIDTH_PLACEHOLDER = "{width}"; + private final Cache s3PropsCache = CacheBuilder.newBuilder() + .maximumSize(50) + .build(); + + private final ReactiveExtensionClient client; + + @Override + public Mono generate(ThumbnailContext thumbnailContext) { + var url = thumbnailContext.getImageUrl().toString(); + var size = thumbnailContext.getSize(); + return getCacheValue(url) + .mapNotNull(cacheValue -> placedPattern(cacheValue.pattern(), size)) + .map(param -> { + if (param.startsWith("?")) { + UriComponentsBuilder.fromHttpUrl(url) + .queryParam(param.substring(1)); + } + return url + param; + }) + .map(URI::create); + } + + private static String placedPattern(String pattern, ThumbnailSize size) { + return StringUtils.replace(pattern, WIDTH_PLACEHOLDER, String.valueOf(size.getWidth())); + } + + @Override + public Mono delete(URL url) { + // do nothing for s3 + return Mono.empty(); + } + + @Override + public Mono supports(ThumbnailContext thumbnailContext) { + var url = thumbnailContext.getImageUrl().toString(); + return getCacheValue(url).hasElement(); + } + + private Mono getCacheValue(String imageUrl) { + return Flux.fromIterable(s3PropsCache.asMap().entrySet()) + .filter(entry -> imageUrl.startsWith(entry.getKey())) + .next() + .map(Map.Entry::getValue) + .switchIfEmpty(Mono.defer(() -> listAllS3ObjectDomain() + .filter(entry -> imageUrl.startsWith(entry.getKey())) + .map(Map.Entry::getValue) + .next() + )); + } + + @Builder + record S3PropsCacheValue(String pattern, String configMapName) { + } + + private Flux> listAllS3ObjectDomain() { + return listS3PolicyTemplateNames() + .collectList() + .flatMapMany(this::listAllS3Policy) + .flatMap(s3Policy -> { + var s3ConfigMapName = s3Policy.getSpec().getConfigMapName(); + return fetchS3PropsByConfigMapName(s3ConfigMapName) + .mapNotNull(properties -> { + var thumbnailParam = properties.getThumbnailParam(); + if (thumbnailParam == null || !thumbnailParam.hasPattern()) { + return null; + } + var objectDomain = properties.toObjectURL(""); + var cacheValue = S3PropsCacheValue.builder() + .pattern(thumbnailParam.pattern()) + .configMapName(s3ConfigMapName) + .build(); + return Map.entry(objectDomain, cacheValue); + }); + }) + .doOnNext(cache -> s3PropsCache.put(cache.getKey(), cache.getValue())); + } + + private Flux listAllS3Policy(List s3PolicyTemplateNames) { + Assert.notNull(s3PolicyTemplateNames, "The s3PolicyTemplateNames must not be null."); + return client.listAll(Policy.class, new ListOptions(), Sort.unsorted()) + .filter(policy -> { + var templateName = policy.getSpec().getTemplateName(); + return s3PolicyTemplateNames.contains(templateName); + }); + } + + private Mono fetchS3PropsByConfigMapName(String name) { + return client.fetch(ConfigMap.class, name) + .map(S3OsProperties::convertFrom); + } + + private Flux listS3PolicyTemplateNames() { + return client.listAll(PolicyTemplate.class, new ListOptions(), Sort.unsorted()) + .filter(policyTemplate -> POLICY_SETTING_NAME.equals(policyTemplate.getSpec().getSettingName())) + .map(template -> template.getMetadata().getName()); + } +} diff --git a/src/main/resources/extensions/ext-definitions.yaml b/src/main/resources/extensions/ext-definitions.yaml new file mode 100644 index 0000000..fbc1d0c --- /dev/null +++ b/src/main/resources/extensions/ext-definitions.yaml @@ -0,0 +1,9 @@ +apiVersion: plugin.halo.run/v1alpha1 +kind: ExtensionDefinition +metadata: + name: s3os-thumbnail-provider +spec: + className: run.halo.s3os.S3ThumbnailProvider + extensionPointName: thumbnail-provider + displayName: "S3 协议 OSS 缩略图生成" + description: "为上传到支持 S3 协议的 OSS 的图片生成缩略图" diff --git a/src/main/resources/extensions/policy-template-s3os.yaml b/src/main/resources/extensions/policy-template-s3os.yaml index 80c7c01..f44e4c5 100644 --- a/src/main/resources/extensions/policy-template-s3os.yaml +++ b/src/main/resources/extensions/policy-template-s3os.yaml @@ -144,4 +144,45 @@ spec: name: urlSuffix label: 网址后缀 placeholder: 例如：?imageMogr2/format/webp - validation: required \ No newline at end of file + validation: required + - $formkit: group + label: 缩略图参数 + name: thumbnailParam + children: + - $formkit: select + name: type + key: type + label: 类型 + options: + - label: 无 + value: "" + - label: 预设参数 + value: preset + - label: 自定义参数 + value: custom + - $formkit: select + if: "$value.type == preset" + name: pattern + key: type + label: 使用预设参数 + help: 请根据您的对象存储服务商选择对应的缩略图参数 + options: + - label: 腾讯云 COS + value: "?imageView2/0/w/{width}" + - label: 七牛云 KODO + value: "?imageView2/0/w/{width}" + - label: 阿里云 OSS + value: "?x-oss-process=image/resize,w_{width},m_lfit" + - label: 百度云 BOS + value: "?x-bce-process=image/resize,m_lfit,w_{width}" + - label: 青云 OSS + value: "?image&action=resize:w_{width},m_2" + - label: 京东云 + value: "?x-oss-process=img/sw/{width}" + - label: 又拍云 + value: "!/fw/{width}" + - $formkit: text + if: "$value.type == custom" + label: 自定义参数 + help: "{width} 为宽度占位符将被替换为所需缩略图宽度值，如: 400，参数需要以 ? 开头，间隔符除外" + name: pattern diff --git a/src/test/java/run/halo/s3os/AttachmentThumbnailReconcilerTest.java b/src/test/java/run/halo/s3os/AttachmentThumbnailReconcilerTest.java new file mode 100644 index 0000000..20d2ca3 --- /dev/null +++ b/src/test/java/run/halo/s3os/AttachmentThumbnailReconcilerTest.java @@ -0,0 +1,25 @@ +package run.halo.s3os; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Duration; +import java.time.Instant; +import org.junit.jupiter.api.Test; + +class AttachmentThumbnailReconcilerTest { + + @Test + void isMadeWithIn1DayTest() { + var madeTime = Instant.now().plusSeconds(2).toString(); + var result = AttachmentThumbnailReconciler.isMadeWithIn1Day(madeTime); + assertThat(result).isTrue(); + + madeTime = Instant.now().plus(Duration.ofDays(2)).toString(); + result = AttachmentThumbnailReconciler.isMadeWithIn1Day(madeTime); + assertThat(result).isTrue(); + + madeTime = Instant.now().minus(Duration.ofHours(25)).toString(); + result = AttachmentThumbnailReconciler.isMadeWithIn1Day(madeTime); + assertThat(result).isFalse(); + } +} From 1e44ad4713ebc68f8244e5517ed0b02fbef0029d Mon Sep 17 00:00:00 2001 From: guqing Date: Wed, 4 Sep 2024 14:35:44 +0800 Subject: [PATCH 2/4] refactor: setting options definition for thumbnail param pattern --- build.gradle | 2 +- .../s3os/AttachmentThumbnailReconciler.java | 74 ------------------- .../java/run/halo/s3os/S3OsProperties.java | 8 +- .../run/halo/s3os/S3ThumbnailProvider.java | 12 +-- .../extensions/policy-template-s3os.yaml | 65 +++++++--------- .../AttachmentThumbnailReconcilerTest.java | 25 ------- 6 files changed, 34 insertions(+), 152 deletions(-) delete mode 100644 src/main/java/run/halo/s3os/AttachmentThumbnailReconciler.java delete mode 100644 src/test/java/run/halo/s3os/AttachmentThumbnailReconcilerTest.java diff --git a/build.gradle b/build.gradle index af92100..755c372 100644 --- a/build.gradle +++ b/build.gradle @@ -40,7 +40,7 @@ configurations.runtimeClasspath { halo { - version = '2.17.0' + version = '2.19' } haloPlugin { diff --git a/src/main/java/run/halo/s3os/AttachmentThumbnailReconciler.java b/src/main/java/run/halo/s3os/AttachmentThumbnailReconciler.java deleted file mode 100644 index d662174..0000000 --- a/src/main/java/run/halo/s3os/AttachmentThumbnailReconciler.java +++ /dev/null @@ -1,74 +0,0 @@ -package run.halo.s3os; - -import static run.halo.app.infra.FileCategoryMatcher.IMAGE; - -import java.time.Duration; -import java.time.Instant; -import lombok.RequiredArgsConstructor; -import org.apache.commons.lang3.StringUtils; -import org.springframework.stereotype.Component; -import org.springframework.util.Assert; -import run.halo.app.core.extension.attachment.Attachment; -import run.halo.app.extension.ExtensionClient; -import run.halo.app.extension.MetadataUtil; -import run.halo.app.extension.controller.Controller; -import run.halo.app.extension.controller.ControllerBuilder; -import run.halo.app.extension.controller.Reconciler; - -/** - *

This {@link Reconciler} used to check thumbnail status are generated - * if not, update the attachment to trigger thumbnail generation by halo

- */ -@Component -@RequiredArgsConstructor -public class AttachmentThumbnailReconciler implements Reconciler { - static final String REQUEST_GEN_THUMBNAIL = "s3os.halo.run/request-gen-thumbnail"; - private final ExtensionClient client; - - @Override - public Result reconcile(Request request) { - client.fetch(Attachment.class, request.name()) - .ifPresent(attachment -> { - var annotations = MetadataUtil.nullSafeAnnotations(attachment); - var requestTime = annotations.get(REQUEST_GEN_THUMBNAIL); - if (isMadeWithIn1Day(requestTime)) { - // skip if request is made within 1 day - return; - } - - if (!isImage(attachment)) { - // skip non-image attachments - return; - } - - var status = attachment.getStatus(); - if (status == null || status.getThumbnails() == null) { - // update to trigger attachment thumbnail generation - annotations.put(REQUEST_GEN_THUMBNAIL, Instant.now().toString()); - client.update(attachment); - } - }); - return Result.doNotRetry(); - } - - static boolean isMadeWithIn1Day(String requestTimeStr) { - if (StringUtils.isBlank(requestTimeStr)) { - return false; - } - var requestTime = Instant.parse(requestTimeStr); - return Duration.between(requestTime, Instant.now()).minusDays(1).isNegative(); - } - - public static boolean isImage(Attachment attachment) { - Assert.notNull(attachment, "Attachment must not be null"); - var mediaType = attachment.getSpec().getMediaType(); - return mediaType != null && IMAGE.match(mediaType); - } - - @Override - public Controller setupWith(ControllerBuilder builder) { - return builder - .extension(new Attachment()) - .build(); - } -} diff --git a/src/main/java/run/halo/s3os/S3OsProperties.java b/src/main/java/run/halo/s3os/S3OsProperties.java index b87c7da..3d10e4a 100644 --- a/src/main/java/run/halo/s3os/S3OsProperties.java +++ b/src/main/java/run/halo/s3os/S3OsProperties.java @@ -50,13 +50,7 @@ public class S3OsProperties { private List urlSuffixes; - private ThumbnailParam thumbnailParam; - - public record ThumbnailParam(String type, String pattern) { - public boolean hasPattern() { - return StringUtils.hasText(type) && StringUtils.hasText(pattern); - } - } + private String thumbnailParamPattern; @Data @AllArgsConstructor diff --git a/src/main/java/run/halo/s3os/S3ThumbnailProvider.java b/src/main/java/run/halo/s3os/S3ThumbnailProvider.java index 0e6ac8c..c0dec7c 100644 --- a/src/main/java/run/halo/s3os/S3ThumbnailProvider.java +++ b/src/main/java/run/halo/s3os/S3ThumbnailProvider.java @@ -43,8 +43,10 @@ public Mono generate(ThumbnailContext thumbnailContext) { .mapNotNull(cacheValue -> placedPattern(cacheValue.pattern(), size)) .map(param -> { if (param.startsWith("?")) { - UriComponentsBuilder.fromHttpUrl(url) - .queryParam(param.substring(1)); + return UriComponentsBuilder.fromHttpUrl(url) + .queryParam(param.substring(1)) + .build() + .toString(); } return url + param; }) @@ -91,13 +93,13 @@ private Flux> listAllS3ObjectDomain() { var s3ConfigMapName = s3Policy.getSpec().getConfigMapName(); return fetchS3PropsByConfigMapName(s3ConfigMapName) .mapNotNull(properties -> { - var thumbnailParam = properties.getThumbnailParam(); - if (thumbnailParam == null || !thumbnailParam.hasPattern()) { + var thumbnailParam = properties.getThumbnailParamPattern(); + if (StringUtils.isBlank(thumbnailParam)) { return null; } var objectDomain = properties.toObjectURL(""); var cacheValue = S3PropsCacheValue.builder() - .pattern(thumbnailParam.pattern()) + .pattern(thumbnailParam) .configMapName(s3ConfigMapName) .build(); return Map.entry(objectDomain, cacheValue); diff --git a/src/main/resources/extensions/policy-template-s3os.yaml b/src/main/resources/extensions/policy-template-s3os.yaml index f44e4c5..437ed34 100644 --- a/src/main/resources/extensions/policy-template-s3os.yaml +++ b/src/main/resources/extensions/policy-template-s3os.yaml @@ -145,44 +145,29 @@ spec: label: 网址后缀 placeholder: 例如：?imageMogr2/format/webp validation: required - - $formkit: group + - $formkit: select + name: thumbnailParamPattern label: 缩略图参数 - name: thumbnailParam - children: - - $formkit: select - name: type - key: type - label: 类型 - options: - - label: 无 - value: "" - - label: 预设参数 - value: preset - - label: 自定义参数 - value: custom - - $formkit: select - if: "$value.type == preset" - name: pattern - key: type - label: 使用预设参数 - help: 请根据您的对象存储服务商选择对应的缩略图参数 - options: - - label: 腾讯云 COS - value: "?imageView2/0/w/{width}" - - label: 七牛云 KODO - value: "?imageView2/0/w/{width}" - - label: 阿里云 OSS - value: "?x-oss-process=image/resize,w_{width},m_lfit" - - label: 百度云 BOS - value: "?x-bce-process=image/resize,m_lfit,w_{width}" - - label: 青云 OSS - value: "?image&action=resize:w_{width},m_2" - - label: 京东云 - value: "?x-oss-process=img/sw/{width}" - - label: 又拍云 - value: "!/fw/{width}" - - $formkit: text - if: "$value.type == custom" - label: 自定义参数 - help: "{width} 为宽度占位符将被替换为所需缩略图宽度值，如: 400，参数需要以 ? 开头，间隔符除外" - name: pattern + allowCreate: true + searchable: true + value: "" + help: | + 请根据您的对象存储服务商选择对应的缩略图参数或自定义参数，{width} 为宽度占位符将被替换为所需缩略图宽度值， + 如: 400，参数需要以 ? 开头，间隔符除外 + options: + - label: 无 + value: "" + - label: 腾讯云 COS + value: "?imageView2/0/w/{width}" + - label: 七牛云 KODO + value: "?imageView2/0/w/{width}" + - label: 阿里云 OSS + value: "?x-oss-process=image/resize,w_{width},m_lfit" + - label: 百度云 BOS + value: "?x-bce-process=image/resize,m_lfit,w_{width}" + - label: 青云 OSS + value: "?image&action=resize:w_{width},m_2" + - label: 京东云 + value: "?x-oss-process=img/sw/{width}" + - label: 又拍云 + value: "!/fw/{width}" diff --git a/src/test/java/run/halo/s3os/AttachmentThumbnailReconcilerTest.java b/src/test/java/run/halo/s3os/AttachmentThumbnailReconcilerTest.java deleted file mode 100644 index 20d2ca3..0000000 --- a/src/test/java/run/halo/s3os/AttachmentThumbnailReconcilerTest.java +++ /dev/null @@ -1,25 +0,0 @@ -package run.halo.s3os; - -import static org.assertj.core.api.Assertions.assertThat; - -import java.time.Duration; -import java.time.Instant; -import org.junit.jupiter.api.Test; - -class AttachmentThumbnailReconcilerTest { - - @Test - void isMadeWithIn1DayTest() { - var madeTime = Instant.now().plusSeconds(2).toString(); - var result = AttachmentThumbnailReconciler.isMadeWithIn1Day(madeTime); - assertThat(result).isTrue(); - - madeTime = Instant.now().plus(Duration.ofDays(2)).toString(); - result = AttachmentThumbnailReconciler.isMadeWithIn1Day(madeTime); - assertThat(result).isTrue(); - - madeTime = Instant.now().minus(Duration.ofHours(25)).toString(); - result = AttachmentThumbnailReconciler.isMadeWithIn1Day(madeTime); - assertThat(result).isFalse(); - } -} From a98a4627ca098f553505b303e6bcc39b3dd23eb0 Mon Sep 17 00:00:00 2001 From: guqing Date: Thu, 19 Sep 2024 11:00:40 +0800 Subject: [PATCH 3/4] chore: update plugin requires --- src/main/resources/plugin.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/plugin.yaml b/src/main/resources/plugin.yaml index 87762cd..fd2bda7 100644 --- a/src/main/resources/plugin.yaml +++ b/src/main/resources/plugin.yaml @@ -4,7 +4,7 @@ metadata: name: PluginS3ObjectStorage spec: enabled: true - requires: ">=2.17.0" + requires: ">=2.19.0" author: name: Halo website: https://github.com/halo-dev From b90c331c08b03a0c9a852f03950f5d373b3a2c5e Mon Sep 17 00:00:00 2001 From: guqing Date: Mon, 23 Sep 2024 11:52:18 +0800 Subject: [PATCH 4/4] fix: code style --- src/main/java/run/halo/s3os/S3OsPlugin.java | 1 - .../run/halo/s3os/S3ThumbnailProvider.java | 28 ++----------------- .../extensions/policy-template-s3os.yaml | 4 +-- 3 files changed, 3 insertions(+), 30 deletions(-) diff --git a/src/main/java/run/halo/s3os/S3OsPlugin.java b/src/main/java/run/halo/s3os/S3OsPlugin.java index 5c3cdbd..7851027 100644 --- a/src/main/java/run/halo/s3os/S3OsPlugin.java +++ b/src/main/java/run/halo/s3os/S3OsPlugin.java @@ -10,7 +10,6 @@ */ @Component public class S3OsPlugin extends BasePlugin { - public static final String POLICY_SETTING_NAME = "s3os-policy-template-setting"; public S3OsPlugin(PluginContext pluginContext) { super(pluginContext); diff --git a/src/main/java/run/halo/s3os/S3ThumbnailProvider.java b/src/main/java/run/halo/s3os/S3ThumbnailProvider.java index c0dec7c..f7cc36e 100644 --- a/src/main/java/run/halo/s3os/S3ThumbnailProvider.java +++ b/src/main/java/run/halo/s3os/S3ThumbnailProvider.java @@ -1,28 +1,20 @@ package run.halo.s3os; -import static run.halo.s3os.S3OsPlugin.POLICY_SETTING_NAME; - import java.net.URI; import java.net.URL; -import java.util.List; import java.util.Map; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import lombok.Builder; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.StringUtils; -import org.springframework.data.domain.Sort; import org.springframework.stereotype.Component; -import org.springframework.util.Assert; import org.springframework.web.util.UriComponentsBuilder; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import run.halo.app.core.attachment.ThumbnailProvider; import run.halo.app.core.attachment.ThumbnailSize; -import run.halo.app.core.extension.attachment.Policy; -import run.halo.app.core.extension.attachment.PolicyTemplate; import run.halo.app.extension.ConfigMap; -import run.halo.app.extension.ListOptions; import run.halo.app.extension.ReactiveExtensionClient; @Component @@ -34,6 +26,7 @@ public class S3ThumbnailProvider implements ThumbnailProvider { .build(); private final ReactiveExtensionClient client; + private final S3LinkService s3LinkService; @Override public Mono generate(ThumbnailContext thumbnailContext) { @@ -86,9 +79,7 @@ record S3PropsCacheValue(String pattern, String configMapName) { } private Flux> listAllS3ObjectDomain() { - return listS3PolicyTemplateNames() - .collectList() - .flatMapMany(this::listAllS3Policy) + return s3LinkService.listS3Policies() .flatMap(s3Policy -> { var s3ConfigMapName = s3Policy.getSpec().getConfigMapName(); return fetchS3PropsByConfigMapName(s3ConfigMapName) @@ -108,23 +99,8 @@ private Flux> listAllS3ObjectDomain() { .doOnNext(cache -> s3PropsCache.put(cache.getKey(), cache.getValue())); } - private Flux listAllS3Policy(List s3PolicyTemplateNames) { - Assert.notNull(s3PolicyTemplateNames, "The s3PolicyTemplateNames must not be null."); - return client.listAll(Policy.class, new ListOptions(), Sort.unsorted()) - .filter(policy -> { - var templateName = policy.getSpec().getTemplateName(); - return s3PolicyTemplateNames.contains(templateName); - }); - } - private Mono fetchS3PropsByConfigMapName(String name) { return client.fetch(ConfigMap.class, name) .map(S3OsProperties::convertFrom); } - - private Flux listS3PolicyTemplateNames() { - return client.listAll(PolicyTemplate.class, new ListOptions(), Sort.unsorted()) - .filter(policyTemplate -> POLICY_SETTING_NAME.equals(policyTemplate.getSpec().getSettingName())) - .map(template -> template.getMetadata().getName()); - } } diff --git a/src/main/resources/extensions/policy-template-s3os.yaml b/src/main/resources/extensions/policy-template-s3os.yaml index 437ed34..ecb61b4 100644 --- a/src/main/resources/extensions/policy-template-s3os.yaml +++ b/src/main/resources/extensions/policy-template-s3os.yaml @@ -157,9 +157,7 @@ spec: options: - label: 无 value: "" - - label: 腾讯云 COS - value: "?imageView2/0/w/{width}" - - label: 七牛云 KODO + - label: 腾讯云 COS / 七牛云 KODO value: "?imageView2/0/w/{width}" - label: 阿里云 OSS value: "?x-oss-process=image/resize,w_{width},m_lfit"