diff --git a/api/src/main/java/run/halo/app/extension/DefaultExtensionMatcher.java b/api/src/main/java/run/halo/app/extension/DefaultExtensionMatcher.java new file mode 100644 index 00000000000..25b32d07fa3 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/DefaultExtensionMatcher.java @@ -0,0 +1,54 @@ +package run.halo.app.extension; + +import static org.apache.commons.lang3.ObjectUtils.defaultIfNull; + +import java.util.Map; +import lombok.Builder; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import run.halo.app.extension.router.selector.FieldSelector; +import run.halo.app.extension.router.selector.LabelSelector; + +@Getter +@RequiredArgsConstructor +@Builder(builderMethodName = "internalBuilder") +public class DefaultExtensionMatcher implements ExtensionMatcher { + private static final SpelExpressionParser PARSER = new SpelExpressionParser(); + + private final GroupVersionKind gvk; + private final LabelSelector labelSelector; + private final FieldSelector fieldSelector; + + public static DefaultExtensionMatcherBuilder builder(GroupVersionKind gvk) { + return internalBuilder().gvk(gvk); + } + + /** + * Match the given extension with the current matcher. + * + * @param extension extension to match + * @return true if the extension matches the current matcher + */ + @Override + public boolean match(Extension extension) { + if (gvk != null && !gvk.equals(extension.groupVersionKind())) { + return false; + } + var labels = defaultIfNull(extension.getMetadata().getLabels(), Map.of()); + if (labelSelector != null && !labelSelector.test(labels)) { + return false; + } + + if (fieldSelector != null) { + for (var matcher : fieldSelector.getMatchers()) { + var fieldValue = PARSER.parseRaw(matcher.getKey()) + .getValue(extension, String.class); + if (!matcher.test(fieldValue)) { + return false; + } + } + } + return true; + } +} diff --git a/api/src/main/java/run/halo/app/extension/ExtensionClient.java b/api/src/main/java/run/halo/app/extension/ExtensionClient.java index b518d7d0e9f..b64dba00830 100644 --- a/api/src/main/java/run/halo/app/extension/ExtensionClient.java +++ b/api/src/main/java/run/halo/app/extension/ExtensionClient.java @@ -4,6 +4,8 @@ import java.util.List; import java.util.Optional; import java.util.function.Predicate; +import org.springframework.data.domain.Sort; +import run.halo.app.extension.index.IndexedQueryEngine; /** * ExtensionClient is an interface which contains some operations on Extension instead of @@ -42,6 +44,11 @@ List list(Class type, Predicate predicate, ListResult list(Class type, Predicate predicate, Comparator comparator, int page, int size); + List listAll(Class type, ListOptions options, Sort sort); + + ListResult listBy(Class type, ListOptions options, + PageRequest page); + /** * Fetches Extension by its type and name. * @@ -82,6 +89,8 @@ ListResult list(Class type, Predicate predicate, */ void delete(E extension); + IndexedQueryEngine indexedQueryEngine(); + void watch(Watcher watcher); } diff --git a/api/src/main/java/run/halo/app/extension/ExtensionMatcher.java b/api/src/main/java/run/halo/app/extension/ExtensionMatcher.java new file mode 100644 index 00000000000..498a113ec9f --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/ExtensionMatcher.java @@ -0,0 +1,14 @@ +package run.halo.app.extension; + +import run.halo.app.extension.router.selector.FieldSelector; +import run.halo.app.extension.router.selector.LabelSelector; + +public interface ExtensionMatcher { + GroupVersionKind getGvk(); + + LabelSelector getLabelSelector(); + + FieldSelector getFieldSelector(); + + boolean match(Extension extension); +} diff --git a/api/src/main/java/run/halo/app/extension/ListOptions.java b/api/src/main/java/run/halo/app/extension/ListOptions.java new file mode 100644 index 00000000000..ca599ecbf1a --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/ListOptions.java @@ -0,0 +1,13 @@ +package run.halo.app.extension; + +import lombok.Data; +import lombok.experimental.Accessors; +import run.halo.app.extension.router.selector.FieldSelector; +import run.halo.app.extension.router.selector.LabelSelector; + +@Data +@Accessors(chain = true) +public class ListOptions { + private LabelSelector labelSelector; + private FieldSelector fieldSelector; +} diff --git a/api/src/main/java/run/halo/app/extension/PageRequest.java b/api/src/main/java/run/halo/app/extension/PageRequest.java new file mode 100644 index 00000000000..d2ed4585b25 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/PageRequest.java @@ -0,0 +1,65 @@ +package run.halo.app.extension; + +import org.springframework.data.domain.Sort; +import org.springframework.util.Assert; + +/** + *

{@link PageRequest} is an interface for pagination information.

+ *

Page number starts from 1.

+ *

if page size is 0, it means no pagination and all results will be returned.

+ * + * @author guqing + * @see PageRequestImpl + * @since 2.12.0 + */ +public interface PageRequest { + int getPageNumber(); + + int getPageSize(); + + PageRequest previous(); + + PageRequest next(); + + /** + * Returns the previous {@link PageRequest} or the first {@link PageRequest} if the current one + * already is the first one. + * + * @return a new {@link org.springframework.data.domain.PageRequest} with + * {@link #getPageNumber()} - 1 as {@link #getPageNumber()} + */ + PageRequest previousOrFirst(); + + /** + * Returns the {@link PageRequest} requesting the first page. + * + * @return a new {@link org.springframework.data.domain.PageRequest} with + * {@link #getPageNumber()} = 1 as {@link #getPageNumber()} + */ + PageRequest first(); + + /** + * Creates a new {@link PageRequest} with {@code pageNumber} applied. + * + * @param pageNumber 1-based page index. + * @return a new {@link org.springframework.data.domain.PageRequest} + */ + PageRequest withPage(int pageNumber); + + PageRequestImpl withSort(Sort sort); + + boolean hasPrevious(); + + Sort getSort(); + + /** + * Returns the current {@link Sort} or the given one if the current one is unsorted. + * + * @param sort must not be {@literal null}. + * @return the current {@link Sort} or the given one if the current one is unsorted. + */ + default Sort getSortOr(Sort sort) { + Assert.notNull(sort, "Fallback Sort must not be null"); + return getSort().isSorted() ? getSort() : sort; + } +} diff --git a/api/src/main/java/run/halo/app/extension/PageRequestImpl.java b/api/src/main/java/run/halo/app/extension/PageRequestImpl.java new file mode 100644 index 00000000000..1d45455e234 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/PageRequestImpl.java @@ -0,0 +1,86 @@ +package run.halo.app.extension; + +import static org.apache.commons.lang3.ObjectUtils.defaultIfNull; + +import org.springframework.data.domain.Sort; +import org.springframework.util.Assert; + +public class PageRequestImpl implements PageRequest { + + private final int pageNumber; + private final int pageSize; + private final Sort sort; + + public PageRequestImpl(int pageNumber, int pageSize, Sort sort) { + Assert.notNull(sort, "Sort must not be null"); + Assert.isTrue(pageNumber >= 0, "Page index must not be less than zero!"); + Assert.isTrue(pageSize >= 0, "Page size must not be less than one!"); + this.pageNumber = pageNumber; + this.pageSize = pageSize; + this.sort = sort; + } + + public static PageRequestImpl of(int pageNumber, int pageSize) { + return of(pageNumber, pageSize, Sort.unsorted()); + } + + public static PageRequestImpl of(int pageNumber, int pageSize, Sort sort) { + return new PageRequestImpl(pageNumber, pageSize, sort); + } + + public static PageRequestImpl ofSize(int pageSize) { + return PageRequestImpl.of(1, pageSize); + } + + @Override + public int getPageNumber() { + return pageNumber; + } + + @Override + public int getPageSize() { + return pageSize; + } + + @Override + public PageRequest previous() { + return getPageNumber() == 0 ? this + : new PageRequestImpl(getPageNumber() - 1, getPageSize(), getSort()); + } + + @Override + public Sort getSort() { + return sort; + } + + @Override + public PageRequest next() { + return new PageRequestImpl(getPageNumber() + 1, getPageSize(), getSort()); + } + + @Override + public PageRequest previousOrFirst() { + return hasPrevious() ? previous() : first(); + } + + @Override + public PageRequest first() { + return new PageRequestImpl(1, getPageSize(), getSort()); + } + + @Override + public PageRequest withPage(int pageNumber) { + return new PageRequestImpl(pageNumber, getPageSize(), getSort()); + } + + @Override + public PageRequestImpl withSort(Sort sort) { + return new PageRequestImpl(getPageNumber(), getPageSize(), + defaultIfNull(sort, Sort.unsorted())); + } + + @Override + public boolean hasPrevious() { + return pageNumber > 1; + } +} diff --git a/api/src/main/java/run/halo/app/extension/ReactiveExtensionClient.java b/api/src/main/java/run/halo/app/extension/ReactiveExtensionClient.java index b023902433d..021d4ca2971 100644 --- a/api/src/main/java/run/halo/app/extension/ReactiveExtensionClient.java +++ b/api/src/main/java/run/halo/app/extension/ReactiveExtensionClient.java @@ -2,8 +2,10 @@ import java.util.Comparator; import java.util.function.Predicate; +import org.springframework.data.domain.Sort; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +import run.halo.app.extension.index.IndexedQueryEngine; /** * ExtensionClient is an interface which contains some operations on Extension instead of @@ -39,6 +41,11 @@ Flux list(Class type, Predicate predicate, Mono> list(Class type, Predicate predicate, Comparator comparator, int page, int size); + Flux listAll(Class type, ListOptions options, Sort sort); + + Mono> listBy(Class type, ListOptions options, + PageRequest pageable); + /** * Fetches Extension by its type and name. * @@ -80,6 +87,8 @@ Mono> list(Class type, Predicate predi */ Mono delete(E extension); + IndexedQueryEngine indexedQueryEngine(); + void watch(Watcher watcher); } diff --git a/api/src/main/java/run/halo/app/extension/Watcher.java b/api/src/main/java/run/halo/app/extension/Watcher.java index 4fa9bc9673a..210189be7f8 100644 --- a/api/src/main/java/run/halo/app/extension/Watcher.java +++ b/api/src/main/java/run/halo/app/extension/Watcher.java @@ -3,9 +3,14 @@ import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import reactor.core.Disposable; +import run.halo.app.extension.controller.Reconciler; public interface Watcher extends Disposable { + default void onAdd(Reconciler.Request request) { + // Do nothing here, just for sync all on start. + } + default void onAdd(Extension extension) { // Do nothing here } diff --git a/api/src/main/java/run/halo/app/extension/WatcherExtensionMatchers.java b/api/src/main/java/run/halo/app/extension/WatcherExtensionMatchers.java new file mode 100644 index 00000000000..ab4a0459a88 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/WatcherExtensionMatchers.java @@ -0,0 +1,48 @@ +package run.halo.app.extension; + +import java.util.Objects; +import lombok.Builder; + +public class WatcherExtensionMatchers { + private final GroupVersionKind gvk; + private final ExtensionMatcher onAddMatcher; + private final ExtensionMatcher onUpdateMatcher; + private final ExtensionMatcher onDeleteMatcher; + + /** + * Constructs a new {@link WatcherExtensionMatchers} with the given + * {@link DefaultExtensionMatcher}. + */ + @Builder(builderMethodName = "internalBuilder") + public WatcherExtensionMatchers(GroupVersionKind gvk, ExtensionMatcher onAddMatcher, + ExtensionMatcher onUpdateMatcher, ExtensionMatcher onDeleteMatcher) { + this.gvk = gvk; + this.onAddMatcher = Objects.requireNonNullElse(onAddMatcher, emptyMatcher(gvk)); + this.onUpdateMatcher = Objects.requireNonNullElse(onUpdateMatcher, emptyMatcher(gvk)); + this.onDeleteMatcher = Objects.requireNonNullElse(onDeleteMatcher, emptyMatcher(gvk)); + } + + public GroupVersionKind getGroupVersionKind() { + return this.gvk; + } + + public ExtensionMatcher onAddMatcher() { + return this.onAddMatcher; + } + + public ExtensionMatcher onUpdateMatcher() { + return this.onUpdateMatcher; + } + + public ExtensionMatcher onDeleteMatcher() { + return this.onDeleteMatcher; + } + + public static WatcherExtensionMatchersBuilder builder(GroupVersionKind gvk) { + return internalBuilder().gvk(gvk); + } + + static ExtensionMatcher emptyMatcher(GroupVersionKind gvk) { + return DefaultExtensionMatcher.builder(gvk).build(); + } +} diff --git a/api/src/main/java/run/halo/app/extension/controller/ControllerBuilder.java b/api/src/main/java/run/halo/app/extension/controller/ControllerBuilder.java index 52f5810cee4..80cf85078e4 100644 --- a/api/src/main/java/run/halo/app/extension/controller/ControllerBuilder.java +++ b/api/src/main/java/run/halo/app/extension/controller/ControllerBuilder.java @@ -2,13 +2,12 @@ import java.time.Duration; import java.time.Instant; -import java.util.function.BiPredicate; -import java.util.function.Predicate; import java.util.function.Supplier; import org.springframework.util.Assert; import run.halo.app.extension.Extension; import run.halo.app.extension.ExtensionClient; -import run.halo.app.extension.WatcherPredicates; +import run.halo.app.extension.ExtensionMatcher; +import run.halo.app.extension.WatcherExtensionMatchers; import run.halo.app.extension.controller.Reconciler.Request; public class ControllerBuilder { @@ -19,17 +18,17 @@ public class ControllerBuilder { private Duration maxDelay; - private Reconciler reconciler; + private final Reconciler reconciler; private Supplier nowSupplier; private Extension extension; - private Predicate onAddPredicate; + private ExtensionMatcher onAddMatcher; - private Predicate onDeletePredicate; + private ExtensionMatcher onDeleteMatcher; - private BiPredicate onUpdatePredicate; + private ExtensionMatcher onUpdateMatcher; private final ExtensionClient client; @@ -65,19 +64,18 @@ public ControllerBuilder extension(Extension extension) { return this; } - public ControllerBuilder onAddPredicate(Predicate onAddPredicate) { - this.onAddPredicate = onAddPredicate; + public ControllerBuilder onAddMatcher(ExtensionMatcher onAddMatcher) { + this.onAddMatcher = onAddMatcher; return this; } - public ControllerBuilder onDeletePredicate(Predicate onDeletePredicate) { - this.onDeletePredicate = onDeletePredicate; + public ControllerBuilder onDeleteMatcher(ExtensionMatcher onDeleteMatcher) { + this.onDeleteMatcher = onDeleteMatcher; return this; } - public ControllerBuilder onUpdatePredicate( - BiPredicate onUpdatePredicate) { - this.onUpdatePredicate = onUpdatePredicate; + public ControllerBuilder onUpdateMatcher(ExtensionMatcher extensionMatcher) { + this.onUpdateMatcher = extensionMatcher; return this; } @@ -107,18 +105,17 @@ public Controller build() { Assert.notNull(reconciler, "Reconciler must not be null"); var queue = new DefaultQueue(nowSupplier, minDelay); - var predicates = new WatcherPredicates.Builder() - .withGroupVersionKind(extension.groupVersionKind()) - .onAddPredicate(onAddPredicate) - .onUpdatePredicate(onUpdatePredicate) - .onDeletePredicate(onDeletePredicate) + var extensionMatchers = WatcherExtensionMatchers.builder(extension.groupVersionKind()) + .onAddMatcher(onAddMatcher) + .onUpdateMatcher(onUpdateMatcher) + .onDeleteMatcher(onDeleteMatcher) .build(); - var watcher = new ExtensionWatcher(queue, predicates); + var watcher = new ExtensionWatcher(queue, extensionMatchers); var synchronizer = new RequestSynchronizer(syncAllOnStart, client, extension, watcher, - predicates.onAddPredicate()); + extensionMatchers.onAddMatcher()); return new DefaultController<>(name, reconciler, queue, synchronizer, minDelay, maxDelay, workerCount); } diff --git a/api/src/main/java/run/halo/app/extension/controller/ExtensionWatcher.java b/api/src/main/java/run/halo/app/extension/controller/ExtensionWatcher.java index 4c3be1ff168..3e0f854b311 100644 --- a/api/src/main/java/run/halo/app/extension/controller/ExtensionWatcher.java +++ b/api/src/main/java/run/halo/app/extension/controller/ExtensionWatcher.java @@ -2,7 +2,7 @@ import run.halo.app.extension.Extension; import run.halo.app.extension.Watcher; -import run.halo.app.extension.WatcherPredicates; +import run.halo.app.extension.WatcherExtensionMatchers; import run.halo.app.extension.controller.Reconciler.Request; public class ExtensionWatcher implements Watcher { @@ -12,16 +12,25 @@ public class ExtensionWatcher implements Watcher { private volatile boolean disposed = false; private Runnable disposeHook; - private final WatcherPredicates predicates; - public ExtensionWatcher(RequestQueue queue, WatcherPredicates predicates) { + private final WatcherExtensionMatchers matchers; + + public ExtensionWatcher(RequestQueue queue, WatcherExtensionMatchers matchers) { this.queue = queue; - this.predicates = predicates; + this.matchers = matchers; + } + + @Override + public void onAdd(Request request) { + if (isDisposed()) { + return; + } + queue.addImmediately(request); } @Override public void onAdd(Extension extension) { - if (isDisposed() || !predicates.onAddPredicate().test(extension)) { + if (isDisposed() || !matchers.onAddMatcher().match(extension)) { return; } // TODO filter the event @@ -30,7 +39,7 @@ public void onAdd(Extension extension) { @Override public void onUpdate(Extension oldExtension, Extension newExtension) { - if (isDisposed() || !predicates.onUpdatePredicate().test(oldExtension, newExtension)) { + if (isDisposed() || !matchers.onUpdateMatcher().match(newExtension)) { return; } // TODO filter the event @@ -39,7 +48,7 @@ public void onUpdate(Extension oldExtension, Extension newExtension) { @Override public void onDelete(Extension extension) { - if (isDisposed() || !predicates.onDeletePredicate().test(extension)) { + if (isDisposed() || !matchers.onDeleteMatcher().match(extension)) { return; } // TODO filter the event diff --git a/api/src/main/java/run/halo/app/extension/controller/RequestSynchronizer.java b/api/src/main/java/run/halo/app/extension/controller/RequestSynchronizer.java index 5285aee515f..5792a37291a 100644 --- a/api/src/main/java/run/halo/app/extension/controller/RequestSynchronizer.java +++ b/api/src/main/java/run/halo/app/extension/controller/RequestSynchronizer.java @@ -1,42 +1,47 @@ package run.halo.app.extension.controller; -import java.util.function.Predicate; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import run.halo.app.extension.Extension; import run.halo.app.extension.ExtensionClient; +import run.halo.app.extension.ExtensionMatcher; +import run.halo.app.extension.GroupVersionKind; +import run.halo.app.extension.ListOptions; import run.halo.app.extension.Watcher; import run.halo.app.extension.controller.Reconciler.Request; +import run.halo.app.extension.index.IndexedQueryEngine; @Slf4j public class RequestSynchronizer implements Synchronizer { private final ExtensionClient client; - private final Class type; + private final GroupVersionKind type; private final boolean syncAllOnStart; private volatile boolean disposed = false; - private volatile boolean started = false; + private final IndexedQueryEngine indexedQueryEngine; private final Watcher watcher; - private final Predicate listPredicate; + private final ExtensionMatcher listMatcher; + + @Getter + private volatile boolean started = false; public RequestSynchronizer(boolean syncAllOnStart, ExtensionClient client, Extension extension, Watcher watcher, - Predicate listPredicate) { + ExtensionMatcher listMatcher) { this.syncAllOnStart = syncAllOnStart; this.client = client; - this.type = extension.getClass(); + this.type = extension.groupVersionKind(); this.watcher = watcher; - if (listPredicate == null) { - listPredicate = e -> true; - } - this.listPredicate = listPredicate; + this.indexedQueryEngine = client.indexedQueryEngine(); + this.listMatcher = listMatcher; } @Override @@ -48,17 +53,18 @@ public void start() { started = true; if (syncAllOnStart) { - client.list(type, listPredicate::test, null) - .forEach(watcher::onAdd); + var listOptions = new ListOptions(); + if (listMatcher != null) { + listOptions.setFieldSelector(listMatcher.getFieldSelector()); + listOptions.setLabelSelector(listMatcher.getLabelSelector()); + } + indexedQueryEngine.retrieveAll(type, listOptions) + .forEach(name -> watcher.onAdd(new Request(name))); } client.watch(this.watcher); log.info("Started request({}) synchronizer.", type); } - public boolean isStarted() { - return started; - } - @Override public void dispose() { disposed = true; diff --git a/api/src/main/java/run/halo/app/extension/index/AbstractIndexAttribute.java b/api/src/main/java/run/halo/app/extension/index/AbstractIndexAttribute.java new file mode 100644 index 00000000000..bfe56a73818 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/AbstractIndexAttribute.java @@ -0,0 +1,33 @@ +package run.halo.app.extension.index; + +import lombok.EqualsAndHashCode; +import org.springframework.util.Assert; +import run.halo.app.extension.Extension; +import run.halo.app.extension.GVK; + +@EqualsAndHashCode +public abstract class AbstractIndexAttribute implements IndexAttribute { + private final Class objectType; + + /** + * Creates a new {@link AbstractIndexAttribute} for the given object type. + * + * @param objectType must not be {@literal null}. + */ + public AbstractIndexAttribute(Class objectType) { + Assert.notNull(objectType, "Object type must not be null"); + Assert.state(isValidExtension(objectType), + "Invalid extension type, make sure you have annotated it with @" + GVK.class + .getSimpleName()); + this.objectType = objectType; + } + + @Override + public Class getObjectType() { + return this.objectType; + } + + boolean isValidExtension(Class type) { + return type.getAnnotation(GVK.class) != null; + } +} diff --git a/api/src/main/java/run/halo/app/extension/index/FunctionalIndexAttribute.java b/api/src/main/java/run/halo/app/extension/index/FunctionalIndexAttribute.java new file mode 100644 index 00000000000..2f9d5423126 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/FunctionalIndexAttribute.java @@ -0,0 +1,54 @@ +package run.halo.app.extension.index; + +import java.util.Set; +import java.util.function.Function; +import lombok.EqualsAndHashCode; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; +import run.halo.app.extension.Extension; +import run.halo.app.extension.Unstructured; + +@EqualsAndHashCode(callSuper = true) +public class FunctionalIndexAttribute + extends AbstractIndexAttribute { + + @EqualsAndHashCode.Exclude + private final Function valueFunc; + + /** + * Creates a new {@link FunctionalIndexAttribute} for the given object type and value function. + * + * @param objectType must not be {@literal null}. + * @param valueFunc value function must not be {@literal null}. + */ + public FunctionalIndexAttribute(Class objectType, + Function valueFunc) { + super(objectType); + Assert.notNull(valueFunc, "Value function must not be null"); + this.valueFunc = valueFunc; + } + + @Override + public Set getValues(Extension object) { + var value = getValue(object); + return value == null ? Set.of() : Set.of(value); + } + + /** + * Gets the value for the given object. + * + * @param object the object to get the value for. + * @return returns the value for the given object. + */ + @Nullable + public String getValue(Extension object) { + if (object instanceof Unstructured unstructured) { + var ext = Unstructured.OBJECT_MAPPER.convertValue(unstructured, getObjectType()); + return valueFunc.apply(ext); + } + if (getObjectType().isInstance(object)) { + return valueFunc.apply(getObjectType().cast(object)); + } + throw new IllegalArgumentException("Object type does not match"); + } +} diff --git a/api/src/main/java/run/halo/app/extension/index/FunctionalMultiValueIndexAttribute.java b/api/src/main/java/run/halo/app/extension/index/FunctionalMultiValueIndexAttribute.java new file mode 100644 index 00000000000..367284693f7 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/FunctionalMultiValueIndexAttribute.java @@ -0,0 +1,46 @@ +package run.halo.app.extension.index; + +import java.util.Set; +import java.util.function.Function; +import lombok.EqualsAndHashCode; +import org.springframework.util.Assert; +import run.halo.app.extension.Extension; +import run.halo.app.extension.Unstructured; + +@EqualsAndHashCode(callSuper = true) +public class FunctionalMultiValueIndexAttribute + extends AbstractIndexAttribute { + + @EqualsAndHashCode.Exclude + private final Function> valueFunc; + + /** + * Creates a new {@link FunctionalIndexAttribute} for the given object type and value function. + * + * @param objectType object type must not be {@literal null}. + * @param valueFunc value function must not be {@literal null}. + */ + public FunctionalMultiValueIndexAttribute(Class objectType, + Function> valueFunc) { + super(objectType); + Assert.notNull(valueFunc, "Value function must not be null"); + this.valueFunc = valueFunc; + } + + @Override + public Set getValues(Extension object) { + if (object instanceof Unstructured unstructured) { + var ext = Unstructured.OBJECT_MAPPER.convertValue(unstructured, getObjectType()); + return getNonNullValues(ext); + } + if (getObjectType().isInstance(object)) { + return getNonNullValues(getObjectType().cast(object)); + } + throw new IllegalArgumentException("Object type does not match"); + } + + private Set getNonNullValues(E object) { + var values = valueFunc.apply(object); + return values == null ? Set.of() : values; + } +} diff --git a/api/src/main/java/run/halo/app/extension/index/IndexAttribute.java b/api/src/main/java/run/halo/app/extension/index/IndexAttribute.java new file mode 100644 index 00000000000..a75e40afd26 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/IndexAttribute.java @@ -0,0 +1,23 @@ +package run.halo.app.extension.index; + +import java.util.Set; +import run.halo.app.extension.Extension; + +public interface IndexAttribute { + + /** + * Specify this class is belonged to which extension. + * + * @return the extension class. + */ + Class getObjectType(); + + /** + * Get the value of the attribute. + * + * @param object the object to get value from. + * @param the type of the object. + * @return the value of the attribute must not be null. + */ + Set getValues(E object); +} diff --git a/api/src/main/java/run/halo/app/extension/index/IndexAttributeFactory.java b/api/src/main/java/run/halo/app/extension/index/IndexAttributeFactory.java new file mode 100644 index 00000000000..81755f09332 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/IndexAttributeFactory.java @@ -0,0 +1,20 @@ +package run.halo.app.extension.index; + +import java.util.Set; +import java.util.function.Function; +import lombok.experimental.UtilityClass; +import run.halo.app.extension.Extension; + +@UtilityClass +public class IndexAttributeFactory { + + public static IndexAttribute simpleAttribute(Class type, + Function valueFunc) { + return new FunctionalIndexAttribute<>(type, valueFunc); + } + + public static IndexAttribute multiValueAttribute(Class type, + Function> valueFunc) { + return new FunctionalMultiValueIndexAttribute<>(type, valueFunc); + } +} diff --git a/api/src/main/java/run/halo/app/extension/index/IndexSpec.java b/api/src/main/java/run/halo/app/extension/index/IndexSpec.java new file mode 100644 index 00000000000..e351d8e7634 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/IndexSpec.java @@ -0,0 +1,39 @@ +package run.halo.app.extension.index; + +import com.google.common.base.Objects; +import lombok.Data; +import lombok.experimental.Accessors; + +@Data +@Accessors(chain = true) +public class IndexSpec { + private String name; + + private IndexAttribute indexFunc; + + private OrderType order; + + private boolean unique; + + public enum OrderType { + ASC, + DESC + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + IndexSpec indexSpec = (IndexSpec) o; + return Objects.equal(name, indexSpec.name); + } + + @Override + public int hashCode() { + return Objects.hashCode(name); + } +} diff --git a/api/src/main/java/run/halo/app/extension/index/IndexSpecRegistry.java b/api/src/main/java/run/halo/app/extension/index/IndexSpecRegistry.java new file mode 100644 index 00000000000..839bd4ab9bd --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/IndexSpecRegistry.java @@ -0,0 +1,56 @@ +package run.halo.app.extension.index; + +import org.springframework.lang.NonNull; +import run.halo.app.extension.Extension; +import run.halo.app.extension.Metadata; +import run.halo.app.extension.Scheme; + +/** + *

{@link IndexSpecRegistry} is a registry for {@link IndexSpecs} to manage {@link IndexSpecs} + * for different {@link Extension}.

+ * + * @author guqing + * @since 2.12.0 + */ +public interface IndexSpecRegistry { + /** + *

Create a new {@link IndexSpecs} for the given {@link Extension} type.

+ *

The returned {@link IndexSpecs} is always includes some default {@link IndexSpec} that + * does not need to be registered again:

+ *
    + *
  • {@link Metadata#getName()} for unique primary index spec named metadata_name
    • + *
  • {@link Metadata#getCreationTimestamp()} for creation_timestamp index spec
    • + *
  • {@link Metadata#getDeletionTimestamp()} for deletion_timestamp index spec
    • + *
  • {@link Metadata#getLabels()} for labels index spec
    • + *
+ * + * @param extensionType must not be {@literal null}. + * @param the extension type + * @return the {@link IndexSpecs} for the given {@link Extension} type. + */ + IndexSpecs indexFor(Class extensionType); + + /** + * Get {@link IndexSpecs} for the given {@link Extension} type registered before. + * + * @param extensionType must not be {@literal null}. + * @param the extension type + * @return the {@link IndexSpecs} for the given {@link Extension} type. + * @throws IllegalArgumentException if no {@link IndexSpecs} found for the given + * {@link Extension} type. + */ + IndexSpecs getIndexSpecs(Class extensionType); + + boolean contains(Class extensionType); + + void removeIndexSpecs(Class extensionType); + + /** + * Get key space for an extension type. + * + * @param scheme is a scheme of an Extension. + * @return key space(never null) + */ + @NonNull + String getKeySpace(Scheme scheme); +} diff --git a/api/src/main/java/run/halo/app/extension/index/IndexSpecs.java b/api/src/main/java/run/halo/app/extension/index/IndexSpecs.java new file mode 100644 index 00000000000..c84bead95c3 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/IndexSpecs.java @@ -0,0 +1,54 @@ +package run.halo.app.extension.index; + +import java.util.List; +import org.springframework.lang.Nullable; + +/** + * An interface that defines a collection of {@link IndexSpec}, and provides methods to add, + * remove, and get {@link IndexSpec}. + * + * @author guqing + * @since 2.12.0 + */ +public interface IndexSpecs { + + /** + * Add a new {@link IndexSpec} to the collection. + * + * @param indexSpec the index spec to add. + * @throws IllegalArgumentException if the index spec with the same name already exists or + * the index spec is invalid + */ + void add(IndexSpec indexSpec); + + /** + * Get all {@link IndexSpec} in the collection. + * + * @return all index specs + */ + List getIndexSpecs(); + + /** + * Get the {@link IndexSpec} with the given name. + * + * @param indexName the name of the index spec to get. + * @return the index spec with the given name, or {@code null} if not found. + */ + @Nullable + IndexSpec getIndexSpec(String indexName); + + /** + * Check if the collection contains the {@link IndexSpec} with the given name. + * + * @param indexName the name of the index spec to check. + * @return {@code true} if the collection contains the index spec with the given name, + */ + boolean contains(String indexName); + + /** + * Remove the {@link IndexSpec} with the given name. + * + * @param name the name of the index spec to remove. + */ + void remove(String name); +} diff --git a/api/src/main/java/run/halo/app/extension/index/IndexedQueryEngine.java b/api/src/main/java/run/halo/app/extension/index/IndexedQueryEngine.java new file mode 100644 index 00000000000..54ac43b98ec --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/IndexedQueryEngine.java @@ -0,0 +1,42 @@ +package run.halo.app.extension.index; + +import java.util.List; +import run.halo.app.extension.GroupVersionKind; +import run.halo.app.extension.ListOptions; +import run.halo.app.extension.ListResult; +import run.halo.app.extension.Metadata; +import run.halo.app.extension.PageRequest; + +/** + *

An interface for querying indexed object records from the index store.

+ *

It provides a way to retrieve the object records by the given {@link GroupVersionKind} and + * {@link ListOptions}, the final result will be ordered by the index what {@link ListOptions} + * used and specified by the {@link PageRequest#getSort()}.

+ * + * @author guqing + * @since 2.12.0 + */ +public interface IndexedQueryEngine { + + /** + * Page retrieve the object records by the given {@link GroupVersionKind} and + * {@link ListOptions}. + * + * @param type the type of the object must exist in + * {@link run.halo.app.extension.SchemeManager}. + * @param options the list options to use for retrieving the object records. + * @param page which page to retrieve and how large the page should be. + * @return a collection of {@link Metadata#getName()} for the given page. + */ + ListResult retrieve(GroupVersionKind type, ListOptions options, PageRequest page); + + /** + * Retrieve all the object records by the given {@link GroupVersionKind} and + * {@link ListOptions}. + * + * @param type the type of the object must exist in {@link run.halo.app.extension.SchemeManager} + * @param options the list options to use for retrieving the object records + * @return a collection of {@link Metadata#getName()} + */ + List retrieveAll(GroupVersionKind type, ListOptions options); +} diff --git a/api/src/main/java/run/halo/app/extension/router/SortableRequest.java b/api/src/main/java/run/halo/app/extension/router/SortableRequest.java index ce1d710741a..f786eb6cae1 100644 --- a/api/src/main/java/run/halo/app/extension/router/SortableRequest.java +++ b/api/src/main/java/run/halo/app/extension/router/SortableRequest.java @@ -3,6 +3,7 @@ import static run.halo.app.extension.Comparators.compareCreationTimestamp; import static run.halo.app.extension.Comparators.compareName; import static run.halo.app.extension.Comparators.nullsComparator; +import static run.halo.app.extension.router.selector.SelectorUtil.labelAndFieldSelectorToListOptions; import static run.halo.app.extension.router.selector.SelectorUtil.labelAndFieldSelectorToPredicate; import io.swagger.v3.oas.annotations.media.ArraySchema; @@ -17,6 +18,9 @@ import org.springframework.web.server.ServerWebExchange; import run.halo.app.core.extension.endpoint.SortResolver; import run.halo.app.extension.Extension; +import run.halo.app.extension.ListOptions; +import run.halo.app.extension.PageRequest; +import run.halo.app.extension.PageRequestImpl; public class SortableRequest extends IListRequest.QueryListRequest { @@ -48,6 +52,19 @@ public Predicate toPredicate() { return labelAndFieldSelectorToPredicate(getLabelSelector(), getFieldSelector()); } + /** + * Build {@link ListOptions} from query params. + * + * @return a list options. + */ + public ListOptions toListOptions() { + return labelAndFieldSelectorToListOptions(getLabelSelector(), getFieldSelector()); + } + + public PageRequest toPageRequest() { + return PageRequestImpl.of(getPage(), getSize(), getSort()); + } + /** * Build comparator from sort. * diff --git a/api/src/main/java/run/halo/app/extension/router/selector/EqualityMatcher.java b/api/src/main/java/run/halo/app/extension/router/selector/EqualityMatcher.java new file mode 100644 index 00000000000..a534891b4bb --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/router/selector/EqualityMatcher.java @@ -0,0 +1,73 @@ +package run.halo.app.extension.router.selector; + +import java.util.function.Function; +import java.util.function.Predicate; + +public class EqualityMatcher implements SelectorMatcher { + private final Operator operator; + private final String key; + private final String value; + + EqualityMatcher(String key, Operator operator, String value) { + this.key = key; + this.operator = operator; + this.value = value; + } + + /** + * The "equal" matcher. Matches a label if the label is present and equal. + * + * @param key the matching label key + * @param value the matching label value + * @return the equality matcher + */ + public static EqualityMatcher equal(String key, String value) { + return new EqualityMatcher(key, Operator.EQUAL, value); + } + + /** + * The "not equal" matcher. Matches a label if the label is not present or not equal. + * + * @param key the matching label key + * @param value the matching label value + * @return the equality matcher + */ + public static EqualityMatcher notEqual(String key, String value) { + return new EqualityMatcher(key, Operator.NOT_EQUAL, value); + } + + @Override + public String toString() { + return key + + " " + + operator.name().toLowerCase() + + " " + + value; + } + + @Override + public boolean test(String s) { + return operator.with(value).test(s); + } + + @Override + public String getKey() { + return key; + } + + protected enum Operator { + EQUAL(arg -> arg::equals), + DOUBLE_EQUAL(arg -> arg::equals), + NOT_EQUAL(arg -> v -> !arg.equals(v)); + + private final Function> matcherFunc; + + Operator(Function> matcherFunc) { + this.matcherFunc = matcherFunc; + } + + Predicate with(String value) { + return matcherFunc.apply(value); + } + } +} diff --git a/api/src/main/java/run/halo/app/extension/router/selector/FieldCriteriaPredicateConverter.java b/api/src/main/java/run/halo/app/extension/router/selector/FieldCriteriaPredicateConverter.java index 7bd95e5f615..f3294eadf8f 100644 --- a/api/src/main/java/run/halo/app/extension/router/selector/FieldCriteriaPredicateConverter.java +++ b/api/src/main/java/run/halo/app/extension/router/selector/FieldCriteriaPredicateConverter.java @@ -5,6 +5,7 @@ import org.springframework.lang.NonNull; import run.halo.app.extension.Extension; +@Deprecated(since = "2.12.0") public class FieldCriteriaPredicateConverter implements Converter> { diff --git a/api/src/main/java/run/halo/app/extension/router/selector/FieldSelector.java b/api/src/main/java/run/halo/app/extension/router/selector/FieldSelector.java new file mode 100644 index 00000000000..4f3990bf9f6 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/router/selector/FieldSelector.java @@ -0,0 +1,49 @@ +package run.halo.app.extension.router.selector; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Predicate; +import lombok.Data; +import lombok.experimental.Accessors; + +@Data +@Accessors(chain = true) +public class FieldSelector implements Predicate { + private List matchers; + + @Override + public boolean test(String fieldValue) { + if (matchers == null || matchers.isEmpty()) { + return true; + } + return matchers.stream() + .allMatch(matcher -> matcher.test(fieldValue)); + } + + public static FieldSelectorBuilder builder() { + return new FieldSelectorBuilder(); + } + + public static class FieldSelectorBuilder { + private final List matchers = new ArrayList<>(); + + public FieldSelectorBuilder eq(String fieldPath, String value) { + matchers.add(EqualityMatcher.equal(fieldPath, value)); + return this; + } + + public FieldSelectorBuilder notEq(String fieldPath, String value) { + matchers.add(EqualityMatcher.notEqual(fieldPath, value)); + return this; + } + + /** + * Build a field selector. + */ + public FieldSelector build() { + FieldSelector fieldSelector = new FieldSelector(); + fieldSelector.setMatchers(matchers); + return fieldSelector; + } + } +} diff --git a/api/src/main/java/run/halo/app/extension/router/selector/FieldSelectorConverter.java b/api/src/main/java/run/halo/app/extension/router/selector/FieldSelectorConverter.java new file mode 100644 index 00000000000..3bfd3b1974e --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/router/selector/FieldSelectorConverter.java @@ -0,0 +1,44 @@ +package run.halo.app.extension.router.selector; + +import static org.apache.commons.lang3.ObjectUtils.defaultIfNull; + +import java.util.Set; +import org.springframework.core.convert.converter.Converter; +import org.springframework.lang.NonNull; +import org.springframework.util.CollectionUtils; + +public class FieldSelectorConverter implements Converter { + + @NonNull + @Override + public SelectorMatcher convert(@NonNull SelectorCriteria criteria) { + var key = criteria.key(); + // compatible with old field selector + if ("name".equals(key)) { + key = "metadata.name"; + } + switch (criteria.operator()) { + case Equals -> { + return EqualityMatcher.equal(key, getSingleValue(criteria)); + } + case NotEquals -> { + return EqualityMatcher.notEqual(key, getSingleValue(criteria)); + } + // compatible with old field selector + case IN -> { + var valueArr = + defaultIfNull(criteria.values(), Set.of()).toArray(new String[0]); + return SetMatcher.in(key, valueArr); + } + default -> throw new IllegalArgumentException( + "Unsupported operator: " + criteria.operator()); + } + } + + String getSingleValue(SelectorCriteria criteria) { + if (CollectionUtils.isEmpty(criteria.values())) { + return null; + } + return criteria.values().iterator().next(); + } +} diff --git a/api/src/main/java/run/halo/app/extension/router/selector/LabelCriteriaPredicateConverter.java b/api/src/main/java/run/halo/app/extension/router/selector/LabelCriteriaPredicateConverter.java index ed1794842e9..5832e7c4f4f 100644 --- a/api/src/main/java/run/halo/app/extension/router/selector/LabelCriteriaPredicateConverter.java +++ b/api/src/main/java/run/halo/app/extension/router/selector/LabelCriteriaPredicateConverter.java @@ -5,6 +5,7 @@ import org.springframework.lang.NonNull; import run.halo.app.extension.Extension; +@Deprecated(since = "2.12.0") public class LabelCriteriaPredicateConverter implements Converter> { diff --git a/api/src/main/java/run/halo/app/extension/router/selector/LabelSelector.java b/api/src/main/java/run/halo/app/extension/router/selector/LabelSelector.java new file mode 100644 index 00000000000..82be7dfab9d --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/router/selector/LabelSelector.java @@ -0,0 +1,73 @@ +package run.halo.app.extension.router.selector; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; +import lombok.Data; +import lombok.experimental.Accessors; +import org.springframework.lang.NonNull; +import org.springframework.util.Assert; + +@Data +@Accessors(chain = true) +public class LabelSelector implements Predicate> { + private List matchers; + + @Override + public boolean test(@NonNull Map labels) { + Assert.notNull(labels, "Labels must not be null"); + if (matchers == null || matchers.isEmpty()) { + return true; + } + return matchers.stream() + .allMatch(matcher -> matcher.test(labels.get(matcher.getKey()))); + } + + public static LabelSelectorBuilder builder() { + return new LabelSelectorBuilder(); + } + + public static class LabelSelectorBuilder { + private final List matchers = new ArrayList<>(); + + public LabelSelectorBuilder eq(String key, String value) { + matchers.add(EqualityMatcher.equal(key, value)); + return this; + } + + public LabelSelectorBuilder notEq(String key, String value) { + matchers.add(EqualityMatcher.notEqual(key, value)); + return this; + } + + public LabelSelectorBuilder in(String key, String... values) { + matchers.add(SetMatcher.in(key, values)); + return this; + } + + public LabelSelectorBuilder notIn(String key, String... values) { + matchers.add(SetMatcher.notIn(key, values)); + return this; + } + + public LabelSelectorBuilder exists(String key) { + matchers.add(SetMatcher.exists(key)); + return this; + } + + public LabelSelectorBuilder notExists(String key) { + matchers.add(SetMatcher.notExists(key)); + return this; + } + + /** + * Build the label selector. + */ + public LabelSelector build() { + var labelSelector = new LabelSelector(); + labelSelector.setMatchers(matchers); + return labelSelector; + } + } +} diff --git a/api/src/main/java/run/halo/app/extension/router/selector/LabelSelectorConverter.java b/api/src/main/java/run/halo/app/extension/router/selector/LabelSelectorConverter.java new file mode 100644 index 00000000000..d3e21dd95ab --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/router/selector/LabelSelectorConverter.java @@ -0,0 +1,44 @@ +package run.halo.app.extension.router.selector; + +import static org.apache.commons.lang3.ObjectUtils.defaultIfNull; + +import java.util.Set; +import org.springframework.core.convert.converter.Converter; +import org.springframework.lang.NonNull; +import org.springframework.util.CollectionUtils; + +public class LabelSelectorConverter implements Converter { + + @NonNull + @Override + public SelectorMatcher convert(@NonNull SelectorCriteria criteria) { + switch (criteria.operator()) { + case Equals -> { + return EqualityMatcher.equal(criteria.key(), getSingleValue(criteria)); + } + case NotEquals -> { + return EqualityMatcher.notEqual(criteria.key(), getSingleValue(criteria)); + } + case NotExist -> { + return SetMatcher.notExists(criteria.key()); + } + case Exist -> { + return SetMatcher.exists(criteria.key()); + } + case IN -> { + var valueArr = + defaultIfNull(criteria.values(), Set.of()).toArray(new String[0]); + return SetMatcher.in(criteria.key(), valueArr); + } + default -> throw new IllegalArgumentException( + "Unsupported operator: " + criteria.operator()); + } + } + + String getSingleValue(SelectorCriteria criteria) { + if (CollectionUtils.isEmpty(criteria.values())) { + return null; + } + return criteria.values().iterator().next(); + } +} diff --git a/api/src/main/java/run/halo/app/extension/router/selector/SelectorMatcher.java b/api/src/main/java/run/halo/app/extension/router/selector/SelectorMatcher.java new file mode 100644 index 00000000000..88eb63b740c --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/router/selector/SelectorMatcher.java @@ -0,0 +1,14 @@ +package run.halo.app.extension.router.selector; + +public interface SelectorMatcher { + + String getKey(); + + /** + * Returns true if a label value matches. + * + * @param s the label value + * @return the boolean + */ + boolean test(String s); +} diff --git a/api/src/main/java/run/halo/app/extension/router/selector/SelectorUtil.java b/api/src/main/java/run/halo/app/extension/router/selector/SelectorUtil.java index 937c53b4b9a..f090faf7f5f 100644 --- a/api/src/main/java/run/halo/app/extension/router/selector/SelectorUtil.java +++ b/api/src/main/java/run/halo/app/extension/router/selector/SelectorUtil.java @@ -1,10 +1,13 @@ package run.halo.app.extension.router.selector; import java.util.List; +import java.util.Objects; +import java.util.Optional; import java.util.function.Predicate; import org.springframework.data.util.Predicates; import org.springframework.web.server.ServerWebInputException; import run.halo.app.extension.Extension; +import run.halo.app.extension.ListOptions; public final class SelectorUtil { @@ -58,4 +61,38 @@ public static Predicate labelAndFieldSelectorToPredicat return SelectorUtil.labelSelectorsToPredicate(labelSelectors) .and(fieldSelectorToPredicate(fieldSelectors)); } + + /** + * Convert label and field selector expressions to {@link ListOptions}. + * + * @param labelSelectorTerms label selector expressions + * @param fieldSelectorTerms field selector expressions + * @return list options(never null) + */ + public static ListOptions labelAndFieldSelectorToListOptions( + List labelSelectorTerms, List fieldSelectorTerms) { + var selectorConverter = new SelectorConverter(); + + var labelConverter = new LabelSelectorConverter(); + var labelMatchers = Optional.ofNullable(labelSelectorTerms) + .map(selectors -> selectors.stream() + .map(selectorConverter::convert) + .filter(Objects::nonNull) + .map(labelConverter::convert) + .toList()) + .orElse(List.of()); + + var fieldConverter = new FieldSelectorConverter(); + var fieldMatchers = Optional.ofNullable(fieldSelectorTerms) + .map(selectors -> selectors.stream() + .map(selectorConverter::convert) + .filter(Objects::nonNull) + .map(fieldConverter::convert) + .toList()) + .orElse(List.of()); + + return new ListOptions() + .setLabelSelector(new LabelSelector().setMatchers(labelMatchers)) + .setFieldSelector(new FieldSelector().setMatchers(fieldMatchers)); + } } diff --git a/api/src/main/java/run/halo/app/extension/router/selector/SetMatcher.java b/api/src/main/java/run/halo/app/extension/router/selector/SetMatcher.java new file mode 100644 index 00000000000..474db19b59a --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/router/selector/SetMatcher.java @@ -0,0 +1,69 @@ +package run.halo.app.extension.router.selector; + +import java.util.Arrays; +import java.util.Objects; +import java.util.function.Function; +import java.util.function.Predicate; + +public class SetMatcher implements SelectorMatcher { + private final SetMatcher.Operator operator; + private final String key; + private final String[] values; + + SetMatcher(String key, SetMatcher.Operator operator) { + this(key, operator, new String[] {}); + } + + SetMatcher(String key, SetMatcher.Operator operator, String[] values) { + this.key = key; + this.operator = operator; + this.values = values; + } + + public static SetMatcher in(String key, String... values) { + return new SetMatcher(key, Operator.IN, values); + } + + public static SetMatcher notIn(String key, String... values) { + return new SetMatcher(key, Operator.NOT_IN, values); + } + + public static SetMatcher exists(String key) { + return new SetMatcher(key, Operator.EXISTS); + } + + public static SetMatcher notExists(String key) { + return new SetMatcher(key, Operator.NOT_EXISTS); + } + + @Override + public String getKey() { + return key; + } + + @Override + public boolean test(String s) { + return operator.with(values).test(s); + } + + private enum Operator { + IN(values -> v -> contains(values, v)), + NOT_IN(values -> v -> !contains(values, v)), + EXISTS(values -> Objects::nonNull), + NOT_EXISTS(values -> Objects::isNull); + + private final Function> matcherFunc; + + Operator(Function> matcherFunc) { + this.matcherFunc = matcherFunc; + } + + private static boolean contains(String[] strArray, String s) { + return Arrays.asList(strArray).contains(s); + } + + Predicate with(String... values) { + return matcherFunc.apply(values); + } + } +} diff --git a/api/src/test/java/run/halo/app/extension/controller/ControllerBuilderTest.java b/api/src/test/java/run/halo/app/extension/controller/ControllerBuilderTest.java index b22fbb99be1..419cc399255 100644 --- a/api/src/test/java/run/halo/app/extension/controller/ControllerBuilderTest.java +++ b/api/src/test/java/run/halo/app/extension/controller/ControllerBuilderTest.java @@ -5,7 +5,6 @@ import java.time.Duration; import java.time.Instant; -import java.util.Objects; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -51,9 +50,9 @@ void buildTest() { .syncAllOnStart(true) .minDelay(Duration.ofMillis(5)) .maxDelay(Duration.ofSeconds(1000)) - .onAddPredicate(Objects::nonNull) - .onUpdatePredicate(Objects::equals) - .onDeletePredicate(Objects::nonNull) + .onAddMatcher(null) + .onUpdateMatcher(null) + .onDeleteMatcher(null) .build() ); } diff --git a/api/src/test/java/run/halo/app/extension/controller/ExtensionWatcherTest.java b/api/src/test/java/run/halo/app/extension/controller/ExtensionWatcherTest.java index 8bcaada2b23..889fb3d9335 100644 --- a/api/src/test/java/run/halo/app/extension/controller/ExtensionWatcherTest.java +++ b/api/src/test/java/run/halo/app/extension/controller/ExtensionWatcherTest.java @@ -13,7 +13,10 @@ import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import run.halo.app.extension.WatcherPredicates; +import run.halo.app.extension.DefaultExtensionMatcher; +import run.halo.app.extension.FakeExtension; +import run.halo.app.extension.GroupVersionKind; +import run.halo.app.extension.WatcherExtensionMatchers; import run.halo.app.extension.controller.Reconciler.Request; @ExtendWith(MockitoExtension.class) @@ -23,17 +26,22 @@ class ExtensionWatcherTest { RequestQueue queue; @Mock - WatcherPredicates predicates; + WatcherExtensionMatchers matchers; @InjectMocks ExtensionWatcher watcher; + private static DefaultExtensionMatcher getEmptyMatcher() { + return DefaultExtensionMatcher.builder(GroupVersionKind.fromExtension(FakeExtension.class)) + .build(); + } + @Test void shouldAddExtensionWhenAddPredicateAlwaysTrue() { - when(predicates.onAddPredicate()).thenReturn(e -> true); + when(matchers.onAddMatcher()).thenReturn(getEmptyMatcher()); watcher.onAdd(createFake("fake-name")); - verify(predicates, times(1)).onAddPredicate(); + verify(matchers, times(1)).onAddMatcher(); verify(queue, times(1)).addImmediately( argThat(request -> request.name().equals("fake-name"))); verify(queue, times(0)).add(any()); @@ -41,10 +49,11 @@ void shouldAddExtensionWhenAddPredicateAlwaysTrue() { @Test void shouldNotAddExtensionWhenAddPredicateAlwaysFalse() { - when(predicates.onAddPredicate()).thenReturn(e -> false); + var type = GroupVersionKind.fromAPIVersionAndKind("v1alpha1", "User"); + when(matchers.onAddMatcher()).thenReturn(DefaultExtensionMatcher.builder(type).build()); watcher.onAdd(createFake("fake-name")); - verify(predicates, times(1)).onAddPredicate(); + verify(matchers, times(1)).onAddMatcher(); verify(queue, times(0)).add(any()); verify(queue, times(0)).addImmediately(any()); } @@ -54,17 +63,17 @@ void shouldNotAddExtensionWhenWatcherIsDisposed() { watcher.dispose(); watcher.onAdd(createFake("fake-name")); - verify(predicates, times(0)).onAddPredicate(); + verify(matchers, times(0)).onAddMatcher(); verify(queue, times(0)).addImmediately(any()); verify(queue, times(0)).add(any()); } @Test void shouldUpdateExtensionWhenUpdatePredicateAlwaysTrue() { - when(predicates.onUpdatePredicate()).thenReturn((e1, e2) -> true); + when(matchers.onUpdateMatcher()).thenReturn(getEmptyMatcher()); watcher.onUpdate(createFake("old-fake-name"), createFake("new-fake-name")); - verify(predicates, times(1)).onUpdatePredicate(); + verify(matchers, times(1)).onUpdateMatcher(); verify(queue, times(1)).addImmediately( argThat(request -> request.name().equals("new-fake-name"))); verify(queue, times(0)).add(any()); @@ -72,10 +81,11 @@ void shouldUpdateExtensionWhenUpdatePredicateAlwaysTrue() { @Test void shouldUpdateExtensionWhenUpdatePredicateAlwaysFalse() { - when(predicates.onUpdatePredicate()).thenReturn((e1, e2) -> false); + var type = GroupVersionKind.fromAPIVersionAndKind("v1alpha1", "User"); + when(matchers.onUpdateMatcher()).thenReturn(DefaultExtensionMatcher.builder(type).build()); watcher.onUpdate(createFake("old-fake-name"), createFake("new-fake-name")); - verify(predicates, times(1)).onUpdatePredicate(); + verify(matchers, times(1)).onUpdateMatcher(); verify(queue, times(0)).add(any()); verify(queue, times(0)).addImmediately(any()); } @@ -85,17 +95,17 @@ void shouldNotUpdateExtensionWhenWatcherIsDisposed() { watcher.dispose(); watcher.onUpdate(createFake("old-fake-name"), createFake("new-fake-name")); - verify(predicates, times(0)).onUpdatePredicate(); + verify(matchers, times(0)).onUpdateMatcher(); verify(queue, times(0)).add(any()); verify(queue, times(0)).addImmediately(any()); } @Test void shouldDeleteExtensionWhenDeletePredicateAlwaysTrue() { - when(predicates.onDeletePredicate()).thenReturn(e -> true); + when(matchers.onDeleteMatcher()).thenReturn(getEmptyMatcher()); watcher.onDelete(createFake("fake-name")); - verify(predicates, times(1)).onDeletePredicate(); + verify(matchers, times(1)).onDeleteMatcher(); verify(queue, times(1)).addImmediately( argThat(request -> request.name().equals("fake-name"))); verify(queue, times(0)).add(any()); @@ -103,10 +113,11 @@ void shouldDeleteExtensionWhenDeletePredicateAlwaysTrue() { @Test void shouldDeleteExtensionWhenDeletePredicateAlwaysFalse() { - when(predicates.onDeletePredicate()).thenReturn(e -> false); + var type = GroupVersionKind.fromAPIVersionAndKind("v1alpha1", "User"); + when(matchers.onDeleteMatcher()).thenReturn(DefaultExtensionMatcher.builder(type).build()); watcher.onDelete(createFake("fake-name")); - verify(predicates, times(1)).onDeletePredicate(); + verify(matchers, times(1)).onDeleteMatcher(); verify(queue, times(0)).add(any()); verify(queue, times(0)).addImmediately(any()); } @@ -116,7 +127,7 @@ void shouldNotDeleteExtensionWhenWatcherIsDisposed() { watcher.dispose(); watcher.onDelete(createFake("fake-name")); - verify(predicates, times(0)).onDeletePredicate(); + verify(matchers, times(0)).onDeleteMatcher(); verify(queue, times(0)).add(any()); verify(queue, times(0)).addImmediately(any()); } diff --git a/api/src/test/java/run/halo/app/extension/controller/RequestSynchronizerTest.java b/api/src/test/java/run/halo/app/extension/controller/RequestSynchronizerTest.java index 53a5258e713..dc6234eec13 100644 --- a/api/src/test/java/run/halo/app/extension/controller/RequestSynchronizerTest.java +++ b/api/src/test/java/run/halo/app/extension/controller/RequestSynchronizerTest.java @@ -3,22 +3,26 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isA; import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.util.List; -import java.util.function.Predicate; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import run.halo.app.extension.Extension; import run.halo.app.extension.ExtensionClient; +import run.halo.app.extension.ExtensionMatcher; import run.halo.app.extension.FakeExtension; +import run.halo.app.extension.GroupVersionKind; +import run.halo.app.extension.ListOptions; import run.halo.app.extension.Watcher; +import run.halo.app.extension.index.IndexedQueryEngine; @ExtendWith(MockitoExtension.class) class RequestSynchronizerTest { @@ -26,41 +30,47 @@ class RequestSynchronizerTest { @Mock ExtensionClient client; + @Mock + IndexedQueryEngine indexedQueryEngine; + @Mock Watcher watcher; @Mock - Predicate listPredicate; + ExtensionMatcher listMatcher; RequestSynchronizer synchronizer; @BeforeEach void setUp() { + when(client.indexedQueryEngine()).thenReturn(indexedQueryEngine); synchronizer = - new RequestSynchronizer(true, client, new FakeExtension(), watcher, listPredicate); + new RequestSynchronizer(true, client, new FakeExtension(), watcher, listMatcher); assertFalse(synchronizer.isDisposed()); assertFalse(synchronizer.isStarted()); } @Test void shouldStartCorrectlyWhenSyncingAllOnStart() { - when(client.list(same(FakeExtension.class), any(), any())).thenReturn( - List.of(FakeExtension.createFake("fake-01"), FakeExtension.createFake("fake-02"))); + var type = GroupVersionKind.fromExtension(FakeExtension.class); + when(indexedQueryEngine.retrieveAll(eq(type), isA(ListOptions.class))) + .thenReturn(List.of("fake-01", "fake-02")); synchronizer.start(); assertTrue(synchronizer.isStarted()); assertFalse(synchronizer.isDisposed()); - verify(client, times(1)).list(same(FakeExtension.class), any(), any()); - verify(watcher, times(2)).onAdd(any()); + verify(indexedQueryEngine, times(1)).retrieveAll(eq(type), + isA(ListOptions.class)); + verify(watcher, times(2)).onAdd(isA(Reconciler.Request.class)); verify(client, times(1)).watch(same(watcher)); } @Test void shouldStartCorrectlyWhenNotSyncingAllOnStart() { synchronizer = - new RequestSynchronizer(false, client, new FakeExtension(), watcher, listPredicate); + new RequestSynchronizer(false, client, new FakeExtension(), watcher, listMatcher); assertFalse(synchronizer.isDisposed()); assertFalse(synchronizer.isStarted()); @@ -70,7 +80,7 @@ void shouldStartCorrectlyWhenNotSyncingAllOnStart() { assertFalse(synchronizer.isDisposed()); verify(client, times(0)).list(any(), any(), any()); - verify(watcher, times(0)).onAdd(any()); + verify(watcher, times(0)).onAdd(isA(Reconciler.Request.class)); verify(client, times(1)).watch(any(Watcher.class)); } @@ -93,7 +103,7 @@ void shouldNotStartAfterDisposing() { synchronizer.start(); verify(client, times(0)).list(any(), any(), any()); - verify(watcher, times(0)).onAdd(any()); + verify(watcher, times(0)).onAdd(isA(Reconciler.Request.class)); verify(client, times(0)).watch(any()); } diff --git a/api/src/test/java/run/halo/app/extension/index/FunctionalMultiValueIndexAttributeTest.java b/api/src/test/java/run/halo/app/extension/index/FunctionalMultiValueIndexAttributeTest.java new file mode 100644 index 00000000000..f12dababa09 --- /dev/null +++ b/api/src/test/java/run/halo/app/extension/index/FunctionalMultiValueIndexAttributeTest.java @@ -0,0 +1,56 @@ +package run.halo.app.extension.index; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.Set; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.junit.jupiter.api.Test; +import run.halo.app.extension.AbstractExtension; +import run.halo.app.extension.GVK; +import run.halo.app.extension.Unstructured; + +/** + * Tests for {@link FunctionalMultiValueIndexAttribute}. + * + * @author guqing + * @since 2.12.0 + */ +class FunctionalMultiValueIndexAttributeTest { + + @Test + void create() { + var attribute = new FunctionalMultiValueIndexAttribute<>(FakeExtension.class, + FakeExtension::getCategories); + assertThat(attribute).isNotNull(); + } + + @Test + void getValues() { + var attribute = new FunctionalMultiValueIndexAttribute<>(FakeExtension.class, + FakeExtension::getCategories); + var fake = new FakeExtension(); + fake.setCategories(Set.of("test", "halo")); + assertThat(attribute.getValues(fake)).isEqualTo(fake.getCategories()); + + var unstructured = Unstructured.OBJECT_MAPPER.convertValue(fake, Unstructured.class); + assertThat(attribute.getValues(unstructured)).isEqualTo(fake.getCategories()); + + var demoExt = new DemoExtension(); + assertThatThrownBy(() -> attribute.getValues(demoExt)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Object type does not match"); + } + + @Data + @EqualsAndHashCode(callSuper = true) + @GVK(group = "test.halo.run", version = "v1", kind = "FakeExtension", plural = "fakes", + singular = "fake") + static class FakeExtension extends AbstractExtension { + Set categories; + } + + class DemoExtension extends AbstractExtension { + } +} diff --git a/api/src/test/java/run/halo/app/extension/index/IndexAttributeFactoryTest.java b/api/src/test/java/run/halo/app/extension/index/IndexAttributeFactoryTest.java new file mode 100644 index 00000000000..04dab7560fb --- /dev/null +++ b/api/src/test/java/run/halo/app/extension/index/IndexAttributeFactoryTest.java @@ -0,0 +1,41 @@ +package run.halo.app.extension.index; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Set; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.junit.jupiter.api.Test; +import run.halo.app.extension.AbstractExtension; +import run.halo.app.extension.GVK; +import run.halo.app.extension.Metadata; + +/** + * Tests for {@link IndexAttributeFactory}. + * + * @author guqing + * @since 2.12.0 + */ +class IndexAttributeFactoryTest { + + @Test + void multiValueAttribute() { + var attribute = IndexAttributeFactory.multiValueAttribute(FakeExtension.class, + FakeExtension::getTags); + assertThat(attribute).isNotNull(); + assertThat(attribute.getObjectType()).isEqualTo(FakeExtension.class); + var extension = new FakeExtension(); + extension.setMetadata(new Metadata()); + extension.getMetadata().setName("fake-name-1"); + extension.setTags(Set.of("tag1", "tag2")); + assertThat(attribute.getValues(extension)).isEqualTo(Set.of("tag1", "tag2")); + } + + @Data + @EqualsAndHashCode(callSuper = true) + @GVK(group = "test", version = "v1", kind = "FakeExtension", plural = "fakes", + singular = "fake") + static class FakeExtension extends AbstractExtension { + Set tags; + } +} diff --git a/api/src/test/java/run/halo/app/extension/index/IndexSpecTest.java b/api/src/test/java/run/halo/app/extension/index/IndexSpecTest.java new file mode 100644 index 00000000000..08f240f2a28 --- /dev/null +++ b/api/src/test/java/run/halo/app/extension/index/IndexSpecTest.java @@ -0,0 +1,76 @@ +package run.halo.app.extension.index; + +import static org.assertj.core.api.Assertions.assertThat; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.junit.jupiter.api.Test; +import run.halo.app.extension.AbstractExtension; +import run.halo.app.extension.GVK; + +/** + * Tests for {@link IndexSpec}. + * + * @author guqing + * @since 2.12.0 + */ +class IndexSpecTest { + + @Test + void equalsVerifier() { + var spec1 = new IndexSpec() + .setName("metadata.name") + .setOrder(IndexSpec.OrderType.ASC) + .setIndexFunc(IndexAttributeFactory.simpleAttribute(FakeExtension.class, + e -> e.getMetadata().getName()) + ) + .setUnique(true); + + var spec2 = new IndexSpec() + .setName("metadata.name") + .setOrder(IndexSpec.OrderType.ASC) + .setIndexFunc(IndexAttributeFactory.simpleAttribute(FakeExtension.class, + e -> e.getMetadata().getName()) + ) + .setUnique(true); + + assertThat(spec1).isEqualTo(spec2); + assertThat(spec1.equals(spec2)).isTrue(); + assertThat(spec1.hashCode()).isEqualTo(spec2.hashCode()); + + var spec3 = new IndexSpec() + .setName("metadata.name") + .setOrder(IndexSpec.OrderType.DESC) + .setIndexFunc(IndexAttributeFactory.simpleAttribute(FakeExtension.class, + e -> e.getMetadata().getName()) + ) + .setUnique(false); + assertThat(spec1).isEqualTo(spec3); + assertThat(spec1.equals(spec3)).isTrue(); + assertThat(spec1.hashCode()).isEqualTo(spec3.hashCode()); + + var spec4 = new IndexSpec() + .setName("slug") + .setOrder(IndexSpec.OrderType.ASC) + .setIndexFunc(IndexAttributeFactory.simpleAttribute(FakeExtension.class, + e -> e.getMetadata().getName()) + ) + .setUnique(true); + assertThat(spec1.equals(spec4)).isFalse(); + assertThat(spec1).isNotEqualTo(spec4); + } + + @Test + void equalAnotherObject() { + var spec3 = new IndexSpec() + .setName("metadata.name"); + assertThat(spec3.equals(new Object())).isFalse(); + } + + @Data + @EqualsAndHashCode(callSuper = true) + @GVK(group = "test", version = "v1", kind = "Fake", plural = "fakes", singular = "fake") + static class FakeExtension extends AbstractExtension { + private String slug; + } +} diff --git a/application/src/main/java/run/halo/app/content/PostIndexInformer.java b/application/src/main/java/run/halo/app/content/PostIndexInformer.java index 0ede05bdfb7..ecb1d571c58 100644 --- a/application/src/main/java/run/halo/app/content/PostIndexInformer.java +++ b/application/src/main/java/run/halo/app/content/PostIndexInformer.java @@ -57,7 +57,7 @@ public PostIndexInformer(ExtensionClient client) { client, new Post(), postWatcher, - this::checkExtension); + null); } private DefaultIndexer.IndexFunc labelIndexFunc() { diff --git a/application/src/main/java/run/halo/app/extension/DelegateExtensionClient.java b/application/src/main/java/run/halo/app/extension/DelegateExtensionClient.java index 571d6519f56..a9a0bf50858 100644 --- a/application/src/main/java/run/halo/app/extension/DelegateExtensionClient.java +++ b/application/src/main/java/run/halo/app/extension/DelegateExtensionClient.java @@ -4,7 +4,9 @@ import java.util.List; import java.util.Optional; import java.util.function.Predicate; +import org.springframework.data.domain.Sort; import org.springframework.stereotype.Component; +import run.halo.app.extension.index.IndexedQueryEngine; /** * DelegateExtensionClient fully delegates ReactiveExtensionClient. @@ -32,6 +34,17 @@ public ListResult list(Class type, Predicate pred return client.list(type, predicate, comparator, page, size).block(); } + @Override + public List listAll(Class type, ListOptions options, Sort sort) { + return client.listAll(type, options, sort).collectList().block(); + } + + @Override + public ListResult listBy(Class type, ListOptions options, + PageRequest page) { + return client.listBy(type, options, page).block(); + } + @Override public Optional fetch(Class type, String name) { return client.fetch(type, name).blockOptional(); @@ -57,6 +70,11 @@ public void delete(E extension) { client.delete(extension).block(); } + @Override + public IndexedQueryEngine indexedQueryEngine() { + return client.indexedQueryEngine(); + } + @Override public void watch(Watcher watcher) { client.watch(watcher); diff --git a/application/src/main/java/run/halo/app/extension/ReactiveExtensionClientImpl.java b/application/src/main/java/run/halo/app/extension/ReactiveExtensionClientImpl.java index 48eda2f2218..116dd7e142e 100644 --- a/application/src/main/java/run/halo/app/extension/ReactiveExtensionClientImpl.java +++ b/application/src/main/java/run/halo/app/extension/ReactiveExtensionClientImpl.java @@ -11,17 +11,34 @@ import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Predicate; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.event.ContextRefreshedEvent; +import org.springframework.context.event.EventListener; import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Sort; import org.springframework.data.util.Predicates; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.util.retry.Retry; import run.halo.app.extension.exception.ExtensionNotFoundException; +import run.halo.app.extension.index.DefaultExtensionIterator; +import run.halo.app.extension.index.ExtensionIterator; +import run.halo.app.extension.index.ExtensionPaginatedLister; +import run.halo.app.extension.index.IndexedQueryEngine; +import run.halo.app.extension.index.IndexerFactory; import run.halo.app.extension.store.ReactiveExtensionStoreClient; +@Slf4j @Component +@RequiredArgsConstructor public class ReactiveExtensionClientImpl implements ReactiveExtensionClient { private final ReactiveExtensionStoreClient client; @@ -30,18 +47,15 @@ public class ReactiveExtensionClientImpl implements ReactiveExtensionClient { private final SchemeManager schemeManager; - private final Watcher.WatcherComposite watchers; + private final Watcher.WatcherComposite watchers = new Watcher.WatcherComposite(); private final ObjectMapper objectMapper; - public ReactiveExtensionClientImpl(ReactiveExtensionStoreClient client, - ExtensionConverter converter, SchemeManager schemeManager, ObjectMapper objectMapper) { - this.client = client; - this.converter = converter; - this.schemeManager = schemeManager; - this.objectMapper = objectMapper; - this.watchers = new Watcher.WatcherComposite(); - } + private final IndexerFactory indexerFactory; + + private final IndexedQueryEngine indexedQueryEngine; + + private final AtomicBoolean ready = new AtomicBoolean(false); @Override public Flux list(Class type, Predicate predicate, @@ -74,6 +88,37 @@ public Mono> list(Class type, Predicate Flux listAll(Class type, ListOptions options, Sort sort) { + return listBy(type, options, PageRequestImpl.ofSize(0).withSort(sort)) + .flatMapIterable(ListResult::getItems); + } + + @Override + public Mono> listBy(Class type, ListOptions options, + PageRequest page) { + var scheme = schemeManager.get(type); + return Mono.fromSupplier( + () -> indexedQueryEngine.retrieve(scheme.groupVersionKind(), options, page) + ) + .flatMap(objectKeys -> { + var storeNames = objectKeys.get() + .map(objectKey -> ExtensionStoreUtil.buildStoreName(scheme, objectKey)) + .toList(); + final long startTimeMs = System.currentTimeMillis(); + return client.listByNames(storeNames) + .map(extensionStore -> converter.convertFrom(type, extensionStore)) + .doFinally(s -> { + log.debug("Successfully retrieved by names from db for {} in {}ms", + scheme.groupVersionKind(), System.currentTimeMillis() - startTimeMs); + }) + .collectList() + .map(result -> new ListResult<>(page.getPageNumber(), page.getPageSize(), + objectKeys.getTotal(), result)); + }) + .defaultIfEmpty(ListResult.emptyResult()); + } + @Override public Mono fetch(Class type, String name) { var storeName = ExtensionStoreUtil.buildStoreName(schemeManager.get(type), name); @@ -103,8 +148,9 @@ private Mono get(GroupVersionKind gvk, String name) { } @Override - @SuppressWarnings("unchecked") + @Transactional public Mono create(E extension) { + checkClientReady(); return Mono.just(extension) .doOnNext(ext -> { var metadata = extension.getMetadata(); @@ -117,7 +163,7 @@ public Mono create(E extension) { if (!hasText(metadata.getGenerateName())) { throw new IllegalArgumentException( "The metadata.generateName must not be blank when metadata.name is " - + "blank"); + + "blank"); } // generate name with random text metadata.setName(metadata.getGenerateName() + randomAlphabetic(5)); @@ -125,18 +171,20 @@ public Mono create(E extension) { extension.setMetadata(metadata); }) .map(converter::convertTo) - .flatMap(extStore -> client.create(extStore.getName(), extStore.getData()) - .map(created -> converter.convertFrom((Class) extension.getClass(), created)) - .doOnNext(watchers::onAdd)) + .flatMap(extStore -> doCreate(extension, extStore.getName(), extStore.getData()) + .doOnNext(watchers::onAdd) + ) .retryWhen(Retry.backoff(3, Duration.ofMillis(100)) // retry when generateName is set .filter(t -> t instanceof DataIntegrityViolationException - && hasText(extension.getMetadata().getGenerateName()))); + && hasText(extension.getMetadata().getGenerateName())) + ); } @Override - @SuppressWarnings("unchecked") + @Transactional public Mono update(E extension) { + checkClientReady(); // Refactor the atomic reference if we have a better solution. return getLatest(extension).flatMap(old -> { var oldJsonExt = new JsonExtension(objectMapper, old); @@ -156,8 +204,7 @@ public Mono update(E extension) { isOnlyStatusChanged(oldJsonExt.getInternal(), newJsonExt.getInternal()); var store = this.converter.convertTo(newJsonExt); - var updated = client.update(store.getName(), store.getVersion(), store.getData()) - .map(ext -> converter.convertFrom((Class) extension.getClass(), ext)); + var updated = doUpdate(extension, store.getName(), store.getVersion(), store.getData()); if (!onlyStatusChanged) { updated = updated.doOnNext(ext -> watchers.onUpdate(old, ext)); } @@ -173,15 +220,58 @@ private Mono getLatest(Extension extension) { } @Override - @SuppressWarnings("unchecked") + @Transactional public Mono delete(E extension) { + checkClientReady(); // set deletionTimestamp extension.getMetadata().setDeletionTimestamp(Instant.now()); var extensionStore = converter.convertTo(extension); - return client.update(extensionStore.getName(), extensionStore.getVersion(), - extensionStore.getData()) - .map(deleted -> converter.convertFrom((Class) extension.getClass(), deleted)) - .doOnNext(watchers::onDelete); + return doUpdate(extension, extensionStore.getName(), + extensionStore.getVersion(), extensionStore.getData() + ).doOnNext(watchers::onDelete); + } + + @Override + public IndexedQueryEngine indexedQueryEngine() { + return this.indexedQueryEngine; + } + + @SuppressWarnings("unchecked") + Mono doCreate(E oldExtension, String name, byte[] data) { + return Mono.defer(() -> { + var gvk = oldExtension.groupVersionKind(); + var type = (Class) oldExtension.getClass(); + var indexer = indexerFactory.getIndexer(gvk); + return client.create(name, data) + .map(created -> converter.convertFrom(type, created)) + .doOnNext(indexer::indexRecord); + }); + } + + @SuppressWarnings("unchecked") + Mono doUpdate(E oldExtension, String name, Long version, byte[] data) { + return Mono.defer(() -> { + var type = (Class) oldExtension.getClass(); + var indexer = indexerFactory.getIndexer(oldExtension.groupVersionKind()); + return client.update(name, version, data) + .map(updated -> converter.convertFrom(type, updated)) + .doOnNext(indexer::updateRecord); + }); + } + + private void checkClientReady() { + if (!ready.get()) { + throw new IllegalStateException("Service is not ready yet"); + } + } + + /** + * Internal method for changing the ready state of the client. + * + * @param ready ready state + */ + void setReady(boolean ready) { + this.ready.set(ready); } @Override @@ -211,4 +301,58 @@ private static boolean isOnlyStatusChanged(ObjectNode oldNode, ObjectNode newNod } return true; } + + @Component + @RequiredArgsConstructor + class IndexBuildsManager { + private final SchemeManager schemeManager; + private final IndexerFactory indexerFactory; + private final ExtensionConverter converter; + private final ReactiveExtensionStoreClient client; + private final SchemeWatcherManager schemeWatcherManager; + + @NonNull + private ExtensionIterator createExtensionIterator(Scheme scheme) { + var type = scheme.type(); + var prefix = ExtensionStoreUtil.buildStoreNamePrefix(scheme); + var lister = new ExtensionPaginatedLister() { + @Override + @SuppressWarnings("unchecked") + public Page list(Pageable pageable) { + return client.listByNamePrefix(prefix, pageable) + .map(page -> page.map( + store -> (E) converter.convertFrom(type, store)) + ) + .block(); + } + }; + return new DefaultExtensionIterator<>(lister); + } + + @EventListener(ContextRefreshedEvent.class) + public void startBuildingIndex() { + final long startTimeMs = System.currentTimeMillis(); + log.info("Start building index for all extensions, please wait..."); + schemeManager.schemes() + .forEach(scheme -> indexerFactory.createIndexerFor(scheme.type(), + createExtensionIterator(scheme))); + + schemeWatcherManager.register(event -> { + if (event instanceof SchemeWatcherManager.SchemeRegistered schemeRegistered) { + var scheme = schemeRegistered.getNewScheme(); + indexerFactory.createIndexerFor(scheme.type(), createExtensionIterator(scheme)); + return; + } + if (event instanceof SchemeWatcherManager.SchemeUnregistered schemeUnregistered) { + var scheme = schemeUnregistered.getDeletedScheme(); + indexerFactory.removeIndexer(scheme); + } + }); + log.info("Successfully built index in {}ms, Preparing to lunch application...", + System.currentTimeMillis() - startTimeMs); + + // mark client as ready + setReady(true); + } + } } diff --git a/application/src/main/java/run/halo/app/extension/gc/GcReconciler.java b/application/src/main/java/run/halo/app/extension/gc/GcReconciler.java index 009871db02f..f805fdef71f 100644 --- a/application/src/main/java/run/halo/app/extension/gc/GcReconciler.java +++ b/application/src/main/java/run/halo/app/extension/gc/GcReconciler.java @@ -16,6 +16,7 @@ import run.halo.app.extension.controller.DefaultController; import run.halo.app.extension.controller.DefaultQueue; import run.halo.app.extension.controller.Reconciler; +import run.halo.app.extension.index.IndexerFactory; import run.halo.app.extension.store.ExtensionStoreClient; @Slf4j @@ -30,21 +31,23 @@ class GcReconciler implements Reconciler { private final SchemeManager schemeManager; + private final IndexerFactory indexerFactory; + private final SchemeWatcherManager schemeWatcherManager; GcReconciler(ExtensionClient client, ExtensionStoreClient storeClient, ExtensionConverter converter, - SchemeManager schemeManager, + SchemeManager schemeManager, IndexerFactory indexerFactory, SchemeWatcherManager schemeWatcherManager) { this.client = client; this.storeClient = storeClient; this.converter = converter; this.schemeManager = schemeManager; + this.indexerFactory = indexerFactory; this.schemeWatcherManager = schemeWatcherManager; } - @Override public Result reconcile(GcRequest request) { log.debug("Extension {} is being deleted", request); @@ -54,6 +57,9 @@ public Result reconcile(GcRequest request) { .ifPresent(extension -> { var extensionStore = converter.convertTo(extension); storeClient.delete(extensionStore.getName(), extensionStore.getVersion()); + // drop index for this extension + var indexer = indexerFactory.getIndexer(extension.groupVersionKind()); + indexer.unIndexRecord(request.name()); log.debug("Extension {} was deleted", request); }); diff --git a/application/src/main/java/run/halo/app/extension/gc/GcSynchronizer.java b/application/src/main/java/run/halo/app/extension/gc/GcSynchronizer.java index 8265bb63676..86173348364 100644 --- a/application/src/main/java/run/halo/app/extension/gc/GcSynchronizer.java +++ b/application/src/main/java/run/halo/app/extension/gc/GcSynchronizer.java @@ -2,9 +2,12 @@ import static run.halo.app.extension.Comparators.compareCreationTimestamp; +import java.util.List; import java.util.function.Predicate; +import org.springframework.data.domain.Sort; import run.halo.app.extension.Extension; import run.halo.app.extension.ExtensionClient; +import run.halo.app.extension.ListOptions; import run.halo.app.extension.Scheme; import run.halo.app.extension.SchemeManager; import run.halo.app.extension.SchemeWatcherManager; @@ -12,6 +15,7 @@ import run.halo.app.extension.Watcher; import run.halo.app.extension.controller.RequestQueue; import run.halo.app.extension.controller.Synchronizer; +import run.halo.app.extension.router.selector.FieldSelector; class GcSynchronizer implements Synchronizer { @@ -58,6 +62,7 @@ public void start() { this.schemeWatcherManager.register(event -> { if (event instanceof SchemeRegistered registeredEvent) { var newScheme = registeredEvent.getNewScheme(); + listDeleted(newScheme.type()).forEach(watcher::onDelete); client.list(newScheme.type(), deleted(), compareCreationTimestamp(true)) .forEach(watcher::onDelete); } @@ -65,8 +70,19 @@ public void start() { client.watch(watcher); schemeManager.schemes().stream() .map(Scheme::type) - .forEach(type -> client.list(type, deleted(), compareCreationTimestamp(true)) - .forEach(watcher::onDelete)); + .forEach(type -> listDeleted(type).forEach(watcher::onDelete)); + } + + List listDeleted(Class type) { + var options = new ListOptions() + .setFieldSelector(FieldSelector.builder() + .notEq("metadata.deletionTimestamp", null) + .build() + ); + return client.listAll(type, options, Sort.by("metadata.creationTimestamp")) + .stream() + .sorted(compareCreationTimestamp(true)) + .toList(); } private Predicate deleted() { diff --git a/application/src/main/java/run/halo/app/extension/index/DefaultExtensionIterator.java b/application/src/main/java/run/halo/app/extension/index/DefaultExtensionIterator.java new file mode 100644 index 00000000000..fd752b4b22c --- /dev/null +++ b/application/src/main/java/run/halo/app/extension/index/DefaultExtensionIterator.java @@ -0,0 +1,62 @@ +package run.halo.app.extension.index; + +import java.util.List; +import java.util.NoSuchElementException; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Sort; +import run.halo.app.extension.Extension; + +/** + * Default implementation of {@link ExtensionIterator}. + * + * @param the type of the extension. + * @author guqing + * @since 2.12.0 + */ +public class DefaultExtensionIterator implements ExtensionIterator { + static final int DEFAULT_PAGE_SIZE = 500; + private final ExtensionPaginatedLister lister; + private Pageable currentPageable; + private List currentData; + private int currentIndex; + + /** + * Constructs a new DefaultExtensionIterator with the given lister. + * + * @param lister the lister to use to load data. + */ + public DefaultExtensionIterator(ExtensionPaginatedLister lister) { + this.lister = lister; + this.currentPageable = PageRequest.of(0, DEFAULT_PAGE_SIZE, Sort.by("name")); + this.currentData = loadData(); + } + + private List loadData() { + Page page = lister.list(currentPageable); + currentPageable = page.hasNext() ? page.nextPageable() : null; + return page.getContent(); + } + + @Override + public boolean hasNext() { + if (currentIndex < currentData.size()) { + return true; + } + if (currentPageable == null) { + return false; + } + currentData = loadData(); + currentIndex = 0; + return !currentData.isEmpty(); + } + + @Override + public E next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return currentData.get(currentIndex++); + } +} diff --git a/application/src/main/java/run/halo/app/extension/index/DefaultIndexSpecs.java b/application/src/main/java/run/halo/app/extension/index/DefaultIndexSpecs.java new file mode 100644 index 00000000000..831eaaefe7a --- /dev/null +++ b/application/src/main/java/run/halo/app/extension/index/DefaultIndexSpecs.java @@ -0,0 +1,64 @@ +package run.halo.app.extension.index; + +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import org.apache.commons.lang3.StringUtils; + +/** + * Default implementation of {@link IndexSpecs}. + * + * @author guqing + * @since 2.12.0 + */ +public class DefaultIndexSpecs implements IndexSpecs { + private final ConcurrentMap indexSpecs; + + public DefaultIndexSpecs() { + this.indexSpecs = new ConcurrentHashMap<>(); + } + + @Override + public void add(IndexSpec indexSpec) { + checkIndexSpec(indexSpec); + var indexName = indexSpec.getName(); + var existingSpec = indexSpecs.putIfAbsent(indexName, indexSpec); + if (existingSpec != null) { + throw new IllegalArgumentException( + "IndexSpec with name " + indexName + " already exists"); + } + } + + @Override + public List getIndexSpecs() { + return List.copyOf(this.indexSpecs.values()); + } + + @Override + public IndexSpec getIndexSpec(String indexName) { + return this.indexSpecs.get(indexName); + } + + @Override + public boolean contains(String indexName) { + return this.indexSpecs.containsKey(indexName); + } + + @Override + public void remove(String name) { + this.indexSpecs.remove(name); + } + + private void checkIndexSpec(IndexSpec indexSpec) { + var order = indexSpec.getOrder(); + if (order == null) { + indexSpec.setOrder(IndexSpec.OrderType.ASC); + } + if (StringUtils.isBlank(indexSpec.getName())) { + throw new IllegalArgumentException("IndexSpec name must not be blank"); + } + if (indexSpec.getIndexFunc() == null) { + throw new IllegalArgumentException("IndexSpec indexFunc must not be null"); + } + } +} diff --git a/application/src/main/java/run/halo/app/extension/index/DefaultIndexer.java b/application/src/main/java/run/halo/app/extension/index/DefaultIndexer.java new file mode 100644 index 00000000000..5697d54e151 --- /dev/null +++ b/application/src/main/java/run/halo/app/extension/index/DefaultIndexer.java @@ -0,0 +1,200 @@ +package run.halo.app.extension.index; + +import static run.halo.app.extension.index.IndexerTransaction.ChangeRecord; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Function; +import org.apache.commons.lang3.BooleanUtils; +import run.halo.app.extension.Extension; + +/** + *

A default implementation of {@link Indexer}.

+ *

It uses the {@link IndexEntryContainer} to store the index entries for the specified + * {@link IndexDescriptor}s.

+ * + * @author guqing + * @since 2.12.0 + */ +public class DefaultIndexer implements Indexer { + private final ReadWriteLock rwl = new ReentrantReadWriteLock(); + private final Lock readLock = rwl.readLock(); + private final Lock writeLock = rwl.writeLock(); + + private final List indexDescriptors; + private final IndexEntryContainer indexEntries; + + /** + * Constructs a new {@link DefaultIndexer} with the given {@link IndexDescriptor}s and + * {@link IndexEntryContainer}. + * + * @param indexDescriptors the index descriptors. + * @param oldIndexEntries must have the same size with the given descriptors + */ + public DefaultIndexer(List indexDescriptors, + IndexEntryContainer oldIndexEntries) { + this.indexDescriptors = new ArrayList<>(indexDescriptors); + this.indexEntries = new IndexEntryContainer(); + for (IndexEntry entry : oldIndexEntries) { + this.indexEntries.add(entry); + } + for (IndexDescriptor indexDescriptor : indexDescriptors) { + if (!indexDescriptor.isReady()) { + throw new IllegalArgumentException( + "Index descriptor is not ready for: " + indexDescriptor.getSpec().getName()); + } + if (!this.indexEntries.contains(indexDescriptor)) { + throw new IllegalArgumentException( + "Index entry not found for: " + indexDescriptor.getSpec().getName()); + } + } + } + + static String getObjectKey(Extension extension) { + return PrimaryKeySpecUtils.getObjectPrimaryKey(extension); + } + + @Override + public void indexRecord(E extension) { + writeLock.lock(); + var txn = new IndexerTransactionImpl(); + try { + txn.begin(); + doIndexRecord(extension).forEach(txn::add); + txn.commit(); + } catch (Throwable e) { + txn.rollback(); + throw e; + } finally { + writeLock.unlock(); + } + } + + @Override + public void updateRecord(E extension) { + writeLock.lock(); + var txn = new IndexerTransactionImpl(); + try { + txn.begin(); + unIndexRecord(getObjectKey(extension)); + indexRecord(extension); + txn.commit(); + } catch (Throwable e) { + txn.rollback(); + throw e; + } finally { + writeLock.unlock(); + } + } + + @Override + public void unIndexRecord(String extensionName) { + writeLock.lock(); + var txn = new IndexerTransactionImpl(); + try { + txn.begin(); + doUnIndexRecord(extensionName).forEach(txn::add); + txn.commit(); + } catch (Throwable e) { + txn.rollback(); + throw e; + } finally { + writeLock.unlock(); + } + } + + private List doUnIndexRecord(String extensionName) { + List changeRecords = new ArrayList<>(); + for (IndexEntry indexEntry : indexEntries) { + indexEntry.entries().forEach(records -> { + var indexKey = records.getKey(); + var objectKey = records.getValue(); + if (objectKey.equals(extensionName)) { + changeRecords.add(ChangeRecord.onRemove(indexEntry, indexKey, objectKey)); + } + }); + } + return changeRecords; + } + + private List doIndexRecord(E extension) { + List changeRecords = new ArrayList<>(); + for (IndexDescriptor indexDescriptor : indexDescriptors) { + var indexEntry = indexEntries.get(indexDescriptor); + var indexFunc = indexDescriptor.getSpec().getIndexFunc(); + Set indexKeys = indexFunc.getValues(extension); + var objectKey = PrimaryKeySpecUtils.getObjectPrimaryKey(extension); + for (String indexKey : indexKeys) { + changeRecords.add(ChangeRecord.onAdd(indexEntry, indexKey, objectKey)); + } + } + return changeRecords; + } + + @Override + public IndexDescriptor findIndexByName(String name) { + readLock.lock(); + try { + return indexDescriptors.stream() + .filter(descriptor -> descriptor.getSpec().getName().equals(name)) + .findFirst() + .orElse(null); + } finally { + readLock.unlock(); + } + } + + @Override + public IndexEntry createIndexEntry(IndexDescriptor descriptor) { + return new IndexEntryImpl(descriptor); + } + + @Override + public void removeIndexRecords(Function matchFn) { + writeLock.lock(); + try { + var iterator = indexEntries.iterator(); + while (iterator.hasNext()) { + var entry = iterator.next(); + if (BooleanUtils.isTrue(matchFn.apply(entry.getIndexDescriptor()))) { + iterator.remove(); + entry.entries().clear(); + indexEntries.add(createIndexEntry(entry.getIndexDescriptor())); + } + } + } finally { + writeLock.unlock(); + } + } + + @Override + public Iterator readyIndexesIterator() { + readLock.lock(); + try { + var readyIndexes = new ArrayList(); + for (IndexEntry entry : indexEntries) { + if (entry.getIndexDescriptor().isReady()) { + readyIndexes.add(entry); + } + } + return readyIndexes.iterator(); + } finally { + readLock.unlock(); + } + } + + @Override + public Iterator allIndexesIterator() { + readLock.lock(); + try { + return indexEntries.iterator(); + } finally { + readLock.unlock(); + } + } +} diff --git a/application/src/main/java/run/halo/app/extension/index/ExtensionIterator.java b/application/src/main/java/run/halo/app/extension/index/ExtensionIterator.java new file mode 100644 index 00000000000..faf693c6d29 --- /dev/null +++ b/application/src/main/java/run/halo/app/extension/index/ExtensionIterator.java @@ -0,0 +1,17 @@ +package run.halo.app.extension.index; + +import java.util.Iterator; +import run.halo.app.extension.Extension; + +/** + * An iterator over a collection of extensions, it is used to iterate extensions in a paginated + * way to avoid loading all extensions into memory at once. + * + * @param the type of the extension. + * @author guqing + * @see DefaultExtensionIterator + * @since 2.12.0 + */ +public interface ExtensionIterator extends Iterator { + +} diff --git a/application/src/main/java/run/halo/app/extension/index/ExtensionPaginatedLister.java b/application/src/main/java/run/halo/app/extension/index/ExtensionPaginatedLister.java new file mode 100644 index 00000000000..dbf406b12bf --- /dev/null +++ b/application/src/main/java/run/halo/app/extension/index/ExtensionPaginatedLister.java @@ -0,0 +1,23 @@ +package run.halo.app.extension.index; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import run.halo.app.extension.Extension; + +/** + * List extensions with pagination, used for {@link ExtensionIterator}. + * + * @author guqing + * @since 2.12.0 + */ +public interface ExtensionPaginatedLister { + + /** + * List extensions with pagination. + * + * @param pageable pageable + * @param extension type + * @return page of extensions + */ + Page list(Pageable pageable); +} diff --git a/application/src/main/java/run/halo/app/extension/index/IndexBuilder.java b/application/src/main/java/run/halo/app/extension/index/IndexBuilder.java new file mode 100644 index 00000000000..4d0f98fb875 --- /dev/null +++ b/application/src/main/java/run/halo/app/extension/index/IndexBuilder.java @@ -0,0 +1,26 @@ +package run.halo.app.extension.index; + +import org.springframework.lang.NonNull; + +/** + * {@link IndexBuilder} is used to build index for a specific + * {@link run.halo.app.extension.Extension} type on startup. + * + * @author guqing + * @since 2.12.0 + */ +public interface IndexBuilder { + /** + * Start building index for a specific {@link run.halo.app.extension.Extension} type. + */ + void startBuildingIndex(); + + /** + * Gets final index entries after building index. + * + * @return index entries must not be null. + * @throws IllegalStateException if any index entries are not ready yet. + */ + @NonNull + IndexEntryContainer getIndexEntries(); +} diff --git a/application/src/main/java/run/halo/app/extension/index/IndexBuilderImpl.java b/application/src/main/java/run/halo/app/extension/index/IndexBuilderImpl.java new file mode 100644 index 00000000000..a165e71b16d --- /dev/null +++ b/application/src/main/java/run/halo/app/extension/index/IndexBuilderImpl.java @@ -0,0 +1,65 @@ +package run.halo.app.extension.index; + +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import org.springframework.lang.NonNull; +import run.halo.app.extension.Extension; + +public class IndexBuilderImpl implements IndexBuilder { + private final List indexDescriptors; + private final ExtensionIterator extensionIterator; + + private final IndexEntryContainer indexEntries = new IndexEntryContainer(); + + public static IndexBuilder of(List indexDescriptors, + ExtensionIterator extensionIterator) { + return new IndexBuilderImpl(indexDescriptors, extensionIterator); + } + + IndexBuilderImpl(List indexDescriptors, + ExtensionIterator extensionIterator) { + this.indexDescriptors = indexDescriptors; + this.extensionIterator = extensionIterator; + indexDescriptors.forEach(indexDescriptor -> { + var indexEntry = new IndexEntryImpl(indexDescriptor); + indexEntries.add(indexEntry); + }); + } + + @Override + public void startBuildingIndex() { + while (extensionIterator.hasNext()) { + var extensionRecord = extensionIterator.next(); + + indexRecords(extensionRecord); + } + + for (IndexDescriptor indexDescriptor : indexDescriptors) { + indexDescriptor.setReady(true); + } + } + + @Override + @NonNull + public IndexEntryContainer getIndexEntries() { + for (IndexEntry indexEntry : indexEntries) { + if (!indexEntry.getIndexDescriptor().isReady()) { + throw new IllegalStateException( + "IndexEntry are not ready yet for index named " + + indexEntry.getIndexDescriptor().getSpec().getName()); + } + } + return indexEntries; + } + + private void indexRecords(E extension) { + for (IndexDescriptor indexDescriptor : indexDescriptors) { + var indexEntry = indexEntries.get(indexDescriptor); + var indexFunc = indexDescriptor.getSpec().getIndexFunc(); + Set indexKeys = indexFunc.getValues(extension); + indexEntry.addEntry(new LinkedList<>(indexKeys), + PrimaryKeySpecUtils.getObjectPrimaryKey(extension)); + } + } +} diff --git a/application/src/main/java/run/halo/app/extension/index/IndexDescriptor.java b/application/src/main/java/run/halo/app/extension/index/IndexDescriptor.java new file mode 100644 index 00000000000..497eea145c2 --- /dev/null +++ b/application/src/main/java/run/halo/app/extension/index/IndexDescriptor.java @@ -0,0 +1,20 @@ +package run.halo.app.extension.index; + +import lombok.Data; +import lombok.ToString; + +@Data +@ToString(callSuper = true) +public class IndexDescriptor { + + private final IndexSpec spec; + + /** + * Record whether the index is ready, managed by {@link IndexBuilder}. + */ + private boolean ready; + + public IndexDescriptor(IndexSpec spec) { + this.spec = spec; + } +} diff --git a/application/src/main/java/run/halo/app/extension/index/IndexEntry.java b/application/src/main/java/run/halo/app/extension/index/IndexEntry.java new file mode 100644 index 00000000000..1e2b76ccdaf --- /dev/null +++ b/application/src/main/java/run/halo/app/extension/index/IndexEntry.java @@ -0,0 +1,93 @@ +package run.halo.app.extension.index; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import run.halo.app.extension.Metadata; + +/** + *

{@link IndexEntry} used to store the mapping between index key and + * {@link Metadata#getName()}.

+ *

For example, if we have a {@link Metadata} with name {@code foo} and labels {@code bar=1} + * and {@code baz=2}, then the index entry will be:

+ * 
+ *     bar=1 -> foo
+ *     baz=2 -> foo
+ *
+ *

And if we have another {@link Metadata} with name {@code bar} and labels {@code bar=1} + * and {@code baz=3}, then the index entry will be:

+ * 
+ *     bar=1 -> foo, bar
+ *     baz=2 -> foo
+ *     baz=3 -> bar
+ *
+ *

{@link #getIndexDescriptor()} describes the owner of this index entry.

+ *

Index entries is ordered by key, and the order is determined by + * {@link IndexSpec#getOrder()}.

+ *

Do not modify the returned result for all methods of this class.

+ *

This class is thread-safe.

+ * + * @author guqing + * @since 2.12.0 + */ +public interface IndexEntry { + + /** + *

Adds a new entry to this index entry.

+ *

For example, if we have a {@link Metadata} with name {@code foo} and labels {@code bar=1} + * and {@code baz=2} and index order is {@link IndexSpec.OrderType#ASC}, then the index entry + * will be:

+ * 
+     *     bar=1 -> foo
+     *     baz=2 -> foo
+     *
+ * + * @param indexKeys index keys + * @param objectKey object key (usually is {@link Metadata#getName()}). + */ + void addEntry(List indexKeys, String objectKey); + + /** + * Removes the entry with the given {@code indexedKey} and {@code objectKey}. + * + * @param indexedKey indexed key + * @param objectKey object key (usually is {@link Metadata#getName()}). + */ + void removeEntry(String indexedKey, String objectKey); + + /** + * Removes all entries with the given {@code objectKey}. + * + * @param objectKey object key(usually is {@link Metadata#getName()}). + */ + void remove(String objectKey); + + /** + * Returns the {@link IndexDescriptor} of this entry. + * + * @return the {@link IndexDescriptor} of this entry. + */ + IndexDescriptor getIndexDescriptor(); + + /** + * Returns the indexed keys of this entry in order. + * + * @return distinct indexed keys of this entry. + */ + Set indexedKeys(); + + /** + * Returns the entries of this entry in order. + * + * @return entries of this entry. + */ + Collection> entries(); + + /** + * Returns the object names of this entry in order. + * + * @return object names of this entry. + */ + List getByIndexKey(String indexKey); +} diff --git a/application/src/main/java/run/halo/app/extension/index/IndexEntryContainer.java b/application/src/main/java/run/halo/app/extension/index/IndexEntryContainer.java new file mode 100644 index 00000000000..dac484a735f --- /dev/null +++ b/application/src/main/java/run/halo/app/extension/index/IndexEntryContainer.java @@ -0,0 +1,71 @@ +package run.halo.app.extension.index; + +import java.util.Iterator; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Consumer; +import org.springframework.lang.NonNull; + +/** + *

A container for {@link IndexEntry}s, it is used to store all {@link IndexEntry}s according + * to the {@link IndexDescriptor}.

+ *

This class is thread-safe.

+ * + * @author guqing + * @see DefaultIndexer + * @since 2.12.0 + */ +public class IndexEntryContainer implements Iterable { + private final ConcurrentMap indexEntryMap; + + public IndexEntryContainer() { + this.indexEntryMap = new ConcurrentHashMap<>(); + } + + /** + * Add an {@link IndexEntry} to this container. + * + * @param entry the entry to add + * @throws IllegalArgumentException if the entry already exists + */ + public void add(IndexEntry entry) { + IndexEntry existing = indexEntryMap.putIfAbsent(entry.getIndexDescriptor(), entry); + if (existing != null) { + throw new IllegalArgumentException( + "Index entry already exists for " + entry.getIndexDescriptor()); + } + } + + /** + * Get the {@link IndexEntry} for the given {@link IndexDescriptor}. + * + * @param indexDescriptor the index descriptor + * @return the index entry + */ + public IndexEntry get(IndexDescriptor indexDescriptor) { + return indexEntryMap.get(indexDescriptor); + } + + public boolean contains(IndexDescriptor indexDescriptor) { + return indexEntryMap.containsKey(indexDescriptor); + } + + public void remove(IndexDescriptor indexDescriptor) { + indexEntryMap.remove(indexDescriptor); + } + + public int size() { + return indexEntryMap.size(); + } + + @Override + @NonNull + public Iterator iterator() { + return indexEntryMap.values().iterator(); + } + + @Override + public void forEach(Consumer action) { + indexEntryMap.values().forEach(action); + } +} diff --git a/application/src/main/java/run/halo/app/extension/index/IndexEntryImpl.java b/application/src/main/java/run/halo/app/extension/index/IndexEntryImpl.java new file mode 100644 index 00000000000..81c7cdcd23e --- /dev/null +++ b/application/src/main/java/run/halo/app/extension/index/IndexEntryImpl.java @@ -0,0 +1,118 @@ +package run.halo.app.extension.index; + +import com.google.common.collect.ListMultimap; +import com.google.common.collect.MultimapBuilder; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import lombok.Data; +import run.halo.app.infra.exception.DuplicateNameException; + +@Data +public class IndexEntryImpl implements IndexEntry { + private final ReadWriteLock rwl = new ReentrantReadWriteLock(); + private final Lock readLock = rwl.readLock(); + private final Lock writeLock = rwl.writeLock(); + + private final IndexDescriptor indexDescriptor; + private final ListMultimap indexKeyObjectNamesMap; + + /** + * Creates a new {@link IndexEntryImpl} for the given {@link IndexDescriptor}. + * + * @param indexDescriptor for which the {@link IndexEntryImpl} is created. + */ + public IndexEntryImpl(IndexDescriptor indexDescriptor) { + this.indexDescriptor = indexDescriptor; + this.indexKeyObjectNamesMap = MultimapBuilder.treeKeys(keyComparator()) + .linkedListValues().build(); + } + + Comparator keyComparator() { + var order = indexDescriptor.getSpec().getOrder(); + if (IndexSpec.OrderType.ASC.equals(order)) { + return Comparator.naturalOrder(); + } else if (IndexSpec.OrderType.DESC.equals(order)) { + return Comparator.reverseOrder(); + } else { + throw new IllegalArgumentException("Invalid order: " + order); + } + } + + @Override + public void addEntry(List keys, String objectName) { + var isUnique = indexDescriptor.getSpec().isUnique(); + for (String key : keys) { + writeLock.lock(); + try { + if (isUnique && indexKeyObjectNamesMap.containsKey(key)) { + throw new DuplicateNameException( + "The value [%s] is already exists for unique index [%s].".formatted( + key, + indexDescriptor.getSpec().getName()), + null, + "problemDetail.index.duplicateKey", + new Object[] {key, indexDescriptor.getSpec().getName()}); + } + this.indexKeyObjectNamesMap.put(key, objectName); + } finally { + writeLock.unlock(); + } + } + } + + @Override + public void removeEntry(String indexedKey, String objectKey) { + writeLock.lock(); + try { + indexKeyObjectNamesMap.remove(indexedKey, objectKey); + } finally { + writeLock.unlock(); + } + } + + @Override + public void remove(String objectName) { + writeLock.lock(); + try { + indexKeyObjectNamesMap.values().removeIf(objectName::equals); + } finally { + writeLock.unlock(); + } + } + + @Override + public Set indexedKeys() { + readLock.lock(); + try { + return indexKeyObjectNamesMap.keySet(); + } finally { + readLock.unlock(); + } + } + + @Override + public Collection> entries() { + readLock.lock(); + try { + return indexKeyObjectNamesMap.entries(); + } finally { + readLock.unlock(); + } + } + + @Override + public List getByIndexKey(String indexKey) { + readLock.lock(); + try { + return indexKeyObjectNamesMap.get(indexKey); + } finally { + readLock.unlock(); + } + } +} diff --git a/application/src/main/java/run/halo/app/extension/index/IndexSpecRegistryImpl.java b/application/src/main/java/run/halo/app/extension/index/IndexSpecRegistryImpl.java new file mode 100644 index 00000000000..4478d4430fb --- /dev/null +++ b/application/src/main/java/run/halo/app/extension/index/IndexSpecRegistryImpl.java @@ -0,0 +1,96 @@ +package run.halo.app.extension.index; + +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import lombok.RequiredArgsConstructor; +import org.springframework.lang.NonNull; +import org.springframework.stereotype.Component; +import run.halo.app.extension.Extension; +import run.halo.app.extension.ExtensionStoreUtil; +import run.halo.app.extension.Scheme; +import run.halo.app.extension.SchemeManager; + +/** + *

A default implementation of {@link IndexSpecRegistry}.

+ * + * @author guqing + * @since 2.12.0 + */ +@Component +@RequiredArgsConstructor +public class IndexSpecRegistryImpl implements IndexSpecRegistry { + private final SchemeManager schemeManager; + private final ConcurrentMap extensionIndexSpecs = new ConcurrentHashMap<>(); + + @Override + public IndexSpecs indexFor(Class extensionType) { + var keySpace = getKeySpace(extensionType); + var indexSpecs = new DefaultIndexSpecs(); + useDefaultIndexSpec(extensionType, indexSpecs); + extensionIndexSpecs.put(keySpace, indexSpecs); + return indexSpecs; + } + + @Override + public IndexSpecs getIndexSpecs(Class extensionType) { + var keySpace = getKeySpace(extensionType); + var result = extensionIndexSpecs.get(keySpace); + if (result == null) { + throw new IllegalArgumentException( + "No index specs found for extension type: " + extensionType.getName() + + ", make sure you have called indexFor() before calling getIndexSpecs()"); + + } + return result; + } + + @Override + public boolean contains(Class extensionType) { + var keySpace = getKeySpace(extensionType); + return extensionIndexSpecs.containsKey(keySpace); + } + + @Override + public void removeIndexSpecs(Class extensionType) { + extensionIndexSpecs.remove(getKeySpace(extensionType)); + } + + @NonNull + private String getKeySpace(Class extensionType) { + var scheme = schemeManager.get(extensionType); + return ExtensionStoreUtil.buildStoreNamePrefix(scheme); + } + + @Override + @NonNull + public String getKeySpace(Scheme scheme) { + return ExtensionStoreUtil.buildStoreNamePrefix(scheme); + } + + void useDefaultIndexSpec(Class extensionType, + IndexSpecs indexSpecs) { + var nameIndexSpec = PrimaryKeySpecUtils.primaryKeyIndexSpec(extensionType); + indexSpecs.add(nameIndexSpec); + + var creationTimestampIndexSpec = new IndexSpec() + .setName("metadata.creationTimestamp") + .setOrder(IndexSpec.OrderType.ASC) + .setUnique(false) + .setIndexFunc(IndexAttributeFactory.simpleAttribute(extensionType, + e -> e.getMetadata().getCreationTimestamp().toString()) + ); + indexSpecs.add(creationTimestampIndexSpec); + + var deletionTimestampIndexSpec = new IndexSpec() + .setName("metadata.deletionTimestamp") + .setOrder(IndexSpec.OrderType.ASC) + .setUnique(false) + .setIndexFunc(IndexAttributeFactory.simpleAttribute(extensionType, + e -> Objects.toString(e.getMetadata().getDeletionTimestamp(), null)) + ); + indexSpecs.add(deletionTimestampIndexSpec); + + indexSpecs.add(LabelIndexSpecUtils.labelIndexSpec(extensionType)); + } +} diff --git a/application/src/main/java/run/halo/app/extension/index/IndexedQueryEngineImpl.java b/application/src/main/java/run/halo/app/extension/index/IndexedQueryEngineImpl.java new file mode 100644 index 00000000000..97789167370 --- /dev/null +++ b/application/src/main/java/run/halo/app/extension/index/IndexedQueryEngineImpl.java @@ -0,0 +1,275 @@ +package run.halo.app.extension.index; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.PriorityQueue; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Sort; +import org.springframework.lang.NonNull; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StopWatch; +import run.halo.app.extension.GroupVersionKind; +import run.halo.app.extension.ListOptions; +import run.halo.app.extension.ListResult; +import run.halo.app.extension.PageRequest; +import run.halo.app.extension.router.selector.FieldSelector; +import run.halo.app.extension.router.selector.SelectorMatcher; + +/** + * A default implementation of {@link IndexedQueryEngine}. + * + * @author guqing + * @since 2.12.0 + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class IndexedQueryEngineImpl implements IndexedQueryEngine { + + private final IndexerFactory indexerFactory; + + private static Map fieldPathIndexEntryMap(Indexer indexer) { + // O(n) time complexity + Map indexEntryMap = new LinkedHashMap<>(); + var iterator = indexer.readyIndexesIterator(); + while (iterator.hasNext()) { + var indexEntry = iterator.next(); + var descriptor = indexEntry.getIndexDescriptor(); + var indexedFieldPath = descriptor.getSpec().getName(); + indexEntryMap.put(indexedFieldPath, indexEntry); + } + return indexEntryMap; + } + + static IndexEntry getIndexEntry(String fieldPath, Map fieldPathEntryMap) { + if (!fieldPathEntryMap.containsKey(fieldPath)) { + throwNotIndexedException(fieldPath); + } + return fieldPathEntryMap.get(fieldPath); + } + + @Override + public ListResult retrieve(GroupVersionKind type, ListOptions options, + PageRequest page) { + var indexer = indexerFactory.getIndexer(type); + var allMatchedResult = doRetrieve(indexer, options, page.getSort()); + var list = ListResult.subList(allMatchedResult, page.getPageNumber(), page.getPageSize()); + return new ListResult<>(page.getPageNumber(), page.getPageSize(), + allMatchedResult.size(), list); + } + + @Override + public List retrieveAll(GroupVersionKind type, ListOptions options) { + var indexer = indexerFactory.getIndexer(type); + return doRetrieve(indexer, options, Sort.unsorted()); + } + + static List intersection(List list1, List list2) { + Set set = new LinkedHashSet<>(list1); + List intersection = new ArrayList<>(); + for (T item : list2) { + if (set.contains(item) && !intersection.contains(item)) { + intersection.add(item); + } + } + return intersection; + } + + static void throwNotIndexedException(String fieldPath) { + throw new IllegalArgumentException( + "No index found for fieldPath: " + fieldPath + + ", make sure you have created an index for this field."); + } + + List retrieveForLabelMatchers(List labelMatchers, + Map fieldPathEntryMap, List allMetadataNames) { + var indexEntry = getIndexEntry(LabelIndexSpecUtils.LABEL_PATH, fieldPathEntryMap); + // O(m) time complexity, m is the number of labelMatchers + var labelKeysToQuery = labelMatchers.stream() + .map(SelectorMatcher::getKey) + .collect(Collectors.toSet()); + + Map> objectNameLabelsMap = new LinkedHashMap<>(); + indexEntry.entries().forEach(entry -> { + // key is labelKey=labelValue, value is objectName + var labelPair = LabelIndexSpecUtils.labelKeyValuePair(entry.getKey()); + if (!labelKeysToQuery.contains(labelPair.getFirst())) { + return; + } + objectNameLabelsMap.computeIfAbsent(entry.getValue(), k -> new LinkedHashMap<>()) + .put(labelPair.getFirst(), labelPair.getSecond()); + }); + // O(p * m) time complexity, p is the number of allMetadataNames + return allMetadataNames.stream() + .filter(objectName -> { + var labels = objectNameLabelsMap.getOrDefault(objectName, Map.of()); + // object match all labels will be returned + return labelMatchers.stream() + .allMatch(matcher -> matcher.test(labels.get(matcher.getKey()))); + }) + .toList(); + } + + List doRetrieve(Indexer indexer, ListOptions options, Sort sort) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("build index entry map"); + var fieldPathEntryMap = fieldPathIndexEntryMap(indexer); + stopWatch.stop(); + var primaryEntry = getIndexEntry(PrimaryKeySpecUtils.PRIMARY_INDEX_NAME, fieldPathEntryMap); + // O(n) time complexity + stopWatch.start("retrieve all metadata names"); + var allMetadataNames = new ArrayList<>(primaryEntry.indexedKeys()); + stopWatch.stop(); + + stopWatch.start("retrieve matched metadata names"); + final Optional> matchedByLabels = + Optional.ofNullable(options.getLabelSelector()) + .filter(selector -> !CollectionUtils.isEmpty(selector.getMatchers())) + .map(labelSelector -> labelSelector.getMatchers() + .stream() + .sorted(Comparator.comparing(SelectorMatcher::getKey)) + .toList() + ) + .map(matchers -> retrieveForLabelMatchers(matchers, + fieldPathEntryMap, allMetadataNames) + ); + stopWatch.stop(); + + stopWatch.start("retrieve matched metadata names by fields"); + Optional> matchedByFields = Optional.ofNullable(options.getFieldSelector()) + .filter(selector -> !CollectionUtils.isEmpty(selector.getMatchers())) + .map(FieldSelector::getMatchers) + .map(matchers -> matchers.stream() + .map(matcher -> { + var fieldPath = matcher.getKey(); + if (!fieldPathEntryMap.containsKey(fieldPath)) { + throwNotIndexedException(fieldPath); + } + var indexEntry = fieldPathEntryMap.get(fieldPath); + var indexedKeys = indexEntry.indexedKeys(); + return indexedKeys.stream() + .filter(matcher::test) + .map(indexEntry::getByIndexKey) + .flatMap(List::stream) + .toList(); + }) + .reduce(IndexedQueryEngineImpl::intersection) + .orElse(List.of()) + ); + stopWatch.stop(); + + stopWatch.start("merge result"); + List foundObjectKeys; + if (matchedByLabels.isEmpty() && matchedByFields.isEmpty()) { + foundObjectKeys = allMetadataNames; + } else if (matchedByLabels.isEmpty()) { + foundObjectKeys = matchedByFields.orElse(allMetadataNames); + } else { + foundObjectKeys = matchedByFields + .map(strings -> intersection(matchedByLabels.get(), strings)) + .orElseGet(() -> matchedByLabels.orElse(allMetadataNames)); + } + stopWatch.stop(); + + stopWatch.start("sort result"); + ResultSorter resultSorter = new ResultSorter(fieldPathEntryMap, foundObjectKeys); + var result = resultSorter.sortBy(sort); + stopWatch.stop(); + log.debug("Retrieve result from indexer, {}", stopWatch.prettyPrint()); + return result; + } + + /** + * Sort the given list by the given {@link Sort}. + */ + static class ResultSorter { + private final Map fieldPathEntryMap; + private final List list; + + public ResultSorter(Map fieldPathEntryMap, List list) { + this.fieldPathEntryMap = fieldPathEntryMap; + this.list = list; + } + + public List sortBy(@NonNull Sort sort) { + if (sort.isUnsorted()) { + return list; + } + var sortedLists = new ArrayList>(); + for (Sort.Order order : sort) { + var indexEntry = fieldPathEntryMap.get(order.getProperty()); + if (indexEntry == null) { + throwNotIndexedException(order.getProperty()); + } + var entries = new ArrayList<>(indexEntry.entries()); + var indexOrder = indexEntry.getIndexDescriptor().getSpec().getOrder(); + var asc = IndexSpec.OrderType.ASC.equals(indexOrder); + if (asc != order.isAscending()) { + Collections.reverse(entries); + } + entries.removeIf(entry -> !list.contains(entry.getValue())); + var objectNames = entries.stream() + .map(Map.Entry::getValue) + .collect(Collectors.toList()); + sortedLists.add(objectNames); + } + return mergeSortedLists(sortedLists); + } + + /** + *

Merge the given sorted lists into one sorted list.

+ *

The time complexity is O(n * log(m)), n is the number of all elements in the + * sortedLists, m is the number of sortedLists.

+ */ + private List mergeSortedLists(List> sortedLists) { + List result = new ArrayList<>(); + // Use a priority queue to store the current element of each list and its index in + // the list + PriorityQueue minHeap = new PriorityQueue<>( + Comparator.comparing(pair -> pair.value)); + + // Initialize the priority queue and add the first element of each list to the queue + for (int i = 0; i < sortedLists.size(); i++) { + if (!sortedLists.get(i).isEmpty()) { + minHeap.add(new Pair(i, 0, sortedLists.get(i).get(0))); + } + } + + while (!minHeap.isEmpty()) { + Pair current = minHeap.poll(); + result.add(current.value()); + + // Add the next element of this list to the priority queue + if (current.indexInList() + 1 < sortedLists.get(current.listIndex()).size()) { + var list = sortedLists.get(current.listIndex()); + minHeap.add(new Pair(current.listIndex(), + current.indexInList() + 1, + list.get(current.indexInList() + 1)) + ); + } + } + return result; + } + + /** + *

A pair of element and its position in the original list.

+ * 
+         * listIndex: column index.
+         * indexInList: element index in the list.
+         * value: element value.
+         *
+ */ + private record Pair(int listIndex, int indexInList, String value) { + } + } +} diff --git a/application/src/main/java/run/halo/app/extension/index/Indexer.java b/application/src/main/java/run/halo/app/extension/index/Indexer.java new file mode 100644 index 00000000000..78702f3c752 --- /dev/null +++ b/application/src/main/java/run/halo/app/extension/index/Indexer.java @@ -0,0 +1,91 @@ +package run.halo.app.extension.index; + +import java.util.Iterator; +import java.util.function.Function; +import run.halo.app.extension.Extension; + +/** + *

The {@link Indexer} is owned by the {@link Extension} and is responsible for the lookup and + * lifetimes of the indexes in a {@link Extension} collection. Every {@link Extension} has + * exactly one instance of this class.

+ *

Callers are expected to have acquired the necessary locks while accessing this interface.

+ * To inspect the contents of this {@link Indexer}, callers may obtain an iterator from + * getIndexIterator(). + * + * @author guqing + * @since 2.12.0 + */ +public interface Indexer { + + /** + *

Index the specified {@link Extension} by {@link IndexDescriptor}s.

+ *

First, the {@link Indexer} will index the {@link Extension} by the + * {@link IndexDescriptor}s and record the index entries to {@link IndexerTransaction} and + * commit the transaction, if any error occurs, the transaction will be rollback to keep the + * {@link Indexer} consistent.

+ * + * @param extension the {@link Extension} to be indexed + * @param the type of the {@link Extension} + */ + void indexRecord(E extension); + + /** + *

Update indexes for the specified {@link Extension} by {@link IndexDescriptor}s.

+ *

First, the {@link Indexer} will remove the index entries of the {@link Extension} by + * the old {@link IndexDescriptor}s and reindex the {@link Extension} to generate change logs + * to {@link IndexerTransaction} and commit the transaction, if any error occurs, the + * transaction will be rollback to keep the {@link Indexer} consistent.

+ * + * @param extension the {@link Extension} to be updated + * @param the type of the {@link Extension} + */ + void updateRecord(E extension); + + /** + *

Remove indexes (index entries) for the specified {@link Extension} already indexed by + * {@link IndexDescriptor}s.

+ * + * @param extensionName the {@link Extension} to be removed + */ + void unIndexRecord(String extensionName); + + /** + *

Find index by name.

+ *

The index name uniquely identifies an index.

+ * + * @param name index name + * @return index descriptor if found, null otherwise + */ + IndexDescriptor findIndexByName(String name); + + /** + *

Create an index entry for the specified {@link IndexDescriptor}.

+ * + * @param descriptor the {@link IndexDescriptor} to be recorded + * @return the {@link IndexEntry} created + */ + IndexEntry createIndexEntry(IndexDescriptor descriptor); + + /** + *

Remove all index entries that match the given {@link IndexDescriptor}.

+ * + * @param matchFn the {@link IndexDescriptor} to be matched + */ + void removeIndexRecords(Function matchFn); + + /** + *

Gets an iterator over all the ready {@link IndexEntry}s, in no particular order.

+ * + * @return an iterator over all the ready {@link IndexEntry}s + * @link {@link IndexDescriptor#isReady()} + */ + Iterator readyIndexesIterator(); + + /** + *

Gets an iterator over all the {@link IndexEntry}s, in no particular order.

+ * + * @return an iterator over all the {@link IndexEntry}s + * @link {@link IndexDescriptor#isReady()} + */ + Iterator allIndexesIterator(); +} diff --git a/application/src/main/java/run/halo/app/extension/index/IndexerFactory.java b/application/src/main/java/run/halo/app/extension/index/IndexerFactory.java new file mode 100644 index 00000000000..11289284a22 --- /dev/null +++ b/application/src/main/java/run/halo/app/extension/index/IndexerFactory.java @@ -0,0 +1,56 @@ +package run.halo.app.extension.index; + +import org.springframework.lang.NonNull; +import run.halo.app.extension.Extension; +import run.halo.app.extension.ExtensionStoreUtil; +import run.halo.app.extension.GroupVersionKind; +import run.halo.app.extension.Scheme; +import run.halo.app.extension.SchemeManager; + +/** + *

{@link IndexerFactory} is used to create {@link Indexer} for {@link Extension} type.

+ *

It's stored {@link Indexer} by key space, the key space is generated by {@link Scheme} like + * {@link ExtensionStoreUtil#buildStoreNamePrefix(Scheme)}.

+ *

E.g. create {@link Indexer} for Post extension, the mapping relationship is:

+ * 
+ *    /registry/content.halo.run/posts -> Indexer
+ *
+ * + * @author guqing + * @since 2.12.0 + */ +public interface IndexerFactory { + + /** + * Create {@link Indexer} for {@link Extension} type. + * + * @param extensionType the extension type must exist in {@link SchemeManager}. + * @param extensionIterator the extension iterator to iterate all records for the extension type + * @return created {@link Indexer} + */ + @NonNull + Indexer createIndexerFor(Class extensionType, + ExtensionIterator extensionIterator); + + /** + * Get {@link Indexer} for {@link GroupVersionKind}. + * + * @param gvk the group version kind must exist in {@link SchemeManager} + * @return the indexer + * @throws IllegalArgumentException if the {@link GroupVersionKind} represents a special + * {@link Extension} not exists in {@link SchemeManager} + */ + @NonNull + Indexer getIndexer(GroupVersionKind gvk); + + boolean contains(GroupVersionKind gvk); + + /** + *

Remove a specific {@link Indexer} by {@link Scheme} that represents a {@link Extension} + * .

+ *

Usually, the specified {@link Scheme} is not in {@link SchemeManager} at this time.

+ * + * @param scheme the scheme represents a {@link Extension} + */ + void removeIndexer(Scheme scheme); +} diff --git a/application/src/main/java/run/halo/app/extension/index/IndexerFactoryImpl.java b/application/src/main/java/run/halo/app/extension/index/IndexerFactoryImpl.java new file mode 100644 index 00000000000..9fe9a641497 --- /dev/null +++ b/application/src/main/java/run/halo/app/extension/index/IndexerFactoryImpl.java @@ -0,0 +1,83 @@ +package run.halo.app.extension.index; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.lang.NonNull; +import org.springframework.stereotype.Component; +import run.halo.app.extension.Extension; +import run.halo.app.extension.ExtensionStoreUtil; +import run.halo.app.extension.GroupVersionKind; +import run.halo.app.extension.Scheme; +import run.halo.app.extension.SchemeManager; + +/** + *

A default implementation of {@link IndexerFactory}.

+ * + * @author guqing + * @since 2.12.0 + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class IndexerFactoryImpl implements IndexerFactory { + + private final ConcurrentMap keySpaceIndexer = new ConcurrentHashMap<>(); + private final IndexSpecRegistry indexSpecRegistry; + private final SchemeManager schemeManager; + + @Override + @NonNull + public Indexer createIndexerFor(Class extensionType, + ExtensionIterator extensionIterator) { + var scheme = schemeManager.get(extensionType); + var keySpace = indexSpecRegistry.getKeySpace(scheme); + if (keySpaceIndexer.containsKey(keySpace)) { + throw new IllegalArgumentException("Indexer already exists for type: " + keySpace); + } + if (!indexSpecRegistry.contains(extensionType)) { + indexSpecRegistry.indexFor(extensionType); + } + var specs = indexSpecRegistry.getIndexSpecs(extensionType); + var indexDescriptors = specs.getIndexSpecs() + .stream() + .map(IndexDescriptor::new) + .toList(); + + final long startTimeMs = System.currentTimeMillis(); + log.info("Start building index for type: {}, please wait...", keySpace); + var indexBuilder = IndexBuilderImpl.of(indexDescriptors, extensionIterator); + indexBuilder.startBuildingIndex(); + var indexer = + new DefaultIndexer(indexDescriptors, indexBuilder.getIndexEntries()); + keySpaceIndexer.put(keySpace, indexer); + log.info("Index for type: {} built successfully, cost {} ms", keySpace, + System.currentTimeMillis() - startTimeMs); + return indexer; + } + + @Override + @NonNull + public Indexer getIndexer(GroupVersionKind gvk) { + var scheme = schemeManager.get(gvk); + var indexer = keySpaceIndexer.get(indexSpecRegistry.getKeySpace(scheme)); + if (indexer == null) { + throw new IllegalArgumentException("No indexer found for type: " + gvk); + } + return indexer; + } + + @Override + public boolean contains(GroupVersionKind gvk) { + var schemeOpt = schemeManager.fetch(gvk); + return schemeOpt.isPresent() + && keySpaceIndexer.containsKey(indexSpecRegistry.getKeySpace(schemeOpt.get())); + } + + @Override + public void removeIndexer(Scheme scheme) { + var keySpace = ExtensionStoreUtil.buildStoreNamePrefix(scheme); + keySpaceIndexer.remove(keySpace); + } +} diff --git a/application/src/main/java/run/halo/app/extension/index/IndexerTransaction.java b/application/src/main/java/run/halo/app/extension/index/IndexerTransaction.java new file mode 100644 index 00000000000..cbb090dbfa4 --- /dev/null +++ b/application/src/main/java/run/halo/app/extension/index/IndexerTransaction.java @@ -0,0 +1,40 @@ +package run.halo.app.extension.index; + +import org.springframework.util.Assert; + +/** + *

{@link IndexerTransaction} is a transactional interface for {@link Indexer} to ensure + * consistency when {@link Indexer} indexes objects.

+ *

It is not supported to call {@link #begin()} twice without calling {@link #commit()} or + * {@link #rollback()} in between and it is not supported to call one of {@link #commit()} or + * {@link #rollback()} in different thread than {@link #begin()} was called.

+ * + * @author guqing + * @since 2.12.0 + */ +public interface IndexerTransaction { + void begin(); + + void commit(); + + void rollback(); + + void add(ChangeRecord changeRecord); + + record ChangeRecord(IndexEntry indexEntry, String key, String value, boolean isAdd) { + + public ChangeRecord { + Assert.notNull(indexEntry, "IndexEntry must not be null"); + Assert.notNull(key, "Key must not be null"); + Assert.notNull(value, "Value must not be null"); + } + + public static ChangeRecord onAdd(IndexEntry indexEntry, String key, String value) { + return new ChangeRecord(indexEntry, key, value, true); + } + + public static ChangeRecord onRemove(IndexEntry indexEntry, String key, String value) { + return new ChangeRecord(indexEntry, key, value, false); + } + } +} diff --git a/application/src/main/java/run/halo/app/extension/index/IndexerTransactionImpl.java b/application/src/main/java/run/halo/app/extension/index/IndexerTransactionImpl.java new file mode 100644 index 00000000000..d90bf922c59 --- /dev/null +++ b/application/src/main/java/run/halo/app/extension/index/IndexerTransactionImpl.java @@ -0,0 +1,105 @@ +package run.halo.app.extension.index; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.List; + +/** + * Implementation of {@link IndexerTransaction}. + * + * @author guqing + * @since 2.12.0 + */ +public class IndexerTransactionImpl implements IndexerTransaction { + private Deque changeRecords; + private boolean inTransaction = false; + private Long threadId; + + @Override + public synchronized void begin() { + if (inTransaction) { + throw new IllegalStateException("Transaction already active"); + } + threadId = Thread.currentThread().getId(); + this.changeRecords = new ArrayDeque<>(); + inTransaction = true; + } + + @Override + public synchronized void commit() { + checkThread(); + if (!inTransaction) { + throw new IllegalStateException("Transaction not started"); + } + Deque committedRecords = new ArrayDeque<>(); + try { + while (!changeRecords.isEmpty()) { + var changeRecord = changeRecords.pop(); + applyChange(changeRecord); + committedRecords.push(changeRecord); + } + // Reset threadId after transaction ends + inTransaction = false; + // Reset threadId after transaction ends + threadId = null; + } catch (Exception e) { + // Rollback the changes that were committed before the error occurred + while (!committedRecords.isEmpty()) { + var changeRecord = committedRecords.pop(); + revertChange(changeRecord); + } + throw e; + } + } + + @Override + public synchronized void rollback() { + checkThread(); + if (!inTransaction) { + throw new IllegalStateException("Transaction not started"); + } + changeRecords.clear(); + inTransaction = false; + // Reset threadId after transaction ends + threadId = null; + } + + @Override + public synchronized void add(ChangeRecord changeRecord) { + if (inTransaction) { + changeRecords.push(changeRecord); + } else { + throw new IllegalStateException("No active transaction to add changes"); + } + } + + private void applyChange(ChangeRecord changeRecord) { + var indexEntry = changeRecord.indexEntry(); + var key = changeRecord.key(); + var value = changeRecord.value(); + var isAdd = changeRecord.isAdd(); + if (isAdd) { + indexEntry.addEntry(List.of(key), value); + } else { + indexEntry.removeEntry(key, value); + } + } + + private void revertChange(ChangeRecord changeRecord) { + var indexEntry = changeRecord.indexEntry(); + var key = changeRecord.key(); + var value = changeRecord.value(); + var isAdd = changeRecord.isAdd(); + if (isAdd) { + indexEntry.removeEntry(key, value); + } else { + indexEntry.addEntry(List.of(key), value); + } + } + + private void checkThread() { + if (threadId != null && !threadId.equals(Thread.currentThread().getId())) { + throw new IllegalStateException("Transaction cannot span multiple threads!"); + } + } +} diff --git a/application/src/main/java/run/halo/app/extension/index/LabelIndexSpecUtils.java b/application/src/main/java/run/halo/app/extension/index/LabelIndexSpecUtils.java new file mode 100644 index 00000000000..4117245e952 --- /dev/null +++ b/application/src/main/java/run/halo/app/extension/index/LabelIndexSpecUtils.java @@ -0,0 +1,55 @@ +package run.halo.app.extension.index; + +import java.util.Set; +import java.util.stream.Collectors; +import lombok.experimental.UtilityClass; +import org.springframework.data.util.Pair; +import run.halo.app.extension.Extension; + +@UtilityClass +public class LabelIndexSpecUtils { + + public static final String LABEL_PATH = "metadata.labels"; + + /** + * Creates a label index spec. + * + * @param extensionType extension type + * @param extension type + * @return label index spec + */ + public static IndexSpec labelIndexSpec(Class extensionType) { + return new IndexSpec() + .setName(LABEL_PATH) + .setOrder(IndexSpec.OrderType.ASC) + .setUnique(false) + .setIndexFunc(IndexAttributeFactory.multiValueAttribute(extensionType, + LabelIndexSpecUtils::labelIndexValueFunc) + ); + } + + /** + * Label key-value pair from indexed label key string, e.g. "key=value". + * + * @param indexedLabelKey indexed label key + * @return label key-value pair + */ + public static Pair labelKeyValuePair(String indexedLabelKey) { + var idx = indexedLabelKey.indexOf('='); + if (idx != -1) { + return Pair.of(indexedLabelKey.substring(0, idx), indexedLabelKey.substring(idx + 1)); + } + throw new IllegalArgumentException("Invalid label key-value pair: " + indexedLabelKey); + } + + static Set labelIndexValueFunc(E obj) { + var labels = obj.getMetadata().getLabels(); + if (labels == null) { + return Set.of(); + } + return labels.entrySet() + .stream() + .map(entry -> entry.getKey() + "=" + entry.getValue()) + .collect(Collectors.toSet()); + } +} diff --git a/application/src/main/java/run/halo/app/extension/index/PrimaryKeySpecUtils.java b/application/src/main/java/run/halo/app/extension/index/PrimaryKeySpecUtils.java new file mode 100644 index 00000000000..6fb11bf00c9 --- /dev/null +++ b/application/src/main/java/run/halo/app/extension/index/PrimaryKeySpecUtils.java @@ -0,0 +1,30 @@ +package run.halo.app.extension.index; + +import lombok.experimental.UtilityClass; +import run.halo.app.extension.Extension; + +@UtilityClass +public class PrimaryKeySpecUtils { + public static final String PRIMARY_INDEX_NAME = "metadata.name"; + + /** + * Primary key index spec. + * + * @param type the type + * @param the type parameter of {@link Extension} + * @return the index spec + */ + public static IndexSpec primaryKeyIndexSpec(Class type) { + return new IndexSpec() + .setName(PRIMARY_INDEX_NAME) + .setOrder(IndexSpec.OrderType.ASC) + .setUnique(true) + .setIndexFunc(IndexAttributeFactory.simpleAttribute(type, + e -> e.getMetadata().getName()) + ); + } + + public static String getObjectPrimaryKey(Extension obj) { + return obj.getMetadata().getName(); + } +} diff --git a/application/src/main/java/run/halo/app/extension/router/ExtensionListHandler.java b/application/src/main/java/run/halo/app/extension/router/ExtensionListHandler.java index 4603801f7b4..f11e979a98f 100644 --- a/application/src/main/java/run/halo/app/extension/router/ExtensionListHandler.java +++ b/application/src/main/java/run/halo/app/extension/router/ExtensionListHandler.java @@ -25,11 +25,10 @@ public ExtensionListHandler(Scheme scheme, ReactiveExtensionClient client) { @NonNull public Mono handle(@NonNull ServerRequest request) { var queryParams = new SortableRequest(request.exchange()); - return client.list(scheme.type(), - queryParams.toPredicate(), - queryParams.toComparator(), - queryParams.getPage(), - queryParams.getSize()) + return client.listBy(scheme.type(), + queryParams.toListOptions(), + queryParams.toPageRequest() + ) .flatMap(listResult -> ServerResponse .ok() .contentType(MediaType.APPLICATION_JSON) diff --git a/application/src/main/java/run/halo/app/extension/store/ExtensionStoreClient.java b/application/src/main/java/run/halo/app/extension/store/ExtensionStoreClient.java index e1ad8e8dfa8..45e3adf988a 100644 --- a/application/src/main/java/run/halo/app/extension/store/ExtensionStoreClient.java +++ b/application/src/main/java/run/halo/app/extension/store/ExtensionStoreClient.java @@ -2,6 +2,8 @@ import java.util.List; import java.util.Optional; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; /** * An interface to query and operate ExtensionStore. @@ -18,6 +20,10 @@ public interface ExtensionStoreClient { */ List listByNamePrefix(String prefix); + Page listByNamePrefix(String prefix, Pageable pageable); + + List listByNames(List names); + /** * Fetches an ExtensionStore by unique name. * diff --git a/application/src/main/java/run/halo/app/extension/store/ExtensionStoreClientJPAImpl.java b/application/src/main/java/run/halo/app/extension/store/ExtensionStoreClientJPAImpl.java index 82b2586de37..e5030709509 100644 --- a/application/src/main/java/run/halo/app/extension/store/ExtensionStoreClientJPAImpl.java +++ b/application/src/main/java/run/halo/app/extension/store/ExtensionStoreClientJPAImpl.java @@ -1,10 +1,10 @@ package run.halo.app.extension.store; -import jakarta.persistence.EntityNotFoundException; import java.util.List; import java.util.Optional; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; -import reactor.core.publisher.Mono; /** * An implementation of ExtensionStoreClient using JPA. @@ -14,44 +14,44 @@ @Service public class ExtensionStoreClientJPAImpl implements ExtensionStoreClient { - private final ExtensionStoreRepository repository; + private final ReactiveExtensionStoreClient storeClient; - public ExtensionStoreClientJPAImpl(ExtensionStoreRepository repository) { - this.repository = repository; + public ExtensionStoreClientJPAImpl(ReactiveExtensionStoreClient storeClient) { + this.storeClient = storeClient; } @Override public List listByNamePrefix(String prefix) { - return repository.findAllByNameStartingWith(prefix).collectList().block(); + return storeClient.listByNamePrefix(prefix).collectList().block(); + } + + @Override + public Page listByNamePrefix(String prefix, Pageable pageable) { + return storeClient.listByNamePrefix(prefix, pageable).block(); + } + + @Override + public List listByNames(List names) { + return storeClient.listByNames(names).collectList().block(); } @Override public Optional fetchByName(String name) { - return repository.findById(name).blockOptional(); + return storeClient.fetchByName(name).blockOptional(); } @Override public ExtensionStore create(String name, byte[] data) { - var store = new ExtensionStore(name, data); - return repository.save(store).block(); + return storeClient.create(name, data).block(); } @Override public ExtensionStore update(String name, Long version, byte[] data) { - var store = new ExtensionStore(name, data, version); - return repository.save(store).block(); + return storeClient.update(name, version, data).block(); } @Override public ExtensionStore delete(String name, Long version) { - return repository.findById(name) - .switchIfEmpty(Mono.error(() -> new EntityNotFoundException( - "Extension store with name " + name + " was not found."))) - .flatMap(deleting -> { - deleting.setVersion(version); - return repository.delete(deleting).thenReturn(deleting); - }) - .block(); + return storeClient.delete(name, version).block(); } - } diff --git a/application/src/main/java/run/halo/app/extension/store/ExtensionStoreRepository.java b/application/src/main/java/run/halo/app/extension/store/ExtensionStoreRepository.java index 1fc1d1476bc..ddf42030e8d 100644 --- a/application/src/main/java/run/halo/app/extension/store/ExtensionStoreRepository.java +++ b/application/src/main/java/run/halo/app/extension/store/ExtensionStoreRepository.java @@ -1,8 +1,11 @@ package run.halo.app.extension.store; +import java.util.List; +import org.springframework.data.domain.Pageable; import org.springframework.data.r2dbc.repository.R2dbcRepository; import org.springframework.stereotype.Repository; import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; /** * This repository contains some basic operations on ExtensionStore entity. @@ -20,4 +23,16 @@ public interface ExtensionStoreRepository extends R2dbcRepository findAllByNameStartingWith(String prefix); + Flux findAllByNameStartingWith(String prefix, Pageable pageable); + + Mono countByNameStartingWith(String prefix); + + /** + *

Finds all ExtensionStore by name in, the result no guarantee the same order as the given + * names, so if you want this, please order the result by yourself.

+ * + * @param names names to find + * @return a flux of extension stores + */ + Flux findByNameIn(List names); } diff --git a/application/src/main/java/run/halo/app/extension/store/ReactiveExtensionStoreClient.java b/application/src/main/java/run/halo/app/extension/store/ReactiveExtensionStoreClient.java index 7024e118d6a..ab7ba7b09f3 100644 --- a/application/src/main/java/run/halo/app/extension/store/ReactiveExtensionStoreClient.java +++ b/application/src/main/java/run/halo/app/extension/store/ReactiveExtensionStoreClient.java @@ -1,5 +1,8 @@ package run.halo.app.extension.store; +import java.util.List; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -7,6 +10,16 @@ public interface ReactiveExtensionStoreClient { Flux listByNamePrefix(String prefix); + Mono> listByNamePrefix(String prefix, Pageable pageable); + + /** + * List stores by names and return data according to given names order. + * + * @param names store names to list + * @return a flux of extension stores + */ + Flux listByNames(List names); + Mono fetchByName(String name); Mono create(String name, byte[] data); diff --git a/application/src/main/java/run/halo/app/extension/store/ReactiveExtensionStoreClientImpl.java b/application/src/main/java/run/halo/app/extension/store/ReactiveExtensionStoreClientImpl.java index b8056d291b0..8e7b4ce8a7f 100644 --- a/application/src/main/java/run/halo/app/extension/store/ReactiveExtensionStoreClientImpl.java +++ b/application/src/main/java/run/halo/app/extension/store/ReactiveExtensionStoreClientImpl.java @@ -1,6 +1,12 @@ package run.halo.app.extension.store; +import java.util.Comparator; +import java.util.List; +import java.util.function.ToIntFunction; import org.springframework.dao.DuplicateKeyException; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; import reactor.core.publisher.Flux; @@ -21,6 +27,22 @@ public Flux listByNamePrefix(String prefix) { return repository.findAllByNameStartingWith(prefix); } + @Override + public Mono> listByNamePrefix(String prefix, Pageable pageable) { + return this.repository.findAllByNameStartingWith(prefix, pageable) + .collectList() + .zipWith(this.repository.countByNameStartingWith(prefix)) + .map(p -> new PageImpl<>(p.getT1(), pageable, p.getT2())); + } + + @Override + public Flux listByNames(List names) { + ToIntFunction comparator = + store -> names.indexOf(store.getName()); + return repository.findByNameIn(names) + .sort(Comparator.comparingInt(comparator)); + } + @Override public Mono fetchByName(String name) { return repository.findById(name); diff --git a/application/src/main/java/run/halo/app/plugin/SharedApplicationContextHolder.java b/application/src/main/java/run/halo/app/plugin/SharedApplicationContextHolder.java index d0b0baec800..87ffc39a181 100644 --- a/application/src/main/java/run/halo/app/plugin/SharedApplicationContextHolder.java +++ b/application/src/main/java/run/halo/app/plugin/SharedApplicationContextHolder.java @@ -8,6 +8,7 @@ import run.halo.app.extension.DefaultSchemeManager; import run.halo.app.extension.ExtensionClient; import run.halo.app.extension.ReactiveExtensionClient; +import run.halo.app.extension.index.IndexSpecRegistry; import run.halo.app.infra.BackupRootGetter; import run.halo.app.infra.ExternalLinkProcessor; import run.halo.app.infra.ExternalUrlSupplier; @@ -62,8 +63,10 @@ SharedApplicationContext createSharedApplicationContext() { // register shared object here var extensionClient = rootApplicationContext.getBean(ExtensionClient.class); var reactiveExtensionClient = rootApplicationContext.getBean(ReactiveExtensionClient.class); + var indexSpecRegistry = rootApplicationContext.getBean(IndexSpecRegistry.class); beanFactory.registerSingleton("extensionClient", extensionClient); beanFactory.registerSingleton("reactiveExtensionClient", reactiveExtensionClient); + beanFactory.registerSingleton("indexSpecRegistry", indexSpecRegistry); DefaultSchemeManager defaultSchemeManager = rootApplicationContext.getBean(DefaultSchemeManager.class); diff --git a/application/src/main/resources/config/i18n/messages.properties b/application/src/main/resources/config/i18n/messages.properties index c32006c1b3a..24afe5e82eb 100644 --- a/application/src/main/resources/config/i18n/messages.properties +++ b/application/src/main/resources/config/i18n/messages.properties @@ -40,7 +40,7 @@ problemDetail.run.halo.app.infra.exception.DuplicateNameException=Duplicate name problemDetail.run.halo.app.infra.exception.PluginAlreadyExistsException=Plugin {0} already exists. problemDetail.run.halo.app.infra.exception.RateLimitExceededException=API rate limit exceeded, please try again later. problemDetail.run.halo.app.infra.exception.EmailVerificationFailed=Invalid email verification code. - +problemDetail.index.duplicateKey=The value of {0} already exists for unique index {1}, please rename it and retry. problemDetail.user.email.verify.maxAttempts=Too many verification attempts, please try again later. problemDetail.user.password.unsatisfied=The password does not meet the specifications. problemDetail.user.username.unsatisfied=The username does not meet the specifications. diff --git a/application/src/main/resources/config/i18n/messages_zh.properties b/application/src/main/resources/config/i18n/messages_zh.properties index b51796f0667..38b31efcd85 100644 --- a/application/src/main/resources/config/i18n/messages_zh.properties +++ b/application/src/main/resources/config/i18n/messages_zh.properties @@ -17,7 +17,7 @@ problemDetail.run.halo.app.infra.exception.DuplicateNameException=检测到有 problemDetail.run.halo.app.infra.exception.PluginAlreadyExistsException=插件 {0} 已经存。 problemDetail.run.halo.app.infra.exception.RateLimitExceededException=请求过于频繁，请稍候再试。 problemDetail.run.halo.app.infra.exception.EmailVerificationFailed=验证码错误或已失效。 - +problemDetail.index.duplicateKey=唯一索引 {1} 中的值 {0} 已存在，请更名后重试。 problemDetail.user.email.verify.maxAttempts=尝试次数过多，请稍候再试。 problemDetail.user.password.unsatisfied=密码不符合规范。 problemDetail.user.username.unsatisfied=用户名不符合规范。 diff --git a/application/src/test/java/run/halo/app/config/ExtensionConfigurationTest.java b/application/src/test/java/run/halo/app/config/ExtensionConfigurationTest.java index 4b8e833021b..ee0ad619e1a 100644 --- a/application/src/test/java/run/halo/app/config/ExtensionConfigurationTest.java +++ b/application/src/test/java/run/halo/app/config/ExtensionConfigurationTest.java @@ -22,17 +22,21 @@ import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.http.MediaType; import org.springframework.security.test.context.support.WithMockUser; +import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.web.reactive.server.WebTestClient; import reactor.core.publisher.Flux; import run.halo.app.core.extension.Role; import run.halo.app.core.extension.service.RoleService; import run.halo.app.extension.ExtensionClient; import run.halo.app.extension.FakeExtension; +import run.halo.app.extension.GroupVersionKind; import run.halo.app.extension.Metadata; import run.halo.app.extension.Scheme; import run.halo.app.extension.SchemeManager; +import run.halo.app.extension.index.IndexerFactory; import run.halo.app.extension.store.ExtensionStoreRepository; +@DirtiesContext @SpringBootTest @AutoConfigureWebTestClient class ExtensionConfigurationTest { @@ -66,8 +70,15 @@ void setUp() { } @AfterEach - void cleanUp(@Autowired ExtensionStoreRepository repository) { - repository.deleteAll().subscribe(); + void cleanUp(@Autowired ExtensionStoreRepository repository, + @Autowired IndexerFactory indexerFactory) { + var gvk = Scheme.buildFromType(FakeExtension.class).groupVersionKind(); + if (indexerFactory.contains(gvk)) { + indexerFactory.getIndexer(gvk).removeIndexRecords(descriptor -> true); + } + repository.deleteAll().block(); + schemeManager.fetch(GroupVersionKind.fromExtension(FakeExtension.class)) + .ifPresent(scheme -> schemeManager.unregister(scheme)); } @Test @@ -116,13 +127,17 @@ class AfterCreatingExtension { @BeforeEach void setUp() { - var metadata = new Metadata(); metadata.setName("my-fake"); metadata.setLabels(Map.of("label-key", "label-value")); var fake = new FakeExtension(); fake.setMetadata(metadata); + webClient.get() + .uri("/apis/fake.halo.run/v1alpha1/fakes/{}", metadata.getName()) + .exchange() + .expectStatus().isNotFound(); + createdFake = webClient.post() .uri("/apis/fake.halo.run/v1alpha1/fakes") .contentType(MediaType.APPLICATION_JSON) diff --git a/application/src/test/java/run/halo/app/extension/ReactiveExtensionClientTest.java b/application/src/test/java/run/halo/app/extension/ReactiveExtensionClientTest.java index 0f83fbe0929..e9a20eb951e 100644 --- a/application/src/test/java/run/halo/app/extension/ReactiveExtensionClientTest.java +++ b/application/src/test/java/run/halo/app/extension/ReactiveExtensionClientTest.java @@ -13,6 +13,7 @@ import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -40,6 +41,8 @@ import reactor.core.publisher.Mono; import reactor.test.StepVerifier; import run.halo.app.extension.exception.SchemeNotFoundException; +import run.halo.app.extension.index.Indexer; +import run.halo.app.extension.index.IndexerFactory; import run.halo.app.extension.store.ExtensionStore; import run.halo.app.extension.store.ReactiveExtensionStoreClient; @@ -57,6 +60,9 @@ class ReactiveExtensionClientTest { @Mock SchemeManager schemeManager; + @Mock + IndexerFactory indexerFactory; + @Spy ObjectMapper objectMapper = JsonMapper.builder() .addModule(new JavaTimeModule()) @@ -67,6 +73,7 @@ class ReactiveExtensionClientTest { @BeforeEach void setUp() { + client.setReady(true); lenient().when(schemeManager.get(eq(FakeExtension.class))) .thenReturn(fakeScheme); lenient().when(schemeManager.get(eq(fakeScheme.groupVersionKind()))).thenReturn(fakeScheme); @@ -126,7 +133,8 @@ class UnRegisteredExtension extends AbstractExtension { .thenThrow(SchemeNotFoundException.class); assertThrows(SchemeNotFoundException.class, - () -> client.list(UnRegisteredExtension.class, null, null)); + () -> client.list(UnRegisteredExtension.class, null, + null)); assertThrows(SchemeNotFoundException.class, () -> client.list(UnRegisteredExtension.class, null, null, 0, 10)); assertThrows(SchemeNotFoundException.class, @@ -172,7 +180,8 @@ void shouldReturnExtensionsWithFilterAndSorter() { List.of(createExtensionStore("fake-01"), createExtensionStore("fake-02")))); // without filter and sorter - var fakes = client.list(FakeExtension.class, null, null); + Flux fakes = + client.list(FakeExtension.class, null, null); StepVerifier.create(fakes) .expectNext(fake1) .expectNext(fake2) @@ -327,6 +336,9 @@ void shouldCreateSuccessfully() { Mono.just(createExtensionStore("/registry/fake.halo.run/fakes/fake"))); when(converter.convertFrom(same(FakeExtension.class), any())).thenReturn(fake); + var indexer = mock(Indexer.class); + when(indexerFactory.getIndexer(eq(fake.groupVersionKind()))).thenReturn(indexer); + StepVerifier.create(client.create(fake)) .expectNext(fake) .verifyComplete(); @@ -334,6 +346,7 @@ void shouldCreateSuccessfully() { verify(converter, times(1)).convertTo(eq(fake)); verify(storeClient, times(1)).create(eq("/registry/fake.halo.run/fakes/fake"), any()); assertNotNull(fake.getMetadata().getCreationTimestamp()); + verify(indexer).indexRecord(eq(fake)); } @Test @@ -347,6 +360,9 @@ void shouldCreateWithGenerateNameSuccessfully() { Mono.just(createExtensionStore("/registry/fake.halo.run/fakes/fake"))); when(converter.convertFrom(same(FakeExtension.class), any())).thenReturn(fake); + var indexer = mock(Indexer.class); + when(indexerFactory.getIndexer(eq(fake.groupVersionKind()))).thenReturn(indexer); + StepVerifier.create(client.create(fake)) .expectNext(fake) .verifyComplete(); @@ -357,6 +373,7 @@ void shouldCreateWithGenerateNameSuccessfully() { })); verify(storeClient, times(1)).create(eq("/registry/fake.halo.run/fakes/fake"), any()); assertNotNull(fake.getMetadata().getCreationTimestamp()); + verify(indexer).indexRecord(eq(fake)); } @Test @@ -393,6 +410,9 @@ void shouldCreateUsingUnstructuredSuccessfully() throws JsonProcessingException Mono.just(createExtensionStore("/registry/fake.halo.run/fakes/fake"))); when(converter.convertFrom(same(Unstructured.class), any())).thenReturn(fake); + var indexer = mock(Indexer.class); + when(indexerFactory.getIndexer(eq(fake.groupVersionKind()))).thenReturn(indexer); + StepVerifier.create(client.create(fake)) .expectNext(fake) .verifyComplete(); @@ -400,6 +420,7 @@ void shouldCreateUsingUnstructuredSuccessfully() throws JsonProcessingException verify(converter, times(1)).convertTo(eq(fake)); verify(storeClient, times(1)).create(eq("/registry/fake.halo.run/fakes/fake"), any()); assertNotNull(fake.getMetadata().getCreationTimestamp()); + verify(indexer).indexRecord(eq(fake)); } @Test @@ -423,6 +444,9 @@ void shouldUpdateSuccessfully() { .thenReturn(oldFake) .thenReturn(updatedFake); + var indexer = mock(Indexer.class); + when(indexerFactory.getIndexer(eq(fake.groupVersionKind()))).thenReturn(indexer); + StepVerifier.create(client.update(fake)) .expectNext(updatedFake) .verifyComplete(); @@ -432,6 +456,7 @@ void shouldUpdateSuccessfully() { verify(converter, times(2)).convertFrom(same(FakeExtension.class), any()); verify(storeClient) .update(eq("/registry/fake.halo.run/fakes/fake"), eq(2L), any()); + verify(indexer).updateRecord(eq(updatedFake)); } @Test @@ -475,6 +500,9 @@ void shouldUpdateIfExtensionStatusChangedOnly() { .thenReturn(oldFake) .thenReturn(updatedFake); + var indexer = mock(Indexer.class); + when(indexerFactory.getIndexer(eq(fake.groupVersionKind()))).thenReturn(indexer); + StepVerifier.create(client.update(fake)) .expectNext(updatedFake) .verifyComplete(); @@ -484,6 +512,7 @@ void shouldUpdateIfExtensionStatusChangedOnly() { verify(converter, times(2)).convertFrom(same(FakeExtension.class), any()); verify(storeClient) .update(eq("/registry/fake.halo.run/fakes/fake"), eq(2L), any()); + verify(indexer).updateRecord(eq(updatedFake)); } @Test @@ -506,6 +535,9 @@ void shouldUpdateUnstructuredSuccessfully() throws JsonProcessingException { .thenReturn(oldFake) .thenReturn(updatedFake); + var indexer = mock(Indexer.class); + when(indexerFactory.getIndexer(eq(fake.groupVersionKind()))).thenReturn(indexer); + StepVerifier.create(client.update(fake)) .expectNext(updatedFake) .verifyComplete(); @@ -515,6 +547,7 @@ void shouldUpdateUnstructuredSuccessfully() throws JsonProcessingException { verify(converter, times(2)).convertFrom(same(Unstructured.class), any()); verify(storeClient) .update(eq("/registry/fake.halo.run/fakes/fake"), eq(12345L), any()); + verify(indexer).updateRecord(eq(updatedFake)); } @Test @@ -526,6 +559,9 @@ void shouldDeleteSuccessfully() { Mono.just(createExtensionStore("/registry/fake.halo.run/fakes/fake"))); when(converter.convertFrom(same(FakeExtension.class), any())).thenReturn(fake); + var indexer = mock(Indexer.class); + when(indexerFactory.getIndexer(eq(fake.groupVersionKind()))).thenReturn(indexer); + StepVerifier.create(client.delete(fake)) .expectNext(fake) .verifyComplete(); @@ -533,6 +569,7 @@ void shouldDeleteSuccessfully() { verify(converter, times(1)).convertTo(any()); verify(storeClient, times(1)).update(any(), any(), any()); verify(storeClient, never()).delete(any(), any()); + verify(indexer).updateRecord(eq(fake)); } @Nested @@ -549,10 +586,10 @@ void setUp() { @Test void shouldWatchOnAddSuccessfully() { - doNothing().when(watcher).onAdd(any()); + doNothing().when(watcher).onAdd(isA(Extension.class)); shouldCreateSuccessfully(); - verify(watcher, times(1)).onAdd(any()); + verify(watcher, times(1)).onAdd(isA(Extension.class)); } @Test diff --git a/application/src/test/java/run/halo/app/extension/gc/GcReconcilerTest.java b/application/src/test/java/run/halo/app/extension/gc/GcReconcilerTest.java index 9ba09df37c5..455b5b830f5 100644 --- a/application/src/test/java/run/halo/app/extension/gc/GcReconcilerTest.java +++ b/application/src/test/java/run/halo/app/extension/gc/GcReconcilerTest.java @@ -2,6 +2,8 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -19,6 +21,8 @@ import run.halo.app.extension.FakeExtension; import run.halo.app.extension.Metadata; import run.halo.app.extension.Unstructured; +import run.halo.app.extension.index.Indexer; +import run.halo.app.extension.index.IndexerFactory; import run.halo.app.extension.store.ExtensionStore; import run.halo.app.extension.store.ExtensionStoreClient; @@ -34,6 +38,9 @@ class GcReconcilerTest { @Mock ExtensionConverter converter; + @Mock + IndexerFactory indexerFactory; + @InjectMocks GcReconciler reconciler; @@ -91,10 +98,14 @@ void shouldDeleteCorrectly() { when(converter.convertTo(any())).thenReturn(store); + var indexer = mock(Indexer.class); + when(indexerFactory.getIndexer(any())).thenReturn(indexer); + var result = reconciler.reconcile(createGcRequest()); assertNull(result); verify(converter).convertTo(any()); verify(storeClient).delete("fake-store-name", 1L); + verify(indexer).unIndexRecord(eq(fake.getMetadata().getName())); } GcRequest createGcRequest() { diff --git a/application/src/test/java/run/halo/app/extension/index/DefaultExtensionIteratorTest.java b/application/src/test/java/run/halo/app/extension/index/DefaultExtensionIteratorTest.java new file mode 100644 index 00000000000..87af097f4aa --- /dev/null +++ b/application/src/test/java/run/halo/app/extension/index/DefaultExtensionIteratorTest.java @@ -0,0 +1,85 @@ +package run.halo.app.extension.index; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.NoSuchElementException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Sort; +import run.halo.app.extension.Extension; + +/** + * Tests for {@link DefaultExtensionIterator}. + * + * @author guqing + * @since 2.12.0 + */ +@ExtendWith(MockitoExtension.class) +class DefaultExtensionIteratorTest { + + @Mock + private ExtensionPaginatedLister lister; + + @Test + @SuppressWarnings("unchecked") + void testConstructor_loadsData() { + Page page = mock(Page.class); + when(page.getContent()).thenReturn(List.of(mock(Extension.class))); + when(page.hasNext()).thenReturn(true); + when(page.nextPageable()).thenReturn( + PageRequest.of(1, DefaultExtensionIterator.DEFAULT_PAGE_SIZE, Sort.by("name"))); + when(lister.list(any())).thenReturn(page); + + var iterator = new DefaultExtensionIterator<>(lister); + + assertThat(iterator.hasNext()).isTrue(); + } + + @Test + @SuppressWarnings("unchecked") + void hasNext_whenNextPageExists_loadsNextPage() { + Page page1 = mock(Page.class); + when(page1.getContent()).thenReturn(List.of(mock(Extension.class))); + when(page1.hasNext()).thenReturn(true); + when(page1.nextPageable()).thenReturn( + PageRequest.of(1, DefaultExtensionIterator.DEFAULT_PAGE_SIZE, Sort.by("name"))); + + Page page2 = mock(Page.class); + when(page2.getContent()).thenReturn(List.of(mock(Extension.class))); + when(page2.hasNext()).thenReturn(false); + + when(lister.list(any(Pageable.class))).thenReturn(page1, page2); + + var iterator = new DefaultExtensionIterator<>(lister); + // consume first page + iterator.next(); + + // should load the next page + assertThat(iterator.hasNext()).isTrue(); + } + + @Test + @SuppressWarnings("unchecked") + void next_whenNoNextElement_throwsException() { + Page page = mock(Page.class); + when(page.getContent()).thenReturn(List.of(mock(Extension.class))); + when(page.hasNext()).thenReturn(false); + when(lister.list(any())).thenReturn(page); + + var iterator = new DefaultExtensionIterator<>(lister); + // consume only element + iterator.next(); + + assertThatThrownBy(iterator::next).isInstanceOf(NoSuchElementException.class); + } +} diff --git a/application/src/test/java/run/halo/app/extension/index/DefaultIndexSpecsTest.java b/application/src/test/java/run/halo/app/extension/index/DefaultIndexSpecsTest.java new file mode 100644 index 00000000000..1b12a81cbe0 --- /dev/null +++ b/application/src/test/java/run/halo/app/extension/index/DefaultIndexSpecsTest.java @@ -0,0 +1,75 @@ +package run.halo.app.extension.index; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static run.halo.app.extension.index.PrimaryKeySpecUtils.primaryKeyIndexSpec; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.junit.jupiter.api.Test; +import run.halo.app.extension.AbstractExtension; +import run.halo.app.extension.GVK; + +/** + * Test for {@link DefaultIndexSpecs}. + * + * @author guqing + * @since 2.12.0 + */ +class DefaultIndexSpecsTest { + + @Test + void add() { + var specs = new DefaultIndexSpecs(); + specs.add(primaryKeyIndexSpec(FakeExtension.class)); + assertThat(specs.contains(PrimaryKeySpecUtils.PRIMARY_INDEX_NAME)).isTrue(); + } + + @Test + void addWithException() { + var specs = new DefaultIndexSpecs(); + var nameSpec = new IndexSpec().setName("test"); + assertThatThrownBy(() -> specs.add(nameSpec)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("IndexSpec indexFunc must not be null"); + nameSpec.setIndexFunc(IndexAttributeFactory.simpleAttribute(FakeExtension.class, + e -> e.getMetadata().getName())); + specs.add(nameSpec); + assertThatThrownBy(() -> specs.add(nameSpec)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("IndexSpec with name test already exists"); + } + + @Test + void getIndexSpecs() { + var specs = new DefaultIndexSpecs(); + specs.add(primaryKeyIndexSpec(FakeExtension.class)); + assertThat(specs.getIndexSpecs()).hasSize(1); + } + + @Test + void getIndexSpec() { + var specs = new DefaultIndexSpecs(); + var nameSpec = primaryKeyIndexSpec(FakeExtension.class); + specs.add(nameSpec); + assertThat(specs.getIndexSpec(PrimaryKeySpecUtils.PRIMARY_INDEX_NAME)).isEqualTo(nameSpec); + } + + @Test + void remove() { + var specs = new DefaultIndexSpecs(); + var nameSpec = primaryKeyIndexSpec(FakeExtension.class); + specs.add(nameSpec); + assertThat(specs.contains(PrimaryKeySpecUtils.PRIMARY_INDEX_NAME)).isTrue(); + specs.remove(PrimaryKeySpecUtils.PRIMARY_INDEX_NAME); + assertThat(specs.contains(PrimaryKeySpecUtils.PRIMARY_INDEX_NAME)).isFalse(); + } + + @Data + @EqualsAndHashCode(callSuper = true) + @GVK(group = "test", version = "v1", kind = "FakeExtension", plural = "fakeextensions", + singular = "fakeextension") + static class FakeExtension extends AbstractExtension { + private String email; + } +} diff --git a/application/src/test/java/run/halo/app/extension/index/DefaultIndexerTest.java b/application/src/test/java/run/halo/app/extension/index/DefaultIndexerTest.java new file mode 100644 index 00000000000..ca6ebf133dd --- /dev/null +++ b/application/src/test/java/run/halo/app/extension/index/DefaultIndexerTest.java @@ -0,0 +1,260 @@ +package run.halo.app.extension.index; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.List; +import java.util.Map; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; +import run.halo.app.extension.AbstractExtension; +import run.halo.app.extension.GVK; +import run.halo.app.extension.Metadata; +import run.halo.app.infra.exception.DuplicateNameException; + +/** + * Tests for {@link DefaultIndexer}. + * + * @author guqing + * @since 2.12.0 + */ +@ExtendWith(MockitoExtension.class) +class DefaultIndexerTest { + + @Test + void constructor() { + var spec = getNameIndexSpec(); + var descriptor = new IndexDescriptor(spec); + descriptor.setReady(true); + var indexContainer = new IndexEntryContainer(); + indexContainer.add(new IndexEntryImpl(descriptor)); + new DefaultIndexer(List.of(descriptor), indexContainer); + } + + @Test + void constructorWithException() { + var spec = getNameIndexSpec(); + var descriptor = new IndexDescriptor(spec); + var indexContainer = new IndexEntryContainer(); + assertThatThrownBy(() -> new DefaultIndexer(List.of(descriptor), indexContainer)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Index descriptor is not ready for: metadata.name"); + descriptor.setReady(true); + assertThatThrownBy(() -> new DefaultIndexer(List.of(descriptor), indexContainer)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Index entry not found for: metadata.name"); + } + + @Test + void getObjectKey() { + var fake = createFakeExtension(); + assertThat(DefaultIndexer.getObjectKey(fake)).isEqualTo("fake-extension"); + } + + private static FakeExtension createFakeExtension() { + var fake = new FakeExtension(); + fake.setMetadata(new Metadata()); + fake.getMetadata().setName("fake-extension"); + fake.setEmail("fake-email"); + return fake; + } + + @Test + void indexRecord() { + var nameIndex = getNameIndexSpec(); + var indexContainer = new IndexEntryContainer(); + var descriptor = new IndexDescriptor(nameIndex); + descriptor.setReady(true); + indexContainer.add(new IndexEntryImpl(descriptor)); + + var indexer = new DefaultIndexer(List.of(descriptor), indexContainer); + indexer.indexRecord(createFakeExtension()); + + var iterator = indexer.allIndexesIterator(); + assertThat(iterator.hasNext()).isTrue(); + var indexEntry = iterator.next(); + var entries = indexEntry.entries(); + assertThat(entries).hasSize(1); + assertThat(entries).contains(Map.entry("fake-extension", "fake-extension")); + } + + private static IndexSpec getNameIndexSpec() { + return getIndexSpec("metadata.name", true, + IndexAttributeFactory.simpleAttribute(FakeExtension.class, + e -> e.getMetadata().getName())); + } + + @Test + void indexRecordWithExceptionShouldRollback() { + var indexContainer = new IndexEntryContainer(); + // add email before name + var emailDescriptor = new IndexDescriptor(getIndexSpec("email", false, + IndexAttributeFactory.simpleAttribute(FakeExtension.class, FakeExtension::getEmail))); + emailDescriptor.setReady(true); + var emailIndexEntry = new IndexEntryImpl(emailDescriptor); + indexContainer.add(emailIndexEntry); + + var descriptor = new IndexDescriptor(getNameIndexSpec()); + descriptor.setReady(true); + var nameIndexEntry = new IndexEntryImpl(descriptor); + indexContainer.add(nameIndexEntry); + + var indexer = new DefaultIndexer(List.of(descriptor, emailDescriptor), indexContainer); + + indexer.indexRecord(createFakeExtension()); + assertThat(emailIndexEntry.entries()).hasSize(1); + assertThat(nameIndexEntry.entries()).hasSize(1); + + var fake2 = createFakeExtension(); + fake2.setEmail("email-2"); + + // email applied to entry then name duplicate + assertThatThrownBy(() -> indexer.indexRecord(fake2)) + .isInstanceOf(DuplicateNameException.class) + .hasMessage( + "400 BAD_REQUEST \"The value [fake-extension] is already exists for unique index " + + "[metadata.name].\""); + + // should be rollback email-2 key + assertThat(emailIndexEntry.entries()).hasSize(1); + assertThat(nameIndexEntry.entries()).hasSize(1); + } + + @Test + void updateRecordWithExceptionShouldRollback() { + var indexContainer = new IndexEntryContainer(); + // add email before name + var emailDescriptor = new IndexDescriptor(getIndexSpec("email", false, + IndexAttributeFactory.simpleAttribute(FakeExtension.class, FakeExtension::getEmail))); + emailDescriptor.setReady(true); + var emailIndexEntry = new IndexEntryImpl(emailDescriptor); + indexContainer.add(emailIndexEntry); + + var descriptor = new IndexDescriptor(getNameIndexSpec()); + descriptor.setReady(true); + var nameIndexEntry = new IndexEntryImpl(descriptor); + indexContainer.add(nameIndexEntry); + + var indexer = new DefaultIndexer(List.of(descriptor, emailDescriptor), indexContainer); + + var fakeExtension = createFakeExtension(); + indexer.indexRecord(fakeExtension); + + assertThat(emailIndexEntry.entries()).hasSize(1); + assertThat(emailIndexEntry.entries()).contains(Map.entry("fake-email", "fake-extension")); + assertThat(nameIndexEntry.entries()).hasSize(1); + assertThat(nameIndexEntry.entries()).contains( + Map.entry("fake-extension", "fake-extension")); + + fakeExtension.setEmail("email-2"); + indexer.updateRecord(fakeExtension); + assertThat(emailIndexEntry.entries()).hasSize(1); + assertThat(emailIndexEntry.entries()).contains(Map.entry("email-2", "fake-extension")); + assertThat(nameIndexEntry.entries()).hasSize(1); + assertThat(nameIndexEntry.entries()).contains( + Map.entry("fake-extension", "fake-extension")); + + fakeExtension.getMetadata().setName("fake-extension-2"); + indexer.updateRecord(fakeExtension); + assertThat(emailIndexEntry.entries()) + .containsExactly(Map.entry("email-2", "fake-extension"), + Map.entry("email-2", "fake-extension-2")); + assertThat(nameIndexEntry.entries()) + .containsExactly(Map.entry("fake-extension", "fake-extension"), + Map.entry("fake-extension-2", "fake-extension-2")); + } + + @Test + void findIndexByName() { + var indexContainer = new IndexEntryContainer(); + // add email before name + var emailDescriptor = new IndexDescriptor(getIndexSpec("email", false, + IndexAttributeFactory.simpleAttribute(FakeExtension.class, FakeExtension::getEmail))); + emailDescriptor.setReady(true); + var emailIndexEntry = new IndexEntryImpl(emailDescriptor); + indexContainer.add(emailIndexEntry); + + var descriptor = new IndexDescriptor(getNameIndexSpec()); + descriptor.setReady(true); + var nameIndexEntry = new IndexEntryImpl(descriptor); + indexContainer.add(nameIndexEntry); + + var indexer = new DefaultIndexer(List.of(descriptor, emailDescriptor), indexContainer); + + var foundNameDescriptor = indexer.findIndexByName("metadata.name"); + assertThat(foundNameDescriptor).isNotNull(); + assertThat(foundNameDescriptor).isEqualTo(descriptor); + + var foundEmailDescriptor = indexer.findIndexByName("email"); + assertThat(foundEmailDescriptor).isNotNull(); + assertThat(foundEmailDescriptor).isEqualTo(emailDescriptor); + } + + @Test + void createIndexEntry() { + var nameSpec = getNameIndexSpec(); + var descriptor = new IndexDescriptor(nameSpec); + descriptor.setReady(true); + var indexContainer = new IndexEntryContainer(); + indexContainer.add(new IndexEntryImpl(descriptor)); + var indexer = new DefaultIndexer(List.of(descriptor), indexContainer); + var indexEntry = indexer.createIndexEntry(descriptor); + assertThat(indexEntry).isNotNull(); + } + + @Test + void removeIndexRecord() { + var nameIndex = getNameIndexSpec(); + var indexContainer = new IndexEntryContainer(); + var descriptor = new IndexDescriptor(nameIndex); + descriptor.setReady(true); + var nameIndexEntry = new IndexEntryImpl(descriptor); + indexContainer.add(nameIndexEntry); + + var indexer = new DefaultIndexer(List.of(descriptor), indexContainer); + indexer.indexRecord(createFakeExtension()); + + assertThat(nameIndexEntry.entries()) + .containsExactly(Map.entry("fake-extension", "fake-extension")); + + indexer.removeIndexRecords(d -> true); + assertThat(nameIndexEntry.entries()).isEmpty(); + } + + @Test + void readyIndexesIterator() { + var indexContainer = new IndexEntryContainer(); + var descriptor = new IndexDescriptor(getNameIndexSpec()); + descriptor.setReady(true); + var nameIndexEntry = new IndexEntryImpl(descriptor); + indexContainer.add(nameIndexEntry); + + var indexer = new DefaultIndexer(List.of(descriptor), indexContainer); + + var iterator = indexer.readyIndexesIterator(); + assertThat(iterator.hasNext()).isTrue(); + + descriptor.setReady(false); + iterator = indexer.readyIndexesIterator(); + assertThat(iterator.hasNext()).isFalse(); + } + + private static IndexSpec getIndexSpec(String name, boolean unique, IndexAttribute attribute) { + return new IndexSpec() + .setName(name) + .setOrder(IndexSpec.OrderType.ASC) + .setUnique(unique) + .setIndexFunc(attribute); + } + + @Data + @EqualsAndHashCode(callSuper = true) + @GVK(group = "test", version = "v1", kind = "FakeExtension", plural = "fakeextensions", + singular = "fakeextension") + static class FakeExtension extends AbstractExtension { + private String email; + } +} diff --git a/application/src/test/java/run/halo/app/extension/index/IndexDescriptorTest.java b/application/src/test/java/run/halo/app/extension/index/IndexDescriptorTest.java new file mode 100644 index 00000000000..08c3e4485e1 --- /dev/null +++ b/application/src/test/java/run/halo/app/extension/index/IndexDescriptorTest.java @@ -0,0 +1,40 @@ +package run.halo.app.extension.index; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; +import run.halo.app.extension.FakeExtension; + +/** + * Tests for {@link IndexDescriptor}. + * + * @author guqing + * @since 2.12.0 + */ +class IndexDescriptorTest { + + @Test + void equalsVerifier() { + var spec1 = new IndexSpec() + .setName("metadata.name") + .setOrder(IndexSpec.OrderType.ASC) + .setIndexFunc(IndexAttributeFactory.simpleAttribute(FakeExtension.class, + e -> e.getMetadata().getName()) + ) + .setUnique(true); + + var descriptor = new IndexDescriptor(spec1); + var descriptor2 = new IndexDescriptor(spec1); + assertThat(descriptor).isEqualTo(descriptor2); + + var spec2 = new IndexSpec() + .setName("metadata.name") + .setOrder(IndexSpec.OrderType.DESC) + .setIndexFunc(IndexAttributeFactory.simpleAttribute(FakeExtension.class, + e -> e.getMetadata().getName()) + ) + .setUnique(false); + var descriptor3 = new IndexDescriptor(spec2); + assertThat(descriptor).isEqualTo(descriptor3); + } +} diff --git a/application/src/test/java/run/halo/app/extension/index/IndexEntryContainerTest.java b/application/src/test/java/run/halo/app/extension/index/IndexEntryContainerTest.java new file mode 100644 index 00000000000..05eb534b1b8 --- /dev/null +++ b/application/src/test/java/run/halo/app/extension/index/IndexEntryContainerTest.java @@ -0,0 +1,56 @@ +package run.halo.app.extension.index; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.junit.jupiter.api.Test; +import run.halo.app.extension.AbstractExtension; +import run.halo.app.extension.GVK; + +/** + * Tests for {@link IndexEntryContainer}. + * + * @author guqing + * @since 2.12.0 + */ +class IndexEntryContainerTest { + + @Test + void add() { + IndexEntryContainer indexEntry = new IndexEntryContainer(); + var spec = PrimaryKeySpecUtils.primaryKeyIndexSpec(FakeExtension.class); + var descriptor = new IndexDescriptor(spec); + var entry = new IndexEntryImpl(descriptor); + indexEntry.add(entry); + assertThat(indexEntry.contains(descriptor)).isTrue(); + + assertThatThrownBy(() -> indexEntry.add(entry)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Index entry already exists for " + descriptor); + } + + @Test + void remove() { + IndexEntryContainer indexEntry = new IndexEntryContainer(); + var spec = PrimaryKeySpecUtils.primaryKeyIndexSpec(FakeExtension.class); + var descriptor = new IndexDescriptor(spec); + var entry = new IndexEntryImpl(descriptor); + indexEntry.add(entry); + assertThat(indexEntry.contains(descriptor)).isTrue(); + assertThat(indexEntry.size()).isEqualTo(1); + + indexEntry.remove(descriptor); + assertThat(indexEntry.contains(descriptor)).isFalse(); + assertThat(indexEntry.size()).isEqualTo(0); + } + + @Data + @EqualsAndHashCode(callSuper = true) + @GVK(group = "test", version = "v1", kind = "FakeExtension", plural = "fakes", + singular = "fake") + static class FakeExtension extends AbstractExtension { + + } +} diff --git a/application/src/test/java/run/halo/app/extension/index/IndexEntryImplTest.java b/application/src/test/java/run/halo/app/extension/index/IndexEntryImplTest.java new file mode 100644 index 00000000000..479b9cf313c --- /dev/null +++ b/application/src/test/java/run/halo/app/extension/index/IndexEntryImplTest.java @@ -0,0 +1,108 @@ +package run.halo.app.extension.index; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +/** + * Tests for {@link IndexEntryImpl}. + * + * @author guqing + * @since 2.12.0 + */ +@ExtendWith(MockitoExtension.class) +class IndexEntryImplTest { + + @Test + void add() { + var spec = + PrimaryKeySpecUtils.primaryKeyIndexSpec(IndexEntryContainerTest.FakeExtension.class); + var descriptor = new IndexDescriptor(spec); + var entry = new IndexEntryImpl(descriptor); + entry.addEntry(List.of("slug-1"), "fake-name-1"); + assertThat(entry.indexedKeys()).containsExactly("slug-1"); + } + + @Test + void remove() { + var spec = + PrimaryKeySpecUtils.primaryKeyIndexSpec(IndexEntryContainerTest.FakeExtension.class); + var descriptor = new IndexDescriptor(spec); + var entry = new IndexEntryImpl(descriptor); + entry.addEntry(List.of("slug-1"), "fake-name-1"); + assertThat(entry.indexedKeys()).containsExactly("slug-1"); + assertThat(entry.entries()).hasSize(1); + + entry.removeEntry("slug-1", "fake-name-1"); + assertThat(entry.indexedKeys()).isEmpty(); + assertThat(entry.entries()).isEmpty(); + } + + @Test + void removeByIndex() { + var spec = + PrimaryKeySpecUtils.primaryKeyIndexSpec(IndexEntryContainerTest.FakeExtension.class); + var descriptor = new IndexDescriptor(spec); + var entry = new IndexEntryImpl(descriptor); + entry.addEntry(List.of("slug-1", "slug-2"), "fake-name-1"); + assertThat(entry.indexedKeys()).containsExactly("slug-1", "slug-2"); + assertThat(entry.entries()).hasSize(2); + + entry.remove("fake-name-1"); + assertThat(entry.indexedKeys()).isEmpty(); + assertThat(entry.entries()).isEmpty(); + } + + @Test + void getByIndexKey() { + var spec = + PrimaryKeySpecUtils.primaryKeyIndexSpec(IndexEntryContainerTest.FakeExtension.class); + var descriptor = new IndexDescriptor(spec); + var entry = new IndexEntryImpl(descriptor); + entry.addEntry(List.of("slug-1", "slug-2"), "fake-name-1"); + assertThat(entry.indexedKeys()).containsExactly("slug-1", "slug-2"); + assertThat(entry.entries()).hasSize(2); + + assertThat(entry.getByIndexKey("slug-1")).isEqualTo(List.of("fake-name-1")); + } + + @Test + void keyOrder() { + var spec = + PrimaryKeySpecUtils.primaryKeyIndexSpec(IndexEntryContainerTest.FakeExtension.class); + spec.setOrder(IndexSpec.OrderType.DESC); + var descriptor = new IndexDescriptor(spec); + var entry = new IndexEntryImpl(descriptor); + entry.addEntry(List.of("slug-1", "slug-2"), "fake-name-1"); + entry.addEntry(List.of("slug-3"), "fake-name-2"); + entry.addEntry(List.of("slug-4"), "fake-name-3"); + entry.addEntry(List.of("slug-5"), "fake-name-1"); + assertThat(entry.entries()) + .containsSequence( + Map.entry("slug-5", "fake-name-1"), + Map.entry("slug-4", "fake-name-3"), + Map.entry("slug-3", "fake-name-2"), + Map.entry("slug-2", "fake-name-1"), + Map.entry("slug-1", "fake-name-1")); + + assertThat(entry.indexedKeys()).containsSequence("slug-4", "slug-3", "slug-2", "slug-1"); + + + spec.setOrder(IndexSpec.OrderType.ASC); + var descriptor2 = new IndexDescriptor(spec); + var entry2 = new IndexEntryImpl(descriptor2); + entry2.addEntry(List.of("slug-1", "slug-2"), "fake-name-1"); + entry2.addEntry(List.of("slug-3"), "fake-name-2"); + entry2.addEntry(List.of("slug-4"), "fake-name-3"); + assertThat(entry2.entries()) + .containsSequence(Map.entry("slug-1", "fake-name-1"), + Map.entry("slug-2", "fake-name-1"), + Map.entry("slug-3", "fake-name-2"), + Map.entry("slug-4", "fake-name-3")); + assertThat(entry2.indexedKeys()).containsSequence("slug-1", "slug-2", "slug-3", "slug-4"); + } +} diff --git a/application/src/test/java/run/halo/app/extension/index/IndexSpecRegistryImplTest.java b/application/src/test/java/run/halo/app/extension/index/IndexSpecRegistryImplTest.java new file mode 100644 index 00000000000..12dc4d84def --- /dev/null +++ b/application/src/test/java/run/halo/app/extension/index/IndexSpecRegistryImplTest.java @@ -0,0 +1,88 @@ +package run.halo.app.extension.index; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + +import java.util.Set; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import run.halo.app.extension.AbstractExtension; +import run.halo.app.extension.GVK; +import run.halo.app.extension.Scheme; +import run.halo.app.extension.SchemeManager; + +/** + * Tests for {@link IndexSpecRegistryImpl}. + * + * @author guqing + * @since 2.12.0 + */ +@ExtendWith(MockitoExtension.class) +class IndexSpecRegistryImplTest { + @Mock + private SchemeManager schemeManager; + + @InjectMocks + private IndexSpecRegistryImpl indexSpecRegistry; + + private Scheme scheme; + + @AfterEach + void tearDown() { + indexSpecRegistry.removeIndexSpecs(FakeExtension.class); + } + + @BeforeEach + void setUp() { + this.scheme = Scheme.buildFromType(FakeExtension.class); + when(schemeManager.get(eq(FakeExtension.class))) + .thenReturn(scheme); + } + + @Test + void indexFor() { + var specs = indexSpecRegistry.indexFor(FakeExtension.class); + assertThat(specs).isNotNull(); + assertThat(specs.getIndexSpecs()).hasSize(4); + } + + @Test + void contains() { + indexSpecRegistry.indexFor(FakeExtension.class); + assertThat(indexSpecRegistry.contains(FakeExtension.class)).isTrue(); + } + + @Test + void getKeySpace() { + var keySpace = indexSpecRegistry.getKeySpace(scheme); + assertThat(keySpace).isEqualTo("/registry/test.halo.run/fakes"); + } + + @Test + void getIndexSpecs() { + assertThatThrownBy(() -> indexSpecRegistry.getIndexSpecs(FakeExtension.class)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("No index specs found for extension type: "); + + indexSpecRegistry.indexFor(FakeExtension.class); + var specs = indexSpecRegistry.getIndexSpecs(FakeExtension.class); + assertThat(specs).isNotNull(); + } + + @Data + @EqualsAndHashCode(callSuper = true) + @GVK(group = "test.halo.run", version = "v1", kind = "FakeExtension", plural = "fakes", + singular = "fake") + static class FakeExtension extends AbstractExtension { + Set tags; + } +} diff --git a/application/src/test/java/run/halo/app/extension/index/IndexedQueryEngineImplTest.java b/application/src/test/java/run/halo/app/extension/index/IndexedQueryEngineImplTest.java new file mode 100644 index 00000000000..20bc8f99de9 --- /dev/null +++ b/application/src/test/java/run/halo/app/extension/index/IndexedQueryEngineImplTest.java @@ -0,0 +1,485 @@ +package run.halo.app.extension.index; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.data.domain.Sort; +import run.halo.app.extension.AbstractExtension; +import run.halo.app.extension.GVK; +import run.halo.app.extension.GroupVersionKind; +import run.halo.app.extension.ListOptions; +import run.halo.app.extension.PageRequest; +import run.halo.app.extension.router.selector.EqualityMatcher; +import run.halo.app.extension.router.selector.FieldSelector; +import run.halo.app.extension.router.selector.LabelSelector; +import run.halo.app.extension.router.selector.SelectorMatcher; + +/** + * Tests for {@link IndexedQueryEngineImpl}. + * + * @author guqing + * @since 2.12.0 + */ +@ExtendWith(MockitoExtension.class) +class IndexedQueryEngineImplTest { + + @Mock + private IndexerFactory indexerFactory; + + @InjectMocks + private IndexedQueryEngineImpl indexedQueryEngine; + + @Test + void getIndexEntry() { + Map indexMap = new HashMap<>(); + assertThatThrownBy(() -> IndexedQueryEngineImpl.getIndexEntry("field1", indexMap)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "No index found for fieldPath: field1, make sure you have created an index for " + + "this field."); + } + + @Test + void retrieve() { + var spyIndexedQueryEngine = spy(indexedQueryEngine); + doReturn(List.of("object1", "object2", "object3")).when(spyIndexedQueryEngine) + .doRetrieve(any(), any(), eq(Sort.unsorted())); + + var gvk = GroupVersionKind.fromExtension(DemoExtension.class); + + when(indexerFactory.getIndexer(eq(gvk))).thenReturn(mock(Indexer.class)); + + var pageRequest = mock(PageRequest.class); + when(pageRequest.getPageNumber()).thenReturn(1); + when(pageRequest.getPageSize()).thenReturn(2); + when(pageRequest.getSort()).thenReturn(Sort.unsorted()); + + var result = spyIndexedQueryEngine.retrieve(gvk, new ListOptions(), pageRequest); + assertThat(result.getItems()).containsExactly("object1", "object2"); + assertThat(result.getTotal()).isEqualTo(3); + + verify(spyIndexedQueryEngine).doRetrieve(any(), any(), eq(Sort.unsorted())); + verify(indexerFactory).getIndexer(eq(gvk)); + verify(pageRequest, times(2)).getPageNumber(); + verify(pageRequest, times(2)).getPageSize(); + verify(pageRequest).getSort(); + } + + @Test + void retrieveAll() { + var spyIndexedQueryEngine = spy(indexedQueryEngine); + doReturn(List.of()).when(spyIndexedQueryEngine) + .doRetrieve(any(), any(), eq(Sort.unsorted())); + + var gvk = GroupVersionKind.fromExtension(DemoExtension.class); + + when(indexerFactory.getIndexer(eq(gvk))).thenReturn(mock(Indexer.class)); + + var result = spyIndexedQueryEngine.retrieveAll(gvk, new ListOptions()); + assertThat(result).isEmpty(); + + verify(spyIndexedQueryEngine).doRetrieve(any(), any(), eq(Sort.unsorted())); + verify(indexerFactory).getIndexer(eq(gvk)); + } + + @Test + void doRetrieve() { + var indexer = mock(Indexer.class); + var labelEntry = mock(IndexEntry.class); + var fieldSlugEntry = mock(IndexEntry.class); + var nameEntry = mock(IndexEntry.class); + when(indexer.readyIndexesIterator()).thenReturn( + List.of(labelEntry, nameEntry, fieldSlugEntry).iterator()); + + when(nameEntry.getIndexDescriptor()) + .thenReturn(new IndexDescriptor( + PrimaryKeySpecUtils.primaryKeyIndexSpec(DemoExtension.class))); + when(nameEntry.indexedKeys()).thenReturn(Set.of("object1", "object2", "object3")); + + when(fieldSlugEntry.getIndexDescriptor()) + .thenReturn(new IndexDescriptor(new IndexSpec() + .setName("slug") + .setOrder(IndexSpec.OrderType.ASC))); + when(fieldSlugEntry.indexedKeys()).thenReturn(Set.of("slug1", "slug2", "slug3")); + when(fieldSlugEntry.getByIndexKey(eq("slug1"))).thenReturn(List.of("object1")); + + when(labelEntry.getIndexDescriptor()) + .thenReturn( + new IndexDescriptor(LabelIndexSpecUtils.labelIndexSpec(DemoExtension.class))); + when(labelEntry.entries()).thenReturn(List.of( + Map.entry("key1=value1", "object1"), + Map.entry("key2=value2", "object1"), + Map.entry("key1=value1", "object2"), + Map.entry("key2=value2", "object2"), + Map.entry("key1=value1", "object3") + )); + var listOptions = new ListOptions(); + listOptions.setLabelSelector(LabelSelector.builder() + .eq("key1", "value1").build()); + listOptions.setFieldSelector(FieldSelector.builder() + .eq("slug", "slug1").build()); + var result = indexedQueryEngine.doRetrieve(indexer, listOptions, Sort.unsorted()); + assertThat(result).containsExactly("object1"); + } + + @Test + void intersection() { + var list1 = Arrays.asList(1, 2, 3, 4); + var list2 = Arrays.asList(3, 4, 5, 6); + var expected = Arrays.asList(3, 4); + assertThat(IndexedQueryEngineImpl.intersection(list1, list2)).isEqualTo(expected); + + list1 = Arrays.asList(1, 2, 3); + list2 = Arrays.asList(4, 5, 6); + expected = List.of(); + assertThat(IndexedQueryEngineImpl.intersection(list1, list2)).isEqualTo(expected); + + list1 = List.of(); + list2 = Arrays.asList(1, 2, 3); + expected = List.of(); + assertThat(IndexedQueryEngineImpl.intersection(list1, list2)).isEqualTo(expected); + + list1 = Arrays.asList(1, 2, 3); + list2 = List.of(); + expected = List.of(); + assertThat(IndexedQueryEngineImpl.intersection(list1, list2)).isEqualTo(expected); + + list1 = List.of(); + list2 = List.of(); + expected = List.of(); + assertThat(IndexedQueryEngineImpl.intersection(list1, list2)).isEqualTo(expected); + + list1 = Arrays.asList(1, 2, 2, 3); + list2 = Arrays.asList(2, 3, 3, 4); + expected = Arrays.asList(2, 3); + assertThat(IndexedQueryEngineImpl.intersection(list1, list2)).isEqualTo(expected); + } + + @Nested + @ExtendWith(MockitoExtension.class) + class LabelMatcherTest { + @InjectMocks + private IndexedQueryEngineImpl indexedQueryEngine; + + @Test + void testRetrieveForLabelMatchers() { + // Setup mocks + IndexEntry indexEntryMock = mock(IndexEntry.class); + Map fieldPathEntryMap = + Map.of(LabelIndexSpecUtils.LABEL_PATH, indexEntryMock); + List allMetadataNames = Arrays.asList("object1", "object2", "object3"); + + // Setup mock behavior + when(indexEntryMock.entries()) + .thenReturn(List.of(Map.entry("key1=value1", "object1"), + Map.entry("key2=value2", "object1"), + Map.entry("key1=value1", "object2"), + Map.entry("key2=value2", "object2"), + Map.entry("key1=value1", "object3"))); + + var matcher1 = EqualityMatcher.equal("key1", "value1"); + var matcher2 = EqualityMatcher.equal("key2", "value2"); + + List labelMatchers = Arrays.asList(matcher1, matcher2); + + // Expected results + List expected = Arrays.asList("object1", "object2"); + + // Test + assertThat(indexedQueryEngine.retrieveForLabelMatchers(labelMatchers, fieldPathEntryMap, + allMetadataNames)) + .isEqualTo(expected); + } + + @Test + void testRetrieveForLabelMatchersNoMatch() { + // Setup mocks + IndexEntry indexEntryMock = mock(IndexEntry.class); + Map fieldPathEntryMap = + Map.of(LabelIndexSpecUtils.LABEL_PATH, indexEntryMock); + List allMetadataNames = Arrays.asList("object1", "object2", "object3"); + + // Setup mock behavior + when(indexEntryMock.entries()) + .thenReturn(List.of(Map.entry("key1=value1", "object1"), + Map.entry("key2=value2", "object2"), + Map.entry("key1=value3", "object3")) + ); + + var matcher1 = EqualityMatcher.equal("key3", "value3"); + List labelMatchers = List.of(matcher1); + + // Expected results + List expected = List.of(); + + // Test + assertThat(indexedQueryEngine.retrieveForLabelMatchers(labelMatchers, fieldPathEntryMap, + allMetadataNames)) + .isEqualTo(expected); + } + } + + @Data + @EqualsAndHashCode(callSuper = true) + @GVK(group = "test", version = "v1", kind = "demo", plural = "demos", singular = "demo") + static class DemoExtension extends AbstractExtension { + + } + + @Nested + @ExtendWith(MockitoExtension.class) + class ResultSorterTest { + + @Test + void testSortByUnsorted() { + var fieldPathEntryMap = Map.of(); + List list = new ArrayList<>(); + var sorter = new IndexedQueryEngineImpl.ResultSorter(fieldPathEntryMap, list); + Sort sort = Sort.unsorted(); + list.add("Item1"); + list.add("Item2"); + + List sortedList = sorter.sortBy(sort); + + assertThat(sortedList).containsExactly("Item1", "Item2"); + } + + @Test + void testSortBySortedAscending() { + final var fieldPathEntryMap = new HashMap(); + var entry = mock(IndexEntry.class); + when(entry.entries()).thenReturn(List.of( + Map.entry("key2", "Item2"), + Map.entry("key1", "Item1") + )); + lenient().when(entry.getIndexDescriptor()) + .thenReturn(new IndexDescriptor(new IndexSpec() + .setName("field1") + .setOrder(IndexSpec.OrderType.DESC))); + fieldPathEntryMap.put("field1", entry); + + var list = new ArrayList<>(Arrays.asList("Item1", "Item2")); + var sorter = new IndexedQueryEngineImpl.ResultSorter(fieldPathEntryMap, list); + var sort = Sort.by(Sort.Order.asc("field1")); + List sortedList = sorter.sortBy(sort); + + assertThat(sortedList).containsExactly("Item1", "Item2"); + } + + @Test + void testSortBySortedDescending() { + final var fieldPathEntryMap = new HashMap(); + var entry = mock(IndexEntry.class); + when(entry.entries()).thenReturn(List.of( + Map.entry("key1", "Item1"), + Map.entry("key2", "Item2") + )); + var indexDescriptor = new IndexDescriptor(new IndexSpec() + .setName("field1") + .setOrder(IndexSpec.OrderType.ASC)); + when(entry.getIndexDescriptor()).thenReturn(indexDescriptor); + fieldPathEntryMap.put("field1", entry); + + var list = new ArrayList(); + list.add("Item1"); + list.add("Item2"); + Sort sort = Sort.by(Sort.Order.desc("field1")); + + var sorter = new IndexedQueryEngineImpl.ResultSorter(fieldPathEntryMap, list); + List sortedList = sorter.sortBy(sort); + + assertThat(sortedList).containsSequence("Item2", "Item1"); + } + + @Test + void testSortByMultipleFields() { + final var fieldPathEntryMap = new LinkedHashMap(); + + var entry1 = mock(IndexEntry.class); + when(entry1.entries()).thenReturn(List.of( + Map.entry("k3", "Item3"), + Map.entry("k2", "Item2") + )); + when(entry1.getIndexDescriptor()).thenReturn(new IndexDescriptor(new IndexSpec() + .setName("field1") + .setOrder(IndexSpec.OrderType.DESC))); + + var entry2 = mock(IndexEntry.class); + lenient().when(entry2.entries()).thenReturn(List.of( + Map.entry("k1", "Item1"), + Map.entry("k3", "Item3") + )); + lenient().when(entry2.getIndexDescriptor()) + .thenReturn(new IndexDescriptor(new IndexSpec() + .setName("field2") + .setOrder(IndexSpec.OrderType.ASC))); + + fieldPathEntryMap.put("field1", entry1); + fieldPathEntryMap.put("field2", entry2); + + final Sort sort = Sort.by(Sort.Order.asc("field1"), + Sort.Order.desc("field2")); + + var list = new ArrayList<>(Arrays.asList("Item1", "Item2", "Item3")); + var sorter = new IndexedQueryEngineImpl.ResultSorter(fieldPathEntryMap, list); + List sortedList = sorter.sortBy(sort); + + assertThat(sortedList).containsSequence("Item2", "Item3", "Item1"); + } + + @Test + void testSortByMultipleFields2() { + final var fieldPathEntryMap = new LinkedHashMap(); + + var entry1 = mock(IndexEntry.class); + when(entry1.entries()).thenReturn(List.of( + Map.entry("John", "John"), + Map.entry("Bob", "Bob"), + Map.entry("Alice", "Alice") + )); + lenient().when(entry1.getIndexDescriptor()) + .thenReturn(new IndexDescriptor(new IndexSpec() + .setName("field1") + .setOrder(IndexSpec.OrderType.DESC))); + + var entry2 = mock(IndexEntry.class); + when(entry2.entries()).thenReturn(List.of( + Map.entry("David", "David"), + Map.entry("Eva", "Eva"), + Map.entry("Frank", "Frank") + )); + lenient().when(entry2.getIndexDescriptor()) + .thenReturn(new IndexDescriptor(new IndexSpec() + .setName("field2") + .setOrder(IndexSpec.OrderType.ASC))); + + var entry3 = mock(IndexEntry.class); + lenient().when(entry3.entries()).thenReturn(List.of( + Map.entry("George", "George"), + Map.entry("Helen", "Helen"), + Map.entry("Ivy", "Ivy") + )); + lenient().when(entry3.getIndexDescriptor()) + .thenReturn(new IndexDescriptor(new IndexSpec() + .setName("field3") + .setOrder(IndexSpec.OrderType.ASC))); + + fieldPathEntryMap.put("field1", entry1); + fieldPathEntryMap.put("field2", entry2); + fieldPathEntryMap.put("field3", entry3); + + var list = new ArrayList<>( + Arrays.asList("Alice", "Bob", "Ivy", "Eva", "George")); + final Sort sort = Sort.by(Sort.Order.desc("field1"), + Sort.Order.asc("field2"), + Sort.Order.asc("field3")); + + var sorter = new IndexedQueryEngineImpl.ResultSorter(fieldPathEntryMap, list); + List sortedList = sorter.sortBy(sort); + + assertThat(sortedList).containsSequence("Bob", "Alice", "Eva", "George", "Ivy"); + } + + @Test + void testSortByWithMissingFieldInMap() { + var fieldPathEntryMap = new LinkedHashMap(); + var list = new ArrayList<>(Arrays.asList("Item1", "Item2")); + var sorter = new IndexedQueryEngineImpl.ResultSorter(fieldPathEntryMap, list); + + Sort sort = Sort.by(Sort.Order.asc("missingField")); + assertThatThrownBy(() -> sorter.sortBy(sort)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "No index found for fieldPath: missingField, make sure you have created an " + + "index for this field."); + } + + @Test + void testSortByWithEmptyMap() { + var fieldPathEntryMap = new LinkedHashMap(); + var entry = mock(IndexEntry.class); + when(entry.entries()).thenReturn(List.of()); + lenient().when(entry.getIndexDescriptor()) + .thenReturn(new IndexDescriptor(new IndexSpec() + .setName("field") + .setOrder(IndexSpec.OrderType.DESC))); + fieldPathEntryMap.put("field", entry); + + var list = new ArrayList<>(Arrays.asList("Item1", "Item2")); + var sorter = new IndexedQueryEngineImpl.ResultSorter(fieldPathEntryMap, list); + Sort sort = Sort.by(Sort.Order.asc("field")); + + List sortedList = sorter.sortBy(sort); + assertThat(sortedList).isEmpty(); + } + + @Test + void testSortByWithEmptyList() { + var fieldPathEntryMap = new LinkedHashMap(); + var entry = mock(IndexEntry.class); + when(entry.entries()).thenReturn(List.of( + Map.entry("John", "John"), + Map.entry("Bob", "Bob") + )); + lenient().when(entry.getIndexDescriptor()) + .thenReturn(new IndexDescriptor(new IndexSpec() + .setName("field") + .setOrder(IndexSpec.OrderType.DESC))); + fieldPathEntryMap.put("field", entry); + + var sorter = new IndexedQueryEngineImpl.ResultSorter(fieldPathEntryMap, List.of()); + Sort sort = Sort.by(Sort.Order.asc("field")); + + List sortedList = sorter.sortBy(sort); + assertThat(sortedList).isEmpty(); + } + + @Test + void testSortByWithItemNotInIndex() { + var fieldPathEntryMap = new LinkedHashMap(); + var entry = mock(IndexEntry.class); + when(entry.entries()).thenReturn(List.of( + Map.entry("Item2", "Item2"), + Map.entry("Item1", "Item1") + )); + lenient().when(entry.getIndexDescriptor()) + .thenReturn(new IndexDescriptor(new IndexSpec() + .setName("field") + .setOrder(IndexSpec.OrderType.DESC))); + + fieldPathEntryMap.put("field", entry); + + // Item3 is not in the index + var list = new ArrayList<>(Arrays.asList("Item1", "Item3")); + var sorter = new IndexedQueryEngineImpl.ResultSorter(fieldPathEntryMap, list); + Sort sort = Sort.by(Sort.Order.asc("field")); + + List sortedList = sorter.sortBy(sort); + assertThat(sortedList).containsExactly("Item1"); + } + } +} \ No newline at end of file diff --git a/application/src/test/java/run/halo/app/extension/index/IndexerFactoryImplTest.java b/application/src/test/java/run/halo/app/extension/index/IndexerFactoryImplTest.java new file mode 100644 index 00000000000..6d496fb0c61 --- /dev/null +++ b/application/src/test/java/run/halo/app/extension/index/IndexerFactoryImplTest.java @@ -0,0 +1,93 @@ +package run.halo.app.extension.index; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Optional; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import run.halo.app.extension.AbstractExtension; +import run.halo.app.extension.GVK; +import run.halo.app.extension.GroupVersionKind; +import run.halo.app.extension.Scheme; +import run.halo.app.extension.SchemeManager; + +/** + * Tests for {@link IndexerFactoryImpl}. + * + * @author guqing + * @since 2.12.0 + */ +@ExtendWith(MockitoExtension.class) +class IndexerFactoryImplTest { + @Mock + private SchemeManager schemeManager; + @Mock + private IndexSpecRegistry indexSpecRegistry; + + @InjectMocks + IndexerFactoryImpl indexerFactory; + + @Test + @SuppressWarnings("unchecked") + void indexFactory() { + var scheme = Scheme.buildFromType(DemoExtension.class); + when(schemeManager.get(eq(DemoExtension.class))) + .thenReturn(scheme); + when(indexSpecRegistry.getKeySpace(scheme)) + .thenReturn("/registry/test/demoextensions"); + when(indexSpecRegistry.contains(eq(DemoExtension.class))) + .thenReturn(false); + var specs = mock(IndexSpecs.class); + when(indexSpecRegistry.getIndexSpecs(eq(DemoExtension.class))) + .thenReturn(specs); + when(specs.getIndexSpecs()) + .thenReturn(List.of(PrimaryKeySpecUtils.primaryKeyIndexSpec(DemoExtension.class))); + ExtensionIterator iterator = mock(ExtensionIterator.class); + when(iterator.hasNext()).thenReturn(false); + // create indexer + var indexer = indexerFactory.createIndexerFor(DemoExtension.class, iterator); + assertThat(indexer).isNotNull(); + + when(schemeManager.fetch(eq(scheme.groupVersionKind()))).thenReturn(Optional.of(scheme)); + when(schemeManager.get(eq(scheme.groupVersionKind()))).thenReturn(scheme); + // contains indexer + var hasIndexer = indexerFactory.contains(scheme.groupVersionKind()); + assertThat(hasIndexer).isTrue(); + + assertThat(indexerFactory.contains( + new GroupVersionKind("test", "v1", "Post"))).isFalse(); + + // get indexer + var foundIndexer = indexerFactory.getIndexer(scheme.groupVersionKind()); + assertThat(foundIndexer).isEqualTo(indexer); + + // remove indexer + indexerFactory.removeIndexer(scheme); + assertThat(indexerFactory.contains(scheme.groupVersionKind())).isFalse(); + + // verify + verify(indexSpecRegistry).indexFor(eq(DemoExtension.class)); + verify(schemeManager).get(eq(DemoExtension.class)); + verify(indexSpecRegistry, times(4)).getKeySpace(eq(scheme)); + verify(indexSpecRegistry).contains(eq(DemoExtension.class)); + } + + @Data + @EqualsAndHashCode(callSuper = true) + @GVK(group = "test", version = "v1", kind = "DemoExtension", plural = "demoextensions", + singular = "demoextension") + static class DemoExtension extends AbstractExtension { + private String email; + } +} diff --git a/application/src/test/java/run/halo/app/extension/index/LabelIndexSpecUtilsTest.java b/application/src/test/java/run/halo/app/extension/index/LabelIndexSpecUtilsTest.java new file mode 100644 index 00000000000..529858b856a --- /dev/null +++ b/application/src/test/java/run/halo/app/extension/index/LabelIndexSpecUtilsTest.java @@ -0,0 +1,48 @@ +package run.halo.app.extension.index; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.Map; +import org.junit.jupiter.api.Test; +import run.halo.app.extension.AbstractExtension; +import run.halo.app.extension.Metadata; + +/** + * Tests for {@link LabelIndexSpecUtils}. + * + * @author guqing + * @since 2.12.0 + */ +class LabelIndexSpecUtilsTest { + + @Test + void labelKeyValuePair() { + var pair = LabelIndexSpecUtils.labelKeyValuePair("key=value"); + assertThat(pair.getFirst()).isEqualTo("key"); + assertThat(pair.getSecond()).isEqualTo("value"); + + pair = LabelIndexSpecUtils.labelKeyValuePair("key=value=1"); + assertThat(pair.getFirst()).isEqualTo("key"); + assertThat(pair.getSecond()).isEqualTo("value=1"); + + assertThatThrownBy(() -> LabelIndexSpecUtils.labelKeyValuePair("key")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid label key-value pair: key"); + } + + @Test + void labelIndexValueFunc() { + var ext = new TestExtension(); + ext.setMetadata(new Metadata()); + assertThat(LabelIndexSpecUtils.labelIndexValueFunc(ext)).isEmpty(); + + ext.getMetadata().setLabels(Map.of("key", "value", "key1", "value1")); + assertThat(LabelIndexSpecUtils.labelIndexValueFunc(ext)).containsExactlyInAnyOrder( + "key=value", "key1=value1"); + } + + static class TestExtension extends AbstractExtension { + + } +} diff --git a/application/src/test/java/run/halo/app/extension/index/PrimaryKeySpecUtilsTest.java b/application/src/test/java/run/halo/app/extension/index/PrimaryKeySpecUtilsTest.java new file mode 100644 index 00000000000..d182b3a45cf --- /dev/null +++ b/application/src/test/java/run/halo/app/extension/index/PrimaryKeySpecUtilsTest.java @@ -0,0 +1,46 @@ +package run.halo.app.extension.index; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Set; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.junit.jupiter.api.Test; +import run.halo.app.extension.AbstractExtension; +import run.halo.app.extension.GVK; +import run.halo.app.extension.Metadata; + +/** + * Tests for {@link PrimaryKeySpecUtils}. + * + * @author guqing + * @since 2.12.0 + */ +class PrimaryKeySpecUtilsTest { + + @Test + void primaryKeyIndexSpec() { + var spec = + PrimaryKeySpecUtils.primaryKeyIndexSpec(FakeExtension.class); + assertThat(spec.getName()).isEqualTo("metadata.name"); + assertThat(spec.getOrder()).isEqualTo(IndexSpec.OrderType.ASC); + assertThat(spec.isUnique()).isTrue(); + assertThat(spec.getIndexFunc()).isNotNull(); + assertThat(spec.getIndexFunc().getObjectType()).isEqualTo(FakeExtension.class); + + var extension = new FakeExtension(); + extension.setMetadata(new Metadata()); + extension.getMetadata().setName("fake-name-1"); + + assertThat(spec.getIndexFunc().getValues(extension)) + .isEqualTo(Set.of("fake-name-1")); + } + + @Data + @EqualsAndHashCode(callSuper = true) + @GVK(group = "test", version = "v1", kind = "FakeExtension", plural = "fakes", + singular = "fake") + static class FakeExtension extends AbstractExtension { + + } +} diff --git a/application/src/test/java/run/halo/app/extension/router/ExtensionListHandlerTest.java b/application/src/test/java/run/halo/app/extension/router/ExtensionListHandlerTest.java index 8e5e5e2c25f..12014a33b3f 100644 --- a/application/src/test/java/run/halo/app/extension/router/ExtensionListHandlerTest.java +++ b/application/src/test/java/run/halo/app/extension/router/ExtensionListHandlerTest.java @@ -3,15 +3,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.util.List; -import java.util.Objects; -import java.util.stream.Stream; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -25,6 +21,7 @@ import reactor.core.publisher.Mono; import reactor.test.StepVerifier; import run.halo.app.extension.FakeExtension; +import run.halo.app.extension.ListOptions; import run.halo.app.extension.ListResult; import run.halo.app.extension.ReactiveExtensionClient; import run.halo.app.extension.Scheme; @@ -53,7 +50,7 @@ void shouldHandleCorrectly() { final var fake01 = FakeExtension.createFake("fake01"); final var fake02 = FakeExtension.createFake("fake02"); var fakeListResult = new ListResult<>(0, 0, 2, List.of(fake01, fake02)); - when(client.list(same(FakeExtension.class), any(), any(), anyInt(), anyInt())) + when(client.listBy(same(FakeExtension.class), any(ListOptions.class), any())) .thenReturn(Mono.just(fakeListResult)); var responseMono = listHandler.handle(serverRequest); @@ -66,10 +63,7 @@ void shouldHandleCorrectly() { assertEquals(fakeListResult, ((EntityResponse) response).entity()); }) .verifyComplete(); - verify(client).list(same(FakeExtension.class), any(), argThat(comp -> { - var sortedFakes = Stream.of(fake01, fake02).sorted(comp).toList(); - return Objects.equals(List.of(fake02, fake01), sortedFakes); - }), anyInt(), anyInt()); + verify(client).listBy(same(FakeExtension.class), any(ListOptions.class), any()); } } diff --git a/application/src/test/java/run/halo/app/extension/store/ExtensionStoreClientJPAImplTest.java b/application/src/test/java/run/halo/app/extension/store/ReactiveExtensionStoreClientImplTest.java similarity index 86% rename from application/src/test/java/run/halo/app/extension/store/ExtensionStoreClientJPAImplTest.java rename to application/src/test/java/run/halo/app/extension/store/ReactiveExtensionStoreClientImplTest.java index f2f12c43b14..5e7f13c4113 100644 --- a/application/src/test/java/run/halo/app/extension/store/ExtensionStoreClientJPAImplTest.java +++ b/application/src/test/java/run/halo/app/extension/store/ReactiveExtensionStoreClientImplTest.java @@ -1,13 +1,11 @@ package run.halo.app.extension.store; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.when; -import jakarta.persistence.EntityNotFoundException; import java.util.List; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -18,13 +16,13 @@ import reactor.core.publisher.Mono; @ExtendWith(MockitoExtension.class) -class ExtensionStoreClientJPAImplTest { +class ReactiveExtensionStoreClientImplTest { @Mock ExtensionStoreRepository repository; @InjectMocks - ExtensionStoreClientJPAImpl client; + ReactiveExtensionStoreClientImpl client; @Test void listByNamePrefix() { @@ -36,7 +34,7 @@ void listByNamePrefix() { when(repository.findAllByNameStartingWith("/registry/posts")) .thenReturn(Flux.fromIterable(expectedExtensions)); - var gotExtensions = client.listByNamePrefix("/registry/posts"); + var gotExtensions = client.listByNamePrefix("/registry/posts").collectList().block(); assertEquals(expectedExtensions, gotExtensions); } @@ -48,7 +46,7 @@ void fetchByName() { when(repository.findById("/registry/posts/hello-halo")) .thenReturn(Mono.just(expectedExtension)); - var gotExtension = client.fetchByName("/registry/posts/hello-halo"); + var gotExtension = client.fetchByName("/registry/posts/hello-halo").blockOptional(); assertTrue(gotExtension.isPresent()); assertEquals(expectedExtension, gotExtension.get()); } @@ -62,7 +60,8 @@ void create() { .thenReturn(Mono.just(expectedExtension)); var createdExtension = - client.create("/registry/posts/hello-halo", "hello halo".getBytes()); + client.create("/registry/posts/hello-halo", "hello halo".getBytes()) + .block(); assertEquals(expectedExtension, createdExtension); } @@ -75,18 +74,17 @@ void update() { when(repository.save(any())).thenReturn(Mono.just(expectedExtension)); var updatedExtension = - client.update("/registry/posts/hello-halo", 1L, "hello halo".getBytes()); + client.update("/registry/posts/hello-halo", 1L, "hello halo".getBytes()) + .block(); assertEquals(expectedExtension, updatedExtension); } @Test - void shouldThrowEntityNotFoundExceptionWhenDeletingNonExistExt() { - + void shouldDoNotThrowExceptionWhenDeletingNonExistExt() { when(repository.findById(anyString())).thenReturn(Mono.empty()); - assertThrows(EntityNotFoundException.class, - () -> client.delete("/registry/posts/hello-halo", 1L)); + client.delete("/registry/posts/hello-halo", 1L).block(); } @Test @@ -97,7 +95,7 @@ void shouldDeleteSuccessfully() { when(repository.findById(anyString())).thenReturn(Mono.just(expectedExtension)); when(repository.delete(any())).thenReturn(Mono.empty()); - var deletedExtension = client.delete("/registry/posts/hello-halo", 2L); + var deletedExtension = client.delete("/registry/posts/hello-halo", 2L).block(); assertEquals(expectedExtension, deletedExtension); } diff --git a/docs/index/README.md b/docs/index/README.md new file mode 100644 index 00000000000..0a49372228e --- /dev/null +++ b/docs/index/README.md @@ -0,0 +1,316 @@ +# 索引机制 RFC + +## 背景 + +目前 Halo 使用 Extension 机制来存储和获取数据以便支持更好的扩展性，所以设计之初就存在查询数据时会将对应 Extension 的所有数据查询到内存中处理的问题，这会导致当分页查询和条件查询时也会有大批量无效数据被加载到内存中，随着 Halo 用户的数据量的增长，如果没有一个方案来解决这样的数据查询问题会对 Halo 用户的服务器内存资源有较高的要求，因此本篇提出使用索引机制来解决数据查询问题，以便提高查询效率和减少内存开销。 + +## 目标 + +- **提高查询效率**：加快数据检索速度。通过使用索引，数据库可以快速定位到数据行的位置，从而减少必须读取的数据量。 +- **减少网络和内存开销：** 没有索引前查询数据会将 Extension 的所有数据都传输到应用对网络和内存开销都很大，通过索引定位确切的数据来减少不必要的消耗。 +- **优化排序操作**：通过索引加速排序操作，因此需要索引本身有序。 +- **索引存储可扩展**：索引虽然能提高查询效率，但它会占用额外的存储空间，如果过大可以考虑在磁盘上读写等方式来减少对内存的占用。 + +## 非目标 + +- 索引的持久化存储，前期只考虑在内存中存储索引，如果后续索引过大可以考虑在磁盘上读写等方式来减少对内存的占用。 +- 索引的自动维护，索引的维护需要考虑到索引的数据是否改变，如果改变则需要更新索引，这个改变的判断不好界定，所以先不考虑索引的自动维护。 +- 索引的前置验证，比如在启动时验证索引的完整性和正确性，但目前每次启动都会重新构建索引，所以先不考虑索引的前置验证。 +- 多线程构建索引，目前索引的构建是单线程的，如果后续索引过大可以考虑多线程构建索引。 + +## 方案 + +索引是一种存储数据结构，可提供对数据集中字段的高效查找。索引将 Extension 中的字段映射到 Extension 的名称，以便在查询特定字段时不需要完整的扫描。 + +### 索引结构 + +每个 Extension 声明的索引都会被创建为一个 keySpace 与索引信息的映射， +类如对附件分组的一个对名称的索引示例如下： + +```javascript +{ + "/registry/storage.halo.run/groups": [{ + name: "specName", + spec: { + // a function that returns the value of the index key + indexFunc: function(doc) { + return doc.spec.name; + }, + order: 'asc', + unique: false + }, + v: 1, + ready: false + }, + { + name: "metadata.labels", + spec: { + indexFunc: function(doc) { + var labels = obj.getMetadata().getLabels(); + if (labels == null) { + return Set.of(); + } + return labels.entrySet() + .stream() + .map(entry -> entry.getKey() + "=" + entry.getValue()) + .collect(Collectors.toSet()); + }, + order: 'asc', + unique: false + }, + v: 1, + ready: true, + }] +} +``` + +- `name: specName` 表示索引的名称，每个 Extension 声明的索引名称不能重复，通常为字段路径如 `metadata.name`。 +- `spec.indexFunc` 用于生成索引键，索引键是一个字符串数组，每个字符串都是一个索引键值，索引键值是一个字符串。 +- `spec.order` 用于记录索引键的排序方式，可选值为 `asc` 或 `desc`，`asc` 表示升序，`desc` 表示降序。 +- `spec.unique` 用于标识是否为唯一索引以在添加索引时进行唯一性检查。 +- `v` 用于记录索引结构的版本以防止后续为优化导致索引结构改变时便于检查重建索引。 +- `ready` 用于记录该索引是否构建完成，当开始构建该索引键索引记录时为 false，如果构建完成则修改为 true，如果因为断电等导致索引构建不完整则 ready 会是 false，下次启动时需要重新开始构建。 + +对于每个 Extension 都有一个默认的唯一索引 `metadata.name` 其 entries 与 Extension 每一条记录唯一对应。 + +### 索引构建 + +索引是通过对 Extension 数据执行完整扫描来构建的。 + +1. **针对特定 Extension 数据集的操作**: 当构建索引时，操作是针对特定的 Extension 数据进行的。将 `ready` 置为 `false` +2. **扫描 Extension 数据集**: 构建索引的关键步骤是扫描 Extension 数据集中的每一条记录。这个扫描过程并不是基于数据库中所有数据的顺序，而是专注于该 Extension 数据集内的数据。当构建索引时会锁定对该 Extension 的写操作。 +3. **生成索引键（KeyString键）**:对于 Extension 数据集中的每个 Extension，会根据其索引字段生成 KeyString 键。String 为 Extension 的 `metadata.name` 用于定位 Extension 在数据库中的位置。 +4. **排序和插入操作**: 生成的键会被插入到一个外部排序器中，以确保它们的顺序。排序后，这些键按顺序批量加载到索引中。 +5. 释放对该 Extension 写操作的锁定完成了索引构建。 + +对于后续 Extension 和索引的更新需要在同一个事务中以确保一致性。 + +```json +{ + "metadata.name": { + "group-1": [] + }, + "specName": { + "zhangsan": [ + "metadata-name-1" + ], + "lisi": [ + "metadata-name-2" + ] + }, + "halo.run/hidden": { + "true": [ + "metadata-name-3" + ], + "false": [ + "metadata-name-4" + ] + } +} +``` + +### 索引前置验证 + +1. 每次启动后先检查索引是否 ready +2. `metadata.name` 索引条目的数量始终与数据库中记录的 Extension 数量一致 +3. 如果排序顺序为升序，则索引条目按递增顺序排列。 +4. 如果排序顺序为降序，则索引条目按降序排列。 +5. 每个索引的索引条目数量不超过数据库中记录的对应 Extension 数量。 + +### 索引在 Extension 的声明 + +手动注册索引 + +```java +public class IndexSpec { + private String name; + + private IndexAttribute indexFunc; + + private OrderType order; + + private boolean unique; + + public enum OrderType { + ASC, + DESC + } + + // Getters and other methods... +} + +IndexSpecs indexSpecs = indexSpecRegistry.indexFor(Person.class); +indexSpecs.add(new IndexSpec() + .setName("spec.name") + .setOrder(IndexSpec.OrderType.DESC) + .setIndexFunc(IndexAttributeFactory.simpleAttribute(Person.class, + e -> e.getSpec().getName()) + ) + .setUnique(false)); +``` + +用于普通索引的注解 + +```java +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE, ElementType.ANNOTATION_TYPE}) // 用于类和注解的注解 +public @interface Index { + String name(); // 索引名称 + String field(); // 需要索引的字段 +} +``` + +Indexes 注解用于声明多个索引 + +```java +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface Indexes { + Index[] value() default {}; // Index注解数组 +} +``` + +```java +@Data +@Indexes({ + @Index(name = "specName", field = "spec.name"), + @Index(name = "creationTimestamp", field = "metadata.creationTimestamp"), +}) +@EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) +@GVK(group = "my-plugin.guqing.io", + version = "v1alpha1", + kind = "Person", + plural = "persons", + singular = "person") +public class Person extends Extension { + + @Schema(description = "The description on name field", maxLength = 100) + private String name; + + @Schema(description = "The description on age field", maximum = "150", minimum = "0") + private Integer age; + + @Schema(description = "The description on gender field") + private Gender gender; + + public enum Gender { + MALE, FEMALE, + } +} +``` + +不论是手动注册索引还是通过注解注册索引都由 IndexSpecRegistry 管理。 + +```java +public interface IndexSpecRegistry { + /** + *

Create a new {@link IndexSpecs} for the given {@link Extension} type.

+ *

The returned {@link IndexSpecs} is always includes some default {@link IndexSpec} that + * does not need to be registered again:

+ *
    + *
  • {@link Metadata#getName()} for unique primary index spec named metadata_name
    • + *
  • {@link Metadata#getCreationTimestamp()} for creation_timestamp index spec
    • + *
  • {@link Metadata#getDeletionTimestamp()} for deletion_timestamp index spec
    • + *
  • {@link Metadata#getLabels()} for labels index spec
    • + *
+ * + * @param extensionType must not be {@literal null}. + * @param the extension type + * @return the {@link IndexSpecs} for the given {@link Extension} type. + */ + IndexSpecs indexFor(Class extensionType); + + /** + * Get {@link IndexSpecs} for the given {@link Extension} type registered before. + * + * @param extensionType must not be {@literal null}. + * @param the extension type + * @return the {@link IndexSpecs} for the given {@link Extension} type. + * @throws IllegalArgumentException if no {@link IndexSpecs} found for the given + * {@link Extension} type. + */ + IndexSpecs getIndexSpecs(Class extensionType); + + boolean contains(Class extensionType); + + void removeIndexSpecs(Class extensionType); + + /** + * Get key space for an extension type. + * + * @param scheme is a scheme of an Extension. + * @return key space(never null) + */ + @NonNull + String getKeySpace(Scheme scheme); +} +``` + +对于添加了索引的 Extension 可以使用 `IndexedQueryEngine` 来查询数据： + +```java +public interface IndexedQueryEngine { + /** + * Page retrieve the object records by the given {@link GroupVersionKind} and + * {@link ListOptions}. + * + * @param type the type of the object must exist in + * {@link run.halo.app.extension.SchemeManager}. + * @param options the list options to use for retrieving the object records. + * @param page which page to retrieve and how large the page should be. + * @return a collection of {@link Metadata#getName()} for the given page. + */ + ListResult retrieve(GroupVersionKind type, ListOptions options, PageRequest page); + + /** + * Retrieve all the object records by the given {@link GroupVersionKind} and + * {@link ListOptions}. + * + * @param type the type of the object must exist in {@link run.halo.app.extension.SchemeManager} + * @param options the list options to use for retrieving the object records + * @return a collection of {@link Metadata#getName()} + */ + List retrieveAll(GroupVersionKind type, ListOptions options); +} +``` + +但为了简便起见，会在 ReactiveExtensionClient 中提供一个 `listBy` 方法来查询数据： + +```java +public interface ReactiveExtensionClient { + //... + Mono> listBy(Class type, ListOptions options, + PageRequest pageable); +} +``` + +其中 `ListOptions` 包含两部分，`LabelSelector` 和 `FieldSelector`，一个常见的手动构建的 `ListOptions` 示例： + +```java +var listOptions = new ListOptions(); +listOptions.setLabelSelector(LabelSelector.builder() + .eq("key1", "value1").build()); +listOptions.setFieldSelector(FieldSelector.builder() + .eq("slug", "slug1").build()); +``` + +为了兼容以前的写法，对于 APIs 中可以继续使用 `run.halo.app.extension.router.IListRequest`，然后使用工具类转换即可得到 `ListOptions` 和 `PageRequest`。 + +```java +class QueryListRequest implements IListRequest { + public ListOptions toListOptions() { + return labelAndFieldSelectorToListOptions(getLabelSelector(), getFieldSelector()); + } + + public PageRequest toPageRequest() { + return PageRequestImpl.of(getPage(), getSize(), getSort()); + } +} +``` + +### Reconciler + +对于 Reconciler 来说，之前每次由 DefaultController 启动对于需要 `syncAllOnStart` 的 Reconciler 都是获取所有对应的 Extension 数据，然后再进行 Reconcile，这样会导致每次都将所有的 Extension 数据加载到内存中，随着数据量的增加导致内存占用过大，当有了索引后只获取所有 Extension 的 `metadata.name` 来触发 reconcile 即可。 + +GcReconciler 也从索引中获取 `metadata.deletionTimestamp` 不为空的 Extension 的 `metadata.name` 来触发 reconcile 以减少全量加载数据的操作。