diff --git a/api/src/main/java/run/halo/app/extension/DefaultExtensionMatcher.java b/api/src/main/java/run/halo/app/extension/DefaultExtensionMatcher.java index 9ca69c2c407..a3026fef34f 100644 --- a/api/src/main/java/run/halo/app/extension/DefaultExtensionMatcher.java +++ b/api/src/main/java/run/halo/app/extension/DefaultExtensionMatcher.java @@ -1,12 +1,10 @@ package run.halo.app.extension; -import static org.apache.commons.lang3.ObjectUtils.defaultIfNull; - -import java.util.Map; import lombok.Builder; import lombok.Getter; import lombok.RequiredArgsConstructor; -import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.util.CollectionUtils; +import run.halo.app.extension.index.query.QueryFactory; import run.halo.app.extension.router.selector.FieldSelector; import run.halo.app.extension.router.selector.LabelSelector; @@ -14,14 +12,14 @@ @RequiredArgsConstructor @Builder(builderMethodName = "internalBuilder") public class DefaultExtensionMatcher implements ExtensionMatcher { - private static final SpelExpressionParser PARSER = new SpelExpressionParser(); - + private final ExtensionClient client; private final GroupVersionKind gvk; private final LabelSelector labelSelector; private final FieldSelector fieldSelector; - public static DefaultExtensionMatcherBuilder builder(GroupVersionKind gvk) { - return internalBuilder().gvk(gvk); + public static DefaultExtensionMatcherBuilder builder(ExtensionClient client, + GroupVersionKind gvk) { + return internalBuilder().client(client).gvk(gvk); } /** @@ -32,18 +30,28 @@ public static DefaultExtensionMatcherBuilder builder(GroupVersionKind gvk) { */ @Override public boolean match(Extension extension) { - if (gvk != null && !gvk.equals(extension.groupVersionKind())) { + if (!gvk.equals(extension.groupVersionKind())) { return false; } - var labels = defaultIfNull(extension.getMetadata().getLabels(), Map.of()); - if (labelSelector != null && !labelSelector.test(labels)) { - return false; + if (!hasFieldSelector() && !hasLabelSelector()) { + return true; } - - if (fieldSelector != null) { - return fieldSelector.test(key -> PARSER.parseRaw(key) - .getValue(extension, String.class)); + var listOptions = new ListOptions(); + listOptions.setLabelSelector(labelSelector); + var fieldQuery = QueryFactory.all(); + if (hasFieldSelector()) { + fieldQuery = QueryFactory.and(fieldQuery, fieldSelector.query()); } - return true; + listOptions.setFieldSelector(new FieldSelector(fieldQuery)); + return client.indexedQueryEngine().retrieve(getGvk(), + listOptions, PageRequestImpl.ofSize(1)).getTotal() > 0; + } + + boolean hasFieldSelector() { + return fieldSelector != null && fieldSelector.query() != null; + } + + boolean hasLabelSelector() { + return labelSelector != null && !CollectionUtils.isEmpty(labelSelector.getMatchers()); } } diff --git a/api/src/main/java/run/halo/app/extension/WatcherExtensionMatchers.java b/api/src/main/java/run/halo/app/extension/WatcherExtensionMatchers.java index ab4a0459a88..0a23f3f352a 100644 --- a/api/src/main/java/run/halo/app/extension/WatcherExtensionMatchers.java +++ b/api/src/main/java/run/halo/app/extension/WatcherExtensionMatchers.java @@ -2,8 +2,12 @@ import java.util.Objects; import lombok.Builder; +import lombok.Getter; +import org.springframework.util.Assert; public class WatcherExtensionMatchers { + @Getter + private final ExtensionClient client; private final GroupVersionKind gvk; private final ExtensionMatcher onAddMatcher; private final ExtensionMatcher onUpdateMatcher; @@ -14,12 +18,19 @@ public class WatcherExtensionMatchers { * {@link DefaultExtensionMatcher}. */ @Builder(builderMethodName = "internalBuilder") - public WatcherExtensionMatchers(GroupVersionKind gvk, ExtensionMatcher onAddMatcher, + public WatcherExtensionMatchers(ExtensionClient client, + GroupVersionKind gvk, ExtensionMatcher onAddMatcher, ExtensionMatcher onUpdateMatcher, ExtensionMatcher onDeleteMatcher) { + Assert.notNull(client, "The client must not be null."); + Assert.notNull(gvk, "The gvk must not be null."); + this.client = client; this.gvk = gvk; - this.onAddMatcher = Objects.requireNonNullElse(onAddMatcher, emptyMatcher(gvk)); - this.onUpdateMatcher = Objects.requireNonNullElse(onUpdateMatcher, emptyMatcher(gvk)); - this.onDeleteMatcher = Objects.requireNonNullElse(onDeleteMatcher, emptyMatcher(gvk)); + this.onAddMatcher = + Objects.requireNonNullElse(onAddMatcher, emptyMatcher(client, gvk)); + this.onUpdateMatcher = + Objects.requireNonNullElse(onUpdateMatcher, emptyMatcher(client, gvk)); + this.onDeleteMatcher = + Objects.requireNonNullElse(onDeleteMatcher, emptyMatcher(client, gvk)); } public GroupVersionKind getGroupVersionKind() { @@ -38,11 +49,13 @@ public ExtensionMatcher onDeleteMatcher() { return this.onDeleteMatcher; } - public static WatcherExtensionMatchersBuilder builder(GroupVersionKind gvk) { - return internalBuilder().gvk(gvk); + public static WatcherExtensionMatchersBuilder builder(ExtensionClient client, + GroupVersionKind gvk) { + return internalBuilder().gvk(gvk).client(client); } - static ExtensionMatcher emptyMatcher(GroupVersionKind gvk) { - return DefaultExtensionMatcher.builder(gvk).build(); + static ExtensionMatcher emptyMatcher(ExtensionClient client, + GroupVersionKind gvk) { + return DefaultExtensionMatcher.builder(client, gvk).build(); } } diff --git a/api/src/main/java/run/halo/app/extension/controller/ControllerBuilder.java b/api/src/main/java/run/halo/app/extension/controller/ControllerBuilder.java index 80cf85078e4..2bfcd9c4c4e 100644 --- a/api/src/main/java/run/halo/app/extension/controller/ControllerBuilder.java +++ b/api/src/main/java/run/halo/app/extension/controller/ControllerBuilder.java @@ -105,7 +105,8 @@ public Controller build() { Assert.notNull(reconciler, "Reconciler must not be null"); var queue = new DefaultQueue(nowSupplier, minDelay); - var extensionMatchers = WatcherExtensionMatchers.builder(extension.groupVersionKind()) + var extensionMatchers = WatcherExtensionMatchers.builder(client, + extension.groupVersionKind()) .onAddMatcher(onAddMatcher) .onUpdateMatcher(onUpdateMatcher) .onDeleteMatcher(onDeleteMatcher) diff --git a/api/src/main/java/run/halo/app/extension/index/query/All.java b/api/src/main/java/run/halo/app/extension/index/query/All.java new file mode 100644 index 00000000000..541d730207e --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/query/All.java @@ -0,0 +1,15 @@ +package run.halo.app.extension.index.query; + +import java.util.NavigableSet; + +public class All extends SimpleQuery { + + public All(String fieldName) { + super(fieldName, null); + } + + @Override + public NavigableSet matches(QueryIndexView indexView) { + return indexView.getAllIdsForField(fieldName); + } +} diff --git a/api/src/main/java/run/halo/app/extension/index/query/And.java b/api/src/main/java/run/halo/app/extension/index/query/And.java new file mode 100644 index 00000000000..ccb9ac4134b --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/query/And.java @@ -0,0 +1,37 @@ +package run.halo.app.extension.index.query; + +import com.google.common.collect.Sets; +import java.util.Collection; +import java.util.NavigableSet; + +public class And extends LogicalQuery { + + /** + * Creates a new And query with the given child queries. + * + * @param childQueries The child queries + */ + public And(Collection childQueries) { + super(childQueries); + if (this.size < 2) { + throw new IllegalStateException( + "An 'And' query cannot have fewer than 2 child queries, " + childQueries.size() + + " were supplied"); + } + } + + @Override + public NavigableSet matches(QueryIndexView indexView) { + NavigableSet resultSet = null; + for (Query query : childQueries) { + NavigableSet currentResult = query.matches(indexView); + indexView.removeAllFieldValuesByIdNotIn(currentResult); + if (resultSet == null) { + resultSet = Sets.newTreeSet(currentResult); + } else { + resultSet.retainAll(currentResult); + } + } + return resultSet == null ? Sets.newTreeSet() : resultSet; + } +} diff --git a/api/src/main/java/run/halo/app/extension/index/query/Between.java b/api/src/main/java/run/halo/app/extension/index/query/Between.java new file mode 100644 index 00000000000..829770b5d87 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/query/Between.java @@ -0,0 +1,35 @@ +package run.halo.app.extension.index.query; + +import com.google.common.collect.Sets; +import java.util.NavigableSet; + +public class Between extends SimpleQuery { + private final String lowerValue; + private final boolean lowerInclusive; + private final String upperValue; + private final boolean upperInclusive; + + public Between(String fieldName, String lowerValue, boolean lowerInclusive, + String upperValue, boolean upperInclusive) { + // value and isFieldRef are not used in Between + super(fieldName, null, false); + this.lowerValue = lowerValue; + this.lowerInclusive = lowerInclusive; + this.upperValue = upperValue; + this.upperInclusive = upperInclusive; + } + + + @Override + public NavigableSet matches(QueryIndexView indexView) { + NavigableSet allValues = indexView.getAllValuesForField(fieldName); + // get all values in the specified range + var subSet = allValues.subSet(lowerValue, lowerInclusive, upperValue, upperInclusive); + + var resultSet = Sets.newTreeSet(); + for (String val : subSet) { + resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val)); + } + return resultSet; + } +} diff --git a/api/src/main/java/run/halo/app/extension/index/query/EqualQuery.java b/api/src/main/java/run/halo/app/extension/index/query/EqualQuery.java new file mode 100644 index 00000000000..8816cd7bf11 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/query/EqualQuery.java @@ -0,0 +1,30 @@ +package run.halo.app.extension.index.query; + +import java.util.NavigableSet; + +public class EqualQuery extends SimpleQuery { + + public EqualQuery(String fieldName, String value) { + super(fieldName, value); + } + + public EqualQuery(String fieldName, String value, boolean isFieldRef) { + super(fieldName, value, isFieldRef); + } + + @Override + public NavigableSet matches(QueryIndexView indexView) { + if (isFieldRef) { + return resultSetForRefValue(indexView); + } + return resultSetForExactValue(indexView); + } + + private NavigableSet resultSetForRefValue(QueryIndexView indexView) { + return indexView.findIdsForFieldValueEqual(fieldName, value); + } + + private NavigableSet resultSetForExactValue(QueryIndexView indexView) { + return indexView.getIdsForFieldValue(fieldName, value); + } +} diff --git a/api/src/main/java/run/halo/app/extension/index/query/GreaterThanQuery.java b/api/src/main/java/run/halo/app/extension/index/query/GreaterThanQuery.java new file mode 100644 index 00000000000..b8670ee4d48 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/query/GreaterThanQuery.java @@ -0,0 +1,41 @@ +package run.halo.app.extension.index.query; + +import com.google.common.collect.Sets; +import java.util.NavigableSet; + +public class GreaterThanQuery extends SimpleQuery { + private final boolean orEqual; + + public GreaterThanQuery(String fieldName, String value, boolean orEqual) { + this(fieldName, value, orEqual, false); + } + + public GreaterThanQuery(String fieldName, String value, boolean orEqual, boolean isFieldRef) { + super(fieldName, value, isFieldRef); + this.orEqual = orEqual; + } + + @Override + public NavigableSet matches(QueryIndexView indexView) { + if (isFieldRef) { + return resultSetForRefValue(indexView); + } + return resultSetForExtractValue(indexView); + } + + private NavigableSet resultSetForRefValue(QueryIndexView indexView) { + return indexView.findIdsForFieldValueGreaterThan(fieldName, value, orEqual); + } + + private NavigableSet resultSetForExtractValue(QueryIndexView indexView) { + var resultSet = Sets.newTreeSet(); + var allValues = indexView.getAllValuesForField(fieldName); + NavigableSet tailSet = + orEqual ? allValues.tailSet(value, true) : allValues.tailSet(value, false); + + for (String val : tailSet) { + resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val)); + } + return resultSet; + } +} diff --git a/api/src/main/java/run/halo/app/extension/index/query/InQuery.java b/api/src/main/java/run/halo/app/extension/index/query/InQuery.java new file mode 100644 index 00000000000..5c743190e5d --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/query/InQuery.java @@ -0,0 +1,23 @@ +package run.halo.app.extension.index.query; + +import com.google.common.collect.Sets; +import java.util.NavigableSet; +import java.util.Set; + +public class InQuery extends SimpleQuery { + private final Set values; + + public InQuery(String columnName, Set values) { + super(columnName, null); + this.values = values; + } + + @Override + public NavigableSet matches(QueryIndexView indexView) { + NavigableSet resultSet = Sets.newTreeSet(); + for (String val : values) { + resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val)); + } + return resultSet; + } +} diff --git a/api/src/main/java/run/halo/app/extension/index/query/LessThanQuery.java b/api/src/main/java/run/halo/app/extension/index/query/LessThanQuery.java new file mode 100644 index 00000000000..4f882cb6487 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/query/LessThanQuery.java @@ -0,0 +1,41 @@ +package run.halo.app.extension.index.query; + +import com.google.common.collect.Sets; +import java.util.NavigableSet; + +public class LessThanQuery extends SimpleQuery { + private final boolean orEqual; + + public LessThanQuery(String fieldName, String value, boolean orEqual) { + this(fieldName, value, orEqual, false); + } + + public LessThanQuery(String fieldName, String value, boolean orEqual, boolean isFieldRef) { + super(fieldName, value, isFieldRef); + this.orEqual = orEqual; + } + + @Override + public NavigableSet matches(QueryIndexView indexView) { + if (isFieldRef) { + return resultSetForRefValue(indexView); + } + return resultSetForExactValue(indexView); + } + + private NavigableSet resultSetForRefValue(QueryIndexView indexView) { + return indexView.findIdsForFieldValueLessThan(fieldName, value, orEqual); + } + + private NavigableSet resultSetForExactValue(QueryIndexView indexView) { + var resultSet = Sets.newTreeSet(); + var allValues = indexView.getAllValuesForField(fieldName); + var headSet = orEqual ? allValues.headSet(value, true) + : allValues.headSet(value, false); + + for (String val : headSet) { + resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val)); + } + return resultSet; + } +} diff --git a/api/src/main/java/run/halo/app/extension/index/query/LogicalQuery.java b/api/src/main/java/run/halo/app/extension/index/query/LogicalQuery.java new file mode 100644 index 00000000000..4d81a1ef8e4 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/query/LogicalQuery.java @@ -0,0 +1,31 @@ +package run.halo.app.extension.index.query; + +import java.util.Collection; +import java.util.Objects; + +public abstract class LogicalQuery implements Query { + protected final Collection childQueries; + protected final int size; + + /** + * Creates a new logical query with the given child queries. + * + * @param childQueries with the given child queries. + */ + public LogicalQuery(Collection childQueries) { + Objects.requireNonNull(childQueries, + "The child queries supplied to a logical query cannot be null"); + for (Query query : childQueries) { + if (!isValid(query)) { + throw new IllegalStateException("Unexpected type of query: " + (query == null ? null + : query + ", " + query.getClass())); + } + } + this.size = childQueries.size(); + this.childQueries = childQueries; + } + + boolean isValid(Query query) { + return query instanceof LogicalQuery || query instanceof SimpleQuery; + } +} diff --git a/api/src/main/java/run/halo/app/extension/index/query/NotEqual.java b/api/src/main/java/run/halo/app/extension/index/query/NotEqual.java new file mode 100644 index 00000000000..6b0aed4c211 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/query/NotEqual.java @@ -0,0 +1,31 @@ +package run.halo.app.extension.index.query; + +import com.google.common.collect.Sets; +import java.util.NavigableSet; + +public class NotEqual extends SimpleQuery { + private final EqualQuery equalQuery; + + public NotEqual(String fieldName, String value) { + this(fieldName, value, false); + } + + public NotEqual(String fieldName, String value, boolean isFieldRef) { + super(fieldName, value, isFieldRef); + this.equalQuery = new EqualQuery(fieldName, value, isFieldRef); + } + + @Override + public NavigableSet matches(QueryIndexView indexView) { + var names = equalQuery.matches(indexView); + var allNames = indexView.getAllIdsForField(fieldName); + + var resultSet = Sets.newTreeSet(); + for (String name : allNames) { + if (!names.contains(name)) { + resultSet.add(name); + } + } + return resultSet; + } +} diff --git a/api/src/main/java/run/halo/app/extension/index/query/Or.java b/api/src/main/java/run/halo/app/extension/index/query/Or.java new file mode 100644 index 00000000000..ec79270ca91 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/query/Or.java @@ -0,0 +1,21 @@ +package run.halo.app.extension.index.query; + +import com.google.common.collect.Sets; +import java.util.Collection; +import java.util.NavigableSet; + +public class Or extends LogicalQuery { + + public Or(Collection childQueries) { + super(childQueries); + } + + @Override + public NavigableSet matches(QueryIndexView indexView) { + var resultSet = Sets.newTreeSet(); + for (Query query : childQueries) { + resultSet.addAll(query.matches(indexView)); + } + return resultSet; + } +} diff --git a/api/src/main/java/run/halo/app/extension/index/query/Query.java b/api/src/main/java/run/halo/app/extension/index/query/Query.java new file mode 100644 index 00000000000..1ad471603a7 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/query/Query.java @@ -0,0 +1,22 @@ +package run.halo.app.extension.index.query; + +import java.util.NavigableSet; +import run.halo.app.extension.Metadata; + +/** + * A {@link Query} is used to match {@link QueryIndexView} objects. + * + * @author guqing + * @since 2.12.0 + */ +public interface Query { + + /** + * Matches the given {@link QueryIndexView} and returns the matched object names see + * {@link Metadata#getName()}. + * + * @param indexView the {@link QueryIndexView} to match. + * @return the matched object names ordered by natural order. + */ + NavigableSet matches(QueryIndexView indexView); +} diff --git a/api/src/main/java/run/halo/app/extension/index/query/QueryFactory.java b/api/src/main/java/run/halo/app/extension/index/query/QueryFactory.java new file mode 100644 index 00000000000..c1847a5840e --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/query/QueryFactory.java @@ -0,0 +1,164 @@ +package run.halo.app.extension.index.query; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import lombok.experimental.UtilityClass; +import org.springframework.util.Assert; + +@UtilityClass +public class QueryFactory { + + public static Query all() { + return new All("metadata.name"); + } + + public static Query all(String fieldName) { + return new All(fieldName); + } + + public static Query notEqual(String fieldName, String attributeValue) { + return new NotEqual(fieldName, attributeValue); + } + + public static Query notEqualOtherField(String fieldName, String otherFieldName) { + return new NotEqual(fieldName, otherFieldName, true); + } + + public static Query equal(String fieldName, String attributeValue) { + return new EqualQuery(fieldName, attributeValue); + } + + public static Query equalOtherField(String fieldName, String otherFieldName) { + return new EqualQuery(fieldName, otherFieldName, true); + } + + public static Query lessThanOtherField(String fieldName, String otherFieldName) { + return new LessThanQuery(fieldName, otherFieldName, false, true); + } + + public static Query lessThanOrEqualOtherField(String fieldName, String otherFieldName) { + return new LessThanQuery(fieldName, otherFieldName, true, true); + } + + public static Query lessThan(String fieldName, String attributeValue) { + return new LessThanQuery(fieldName, attributeValue, false); + } + + public static Query lessThanOrEqual(String fieldName, String attributeValue) { + return new LessThanQuery(fieldName, attributeValue, true); + } + + public static Query greaterThan(String fieldName, String attributeValue) { + return new GreaterThanQuery(fieldName, attributeValue, false); + } + + public static Query greaterThanOrEqual(String fieldName, String attributeValue) { + return new GreaterThanQuery(fieldName, attributeValue, true); + } + + public static Query greaterThanOtherField(String fieldName, String otherFieldName) { + return new GreaterThanQuery(fieldName, otherFieldName, false, true); + } + + public static Query greaterThanOrEqualOtherField(String fieldName, + String otherFieldName) { + return new GreaterThanQuery(fieldName, otherFieldName, true, true); + } + + public static Query in(String fieldName, String... attributeValues) { + return in(fieldName, Set.of(attributeValues)); + } + + public static Query in(String fieldName, Collection values) { + Assert.notNull(values, "Values must not be null"); + if (values.size() == 1) { + String singleValue = values.iterator().next(); + return equal(fieldName, singleValue); + } + // Copy the values into a Set if necessary... + var valueSet = (values instanceof Set ? (Set) values + : new HashSet<>(values)); + return new InQuery(fieldName, valueSet); + } + + public static Query and(Collection queries) { + Assert.notEmpty(queries, "Queries must not be empty"); + if (queries.size() == 1) { + return queries.iterator().next(); + } + return new And(queries); + } + + public static And and(Query query1, Query query2) { + Collection queries = Arrays.asList(query1, query2); + return new And(queries); + } + + public static Query and(Query query1, Query query2, Query... additionalQueries) { + var queries = new ArrayList(2 + additionalQueries.length); + queries.add(query1); + queries.add(query2); + Collections.addAll(queries, additionalQueries); + return new And(queries); + } + + public static Query and(Query query1, Query query2, Collection additionalQueries) { + var queries = new ArrayList(2 + additionalQueries.size()); + queries.add(query1); + queries.add(query2); + queries.addAll(additionalQueries); + return new And(queries); + } + + public static Query or(Query query1, Query query2) { + Collection queries = Arrays.asList(query1, query2); + return new Or(queries); + } + + public static Query or(Query query1, Query query2, Query... additionalQueries) { + var queries = new ArrayList(2 + additionalQueries.length); + queries.add(query1); + queries.add(query2); + Collections.addAll(queries, additionalQueries); + return new Or(queries); + } + + public static Query or(Query query1, Query query2, Collection additionalQueries) { + var queries = new ArrayList(2 + additionalQueries.size()); + queries.add(query1); + queries.add(query2); + queries.addAll(additionalQueries); + return new Or(queries); + } + + public static Query betweenLowerExclusive(String fieldName, String lowerValue, + String upperValue) { + return new Between(fieldName, lowerValue, false, upperValue, true); + } + + public static Query betweenUpperExclusive(String fieldName, String lowerValue, + String upperValue) { + return new Between(fieldName, lowerValue, true, upperValue, false); + } + + public static Query betweenExclusive(String fieldName, String lowerValue, + String upperValue) { + return new Between(fieldName, lowerValue, false, upperValue, false); + } + + public static Query between(String fieldName, String lowerValue, String upperValue) { + return new Between(fieldName, lowerValue, true, upperValue, true); + } + + public static Query startsWith(String fieldName, String value) { + return new StringStartsWith(fieldName, value); + } + + public static Query endsWith(String fieldName, String value) { + return new StringEndsWith(fieldName, value); + } +} diff --git a/api/src/main/java/run/halo/app/extension/index/query/QueryIndexView.java b/api/src/main/java/run/halo/app/extension/index/query/QueryIndexView.java new file mode 100644 index 00000000000..dcf6e02a92d --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/query/QueryIndexView.java @@ -0,0 +1,57 @@ +package run.halo.app.extension.index.query; + +import java.util.NavigableSet; +import run.halo.app.extension.Metadata; +import run.halo.app.extension.index.IndexSpec; + +/** + *

A view of an index entries that can be queried.

+ *

Explanation of naming:

+ *
    + *
  • fieldName: a field of an index, usually {@link IndexSpec#getName()}
    • + *
  • fieldValue: a value of a field, e.g. a value of a field "name" could be "foo"
    • + *
  • id: the id of an object pointing to object position, see {@link Metadata#getName()}
    • + *
+ * + * @author guqing + * @since 2.12.0 + */ +public interface QueryIndexView { + /** + * Gets all object ids for a given field name and field value. + * + * @param fieldName the field name + * @param fieldValue the field value + * @return all indexed object ids associated with the given field name and field value + * @throws IllegalArgumentException if the field name is not indexed + */ + NavigableSet getIdsForFieldValue(String fieldName, String fieldValue); + + /** + * Gets all field values for a given field name. + * + * @param fieldName the field name + * @return all field values for the given field name + * @throws IllegalArgumentException if the field name is not indexed + */ + NavigableSet getAllValuesForField(String fieldName); + + /** + * Gets all object ids for a given field name. + * + * @param fieldName the field name + * @return all indexed object ids for the given field name + * @throws IllegalArgumentException if the field name is not indexed + */ + NavigableSet getAllIdsForField(String fieldName); + + NavigableSet findIdsForFieldValueEqual(String fieldName1, String fieldName2); + + NavigableSet findIdsForFieldValueGreaterThan(String fieldName1, String fieldName2, + boolean orEqual); + + NavigableSet findIdsForFieldValueLessThan(String fieldName1, String fieldName2, + boolean orEqual); + + void removeAllFieldValuesByIdNotIn(NavigableSet ids); +} diff --git a/api/src/main/java/run/halo/app/extension/index/query/QueryIndexViewImpl.java b/api/src/main/java/run/halo/app/extension/index/query/QueryIndexViewImpl.java new file mode 100644 index 00000000000..2b32f7c141f --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/query/QueryIndexViewImpl.java @@ -0,0 +1,194 @@ +package run.halo.app.extension.index.query; + +import com.google.common.collect.MultimapBuilder; +import com.google.common.collect.Multimaps; +import com.google.common.collect.SetMultimap; +import com.google.common.collect.Sets; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.NavigableSet; +import java.util.TreeSet; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +/** + * A default implementation for {@link QueryIndexView}. + * + * @author guqing + * @since 2.12.0 + */ +public class QueryIndexViewImpl implements QueryIndexView { + private final Lock lock = new ReentrantLock(); + private final Map> orderedMatches; + + /** + * Creates a new {@link QueryIndexViewImpl} for the given {@link Map} of index entries. + * + * @param indexEntries index entries from indexer to create the view for. + */ + public QueryIndexViewImpl(Map>> indexEntries) { + this.orderedMatches = new HashMap<>(); + for (var entry : indexEntries.entrySet()) { + // do not use stream collect here as it is slower + this.orderedMatches.put(entry.getKey(), createSetMultiMap(entry.getValue())); + } + } + + @Override + public NavigableSet getIdsForFieldValue(String fieldName, String fieldValue) { + lock.lock(); + try { + checkFieldNameIndexed(fieldName); + SetMultimap fieldMap = orderedMatches.get(fieldName); + return fieldMap != null ? new TreeSet<>(fieldMap.get(fieldValue)) : emptySet(); + } finally { + lock.unlock(); + } + } + + @Override + public NavigableSet getAllValuesForField(String fieldName) { + lock.lock(); + try { + checkFieldNameIndexed(fieldName); + SetMultimap fieldMap = orderedMatches.get(fieldName); + return fieldMap != null ? new TreeSet<>(fieldMap.keySet()) : emptySet(); + } finally { + lock.unlock(); + } + } + + @Override + public NavigableSet getAllIdsForField(String fieldName) { + lock.lock(); + try { + checkFieldNameIndexed(fieldName); + SetMultimap fieldMap = orderedMatches.get(fieldName); + return fieldMap != null ? new TreeSet<>(fieldMap.values()) : emptySet(); + } finally { + lock.unlock(); + } + } + + @Override + public NavigableSet findIdsForFieldValueEqual(String fieldName1, String fieldName2) { + lock.lock(); + try { + checkFieldNameIndexed(fieldName1); + checkFieldNameIndexed(fieldName2); + var index1 = orderedMatches.get(fieldName1); + var index2 = orderedMatches.get(fieldName2); + + var idFieldValuesForIndex2Map = + Multimaps.invertFrom(index2, MultimapBuilder.treeKeys().treeSetValues().build()); + var result = Sets.newTreeSet(); + for (Map.Entry entryForIndex1 : index1.entries()) { + var fieldValues = idFieldValuesForIndex2Map.get(entryForIndex1.getValue()); + for (String item : fieldValues) { + if (entryForIndex1.getKey().equals(item)) { + result.add(entryForIndex1.getValue()); + } + } + } + return result; + } finally { + lock.unlock(); + } + } + + @Override + public NavigableSet findIdsForFieldValueGreaterThan(String fieldName1, + String fieldName2, boolean orEqual) { + lock.lock(); + try { + checkFieldNameIndexed(fieldName1); + checkFieldNameIndexed(fieldName2); + + var index1 = orderedMatches.get(fieldName1); + var index2 = orderedMatches.get(fieldName2); + + var idFieldValuesForIndex2Map = + Multimaps.invertFrom(index2, MultimapBuilder.treeKeys().treeSetValues().build()); + + var result = Sets.newTreeSet(); + for (Map.Entry entryForIndex1 : index1.entries()) { + var fieldValues = idFieldValuesForIndex2Map.get(entryForIndex1.getValue()); + for (String item : fieldValues) { + int compare = entryForIndex1.getKey().compareTo(item); + if (orEqual ? compare >= 0 : compare > 0) { + result.add(entryForIndex1.getValue()); + } + } + } + return result; + } finally { + lock.unlock(); + } + } + + @Override + public NavigableSet findIdsForFieldValueLessThan(String fieldName1, String fieldName2, + boolean orEqual) { + lock.lock(); + try { + checkFieldNameIndexed(fieldName1); + checkFieldNameIndexed(fieldName2); + SetMultimap index1 = orderedMatches.get(fieldName1); + SetMultimap index2 = orderedMatches.get(fieldName2); + + var idFieldValuesForIndex2Map = + Multimaps.invertFrom(index2, MultimapBuilder.treeKeys().treeSetValues().build()); + + var result = Sets.newTreeSet(); + for (Map.Entry entryForIndex1 : index1.entries()) { + var fieldValues = idFieldValuesForIndex2Map.get(entryForIndex1.getValue()); + for (String item : fieldValues) { + int compare = entryForIndex1.getKey().compareTo(item); + if (orEqual ? compare <= 0 : compare < 0) { + result.add(entryForIndex1.getValue()); + } + } + } + return result; + } finally { + lock.unlock(); + } + } + + @Override + public void removeAllFieldValuesByIdNotIn(NavigableSet ids) { + lock.lock(); + try { + for (var fieldNameValuesEntry : orderedMatches.entrySet()) { + SetMultimap indicates = fieldNameValuesEntry.getValue(); + indicates.entries().removeIf(entry -> !ids.contains(entry.getValue())); + } + } finally { + lock.unlock(); + } + } + + private void checkFieldNameIndexed(String fieldName) { + if (!orderedMatches.containsKey(fieldName)) { + throw new IllegalArgumentException("Field name " + fieldName + + " is not indexed, please ensure it added to the index spec before querying"); + } + } + + private TreeSet emptySet() { + return new TreeSet<>(); + } + + private SetMultimap createSetMultiMap( + Collection> entries) { + + SetMultimap multiMap = MultimapBuilder.hashKeys() + .hashSetValues() + .build(); + for (Map.Entry entry : entries) { + multiMap.put(entry.getKey(), entry.getValue()); + } + return multiMap; + } +} diff --git a/api/src/main/java/run/halo/app/extension/index/query/SimpleQuery.java b/api/src/main/java/run/halo/app/extension/index/query/SimpleQuery.java new file mode 100644 index 00000000000..a0c91db2005 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/query/SimpleQuery.java @@ -0,0 +1,21 @@ +package run.halo.app.extension.index.query; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.util.Assert; + +public abstract class SimpleQuery implements Query { + protected final String fieldName; + protected final String value; + protected final boolean isFieldRef; + + protected SimpleQuery(String fieldName, String value) { + this(fieldName, value, false); + } + + protected SimpleQuery(String fieldName, String value, boolean isFieldRef) { + Assert.isTrue(StringUtils.isNotBlank(fieldName), "fieldName cannot be blank."); + this.fieldName = fieldName; + this.value = value; + this.isFieldRef = isFieldRef; + } +} diff --git a/api/src/main/java/run/halo/app/extension/index/query/StringEndsWith.java b/api/src/main/java/run/halo/app/extension/index/query/StringEndsWith.java new file mode 100644 index 00000000000..b6e2bed00f5 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/query/StringEndsWith.java @@ -0,0 +1,22 @@ +package run.halo.app.extension.index.query; + +import com.google.common.collect.Sets; +import java.util.NavigableSet; + +public class StringEndsWith extends SimpleQuery { + public StringEndsWith(String fieldName, String value) { + super(fieldName, value); + } + + @Override + public NavigableSet matches(QueryIndexView indexView) { + var resultSet = Sets.newTreeSet(); + var fieldValues = indexView.getAllValuesForField(fieldName); + for (String val : fieldValues) { + if (val.endsWith(value)) { + resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val)); + } + } + return resultSet; + } +} diff --git a/api/src/main/java/run/halo/app/extension/index/query/StringStartsWith.java b/api/src/main/java/run/halo/app/extension/index/query/StringStartsWith.java new file mode 100644 index 00000000000..5d0fd5a46e8 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/query/StringStartsWith.java @@ -0,0 +1,23 @@ +package run.halo.app.extension.index.query; + +import com.google.common.collect.Sets; +import java.util.NavigableSet; + +public class StringStartsWith extends SimpleQuery { + public StringStartsWith(String fieldName, String value) { + super(fieldName, value); + } + + @Override + public NavigableSet matches(QueryIndexView indexView) { + var resultSet = Sets.newTreeSet(); + var allValues = indexView.getAllValuesForField(fieldName); + + for (String val : allValues) { + if (val.startsWith(value)) { + resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val)); + } + } + return resultSet; + } +} diff --git a/api/src/main/java/run/halo/app/extension/router/selector/AndSelectorMatcher.java b/api/src/main/java/run/halo/app/extension/router/selector/AndSelectorMatcher.java deleted file mode 100644 index c12090b12d0..00000000000 --- a/api/src/main/java/run/halo/app/extension/router/selector/AndSelectorMatcher.java +++ /dev/null @@ -1,18 +0,0 @@ -package run.halo.app.extension.router.selector; - -import lombok.Getter; - -@Getter -public class AndSelectorMatcher extends LogicalMatcher { - private final SelectorMatcher left; - private final SelectorMatcher right; - - public AndSelectorMatcher(SelectorMatcher left, SelectorMatcher right) { - this.left = left; - this.right = right; - } - - public boolean test(String s) { - return left.test(s) && right.test(s); - } -} diff --git a/api/src/main/java/run/halo/app/extension/router/selector/AnySelectorMatcher.java b/api/src/main/java/run/halo/app/extension/router/selector/AnySelectorMatcher.java deleted file mode 100644 index 06b1094a868..00000000000 --- a/api/src/main/java/run/halo/app/extension/router/selector/AnySelectorMatcher.java +++ /dev/null @@ -1,9 +0,0 @@ -package run.halo.app.extension.router.selector; - -public class AnySelectorMatcher extends LogicalMatcher { - - @Override - public boolean test(String s) { - return true; - } -} diff --git a/api/src/main/java/run/halo/app/extension/router/selector/FieldSelector.java b/api/src/main/java/run/halo/app/extension/router/selector/FieldSelector.java index b82e6f52994..a0a90252a94 100644 --- a/api/src/main/java/run/halo/app/extension/router/selector/FieldSelector.java +++ b/api/src/main/java/run/halo/app/extension/router/selector/FieldSelector.java @@ -1,117 +1,15 @@ package run.halo.app.extension.router.selector; import java.util.Objects; -import java.util.function.UnaryOperator; -import lombok.Data; -import lombok.experimental.Accessors; -import org.springframework.util.Assert; +import run.halo.app.extension.index.query.Query; +import run.halo.app.extension.index.query.QueryFactory; -@Data -@Accessors(chain = true) -public class FieldSelector { - private SelectorMatcher matcher; - - public static FieldSelectorBuilder builder() { - return new FieldSelectorBuilder(null); - } - - public static FieldSelectorBuilder builder(SelectorMatcher rootMatcher) { - return new FieldSelectorBuilder(rootMatcher); - } - - public boolean test(UnaryOperator valueForKeyFunc) { - return evaluate(matcher, valueForKeyFunc); +public record FieldSelector(Query query) { + public FieldSelector(Query query) { + this.query = Objects.requireNonNullElseGet(query, QueryFactory::all); } - boolean evaluate(SelectorMatcher matcher, UnaryOperator valueForKeyFunc) { - if (matcher instanceof LogicalMatcher) { - if (matcher instanceof AndSelectorMatcher andNode) { - return evaluate(andNode.getLeft(), valueForKeyFunc) - && evaluate(andNode.getRight(), valueForKeyFunc); - } else if (matcher instanceof OrSelectorMatcher orNode) { - return evaluate(orNode.getLeft(), valueForKeyFunc) - || evaluate(orNode.getRight(), valueForKeyFunc); - } else if (matcher instanceof AnySelectorMatcher) { - return true; - } - } - String valueToTest = valueForKeyFunc.apply(matcher.getKey()); - return matcher.test(valueToTest); - } - - public static class FieldSelectorBuilder { - private SelectorMatcher rootMatcher; - - public FieldSelectorBuilder(SelectorMatcher rootMatcher) { - this.rootMatcher = rootMatcher; - } - - public FieldSelectorBuilder eq(String fieldName, String fieldValue) { - return and(EqualityMatcher.equal(fieldName, fieldValue)); - } - - public FieldSelectorBuilder notEq(String fieldName, String fieldValue) { - return and(EqualityMatcher.notEqual(fieldName, fieldValue)); - } - - public FieldSelectorBuilder in(String fieldName, String... fieldValues) { - return and(SetMatcher.in(fieldName, fieldValues)); - } - - public FieldSelectorBuilder notIn(String fieldName, String... fieldValues) { - return and(SetMatcher.notIn(fieldName, fieldValues)); - } - - public FieldSelectorBuilder exists(String fieldName) { - return and(SetMatcher.exists(fieldName)); - } - - public FieldSelectorBuilder notExists(String fieldName) { - return and(SetMatcher.notExists(fieldName)); - } - - /** - * Combine the current selector matcher with another one with AND. - * - * @param other another selector matcher to be combined with the current one with AND - * @return the current selector matcher builder - */ - public FieldSelectorBuilder and(SelectorMatcher other) { - Assert.notNull(other, "Other selector matcher must not be null"); - if (rootMatcher == null) { - this.rootMatcher = other; - return this; - } - this.rootMatcher = new AndSelectorMatcher(rootMatcher, other); - return this; - } - - /** - * Combine the current selector matcher with another one with OR. - * - * @param other another selector matcher to be combined with the current one with OR - * @return the current selector matcher builder - */ - public FieldSelectorBuilder or(SelectorMatcher other) { - Assert.notNull(other, "Other selector matcher must not be null"); - if (rootMatcher == null) { - rootMatcher = other; - } - rootMatcher = new OrSelectorMatcher(rootMatcher, other); - return this; - } - - /** - * Build the selector matcher. - */ - public FieldSelector build() { - var fieldSelector = new FieldSelector(); - fieldSelector.setMatcher(buildMatcher()); - return fieldSelector; - } - - public SelectorMatcher buildMatcher() { - return Objects.requireNonNullElseGet(rootMatcher, AnySelectorMatcher::new); - } + public static FieldSelector of(Query query) { + return new FieldSelector(query); } } diff --git a/api/src/main/java/run/halo/app/extension/router/selector/FieldSelectorConverter.java b/api/src/main/java/run/halo/app/extension/router/selector/FieldSelectorConverter.java index 3bfd3b1974e..8d630b8214a 100644 --- a/api/src/main/java/run/halo/app/extension/router/selector/FieldSelectorConverter.java +++ b/api/src/main/java/run/halo/app/extension/router/selector/FieldSelectorConverter.java @@ -6,12 +6,14 @@ import org.springframework.core.convert.converter.Converter; import org.springframework.lang.NonNull; import org.springframework.util.CollectionUtils; +import run.halo.app.extension.index.query.Query; +import run.halo.app.extension.index.query.QueryFactory; -public class FieldSelectorConverter implements Converter { +public class FieldSelectorConverter implements Converter { @NonNull @Override - public SelectorMatcher convert(@NonNull SelectorCriteria criteria) { + public Query convert(@NonNull SelectorCriteria criteria) { var key = criteria.key(); // compatible with old field selector if ("name".equals(key)) { @@ -19,16 +21,15 @@ public SelectorMatcher convert(@NonNull SelectorCriteria criteria) { } switch (criteria.operator()) { case Equals -> { - return EqualityMatcher.equal(key, getSingleValue(criteria)); + return QueryFactory.equal(key, getSingleValue(criteria)); } case NotEquals -> { - return EqualityMatcher.notEqual(key, getSingleValue(criteria)); + return QueryFactory.notEqual(key, getSingleValue(criteria)); } // compatible with old field selector case IN -> { - var valueArr = - defaultIfNull(criteria.values(), Set.of()).toArray(new String[0]); - return SetMatcher.in(key, valueArr); + var valueArr = defaultIfNull(criteria.values(), Set.of()); + return QueryFactory.in(key, valueArr); } default -> throw new IllegalArgumentException( "Unsupported operator: " + criteria.operator()); diff --git a/api/src/main/java/run/halo/app/extension/router/selector/LogicalMatcher.java b/api/src/main/java/run/halo/app/extension/router/selector/LogicalMatcher.java deleted file mode 100644 index 62ee67419d4..00000000000 --- a/api/src/main/java/run/halo/app/extension/router/selector/LogicalMatcher.java +++ /dev/null @@ -1,9 +0,0 @@ -package run.halo.app.extension.router.selector; - -public abstract class LogicalMatcher implements SelectorMatcher { - - @Override - public String getKey() { - throw new UnsupportedOperationException(); - } -} diff --git a/api/src/main/java/run/halo/app/extension/router/selector/OrSelectorMatcher.java b/api/src/main/java/run/halo/app/extension/router/selector/OrSelectorMatcher.java deleted file mode 100644 index 0fd41bb56ac..00000000000 --- a/api/src/main/java/run/halo/app/extension/router/selector/OrSelectorMatcher.java +++ /dev/null @@ -1,24 +0,0 @@ -package run.halo.app.extension.router.selector; - -import lombok.Getter; - -@Getter -public class OrSelectorMatcher extends LogicalMatcher { - private final SelectorMatcher left; - private final SelectorMatcher right; - - public OrSelectorMatcher(SelectorMatcher left, SelectorMatcher right) { - this.left = left; - this.right = right; - } - - @Override - public String getKey() { - return null; - } - - @Override - public boolean test(String s) { - return left.test(s) || right.test(s); - } -} diff --git a/api/src/main/java/run/halo/app/extension/router/selector/SelectorMatcher.java b/api/src/main/java/run/halo/app/extension/router/selector/SelectorMatcher.java index 88eb63b740c..3e4110fd7a5 100644 --- a/api/src/main/java/run/halo/app/extension/router/selector/SelectorMatcher.java +++ b/api/src/main/java/run/halo/app/extension/router/selector/SelectorMatcher.java @@ -5,9 +5,9 @@ public interface SelectorMatcher { String getKey(); /** - * Returns true if a label value matches. + * Returns true if a field value matches. * - * @param s the label value + * @param s the field value * @return the boolean */ boolean test(String s); diff --git a/api/src/main/java/run/halo/app/extension/router/selector/SelectorUtil.java b/api/src/main/java/run/halo/app/extension/router/selector/SelectorUtil.java index f1ba20106fd..bad729cbc24 100644 --- a/api/src/main/java/run/halo/app/extension/router/selector/SelectorUtil.java +++ b/api/src/main/java/run/halo/app/extension/router/selector/SelectorUtil.java @@ -8,6 +8,7 @@ import org.springframework.web.server.ServerWebInputException; import run.halo.app.extension.Extension; import run.halo.app.extension.ListOptions; +import run.halo.app.extension.index.query.QueryFactory; public final class SelectorUtil { @@ -83,19 +84,19 @@ public static ListOptions labelAndFieldSelectorToListOptions( .orElse(List.of()); var fieldConverter = new FieldSelectorConverter(); - var fieldMatchers = Optional.ofNullable(fieldSelectorTerms) + var fieldQuery = Optional.ofNullable(fieldSelectorTerms) .map(selectors -> selectors.stream() .map(selectorConverter::convert) .filter(Objects::nonNull) .map(fieldConverter::convert) .toList() ) - .orElse(List.of()) - .stream() - .reduce(AndSelectorMatcher::new); - - return new ListOptions() - .setLabelSelector(new LabelSelector().setMatchers(labelMatchers)) - .setFieldSelector(new FieldSelector().setMatcher(fieldMatchers.orElse(null))); + .orElse(List.of()); + var listOptions = new ListOptions(); + listOptions.setLabelSelector(new LabelSelector().setMatchers(labelMatchers)); + if (!fieldQuery.isEmpty()) { + listOptions.setFieldSelector(FieldSelector.of(QueryFactory.and(fieldQuery))); + } + return listOptions; } } diff --git a/api/src/test/java/run/halo/app/extension/controller/ControllerBuilderTest.java b/api/src/test/java/run/halo/app/extension/controller/ControllerBuilderTest.java index 419cc399255..63dc39d6274 100644 --- a/api/src/test/java/run/halo/app/extension/controller/ControllerBuilderTest.java +++ b/api/src/test/java/run/halo/app/extension/controller/ControllerBuilderTest.java @@ -2,15 +2,18 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.lenient; import java.time.Duration; import java.time.Instant; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import run.halo.app.extension.ExtensionClient; import run.halo.app.extension.FakeExtension; +import run.halo.app.extension.index.IndexedQueryEngine; @ExtendWith(MockitoExtension.class) class ControllerBuilderTest { @@ -18,6 +21,14 @@ class ControllerBuilderTest { @Mock ExtensionClient client; + @Mock + IndexedQueryEngine indexedQueryEngine; + + @BeforeEach + void setUp() { + lenient().when(client.indexedQueryEngine()).thenReturn(indexedQueryEngine); + } + @Test void buildWithNullReconciler() { assertThrows(IllegalArgumentException.class, diff --git a/api/src/test/java/run/halo/app/extension/controller/ExtensionWatcherTest.java b/api/src/test/java/run/halo/app/extension/controller/ExtensionWatcherTest.java index 889fb3d9335..ec5cbec211a 100644 --- a/api/src/test/java/run/halo/app/extension/controller/ExtensionWatcherTest.java +++ b/api/src/test/java/run/halo/app/extension/controller/ExtensionWatcherTest.java @@ -14,6 +14,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import run.halo.app.extension.DefaultExtensionMatcher; +import run.halo.app.extension.ExtensionClient; import run.halo.app.extension.FakeExtension; import run.halo.app.extension.GroupVersionKind; import run.halo.app.extension.WatcherExtensionMatchers; @@ -25,14 +26,18 @@ class ExtensionWatcherTest { @Mock RequestQueue queue; + @Mock + ExtensionClient client; + @Mock WatcherExtensionMatchers matchers; @InjectMocks ExtensionWatcher watcher; - private static DefaultExtensionMatcher getEmptyMatcher() { - return DefaultExtensionMatcher.builder(GroupVersionKind.fromExtension(FakeExtension.class)) + private DefaultExtensionMatcher getEmptyMatcher() { + return DefaultExtensionMatcher.builder(client, + GroupVersionKind.fromExtension(FakeExtension.class)) .build(); } @@ -50,7 +55,8 @@ void shouldAddExtensionWhenAddPredicateAlwaysTrue() { @Test void shouldNotAddExtensionWhenAddPredicateAlwaysFalse() { var type = GroupVersionKind.fromAPIVersionAndKind("v1alpha1", "User"); - when(matchers.onAddMatcher()).thenReturn(DefaultExtensionMatcher.builder(type).build()); + when(matchers.onAddMatcher()).thenReturn( + DefaultExtensionMatcher.builder(client, type).build()); watcher.onAdd(createFake("fake-name")); verify(matchers, times(1)).onAddMatcher(); @@ -82,7 +88,8 @@ void shouldUpdateExtensionWhenUpdatePredicateAlwaysTrue() { @Test void shouldUpdateExtensionWhenUpdatePredicateAlwaysFalse() { var type = GroupVersionKind.fromAPIVersionAndKind("v1alpha1", "User"); - when(matchers.onUpdateMatcher()).thenReturn(DefaultExtensionMatcher.builder(type).build()); + when(matchers.onUpdateMatcher()).thenReturn( + DefaultExtensionMatcher.builder(client, type).build()); watcher.onUpdate(createFake("old-fake-name"), createFake("new-fake-name")); verify(matchers, times(1)).onUpdateMatcher(); @@ -114,7 +121,8 @@ void shouldDeleteExtensionWhenDeletePredicateAlwaysTrue() { @Test void shouldDeleteExtensionWhenDeletePredicateAlwaysFalse() { var type = GroupVersionKind.fromAPIVersionAndKind("v1alpha1", "User"); - when(matchers.onDeleteMatcher()).thenReturn(DefaultExtensionMatcher.builder(type).build()); + when(matchers.onDeleteMatcher()).thenReturn( + DefaultExtensionMatcher.builder(client, type).build()); watcher.onDelete(createFake("fake-name")); verify(matchers, times(1)).onDeleteMatcher(); diff --git a/api/src/test/java/run/halo/app/extension/index/query/AndTest.java b/api/src/test/java/run/halo/app/extension/index/query/AndTest.java new file mode 100644 index 00000000000..f7e7816371c --- /dev/null +++ b/api/src/test/java/run/halo/app/extension/index/query/AndTest.java @@ -0,0 +1,93 @@ +package run.halo.app.extension.index.query; + +import static org.assertj.core.api.Assertions.assertThat; +import static run.halo.app.extension.index.query.QueryFactory.and; +import static run.halo.app.extension.index.query.QueryFactory.equal; +import static run.halo.app.extension.index.query.QueryFactory.greaterThan; +import static run.halo.app.extension.index.query.QueryFactory.or; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +/** + * Tests for the {@link And} query. + * + * @author guqing + * @since 2.12.0 + */ +public class AndTest { + + @Test + void testMatches() { + Collection> deptEntry = List.of(Map.entry("A", "guqing"), + Map.entry("A", "halo"), + Map.entry("B", "lisi"), + Map.entry("B", "zhangsan"), + Map.entry("C", "ryanwang"), + Map.entry("C", "johnniang") + ); + Collection> ageEntry = List.of(Map.entry("19", "halo"), + Map.entry("19", "guqing"), + Map.entry("18", "zhangsan"), + Map.entry("17", "lisi"), + Map.entry("17", "ryanwang"), + Map.entry("17", "johnniang") + ); + var entries = Map.of("dept", deptEntry, "age", ageEntry); + var indexView = new QueryIndexViewImpl(entries); + + var query = and(equal("dept", "B"), equal("age", "18")); + var resultSet = query.matches(indexView); + assertThat(resultSet).containsExactly("zhangsan"); + + query = and(equal("dept", "C"), equal("age", "18")); + resultSet = query.matches(indexView); + assertThat(resultSet).isEmpty(); + + query = and( + // guqing, halo, lisi, zhangsan + or(equal("dept", "A"), equal("dept", "B")), + // guqing, halo, zhangsan + or(equal("age", "19"), equal("age", "18")) + ); + resultSet = query.matches(indexView); + assertThat(resultSet).containsExactlyInAnyOrder("guqing", "halo", "zhangsan"); + + query = and( + // guqing, halo, lisi, zhangsan + or(equal("dept", "A"), equal("dept", "B")), + // guqing, halo, zhangsan + or(equal("age", "19"), equal("age", "18")) + ); + resultSet = query.matches(indexView); + assertThat(resultSet).containsExactlyInAnyOrder("guqing", "halo", "zhangsan"); + + query = and( + // guqing, halo, lisi, zhangsan + or(equal("dept", "A"), equal("dept", "C")), + // guqing, halo, zhangsan + and(equal("age", "17"), equal("age", "17")) + ); + resultSet = query.matches(indexView); + assertThat(resultSet).containsExactlyInAnyOrder("ryanwang", "johnniang"); + } + + @Test + void andMatch2() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var query = and(equal("lastName", "Fay"), + and( + equal("hireDate", "17"), + and(greaterThan("salary", "1000"), + and(equal("managerId", "101"), + equal("departmentId", "50") + ) + ) + ) + ); + var resultSet = query.matches(indexView); + assertThat(resultSet).containsExactly("100"); + } +} diff --git a/api/src/test/java/run/halo/app/extension/index/query/EmployeeDataSet.java b/api/src/test/java/run/halo/app/extension/index/query/EmployeeDataSet.java new file mode 100644 index 00000000000..bd2b205e38a --- /dev/null +++ b/api/src/test/java/run/halo/app/extension/index/query/EmployeeDataSet.java @@ -0,0 +1,98 @@ +package run.halo.app.extension.index.query; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +public class EmployeeDataSet { + + /** + * Create a {@link QueryIndexView} for employee to test. + * + * @return a {@link QueryIndexView} for employee to test + */ + public static QueryIndexView createEmployeeIndexView() { + /* + * id firstName lastName email hireDate salary managerId departmentId + * 100 Pat Fay p 17 2600 101 50 + * 101 Lee Day l 17 2400 102 40 + * 102 William Jay w 19 2200 102 50 + * 103 Mary Day p 17 2000 103 50 + * 104 John Fay j 17 1800 103 50 + * 105 Gon Fay p 18 1900 101 40 + */ + Collection> idEntry = List.of( + Map.entry("100", "100"), + Map.entry("101", "101"), + Map.entry("102", "102"), + Map.entry("103", "103"), + Map.entry("104", "104"), + Map.entry("105", "105") + ); + Collection> firstNameEntry = List.of( + Map.entry("Pat", "100"), + Map.entry("Lee", "101"), + Map.entry("William", "102"), + Map.entry("Mary", "103"), + Map.entry("John", "104"), + Map.entry("Gon", "105") + ); + Collection> lastNameEntry = List.of( + Map.entry("Fay", "100"), + Map.entry("Day", "101"), + Map.entry("Jay", "102"), + Map.entry("Day", "103"), + Map.entry("Fay", "104"), + Map.entry("Fay", "105") + ); + Collection> emailEntry = List.of( + Map.entry("p", "100"), + Map.entry("l", "101"), + Map.entry("w", "102"), + Map.entry("p", "103"), + Map.entry("j", "104"), + Map.entry("p", "105") + ); + Collection> hireDateEntry = List.of( + Map.entry("17", "100"), + Map.entry("17", "101"), + Map.entry("19", "102"), + Map.entry("17", "103"), + Map.entry("17", "104"), + Map.entry("18", "105") + ); + Collection> salaryEntry = List.of( + Map.entry("2600", "100"), + Map.entry("2400", "101"), + Map.entry("2200", "102"), + Map.entry("2000", "103"), + Map.entry("1800", "104"), + Map.entry("1900", "105") + ); + Collection> managerIdEntry = List.of( + Map.entry("101", "100"), + Map.entry("102", "101"), + Map.entry("102", "102"), + Map.entry("103", "103"), + Map.entry("103", "104"), + Map.entry("101", "105") + ); + Collection> departmentIdEntry = List.of( + Map.entry("50", "100"), + Map.entry("40", "101"), + Map.entry("50", "102"), + Map.entry("50", "103"), + Map.entry("50", "104"), + Map.entry("40", "105") + ); + var entries = Map.of("id", idEntry, + "firstName", firstNameEntry, + "lastName", lastNameEntry, + "email", emailEntry, + "hireDate", hireDateEntry, + "salary", salaryEntry, + "managerId", managerIdEntry, + "departmentId", departmentIdEntry); + return new QueryIndexViewImpl(entries); + } +} diff --git a/api/src/test/java/run/halo/app/extension/index/query/QueryFactoryTest.java b/api/src/test/java/run/halo/app/extension/index/query/QueryFactoryTest.java new file mode 100644 index 00000000000..e2b101424c4 --- /dev/null +++ b/api/src/test/java/run/halo/app/extension/index/query/QueryFactoryTest.java @@ -0,0 +1,221 @@ +package run.halo.app.extension.index.query; + +import static org.assertj.core.api.Assertions.assertThat; +import static run.halo.app.extension.index.query.QueryFactory.all; +import static run.halo.app.extension.index.query.QueryFactory.between; +import static run.halo.app.extension.index.query.QueryFactory.equal; +import static run.halo.app.extension.index.query.QueryFactory.equalOtherField; +import static run.halo.app.extension.index.query.QueryFactory.notEqual; +import static run.halo.app.extension.index.query.QueryFactory.notEqualOtherField; + +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link QueryFactory}. + * + * @author guqing + * @since 2.12.0 + */ +class QueryFactoryTest { + + + @Test + void allTest() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var resultSet = all("firstName").matches(indexView); + assertThat(resultSet).containsExactlyInAnyOrder( + "100", "101", "102", "103", "104", "105" + ); + } + + @Test + void equalTest() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var resultSet = equal("lastName", "Fay").matches(indexView); + assertThat(resultSet).containsExactlyInAnyOrder( + "100", "104", "105" + ); + } + + @Test + void equalOtherFieldTest() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var resultSet = equalOtherField("managerId", "id").matches(indexView); + assertThat(resultSet).containsExactlyInAnyOrder( + "102", "103" + ); + } + + @Test + void notEqualTest() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var resultSet = notEqual("lastName", "Fay").matches(indexView); + assertThat(resultSet).containsExactlyInAnyOrder( + "101", "102", "103" + ); + } + + @Test + void notEqualOtherFieldTest() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var resultSet = notEqualOtherField("managerId", "id").matches(indexView); + // 103 102 is equal + assertThat(resultSet).containsExactlyInAnyOrder( + "100", "101", "104", "105" + ); + } + + @Test + void lessThanTest() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var resultSet = QueryFactory.lessThan("id", "103").matches(indexView); + assertThat(resultSet).containsExactlyInAnyOrder( + "100", "101", "102" + ); + } + + @Test + void lessThanOtherFieldTest() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var resultSet = QueryFactory.lessThanOtherField("id", "managerId").matches(indexView); + assertThat(resultSet).containsExactlyInAnyOrder( + "100", "101" + ); + } + + @Test + void lessThanOrEqualTest() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var resultSet = QueryFactory.lessThanOrEqual("id", "103").matches(indexView); + assertThat(resultSet).containsExactlyInAnyOrder( + "100", "101", "102", "103" + ); + } + + @Test + void lessThanOrEqualOtherFieldTest() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var resultSet = + QueryFactory.lessThanOrEqualOtherField("id", "managerId").matches(indexView); + assertThat(resultSet).containsExactlyInAnyOrder( + "100", "101", "102", "103" + ); + } + + @Test + void greaterThanTest() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var resultSet = QueryFactory.greaterThan("id", "103").matches(indexView); + assertThat(resultSet).containsExactlyInAnyOrder( + "104", "105" + ); + } + + @Test + void greaterThanOtherFieldTest() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var resultSet = QueryFactory.greaterThanOtherField("id", "managerId").matches(indexView); + assertThat(resultSet).containsExactlyInAnyOrder( + "104", "105" + ); + } + + @Test + void greaterThanOrEqualTest() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var resultSet = QueryFactory.greaterThanOrEqual("id", "103").matches(indexView); + assertThat(resultSet).containsExactlyInAnyOrder( + "103", "104", "105" + ); + } + + @Test + void greaterThanOrEqualOtherFieldTest() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var resultSet = + QueryFactory.greaterThanOrEqualOtherField("id", "managerId").matches(indexView); + assertThat(resultSet).containsExactlyInAnyOrder( + "102", "103", "104", "105" + ); + } + + @Test + void inTest() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var resultSet = QueryFactory.in("id", "103", "104").matches(indexView); + assertThat(resultSet).containsExactlyInAnyOrder( + "103", "104" + ); + } + + @Test + void inTest2() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var resultSet = QueryFactory.in("lastName", "Fay").matches(indexView); + assertThat(resultSet).containsExactlyInAnyOrder( + "100", "104", "105" + ); + } + + @Test + void betweenTest() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var resultSet = between("id", "103", "105").matches(indexView); + assertThat(resultSet).containsExactlyInAnyOrder( + "103", "104", "105" + ); + + indexView = EmployeeDataSet.createEmployeeIndexView(); + resultSet = between("salary", "2000", "2400").matches(indexView); + assertThat(resultSet).containsExactlyInAnyOrder( + "101", "102", "103" + ); + } + + @Test + void betweenLowerExclusive() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var resultSet = + QueryFactory.betweenLowerExclusive("salary", "2000", "2400").matches(indexView); + assertThat(resultSet).containsExactlyInAnyOrder( + "101", "102" + ); + } + + @Test + void betweenUpperExclusive() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var resultSet = + QueryFactory.betweenUpperExclusive("salary", "2000", "2400").matches(indexView); + assertThat(resultSet).containsExactlyInAnyOrder( + "102", "103" + ); + } + + @Test + void betweenExclusive() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var resultSet = QueryFactory.betweenExclusive("salary", "2000", "2400").matches(indexView); + assertThat(resultSet).containsExactlyInAnyOrder( + "102" + ); + } + + @Test + void startsWithTest() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var resultSet = QueryFactory.startsWith("firstName", "W").matches(indexView); + assertThat(resultSet).containsExactlyInAnyOrder( + "102" + ); + } + + @Test + void endsWithTest() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var resultSet = QueryFactory.endsWith("firstName", "y").matches(indexView); + assertThat(resultSet).containsExactlyInAnyOrder( + "103" + ); + } +} \ No newline at end of file diff --git a/api/src/test/java/run/halo/app/extension/index/query/QueryIndexViewImplTest.java b/api/src/test/java/run/halo/app/extension/index/query/QueryIndexViewImplTest.java new file mode 100644 index 00000000000..61d998966b4 --- /dev/null +++ b/api/src/test/java/run/halo/app/extension/index/query/QueryIndexViewImplTest.java @@ -0,0 +1,83 @@ +package run.halo.app.extension.index.query; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link QueryIndexViewImpl}. + * + * @author guqing + * @since 2.12.0 + */ +class QueryIndexViewImplTest { + + @Test + void findIdsForFieldValueEqualTest() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var resultSet = indexView.findIdsForFieldValueEqual("managerId", "id"); + assertThat(resultSet).containsExactlyInAnyOrder( + "102", "103" + ); + } + + @Test + void findIdsForFieldValueGreaterThanTest() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var resultSet = indexView.findIdsForFieldValueGreaterThan("id", "managerId", false); + assertThat(resultSet).containsExactlyInAnyOrder( + "104", "105" + ); + + indexView = EmployeeDataSet.createEmployeeIndexView(); + resultSet = indexView.findIdsForFieldValueGreaterThan("id", "managerId", true); + assertThat(resultSet).containsExactlyInAnyOrder( + "103", "102", "104", "105" + ); + } + + @Test + void findIdsForFieldValueGreaterThanTest2() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var resultSet = indexView.findIdsForFieldValueGreaterThan("managerId", "id", false); + assertThat(resultSet).containsExactlyInAnyOrder( + "100", "101" + ); + + indexView = EmployeeDataSet.createEmployeeIndexView(); + resultSet = indexView.findIdsForFieldValueGreaterThan("managerId", "id", true); + assertThat(resultSet).containsExactlyInAnyOrder( + "100", "101", "102", "103" + ); + } + + @Test + void findIdsForFieldValueLessThanTest() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var resultSet = indexView.findIdsForFieldValueLessThan("id", "managerId", false); + assertThat(resultSet).containsExactlyInAnyOrder( + "100", "101" + ); + + indexView = EmployeeDataSet.createEmployeeIndexView(); + resultSet = indexView.findIdsForFieldValueLessThan("id", "managerId", true); + assertThat(resultSet).containsExactlyInAnyOrder( + "100", "101", "102", "103" + ); + } + + @Test + void findIdsForFieldValueLessThanTest2() { + var indexView = EmployeeDataSet.createEmployeeIndexView(); + var resultSet = indexView.findIdsForFieldValueLessThan("managerId", "id", false); + assertThat(resultSet).containsExactlyInAnyOrder( + "104", "105" + ); + + indexView = EmployeeDataSet.createEmployeeIndexView(); + resultSet = indexView.findIdsForFieldValueLessThan("managerId", "id", true); + assertThat(resultSet).containsExactlyInAnyOrder( + "103", "102", "104", "105" + ); + } +} diff --git a/api/src/test/java/run/halo/app/extension/router/selector/FieldSelectorTest.java b/api/src/test/java/run/halo/app/extension/router/selector/FieldSelectorTest.java deleted file mode 100644 index 2f209fcecb4..00000000000 --- a/api/src/test/java/run/halo/app/extension/router/selector/FieldSelectorTest.java +++ /dev/null @@ -1,104 +0,0 @@ -package run.halo.app.extension.router.selector; - -import static org.assertj.core.api.Assertions.assertThat; - -import java.util.Map; -import org.junit.jupiter.api.Test; - -/** - * Tests for {@link FieldSelector}. - * - * @author guqing - * @since 2.12.0 - */ -class FieldSelectorTest { - - @Test - void testEq() { - var fieldSelector = FieldSelector.builder() - .eq("name", "guqing").build(); - assertThat(fieldSelector.test(key -> "guqing")).isTrue(); - assertThat(fieldSelector.test(key -> "halo")).isFalse(); - } - - @Test - void testNotEq() { - var fieldSelector = FieldSelector.builder() - .notEq("name", "guqing").build(); - assertThat(fieldSelector.test(key -> "guqing")).isFalse(); - assertThat(fieldSelector.test(key -> "halo")).isTrue(); - } - - @Test - void testIn() { - var fieldSelector = FieldSelector.builder() - .in("name", "guqing", "guqing1").build(); - assertThat(fieldSelector.test(key -> "guqing")).isTrue(); - assertThat(fieldSelector.test(key -> "halo")).isFalse(); - assertThat(fieldSelector.test(key -> "blog")).isFalse(); - } - - @Test - void testNotIn() { - var fieldSelector = FieldSelector.builder() - .notIn("name", "guqing", "guqing1").build(); - assertThat(fieldSelector.test(key -> "guqing")).isFalse(); - assertThat(fieldSelector.test(key -> "halo")).isTrue(); - assertThat(fieldSelector.test(key -> "blog")).isTrue(); - } - - @Test - void testExists() { - var fieldSelector = FieldSelector.builder() - .exists("name").build(); - assertThat(fieldSelector.test(key -> "guqing")).isTrue(); - assertThat(fieldSelector.test(key -> "halo")).isTrue(); - assertThat(fieldSelector.test(key -> "blog")).isTrue(); - } - - @Test - void testNotExists() { - var fieldSelector = FieldSelector.builder() - .notExists("name").build(); - assertThat(fieldSelector.test(key -> "guqing")).isFalse(); - assertThat(fieldSelector.test(key -> "halo")).isFalse(); - assertThat(fieldSelector.test(key -> "blog")).isFalse(); - } - - @Test - void testAnd() { - var fieldSelector = FieldSelector.builder() - .eq("name", "guqing") - .and(FieldSelector.builder().eq("age", "18").build().getMatcher()) - .build(); - assertThat(fieldSelector.test(key -> "guqing")).isFalse(); - assertThat(fieldSelector.test(key -> "halo")).isFalse(); - assertThat(fieldSelector.test(key -> "18")).isFalse(); - assertThat(fieldSelector.test(key -> "guqing18")).isFalse(); - assertThat(fieldSelector.test(key -> { - var map = Map.of("name", "guqing", "age", "18"); - return map.get(key); - })).isTrue(); - assertThat(fieldSelector.test(key -> { - var map = Map.of("name", "guqing", "age", "19"); - return map.get(key); - })).isFalse(); - } - - @Test - void testOr() { - var fieldSelector = FieldSelector.builder() - .eq("name", "guqing") - .or(FieldSelector.builder().eq("age", "18").build().getMatcher()) - .build(); - assertThat(fieldSelector.test(key -> "guqing")).isTrue(); - assertThat(fieldSelector.test(key -> "halo")).isFalse(); - assertThat(fieldSelector.test(key -> "blog")).isFalse(); - assertThat(fieldSelector.test(key -> "18")).isTrue(); - assertThat(fieldSelector.test(key -> "guqing18")).isFalse(); - assertThat(fieldSelector.test(key -> { - var map = Map.of("name", "guqing", "age", "18"); - return map.get(key); - })).isTrue(); - } -} \ No newline at end of file diff --git a/application/src/main/java/run/halo/app/extension/gc/GcSynchronizer.java b/application/src/main/java/run/halo/app/extension/gc/GcSynchronizer.java index 86173348364..bea81ae0a4d 100644 --- a/application/src/main/java/run/halo/app/extension/gc/GcSynchronizer.java +++ b/application/src/main/java/run/halo/app/extension/gc/GcSynchronizer.java @@ -15,6 +15,7 @@ import run.halo.app.extension.Watcher; import run.halo.app.extension.controller.RequestQueue; import run.halo.app.extension.controller.Synchronizer; +import run.halo.app.extension.index.query.QueryFactory; import run.halo.app.extension.router.selector.FieldSelector; class GcSynchronizer implements Synchronizer { @@ -75,9 +76,8 @@ public void start() { List listDeleted(Class type) { var options = new ListOptions() - .setFieldSelector(FieldSelector.builder() - .notEq("metadata.deletionTimestamp", null) - .build() + .setFieldSelector( + FieldSelector.of(QueryFactory.all("metadata.deletionTimestamp")) ); return client.listAll(type, options, Sort.by("metadata.creationTimestamp")) .stream() diff --git a/application/src/main/java/run/halo/app/extension/index/IndexEntry.java b/application/src/main/java/run/halo/app/extension/index/IndexEntry.java index 076bcd54adf..ececd000a3b 100644 --- a/application/src/main/java/run/halo/app/extension/index/IndexEntry.java +++ b/application/src/main/java/run/halo/app/extension/index/IndexEntry.java @@ -33,6 +33,18 @@ */ public interface IndexEntry { + /** + * Acquires the read lock for reading such as {@link #getByIndexKey(String)}, + * {@link #entries()}, {@link #indexedKeys()}, because the returned result set of these + * methods is not immutable. + */ + void acquireReadLock(); + + /** + * Releases the read lock. + */ + void releaseReadLock(); + /** *

Adds a new entry to this index entry.

*

For example, if we have a {@link Metadata} with name {@code foo} and labels {@code bar=1} @@ -78,12 +90,22 @@ public interface IndexEntry { Set indexedKeys(); /** - * Returns the entries of this entry in order. + *

Returns the entries of this entry in order.

+ *

Note That: Any modification to the returned result will affect the original data + * directly.

* * @return entries of this entry. */ Collection> entries(); + /** + * Returns the immutable entries of this entry in order, it is safe to modify the returned + * result, but extra cost is made. + * + * @return immutable entries of this entry. + */ + Collection> immutableEntries(); + /** * Returns the object names of this entry in order. * diff --git a/application/src/main/java/run/halo/app/extension/index/IndexEntryImpl.java b/application/src/main/java/run/halo/app/extension/index/IndexEntryImpl.java index c89c5826298..b7ea812a17b 100644 --- a/application/src/main/java/run/halo/app/extension/index/IndexEntryImpl.java +++ b/application/src/main/java/run/halo/app/extension/index/IndexEntryImpl.java @@ -1,10 +1,10 @@ package run.halo.app.extension.index; +import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ListMultimap; import com.google.common.collect.MultimapBuilder; import java.util.Collection; import java.util.Comparator; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -45,6 +45,16 @@ Comparator keyComparator() { } } + @Override + public void acquireReadLock() { + this.rwl.readLock().lock(); + } + + @Override + public void releaseReadLock() { + this.rwl.readLock().unlock(); + } + @Override public void addEntry(List keys, String objectName) { var isUnique = indexDescriptor.getSpec().isUnique(); @@ -91,7 +101,7 @@ public void remove(String objectName) { public Set indexedKeys() { readLock.lock(); try { - return new LinkedHashSet<>(indexKeyObjectNamesMap.keySet()); + return indexKeyObjectNamesMap.keySet(); } finally { readLock.unlock(); } @@ -101,7 +111,17 @@ public Set indexedKeys() { public Collection> entries() { readLock.lock(); try { - return List.copyOf(indexKeyObjectNamesMap.entries()); + return indexKeyObjectNamesMap.entries(); + } finally { + readLock.unlock(); + } + } + + @Override + public Collection> immutableEntries() { + readLock.lock(); + try { + return ImmutableListMultimap.copyOf(indexKeyObjectNamesMap).entries(); } finally { readLock.unlock(); } @@ -111,7 +131,7 @@ public Collection> entries() { public List getByIndexKey(String indexKey) { readLock.lock(); try { - return List.copyOf(indexKeyObjectNamesMap.get(indexKey)); + return indexKeyObjectNamesMap.get(indexKey); } finally { readLock.unlock(); } diff --git a/application/src/main/java/run/halo/app/extension/index/IndexedQueryEngineImpl.java b/application/src/main/java/run/halo/app/extension/index/IndexedQueryEngineImpl.java index 467b728bfff..5dd092e97a0 100644 --- a/application/src/main/java/run/halo/app/extension/index/IndexedQueryEngineImpl.java +++ b/application/src/main/java/run/halo/app/extension/index/IndexedQueryEngineImpl.java @@ -1,14 +1,14 @@ package run.halo.app.extension.index; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.PriorityQueue; import java.util.Set; import java.util.stream.Collectors; @@ -23,10 +23,10 @@ import run.halo.app.extension.ListOptions; import run.halo.app.extension.ListResult; import run.halo.app.extension.PageRequest; -import run.halo.app.extension.router.selector.AndSelectorMatcher; -import run.halo.app.extension.router.selector.AnySelectorMatcher; -import run.halo.app.extension.router.selector.LogicalMatcher; -import run.halo.app.extension.router.selector.OrSelectorMatcher; +import run.halo.app.extension.index.query.Query; +import run.halo.app.extension.index.query.QueryIndexViewImpl; +import run.halo.app.extension.router.selector.FieldSelector; +import run.halo.app.extension.router.selector.LabelSelector; import run.halo.app.extension.router.selector.SelectorMatcher; /** @@ -44,7 +44,7 @@ public class IndexedQueryEngineImpl implements IndexedQueryEngine { private static Map fieldPathIndexEntryMap(Indexer indexer) { // O(n) time complexity - Map indexEntryMap = new LinkedHashMap<>(); + Map indexEntryMap = new HashMap<>(); var iterator = indexer.readyIndexesIterator(); while (iterator.hasNext()) { var indexEntry = iterator.next(); @@ -89,12 +89,6 @@ static List intersection(List list1, List list2) { return intersection; } - static List union(List list1, List list2) { - Set set = new LinkedHashSet<>(list1); - set.addAll(list2); - return new ArrayList<>(set); - } - static void throwNotIndexedException(String fieldPath) { throw new IllegalArgumentException( "No index found for fieldPath: " + fieldPath @@ -106,19 +100,25 @@ List retrieveForLabelMatchers(List labelMatchers, var indexEntry = getIndexEntry(LabelIndexSpecUtils.LABEL_PATH, fieldPathEntryMap); // O(m) time complexity, m is the number of labelMatchers var labelKeysToQuery = labelMatchers.stream() + .sorted(Comparator.comparing(SelectorMatcher::getKey)) .map(SelectorMatcher::getKey) .collect(Collectors.toSet()); - Map> objectNameLabelsMap = new LinkedHashMap<>(); - indexEntry.entries().forEach(entry -> { - // key is labelKey=labelValue, value is objectName - var labelPair = LabelIndexSpecUtils.labelKeyValuePair(entry.getKey()); - if (!labelKeysToQuery.contains(labelPair.getFirst())) { - return; - } - objectNameLabelsMap.computeIfAbsent(entry.getValue(), k -> new LinkedHashMap<>()) - .put(labelPair.getFirst(), labelPair.getSecond()); - }); + Map> objectNameLabelsMap = new HashMap<>(); + indexEntry.acquireReadLock(); + try { + indexEntry.entries().forEach(entry -> { + // key is labelKey=labelValue, value is objectName + var labelPair = LabelIndexSpecUtils.labelKeyValuePair(entry.getKey()); + if (!labelKeysToQuery.contains(labelPair.getFirst())) { + return; + } + objectNameLabelsMap.computeIfAbsent(entry.getValue(), k -> new HashMap<>()) + .put(labelPair.getFirst(), labelPair.getSecond()); + }); + } finally { + indexEntry.releaseReadLock(); + } // O(p * m) time complexity, p is the number of allMetadataNames return allMetadataNames.stream() .filter(objectName -> { @@ -138,44 +138,38 @@ List doRetrieve(Indexer indexer, ListOptions options, Sort sort) { var primaryEntry = getIndexEntry(PrimaryKeySpecUtils.PRIMARY_INDEX_NAME, fieldPathEntryMap); // O(n) time complexity stopWatch.start("retrieve all metadata names"); - var allMetadataNames = new ArrayList<>(primaryEntry.indexedKeys()); + var allMetadataNames = new ArrayList(); + primaryEntry.acquireReadLock(); + try { + allMetadataNames.addAll(primaryEntry.indexedKeys()); + } finally { + primaryEntry.releaseReadLock(); + } stopWatch.stop(); stopWatch.start("retrieve matched metadata names"); - final Optional> matchedByLabels = - Optional.ofNullable(options.getLabelSelector()) - .filter(selector -> !CollectionUtils.isEmpty(selector.getMatchers())) - .map(labelSelector -> labelSelector.getMatchers() - .stream() - .sorted(Comparator.comparing(SelectorMatcher::getKey)) - .toList() - ) - .map(matchers -> retrieveForLabelMatchers(matchers, - fieldPathEntryMap, allMetadataNames) - ); + var hasLabelSelector = hasLabelSelector(options.getLabelSelector()); + final List matchedByLabels = hasLabelSelector + ? retrieveForLabelMatchers(options.getLabelSelector().getMatchers(), fieldPathEntryMap, + allMetadataNames) + : allMetadataNames; stopWatch.stop(); stopWatch.start("retrieve matched metadata names by fields"); - Optional> matchedByFields = Optional.ofNullable(options.getFieldSelector()) - .filter(fieldSelector -> fieldSelector.getMatcher() != null) - .map(fieldSelector -> { - var values = evaluate(fieldSelector.getMatcher(), fieldPathEntryMap, - allMetadataNames); - var uniqueValues = new LinkedHashSet<>(values); - return new ArrayList<>(uniqueValues); - }); + final var hasFieldSelector = hasFieldSelector(options.getFieldSelector()); + var matchedByFields = hasFieldSelector + ? retrieveForFieldSelector(options.getFieldSelector().query(), fieldPathEntryMap) + : allMetadataNames; stopWatch.stop(); stopWatch.start("merge result"); List foundObjectKeys; - if (matchedByLabels.isEmpty() && matchedByFields.isEmpty()) { + if (!hasLabelSelector && !hasFieldSelector) { foundObjectKeys = allMetadataNames; - } else if (matchedByLabels.isEmpty()) { - foundObjectKeys = matchedByFields.orElse(allMetadataNames); + } else if (!hasLabelSelector) { + foundObjectKeys = matchedByFields; } else { - foundObjectKeys = matchedByFields - .map(strings -> intersection(matchedByLabels.get(), strings)) - .orElseGet(() -> matchedByLabels.orElse(allMetadataNames)); + foundObjectKeys = intersection(matchedByFields, matchedByLabels); } stopWatch.stop(); @@ -183,37 +177,30 @@ List doRetrieve(Indexer indexer, ListOptions options, Sort sort) { ResultSorter resultSorter = new ResultSorter(fieldPathEntryMap, foundObjectKeys); var result = resultSorter.sortBy(sort); stopWatch.stop(); - log.debug("Retrieve result from indexer, {}", stopWatch.prettyPrint()); + if (log.isTraceEnabled()) { + log.trace("Retrieve result from indexer, {}", stopWatch.prettyPrint()); + } return result; } - List evaluate(SelectorMatcher matcher, Map fieldPathEntryMap, - List allNames) { - if (matcher == null) { - return allNames; - } - if (matcher instanceof LogicalMatcher) { - if (matcher instanceof AndSelectorMatcher andNode) { - var left = evaluate(andNode.getLeft(), fieldPathEntryMap, allNames); - var right = evaluate(andNode.getRight(), fieldPathEntryMap, allNames); - return intersection(left, right); - } else if (matcher instanceof OrSelectorMatcher orNode) { - var left = evaluate(orNode.getLeft(), fieldPathEntryMap, allNames); - var right = evaluate(orNode.getRight(), fieldPathEntryMap, allNames); - return union(left, right); - } else if (matcher instanceof AnySelectorMatcher) { - return allNames; - } - } - var indexEntry = getIndexEntry(matcher.getKey(), fieldPathEntryMap); - var indexedKeys = indexEntry.indexedKeys(); - return indexedKeys.stream() - .filter(matcher::test) - .map(indexEntry::getByIndexKey) - .flatMap(List::stream) - .toList(); + boolean hasLabelSelector(LabelSelector labelSelector) { + return labelSelector != null && !CollectionUtils.isEmpty(labelSelector.getMatchers()); + } + + boolean hasFieldSelector(FieldSelector fieldSelector) { + return fieldSelector != null && fieldSelector.query() != null; } + List retrieveForFieldSelector(Query query, Map fieldPathEntryMap) { + Map>> indexView = new HashMap<>(); + for (Map.Entry entry : fieldPathEntryMap.entrySet()) { + indexView.put(entry.getKey(), entry.getValue().immutableEntries()); + } + // TODO optimize build indexView time + var queryIndexView = new QueryIndexViewImpl(indexView); + var resultSet = query.matches(queryIndexView); + return new ArrayList<>(resultSet); + } /** * Sort the given list by the given {@link Sort}. @@ -238,12 +225,18 @@ public List sortBy(@NonNull Sort sort) { throwNotIndexedException(order.getProperty()); } var set = new HashSet<>(list); - // 8w data about cost 11ms, maybe we can improve it better - final var objectNames = indexEntry.entries().stream() - .map(Map.Entry::getValue) - .filter(set::contains) - .collect(Collectors.toList()); - + var objectNames = new ArrayList(); + indexEntry.acquireReadLock(); + try { + for (var entry : indexEntry.entries()) { + var objectName = entry.getValue(); + if (set.contains(objectName)) { + objectNames.add(objectName); + } + } + } finally { + indexEntry.releaseReadLock(); + } var indexOrder = indexEntry.getIndexDescriptor().getSpec().getOrder(); var asc = IndexSpec.OrderType.ASC.equals(indexOrder); if (asc != order.isAscending()) { diff --git a/application/src/test/java/run/halo/app/extension/index/IndexedQueryEngineImplTest.java b/application/src/test/java/run/halo/app/extension/index/IndexedQueryEngineImplTest.java index 20bc8f99de9..7f99216d7d3 100644 --- a/application/src/test/java/run/halo/app/extension/index/IndexedQueryEngineImplTest.java +++ b/application/src/test/java/run/halo/app/extension/index/IndexedQueryEngineImplTest.java @@ -11,6 +11,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static run.halo.app.extension.index.query.QueryFactory.equal; import java.util.ArrayList; import java.util.Arrays; @@ -124,8 +125,8 @@ void doRetrieve() { .thenReturn(new IndexDescriptor(new IndexSpec() .setName("slug") .setOrder(IndexSpec.OrderType.ASC))); - when(fieldSlugEntry.indexedKeys()).thenReturn(Set.of("slug1", "slug2", "slug3")); - when(fieldSlugEntry.getByIndexKey(eq("slug1"))).thenReturn(List.of("object1")); + when((fieldSlugEntry.immutableEntries())).thenReturn( + List.of(Map.entry("slug1", "object1"), Map.entry("slug2", "object2"))); when(labelEntry.getIndexDescriptor()) .thenReturn( @@ -140,8 +141,7 @@ void doRetrieve() { var listOptions = new ListOptions(); listOptions.setLabelSelector(LabelSelector.builder() .eq("key1", "value1").build()); - listOptions.setFieldSelector(FieldSelector.builder() - .eq("slug", "slug1").build()); + listOptions.setFieldSelector(FieldSelector.of(equal("slug", "slug1"))); var result = indexedQueryEngine.doRetrieve(indexer, listOptions, Sort.unsorted()); assertThat(result).containsExactly("object1"); }