Skip to content

Commit

Permalink
refactor: using indexes to optimize queries
Browse files Browse the repository at this point in the history
  • Loading branch information
@guqing
guqing committed Jan 26, 2024
1 parent 096b1b3 commit bd0faad
Show file tree
Hide file tree
Showing 11 changed files with 227 additions and 349 deletions.
3 changes: 2 additions & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,13 @@ group 'run.halo.moments'
sourceCompatibility = JavaVersion.VERSION_17

repositories {
mavenLocal()
mavenCentral()
maven { url 'https://s01.oss.sonatype.org/content/repositories/snapshots' }
}

dependencies {
implementation platform('run.halo.tools.platform:plugin:2.11.0-SNAPSHOT')
implementation platform('run.halo.tools.platform:plugin:2.12.0-SNAPSHOT')
compileOnly 'run.halo.app:api'

testImplementation 'run.halo.app:api'
Expand Down
8 changes: 8 additions & 0 deletions src/main/java/run/halo/moments/Moment.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ public class Moment extends AbstractExtension {
@Schema(required = true)
private MomentSpec spec;

private Status status;

@Data
public static class MomentSpec {

Expand All @@ -41,6 +43,12 @@ public static class MomentSpec {
private Set<String> tags;
}

@Data
@Schema(name = "MomentStatus")
public static class Status {
private long observedVersion;
}

@Data
public static class MomentContent {

Expand Down
7 changes: 2 additions & 5 deletions src/main/java/run/halo/moments/MomentEndpoint.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
import org.springframework.web.reactive.function.server.RouterFunction;
import org.springframework.web.reactive.function.server.ServerRequest;
import org.springframework.web.reactive.function.server.ServerResponse;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import run.halo.app.core.extension.endpoint.CustomEndpoint;
import run.halo.app.extension.GroupVersion;
Expand All @@ -35,8 +34,6 @@ public class MomentEndpoint implements CustomEndpoint {

private final MomentService momentService;

private final TagMomentIndexer tagMomentIndexer;

@Override
public RouterFunction<ServerResponse> endpoint() {
final var tag = "api.plugin.halo.run/v1alpha1/Moment";
Expand Down Expand Up @@ -93,14 +90,14 @@ private Mono<ServerResponse> createMoment(ServerRequest serverRequest) {
}

private Mono<ServerResponse> listMoment(ServerRequest serverRequest) {
MomentQuery query = new MomentQuery(serverRequest.queryParams());
MomentQuery query = new MomentQuery(serverRequest.exchange());
return momentService.listMoment(query)
.flatMap(listedMoments -> ServerResponse.ok().bodyValue(listedMoments));
}

private Mono<ServerResponse> listTags(ServerRequest request) {
String name = request.queryParam("name").orElse(null);
return Flux.fromIterable(tagMomentIndexer.listAllTags())
return momentService.listAllTags()
.filter(tagName -> StringUtils.isBlank(name) || StringUtils.containsIgnoreCase(tagName,
name))
.collectList()
Expand Down
79 changes: 61 additions & 18 deletions src/main/java/run/halo/moments/MomentQuery.java
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,38 @@
package run.halo.moments;

import static run.halo.app.extension.index.query.QueryFactory.all;
import static run.halo.app.extension.index.query.QueryFactory.and;
import static run.halo.app.extension.index.query.QueryFactory.contains;
import static run.halo.app.extension.index.query.QueryFactory.equal;
import static run.halo.app.extension.index.query.QueryFactory.greaterThanOrEqual;
import static run.halo.app.extension.index.query.QueryFactory.lessThanOrEqual;
import static run.halo.app.extension.router.selector.SelectorUtil.labelAndFieldSelectorToListOptions;

import io.swagger.v3.oas.annotations.media.Schema;
import java.time.Instant;
import org.apache.commons.lang3.StringUtils;
import org.springframework.data.domain.Sort;
import org.springframework.lang.Nullable;
import org.springframework.util.MultiValueMap;
import run.halo.app.extension.router.IListRequest;
import org.springframework.web.server.ServerWebExchange;
import run.halo.app.extension.ListOptions;
import run.halo.app.extension.PageRequest;
import run.halo.app.extension.PageRequestImpl;
import run.halo.app.extension.router.SortableRequest;
import run.halo.app.extension.router.selector.FieldSelector;

/**
* A query object for {@link Moment} list.
*
* @author LIlGG
* @since 1.0.0
*/
public class MomentQuery extends IListRequest.QueryListRequest {
public class MomentQuery extends SortableRequest {
private final MultiValueMap<String, String> queryParams;

public MomentQuery(MultiValueMap<String, String> queryParams) {
super(queryParams);
public MomentQuery(ServerWebExchange exchange) {
super(exchange);
this.queryParams = exchange.getRequest().getQueryParams();
}

@Nullable
Expand All @@ -42,12 +58,6 @@ public Moment.MomentVisible getVisible() {
return Moment.MomentVisible.from(visible);
}

@Schema(description = "Moment collation.")
public MomentSorter getSort() {
String sort = queryParams.getFirst("sort");
return MomentSorter.convertFrom(sort);
}

@Schema
public Instant getStartDate() {
String startDate = queryParams.getFirst("startDate");
Expand All @@ -60,19 +70,52 @@ public Instant getEndDate() {
return convertInstantOrNull(endDate);
}

@Schema(description = "ascending order If it is true; otherwise, it is in descending order.")
public Boolean getSortOrder() {
String sortOrder = queryParams.getFirst("sortOrder");
return convertBooleanOrNull(sortOrder);
/**
* Build {@link ListOptions} from query params.
*
* @return a list options.
*/
public ListOptions toListOptions() {
var listOptions =
labelAndFieldSelectorToListOptions(getLabelSelector(), getFieldSelector());
var query = all();
if (StringUtils.isNotBlank(getOwnerName())) {
query = and(query, equal("spec.owner", getOwnerName()));
}
if (StringUtils.isNotBlank(getTag())) {
query = and(query, equal("spec.tags", getTag()));
}
if (getVisible() != null) {
query = and(query, equal("spec.visible", getVisible().name()));
}

if (getStartDate() != null) {
query = and(query, greaterThanOrEqual("spec.releaseTime", getStartDate().toString()));
}
if (getEndDate() != null) {
query = and(query, lessThanOrEqual("spec.releaseTime", getEndDate().toString()));
}

if (listOptions.getFieldSelector() != null
&& listOptions.getFieldSelector().query() != null) {
query = and(query, listOptions.getFieldSelector().query());
}
if (StringUtils.isNotBlank(getKeyword())) {
query = and(query, contains("spec.owner", getKeyword()));
}
listOptions.setFieldSelector(FieldSelector.of(query));
return listOptions;
}

private Boolean convertBooleanOrNull(String value) {
return StringUtils.isBlank(value) ? null : Boolean.parseBoolean(value);
public PageRequest toPageRequest() {
var sort = getSort();
if (sort.isUnsorted()) {
sort = Sort.by("spec.releaseTime").descending();
}
return PageRequestImpl.of(getPage(), getSize(), sort);
}

private Instant convertInstantOrNull(String timeStr) {
return StringUtils.isBlank(timeStr) ? null : Instant.parse(timeStr);
}


}
18 changes: 16 additions & 2 deletions src/main/java/run/halo/moments/MomentReconciler.java
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,18 @@
package run.halo.moments;

import static run.halo.app.extension.index.query.QueryFactory.equal;

import java.util.Set;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Component;
import run.halo.app.core.extension.notification.Subscription;
import run.halo.app.extension.DefaultExtensionMatcher;
import run.halo.app.extension.ExtensionClient;
import run.halo.app.extension.ExtensionUtil;
import run.halo.app.extension.controller.Controller;
import run.halo.app.extension.controller.ControllerBuilder;
import run.halo.app.extension.controller.Reconciler;
import run.halo.app.extension.router.selector.FieldSelector;
import run.halo.app.notification.NotificationCenter;

/**
Expand Down Expand Up @@ -37,8 +41,14 @@ public Result reconcile(Request request) {
if (ExtensionUtil.addFinalizers(moment.getMetadata(), Set.of(FINALIZER))) {
// auto subscribe to new comment on moment
createCommentSubscriptionForMoment(moment);
client.update(moment);
}
var status = moment.getStatus();
if (status == null) {
status = new Moment.Status();
moment.setStatus(status);
}
status.setObservedVersion(moment.getMetadata().getVersion() + 1);
client.update(moment);
});
return Result.doNotRetry();
}
Expand All @@ -57,9 +67,13 @@ void createCommentSubscriptionForMoment(Moment moment) {

@Override
public Controller setupWith(ControllerBuilder builder) {
final var moment = new Moment();
return builder
.extension(new Moment())
.extension(moment)
.workerCount(5)
.onAddMatcher(DefaultExtensionMatcher.builder(client, moment.groupVersionKind())
.fieldSelector(FieldSelector.of(equal("needsSyncOnStartup", "true"))).build()
)
.build();
}
}
54 changes: 50 additions & 4 deletions src/main/java/run/halo/moments/MomentsPlugin.java
View file
Original file line number Diff line number Diff line change
@@ -1,23 +1,69 @@
package run.halo.moments;

import org.pf4j.PluginWrapper;
import static run.halo.app.extension.index.IndexAttributeFactory.multiValueAttribute;
import static run.halo.app.extension.index.IndexAttributeFactory.simpleAttribute;

import java.util.Optional;
import java.util.Set;
import org.apache.commons.lang3.BooleanUtils;
import org.springframework.stereotype.Component;
import run.halo.app.extension.SchemeManager;
import run.halo.app.extension.index.IndexSpec;
import run.halo.app.plugin.BasePlugin;
import run.halo.app.plugin.PluginContext;

@Component
public class MomentsPlugin extends BasePlugin {

private final SchemeManager schemeManager;

public MomentsPlugin(PluginWrapper wrapper, SchemeManager schemeManager) {
super(wrapper);
public MomentsPlugin(PluginContext pluginContext, SchemeManager schemeManager) {
super(pluginContext);
this.schemeManager = schemeManager;
}

@Override
public void start() {
schemeManager.register(Moment.class);
schemeManager.register(Moment.class, indexSpecs -> {
indexSpecs.add(new IndexSpec()
.setName("spec.tags")
.setIndexFunc(multiValueAttribute(Moment.class, moment -> {
var tags = moment.getSpec().getTags();
return tags == null ? Set.of() : tags;
}))
);
indexSpecs.add(new IndexSpec()
.setName("spec.owner")
.setIndexFunc(
simpleAttribute(Moment.class, moment -> moment.getSpec().getOwner())));
indexSpecs.add(new IndexSpec()
.setName("spec.releaseTime")
.setIndexFunc(simpleAttribute(Moment.class, moment -> {
var releaseTime = moment.getSpec().getReleaseTime();
return releaseTime == null ? null : releaseTime.toString();
}))
);

indexSpecs.add(new IndexSpec()
.setName("spec.visible")
.setIndexFunc(simpleAttribute(Moment.class, moment -> {
var visible = moment.getSpec().getVisible();
return visible == null ? null : visible.toString();
}))
);
indexSpecs.add(new IndexSpec()
.setName("needsSyncOnStartup")
.setIndexFunc(simpleAttribute(Moment.class, moment -> {
var observedVersion = Optional.ofNullable(moment.getStatus())
.map(Moment.Status::getObservedVersion)
.orElse(-1L);
if (observedVersion < moment.getMetadata().getVersion()) {
return BooleanUtils.TRUE;
}
// don't care about the false case
return null;
})));
});
}

@Override
Expand Down
Loading

0 comments on commit bd0faad

Please sign in to comment.