From 6df10603a6a1ec38dc437d4b8c58f868d98fea45 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 4 Jan 2024 14:45:04 +0100 Subject: [PATCH 01/12] initial draft hoping to avoid keeping SecondaryIdQuery subPlans, that are not needed --- .../filter/AggregationResultFilterNode.java | 5 +- .../query/queryplan/ConceptQueryPlan.java | 4 +- .../query/queryplan/DateAggregator.java | 2 +- .../query/queryplan/EventIterating.java | 2 +- .../models/query/queryplan/QPNode.java | 2 +- .../models/query/queryplan/QPParentNode.java | 7 +- .../query/queryplan/SecondaryIdQueryPlan.java | 122 ++++++++++-------- .../queryplan/aggregators/Aggregator.java | 11 +- .../aggregators/ColumnAggregator.java | 2 +- .../DistinctValuesWrapperAggregator.java | 4 +- .../specific/ConstantValueAggregator.java | 3 +- .../aggregators/specific/CountAggregator.java | 2 +- .../CountQuartersOfDateRangeAggregator.java | 2 +- .../CountQuartersOfDatesAggregator.java | 2 +- .../specific/DateDistanceAggregator.java | 2 +- .../specific/DateUnionAggregator.java | 2 +- .../specific/DurationSumAggregator.java | 2 +- .../specific/EventDateUnionAggregator.java | 2 +- .../specific/EventDurationSumAggregator.java | 2 +- .../specific/ExistsAggregator.java | 2 +- .../aggregators/specific/FlagsAggregator.java | 2 +- .../specific/MultiSelectAggregator.java | 2 +- .../specific/PrefixTextAggregator.java | 2 +- .../specific/QuarterAggregator.java | 2 +- .../specific/QuartersInYearAggregator.java | 2 +- .../specific/SelectAggregator.java | 2 +- .../specific/SpecialDateUnion.java | 2 +- .../diffsum/DecimalDiffSumAggregator.java | 2 +- .../diffsum/IntegerDiffSumAggregator.java | 2 +- .../diffsum/MoneyDiffSumAggregator.java | 2 +- .../diffsum/RealDiffSumAggregator.java | 2 +- .../specific/sum/DecimalSumAggregator.java | 2 +- .../specific/sum/IntegerSumAggregator.java | 2 +- .../specific/sum/MoneySumAggregator.java | 2 +- .../specific/sum/RealSumAggregator.java | 2 +- .../specific/value/AllValuesAggregator.java | 2 +- .../value/ConceptElementsAggregator.java | 2 +- .../value/ConceptValuesAggregator.java | 2 +- .../specific/value/FirstValueAggregator.java | 2 +- .../specific/value/LastValueAggregator.java | 2 +- .../specific/value/RandomValueAggregator.java | 2 +- .../queryplan/filter/EventFilterNode.java | 3 +- .../query/queryplan/specific/ConceptNode.java | 33 ++--- .../specific/DateRestrictingNode.java | 4 +- .../queryplan/specific/ExternalNode.java | 3 +- .../query/queryplan/specific/FiltersNode.java | 8 +- .../models/query/queryplan/specific/Leaf.java | 3 +- .../queryplan/specific/NegatingNode.java | 4 +- .../queryplan/specific/ValidityDateNode.java | 6 +- .../models/query/queryplan/specific/Yes.java | 4 +- .../conquery/io/result/ResultTestUtil.java | 2 +- 51 files changed, 158 insertions(+), 136 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/AggregationResultFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/AggregationResultFilterNode.java index fa673d1272..4c592b156f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/AggregationResultFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/AggregationResultFilterNode.java @@ -51,8 +51,9 @@ public void nextBlock(Bucket bucket) { } @Override - public void acceptEvent(Bucket bucket, int event) { - aggregator.acceptEvent(bucket, event); + public boolean acceptEvent(Bucket bucket, int event) { + aggregator.consumeEvent(bucket, event); + return true; // this is ignored for non-EventFilterNodes } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java index 2cd46a035e..3e943f2dbe 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java @@ -63,8 +63,8 @@ public void init(QueryExecutionContext ctx, Entity entity) { child.init(entity, ctx); } - public void nextEvent(Bucket bucket, int event) { - getChild().acceptEvent(bucket, event); + public boolean nextEvent(Bucket bucket, int event) { + return getChild().acceptEvent(bucket, event); } protected SinglelineEntityResult createResult() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java index 087fbc7539..8f7978d967 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java @@ -48,7 +48,7 @@ public void init(Entity entity, QueryExecutionContext context) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { throw new UnsupportedOperationException("This Aggregator uses the result of its siblings and does not accept events"); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/EventIterating.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/EventIterating.java index 8e6687c6ef..57f7af4fdd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/EventIterating.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/EventIterating.java @@ -42,7 +42,7 @@ public void nextTable(QueryExecutionContext ctx, Table currentTable) { public void nextBlock(Bucket bucket) { } - public abstract void acceptEvent(Bucket bucket, int event); + public abstract boolean acceptEvent(Bucket bucket, int event); public boolean isOfInterest(Bucket bucket) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPNode.java index 092ffbe59a..42c7ba6a43 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPNode.java @@ -34,7 +34,7 @@ public void nextTable(QueryExecutionContext ctx, Table currentTable) { } @Override - public abstract void acceptEvent(Bucket bucket, int event); + public abstract boolean acceptEvent(Bucket bucket, int event); public abstract boolean isContained(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPParentNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPParentNode.java index 063db34234..529e86860e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPParentNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPParentNode.java @@ -117,10 +117,13 @@ public boolean isOfInterest(Entity entity) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public boolean acceptEvent(Bucket bucket, int event) { + boolean consumed = false; for (QPNode currentTableChild : currentTableChildren) { - currentTableChild.acceptEvent(bucket, event); + consumed |= currentTableChild.acceptEvent(bucket, event); } + + return consumed; } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java index 455c069d15..a6eea61b89 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java @@ -57,8 +57,11 @@ public class SecondaryIdQueryPlan implements QueryPlan { private final Map childPerKey = new HashMap<>(); - - + /** + * This helps us avoid allocations, instead allowing us to reuse the queries. + */ + @Getter(AccessLevel.NONE) + private final Queue childPlanReusePool = new LinkedList<>(); /** * This is the same execution as a typical ConceptQueryPlan. The difference @@ -89,58 +92,18 @@ public Optional execute(QueryExecutionContext ctx, Entity return createResult(entity); } - /** - * For each secondaryId that is included, create a line, return {@link MultilineEntityResult} containing all results. - */ - private Optional createResult(Entity entity) { - List result = new ArrayList<>(childPerKey.values().size()); - - // Prepend the key (ie the actual SecondaryId) to the result. - for (Map.Entry child : childPerKey.entrySet()) { - if (!child.getValue().isContained()) { - continue; - } - - // Prepend SecondaryId to result-line. - result.add(ArrayUtils.insert(0, child.getValue().createResult().getValues(), child.getKey())); - } - - if (result.isEmpty()) { - return Optional.empty(); - } - - return Optional.of(new MultilineEntityResult(entity.getId(), result)); - } - - /** - * This helps us avoid allocations, instead allowing us to reuse the queries. - */ - @Getter(AccessLevel.NONE) - private final Queue childPlanReusePool = new LinkedList<>(); - - @Override - public void init(QueryExecutionContext ctx, Entity entity) { - queryPlan.init(ctx, entity); - - // Dump the created children into reuse-pool - childPlanReusePool.addAll(childPerKey.values()); - - childPerKey.clear(); - } - - private void executeQueriesWithSecondaryId(QueryExecutionContext ctx, Entity entity, Column secondaryIdColumnId) { - QueryExecutionContext ctxWithPhase = ctx.withActiveSecondaryId(getSecondaryId()); + final QueryExecutionContext ctxWithPhase = ctx.withActiveSecondaryId(getSecondaryId()); - Table currentTable = secondaryIdColumnId.getTable(); + final Table currentTable = secondaryIdColumnId.getTable(); nextTable(ctxWithPhase, currentTable); final List tableBuckets = ctx.getBucketManager().getEntityBucketsForTable(entity, currentTable); for (Bucket bucket : tableBuckets) { - int entityId = entity.getId(); + final int entityId = entity.getId(); nextBlock(bucket); @@ -152,8 +115,8 @@ private void executeQueriesWithSecondaryId(QueryExecutionContext ctx, Entity ent continue; } - int start = bucket.getEntityStart(entityId); - int end = bucket.getEntityEnd(entityId); + final int start = bucket.getEntityStart(entityId); + final int end = bucket.getEntityEnd(entityId); for (int event = start; event < end; event++) { //we ignore events with no value in the secondaryIdColumn @@ -161,9 +124,23 @@ private void executeQueriesWithSecondaryId(QueryExecutionContext ctx, Entity ent continue; } - String key = ((String) bucket.createScriptValue(event, secondaryIdColumnId)); - final ConceptQueryPlan plan = childPerKey.computeIfAbsent(key, k -> createChild(ctxWithPhase, bucket)); - plan.nextEvent(bucket, event); + final String key = ((String) bucket.createScriptValue(event, secondaryIdColumnId)); + + if (childPerKey.containsKey(key)) { + final ConceptQueryPlan plan = childPerKey.get(key); + plan.nextEvent(bucket, event); + } + else { + final ConceptQueryPlan plan = createChild(ctxWithPhase, bucket); + final boolean consumed = plan.nextEvent(bucket, event); + + if (consumed) { + childPerKey.put(key, plan); + } + else { + childPlanReusePool.add(plan); + } + } } } } @@ -175,14 +152,14 @@ private void executeQueriesWithoutSecondaryId(QueryExecutionContext ctx, Entity final List tableBuckets = ctx.getBucketManager().getEntityBucketsForTable(entity, currentTable); for (Bucket bucket : tableBuckets) { - int entityId = entity.getId(); + final int entityId = entity.getId(); nextBlock(bucket); if (!bucket.containsEntity(entityId) || !isOfInterest(bucket)) { continue; } - int start = bucket.getEntityStart(entityId); - int end = bucket.getEntityEnd(entityId); + final int start = bucket.getEntityStart(entityId); + final int end = bucket.getEntityEnd(entityId); for (int event = start; event < end; event++) { for (ConceptQueryPlan child : childPerKey.values()) { @@ -192,10 +169,33 @@ private void executeQueriesWithoutSecondaryId(QueryExecutionContext ctx, Entity } } + /** + * For each secondaryId that is included, create a line, return {@link MultilineEntityResult} containing all results. + */ + private Optional createResult(Entity entity) { + final List result = new ArrayList<>(childPerKey.values().size()); + + // Prepend the key (ie the actual SecondaryId) to the result. + for (Map.Entry child : childPerKey.entrySet()) { + if (!child.getValue().isContained()) { + continue; + } + + // Prepend SecondaryId to result-line. + result.add(ArrayUtils.insert(0, child.getValue().createResult().getValues(), child.getKey())); + } + + if (result.isEmpty()) { + return Optional.empty(); + } + + return Optional.of(new MultilineEntityResult(entity.getId(), result)); + } + private void nextTable(QueryExecutionContext ctx, Table currentTable) { queryPlan.nextTable(ctx, currentTable); for (ConceptQueryPlan c : childPerKey.values()) { - QueryExecutionContext context = QueryUtils.determineDateAggregatorForContext(ctx, c::getValidityDateAggregator); + final QueryExecutionContext context = QueryUtils.determineDateAggregatorForContext(ctx, c::getValidityDateAggregator); c.nextTable(context, currentTable); } } @@ -220,7 +220,7 @@ private ConceptQueryPlan createChild(QueryExecutionContext currentContext, Bucke ConceptQueryPlan plan; // Try to reuse old child plan first before allocating new ones - if((plan = childPlanReusePool.poll()) == null) { + if ((plan = childPlanReusePool.poll()) == null) { plan = query.createQueryPlan(queryPlanContext.withSelectedSecondaryId(secondaryId)); } @@ -234,6 +234,16 @@ private ConceptQueryPlan createChild(QueryExecutionContext currentContext, Bucke return plan; } + @Override + public void init(QueryExecutionContext ctx, Entity entity) { + queryPlan.init(ctx, entity); + + // Dump the created children into reuse-pool + childPlanReusePool.addAll(childPerKey.values()); + + childPerKey.clear(); + } + @Override public boolean isOfInterest(Entity entity) { return queryPlan.isOfInterest(entity); @@ -245,7 +255,7 @@ public Optional> getValidityDateAggregator() { return Optional.empty(); } - DateAggregator agg = new DateAggregator(DateAggregationAction.MERGE); + final DateAggregator agg = new DateAggregator(DateAggregationAction.MERGE); childPerKey.values().forEach(c -> c.getValidityDateAggregator().ifPresent(agg::register)); return agg.hasChildren() ? Optional.of(agg) : Optional.empty(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/Aggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/Aggregator.java index 830c2614fa..c73bb6680d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/Aggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/Aggregator.java @@ -6,7 +6,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; /** - * An aggregator iterates over events, computing a value alongside. Values are fed through {@link Aggregator#acceptEvent(Bucket, int)}, and the result can be queried at {@link Aggregator#createAggregationResult()}. + * An aggregator iterates over events, computing a value alongside. Values are fed through {@link Aggregator#consumeEvent(Bucket, int)}, and the result can be queried at {@link Aggregator#createAggregationResult()}. *

* Every Aggregator has an associated {@code ResultType} that is used for rendering purposes. *

@@ -27,6 +27,15 @@ public abstract class Aggregator extends EventIterating { */ public abstract T createAggregationResult(); + @Override + public boolean acceptEvent(Bucket bucket, int event) { + consumeEvent(bucket, event); + return true; + } + + public abstract void consumeEvent(Bucket bucket, int event) ; + + /** * Specific type of the result used for rendering. */ diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/ColumnAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/ColumnAggregator.java index 6ca3798feb..d47703c3c9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/ColumnAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/ColumnAggregator.java @@ -22,7 +22,7 @@ public void collectRequiredTables(Set out) { public abstract List getRequiredColumns(); @Override - public abstract void acceptEvent(Bucket bucket, int event); + public abstract void consumeEvent(Bucket bucket, int event); /** * Skip all buckets where none of the required columns have values. diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/DistinctValuesWrapperAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/DistinctValuesWrapperAggregator.java index bd8ab46906..82d12a3a58 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/DistinctValuesWrapperAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/DistinctValuesWrapperAggregator.java @@ -62,7 +62,7 @@ public void init(Entity entity, QueryExecutionContext context) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { final List incoming = new ArrayList<>(getColumns().size()); // Do not accept completely empty lines @@ -79,7 +79,7 @@ public void acceptEvent(Bucket bucket, int event) { } if (anyPresent && observed.add(incoming)) { - aggregator.acceptEvent(bucket, event); + aggregator.consumeEvent(bucket, event); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ConstantValueAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ConstantValueAggregator.java index 3b777c833b..9e76f4e71f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ConstantValueAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ConstantValueAggregator.java @@ -33,7 +33,8 @@ public void init(Entity entity, QueryExecutionContext context) { } @Override - public void acceptEvent(Bucket bucket, int event) {} + public void consumeEvent(Bucket bucket, int event) { + } @Override public ResultType getResultType() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountAggregator.java index 98139d420e..7c11065e6a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountAggregator.java @@ -39,7 +39,7 @@ public void init(Entity entity, QueryExecutionContext context) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { // When no column is set, count all events (supposedly someone else is filtering for us) // When column is set, count only the events where column has entries if (column == null || bucket.has(event, column)) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDateRangeAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDateRangeAggregator.java index 496b0659b3..4ff5191435 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDateRangeAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDateRangeAggregator.java @@ -46,7 +46,7 @@ public void nextTable(QueryExecutionContext ctx, Table currentTable) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { return; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDatesAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDatesAggregator.java index c4da7accec..9f14c30acd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDatesAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDatesAggregator.java @@ -32,7 +32,7 @@ public void init(Entity entity, QueryExecutionContext context) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { return; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateDistanceAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateDistanceAggregator.java index 70809fd828..32a801dbc3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateDistanceAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateDistanceAggregator.java @@ -51,7 +51,7 @@ public Long createAggregationResult() { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { return; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateUnionAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateUnionAggregator.java index 8808c08d85..ee84e25693 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateUnionAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateUnionAggregator.java @@ -38,7 +38,7 @@ public void nextTable(QueryExecutionContext ctx, Table currentTable) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { return; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DurationSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DurationSumAggregator.java index d3f95f7861..16b0bb1eed 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DurationSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DurationSumAggregator.java @@ -39,7 +39,7 @@ public void nextTable(QueryExecutionContext ctx, Table currentTable) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { return; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDateUnionAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDateUnionAggregator.java index 0f3344f648..75a5e531b3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDateUnionAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDateUnionAggregator.java @@ -53,7 +53,7 @@ public CDateSet createAggregationResult() { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { if(validityDateColumn == null) { set.addAll(dateRestriction); return; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDurationSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDurationSumAggregator.java index 5f563588cc..14733d3075 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDurationSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDurationSumAggregator.java @@ -43,7 +43,7 @@ public void nextTable(QueryExecutionContext ctx, Table currentTable) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { if (validityDateColumn == null) { return; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ExistsAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ExistsAggregator.java index f606ed5038..6ad7d93dcc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ExistsAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ExistsAggregator.java @@ -29,7 +29,7 @@ public void setReference(QPNode ref) { private QPNode reference; @Override - public void acceptEvent(Bucket bucket, int event) { } + public void consumeEvent(Bucket bucket, int event) { } @Override public Boolean createAggregationResult() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/FlagsAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/FlagsAggregator.java index f29dd2bebe..0de868dadd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/FlagsAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/FlagsAggregator.java @@ -25,7 +25,7 @@ public void init(Entity entity, QueryExecutionContext context) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { for (Map.Entry entry : labels.entrySet()) { final Column column = entry.getValue(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/MultiSelectAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/MultiSelectAggregator.java index c5f9140ba2..3797d78e0f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/MultiSelectAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/MultiSelectAggregator.java @@ -45,7 +45,7 @@ public void nextBlock(Bucket bucket) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { return; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/PrefixTextAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/PrefixTextAggregator.java index f24989bbf9..237bd3ffb9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/PrefixTextAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/PrefixTextAggregator.java @@ -32,7 +32,7 @@ public void init(Entity entity, QueryExecutionContext context) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { return; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuarterAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuarterAggregator.java index 4e09852192..8e054d10db 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuarterAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuarterAggregator.java @@ -53,7 +53,7 @@ public void nextTable(QueryExecutionContext ctx, Table currentTable) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { final CDateRange dateRange = validityDate.getValidityDate(event, bucket); if (dateRange == null){ diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuartersInYearAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuartersInYearAggregator.java index 622ed722e9..18728488f0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuartersInYearAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuartersInYearAggregator.java @@ -35,7 +35,7 @@ public void init(Entity entity, QueryExecutionContext context) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { return; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SelectAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SelectAggregator.java index 6ba351f32e..9b8ee495e3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SelectAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SelectAggregator.java @@ -36,7 +36,7 @@ public void nextBlock(Bucket bucket) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { if (selectedId == -1) { return; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SpecialDateUnion.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SpecialDateUnion.java index e2ad370f73..544fd14b4d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SpecialDateUnion.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SpecialDateUnion.java @@ -37,7 +37,7 @@ public void nextTable(QueryExecutionContext ctx, Table table) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { if (validityDate == null) { set.addAll(dateRestriction); return; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/DecimalDiffSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/DecimalDiffSumAggregator.java index 46d33ed95f..6f7136b332 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/DecimalDiffSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/DecimalDiffSumAggregator.java @@ -50,7 +50,7 @@ public List getRequiredColumns() { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { if (!bucket.has(event, getAddendColumn()) && !bucket.has(event, getSubtrahendColumn())) { return; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/IntegerDiffSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/IntegerDiffSumAggregator.java index 01a2ad9783..25f4036cfb 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/IntegerDiffSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/IntegerDiffSumAggregator.java @@ -48,7 +48,7 @@ public List getRequiredColumns() { return out; } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { if (!bucket.has(event, getAddendColumn()) && !bucket.has(event, getSubtrahendColumn())) { return; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/MoneyDiffSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/MoneyDiffSumAggregator.java index da3afd9017..5bce42bda8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/MoneyDiffSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/MoneyDiffSumAggregator.java @@ -48,7 +48,7 @@ public List getRequiredColumns() { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { if (!bucket.has(event, getAddendColumn()) && !bucket.has(event, getSubtrahendColumn())) { return; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/RealDiffSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/RealDiffSumAggregator.java index a3740b20e4..0d422ef091 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/RealDiffSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/RealDiffSumAggregator.java @@ -49,7 +49,7 @@ public List getRequiredColumns() { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { if (!bucket.has(event, getAddendColumn()) && !bucket.has(event, getSubtrahendColumn())) { return; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/DecimalSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/DecimalSumAggregator.java index 1697f75920..2f7042ad83 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/DecimalSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/DecimalSumAggregator.java @@ -30,7 +30,7 @@ public void init(Entity entity, QueryExecutionContext context) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { return; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/IntegerSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/IntegerSumAggregator.java index 4ab277a443..847caa6769 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/IntegerSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/IntegerSumAggregator.java @@ -28,7 +28,7 @@ public void init(Entity entity, QueryExecutionContext context) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { return; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/MoneySumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/MoneySumAggregator.java index 57634741a3..82dde927d9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/MoneySumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/MoneySumAggregator.java @@ -29,7 +29,7 @@ public void init(Entity entity, QueryExecutionContext context) { @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { return; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/RealSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/RealSumAggregator.java index 8a9354f4ff..a7b4489fcc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/RealSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/RealSumAggregator.java @@ -29,7 +29,7 @@ public void init(Entity entity, QueryExecutionContext context) { @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { return; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/AllValuesAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/AllValuesAggregator.java index feaea1416f..28b264742b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/AllValuesAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/AllValuesAggregator.java @@ -32,7 +32,7 @@ public void init(Entity entity, QueryExecutionContext context) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { if (bucket.has(event, getColumn())) { entries.add((VALUE) bucket.createScriptValue(event, getColumn())); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/ConceptElementsAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/ConceptElementsAggregator.java index 311e77c384..2f92379a7b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/ConceptElementsAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/ConceptElementsAggregator.java @@ -72,7 +72,7 @@ public void init(Entity entity, QueryExecutionContext context) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { if (!bucket.has(event, column)) { return; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/ConceptValuesAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/ConceptValuesAggregator.java index 32a827ad32..cfaae0cdcb 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/ConceptValuesAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/ConceptValuesAggregator.java @@ -59,7 +59,7 @@ public void init(Entity entity, QueryExecutionContext context) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { if (!bucket.has(event, column)) { return; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/FirstValueAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/FirstValueAggregator.java index 7e70495cc3..c4d6864552 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/FirstValueAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/FirstValueAggregator.java @@ -46,7 +46,7 @@ public void nextTable(QueryExecutionContext ctx, Table currentTable) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { return; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/LastValueAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/LastValueAggregator.java index 0e359c3f98..9bcee0a0be 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/LastValueAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/LastValueAggregator.java @@ -46,7 +46,7 @@ public void nextTable(QueryExecutionContext ctx, Table currentTable) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { return; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/RandomValueAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/RandomValueAggregator.java index 0a6c880f4a..7c44470220 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/RandomValueAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/RandomValueAggregator.java @@ -48,7 +48,7 @@ public void init(Entity entity, QueryExecutionContext context) { * @param event */ @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { return; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/EventFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/EventFilterNode.java index 2d1410d2b2..8311becd68 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/EventFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/EventFilterNode.java @@ -15,8 +15,9 @@ public EventFilterNode(FILTER_VALUE filterValue) { public abstract boolean checkEvent(Bucket bucket, int event); @Override - public final void acceptEvent(Bucket bucket, int event) { + public final boolean acceptEvent(Bucket bucket, int event) { hit = true; + return true; } public final boolean isContained() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ConceptNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ConceptNode.java index ff0bbab2db..714af36af1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ConceptNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ConceptNode.java @@ -27,9 +27,9 @@ public class ConceptNode extends QPChainNode { private final long requiredBits; private final CQTable table; private final SecondaryIdDescription selectedSecondaryId; - private boolean tableActive = false; - private Map preCurrentRow = null; - private CBlock currentRow = null; + private boolean tableActive; + private Map preCurrentRow; + private CBlock currentRow; public ConceptNode(QPNode child, List> concepts, CQTable table, SecondaryIdDescription selectedSecondaryId) { this(child, concepts, calculateBitMask(concepts), table, selectedSecondaryId); @@ -94,7 +94,7 @@ public boolean isOfInterest(Bucket bucket) { return false; } - CBlock cBlock = Objects.requireNonNull(preCurrentRow.get(bucket)); + final CBlock cBlock = Objects.requireNonNull(preCurrentRow.get(bucket)); if(cBlock.isConceptIncluded(entity.getId(), requiredBits)) { return super.isOfInterest(bucket); @@ -103,30 +103,23 @@ public boolean isOfInterest(Bucket bucket) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public boolean acceptEvent(Bucket bucket, int event) { if (!tableActive) { - return; + return false; } //check concepts - int[] mostSpecificChildren = currentRow.getPathToMostSpecificChild(event); - if (mostSpecificChildren == null) { - for (ConceptElement ce : concepts) { - // having no specific child set maps directly to root. - // This means we likely have a VirtualConcept - if (ce.getConcept() == ce) { - getChild().acceptEvent(bucket, event); - } - } - return; - } + final int[] mostSpecificChildren = currentRow.getPathToMostSpecificChild(event); + + boolean consumed = false; for (ConceptElement ce : concepts) { - //see #177 we could improve this by building a prefix tree over concepts.prefix - if (ce.matchesPrefix(mostSpecificChildren)) { - getChild().acceptEvent(bucket, event); + if ((mostSpecificChildren != null && ce.matchesPrefix(mostSpecificChildren)) || ce.getConcept() == ce) { + consumed |= getChild().acceptEvent(bucket, event); } } + + return consumed; } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/DateRestrictingNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/DateRestrictingNode.java index 2094fefd0b..04c898b9a0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/DateRestrictingNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/DateRestrictingNode.java @@ -37,8 +37,8 @@ public void nextTable(QueryExecutionContext ctx, Table currentTable) { } @Override - public void acceptEvent(Bucket bucket, int event) { - getChild().acceptEvent(bucket, event); + public boolean acceptEvent(Bucket bucket, int event) { + return getChild().acceptEvent(bucket, event); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java index 292e11d736..f0a5f6a4cf 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java @@ -77,8 +77,9 @@ public void nextTable(QueryExecutionContext ctx, Table currentTable) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public boolean acceptEvent(Bucket bucket, int event) { // Nothing to do + return true; } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/FiltersNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/FiltersNode.java index 34892ae10c..c799ca3e58 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/FiltersNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/FiltersNode.java @@ -94,17 +94,19 @@ public void nextBlock(Bucket bucket) { } @Override - public final void acceptEvent(Bucket bucket, int event) { + public final boolean acceptEvent(Bucket bucket, int event) { for (EventFilterNode f : eventFilters) { if (!f.checkEvent(bucket, event)) { - return; + return false; } } filters.forEach(f -> f.acceptEvent(bucket, event)); - aggregators.forEach(a -> a.acceptEvent(bucket, event)); + aggregators.forEach(a -> a.consumeEvent(bucket, event)); hit = true; + + return true; } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Leaf.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Leaf.java index ee68ad35e8..2f38931471 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Leaf.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Leaf.java @@ -23,8 +23,9 @@ public void init(Entity entity, QueryExecutionContext context) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public boolean acceptEvent(Bucket bucket, int event) { triggered = true; + return true; } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/NegatingNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/NegatingNode.java index e870c37c5d..c9989ab2e4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/NegatingNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/NegatingNode.java @@ -31,8 +31,8 @@ private NegatingNode(@NonNull QPNode child, @NonNull DateAggregator dateAggregat } @Override - public void acceptEvent(Bucket bucket, int event) { - getChild().acceptEvent(bucket, event); + public boolean acceptEvent(Bucket bucket, int event) { + return getChild().acceptEvent(bucket, event); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ValidityDateNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ValidityDateNode.java index 9d29a7f8dd..eba1af8b94 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ValidityDateNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ValidityDateNode.java @@ -30,15 +30,15 @@ public ValidityDateNode(ValidityDate validityDate, QPNode child) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public boolean acceptEvent(Bucket bucket, int event) { //no dateRestriction or event is in date restriction final boolean contained = bucket.eventIsContainedIn(event, validityDate, context.getDateRestriction()); if (!contained){ - return; + return false; } - getChild().acceptEvent(bucket, event); + return getChild().acceptEvent(bucket, event); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Yes.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Yes.java index fd2c228a9b..ec8e65c15b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Yes.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Yes.java @@ -26,8 +26,8 @@ public void init(Entity entity, QueryExecutionContext context) { } @Override - public void acceptEvent(Bucket bucket, int event) { - + public boolean acceptEvent(Bucket bucket, int event) { + return true; } @Override diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java index e2aa4112c6..7127e8d27c 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java @@ -108,7 +108,7 @@ public void init(Entity entity, QueryExecutionContext context) { } @Override - public void acceptEvent(Bucket bucket, int event) { + public void consumeEvent(Bucket bucket, int event) { throw new UnsupportedOperationException(); } From 6ba17b568d84c423063f9f9396d7769f7f43c1ba Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 18 Jan 2024 16:23:38 +0100 Subject: [PATCH 02/12] cleanup --- .../conquery/models/query/queryplan/SecondaryIdQueryPlan.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java index a6eea61b89..90025de01c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java @@ -217,10 +217,10 @@ private boolean isOfInterest(Bucket bucket) { */ private ConceptQueryPlan createChild(QueryExecutionContext currentContext, Bucket currentBucket) { - ConceptQueryPlan plan; + ConceptQueryPlan plan = childPlanReusePool.poll(); // Try to reuse old child plan first before allocating new ones - if ((plan = childPlanReusePool.poll()) == null) { + if (plan == null) { plan = query.createQueryPlan(queryPlanContext.withSelectedSecondaryId(secondaryId)); } From 3e3cf30c42a739cceaff6f73e973249058ef82db Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 18 Jan 2024 16:46:08 +0100 Subject: [PATCH 03/12] fully split EventFilterNode and AggregationResultFilterNode --- .../filter/CollectionNotEmptyFilterNode.java | 1 + .../models/query/filter/RangeFilterNode.java | 1 + .../query/queryplan/EventIterating.java | 3 +++ .../filter/AggregationResultFilterNode.java | 6 ++---- .../queryplan/filter/EventFilterNode.java | 13 +++--------- .../query/queryplan/filter/FilterNode.java | 3 +-- .../query/queryplan/specific/FiltersNode.java | 21 ++++++++++++------- 7 files changed, 25 insertions(+), 23 deletions(-) rename backend/src/main/java/com/bakdata/conquery/models/query/{ => queryplan}/filter/AggregationResultFilterNode.java (85%) diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/CollectionNotEmptyFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/CollectionNotEmptyFilterNode.java index 898196c01e..4855e57c1f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/CollectionNotEmptyFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/CollectionNotEmptyFilterNode.java @@ -3,6 +3,7 @@ import java.util.Collection; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; +import com.bakdata.conquery.models.query.queryplan.filter.AggregationResultFilterNode; import lombok.ToString; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/RangeFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/RangeFilterNode.java index 9004534aa0..f52cbf0acf 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/RangeFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/RangeFilterNode.java @@ -2,6 +2,7 @@ import com.bakdata.conquery.models.common.IRange; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; +import com.bakdata.conquery.models.query.queryplan.filter.AggregationResultFilterNode; import lombok.ToString; /** diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/EventIterating.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/EventIterating.java index 57f7af4fdd..9bd7e76b6c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/EventIterating.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/EventIterating.java @@ -42,6 +42,9 @@ public void nextTable(QueryExecutionContext ctx, Table currentTable) { public void nextBlock(Bucket bucket) { } + /** + * @implSpec returning false signals that the event may be discarded. + */ public abstract boolean acceptEvent(Bucket bucket, int event); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/AggregationResultFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/AggregationResultFilterNode.java similarity index 85% rename from backend/src/main/java/com/bakdata/conquery/models/query/filter/AggregationResultFilterNode.java rename to backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/AggregationResultFilterNode.java index 4c592b156f..56179054d5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/AggregationResultFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/AggregationResultFilterNode.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.models.query.filter; +package com.bakdata.conquery.models.query.queryplan.filter; import java.util.Set; @@ -7,7 +7,6 @@ import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.filter.FilterNode; import lombok.Getter; import lombok.ToString; @@ -18,7 +17,7 @@ * @param Type of the used FilterValue */ @ToString(callSuper = true) -public abstract class AggregationResultFilterNode, FILTER_VALUE> extends FilterNode { +public abstract non-sealed class AggregationResultFilterNode, FILTER_VALUE> extends FilterNode { @Getter private AGGREGATOR aggregator; @@ -56,7 +55,6 @@ public boolean acceptEvent(Bucket bucket, int event) { return true; // this is ignored for non-EventFilterNodes } - @Override public abstract boolean isContained(); @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/EventFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/EventFilterNode.java index 8311becd68..136f4b331c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/EventFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/EventFilterNode.java @@ -4,29 +4,22 @@ import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; -public abstract class EventFilterNode extends FilterNode { - - private boolean hit = false; +public abstract non-sealed class EventFilterNode extends FilterNode { public EventFilterNode(FILTER_VALUE filterValue) { super(filterValue); } + //TODO rename to acceptEvent? public abstract boolean checkEvent(Bucket bucket, int event); @Override public final boolean acceptEvent(Bucket bucket, int event) { - hit = true; - return true; - } - - public final boolean isContained() { - return hit; + throw new IllegalStateException("May not be called."); } @Override public void init(Entity entity, QueryExecutionContext context) { - hit = false; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/FilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/FilterNode.java index 127b527a69..42b4e6d0bd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/FilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/FilterNode.java @@ -8,11 +8,10 @@ @AllArgsConstructor @ToString -public abstract class FilterNode extends EventIterating { +public abstract sealed class FilterNode extends EventIterating permits EventFilterNode, AggregationResultFilterNode { @Setter @Getter protected FILTER_VALUE filterValue; - public abstract boolean isContained(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/FiltersNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/FiltersNode.java index c799ca3e58..3ae62e89db 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/FiltersNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/FiltersNode.java @@ -12,6 +12,7 @@ import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; +import com.bakdata.conquery.models.query.queryplan.filter.AggregationResultFilterNode; import com.bakdata.conquery.models.query.queryplan.filter.EventFilterNode; import com.bakdata.conquery.models.query.queryplan.filter.FilterNode; import lombok.AccessLevel; @@ -37,6 +38,9 @@ public class FiltersNode extends QPNode { @Setter(AccessLevel.PRIVATE) private List> eventFilters; + @Setter(AccessLevel.PRIVATE) + private List aggregationFilters; + @Setter(AccessLevel.PRIVATE) private List> eventDateAggregators; @@ -59,14 +63,16 @@ public static FiltersNode create(List> filters, List> eventFilters = new ArrayList<>(filters.size()); + final List aggregationFilters = new ArrayList<>(filters.size()); - // Select only Event Filtering nodes as they are used differently. + // Event and AggregationResultFilterNodes are used differently for (FilterNode filter : filters) { - if (!(filter instanceof EventFilterNode)) { - continue; + if (filter instanceof EventFilterNode ef) { + eventFilters.add(ef); + } + else if (filter instanceof AggregationResultFilterNode af){ + aggregationFilters.add(af); } - - eventFilters.add((EventFilterNode) filter); } final FiltersNode filtersNode = new FiltersNode(); @@ -74,6 +80,7 @@ public static FiltersNode create(List> filters, List f.acceptEvent(bucket, event)); + aggregationFilters.forEach(f -> f.acceptEvent(bucket, event)); aggregators.forEach(a -> a.consumeEvent(bucket, event)); hit = true; @@ -111,7 +118,7 @@ public final boolean acceptEvent(Bucket bucket, int event) { @Override public boolean isContained() { - for (FilterNode f : filters) { + for (AggregationResultFilterNode f : aggregationFilters) { if (!f.isContained()) { return false; } From ab846554c131b8d755e9ddb107c830f3b450b4b7 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 18 Jan 2024 16:48:58 +0100 Subject: [PATCH 04/12] person was not supposed to be included --- .../test/resources/tests/query/SECONDARY_ID_MIXED/expected.csv | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/src/test/resources/tests/query/SECONDARY_ID_MIXED/expected.csv b/backend/src/test/resources/tests/query/SECONDARY_ID_MIXED/expected.csv index c80cfd0930..e55cf083b5 100644 --- a/backend/src/test/resources/tests/query/SECONDARY_ID_MIXED/expected.csv +++ b/backend/src/test/resources/tests/query/SECONDARY_ID_MIXED/expected.csv @@ -1,5 +1,4 @@ result,secondary,dates -a,f_a3,"{2024-06-30/2025-06-30,2026-06-30/2026-06-30}" a,f_a2,"{2014-06-30/2015-06-30,2024-06-30/2025-06-30,2026-06-30/2026-06-30}" a,f_a1,"{2014-06-30/2015-06-30,2016-06-30/2016-06-30,2024-06-30/2025-06-30,2026-06-30/2026-06-30}" b,f_b1,"{2015-02-03/2015-06-30,2025-02-03/2025-06-30}" \ No newline at end of file From e4b0f7e6f1b3e31ebacf71d04b509318d5645b50 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Mon, 22 Jan 2024 15:32:26 +0100 Subject: [PATCH 05/12] fix test that used secondaryIdMixed as dep --- .../conquery/integration/tests/ReusedQueryTest.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java index c774dfeef0..81f9e9ddac 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java @@ -86,7 +86,9 @@ public void execute(String name, TestConquery testConquery) throws Exception { final SecondaryIdQuery query = (SecondaryIdQuery) IntegrationUtils.parseQuery(conquery, test.getRawQuery()); - final ManagedExecutionId id = IntegrationUtils.assertQueryResult(conquery, query, 4L, ExecutionState.DONE, conquery.getTestUser(), 201); + final long expectedSize = 3L; + + final ManagedExecutionId id = IntegrationUtils.assertQueryResult(conquery, query, expectedSize, ExecutionState.DONE, conquery.getTestUser(), 201); assertThat(id).isNotNull(); @@ -126,7 +128,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { reused.setSecondaryId(query.getSecondaryId()); - IntegrationUtils.assertQueryResult(conquery, reused, 4L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, reused, expectedSize, ExecutionState.DONE, conquery.getTestUser(), 201); } // Reuse in SecondaryId, but do exclude @@ -170,7 +172,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { reused1.setSecondaryId(query.getSecondaryId()); - final ManagedExecutionId reused1Id = IntegrationUtils.assertQueryResult(conquery, reused1, 4L, ExecutionState.DONE, conquery.getTestUser(), 201); + final ManagedExecutionId reused1Id = IntegrationUtils.assertQueryResult(conquery, reused1, expectedSize, ExecutionState.DONE, conquery.getTestUser(), 201); final ManagedQuery execution1 = (ManagedQuery) metaStorage.getExecution(reused1Id); { final SecondaryIdQuery reused2 = new SecondaryIdQuery(); @@ -180,7 +182,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { final ManagedExecutionId reused2Id = - IntegrationUtils.assertQueryResult(conquery, reused2, 4L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, reused2, expectedSize, ExecutionState.DONE, conquery.getTestUser(), 201); final ManagedQuery execution2 = (ManagedQuery) metaStorage.getExecution(reused2Id); assertThat(reused2Id) @@ -227,7 +229,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { execution.createPermission(Set.of(Ability.READ)) )); - ManagedExecutionId copyId = IntegrationUtils.assertQueryResult(conquery, reused, 4L, ExecutionState.DONE, shareHolder, 201); + ManagedExecutionId copyId = IntegrationUtils.assertQueryResult(conquery, reused, expectedSize, ExecutionState.DONE, shareHolder, 201); ManagedExecution copy = metaStorage.getExecution(copyId); From 2e5fd27a1026f123bf245fec1e75db2e499b83a7 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Mon, 22 Jan 2024 15:58:14 +0100 Subject: [PATCH 06/12] adds documentation and removes unused class --- .../filter/CollectionNotEmptyFilterNode.java | 24 ------------------- .../query/queryplan/EventIterating.java | 6 +++-- 2 files changed, 4 insertions(+), 26 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/filter/CollectionNotEmptyFilterNode.java diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/CollectionNotEmptyFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/CollectionNotEmptyFilterNode.java deleted file mode 100644 index 4855e57c1f..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/CollectionNotEmptyFilterNode.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.bakdata.conquery.models.query.filter; - -import java.util.Collection; - -import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.filter.AggregationResultFilterNode; -import lombok.ToString; - - -/** - * Entity is included, when the collection is not empty. - */ -@ToString(callSuper = true) -public class CollectionNotEmptyFilterNode extends AggregationResultFilterNode>, FILTER_VALUE> { - - public CollectionNotEmptyFilterNode(Aggregator> aggregator) { - super(aggregator, null); - } - - @Override - public boolean isContained() { - return !getAggregator().createAggregationResult().isEmpty(); - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/EventIterating.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/EventIterating.java index 9bd7e76b6c..1b3569f23b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/EventIterating.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/EventIterating.java @@ -25,7 +25,7 @@ public abstract class EventIterating { public void collectRequiredTables(Set
requiredTables) { } - public Set
collectRequiredTables() { + public final Set
collectRequiredTables() { Set
out = new HashSet<>(); collectRequiredTables(out); return out; @@ -43,7 +43,9 @@ public void nextBlock(Bucket bucket) { } /** - * @implSpec returning false signals that the event may be discarded. + * Consume the event of the bucket. + * + * @implSpec If the event was not consumed, may return false. This can be used to discard intermediate results. Currently, only relevant for {@link SecondaryIdQueryPlan}, where subplans are discarded if no event was consumed. */ public abstract boolean acceptEvent(Bucket bucket, int event); From aba462edbe58fa49af277387f7f177e952c11397 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Mon, 29 Jan 2024 15:39:22 +0100 Subject: [PATCH 07/12] implement limit on SecondaryIdQuery subplans --- .../apiv1/query/SecondaryIdQuery.java | 4 ++- .../bakdata/conquery/commands/ShardNode.java | 3 +- .../conquery/models/config/QueryConfig.java | 2 ++ .../namespaces/specific/ExecuteForm.java | 2 +- .../namespaces/specific/ExecuteQuery.java | 2 +- .../conquery/models/query/QueryExecutor.java | 8 +++-- .../models/query/QueryPlanContext.java | 2 ++ .../query/queryplan/SecondaryIdQueryPlan.java | 34 +++++++++++++------ .../conquery/models/worker/Worker.java | 8 ++--- .../conquery/models/worker/Workers.java | 31 +++++++++-------- 10 files changed, 60 insertions(+), 36 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java index 31495a2cf2..b9eb628ddf 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java @@ -26,6 +26,7 @@ import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.RequiredEntities; import com.bakdata.conquery.models.query.Visitable; +import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; import com.bakdata.conquery.models.query.queryplan.SecondaryIdQueryPlan; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.SimpleResultInfo; @@ -66,8 +67,9 @@ public class SecondaryIdQuery extends Query { @Override public SecondaryIdQueryPlan createQueryPlan(QueryPlanContext context) { + final ConceptQueryPlan queryPlan = query.createQueryPlan(context.withSelectedSecondaryId(secondaryId)); - return new SecondaryIdQueryPlan(query, context, secondaryId, withSecondaryId, withoutSecondaryId, query.createQueryPlan(context.withSelectedSecondaryId(secondaryId))); + return new SecondaryIdQueryPlan(query, context, secondaryId, withSecondaryId, withoutSecondaryId, queryPlan, context.getSecondaryIdSubPlanLimit()); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java index a2bb9e62a2..c7c3d92138 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java @@ -106,7 +106,8 @@ protected void run(Environment environment, Namespace namespace, ConqueryConfig getConfig().getQueries().getExecutionPool(), () -> createInternalObjectMapper(View.Persistence.Shard.class), () -> createInternalObjectMapper(View.InternalCommunication.class), - getConfig().getCluster().getEntityBucketSize() + getConfig().getCluster().getEntityBucketSize(), + getConfig().getQueries().getMaxSecondaryIdSubPlans() ); final Collection workerStorages = config.getStorage().discoverWorkerStorages(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/QueryConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/QueryConfig.java index aa51f78a64..aaeac45a9a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/QueryConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/QueryConfig.java @@ -11,4 +11,6 @@ public class QueryConfig { private ThreadPoolDefinition executionPool = new ThreadPoolDefinition(); private Duration oldQueriesTime = Duration.days(30); + + private int maxSecondaryIdSubPlans = 30; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteForm.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteForm.java index 2b5e5184c5..2c0dc49022 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteForm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteForm.java @@ -66,7 +66,7 @@ public void react(Worker worker) throws Exception { // Before we start the query, we create it once to test if it will succeed before creating it multiple times for evaluation per core. try { - query.createQueryPlan(new QueryPlanContext(worker)); + query.createQueryPlan(new QueryPlanContext(worker, queryExecutor.getSecondaryIdSubPlanLimit())); } catch (Exception e) { ConqueryError err = asConqueryError(e); diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteQuery.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteQuery.java index 143f223faf..ecf6924645 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteQuery.java @@ -52,7 +52,7 @@ public void react(Worker worker) throws Exception { // Before we start the query, we create it once to test if it will succeed before creating it multiple times for evaluation per core. try { - query.createQueryPlan(new QueryPlanContext(worker)); + query.createQueryPlan(new QueryPlanContext(worker, queryExecutor.getSecondaryIdSubPlanLimit())); } catch (Exception e) { ConqueryError err = asConqueryError(e); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/QueryExecutor.java b/backend/src/main/java/com/bakdata/conquery/models/query/QueryExecutor.java index a2dda784be..ddee9379a4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/QueryExecutor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/QueryExecutor.java @@ -25,17 +25,19 @@ import com.bakdata.conquery.models.query.results.ShardResult; import com.bakdata.conquery.models.worker.Worker; import com.google.common.util.concurrent.MoreExecutors; -import lombok.RequiredArgsConstructor; +import lombok.Data; import lombok.extern.slf4j.Slf4j; @Slf4j -@RequiredArgsConstructor +@Data public class QueryExecutor implements Closeable { private final Worker worker; private final ThreadPoolExecutor executor; + private final int secondaryIdSubPlanLimit; + private final Set cancelledQueries = new HashSet<>(); public void unsetQueryCancelled(ManagedExecutionId query) { @@ -52,7 +54,7 @@ public boolean isCancelled(ManagedExecutionId query) { public boolean execute(Query query, QueryExecutionContext executionContext, ShardResult result, Set entities) { - final ThreadLocal> plan = ThreadLocal.withInitial(() -> query.createQueryPlan(new QueryPlanContext(worker))); + final ThreadLocal> plan = ThreadLocal.withInitial(() -> query.createQueryPlan(new QueryPlanContext(worker, secondaryIdSubPlanLimit))); if (entities.isEmpty()) { log.warn("Entities for query are empty"); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/QueryPlanContext.java b/backend/src/main/java/com/bakdata/conquery/models/query/QueryPlanContext.java index 2d61ca4d9c..a612253f06 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/QueryPlanContext.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/QueryPlanContext.java @@ -20,9 +20,11 @@ public class QueryPlanContext { @Getter(AccessLevel.NONE) private final Worker worker; + private final int secondaryIdSubPlanLimit; private CDateRange dateRestriction = CDateRange.all(); + /** * Set if in {@link com.bakdata.conquery.models.query.queryplan.SecondaryIdQueryPlan}, to the query-active {@link SecondaryIdDescriptionId}. */ diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java index 90025de01c..784f5e3565 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java @@ -15,6 +15,7 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.SecondaryIdDescription; import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId; import com.bakdata.conquery.models.query.QueryExecutionContext; @@ -62,6 +63,7 @@ public class SecondaryIdQueryPlan implements QueryPlan { */ @Getter(AccessLevel.NONE) private final Queue childPlanReusePool = new LinkedList<>(); + private final int subPlanLimit; /** * This is the same execution as a typical ConceptQueryPlan. The difference @@ -126,25 +128,31 @@ private void executeQueriesWithSecondaryId(QueryExecutionContext ctx, Entity ent final String key = ((String) bucket.createScriptValue(event, secondaryIdColumnId)); - if (childPerKey.containsKey(key)) { - final ConceptQueryPlan plan = childPerKey.get(key); + ConceptQueryPlan plan = childPerKey.get(key); + + if (plan != null) { plan.nextEvent(bucket, event); + continue; + } + + plan = createChild(ctxWithPhase, bucket); + final boolean consumed = plan.nextEvent(bucket, event); + + if (consumed) { + childPerKey.put(key, plan); } else { - final ConceptQueryPlan plan = createChild(ctxWithPhase, bucket); - final boolean consumed = plan.nextEvent(bucket, event); - - if (consumed) { - childPerKey.put(key, plan); - } - else { - childPlanReusePool.add(plan); - } + discardSubPlan(plan); } + } } } + private boolean discardSubPlan(ConceptQueryPlan plan) { + return childPlanReusePool.add(plan); + } + private void executeQueriesWithoutSecondaryId(QueryExecutionContext ctx, Entity entity, Table currentTable) { nextTable(ctx, currentTable); @@ -217,6 +225,10 @@ private boolean isOfInterest(Bucket bucket) { */ private ConceptQueryPlan createChild(QueryExecutionContext currentContext, Bucket currentBucket) { + if (childPerKey.size() >= subPlanLimit){ + throw new ConqueryError.ExecutionProcessingError();//TODO proper message + } + ConceptQueryPlan plan = childPlanReusePool.poll(); // Try to reuse old child plan first before allocating new ones diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java index 37c066e660..64c6873d59 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java @@ -71,7 +71,7 @@ public Worker( boolean failOnError, int entityBucketSize, ObjectMapper persistenceMapper, - ObjectMapper communicationMapper) { + ObjectMapper communicationMapper, int secondaryIdSubPlanLimit) { this.storage = storage; this.jobsExecutorService = jobsExecutorService; this.communicationMapper = communicationMapper; @@ -81,7 +81,7 @@ public Worker( storage.loadData(); jobManager = new JobManager(storage.getWorker().getName(), failOnError); - queryExecutor = new QueryExecutor(this, queryThreadPoolDefinition.createService("QueryExecutor %d")); + queryExecutor = new QueryExecutor(this, queryThreadPoolDefinition.createService("QueryExecutor %d"), secondaryIdSubPlanLimit); bucketManager = BucketManager.create(this, storage, entityBucketSize); } @@ -96,7 +96,7 @@ public static Worker newWorker( boolean failOnError, int entityBucketSize, ObjectMapper persistenceMapper, - ObjectMapper communicationMapper) { + ObjectMapper communicationMapper, int secondaryIdSubPlanLimit) { WorkerStorage workerStorage = new WorkerStorage(config, validator, directory); @@ -112,7 +112,7 @@ public static Worker newWorker( workerStorage.setWorker(info); workerStorage.close(); - return new Worker(queryThreadPoolDefinition, workerStorage, jobsExecutorService, failOnError, entityBucketSize, persistenceMapper, communicationMapper); + return new Worker(queryThreadPoolDefinition, workerStorage, jobsExecutorService, failOnError, entityBucketSize, persistenceMapper, communicationMapper, secondaryIdSubPlanLimit); } public ModificationShieldedWorkerStorage getStorage() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java index d6592fd7d9..a816cdad0b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java @@ -35,9 +35,9 @@ public class Workers extends IdResolveContext { @Getter @Setter private AtomicInteger nextWorker = new AtomicInteger(0); @Getter - private ConcurrentHashMap workers = new ConcurrentHashMap<>(); + private final ConcurrentHashMap workers = new ConcurrentHashMap<>(); @JsonIgnore - private transient Map dataset2Worker = new HashMap<>(); + private final transient Map dataset2Worker = new HashMap<>(); /** * Shared ExecutorService among Workers for Jobs. @@ -50,8 +50,10 @@ public class Workers extends IdResolveContext { private final int entityBucketSize; + private final int secondaryIdSubPlanLimit; + - public Workers(ThreadPoolDefinition queryThreadPoolDefinition, Supplier persistenceMapperSupplier, Supplier communicationMapperSupplier, int entityBucketSize) { + public Workers(ThreadPoolDefinition queryThreadPoolDefinition, Supplier persistenceMapperSupplier, Supplier communicationMapperSupplier, int entityBucketSize, int secondaryIdSubPlanLimit) { this.queryThreadPoolDefinition = queryThreadPoolDefinition; jobsThreadPool = queryThreadPoolDefinition.createService("Workers"); @@ -59,20 +61,21 @@ public Workers(ThreadPoolDefinition queryThreadPoolDefinition, Supplier Date: Mon, 29 Jan 2024 17:49:37 +0100 Subject: [PATCH 08/12] dont cache all subPlans --- .../query/queryplan/SecondaryIdQueryPlan.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java index 784f5e3565..1da42ef997 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java @@ -15,7 +15,6 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.SecondaryIdDescription; import com.bakdata.conquery.models.datasets.Table; -import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId; import com.bakdata.conquery.models.query.QueryExecutionContext; @@ -57,12 +56,17 @@ public class SecondaryIdQueryPlan implements QueryPlan { private final ConceptQueryPlan queryPlan; - private final Map childPerKey = new HashMap<>(); + private Map childPerKey; + + /** + * TODO borrow these from {@link QueryExecutionContext} + * * This helps us avoid allocations, instead allowing us to reuse the queries. */ @Getter(AccessLevel.NONE) private final Queue childPlanReusePool = new LinkedList<>(); + private final int subPlanLimit; /** @@ -225,9 +229,7 @@ private boolean isOfInterest(Bucket bucket) { */ private ConceptQueryPlan createChild(QueryExecutionContext currentContext, Bucket currentBucket) { - if (childPerKey.size() >= subPlanLimit){ - throw new ConqueryError.ExecutionProcessingError();//TODO proper message - } + //TODO consider limiting to a global amount of available subQueryPlans ConceptQueryPlan plan = childPlanReusePool.poll(); @@ -251,9 +253,9 @@ public void init(QueryExecutionContext ctx, Entity entity) { queryPlan.init(ctx, entity); // Dump the created children into reuse-pool - childPlanReusePool.addAll(childPerKey.values()); + childPerKey.values().stream().limit(10/*TODO Pull from config */).forEach(childPlanReusePool::add); - childPerKey.clear(); + childPerKey = new HashMap<>(); } @Override From 846ab243f3b86236191410653cfd37a9510f18e5 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 30 Jan 2024 10:21:24 +0100 Subject: [PATCH 09/12] limit retention of subPlans --- .../bakdata/conquery/apiv1/query/SecondaryIdQuery.java | 2 +- .../java/com/bakdata/conquery/commands/ShardNode.java | 2 +- .../bakdata/conquery/models/config/QueryConfig.java | 2 +- .../conquery/models/query/QueryPlanContext.java | 6 ++++-- .../models/query/queryplan/SecondaryIdQueryPlan.java | 5 +++-- .../com/bakdata/conquery/models/worker/Workers.java | 10 +++++----- 6 files changed, 15 insertions(+), 12 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java index b9eb628ddf..8a9138bf05 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java @@ -69,7 +69,7 @@ public class SecondaryIdQuery extends Query { public SecondaryIdQueryPlan createQueryPlan(QueryPlanContext context) { final ConceptQueryPlan queryPlan = query.createQueryPlan(context.withSelectedSecondaryId(secondaryId)); - return new SecondaryIdQueryPlan(query, context, secondaryId, withSecondaryId, withoutSecondaryId, queryPlan, context.getSecondaryIdSubPlanLimit()); + return new SecondaryIdQueryPlan(query, context, secondaryId, withSecondaryId, withoutSecondaryId, queryPlan, context.getSecondaryIdSubPlanRetention()); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java index c7c3d92138..0507eb8536 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java @@ -107,7 +107,7 @@ protected void run(Environment environment, Namespace namespace, ConqueryConfig () -> createInternalObjectMapper(View.Persistence.Shard.class), () -> createInternalObjectMapper(View.InternalCommunication.class), getConfig().getCluster().getEntityBucketSize(), - getConfig().getQueries().getMaxSecondaryIdSubPlans() + getConfig().getQueries().getSecondaryIdSubPlanRetention() ); final Collection workerStorages = config.getStorage().discoverWorkerStorages(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/QueryConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/QueryConfig.java index aaeac45a9a..ec8bdbad4f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/QueryConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/QueryConfig.java @@ -12,5 +12,5 @@ public class QueryConfig { private Duration oldQueriesTime = Duration.days(30); - private int maxSecondaryIdSubPlans = 30; + private int secondaryIdSubPlanRetention = 15; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/QueryPlanContext.java b/backend/src/main/java/com/bakdata/conquery/models/query/QueryPlanContext.java index a612253f06..38b258928f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/QueryPlanContext.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/QueryPlanContext.java @@ -11,16 +11,18 @@ import com.bakdata.conquery.models.worker.Worker; import lombok.AccessLevel; import lombok.AllArgsConstructor; +import lombok.Data; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.With; -@RequiredArgsConstructor @AllArgsConstructor @Getter @With +@Data @With +@AllArgsConstructor @RequiredArgsConstructor public class QueryPlanContext { @Getter(AccessLevel.NONE) private final Worker worker; - private final int secondaryIdSubPlanLimit; + private final int secondaryIdSubPlanRetention; private CDateRange dateRestriction = CDateRange.all(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java index 1da42ef997..ff3492d3b1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java @@ -44,6 +44,7 @@ @ToString public class SecondaryIdQueryPlan implements QueryPlan { + public static final int VALIDITY_DATE_POSITION = ConceptQueryPlan.VALIDITY_DATE_POSITION + 1; private final ConceptQuery query; private final QueryPlanContext queryPlanContext; @@ -67,7 +68,7 @@ public class SecondaryIdQueryPlan implements QueryPlan { @Getter(AccessLevel.NONE) private final Queue childPlanReusePool = new LinkedList<>(); - private final int subPlanLimit; + private final int subPlanRetentionLimit; /** * This is the same execution as a typical ConceptQueryPlan. The difference @@ -253,7 +254,7 @@ public void init(QueryExecutionContext ctx, Entity entity) { queryPlan.init(ctx, entity); // Dump the created children into reuse-pool - childPerKey.values().stream().limit(10/*TODO Pull from config */).forEach(childPlanReusePool::add); + childPerKey.values().stream().limit(subPlanRetentionLimit).forEach(childPlanReusePool::add); childPerKey = new HashMap<>(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java index a816cdad0b..f92900ecdb 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java @@ -50,10 +50,10 @@ public class Workers extends IdResolveContext { private final int entityBucketSize; - private final int secondaryIdSubPlanLimit; + private final int secondaryIdSubPlanRetention; - public Workers(ThreadPoolDefinition queryThreadPoolDefinition, Supplier persistenceMapperSupplier, Supplier communicationMapperSupplier, int entityBucketSize, int secondaryIdSubPlanLimit) { + public Workers(ThreadPoolDefinition queryThreadPoolDefinition, Supplier persistenceMapperSupplier, Supplier communicationMapperSupplier, int entityBucketSize, int secondaryIdSubPlanRetention) { this.queryThreadPoolDefinition = queryThreadPoolDefinition; jobsThreadPool = queryThreadPoolDefinition.createService("Workers"); @@ -61,7 +61,7 @@ public Workers(ThreadPoolDefinition queryThreadPoolDefinition, Supplier Date: Tue, 30 Jan 2024 11:47:51 +0100 Subject: [PATCH 10/12] fix NPE when resetting SecondaryIdQueryPlan --- .../conquery/models/query/queryplan/SecondaryIdQueryPlan.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java index ff3492d3b1..d48ad6d384 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java @@ -57,7 +57,7 @@ public class SecondaryIdQueryPlan implements QueryPlan { private final ConceptQueryPlan queryPlan; - private Map childPerKey; + private Map childPerKey = new HashMap<>(); /** From e0de079b430dd55be645ebe8a5c6dced7a694bc1 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 30 Jan 2024 11:49:09 +0100 Subject: [PATCH 11/12] also clear queue --- .../conquery/models/query/queryplan/SecondaryIdQueryPlan.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java index d48ad6d384..e1398e20df 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java @@ -254,6 +254,8 @@ public void init(QueryExecutionContext ctx, Entity entity) { queryPlan.init(ctx, entity); // Dump the created children into reuse-pool + childPlanReusePool.clear(); + childPerKey.values().stream().limit(subPlanRetentionLimit).forEach(childPlanReusePool::add); childPerKey = new HashMap<>(); From a1d61e5b7776c01b6fef556aff77ac27f24bd780 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Mon, 5 Feb 2024 14:13:16 +0100 Subject: [PATCH 12/12] adds docs --- .../com/bakdata/conquery/models/config/QueryConfig.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/QueryConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/QueryConfig.java index ec8bdbad4f..1cb33d7dfa 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/QueryConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/QueryConfig.java @@ -12,5 +12,11 @@ public class QueryConfig { private Duration oldQueriesTime = Duration.days(30); + /** + * Limits how many subQuery-Plans should be cached between executions: + * This number limits how many sub-plans are cached per core so that outliers do not cause massive memory overhead. + * + * TODO Implement global limit of active secondaryId sub plans + */ private int secondaryIdSubPlanRetention = 15; }