From e688d98caa25ae65a4919aec7566ce4402d5f3b5 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 31 Jul 2024 17:00:03 +0200 Subject: [PATCH 01/54] restructures ResultType and decouples printing into ResultPrinters: - ResultType is now very simple mix of primitive enum and LIST. (We hope to remove even more logic from it) - Result printing was always weird with Mappings embedded in StringT. We've moved that responsibility into ResultInfo via SelectResultInfo for mapped Selects. - This PR opens the possibility to embed anonymization into the ResultInfo. - PrintSettings now caches the C10n calls as they were before that sometimes invoked per cell, which should make rendering faster. --- .../bakdata/conquery/ConqueryConstants.java | 19 +- .../conquery/apiv1/query/CQElement.java | 8 +- .../apiv1/query/SecondaryIdQuery.java | 6 +- .../apiv1/query/TableExportQuery.java | 26 +- .../apiv1/query/concept/specific/CQAnd.java | 14 +- .../query/concept/specific/CQConcept.java | 4 +- .../apiv1/query/concept/specific/CQOr.java | 14 +- .../concept/specific/ResultInfoDecorator.java | 82 ---- .../concept/specific/external/CQExternal.java | 4 +- .../io/result/arrow/ArrowRenderer.java | 19 +- .../conquery/io/result/arrow/ArrowUtil.java | 29 +- .../conquery/io/result/csv/CsvRenderer.java | 2 +- .../io/result/excel/ExcelRenderer.java | 40 +- .../parquet/EntityResultWriteSupport.java | 53 +-- .../mode/local/LocalNamespaceHandler.java | 2 +- .../models/config/IdColumnConfig.java | 4 +- .../models/datasets/concepts/Concept.java | 2 +- .../datasets/concepts/select/Select.java | 6 +- .../select/concept/ConceptColumnSelect.java | 28 +- .../specific/EventDateUnionSelect.java | 4 +- .../specific/EventDurationSumSelect.java | 4 +- .../select/concept/specific/ExistsSelect.java | 4 +- .../concept/specific/QuarterSelect.java | 4 +- .../select/connector/SingleColumnSelect.java | 2 +- .../specific/CountOccurencesSelect.java | 2 +- .../specific/CountQuartersSelect.java | 5 +- .../connector/specific/CountSelect.java | 5 +- .../specific/DateDistanceSelect.java | 4 +- .../connector/specific/DateUnionSelect.java | 4 +- .../connector/specific/DurationSumSelect.java | 4 +- .../select/connector/specific/FlagSelect.java | 4 +- .../specific/MappableSingleColumnSelect.java | 31 +- .../connector/specific/PrefixSelect.java | 4 +- .../specific/QuartersInYearSelect.java | 4 +- .../select/connector/specific/SumSelect.java | 14 +- .../datasets/concepts/tree/TreeConcept.java | 2 +- .../forms/export/RelExportGenerator.java | 21 +- .../conquery/models/query/PrintSettings.java | 10 + .../models/query/SingleTableResult.java | 2 +- .../query/preview/EntityPreviewExecution.java | 58 +-- .../queryplan/specific/ExternalNode.java | 5 +- .../query/resultinfo/ColumnResultInfo.java | 14 +- .../LocalizedDefaultResultInfo.java | 37 +- .../models/query/resultinfo/ResultInfo.java | 24 +- .../query/resultinfo/SelectResultInfo.java | 10 +- .../query/resultinfo/SimpleResultInfo.java | 10 +- .../resultinfo/printers/ResultPrinters.java | 179 +++++++++ .../printers/SecondaryIdResultInfo.java | 52 +++ .../BooleanColumnStatsCollector.java | 6 +- .../statistics/ColumnStatsCollector.java | 29 +- .../statistics/DateColumnStatsCollector.java | 14 +- .../NumberColumnStatsCollector.java | 18 +- .../query/statistics/ResultStatistics.java | 6 +- .../StringColumnStatsCollector.java | 12 +- .../conquery/models/types/ResultType.java | 379 +++++------------- .../sql/conquery/SqlExecutionManager.java | 12 +- .../sql/conversion/NodeConversions.java | 8 +- .../conquery/sql/conversion/SqlConverter.java | 10 +- .../cqelement/ConversionContext.java | 3 + .../cqelement/concept/TablePath.java | 10 +- .../sql/conversion/model/NameGenerator.java | 10 +- .../query/TableExportQueryConverter.java | 2 +- .../sql/execution/SqlExecutionService.java | 6 +- .../com/bakdata/conquery/util/QueryUtils.java | 12 +- .../DefaultSqlCDateSetParserTest.java | 3 +- .../integration/tests/EntityExportTest.java | 4 +- .../conquery/io/result/ResultTestUtil.java | 24 +- .../arrow/ArrowResultGenerationTest.java | 2 +- .../result/csv/CsvResultGenerationTest.java | 2 +- .../result/excel/ExcelResultRenderTest.java | 6 +- .../models/query/DefaultColumnNameTest.java | 2 +- .../conquery/models/types/ResultTypeTest.java | 83 ++-- 72 files changed, 743 insertions(+), 805 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/ResultInfoDecorator.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/SecondaryIdResultInfo.java diff --git a/backend/src/main/java/com/bakdata/conquery/ConqueryConstants.java b/backend/src/main/java/com/bakdata/conquery/ConqueryConstants.java index 4c21a4cc77..cd4484a97f 100644 --- a/backend/src/main/java/com/bakdata/conquery/ConqueryConstants.java +++ b/backend/src/main/java/com/bakdata/conquery/ConqueryConstants.java @@ -2,10 +2,7 @@ import java.util.Set; -import c10n.C10N; -import com.bakdata.conquery.apiv1.forms.FeatureGroup; import com.bakdata.conquery.internationalization.ResultHeadersC10n; -import com.bakdata.conquery.models.forms.util.Resolution; import com.bakdata.conquery.models.query.resultinfo.LocalizedDefaultResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.types.ResultType; @@ -20,30 +17,30 @@ public class ConqueryConstants { public static final String EXTENSION_DESCRIPTION = ".import.json"; public static final ResultInfo DATES_INFO = - new LocalizedDefaultResultInfo((l) -> C10N.get(ResultHeadersC10n.class, l).dates(), new ResultType.ListT(ResultType.DateRangeT.INSTANCE), Set.of(new SemanticType.EventDateT())); + new LocalizedDefaultResultInfo((l) -> l.getLocalized(ResultHeadersC10n.class).dates(), new ResultType.ListT<>(ResultType.Primitive.DATE_RANGE), Set.of(new SemanticType.EventDateT())); public static final ResultInfo DATES_INFO_HISTORY = - new LocalizedDefaultResultInfo((l) -> C10N.get(ResultHeadersC10n.class, l).dates(), new ResultType.ListT(ResultType.DateRangeT.INSTANCE), Set.of(new SemanticType.EventDateT(), new SemanticType.GroupT())); + new LocalizedDefaultResultInfo((l) -> l.getLocalized(ResultHeadersC10n.class).dates(), new ResultType.ListT<>(ResultType.Primitive.DATE_RANGE), Set.of(new SemanticType.EventDateT(), new SemanticType.GroupT())); public static final ResultInfo SOURCE_INFO = - new LocalizedDefaultResultInfo((l) -> C10N.get(ResultHeadersC10n.class, l).source(), ResultType.StringT.INSTANCE, Set.of(new SemanticType.SourcesT(), new SemanticType.CategoricalT(), new SemanticType.GroupT())); + new LocalizedDefaultResultInfo((l) -> l.getLocalized(ResultHeadersC10n.class).source(), ResultType.Primitive.STRING, Set.of(new SemanticType.SourcesT(), new SemanticType.CategoricalT(), new SemanticType.GroupT())); // Form related constants public static final String SINGLE_RESULT_TABLE_NAME = "results"; public static final ResultInfo CONTEXT_INDEX_INFO = - new LocalizedDefaultResultInfo((l) -> C10N.get(ResultHeadersC10n.class, l).index(), ResultType.IntegerT.INSTANCE, Set.of()); + new LocalizedDefaultResultInfo((l) -> l.getLocalized(ResultHeadersC10n.class).index(), ResultType.Primitive.INTEGER, Set.of()); public static final ResultInfo DATE_RANGE_INFO = - new LocalizedDefaultResultInfo((l) -> C10N.get(ResultHeadersC10n.class, l).dateRange(), ResultType.DateRangeT.INSTANCE, Set.of()); + new LocalizedDefaultResultInfo((l) -> l.getLocalized(ResultHeadersC10n.class).dateRange(), ResultType.Primitive.DATE_RANGE, Set.of()); public static final ResultInfo RESOLUTION_INFO = - new LocalizedDefaultResultInfo((l) -> C10N.get(ResultHeadersC10n.class, l).resolution(), new ResultType.StringT(Resolution::localizeValue), Set.of(new SemanticType.ResolutionT())); + new LocalizedDefaultResultInfo((l) -> l.getLocalized(ResultHeadersC10n.class).resolution(), ResultType.Primitive.STRING, Set.of(new SemanticType.ResolutionT())); public static final ResultInfo EVENT_DATE_INFO = - new LocalizedDefaultResultInfo((l) -> C10N.get(ResultHeadersC10n.class, l).eventDate(), ResultType.DateT.INSTANCE, Set.of()); + new LocalizedDefaultResultInfo((l) -> l.getLocalized(ResultHeadersC10n.class).eventDate(), ResultType.Primitive.DATE, Set.of()); public static final ResultInfo OBSERVATION_SCOPE_INFO = - new LocalizedDefaultResultInfo((l) -> C10N.get(ResultHeadersC10n.class, l).observationScope(), new ResultType.StringT(FeatureGroup::localizeValue), Set.of()); + new LocalizedDefaultResultInfo((l) -> l.getLocalized(ResultHeadersC10n.class).observationScope(), ResultType.Primitive.STRING, Set.of()); /** * Drawn from random.org diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQElement.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQElement.java index 61806adfb4..17d7e4424b 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQElement.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQElement.java @@ -2,7 +2,6 @@ import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Set; import java.util.function.Consumer; @@ -11,6 +10,7 @@ import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -38,16 +38,16 @@ public abstract class CQElement implements Visitable { @Getter private String label; - public String getUserOrDefaultLabel(Locale locale) { + public String getUserOrDefaultLabel(PrintSettings printSettings) { // Prefer the user label if (label != null) { return label; } - return defaultLabel(locale); + return defaultLabel(printSettings); } @NotNull - public String defaultLabel(Locale locale) { + public String defaultLabel(PrintSettings printSettings) { // Fallback to CPSType#id() implementation is provided or class name final CPSType type = getClass().getAnnotation(CPSType.class); if (type != null) { diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java index ec3918246d..b294f984ac 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java @@ -26,9 +26,7 @@ import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; import com.bakdata.conquery.models.query.queryplan.SecondaryIdQueryPlan; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.models.query.resultinfo.SimpleResultInfo; -import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.models.types.SemanticType; +import com.bakdata.conquery.models.query.resultinfo.printers.SecondaryIdResultInfo; import com.fasterxml.jackson.annotation.JsonView; import jakarta.validation.constraints.NotNull; import lombok.Getter; @@ -129,7 +127,7 @@ public void resolve(final QueryResolveContext context) { public List getResultInfos() { final List resultInfos = new ArrayList<>(); - resultInfos.add(new SimpleResultInfo(secondaryId.getLabel(), ResultType.StringT.INSTANCE, secondaryId.getDescription(), Set.of(new SemanticType.SecondaryIdT(getSecondaryId())))); + resultInfos.add(new SecondaryIdResultInfo(secondaryId)); resultInfos.addAll(query.getResultInfos()); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java index 8f6576d08a..4d52f00f4a 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java @@ -41,7 +41,8 @@ import com.bakdata.conquery.models.query.queryplan.TableExportQueryPlan; import com.bakdata.conquery.models.query.resultinfo.ColumnResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.models.query.resultinfo.SimpleResultInfo; +import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.SecondaryIdResultInfo; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import com.fasterxml.jackson.annotation.JsonCreator; @@ -229,22 +230,7 @@ private List createResultInfos(Map final SecondaryIdDescription desc = e.getKey(); final Integer pos = e.getValue(); - // If mapping is available, values are mapped - final ResultType.StringT resultType = - desc.getMapping() != null - ? new ResultType.StringT((internal, printSettings) -> { - if (internal == null) { - return null; - } - return desc.getMapping().external((String) internal); - }) - : ResultType.StringT.INSTANCE; - - final Set semantics = new HashSet<>(); - - semantics.add(new SemanticType.SecondaryIdT(desc)); - - infos[pos] = new SimpleResultInfo(desc.getLabel(), resultType, desc.getDescription(), semantics); + infos[pos] = new SecondaryIdResultInfo(desc); } @@ -275,6 +261,7 @@ private List createResultInfos(Map final Set semantics = new HashSet<>(); ResultType resultType = ResultType.resolveResultType(column.getType()); + ResultPrinters.Printer printer = ResultPrinters.defaultPrinter(resultType); if (connectorColumns.containsKey(column)) { // Additionally, Concept Columns are returned as ConceptElementId, when rawConceptColumns is not set. @@ -285,7 +272,8 @@ private List createResultInfos(Map semantics.add(new SemanticType.ConceptColumnT(concept)); if (!isRawConceptValues()) { - resultType = new ResultType.StringT(concept::printConceptLocalId); + resultType = ResultType.Primitive.STRING; + printer = concept::printConceptLocalId; } } else { @@ -293,7 +281,7 @@ private List createResultInfos(Map semantics.add(new SemanticType.ColumnT(column)); } - infos[position] = new ColumnResultInfo(column, resultType, semantics); + infos[position] = new ColumnResultInfo(column, resultType, semantics, printer, column.getDescription()); } return List.of(infos); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java index f598846635..78ad6a3939 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java @@ -3,18 +3,17 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Locale; import java.util.Optional; import java.util.Set; import java.util.function.Consumer; -import c10n.C10N; import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.internationalization.CQElementC10n; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -27,6 +26,7 @@ import com.bakdata.conquery.models.query.queryplan.specific.AndNode; import com.bakdata.conquery.models.query.resultinfo.LocalizedDefaultResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.util.QueryUtils; import com.fasterxml.jackson.annotation.JsonView; @@ -116,25 +116,25 @@ public List getResultInfos() { } if (createExists()) { - resultInfos.add(new LocalizedDefaultResultInfo(this::getUserOrDefaultLabel, this::defaultLabel, ResultType.BooleanT.INSTANCE, Set.of())); + resultInfos.add(new LocalizedDefaultResultInfo(this::getUserOrDefaultLabel, this::defaultLabel, ResultType.Primitive.BOOLEAN, new ResultPrinters.BooleanPrinter(), Set.of())); } return resultInfos; } @Override - public String getUserOrDefaultLabel(Locale locale) { + public String getUserOrDefaultLabel(PrintSettings printSettings) { // Prefer the user label if (getLabel() != null) { return getLabel(); } - return QueryUtils.createDefaultMultiLabel(children, " " + C10N.get(CQElementC10n.class, locale).and() + " ", locale); + return QueryUtils.createDefaultMultiLabel(children, " " + printSettings.getLocalized(CQElementC10n.class).and() + " ", printSettings); } @Override - public String defaultLabel(Locale locale) { + public String defaultLabel(PrintSettings printSettings) { // This forces the default label on children even if there was a user label - return QueryUtils.createTotalDefaultMultiLabel(children, " " + C10N.get(CQElementC10n.class, locale).and() + " ", locale); + return QueryUtils.createTotalDefaultMultiLabel(children, " " + printSettings.getLocalized(CQElementC10n.class).and() + " ", printSettings); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java index be0e5814e9..bec74db77b 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java @@ -3,7 +3,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Locale; import java.util.Set; import java.util.stream.Collectors; @@ -24,6 +23,7 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.DateAggregationMode; import com.bakdata.conquery.models.query.NamespacedIdentifiableHolding; +import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -128,7 +128,7 @@ public static CQConcept forConnector(Connector source) { } @Override - public String defaultLabel(Locale locale) { + public String defaultLabel(PrintSettings printSettings) { if (elements.isEmpty()) { return null; } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java index e93562016d..0454ccd297 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java @@ -3,18 +3,17 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Locale; import java.util.Optional; import java.util.Set; import java.util.function.Consumer; -import c10n.C10N; import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.internationalization.CQElementC10n; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -27,6 +26,7 @@ import com.bakdata.conquery.models.query.queryplan.specific.OrNode; import com.bakdata.conquery.models.query.resultinfo.LocalizedDefaultResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.util.QueryUtils; import com.fasterxml.jackson.annotation.JsonView; @@ -121,25 +121,25 @@ public List getResultInfos() { } if (createExists()) { - resultInfos.add(new LocalizedDefaultResultInfo(this::getUserOrDefaultLabel, this::defaultLabel, ResultType.BooleanT.INSTANCE, Set.of())); + resultInfos.add(new LocalizedDefaultResultInfo(this::getUserOrDefaultLabel, this::defaultLabel, ResultType.Primitive.BOOLEAN, new ResultPrinters.BooleanPrinter(), Set.of())); } return resultInfos; } @Override - public String getUserOrDefaultLabel(Locale locale) { + public String getUserOrDefaultLabel(PrintSettings printSettings) { // Prefer the user label if (getLabel() != null) { return getLabel(); } - return QueryUtils.createDefaultMultiLabel(children, " " + C10N.get(CQElementC10n.class, locale).or() + " ", locale); + return QueryUtils.createDefaultMultiLabel(children, " " + printSettings.getLocalized(CQElementC10n.class).or() + " ", printSettings); } @Override - public String defaultLabel(Locale locale) { + public String defaultLabel(PrintSettings printSettings) { // This forces the default label on children even if there was a user label - return QueryUtils.createTotalDefaultMultiLabel(children, " " + C10N.get(CQElementC10n.class, locale).or() + " ", locale); + return QueryUtils.createTotalDefaultMultiLabel(children, " " + printSettings.getLocalized(CQElementC10n.class).or() + " ", printSettings); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/ResultInfoDecorator.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/ResultInfoDecorator.java deleted file mode 100644 index 3f7b5454cd..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/ResultInfoDecorator.java +++ /dev/null @@ -1,82 +0,0 @@ -package com.bakdata.conquery.apiv1.query.concept.specific; - -import java.util.List; -import java.util.Set; -import java.util.function.Consumer; - -import com.bakdata.conquery.apiv1.query.CQElement; -import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; -import com.bakdata.conquery.models.query.QueryExecutionContext; -import com.bakdata.conquery.models.query.QueryPlanContext; -import com.bakdata.conquery.models.query.QueryResolveContext; -import com.bakdata.conquery.models.query.RequiredEntities; -import com.bakdata.conquery.models.query.Visitable; -import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; -import com.bakdata.conquery.models.query.queryplan.QPNode; -import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.google.common.collect.ClassToInstanceMap; -import com.google.common.collect.MutableClassToInstanceMap; -import jakarta.validation.constraints.NotNull; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.ToString; - -/** - * A wrapper for {@link CQElement}s to provide additional infos to parts of a query. - */ -@Getter -@Setter -@ToString -@NoArgsConstructor -@AllArgsConstructor -@CPSType(id = "RESULT_INFO_DECORATOR", base = CQElement.class) -public class ResultInfoDecorator extends CQElement { - - @NotNull - private ClassToInstanceMap values = MutableClassToInstanceMap.create(); - @NotNull - private CQElement child; - - @SuppressWarnings({"rawtypes", "unchecked"}) - @Override - public List getResultInfos() { - List resultInfos = child.getResultInfos(); - resultInfos.listIterator() - .forEachRemaining(sd -> { - for (Class entry : values.keySet()) { - sd.addAppendix(entry, values.getInstance(entry)); - } - }); - return resultInfos; - } - - @Override - public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { - return child.createQueryPlan(context, plan); - } - - @Override - public void visit(Consumer visitor) { - super.visit(visitor); - child.visit(visitor); - } - - @Override - public void collectRequiredQueries(Set requiredQueries) { - child.collectRequiredQueries(requiredQueries); - } - - @Override - public void resolve(QueryResolveContext context) { - child.resolve(context); - } - - - @Override - public RequiredEntities collectRequiredEntities(QueryExecutionContext context) { - return getChild().collectRequiredEntities(context); - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java index 2ba0d02a0b..1ed2427791 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -227,8 +227,8 @@ public List getResultInfos() { String column = headers[col]; resultInfos.add(new SimpleResultInfo(column, onlySingles ? - ResultType.StringT.INSTANCE : - new ResultType.ListT(ResultType.StringT.INSTANCE))); + ResultType.Primitive.STRING : + new ResultType.ListT(ResultType.Primitive.STRING))); } return resultInfos; diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java index 826c7a6133..9c54fa1b0f 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java @@ -15,7 +15,6 @@ import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; import com.bakdata.conquery.models.query.results.EntityResult; -import com.bakdata.conquery.models.types.ResultType; import lombok.extern.slf4j.Slf4j; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.BitVector; @@ -232,22 +231,22 @@ public static RowConsumer[] generateWriterPipeline(VectorSchemaRoot root, int ve RowConsumer[] builder = new RowConsumer[numVectors]; for ( - int vecI = vectorOffset; - (vecI < root.getFieldVectors().size()) && (vecI < vectorOffset + numVectors); - vecI++ - ) { + int vecI = vectorOffset; + (vecI < root.getFieldVectors().size()) && (vecI < vectorOffset + numVectors); + vecI++ + ) { final int pos = vecI - vectorOffset; final FieldVector vector = root.getVector(vecI); final ResultInfo resultInfo = resultInfos.get(pos); builder[pos] = - generateVectorFiller(pos, vector, settings, resultInfo.getType()); + generateVectorFiller(pos, vector, settings, resultInfo); } return builder; } - private static RowConsumer generateVectorFiller(int pos, ValueVector vector, final PrintSettings settings, ResultType resultType) { + private static RowConsumer generateVectorFiller(int pos, ValueVector vector, final PrintSettings settings, ResultInfo resultInfo) { //TODO When Pattern-matching lands, clean this up. (Think Java 12?) if (vector instanceof IntVector) { return intVectorFiller((IntVector) vector, (line) -> (Integer) line[pos]); @@ -265,7 +264,7 @@ private static RowConsumer generateVectorFiller(int pos, ValueVector vector, fin // If there is no value, we don't want to have it displayed as an empty string (see next if) return null; } - return resultType.printNullable(settings, line[pos]); + return resultInfo.printNullable(settings, line[pos]); }); } @@ -290,7 +289,7 @@ private static RowConsumer generateVectorFiller(int pos, ValueVector vector, fin List nestedVectors = structVector.getPrimitiveVectors(); RowConsumer [] nestedConsumers = new RowConsumer[nestedVectors.size()]; for (int i = 0; i < nestedVectors.size(); i++) { - nestedConsumers[i] = generateVectorFiller(i, nestedVectors.get(i), settings, resultType); + nestedConsumers[i] = generateVectorFiller(i, nestedVectors.get(i), settings, resultInfo); } return structVectorFiller(structVector, nestedConsumers, (line) -> (List) line[pos]); } @@ -300,7 +299,7 @@ private static RowConsumer generateVectorFiller(int pos, ValueVector vector, fin ValueVector nestedVector = listVector.getDataVector(); // pos = 0 is a workaround for now - return listVectorFiller(listVector, generateVectorFiller(0, nestedVector, settings, ((ResultType.ListT) (resultType)).getElementType()), (line) -> (List) line[pos]); + return listVectorFiller(listVector, generateVectorFiller(0, nestedVector, settings, resultInfo), (line) -> (List) line[pos]); } throw new IllegalArgumentException("Unsupported vector type " + vector); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java index 12458db1cc..2253cdbacb 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java @@ -2,7 +2,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.function.BiFunction; import java.util.stream.Collectors; @@ -24,15 +23,21 @@ public class ArrowUtil { public final static RootAllocator ROOT_ALLOCATOR = new RootAllocator(); - private final static Map, BiFunction> FIELD_MAP = Map.of( - ResultType.BooleanT.class, ArrowUtil::boolField, - ResultType.IntegerT.class, ArrowUtil::integerField, - ResultType.NumericT.class, ArrowUtil::floatField, - ResultType.DateT.class, ArrowUtil::dateField, - ResultType.DateRangeT.class, ArrowUtil::dateRangeField, - ResultType.MoneyT.class, ArrowUtil::integerField, - ResultType.ListT.class, ArrowUtil::listField - ); + public BiFunction fieldFor(ResultType type) { + if (type instanceof ResultType.ListT) { + return ArrowUtil::listField; + } + + return switch (((ResultType.Primitive) type)) { + case BOOLEAN -> ArrowUtil::boolField; + case INTEGER, MONEY -> ArrowUtil::integerField; + case NUMERIC -> ArrowUtil::floatField; + case DATE -> ArrowUtil::dateField; + case DATE_RANGE -> ArrowUtil::dateRangeField; + case STRING -> ArrowUtil::stringField; + }; + } + private static Field stringField(ResultInfo info, @NonNull String uniqueName) { return new Field(uniqueName, FieldType.nullable(new ArrowType.Utf8()), null); @@ -70,7 +75,7 @@ private static Field listField(ResultInfo info, @NonNull String uniqueName) { } final ResultType elementType = ((ResultType.ListT) info.getType()).getElementType(); - BiFunction nestedFieldCreator = FIELD_MAP.getOrDefault(elementType.getClass(), ArrowUtil::stringField); + BiFunction nestedFieldCreator = fieldFor(elementType); final Field nestedField = nestedFieldCreator.apply(info, uniqueName); return new Field( uniqueName, @@ -88,7 +93,7 @@ private static Field listField(ResultInfo info, @NonNull String uniqueName) { */ public Field createField(ResultInfo info, UniqueNamer collector) { // Fallback to string field if type is not explicitly registered - BiFunction fieldCreator = FIELD_MAP.getOrDefault(info.getType().getClass(), ArrowUtil::stringField); + BiFunction fieldCreator = fieldFor(info.getType()); return fieldCreator.apply(info, collector.getUniqueName(info)); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java index 3eca172f01..f2c6bb74a6 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java @@ -47,7 +47,7 @@ public void printLine(PrintSettings cfg, EntityPrintId entity, List writer.addValues((Object[]) entity.getExternalId()); try { for (int i = 0; i < infos.size(); i++) { - writer.addValue(infos.get(i).getType().printNullable(cfg, value[i])); + writer.addValue(infos.get(i).printNullable(cfg, value[i])); } } catch (Exception e) { diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java index b38c9d4735..2331c00940 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java @@ -43,13 +43,31 @@ public class ExcelRenderer { public static final int MAX_LINES = 1_048_576; - private static final Map, TypeWriter> TYPE_WRITER_MAP = Map.of( - ResultType.DateT.class, ExcelRenderer::writeDateCell, - ResultType.IntegerT.class, ExcelRenderer::writeIntegerCell, - ResultType.MoneyT.class, ExcelRenderer::writeMoneyCell, - ResultType.NumericT.class, ExcelRenderer::writeNumericCell, - ResultType.BooleanT.class, ExcelRenderer::writeBooleanCell - ); +// private static final Map, TypeWriter> TYPE_WRITER_MAP = Map.of( +// ResultType.DateT.class, ExcelRenderer::writeDateCell, +// ResultType.IntegerT.class, ExcelRenderer::writeIntegerCell, +// ResultType.MoneyT.class, ExcelRenderer::writeMoneyCell, +// ResultType.NumericT.class, ExcelRenderer::writeNumericCell, +// ResultType.BooleanT.class, ExcelRenderer::writeBooleanCell +// ); + + public TypeWriter writer(ResultType type) { + if(!(type instanceof ResultType.ListT)){ + //TODO this does not seem right + return ExcelRenderer::writeStringCell; + } + + return switch (((ResultType.Primitive) type)) { + case BOOLEAN -> ExcelRenderer::writeBooleanCell; + case INTEGER -> ExcelRenderer::writeIntegerCell; + case MONEY -> ExcelRenderer::writeMoneyCell; + case NUMERIC -> ExcelRenderer::writeNumericCell; + case DATE -> ExcelRenderer::writeDateCell; + case STRING -> ExcelRenderer::writeStringCell; + default -> ExcelRenderer::writeStringCell; + }; + } + public static final int CHARACTER_WIDTH_DIVISOR = 256; public static final int AUTOFILTER_SPACE_WIDTH = 3; @@ -258,7 +276,7 @@ private int writeRowsForEntity( } // Fallback to string if type is not explicitly registered - TypeWriter typeWriter = TYPE_WRITER_MAP.getOrDefault(resultInfo.getType().getClass(), ExcelRenderer::writeStringCell); + TypeWriter typeWriter = writer(resultInfo.getType()); typeWriter.writeCell(resultInfo, settings, dataCell, resultValue, styles); } @@ -298,14 +316,14 @@ private void setColumnWidthsAndUntrack(SXSSFSheet sheet) { // Type specific cell writers private static void writeStringCell(ResultInfo info, PrintSettings settings, Cell cell, Object value, Map styles) { cell.setCellValue( - info.getType().printNullable( + info.printNullable( settings, value )); } /** - * This writer is only used on Columns with the result type {@link ResultType.BooleanT}, not on complex types such as `LIST[BOOLEAN]`, + * This writer is only used on Columns with the result type {@link ResultType.Primitive#BOOLEAN}, not on complex types such as `LIST[BOOLEAN]`, * because MS Excel can only represent those as strings */ private static void writeBooleanCell(ResultInfo info, PrintSettings settings, Cell cell, Object value, Map styles) { @@ -313,7 +331,7 @@ private static void writeBooleanCell(ResultInfo info, PrintSettings settings, Ce cell.setCellValue(aBoolean); return; } - cell.setCellValue(info.getType().printNullable(settings, value)); + cell.setCellValue(info.printNullable(settings, value)); } private static void writeDateCell(ResultInfo info, PrintSettings settings, Cell cell, Object value, Map styles) { diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java index 95053843f3..acfc29a2fa 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java @@ -9,9 +9,11 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; +import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.MultilineEntityResult; import com.bakdata.conquery.models.types.ResultType; +import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.arrow.vector.types.pojo.Schema; @@ -118,15 +120,15 @@ public static MessageType generateSchema( } - @RequiredArgsConstructor + @Data private static class StringTColumnConsumer implements ColumnConsumer { - private final ResultType.StringT resultType; + private final ResultPrinters.Printer printer; private final PrintSettings printSettings; @Override public void accept(RecordConsumer recordConsumer, Object o) { - final String printValue = resultType.printNullable(printSettings, o); + final String printValue = getPrinter().print(printSettings, o); recordConsumer.addBinary(Binary.fromString(printValue)); } } @@ -190,13 +192,11 @@ public void accept(RecordConsumer recordConsumer, Object o) { @RequiredArgsConstructor private static class ListTColumnConsumer implements ColumnConsumer { - private final ResultType.ListT resultType; + private final ColumnConsumer elementConsumer; private final PrintSettings printSettings; @Override public void accept(RecordConsumer recordConsumer, Object o) { - final ResultType elementType = resultType.getElementType(); - final ColumnConsumer elementConsumer = getForResultType(elementType, printSettings); List list = (List) o; @@ -225,41 +225,28 @@ public void accept(RecordConsumer recordConsumer, Object o) { private static List generateColumnConsumers(List idHeaders, List resultInfos, PrintSettings printSettings) { final List consumers = new ArrayList<>(); for (ResultInfo idHeader : idHeaders) { - consumers.add(getForResultType(idHeader.getType(), printSettings)); + consumers.add(getForResultType(idHeader.getType(), idHeader, printSettings)); } for (ResultInfo resultInfo : resultInfos) { - consumers.add(getForResultType(resultInfo.getType(), printSettings)); + consumers.add(getForResultType(resultInfo.getType(), resultInfo, printSettings)); } return consumers; } - private static ColumnConsumer getForResultType(ResultType resultType, PrintSettings printSettings) { - if (resultType instanceof ResultType.StringT) { - return new StringTColumnConsumer((ResultType.StringT) resultType, printSettings); - } - else if (resultType instanceof ResultType.BooleanT) { - return new BooleanTColumnConsumer(); - } - else if (resultType instanceof ResultType.IntegerT) { - return new IntegerTColumnConsumer(); - } - else if (resultType instanceof ResultType.NumericT) { - return new NumericTColumnConsumer(); - } - else if (resultType instanceof ResultType.MoneyT) { - return new IntegerTColumnConsumer(); - } - else if (resultType instanceof ResultType.DateT) { - return new IntegerTColumnConsumer(); - } - else if (resultType instanceof ResultType.DateRangeT) { - return new DateRangeTColumnConsumer(); - } - else if (resultType instanceof ResultType.ListT) { - return new ListTColumnConsumer((ResultType.ListT) resultType, printSettings); + private static ColumnConsumer getForResultType(ResultType resultType, ResultInfo resultInfo, PrintSettings printSettings) { + + if (resultType instanceof ResultType.ListT listT) { + //TODO ensure that this works with recursion and mappings + return new ListTColumnConsumer(getForResultType(listT.getElementType(), resultInfo, printSettings), printSettings); } - throw new IllegalArgumentException(String.format("Cannot support ResultType %s", resultType)); + return switch (((ResultType.Primitive) resultType)) { + case BOOLEAN -> new BooleanTColumnConsumer(); + case INTEGER, DATE, MONEY -> new IntegerTColumnConsumer(); + case NUMERIC -> new NumericTColumnConsumer(); + case DATE_RANGE -> new DateRangeTColumnConsumer(); + case STRING -> new StringTColumnConsumer(resultInfo::printNullable, printSettings); + }; } } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java index a0f7f23c2d..dc973f2832 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java @@ -52,7 +52,7 @@ public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaSto ResultSetProcessor resultSetProcessor = ResultSetProcessorFactory.create(config, sqlDialect); SqlExecutionService sqlExecutionService = new SqlExecutionService(dslContext, resultSetProcessor); NodeConversions nodeConversions = new NodeConversions(idColumns, sqlDialect, dslContext, databaseConfig, sqlExecutionService); - SqlConverter sqlConverter = new SqlConverter(nodeConversions); + SqlConverter sqlConverter = new SqlConverter(nodeConversions, config); ExecutionManager executionManager = new SqlExecutionManager(sqlConverter, sqlExecutionService, metaStorage); SqlStorageHandler sqlStorageHandler = new SqlStorageHandler(sqlExecutionService); SqlEntityResolver sqlEntityResolver = new SqlEntityResolver(idColumns, dslContext, sqlDialect, sqlExecutionService); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java index 9c8d1202f6..d2cd559bee 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java @@ -14,6 +14,7 @@ import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.query.resultinfo.LocalizedDefaultResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -139,7 +140,8 @@ public List getIdResultInfos() { ); }, locale -> col.getField(), - ResultType.StringT.getINSTANCE(), + ResultType.Primitive.STRING, + ResultPrinters::printString, //TODO we can now hook our anonymizers into this Set.of(new SemanticType.IdT(col.getName())) )) .collect(Collectors.toUnmodifiableList()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java index b7f3d6a27d..0ecbf8869c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java @@ -66,7 +66,7 @@ public abstract class Concept extends ConceptElemen *

* If {@link PrintSettings#isPrettyPrint()} is false, {@link ConceptElement#getId()} is used to print. */ - public abstract String printConceptLocalId(Object rawValue, PrintSettings printSettings); + public abstract String printConceptLocalId(PrintSettings printSettings, Object rawValue); public List getDefaultSelects() { return getSelects().stream().filter(Select::isDefault).collect(Collectors.toList()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/ConceptElement.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/ConceptElement.java index 68efca1c56..3b458d9696 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/ConceptElement.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/ConceptElement.java @@ -6,11 +6,9 @@ import javax.annotation.CheckForNull; import com.bakdata.conquery.apiv1.KeyValue; -import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeChild; import com.bakdata.conquery.models.identifiable.Labeled; import com.bakdata.conquery.models.identifiable.ids.NamespacedIdentifiable; import com.bakdata.conquery.models.identifiable.ids.specific.ConceptElementId; -import com.bakdata.conquery.models.identifiable.ids.specific.ConceptTreeChildId; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -37,10 +35,6 @@ public abstract class ConceptElement createAggregator() { @Override public ResultPrinters.Printer createPrinter(PrintSettings printSettings) { if (isAsIds()) { - return new ResultPrinters.ListPrinter((rawValue) -> getHolder().findConcept().printConceptLocalId(printSettings, rawValue), printSettings); + return new ResultPrinters.ListPrinter(new ResultPrinters.ConceptIdPrinter(getHolder().findConcept(), printSettings), printSettings); } return new ResultPrinters.ListPrinter(new ResultPrinters.StringPrinter(), printSettings); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/TreeConcept.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/TreeConcept.java index 0fc259936c..8cbc0d80f4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/TreeConcept.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/TreeConcept.java @@ -6,8 +6,6 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Stream; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.Initializing; @@ -18,11 +16,12 @@ import com.bakdata.conquery.models.datasets.concepts.select.concept.UniversalSelect; import com.bakdata.conquery.models.exceptions.ConceptConfigurationException; import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId; -import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.util.CalculatedValue; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonManagedReference; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -209,33 +208,5 @@ public ConceptTreeNode getElementByLocalId(int localId) { return localIdMap.get(localId); } - /** - * rawValue is expected to be an Integer, expressing a localId for {@link TreeConcept#getElementByLocalId(int)}. - *

- * If {@link PrintSettings#isPrettyPrint()} is true, {@link ConceptElement#getLabel()} is used to print. - * If {@link PrintSettings#isPrettyPrint()} is false, {@link ConceptElement#getId()} is used to print. - */ - public String printConceptLocalId(PrintSettings printSettings, Object rawValue) { - - if (rawValue == null) { - return null; - } - - final int localId = (int) rawValue; - - final ConceptTreeNode node = getElementByLocalId(localId); - - if (!printSettings.isPrettyPrint()) { - return node.getId().toString(); - } - - if (node.getDescription() == null) { - return node.getLabel(); - } - - return node.getLabel() + " - " + node.getDescription(); - - } - public static class TreeConceptInitializer extends Initializing.Converter {} } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java index 4b454bb79e..1f1f87983e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java @@ -10,13 +10,13 @@ import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.config.LocaleConfig; import com.bakdata.conquery.models.datasets.concepts.Concept; +import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeNode; +import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.models.index.InternToExternMapper; import com.bakdata.conquery.models.query.C10nCache; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.types.ResultType; import com.google.common.base.Preconditions; -import lombok.RequiredArgsConstructor; -import lombok.ToString; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; @@ -48,8 +48,8 @@ public interface Printer { String print(Object f); } - @ToString() - public static class StringPrinter implements Printer { + + public record StringPrinter() implements Printer { @Override public String print(Object f) { return Objects.toString(f); @@ -57,10 +57,7 @@ public String print(Object f) { } - @ToString() - @RequiredArgsConstructor - public static class IntegerPrinter implements Printer { - private final PrintSettings cfg; + public record IntegerPrinter(PrintSettings cfg) implements Printer { @Override public String print(Object f) { @@ -72,10 +69,7 @@ public String print(Object f) { } } - @ToString() - @RequiredArgsConstructor - public static class NumericPrinter implements Printer { - private final PrintSettings cfg; + public record NumericPrinter(PrintSettings cfg) implements Printer { @Override public String print(Object f) { @@ -87,10 +81,7 @@ public String print(Object f) { } } - @ToString() - @RequiredArgsConstructor - public static class MoneyPrinter implements Printer { - private final PrintSettings cfg; + public record MoneyPrinter(PrintSettings cfg) implements Printer { @Override public String print(Object f) { @@ -104,10 +95,7 @@ public String print(Object f) { } } - @ToString() - @RequiredArgsConstructor - public static class DatePrinter implements Printer { - private final PrintSettings cfg; + public record DatePrinter(PrintSettings cfg) implements Printer { @Override public String print(Object f) { @@ -118,18 +106,13 @@ public String print(Object f) { } } - @ToString() - public static class DateRangePrinter implements Printer { - private final DatePrinter datePrinter; - private final PrintSettings cfg; + public record DateRangePrinter(DatePrinter datePrinter, PrintSettings cfg) implements Printer { - public DateRangePrinter(PrintSettings printSettings){ - datePrinter = new DatePrinter(printSettings); - cfg = printSettings; + public DateRangePrinter(PrintSettings printSettings) { + this(new DatePrinter(printSettings), printSettings); } - @Override public String print(Object f) { Preconditions.checkArgument(f instanceof List, "Expected a List got %s (Type: %s, as string: %s)", f, f.getClass().getName(), f); @@ -155,23 +138,13 @@ public String print(Object f) { } } - @ToString() - public static class BooleanPrinter implements Printer { - private final PrintSettings cfg; - - private final String trueVal; - private final String falseVal; + public record BooleanPrinter(PrintSettings cfg, String trueVal, String falseVal) implements Printer { public BooleanPrinter(PrintSettings cfg) { - this.cfg = cfg; - if (!cfg.isPrettyPrint()) { - trueVal = "1"; - falseVal = "0"; - } - else { - trueVal = C10nCache.getLocalized(Results.class, cfg.getLocale()).True(); - falseVal = C10nCache.getLocalized(Results.class, cfg.getLocale()).False(); - } + this(cfg, cfg.isPrettyPrint() ? C10nCache.getLocalized(Results.class, cfg.getLocale()).True() : "1", cfg.isPrettyPrint() + ? C10nCache.getLocalized(Results.class, cfg.getLocale()) + .False() + : "0"); } @Override @@ -196,8 +169,24 @@ public String print(Object f) { public record ConceptIdPrinter(Concept concept, PrintSettings cfg) implements Printer { @Override - public String print(Object f) { - return concept.printConceptLocalId(cfg, f); + public String print(Object rawValue) { + if (rawValue == null) { + return null; + } + + final int localId = (int) rawValue; + + final ConceptTreeNode node = ((TreeConcept) concept).getElementByLocalId(localId); + + if (!cfg.isPrettyPrint()) { + return node.getId().toString(); + } + + if (node.getDescription() == null) { + return node.getLabel(); + } + + return node.getLabel() + " - " + node.getDescription(); } } From cb0c47e4033e9d6a96d38da3d92c87185c8bb01d Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 20 Aug 2024 15:00:05 +0200 Subject: [PATCH 14/54] fixes StoredQueriesProcessorTest --- .../conquery/models/query/preview/EntityPreviewExecution.java | 2 ++ .../com/bakdata/conquery/api/StoredQueriesProcessorTest.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java index 9398f9aa39..af4403f700 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java @@ -133,6 +133,8 @@ private static Function> createLineToMapTransforme final int size = resultInfos.size(); final String[] columnNames = new String[size]; + //TODO pull renderValue logic into outer loop, only use array as lookup + for (int index = 0; index < size; index++) { final ResultInfo resultInfo = resultInfos.get(index); diff --git a/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java b/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java index ded8072654..10b3307d7d 100644 --- a/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java +++ b/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java @@ -198,6 +198,7 @@ private static ManagedQuery mockManagedQuery(Query queryDescription, User user, setCreationTime(LocalDateTime.MIN); setQueryId(id.getExecution()); setLastResultCount(resultCount); + setConfig(CONFIG); } @Override @@ -217,6 +218,7 @@ private static ExecutionStatus makeState(ManagedExecutionId id, User owner, User { setQueryId(id.getExecution()); setLastResultCount(resultCount); + setConfig(CONFIG); } @Override From 3bd740721657cdfa887c4bebc69d116cfb346c7f Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 20 Aug 2024 15:24:10 +0200 Subject: [PATCH 15/54] cleanup superfluous c'tor --- .../conquery/models/datasets/concepts/select/Select.java | 3 ++- .../datasets/concepts/select/concept/ConceptColumnSelect.java | 3 ++- .../concepts/select/connector/SingleColumnSelect.java | 3 ++- .../select/connector/specific/MappableSingleColumnSelect.java | 3 ++- .../java/com/bakdata/conquery/models/query/PrintSettings.java | 1 - .../conquery/models/query/resultinfo/SelectResultInfo.java | 4 ---- .../java/com/bakdata/conquery/io/result/ResultTestUtil.java | 2 +- .../conquery/io/result/arrow/ArrowResultGenerationTest.java | 3 ++- .../conquery/io/result/excel/ExcelResultRenderTest.java | 3 ++- .../io/result/parquet/ParquetResultGenerationTest.java | 3 ++- .../bakdata/conquery/models/query/DefaultColumnNameTest.java | 3 ++- 11 files changed, 17 insertions(+), 14 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java index c14e41eede..c3d18b3373 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.datasets.concepts.select; +import java.util.Collections; import java.util.List; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; @@ -87,7 +88,7 @@ public String getColumnName() { } public SelectResultInfo getResultInfo(CQConcept cqConcept, PrintSettings settings) { - return new SelectResultInfo(this, cqConcept, settings); + return new SelectResultInfo(this, cqConcept, Collections.emptySet(), settings); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java index e60a700f2d..47203e23a1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.datasets.concepts.select.concept; +import java.util.Collections; import java.util.Set; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; @@ -54,7 +55,7 @@ public ResultPrinters.Printer createPrinter(PrintSettings printSettings) { @Override public SelectResultInfo getResultInfo(CQConcept cqConcept, PrintSettings settings) { if (!isAsIds()) { - return new SelectResultInfo(this, cqConcept, settings); + return new SelectResultInfo(this, cqConcept, Collections.emptySet(), settings); } return new SelectResultInfo(this, cqConcept, Set.of(new SemanticType.ConceptColumnT(cqConcept.getConcept())), settings); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/SingleColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/SingleColumnSelect.java index 858fc98490..a8a5e16ff0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/SingleColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/SingleColumnSelect.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector; +import java.util.Collections; import java.util.EnumSet; import java.util.List; import java.util.Set; @@ -55,7 +56,7 @@ public SelectResultInfo getResultInfo(CQConcept cqConcept, PrintSettings setting return new SelectResultInfo(this, cqConcept, Set.of(new SemanticType.CategoricalT()), settings); } - return new SelectResultInfo(this, cqConcept, settings); + return new SelectResultInfo(this, cqConcept, Collections.emptySet(), settings); } @Nullable diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java index 2c306e9340..4d9ea4faa6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector.specific; +import java.util.Collections; import java.util.Set; import javax.annotation.Nullable; @@ -49,7 +50,7 @@ public ResultPrinters.Printer createPrinter(PrintSettings printSettings) { public SelectResultInfo getResultInfo(CQConcept cqConcept, PrintSettings settings) { if (!isCategorical()) { - return new SelectResultInfo(this, cqConcept, settings); + return new SelectResultInfo(this, cqConcept, Collections.emptySet(), settings); } return new SelectResultInfo(this, cqConcept, Set.of(new SemanticType.CategoricalT()), settings); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java b/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java index b7381b6753..ba494fe2e2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java @@ -86,7 +86,6 @@ public PrintSettings(boolean prettyPrint, Locale locale, Namespace namespace, Co this.currencyFormat = DecimalFormat.getCurrencyInstance(locale); currencyFormat.setCurrency(currency); currencyFormat.setMaximumFractionDigits(currency.getDefaultFractionDigits()); - } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java index 6c857f9c63..4899715639 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java @@ -28,10 +28,6 @@ public SelectResultInfo(Select select, CQConcept cqConcept, Set se this.cqConcept = cqConcept; } - public SelectResultInfo(Select select, CQConcept cqConcept, PrintSettings settings) { - this(select, cqConcept, Set.of(), settings); - } - @Override public String getDescription() { diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java index 0a1fcd5333..793beca635 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java @@ -63,7 +63,7 @@ public static ManagedQuery getTestQuery() { public List getResultInfos(PrintSettings printSettings) { return getResultTypes().stream() .map(resultType -> new TypedSelectDummy(resultType)) - .map(select -> new SelectResultInfo(select, new CQConcept(), PRINT_SETTINGS)) + .map(select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet(), PRINT_SETTINGS)) .collect(Collectors.toList()); } diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java index 98fcabcf50..b7640e817b 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java @@ -12,6 +12,7 @@ import java.io.InputStream; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Objects; @@ -79,7 +80,7 @@ void generateFieldsValue() { List resultInfos = getResultTypes().stream().map(TypedSelectDummy::new) - .map(select -> new SelectResultInfo(select, new CQConcept(), PRINT_SETTINGS)).collect(Collectors.toList()); + .map(select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet(), PRINT_SETTINGS)).collect(Collectors.toList()); List fields = generateFields( resultInfos, diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java index a95f4f388d..58d1b710f4 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java @@ -9,6 +9,7 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.OptionalLong; @@ -69,7 +70,7 @@ void writeAndRead() throws IOException { public List getResultInfos(PrintSettings printSettings) { return getResultTypes().stream() .map(ResultTestUtil.TypedSelectDummy::new) - .map(select -> new SelectResultInfo(select, new CQConcept(), printSettings)) + .map(select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet(), printSettings)) .collect(Collectors.toList()); } diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java index 978de6d10b..8751490399 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java @@ -8,6 +8,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.OptionalLong; @@ -55,7 +56,7 @@ void generateSchema() { final UniqueNamer uniqueNamer = new UniqueNamer(PRINT_SETTINGS); List resultInfos = getResultTypes().stream().map(TypedSelectDummy::new) - .map(select -> new SelectResultInfo(select, new CQConcept(), PRINT_SETTINGS)).collect(Collectors.toList()); + .map(select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet(), PRINT_SETTINGS)).collect(Collectors.toList()); final MessageType messageType = EntityResultWriteSupport.generateSchema(ResultTestUtil.ID_FIELDS, resultInfos, uniqueNamer); diff --git a/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java b/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java index ed3f850420..aabd30244f 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java @@ -6,6 +6,7 @@ import static org.mockito.Mockito.mock; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Locale; @@ -168,7 +169,7 @@ void checkCombinations(TestConcept concept, boolean hasCQConceptLabel, String ex final CQConcept cqConcept = concept.createCQConcept(hasCQConceptLabel); final UniqueNamer uniqNamer = new UniqueNamer(SETTINGS); - SelectResultInfo info = new SelectResultInfo(concept.extractSelect(cqConcept), cqConcept, SETTINGS); + SelectResultInfo info = new SelectResultInfo(concept.extractSelect(cqConcept), cqConcept, Collections.emptySet(), SETTINGS); assertThat(uniqNamer.getUniqueName(info)).isEqualTo(expectedColumnName); } From d4692bf0cba4881001883bf3c00f2e461026b6f5 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 20 Aug 2024 16:49:14 +0200 Subject: [PATCH 16/54] cleanup superfluous PrintSettings c'tor --- .../conquery/io/result/arrow/ResultArrowProcessor.java | 8 +------- .../conquery/io/result/csv/ResultCsvProcessor.java | 2 +- .../conquery/io/result/excel/ResultExcelProcessor.java | 2 +- .../io/result/parquet/ResultParquetProcessor.java | 2 +- .../conquery/models/execution/ManagedExecution.java | 2 +- .../bakdata/conquery/models/query/PrintSettings.java | 10 +++------- .../conquery/models/query/SingleTableResult.java | 2 +- .../query/resultinfo/printers/ResultPrinters.java | 7 +++++-- .../conquery/sql/conversion/NodeConversions.java | 2 +- .../execution/DefaultSqlCDateSetParserTest.java | 2 +- .../integration/json/AbstractQueryEngineTest.java | 2 +- .../bakdata/conquery/integration/json/FormTest.java | 8 +------- .../conquery/models/query/DefaultColumnNameTest.java | 2 +- .../bakdata/conquery/models/query/UniqueNameTest.java | 2 +- .../bakdata/conquery/models/types/ResultTypeTest.java | 6 +++--- 15 files changed, 23 insertions(+), 36 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java index 678305a10d..7e2c02ada1 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java @@ -95,13 +95,7 @@ public static Response getArrow IdPrinter idPrinter = IdColumnUtil.getIdPrinter(subject, exec, namespace, config.getIdColumns().getIds()); final Locale locale = I18n.LOCALE.get(); - PrintSettings settings = new PrintSettings( - pretty, - locale, - namespace, - config, - idPrinter::createId - ); + PrintSettings settings = new PrintSettings(pretty, locale, namespace, config, idPrinter::createId, null); // Collect ResultInfos for id columns and result columns diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java index 348246e969..0eb59cca95 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java @@ -57,7 +57,7 @@ public Response createResult(Su // Get the locale extracted by the LocaleFilter final Locale locale = I18n.LOCALE.get(); - final PrintSettings settings = new PrintSettings(pretty, locale, namespace, config, idPrinter::createId); + final PrintSettings settings = new PrintSettings(pretty, locale, namespace, config, idPrinter::createId, null); final StreamingOutput out = os -> { try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(os, charset))) { diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java index 24b226fc62..89194e75c8 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java @@ -52,7 +52,7 @@ public Response createResult(Su final IdPrinter idPrinter = IdColumnUtil.getIdPrinter(subject, exec, namespace, conqueryConfig.getIdColumns().getIds()); final Locale locale = I18n.LOCALE.get(); - final PrintSettings settings = new PrintSettings(pretty, locale, namespace, conqueryConfig, idPrinter::createId); + final PrintSettings settings = new PrintSettings(pretty, locale, namespace, conqueryConfig, idPrinter::createId, null); final ExcelRenderer excelRenderer = new ExcelRenderer(excelConfig, settings); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java index 8080e8616e..6dfb0e61e0 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java @@ -52,7 +52,7 @@ public Response createResultFile(Subject subject, ManagedExecution exec, boolean final IdPrinter idPrinter = IdColumnUtil.getIdPrinter(subject, exec, namespace, config.getIdColumns().getIds()); final Locale locale = I18n.LOCALE.get(); - final PrintSettings settings = new PrintSettings(pretty, locale, namespace, config, idPrinter::createId); + final PrintSettings settings = new PrintSettings(pretty, locale, namespace, config, idPrinter::createId, null); final StreamingOutput out = output -> { diff --git a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java index 03b9a0d1fd..ae1887bef7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java @@ -168,7 +168,7 @@ public final void initExecutable(Namespace namespace, ConqueryConfig config) { } if (label == null) { // IdMapper is not necessary here - label = makeAutoLabel(new PrintSettings(true, I18n.LOCALE.get(), namespace, config, null)); + label = makeAutoLabel(new PrintSettings(true, I18n.LOCALE.get(), namespace, config, null, null)); } this.namespace = namespace; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java b/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java index ba494fe2e2..6379049aee 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java @@ -13,14 +13,14 @@ import com.bakdata.conquery.models.identifiable.mapping.PrintIdMapper; import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; import com.bakdata.conquery.models.worker.Namespace; -import lombok.AccessLevel; -import lombok.AllArgsConstructor; import lombok.Getter; import lombok.ToString; +/** + * @implNote eager cache everything here, this helps avoid mistakes when rendering values. + */ @Getter @ToString(onlyExplicitlyIncluded = true) -@AllArgsConstructor(access = AccessLevel.PRIVATE) public class PrintSettings { private static final Function NUMBER_FORMAT = NumberFormat::getNumberInstance; @@ -58,10 +58,6 @@ public class PrintSettings { private final PrintIdMapper idMapper; - public PrintSettings(boolean prettyPrint, Locale locale, Namespace namespace, ConqueryConfig config, PrintIdMapper idMapper) { - this(prettyPrint, locale, namespace, config, idMapper, null); - } - public PrintSettings(boolean prettyPrint, Locale locale, Namespace namespace, ConqueryConfig config, PrintIdMapper idMapper, Function columnNamer) { this(prettyPrint, locale, namespace, config, idMapper, columnNamer, DECIMAL_FORMAT.apply(locale), NUMBER_FORMAT.apply(locale)); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/SingleTableResult.java b/backend/src/main/java/com/bakdata/conquery/models/query/SingleTableResult.java index 9526d5838f..c647021640 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/SingleTableResult.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/SingleTableResult.java @@ -24,7 +24,7 @@ default List generateColumnDescriptions(boolean isInitialized, final Locale locale = I18n.LOCALE.get(); - PrintSettings settings = new PrintSettings(true, locale, getNamespace(), config, null); + PrintSettings settings = new PrintSettings(true, locale, getNamespace(), config, null, null); UniqueNamer uniqNamer = new UniqueNamer(settings); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java index 1f1f87983e..f20d274f27 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java @@ -190,7 +190,11 @@ public String print(Object rawValue) { } } - public record ListPrinter(Printer elementPrinter, PrintSettings cfg) implements Printer { + public record ListPrinter(Printer elementPrinter, PrintSettings cfg, LocaleConfig.ListFormat listFormat) implements Printer { + + public ListPrinter(Printer elementPrinter, PrintSettings cfg){ + this(elementPrinter, cfg, cfg.getListFormat()); + } @Override public String print(Object f) { @@ -198,7 +202,6 @@ public String print(Object f) { // Jackson deserializes collections as lists instead of an array, if the type is not given Preconditions.checkArgument(f instanceof List, "Expected a List got %s (as String `%s` )".formatted(f.getClass().getName(), f)); - final LocaleConfig.ListFormat listFormat = cfg.getListFormat(); final StringJoiner joiner = listFormat.createListJoiner(); for (Object obj : (List) f) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java index 78e15186d1..be5564974b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java @@ -44,7 +44,7 @@ public NodeConversions( public ConversionContext convert(QueryDescription queryDescription, Namespace namespace, ConqueryConfig conqueryConfig) { ConversionContext initialCtx = ConversionContext.builder() .idColumns(idColumns) - .sqlPrintSettings(new PrintSettings(false, Locale.ROOT, namespace, conqueryConfig, null)) + .sqlPrintSettings(new PrintSettings(false, Locale.ROOT, namespace, conqueryConfig, null, null)) .config(config) .nameGenerator(nameGenerator) .nodeConversions(this) diff --git a/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java b/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java index ab31adf410..ac08f50d4a 100644 --- a/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java +++ b/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java @@ -18,7 +18,7 @@ class DefaultSqlCDateSetParserTest { private static final DefaultSqlCDateSetParser parser = new DefaultSqlCDateSetParser(); private static final ConqueryConfig CONFIG = new ConqueryConfig(); - private static final PrintSettings PLAIN = new PrintSettings(false, Locale.ENGLISH, null, CONFIG, null); + private static final PrintSettings PLAIN = new PrintSettings(false, Locale.ENGLISH, null, CONFIG, null, null); @ParameterizedTest @MethodSource("testToEpochDayRangeListProvider") diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java index 3c24087b70..c80ac829cf 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java @@ -57,7 +57,7 @@ public void executeTest(StandaloneSupport standaloneSupport) throws IOException SingleTableResult executionResult = (SingleTableResult) execution; //check result info size - PrintSettings printSettings = new PrintSettings(true, Locale.ROOT, standaloneSupport.getNamespace(), standaloneSupport.getConfig(), null); + PrintSettings printSettings = new PrintSettings(true, Locale.ROOT, standaloneSupport.getNamespace(), standaloneSupport.getConfig(), null, null); List resultInfos = executionResult.getResultInfos(printSettings); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java index c759f9d33c..cfa9728123 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java @@ -129,13 +129,7 @@ private void checkResults(StandaloneSupport standaloneSupport, ManagedInternalFo final ConqueryConfig config = standaloneSupport.getConfig(); PrintSettings printSettings = - new PrintSettings( - false, - Locale.ENGLISH, - standaloneSupport.getNamespace(), - config, - idPrinter::createId - ); + new PrintSettings(false, Locale.ENGLISH, standaloneSupport.getNamespace(), config, idPrinter::createId, null); checkSingleResult(managedForm, config, printSettings); diff --git a/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java b/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java index aabd30244f..8782a45152 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java @@ -46,7 +46,7 @@ @Slf4j public class DefaultColumnNameTest { private static final Namespace NAMESPACE = mock(LocalNamespace.class); - private static final PrintSettings SETTINGS = new PrintSettings(false, Locale.ENGLISH, NAMESPACE, new ConqueryConfig(), null); + private static final PrintSettings SETTINGS = new PrintSettings(false, Locale.ENGLISH, NAMESPACE, new ConqueryConfig(), null, null); private static final Validator VALIDATOR = Validators.newValidator(); private static final BiFunction CONCEPT_SELECT_SELECTOR = diff --git a/backend/src/test/java/com/bakdata/conquery/models/query/UniqueNameTest.java b/backend/src/test/java/com/bakdata/conquery/models/query/UniqueNameTest.java index 7bd7e81312..441b6ef2bf 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/query/UniqueNameTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/query/UniqueNameTest.java @@ -13,7 +13,7 @@ public class UniqueNameTest { @Test void testNameCollision() { - PrintSettings settings = new PrintSettings(true, Locale.ROOT, null, new ConqueryConfig(), null); + PrintSettings settings = new PrintSettings(true, Locale.ROOT, null, new ConqueryConfig(), null, null); final UniqueNamer uniqueNamer = new UniqueNamer(settings); final ExternalResultInfo info1 = new ExternalResultInfo("test", ResultType.Primitive.STRING, settings); diff --git a/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java b/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java index fd560d9007..283255fe98 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java @@ -26,9 +26,9 @@ public class ResultTypeTest { public static final ConqueryConfig CONFIG = new ConqueryConfig(); - private static final PrintSettings PRETTY = new PrintSettings(true, Locale.ENGLISH, null, CONFIG, null); - private static final PrintSettings PRETTY_DE = new PrintSettings(true, Locale.GERMANY, null, CONFIG, null); - private static final PrintSettings PLAIN = new PrintSettings(false, Locale.ENGLISH, null, CONFIG, null); + private static final PrintSettings PRETTY = new PrintSettings(true, Locale.ENGLISH, null, CONFIG, null, null); + private static final PrintSettings PRETTY_DE = new PrintSettings(true, Locale.GERMANY, null, CONFIG, null, null); + private static final PrintSettings PLAIN = new PrintSettings(false, Locale.ENGLISH, null, CONFIG, null, null); static { // Initialization of the internationalization From e95bb29cbea877a31d3b235de158f26ac360d8ab Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 22 Aug 2024 10:09:15 +0200 Subject: [PATCH 17/54] Cleanup Localized Printers --- .../com/bakdata/conquery/ResultHeaders.java | 18 +++--- .../conquery/apiv1/forms/FeatureGroup.java | 29 +-------- .../apiv1/query/concept/specific/CQAnd.java | 4 +- .../apiv1/query/concept/specific/CQOr.java | 4 +- .../models/common/LocalizedToString.java | 7 +++ .../models/config/IdColumnConfig.java | 4 +- .../models/forms/util/Resolution.java | 60 +++++-------------- ...ultInfo.java => FixedLabelResultInfo.java} | 6 +- .../resultinfo/printers/ResultPrinters.java | 32 +++++++--- 9 files changed, 64 insertions(+), 100 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/models/common/LocalizedToString.java rename backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/{LocalizedDefaultResultInfo.java => FixedLabelResultInfo.java} (82%) diff --git a/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java b/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java index 24d6d3c7bc..656c180d6f 100644 --- a/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java +++ b/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.models.forms.util.Resolution; import com.bakdata.conquery.models.query.C10nCache; import com.bakdata.conquery.models.query.PrintSettings; -import com.bakdata.conquery.models.query.resultinfo.LocalizedDefaultResultInfo; +import com.bakdata.conquery.models.query.resultinfo.FixedLabelResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; import com.bakdata.conquery.models.types.ResultType; @@ -21,7 +21,7 @@ public static ResultInfo datesInfo(PrintSettings settings) { final ResultType.ListT type = new ResultType.ListT<>(ResultType.Primitive.DATE_RANGE); - return new LocalizedDefaultResultInfo(label, label, type, Set.of(new SemanticType.EventDateT()), settings, ResultPrinters.defaultPrinter(type, settings)); + return new FixedLabelResultInfo(label, label, type, Set.of(new SemanticType.EventDateT()), settings, ResultPrinters.defaultPrinter(type, settings)); } public static ResultInfo historyDatesInfo(PrintSettings settings) { @@ -29,44 +29,44 @@ public static ResultInfo historyDatesInfo(PrintSettings settings) { final ResultType.ListT type = new ResultType.ListT<>(ResultType.Primitive.DATE_RANGE); - return new LocalizedDefaultResultInfo(label, label, type, Set.of(new SemanticType.EventDateT(), new SemanticType.GroupT()), settings, ResultPrinters.defaultPrinter(type, settings)); + return new FixedLabelResultInfo(label, label, type, Set.of(new SemanticType.EventDateT(), new SemanticType.GroupT()), settings, ResultPrinters.defaultPrinter(type, settings)); } public static ResultInfo sourceInfo(PrintSettings settings) { final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()).source(); - return new LocalizedDefaultResultInfo(label, label, ResultType.Primitive.STRING, Set.of(new SemanticType.SourcesT(), new SemanticType.CategoricalT(), new SemanticType.GroupT()), settings, ResultPrinters.defaultPrinter(ResultType.Primitive.STRING, settings)); + return new FixedLabelResultInfo(label, label, ResultType.Primitive.STRING, Set.of(new SemanticType.SourcesT(), new SemanticType.CategoricalT(), new SemanticType.GroupT()), settings, ResultPrinters.defaultPrinter(ResultType.Primitive.STRING, settings)); } public static ResultInfo formContextInfo(PrintSettings settings) { final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()).index(); - return new LocalizedDefaultResultInfo(label, label, ResultType.Primitive.INTEGER, Set.of(), settings, ResultPrinters.defaultPrinter(ResultType.Primitive.INTEGER, settings)); + return new FixedLabelResultInfo(label, label, ResultType.Primitive.INTEGER, Set.of(), settings, ResultPrinters.defaultPrinter(ResultType.Primitive.INTEGER, settings)); } public static ResultInfo formDateRangeInfo(PrintSettings settings) { final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()) .dateRange(); - return new LocalizedDefaultResultInfo(label, label, ResultType.Primitive.DATE_RANGE, Set.of(), settings, ResultPrinters.defaultPrinter(ResultType.Primitive.DATE_RANGE, settings)); + return new FixedLabelResultInfo(label, label, ResultType.Primitive.DATE_RANGE, Set.of(), settings, ResultPrinters.defaultPrinter(ResultType.Primitive.DATE_RANGE, settings)); } public static ResultInfo formResolutionInfo(PrintSettings settings) { final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()).resolution(); - return new LocalizedDefaultResultInfo(label, label, ResultType.Primitive.STRING, Set.of(), settings, new Resolution.LocalizingPrinter(settings)); + return new FixedLabelResultInfo(label, label, ResultType.Primitive.STRING, Set.of(), settings, new ResultPrinters.LocalizedEnumPrinter<>(settings, Resolution.class)); } public static ResultInfo formEventDateInfo(PrintSettings settings) { final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()) .eventDate(); - return new LocalizedDefaultResultInfo(label, label, ResultType.Primitive.DATE, Set.of(), settings, ResultPrinters.defaultPrinter(ResultType.Primitive.DATE, settings)); + return new FixedLabelResultInfo(label, label, ResultType.Primitive.DATE, Set.of(), settings, ResultPrinters.defaultPrinter(ResultType.Primitive.DATE, settings)); } public static ResultInfo formObservationScopeInfo(PrintSettings settings) { final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()).observationScope(); - return new LocalizedDefaultResultInfo(label, label, ResultType.Primitive.STRING, Set.of(), settings, new FeatureGroup.LocalizingPrinter(settings)); + return new FixedLabelResultInfo(label, label, ResultType.Primitive.STRING, Set.of(), settings, new ResultPrinters.LocalizedEnumPrinter<>(settings, FeatureGroup.class)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/FeatureGroup.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/FeatureGroup.java index 3c2927a497..80c329348f 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/FeatureGroup.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/FeatureGroup.java @@ -4,9 +4,7 @@ import c10n.C10N; import com.bakdata.conquery.internationalization.ResultHeadersC10n; -import com.bakdata.conquery.models.forms.util.Resolution; -import com.bakdata.conquery.models.query.PrintSettings; -import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; +import com.bakdata.conquery.models.common.LocalizedToString; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -17,7 +15,7 @@ */ @Getter @RequiredArgsConstructor -public enum FeatureGroup { +public enum FeatureGroup implements LocalizedToString { FEATURE() { @Override public String toString(Locale locale) { @@ -35,28 +33,5 @@ public String toString(Locale locale) { public String toString(Locale locale) { return ""; } - }; - - public abstract String toString(Locale locale); - - @RequiredArgsConstructor - public static class LocalizingPrinter implements ResultPrinters.Printer { - private final PrintSettings cfg; - - @Override - public String print(Object f) { - if (f instanceof Resolution) { - return ((Resolution) f).toString(cfg.getLocale()); - } - try { - // If the object was parsed as a simple string, try to convert it to a - // FeatureGroup to get Internationalization - return FeatureGroup.valueOf((String) f).toString(cfg.getLocale()); - } - catch (Exception e) { - throw new IllegalArgumentException(f + " is not a valid resolution.", e); - } - } } - } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java index 7a69addbcf..f49ff668f9 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java @@ -26,7 +26,7 @@ import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.ExistsAggregator; import com.bakdata.conquery.models.query.queryplan.specific.AndNode; -import com.bakdata.conquery.models.query.resultinfo.LocalizedDefaultResultInfo; +import com.bakdata.conquery.models.query.resultinfo.FixedLabelResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; import com.bakdata.conquery.models.types.ResultType; @@ -124,7 +124,7 @@ public List getResultInfos(PrintSettings settings) { final String userOrDefaultLabel = getUserOrDefaultLabel(settings.getLocale()); final String defaultLabel = defaultLabel(settings.getLocale()); - resultInfos.add(new LocalizedDefaultResultInfo(userOrDefaultLabel, defaultLabel, ResultType.Primitive.BOOLEAN, Set.of(), settings, printer)); + resultInfos.add(new FixedLabelResultInfo(userOrDefaultLabel, defaultLabel, ResultType.Primitive.BOOLEAN, Set.of(), settings, printer)); } return resultInfos; } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java index ddedab3492..6037200bea 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java @@ -26,7 +26,7 @@ import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.ExistsAggregator; import com.bakdata.conquery.models.query.queryplan.specific.OrNode; -import com.bakdata.conquery.models.query.resultinfo.LocalizedDefaultResultInfo; +import com.bakdata.conquery.models.query.resultinfo.FixedLabelResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; import com.bakdata.conquery.models.types.ResultType; @@ -129,7 +129,7 @@ public List getResultInfos(PrintSettings settings) { final String userOrDefaultLabel = getUserOrDefaultLabel(settings.getLocale()); final String defaultLabel = defaultLabel(settings.getLocale()); - resultInfos.add(new LocalizedDefaultResultInfo(userOrDefaultLabel, defaultLabel, ResultType.Primitive.BOOLEAN, Set.of(), settings, printer)); + resultInfos.add(new FixedLabelResultInfo(userOrDefaultLabel, defaultLabel, ResultType.Primitive.BOOLEAN, Set.of(), settings, printer)); } return resultInfos; diff --git a/backend/src/main/java/com/bakdata/conquery/models/common/LocalizedToString.java b/backend/src/main/java/com/bakdata/conquery/models/common/LocalizedToString.java new file mode 100644 index 0000000000..e61a29c76c --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/common/LocalizedToString.java @@ -0,0 +1,7 @@ +package com.bakdata.conquery.models.common; + +import java.util.Locale; + +public interface LocalizedToString { + String toString(Locale locale); +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java index b8167f53b5..2863c20894 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java @@ -13,7 +13,7 @@ import com.bakdata.conquery.apiv1.query.concept.specific.external.DateFormat; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.query.PrintSettings; -import com.bakdata.conquery.models.query.resultinfo.LocalizedDefaultResultInfo; +import com.bakdata.conquery.models.query.resultinfo.FixedLabelResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; import com.bakdata.conquery.models.types.ResultType; @@ -136,7 +136,7 @@ public List getIdResultInfos(PrintSettings printSettings) { ), col.getField()); //TODO we can now hook our anonymizers into this - return new LocalizedDefaultResultInfo(label, label, ResultType.Primitive.STRING, Set.of(new SemanticType.IdT(col.getName())), printSettings, ResultPrinters.defaultPrinter(ResultType.Primitive.STRING, printSettings)); + return new FixedLabelResultInfo(label, label, ResultType.Primitive.STRING, Set.of(new SemanticType.IdT(col.getName())), printSettings, ResultPrinters.defaultPrinter(ResultType.Primitive.STRING, printSettings)); }).collect(Collectors.toUnmodifiableList()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/util/Resolution.java b/backend/src/main/java/com/bakdata/conquery/models/forms/util/Resolution.java index 8d083eaabe..01954e4e22 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/util/Resolution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/util/Resolution.java @@ -9,8 +9,7 @@ import c10n.C10N; import com.bakdata.conquery.apiv1.forms.FeatureGroup; import com.bakdata.conquery.internationalization.DateContextResolutionC10n; -import com.bakdata.conquery.models.query.PrintSettings; -import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; +import com.bakdata.conquery.models.common.LocalizedToString; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.RequiredArgsConstructor; @@ -23,7 +22,7 @@ * edge context. */ @RequiredArgsConstructor -public enum Resolution { +public enum Resolution implements LocalizedToString { /** * For returning contexts with a single {@link com.bakdata.conquery.models.common.daterange.CDateRange} for the entire * {@link FeatureGroup}. @@ -48,17 +47,12 @@ protected List getCompatibleAlignments() { YEARS(COMPLETE) { @Override public String toString(Locale locale) { - return C10N.get(DateContextResolutionC10n.class, locale).year(); } @Override protected List getCompatibleAlignments() { - return List.of( - Alignment.YEAR, - Alignment.QUARTER, - Alignment.DAY - ); + return List.of(Alignment.YEAR, Alignment.QUARTER, Alignment.DAY); } }, @@ -75,10 +69,7 @@ public String toString(Locale locale) { @Override protected List getCompatibleAlignments() { - return List.of( - Alignment.QUARTER, - Alignment.DAY - ); + return List.of(Alignment.QUARTER, Alignment.DAY); } }, @@ -105,7 +96,11 @@ protected List getCompatibleAlignments() { private List thisAndCoarserSubdivisions; - public abstract String toString(Locale locale); + @JsonIgnore + public Alignment getDefaultAlignment() { + // The first alignment is considered the default + return getCompatibleAlignments().get(0); + } /** * Returns the alignments, that are compatible with this resolution. @@ -115,17 +110,6 @@ protected List getCompatibleAlignments() { @JsonIgnore protected abstract List getCompatibleAlignments(); - @JsonIgnore - public boolean isAlignmentSupported(Alignment alignment) { - return getCompatibleAlignments().contains(alignment); - } - - @JsonIgnore - public Alignment getDefaultAlignment() { - // The first alignment is considered the default - return getCompatibleAlignments().get(0); - } - /** * Returns the amount of calendar alignment sub date ranges that would fit in to this resolution. */ @@ -137,6 +121,11 @@ public OptionalInt getAmountForAlignment(Alignment alignment) { return alignment.getAmountForResolution(this); } + @JsonIgnore + public boolean isAlignmentSupported(Alignment alignment) { + return getCompatibleAlignments().contains(alignment); + } + @JsonIgnore public List getThisAndCoarserSubdivisions() { if (thisAndCoarserSubdivisions != null) { @@ -150,25 +139,4 @@ public List getThisAndCoarserSubdivisions() { return thisAndCoarserSubdivisions = Collections.unmodifiableList(thisAndCoarser); } - @RequiredArgsConstructor - public static class LocalizingPrinter implements ResultPrinters.Printer{ - - private final PrintSettings cfg; - - @Override - public String print(Object f) { - if (f instanceof Resolution) { - return ((Resolution) f).toString(cfg.getLocale()); - } - try { - // If the object was parsed as a simple string, try to convert it to a - // DateContextMode to get Internationalization - return Resolution.valueOf(f.toString()).toString(cfg.getLocale()); - } - catch (Exception e) { - throw new IllegalArgumentException(f + " is not a valid resolution.", e); - } - } - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/LocalizedDefaultResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/FixedLabelResultInfo.java similarity index 82% rename from backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/LocalizedDefaultResultInfo.java rename to backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/FixedLabelResultInfo.java index 5843b7dbc6..8a4b898115 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/LocalizedDefaultResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/FixedLabelResultInfo.java @@ -15,7 +15,7 @@ /** * Allows to generate result names, e.g. for CSV-headers, depending on the * provided locale. - * The {@link LocalizedDefaultResultInfo#localizedDefaultLabelProvider} is expected to + * The {@link FixedLabelResultInfo#localizedDefaultLabelProvider} is expected to * use {@link C10N} (Cosmopolitan) like this: *
  *  (locale) -> C10N.get(ExampleC10n.class, locale).example()
@@ -34,7 +34,7 @@
  */
 @EqualsAndHashCode(callSuper = true)
 @ToString
-public class LocalizedDefaultResultInfo extends ResultInfo {
+public class FixedLabelResultInfo extends ResultInfo {
 
 	@NonNull
 	private final String localizedLabel;
@@ -46,7 +46,7 @@ public class LocalizedDefaultResultInfo extends ResultInfo {
 	@Getter
 	private final ResultPrinters.Printer printer;
 
-	public LocalizedDefaultResultInfo(String label, String defaultLabel, ResultType type, Set semantics, PrintSettings settings, ResultPrinters.Printer printer) {
+	public FixedLabelResultInfo(String label, String defaultLabel, ResultType type, Set semantics, PrintSettings settings, ResultPrinters.Printer printer) {
 		super(semantics, settings);
 		this.localizedLabel = label;
 		this.localizedDefaultLabel = defaultLabel;
diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java
index f20d274f27..f1ddeddda5 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java
@@ -7,6 +7,7 @@
 
 import com.bakdata.conquery.internationalization.Results;
 import com.bakdata.conquery.models.common.CDate;
+import com.bakdata.conquery.models.common.LocalizedToString;
 import com.bakdata.conquery.models.common.daterange.CDateRange;
 import com.bakdata.conquery.models.config.LocaleConfig;
 import com.bakdata.conquery.models.datasets.concepts.Concept;
@@ -48,7 +49,6 @@ public interface Printer {
 		String print(Object f);
 	}
 
-
 	public record StringPrinter() implements Printer {
 		@Override
 		public String print(Object f) {
@@ -56,7 +56,6 @@ public String print(Object f) {
 		}
 	}
 
-
 	public record IntegerPrinter(PrintSettings cfg) implements Printer {
 
 		@Override
@@ -106,7 +105,6 @@ public String print(Object f) {
 		}
 	}
 
-
 	public record DateRangePrinter(DatePrinter datePrinter, PrintSettings cfg) implements Printer {
 
 		public DateRangePrinter(PrintSettings printSettings) {
@@ -141,10 +139,11 @@ public String print(Object f) {
 	public record BooleanPrinter(PrintSettings cfg, String trueVal, String falseVal) implements Printer {
 
 		public BooleanPrinter(PrintSettings cfg) {
-			this(cfg, cfg.isPrettyPrint() ? C10nCache.getLocalized(Results.class, cfg.getLocale()).True() : "1", cfg.isPrettyPrint()
-																												 ? C10nCache.getLocalized(Results.class, cfg.getLocale())
-																															.False()
-																												 : "0");
+			this(
+					cfg,
+					cfg.isPrettyPrint() ? C10nCache.getLocalized(Results.class, cfg.getLocale()).True() : "1",
+					cfg.isPrettyPrint() ? C10nCache.getLocalized(Results.class, cfg.getLocale()).False() : "0"
+			);
 		}
 
 		@Override
@@ -165,7 +164,6 @@ public String print(Object f) {
 		}
 	}
 
-
 	public record ConceptIdPrinter(Concept concept, PrintSettings cfg) implements Printer {
 
 		@Override
@@ -192,7 +190,7 @@ public String print(Object rawValue) {
 
 	public record ListPrinter(Printer elementPrinter, PrintSettings cfg, LocaleConfig.ListFormat listFormat) implements Printer {
 
-		public ListPrinter(Printer elementPrinter, PrintSettings cfg){
+		public ListPrinter(Printer elementPrinter, PrintSettings cfg) {
 			this(elementPrinter, cfg, cfg.getListFormat());
 		}
 
@@ -210,4 +208,20 @@ public String print(Object f) {
 			return joiner.toString();
 		}
 	}
+
+	public record LocalizedEnumPrinter & LocalizedToString>(PrintSettings cfg, Class clazz) implements Printer {
+		@Override
+		public String print(Object f) {
+
+			if (clazz.isInstance(f)) {
+				return clazz.cast(f).toString(cfg.getLocale());
+			}
+			try {
+				return Enum.valueOf(clazz, f.toString()).toString(cfg.getLocale());
+			}
+			catch (Exception e) {
+				throw new IllegalArgumentException("%s is not a valid %s.".formatted(f, clazz), e);
+			}
+		}
+	}
 }

From 6d032338b754cddbd61127dc788d3a9a95356317 Mon Sep 17 00:00:00 2001
From: awildturtok <1553491+awildturtok@users.noreply.github.com>
Date: Thu, 22 Aug 2024 10:22:26 +0200
Subject: [PATCH 18/54] Code Style Issues and cleanup

---
 .../java/com/bakdata/conquery/ResultHeaders.java  | 12 ++++++------
 .../conquery/apiv1/query/TableExportQuery.java    |  4 ++--
 .../conquery/io/result/arrow/ArrowUtil.java       |  4 ++--
 .../conquery/io/result/excel/ExcelRenderer.java   |  9 ++++-----
 .../conquery/models/config/IdColumnConfig.java    |  2 +-
 .../models/datasets/concepts/select/Select.java   |  2 +-
 .../models/query/resultinfo/ColumnResultInfo.java | 15 ++-------------
 .../query/resultinfo/ExternalResultInfo.java      |  2 +-
 .../models/query/resultinfo/ResultInfo.java       |  4 ++--
 .../models/query/resultinfo/SelectResultInfo.java | 12 ------------
 .../query/resultinfo/printers/ResultPrinters.java |  4 ++--
 .../printers/SecondaryIdResultInfo.java           | 15 +--------------
 .../execution/DefaultSqlCDateSetParserTest.java   |  2 +-
 13 files changed, 25 insertions(+), 62 deletions(-)

diff --git a/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java b/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java
index 656c180d6f..9d3eb527e3 100644
--- a/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java
+++ b/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java
@@ -21,7 +21,7 @@ public static ResultInfo datesInfo(PrintSettings settings) {
 
 		final ResultType.ListT type = new ResultType.ListT<>(ResultType.Primitive.DATE_RANGE);
 
-		return new FixedLabelResultInfo(label, label, type, Set.of(new SemanticType.EventDateT()), settings, ResultPrinters.defaultPrinter(type, settings));
+		return new FixedLabelResultInfo(label, label, type, Set.of(new SemanticType.EventDateT()), settings, ResultPrinters.printerFor(type, settings));
 	}
 
 	public static ResultInfo historyDatesInfo(PrintSettings settings) {
@@ -29,26 +29,26 @@ public static ResultInfo historyDatesInfo(PrintSettings settings) {
 
 		final ResultType.ListT type = new ResultType.ListT<>(ResultType.Primitive.DATE_RANGE);
 
-		return new FixedLabelResultInfo(label, label, type, Set.of(new SemanticType.EventDateT(), new SemanticType.GroupT()), settings, ResultPrinters.defaultPrinter(type, settings));
+		return new FixedLabelResultInfo(label, label, type, Set.of(new SemanticType.EventDateT(), new SemanticType.GroupT()), settings, ResultPrinters.printerFor(type, settings));
 	}
 
 	public static ResultInfo sourceInfo(PrintSettings settings) {
 		final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()).source();
 
-		return new FixedLabelResultInfo(label, label, ResultType.Primitive.STRING, Set.of(new SemanticType.SourcesT(), new SemanticType.CategoricalT(), new SemanticType.GroupT()), settings, ResultPrinters.defaultPrinter(ResultType.Primitive.STRING, settings));
+		return new FixedLabelResultInfo(label, label, ResultType.Primitive.STRING, Set.of(new SemanticType.SourcesT(), new SemanticType.CategoricalT(), new SemanticType.GroupT()), settings, ResultPrinters.printerFor(ResultType.Primitive.STRING, settings));
 	}
 
 	public static ResultInfo formContextInfo(PrintSettings settings) {
 		final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()).index();
 
-		return new FixedLabelResultInfo(label, label, ResultType.Primitive.INTEGER, Set.of(), settings, ResultPrinters.defaultPrinter(ResultType.Primitive.INTEGER, settings));
+		return new FixedLabelResultInfo(label, label, ResultType.Primitive.INTEGER, Set.of(), settings, ResultPrinters.printerFor(ResultType.Primitive.INTEGER, settings));
 	}
 
 	public static ResultInfo formDateRangeInfo(PrintSettings settings) {
 		final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale())
 									  .dateRange();
 
-		return new FixedLabelResultInfo(label, label, ResultType.Primitive.DATE_RANGE, Set.of(), settings, ResultPrinters.defaultPrinter(ResultType.Primitive.DATE_RANGE, settings));
+		return new FixedLabelResultInfo(label, label, ResultType.Primitive.DATE_RANGE, Set.of(), settings, ResultPrinters.printerFor(ResultType.Primitive.DATE_RANGE, settings));
 	}
 
 	public static ResultInfo formResolutionInfo(PrintSettings settings) {
@@ -61,7 +61,7 @@ public static ResultInfo formEventDateInfo(PrintSettings settings) {
 		final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale())
 									  .eventDate();
 
-		return new FixedLabelResultInfo(label, label, ResultType.Primitive.DATE, Set.of(), settings, ResultPrinters.defaultPrinter(ResultType.Primitive.DATE, settings));
+		return new FixedLabelResultInfo(label, label, ResultType.Primitive.DATE, Set.of(), settings, ResultPrinters.printerFor(ResultType.Primitive.DATE, settings));
 	}
 
 	public static ResultInfo formObservationScopeInfo(PrintSettings settings) {
diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java
index 9aa318bf8e..c8f88d8dbb 100644
--- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java
+++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java
@@ -259,14 +259,14 @@ private List createResultInfos(Set conceptColumns, PrintSett
 
 			// SecondaryIds and date columns are pulled to the front, thus already covered.
 			if (column.getSecondaryId() != null && !conceptColumns.contains(column)) {
-				infos[secondaryIdPositions.get(column.getSecondaryId())].addSemantic(new SemanticType.ColumnT(column));
+				infos[secondaryIdPositions.get(column.getSecondaryId())].addSemantics(new SemanticType.ColumnT(column));
 				continue;
 			}
 
 			final Set semantics = new HashSet<>();
 
 			ResultType resultType = ResultType.resolveResultType(column.getType());
-			ResultPrinters.Printer printer = ResultPrinters.defaultPrinter(resultType, printSettings);
+			ResultPrinters.Printer printer = ResultPrinters.printerFor(resultType, printSettings);
 
 			if (connectorColumns.containsKey(column)) {
 				// Additionally, Concept Columns are returned as ConceptElementId, when rawConceptColumns is not set.
diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java
index 2253cdbacb..7d2bc1ffcd 100644
--- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java
+++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java
@@ -21,9 +21,9 @@
 @UtilityClass
 public class ArrowUtil {
 
-	public final static RootAllocator ROOT_ALLOCATOR = new RootAllocator();
+	public static final RootAllocator ROOT_ALLOCATOR = new RootAllocator();
 
-	public BiFunction fieldFor(ResultType type) {
+	private BiFunction fieldFor(ResultType type) {
 		if (type instanceof ResultType.ListT) {
 			return ArrowUtil::listField;
 		}
diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java
index e2aacaed24..71e10f3ae1 100644
--- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java
+++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java
@@ -43,20 +43,19 @@ public class ExcelRenderer {
 
 	public static final int MAX_LINES = 1_048_576;
 
-	public TypeWriter writer(ResultType type) {
+	private static TypeWriter writer(ResultType type) {
 		if(!(type instanceof ResultType.Primitive)){
 			//Excel cannot handle complex types so we just toString them.
-			return (info, cell, value, styles1, styles12) -> writeStringCell(info, value, styles1, styles12);
+			return (info, settings, cell, value, styles) -> writeStringCell(info, cell, value, styles);
 		}
 
 		return switch (((ResultType.Primitive) type)) {
-			case BOOLEAN -> (info, cell, value, styles1, styles12) -> writeBooleanCell(info, value, styles1, styles12);
+			case BOOLEAN -> (info, settings, cell, value, styles) -> writeBooleanCell(info, cell, value, styles);
 			case INTEGER -> ExcelRenderer::writeIntegerCell;
 			case MONEY -> ExcelRenderer::writeMoneyCell;
 			case NUMERIC -> ExcelRenderer::writeNumericCell;
 			case DATE -> ExcelRenderer::writeDateCell;
-			case STRING -> (info, cell, value, styles1, styles12) -> writeStringCell(info, value, styles1, styles12);
-			default -> (info, cell, value, styles1, styles12) -> writeStringCell(info, value, styles1, styles12);
+			default -> (info, settings, cell, value, styles) -> writeStringCell(info, cell, value, styles);
 		};
 	}
 
diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java
index 2863c20894..ebac0b4a46 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java
@@ -136,7 +136,7 @@ public List getIdResultInfos(PrintSettings printSettings) {
 			), col.getField());
 
 			//TODO we can now hook our anonymizers into this
-			return new FixedLabelResultInfo(label, label, ResultType.Primitive.STRING, Set.of(new SemanticType.IdT(col.getName())), printSettings, ResultPrinters.defaultPrinter(ResultType.Primitive.STRING, printSettings));
+			return new FixedLabelResultInfo(label, label, ResultType.Primitive.STRING, Set.of(new SemanticType.IdT(col.getName())), printSettings, ResultPrinters.printerFor(ResultType.Primitive.STRING, printSettings));
 		}).collect(Collectors.toUnmodifiableList());
 	}
 
diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java
index c3d18b3373..c7c5b026d7 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java
@@ -129,6 +129,6 @@ public boolean isEventDateSelect() {
 	}
 
 	public ResultPrinters.Printer createPrinter(PrintSettings printSettings) {
-		return ResultPrinters.defaultPrinter(getResultType(), printSettings);
+		return ResultPrinters.printerFor(getResultType(), printSettings);
 	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java
index b42fdec50e..b1d8abc4f5 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java
@@ -3,7 +3,6 @@
 import java.util.Set;
 
 import com.bakdata.conquery.models.datasets.Column;
-import com.bakdata.conquery.models.query.ColumnDescriptor;
 import com.bakdata.conquery.models.query.PrintSettings;
 import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters;
 import com.bakdata.conquery.models.types.ResultType;
@@ -33,22 +32,12 @@ public ColumnResultInfo(Column column, ResultType type, Set semant
 
 	@Override
 	public String userColumnName() {
-		return null;
+		return column.getTable().getLabel() + " " + column.getLabel();
 	}
 
 	@Override
 	public String defaultColumnName() {
-		return column.getTable().getLabel() + " " + column.getLabel();
+		return userColumnName();
 	}
 
-	@Override
-	public ColumnDescriptor asColumnDescriptor(UniqueNamer collector) {
-		return ColumnDescriptor.builder()
-							   .label(defaultColumnName())
-							   .defaultLabel(getColumn().getLabel())
-							   .type(getType().typeInfo())
-							   .semantics(getSemantics())
-							   .description(getDescription())
-							   .build();
-	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ExternalResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ExternalResultInfo.java
index 38acd09eae..984840fd5c 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ExternalResultInfo.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ExternalResultInfo.java
@@ -20,7 +20,7 @@ public class ExternalResultInfo extends ResultInfo {
 	private final ResultPrinters.Printer printer;
 
 	public ExternalResultInfo(String name, ResultType type, PrintSettings settings) {
-		this(name, type, null, ResultPrinters.defaultPrinter(type, settings), Collections.emptySet(), settings);
+		this(name, type, null, ResultPrinters.printerFor(type, settings), Collections.emptySet(), settings);
 	}
 	public ExternalResultInfo(String name, ResultType type, String description, ResultPrinters.Printer printer, Set semantics, PrintSettings settings) {
 		super(semantics, settings);
diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java
index 8b6a850a74..9122d20a92 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java
@@ -34,13 +34,13 @@ protected ResultInfo(Collection semantics, PrintSettings settings)
 		this.semantics.addAll(semantics);
 	}
 
-	public final void addSemantic(SemanticType... incoming) {
+	public final void addSemantics(SemanticType... incoming) {
 		semantics.addAll(Arrays.asList(incoming));
 	}
 
 	public abstract String userColumnName();
 
-	public ColumnDescriptor asColumnDescriptor(UniqueNamer collector) {
+	public final ColumnDescriptor asColumnDescriptor(UniqueNamer collector) {
 		return ColumnDescriptor.builder()
 							   .label(collector.getUniqueName(this))
 							   .defaultLabel(defaultColumnName())
diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java
index 4899715639..b8873c64d4 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java
@@ -4,7 +4,6 @@
 
 import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept;
 import com.bakdata.conquery.models.datasets.concepts.select.Select;
-import com.bakdata.conquery.models.query.ColumnDescriptor;
 import com.bakdata.conquery.models.query.PrintSettings;
 import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters;
 import com.bakdata.conquery.models.types.ResultType;
@@ -44,17 +43,6 @@ public ResultType getType() {
 		return select.getResultType();
 	}
 
-	@Override
-	public ColumnDescriptor asColumnDescriptor(UniqueNamer uniqueNamer) {
-		return ColumnDescriptor.builder()
-							   .label(uniqueNamer.getUniqueName(this))
-							   .defaultLabel(defaultColumnName())
-							   .type(getType().typeInfo())
-							   .semantics(getSemantics())
-							   .description(getSelect().getDescription())
-							   .build();
-	}
-
 	@Override
 	public String userColumnName() {
 
diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java
index f1ddeddda5..6f1a4b6d97 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java
@@ -25,9 +25,9 @@
 @Slf4j
 public class ResultPrinters {
 
-	public Printer defaultPrinter(ResultType type, PrintSettings printSettings) {
+	public Printer printerFor(ResultType type, PrintSettings printSettings) {
 		if (type instanceof ResultType.ListT listT) {
-			return new ListPrinter(defaultPrinter(listT.getElementType(), printSettings), printSettings);
+			return new ListPrinter(printerFor(listT.getElementType(), printSettings), printSettings);
 		}
 
 		return switch (((ResultType.Primitive) type)) {
diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/SecondaryIdResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/SecondaryIdResultInfo.java
index 2fd67cad74..6f33bbb124 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/SecondaryIdResultInfo.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/SecondaryIdResultInfo.java
@@ -3,10 +3,8 @@
 import java.util.Set;
 
 import com.bakdata.conquery.models.datasets.SecondaryIdDescription;
-import com.bakdata.conquery.models.query.ColumnDescriptor;
 import com.bakdata.conquery.models.query.PrintSettings;
 import com.bakdata.conquery.models.query.resultinfo.ResultInfo;
-import com.bakdata.conquery.models.query.resultinfo.UniqueNamer;
 import com.bakdata.conquery.models.types.ResultType;
 import com.bakdata.conquery.models.types.SemanticType;
 import lombok.Getter;
@@ -41,17 +39,6 @@ public String userColumnName() {
 
 	@Override
 	public String defaultColumnName() {
-		return secondaryId.getLabel();
-	}
-
-	@Override
-	public ColumnDescriptor asColumnDescriptor(UniqueNamer collector) {
-		return ColumnDescriptor.builder()
-							   .label(defaultColumnName())
-							   .defaultLabel(secondaryId.getLabel())
-							   .type(getType().typeInfo())
-							   .semantics(getSemantics())
-							   .description(secondaryId.getDescription())
-							   .build();
+		return userColumnName();
 	}
 }
diff --git a/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java b/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java
index ac08f50d4a..d6643ae4bb 100644
--- a/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java
+++ b/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java
@@ -24,7 +24,7 @@ class DefaultSqlCDateSetParserTest {
 	@MethodSource("testToEpochDayRangeListProvider")
 	public void testToEpochDayRangeList(String input, String expected, String message) {
 		List> epochDayRangeList = parser.toEpochDayRangeList(input);
-		String actual = ResultPrinters.defaultPrinter(new ResultType.ListT(ResultType.Primitive.DATE_RANGE), PLAIN).print(epochDayRangeList);
+		String actual = ResultPrinters.printerFor(new ResultType.ListT(ResultType.Primitive.DATE_RANGE), PLAIN).print(epochDayRangeList);
 		Assertions.assertEquals(expected, actual, message);
 	}
 

From 15237c90fa044a222cde98c647eedeb223fa6b17 Mon Sep 17 00:00:00 2001
From: awildturtok <1553491+awildturtok@users.noreply.github.com>
Date: Thu, 22 Aug 2024 15:00:17 +0200
Subject: [PATCH 19/54] reworks Money to no longer be "cent" based and instead
 just use it as storage primitive. Is now BigDecimal

---
 .../conquery/io/result/arrow/ArrowUtil.java   |   8 +-
 .../io/result/excel/ExcelRenderer.java        |   8 +-
 .../parquet/EntityResultWriteSupport.java     |   3 +-
 .../bakdata/conquery/models/common/Range.java | 195 ++++++++++--------
 .../filters/specific/NumberFilter.java        |  15 +-
 .../conquery/models/events/Bucket.java        |   2 +-
 .../conquery/models/events/EmptyBucket.java   |   2 +-
 .../conquery/models/events/EmptyStore.java    |   6 +-
 .../models/events/stores/root/MoneyStore.java |   6 +-
 .../events/stores/specific/MoneyIntStore.java |  28 ++-
 .../preproc/parser/specific/MoneyParser.java  |  41 ++--
 .../filter/event/number/MoneyFilterNode.java  |   4 +-
 .../query/preview/EntityPreviewExecution.java |   3 +-
 .../diffsum/MoneyDiffSumAggregator.java       |  15 +-
 .../specific/sum/MoneySumAggregator.java      |  14 +-
 .../resultinfo/printers/ResultPrinters.java   |   8 +-
 .../NumberColumnStatsCollector.java           |   9 +-
 .../model/filter/NumberFilterConverter.java   |  23 +--
 .../conquery/io/result/ResultTestUtil.java    |   9 +-
 .../result/excel/ExcelResultRenderTest.java   |  66 +++---
 .../types/ColumnStoreSerializationTests.java  |   2 +-
 .../conquery/models/types/ResultTypeTest.java |   6 +-
 22 files changed, 233 insertions(+), 240 deletions(-)

diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java
index 7d2bc1ffcd..f5c755e020 100644
--- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java
+++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java
@@ -30,7 +30,8 @@ private BiFunction fieldFor(ResultType type) {
 
 		return switch (((ResultType.Primitive) type)) {
 			case BOOLEAN -> ArrowUtil::boolField;
-			case INTEGER, MONEY -> ArrowUtil::integerField;
+			case INTEGER -> ArrowUtil::integerField;
+			case MONEY -> ArrowUtil::integerField;
 			case NUMERIC -> ArrowUtil::floatField;
 			case DATE -> ArrowUtil::dateField;
 			case DATE_RANGE -> ArrowUtil::dateRangeField;
@@ -51,6 +52,11 @@ private static Field integerField(ResultInfo info, @NonNull String uniqueName) {
 		return new Field(uniqueName, FieldType.nullable(new ArrowType.Int(32, true)), null);
 	}
 
+	private static Field moneyField(ResultInfo info, @NonNull String uniqueName) {
+		//TODO shift left
+		return new Field(uniqueName, FieldType.nullable(new ArrowType.Int(32, true)), null);
+	}
+
 	private static Field floatField(ResultInfo info, @NonNull String uniqueName) {
 		return new Field(uniqueName, FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), null);
 	}
diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java
index 71e10f3ae1..08ba06e063 100644
--- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java
+++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java
@@ -344,15 +344,13 @@ public static void writeNumericCell(ResultInfo info, PrintSettings settings, Cel
 	}
 
 	public static void writeMoneyCell(ResultInfo info, PrintSettings settings, Cell cell, Object value, Map styles) {
-		CellStyle currencyStyle = styles.get(ExcelConfig.CURRENCY_STYLE_PREFIX + settings.getCurrency().getCurrencyCode());
+		final CellStyle currencyStyle = styles.get(ExcelConfig.CURRENCY_STYLE_PREFIX + settings.getCurrency().getCurrencyCode());
 		if (currencyStyle == null) {
 			// Print as cents or what ever the minor currency unit is
-			cell.setCellValue(value.toString());
+			cell.setCellValue(((BigDecimal) value).movePointRight(settings.getCurrency().getDefaultFractionDigits()).intValue());
 			return;
 		}
 		cell.setCellStyle(currencyStyle);
-		cell.setCellValue(
-				new BigDecimal(((Number) value).longValue()).movePointLeft(settings.getCurrency().getDefaultFractionDigits()).doubleValue()
-		);
+		cell.setCellValue(((BigDecimal) value).doubleValue());
 	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java
index 8dcad2291d..4208f5a772 100644
--- a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java
+++ b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java
@@ -242,7 +242,8 @@ private static ColumnConsumer getForResultType(ResultType resultType, ResultPrin
 
 		return switch (((ResultType.Primitive) resultType)) {
 			case BOOLEAN -> new BooleanTColumnConsumer();
-			case INTEGER, DATE, MONEY -> new IntegerTColumnConsumer();
+			case INTEGER, DATE -> new IntegerTColumnConsumer();
+			case MONEY -> new IntegerTColumnConsumer(); //TODO shift point left
 			case NUMERIC -> new NumericTColumnConsumer();
 			case DATE_RANGE -> new DateRangeTColumnConsumer();
 			case STRING -> new StringTColumnConsumer(printer, printSettings);
diff --git a/backend/src/main/java/com/bakdata/conquery/models/common/Range.java b/backend/src/main/java/com/bakdata/conquery/models/common/Range.java
index 49bac9bf77..930507ca07 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/common/Range.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/common/Range.java
@@ -1,7 +1,9 @@
 package com.bakdata.conquery.models.common;
 
+import java.math.BigDecimal;
 import java.util.Optional;
 
+import com.bakdata.conquery.models.config.FrontendConfig;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -10,7 +12,6 @@
 import lombok.Getter;
 import lombok.NonNull;
 import lombok.With;
-import lombok.experimental.Wither;
 
 @With
 @Getter
@@ -20,30 +21,25 @@ public class Range implements IRange> {
 	private final T min;
 	private final T max;
 
-	public Range(T min, T max){
+	public Range(T min, T max) {
 		this.min = min;
 		this.max = max;
 
-		if(!isOrdered()) {
+		if (!isOrdered()) {
 			throw new IllegalArgumentException(String.format("min '%s' is not less than max '%s'", min, max));
 		}
 	}
 
-	@Override
-	public String toString() {
-		if (isExactly()) {
-			return "[" + getMin() + "]";
-		}
-
-		if (isAtLeast()) {
-			return "[" + getMin() + ", +∞)";
-		}
-
-		if (isAtMost()) {
-			return "(-∞, " + getMax() + "]";
-		}
+	@ValidationMethod(message = "If a range is not open in one direction, min needs to be less or equal to max")
+	@JsonIgnore
+	public final boolean isOrdered() {
+		return isOpen() || min.compareTo(max) <= 0;
+	}
 
-		return "[" + getMin() + ", " + getMax() + "]";
+	@Override
+	@JsonIgnore
+	public boolean isOpen() {
+		return max == null || min == null;
 	}
 
 	public static > Range exactly(T exactly) {
@@ -67,6 +63,23 @@ public static > Range all() {
 		return new Range<>(null, null);
 	}
 
+	@Override
+	public String toString() {
+		if (isExactly()) {
+			return "[" + getMin() + "]";
+		}
+
+		if (isAtLeast()) {
+			return "[" + getMin() + ", +∞)";
+		}
+
+		if (isAtMost()) {
+			return "(-∞, " + getMax() + "]";
+		}
+
+		return "[" + getMin() + ", " + getMax() + "]";
+	}
+
 	@Override
 	@JsonIgnore
 	public boolean isExactly() {
@@ -85,18 +98,6 @@ public boolean isAtMost() {
 		return max != null && min == null;
 	}
 
-	@Override
-	@JsonIgnore
-	public boolean isAll() {
-		return max == null && min == null;
-	}
-
-	@Override
-	@JsonIgnore
-	public boolean isOpen() {
-		return max == null || min == null;
-	}
-
 	@Override
 	public boolean contains(Range other) {
 		if (other == null) {
@@ -126,12 +127,24 @@ public boolean contains(Range other) {
 		return contains(other.getMin()) && contains(other.getMax());
 	}
 
-	@ValidationMethod(message = "If a range is not open in one direction, min needs to be less or equal to max")
+	@Override
 	@JsonIgnore
-	public final boolean isOrdered() {
-		return isOpen() || min.compareTo(max) <= 0;
+	public boolean isAll() {
+		return max == null && min == null;
 	}
 
+	@Override
+	public boolean contains(T value) {
+		if (value == null) {
+			return false;
+		}
+
+		if (getMin() != null && value.compareTo(getMin()) < 0) {
+			return false;
+		}
+
+		return getMax() == null || value.compareTo(getMax()) <= 0;
+	}
 
 	@Override
 	public Range span(@NonNull Range other) {
@@ -148,28 +161,15 @@ public Range span(@NonNull Range other) {
 		return out;
 	}
 
-	@Override
-	public boolean contains(T value) {
-		if(value == null) {
-			return false;
-		}
-
-		if (getMin() != null && value.compareTo(getMin()) < 0) {
-			return false;
-		}
-
-		return getMax() == null || value.compareTo(getMax()) <= 0;
-	}
-
 	public static class IntegerRange extends Range {
 		public IntegerRange(Integer min, Integer max) {
 			super(min, max);
 		}
 
-		public static IntegerRange fromNumberRange(IRange orig){
-			return new Range.IntegerRange(
-				Optional.ofNullable(orig.getMin()).map(Number::intValue).orElse(null),
-				Optional.ofNullable(orig.getMax()).map(Number::intValue).orElse(null));
+		public static IntegerRange fromNumberRange(IRange orig) {
+			return new Range.IntegerRange(Optional.ofNullable(orig.getMin()).map(Number::intValue).orElse(null), Optional.ofNullable(orig.getMax())
+																														 .map(Number::intValue)
+																														 .orElse(null));
 		}
 
 		@Override
@@ -177,30 +177,30 @@ public boolean contains(Integer value) {
 			return value != null && contains(value.intValue());
 		}
 
-		public boolean contains(Number value) {
-			return value != null && contains(value.intValue());
-		}
-
 		public boolean contains(int value) {
-			if(getMin() != null && value < getMin()) {
+			if (getMin() != null && value < getMin()) {
 				return false;
 			}
-			if(getMax() != null && value > getMax()) {
+			if (getMax() != null && value > getMax()) {
 				return false;
 			}
 			return true;
 		}
+
+		public boolean contains(Number value) {
+			return value != null && contains(value.intValue());
+		}
 	}
 
 	public static class LongRange extends Range {
-		public LongRange (Long min, Long max) {
+		public LongRange(Long min, Long max) {
 			super(min, max);
 		}
 
-		public static LongRange fromNumberRange(IRange orig){
-			return new Range.LongRange(
-				Optional.ofNullable(orig.getMin()).map(Number::longValue).orElse(null),
-				Optional.ofNullable(orig.getMax()).map(Number::longValue).orElse(null));
+		public static LongRange fromNumberRange(IRange orig) {
+			return new Range.LongRange(Optional.ofNullable(orig.getMin()).map(Number::longValue).orElse(null), Optional.ofNullable(orig.getMax())
+																													   .map(Number::longValue)
+																													   .orElse(null));
 		}
 
 		@Override
@@ -208,19 +208,19 @@ public boolean contains(Long value) {
 			return value != null && contains(value.longValue());
 		}
 
-		public boolean contains(Number value) {
-			return value != null && contains(value.longValue());
-		}
-
 		public boolean contains(long value) {
-			if(getMin() != null && value < getMin()) {
+			if (getMin() != null && value < getMin()) {
 				return false;
 			}
-			if(getMax() != null && value > getMax()) {
+			if (getMax() != null && value > getMax()) {
 				return false;
 			}
 			return true;
 		}
+
+		public boolean contains(Number value) {
+			return value != null && contains(value.longValue());
+		}
 	}
 
 	public static class FloatRange extends Range {
@@ -228,10 +228,10 @@ public FloatRange(Float min, Float max) {
 			super(min, max);
 		}
 
-		public static FloatRange fromNumberRange(IRange orig){
-			return new Range.FloatRange(
-				Optional.ofNullable(orig.getMin()).map(Number::floatValue).orElse(null),
-				Optional.ofNullable(orig.getMax()).map(Number::floatValue).orElse(null));
+		public static FloatRange fromNumberRange(IRange orig) {
+			return new Range.FloatRange(Optional.ofNullable(orig.getMin()).map(Number::floatValue).orElse(null), Optional.ofNullable(orig.getMax())
+																														 .map(Number::floatValue)
+																														 .orElse(null));
 		}
 
 		@Override
@@ -239,22 +239,22 @@ public boolean contains(Float value) {
 			return value != null && contains(value.floatValue());
 		}
 
-		public boolean contains(Number value) {
-			return value != null && contains(value.floatValue());
-		}
-
 		public boolean contains(float value) {
-			if(getMin() != null && value < getMin()) {
+			if (getMin() != null && value < getMin()) {
 				return false;
 			}
-			if(getMax() != null && value > getMax()) {
+			if (getMax() != null && value > getMax()) {
 				return false;
 			}
-			if(Float.isNaN(value)) {
+			if (Float.isNaN(value)) {
 				return false;
 			}
 			return true;
 		}
+
+		public boolean contains(Number value) {
+			return value != null && contains(value.floatValue());
+		}
 	}
 
 	public static class DoubleRange extends Range {
@@ -262,10 +262,10 @@ public DoubleRange(Double min, Double max) {
 			super(min, max);
 		}
 
-		public static DoubleRange fromNumberRange(IRange orig){
-			return new Range.DoubleRange(
-				Optional.ofNullable(orig.getMin()).map(Number::doubleValue).orElse(null),
-				Optional.ofNullable(orig.getMax()).map(Number::doubleValue).orElse(null));
+		public static DoubleRange fromNumberRange(IRange orig) {
+			return new Range.DoubleRange(Optional.ofNullable(orig.getMin()).map(Number::doubleValue).orElse(null), Optional.ofNullable(orig.getMax())
+																														   .map(Number::doubleValue)
+																														   .orElse(null));
 		}
 
 		@Override
@@ -273,21 +273,40 @@ public boolean contains(Double value) {
 			return value != null && contains(value.doubleValue());
 		}
 
-		public boolean contains(Number value) {
-			return value != null && contains(value.doubleValue());
-		}
-
 		public boolean contains(double value) {
-			if(getMin() != null && value < getMin()) {
+			if (getMin() != null && value < getMin()) {
 				return false;
 			}
-			if(getMax() != null && value > getMax()) {
+			if (getMax() != null && value > getMax()) {
 				return false;
 			}
-			if(Double.isNaN(value)) {
+			if (Double.isNaN(value)) {
 				return false;
 			}
 			return true;
 		}
+
+		public boolean contains(Number value) {
+			return value != null && contains(value.doubleValue());
+		}
 	}
+
+	public static class MoneyRange extends Range {
+		public MoneyRange(BigDecimal min, BigDecimal max) {
+			super(min, max);
+		}
+
+		public static MoneyRange from(IRange orig, FrontendConfig.CurrencyConfig currency) {
+			BigDecimal mappedMin = Optional.ofNullable(orig.getMin())
+									  .map(val -> new BigDecimal(val.longValue()).movePointLeft(currency.getDecimalScale()))
+									  .orElse(null);
+			BigDecimal mappedMax = Optional.ofNullable(orig.getMax())
+									  .map(val -> new BigDecimal(val.longValue()).movePointLeft(currency.getDecimalScale()))
+									  .orElse(null);
+			return new Range.MoneyRange(mappedMin, mappedMax);
+		}
+
+
+	}
+
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java
index 97097425e7..356c74fd73 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java
@@ -18,6 +18,11 @@
 import com.bakdata.conquery.models.query.queryplan.filter.FilterNode;
 import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter;
 import com.bakdata.conquery.sql.conversion.model.filter.NumberFilterConverter;
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.OptBoolean;
+import jakarta.validation.constraints.NotNull;
+import lombok.EqualsAndHashCode;
 import lombok.Getter;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
@@ -31,6 +36,12 @@
 @CPSType(id = "NUMBER", base = Filter.class)
 public class NumberFilter> extends SingleColumnFilter {
 
+	@JsonIgnore
+	@JacksonInject(useInput = OptBoolean.FALSE)
+	@NotNull
+	@EqualsAndHashCode.Exclude
+	private ConqueryConfig config;
+
 	@Override
 	public void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig conqueryConfig) throws ConceptConfigurationException {
 		final String type = switch (getColumn().getType()) {
@@ -46,8 +57,8 @@ public void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig
 
 	@Override
 	public FilterNode createFilterNode(RANGE value) {
-		return switch (getColumn().getType()) {
-			case MONEY -> new MoneyFilterNode(getColumn(), (Range.LongRange) value);
+				return switch (getColumn().getType()) {
+			case MONEY -> new MoneyFilterNode(getColumn(), Range.MoneyRange.from(value, config.getFrontend().getCurrency()));
 			case INTEGER -> new IntegerFilterNode(getColumn(), (Range.LongRange) value);
 			case DECIMAL -> new DecimalFilterNode(getColumn(), ((Range) value));
 			case REAL -> new RealFilterNode(getColumn(), Range.DoubleRange.fromNumberRange(value));
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/Bucket.java b/backend/src/main/java/com/bakdata/conquery/models/events/Bucket.java
index 530aa94187..0972141c42 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/Bucket.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/Bucket.java
@@ -162,7 +162,7 @@ public BigDecimal getDecimal(int event, @NotNull Column column) {
 		return ((DecimalStore) getStore(column)).getDecimal(event);
 	}
 
-	public long getMoney(int event, @NotNull Column column) {
+	public BigDecimal getMoney(int event, @NotNull Column column) {
 		return ((MoneyStore) getStore(column)).getMoney(event);
 	}
 
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/EmptyBucket.java b/backend/src/main/java/com/bakdata/conquery/models/events/EmptyBucket.java
index 5d1551cb8d..418f5f74ce 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/EmptyBucket.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/EmptyBucket.java
@@ -73,7 +73,7 @@ public BigDecimal getDecimal(int event, Column column) {
 	}
 
 	@Override
-	public long getMoney(int event, Column column) {
+	public BigDecimal getMoney(int event, Column column) {
 		throw new IllegalStateException("Bucket for ALL_IDS_TABLE may not be evaluated.");
 	}
 
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/EmptyStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/EmptyStore.java
index 7201ae0ac9..aa32018ac3 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/EmptyStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/EmptyStore.java
@@ -114,12 +114,12 @@ public void setInteger(int event, long value) {
 	}
 
 	@Override
-	public long getMoney(int event) {
-		return 0;
+	public BigDecimal getMoney(int event) {
+		return BigDecimal.ZERO;
 	}
 
 	@Override
-	public void setMoney(int event, long money) {
+	public void setMoney(int event, BigDecimal money) {
 
 	}
 
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/MoneyStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/MoneyStore.java
index 96c048f281..c520e21e5e 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/MoneyStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/MoneyStore.java
@@ -1,6 +1,8 @@
 package com.bakdata.conquery.models.events.stores.root;
 
 
+import java.math.BigDecimal;
+
 import com.bakdata.conquery.models.events.MajorTypeId;
 
 /**
@@ -9,8 +11,8 @@
  */
 public interface MoneyStore extends ColumnStore {
 
-	long getMoney(int event);
-	void setMoney(int event, long money);
+	BigDecimal getMoney(int event);
+	void setMoney(int event, BigDecimal money);
 
 	@Override
 	default Object createScriptValue(int event) {
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java
index c66faaae0e..8a589b80bf 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java
@@ -1,27 +1,23 @@
 package com.bakdata.conquery.models.events.stores.specific;
 
+import java.math.BigDecimal;
+
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.IntegerStore;
 import com.bakdata.conquery.models.events.stores.root.MoneyStore;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import lombok.Getter;
-import lombok.Setter;
+import lombok.Data;
 import lombok.ToString;
 
 @CPSType(base = ColumnStore.class, id = "MONEY_VARINT")
-@Getter
-@Setter
+@Data
 @ToString(of = "numberType")
 public class MoneyIntStore implements MoneyStore {
 
-	protected IntegerStore numberType;
+	private final IntegerStore numberType;
+	private final int decimalShift; //TODO this might require preprocessing, consider injecting this.
 
-	@JsonCreator
-	public MoneyIntStore(IntegerStore numberType) {
-		this.numberType = numberType;
-	}
 
 	@Override
 	public int getLines() {
@@ -30,17 +26,17 @@ public int getLines() {
 
 	@Override
 	public MoneyIntStore createDescription() {
-		return new MoneyIntStore(numberType.createDescription());
+		return new MoneyIntStore(numberType.createDescription(), getDecimalShift());
 	}
 
 	@Override
 	public MoneyIntStore select(int[] starts, int[] length) {
-		return new MoneyIntStore(numberType.select(starts, length));
+		return new MoneyIntStore(numberType.select(starts, length), getDecimalShift());
 	}
 
 	@Override
-	public long getMoney(int event) {
-		return numberType.getInteger(event);
+	public BigDecimal getMoney(int event) {
+		return BigDecimal.valueOf(numberType.getInteger(event)).movePointLeft(decimalShift);
 	}
 
 	@Override
@@ -49,8 +45,8 @@ public long estimateEventBits() {
 	}
 
 	@Override
-	public void setMoney(int event, long value) {
-		numberType.setInteger(event, value);
+	public void setMoney(int event, BigDecimal value) {
+		numberType.setInteger(event, value.movePointRight(decimalShift).longValue());
 	}
 
 	@Override
diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/MoneyParser.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/MoneyParser.java
index 3d8049b4d0..31e6675686 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/MoneyParser.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/MoneyParser.java
@@ -3,7 +3,6 @@
 import java.math.BigDecimal;
 
 import com.bakdata.conquery.models.config.ConqueryConfig;
-import com.bakdata.conquery.models.config.ParserConfig;
 import com.bakdata.conquery.models.events.stores.root.IntegerStore;
 import com.bakdata.conquery.models.events.stores.root.MoneyStore;
 import com.bakdata.conquery.models.events.stores.specific.MoneyIntStore;
@@ -11,62 +10,58 @@
 import com.bakdata.conquery.models.preproc.parser.ColumnValues;
 import com.bakdata.conquery.models.preproc.parser.Parser;
 import com.bakdata.conquery.util.NumberParsing;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import lombok.Getter;
 import lombok.ToString;
 
 @ToString(callSuper = true)
-public class MoneyParser extends Parser {
+public class MoneyParser extends Parser {
 
-	private long maxValue = Long.MIN_VALUE;
-	private long minValue = Long.MAX_VALUE;
-
-	@JsonIgnore
-	private final BigDecimal moneyFactor;
+	private final int defaultFractionDigits;
+	private BigDecimal maxValue = null;
+	private BigDecimal minValue = null;
 
 	public MoneyParser(ConqueryConfig config) {
 		super(config);
-		moneyFactor = BigDecimal.valueOf(10).pow(config.getPreprocessor().getParsers().getCurrency().getDefaultFractionDigits());
+		defaultFractionDigits = config.getPreprocessor().getParsers().getCurrency().getDefaultFractionDigits();
 	}
 
 	@Override
-	protected Long parseValue(String value) throws ParsingException {
-		return NumberParsing
-					   .parseMoney(value)
-					   .multiply(moneyFactor)
-					   .longValueExact();
+	protected BigDecimal parseValue(String value) throws ParsingException {
+		return NumberParsing.parseMoney(value);
 	}
 
 	@Override
-	protected void registerValue(Long v) {
-		if (v > maxValue) {
+	protected void registerValue(BigDecimal v) {
+		if (maxValue == null){
 			maxValue = v;
 		}
-		if (v < minValue) {
+		if(minValue == null){
 			minValue = v;
 		}
+
+		maxValue = maxValue.max(v);
+		minValue = minValue.min(v);
 	}
 
 	@Override
 	protected MoneyStore decideType() {
 		IntegerParser subParser = new IntegerParser(getConfig());
-		subParser.registerValue(maxValue);
-		subParser.registerValue(minValue);
+		subParser.registerValue(maxValue.movePointRight(defaultFractionDigits).longValue());
+		subParser.registerValue(minValue.movePointRight(defaultFractionDigits).longValue());
 		subParser.setLines(getLines());
 		subParser.setNullLines(getNullLines());
 		IntegerStore subDecision = subParser.findBestType();
 
-		return new MoneyIntStore(subDecision);
+		return new MoneyIntStore(subDecision, defaultFractionDigits);
 	}
 
 	@Override
-	public void setValue(MoneyStore store, int event, Long value) {
+	public void setValue(MoneyStore store, int event, BigDecimal value) {
 		store.setMoney(event, value);
 	}
 
 	@Override
 	public ColumnValues createColumnValues() {
-		return new LongColumnValues();
+		return new ListColumnValues();
 	}
 
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/number/MoneyFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/number/MoneyFilterNode.java
index 9d1d8993d8..b6cbc41351 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/number/MoneyFilterNode.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/number/MoneyFilterNode.java
@@ -6,9 +6,9 @@
 import lombok.ToString;
 
 @ToString(callSuper = true)
-public class MoneyFilterNode extends NumberFilterNode {
+public class MoneyFilterNode extends NumberFilterNode {
 
-	public MoneyFilterNode(Column column, Range.LongRange filterValue) {
+	public MoneyFilterNode(Column column, Range.MoneyRange filterValue) {
 		super(column, filterValue);
 	}
 
diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java
index af4403f700..7ddfa024c8 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java
@@ -182,11 +182,10 @@ private static Object renderValue(Object value, ResultType type, PrintSettings p
 		return switch (((ResultType.Primitive) type)) {
 			case BOOLEAN -> BooleanNode.valueOf((Boolean) value);
 			case INTEGER -> new IntNode((Integer) value);
-			case NUMERIC -> DecimalNode.valueOf((BigDecimal) value);
+			case NUMERIC,MONEY -> DecimalNode.valueOf((BigDecimal) value);
 			case DATE -> new TextNode(new ResultPrinters.DatePrinter(printSettings).print(value)); //TODO bind printers in outer loop
 			case DATE_RANGE -> new TextNode(new ResultPrinters.DateRangePrinter(printSettings).print(value)); //TODO bind printers in outer loop
 			case STRING -> new TextNode(value.toString()); //TODO mapping
-			case MONEY -> ResultPrinters.readMoney(printSettings, ((Number) value));
 		};
 	}
 
diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/MoneyDiffSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/MoneyDiffSumAggregator.java
index 1a1edf8d17..7360e7ad0c 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/MoneyDiffSumAggregator.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/MoneyDiffSumAggregator.java
@@ -1,5 +1,6 @@
 package com.bakdata.conquery.models.query.queryplan.aggregators.specific.diffsum;
 
+import java.math.BigDecimal;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -15,13 +16,13 @@
  * Aggregator summing over {@code addendColumn} and subtracting over {@code subtrahendColumn}, for money columns.
  */
 @ToString(of = {"addendColumn", "subtrahendColumn"})
-public class MoneyDiffSumAggregator extends ColumnAggregator {
+public class MoneyDiffSumAggregator extends ColumnAggregator {
 
 	@Getter
 	private final Column addendColumn;
 	@Getter
 	private final Column subtrahendColumn;
-	private long sum;
+	private BigDecimal sum;
 	private boolean hit;
 
 	public MoneyDiffSumAggregator(Column addend, Column subtrahend) {
@@ -32,7 +33,7 @@ public MoneyDiffSumAggregator(Column addend, Column subtrahend) {
 	@Override
 	public void init(Entity entity, QueryExecutionContext context) {
 		hit = false;
-		sum = 0;
+		sum = BigDecimal.ZERO;
 	}
 
 
@@ -55,15 +56,15 @@ public void consumeEvent(Bucket bucket, int event) {
 
 		hit = true;
 
-		long addend = bucket.has(event, getAddendColumn()) ? bucket.getMoney(event, getAddendColumn()) : 0;
+		final BigDecimal addend = bucket.has(event, getAddendColumn()) ? bucket.getMoney(event, getAddendColumn()) : BigDecimal.ZERO;
 
-		long subtrahend = bucket.has(event, getSubtrahendColumn()) ? bucket.getMoney(event, getSubtrahendColumn()) : 0;
+		final BigDecimal subtrahend = bucket.has(event, getSubtrahendColumn()) ? bucket.getMoney(event, getSubtrahendColumn()) : BigDecimal.ZERO;
 
-		sum = sum + addend - subtrahend;
+		sum = sum.add(addend).subtract(subtrahend);
 	}
 
 	@Override
-	public Long createAggregationResult() {
+	public BigDecimal createAggregationResult() {
 		return hit ? sum : null;
 	}
 
diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/MoneySumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/MoneySumAggregator.java
index f9d0fc46be..facec8f618 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/MoneySumAggregator.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/MoneySumAggregator.java
@@ -1,5 +1,7 @@
 package com.bakdata.conquery.models.query.queryplan.aggregators.specific.sum;
 
+import java.math.BigDecimal;
+
 import com.bakdata.conquery.models.datasets.Column;
 import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.query.QueryExecutionContext;
@@ -11,10 +13,10 @@
  * Aggregator implementing a sum over {@code column}, for money columns.
  */
 @ToString(callSuper = true, onlyExplicitlyIncluded = true)
-public class MoneySumAggregator extends SingleColumnAggregator {
+public class MoneySumAggregator extends SingleColumnAggregator {
 
 	private boolean hit = false;
-	private long sum = 0L;
+	private BigDecimal sum;
 
 	public MoneySumAggregator(Column column) {
 		super(column);
@@ -23,7 +25,7 @@ public MoneySumAggregator(Column column) {
 	@Override
 	public void init(Entity entity, QueryExecutionContext context) {
 		hit = false;
-		sum = 0;
+		sum = BigDecimal.ZERO;
 	}
 
 
@@ -35,13 +37,13 @@ public void consumeEvent(Bucket bucket, int event) {
 
 		hit = true;
 
-		long addend = bucket.getMoney(event, getColumn());
+		final BigDecimal addend = bucket.getMoney(event, getColumn());
 
-		sum = sum + addend;
+		sum = sum.add(addend);
 	}
 
 	@Override
-	public Long createAggregationResult() {
+	public BigDecimal createAggregationResult() {
 		return hit ? sum : null;
 	}
 
diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java
index 6f1a4b6d97..888339188e 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java
@@ -1,6 +1,5 @@
 package com.bakdata.conquery.models.query.resultinfo.printers;
 
-import java.math.BigDecimal;
 import java.util.List;
 import java.util.Objects;
 import java.util.StringJoiner;
@@ -41,10 +40,6 @@ public Printer printerFor(ResultType type, PrintSettings printSettings) {
 		};
 	}
 
-	public BigDecimal readMoney(PrintSettings cfg, Number value) {
-		return new BigDecimal(value.longValue()).movePointLeft(cfg.getCurrency().getDefaultFractionDigits());
-	}
-
 	public interface Printer {
 		String print(Object f);
 	}
@@ -86,8 +81,7 @@ public record MoneyPrinter(PrintSettings cfg) implements Printer {
 		public String print(Object f) {
 
 			if (cfg.isPrettyPrint()) {
-
-				return cfg.getDecimalFormat().format(readMoney(cfg, (Number) f));
+				return cfg.getDecimalFormat().format(f);
 			}
 
 			return f.toString();
diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/NumberColumnStatsCollector.java b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/NumberColumnStatsCollector.java
index 5cf76a8cf3..ed7bee6d8f 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/NumberColumnStatsCollector.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/NumberColumnStatsCollector.java
@@ -11,7 +11,6 @@
 
 import c10n.C10N;
 import com.bakdata.conquery.models.query.PrintSettings;
-import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters;
 import com.bakdata.conquery.models.types.ResultType;
 import com.google.common.collect.Range;
 import lombok.Getter;
@@ -98,13 +97,7 @@ public void consume(Object value) {
 			return;
 		}
 
-		Number number = (Number) value;
-
-		if (ResultType.Primitive.MONEY.equals(getType())) {
-			number = ResultPrinters.readMoney(getPrintSettings(), number);
-		}
-
-		statistics.addValue(number.doubleValue());
+		statistics.addValue(((Number) value).doubleValue());
 	}
 
 	@Override
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java
index 1eb1daa5fe..6903e2dfd4 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java
@@ -1,13 +1,10 @@
 package com.bakdata.conquery.sql.conversion.model.filter;
 
-import java.math.BigDecimal;
 import java.util.List;
 
 import com.bakdata.conquery.models.common.IRange;
-import com.bakdata.conquery.models.common.Range;
 import com.bakdata.conquery.models.datasets.Column;
 import com.bakdata.conquery.models.datasets.concepts.filters.specific.NumberFilter;
-import com.bakdata.conquery.models.events.MajorTypeId;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext;
@@ -30,7 +27,7 @@ public SqlFilters convertToSqlFilter(NumberFilter filter, FilterContext rootSelect = new ExtractingSqlSelect<>(tables.getRootTable(), column.getName(), numberClass);
 
 		Field eventFilterCtePredecessor = rootSelect.qualify(tables.getPredecessor(ConceptCteStep.EVENT_FILTER)).select();
-		IRange filterValue = prepareFilterValue(column, filterContext.getValue());
+		IRange filterValue = filterContext.getValue();
 		NumberCondition condition = new NumberCondition(eventFilterCtePredecessor, filterValue);
 
 		ConnectorSqlSelects selects = ConnectorSqlSelects.builder()
@@ -52,22 +49,4 @@ public Condition convertForTableExport(NumberFilter filter, FilterContext
 		Field field = DSL.field(DSL.name(tableName, columnName), Number.class);
 		return new NumberCondition(field, filterContext.getValue()).condition();
 	}
-
-	/**
-	 * If there is a long range filter on a column of type MONEY, the filter value will represent a decimal with the point moved right 2 places right.
-	 * 

- * For example, the filter value {@code {min: 1000€, max: 2000€}} will be converted to {@code {min: 10,00€, max: 20,00€}} - */ - private static IRange prepareFilterValue(Column column, IRange filterValue) { - if (column.getType() != MajorTypeId.MONEY || !(filterValue instanceof Range.LongRange)) { - return filterValue; - } - Long min = (Long) filterValue.getMin(); - Long max = (Long) filterValue.getMax(); - return Range.LongRange.of( - BigDecimal.valueOf(min).movePointLeft(2), - BigDecimal.valueOf(max).movePointLeft(2) - ); - } - } diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java index 793beca635..22778470c9 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.io.result; +import java.math.BigDecimal; import java.util.Collections; import java.util.List; import java.util.Locale; @@ -81,7 +82,13 @@ public static List getResultTypes() { @NotNull public static List getTestEntityResults() { - return List.of(new SinglelineEntityResult("1", new Object[]{Boolean.TRUE, 2345634, 123423.34, 5646, List.of(345, 534), "test_string", 4521, List.of(true, false), List.of(List.of(345, 534), List.of(1, 2)), List.of("fizz", "buzz")}), new SinglelineEntityResult("2", new Object[]{Boolean.FALSE, null, null, null, null, null, null, List.of(), List.of(List.of(1234, Integer.MAX_VALUE)), List.of()}), new SinglelineEntityResult("2", new Object[]{Boolean.TRUE, null, null, null, null, null, null, List.of(false, false), null, null}), new MultilineEntityResult("3", List.of(new Object[]{Boolean.FALSE, null, null, null, null, null, null, List.of(false), null, null}, new Object[]{Boolean.TRUE, null, null, null, null, null, null, null, null, null}, new Object[]{Boolean.TRUE, null, null, null, null, null, 4, List.of(true, false, true, false), null, null}))); + return List.of(new SinglelineEntityResult("1", new Object[]{Boolean.TRUE, 2345634, 123423.34, 5646, List.of(345, 534), "test_string", new BigDecimal("45.21"), List.of(true, false), List.of(List.of(345, 534), List.of(1, 2)), List.of("fizz", "buzz")}), + new SinglelineEntityResult("2", new Object[]{Boolean.FALSE, null, null, null, null, null, null, List.of(), List.of(List.of(1234, Integer.MAX_VALUE)), List.of()}), + new SinglelineEntityResult("2", new Object[]{Boolean.TRUE, null, null, null, null, null, null, List.of(false, false), null, null}), + new MultilineEntityResult("3", + List.of(new Object[]{Boolean.FALSE, null, null, null, null, null, null, List.of(false), null, null}, + new Object[]{Boolean.TRUE, null, null, null, null, null, null, null, null, null}, + new Object[]{Boolean.TRUE, null, null, null, null, null, new BigDecimal(4), List.of(true, false, true, false), null, null}))); } public static class TypedSelectDummy extends Select { diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java index 58d1b710f4..b68cdcbb51 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java @@ -42,27 +42,23 @@ @Slf4j public class ExcelResultRenderTest { - static { - I18n.init(); - } - - public static final ConqueryConfig CONFIG = new ConqueryConfig(){{ + public static final ConqueryConfig CONFIG = new ConqueryConfig() {{ // Suppress java.lang.NoClassDefFoundError: com/bakdata/conquery/io/jackson/serializer/CurrencyUnitDeserializer setStorage(new NonPersistentStoreFactory()); }}; private static final List printIdFields = List.of("id1", "id2"); + static { + I18n.init(); + } @Test void writeAndRead() throws IOException { // Prepare every input data - PrintSettings printSettings = new PrintSettings( - true, - Locale.GERMAN, - null, - CONFIG, - (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), - (selectInfo) -> selectInfo.getSelect().getLabel()); + PrintSettings + printSettings = + new PrintSettings(true, Locale.GERMAN, null, CONFIG, (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), (selectInfo) -> selectInfo.getSelect() + .getLabel()); // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized List results = getTestEntityResults(); @@ -83,13 +79,9 @@ public Stream streamResults(OptionalLong maybeLimit) { // First we write to the buffer, than we read from it and parse it as TSV ByteArrayOutputStream output = new ByteArrayOutputStream(); - ExcelRenderer renderer = new ExcelRenderer(new ExcelConfig(),printSettings); + ExcelRenderer renderer = new ExcelRenderer(new ExcelConfig(), printSettings); - renderer.renderToStream( - ResultTestUtil.ID_FIELDS, - mquery, - output, OptionalLong.empty(), printSettings - ); + renderer.renderToStream(ResultTestUtil.ID_FIELDS, mquery, output, OptionalLong.empty(), printSettings); InputStream inputStream = new ByteArrayInputStream(output.toByteArray()); @@ -137,26 +129,24 @@ private List readComputed(InputStream inputStream, PrintSettings setting private List generateExpectedTSV(List results, List resultInfos) { List expected = new ArrayList<>(); expected.add(String.join("\t", printIdFields) + "\t" + getResultTypes().stream().map(ResultType::typeInfo).collect(Collectors.joining("\t"))); - results.stream() - .map(EntityResult.class::cast) - .forEach(res -> { - - for (Object[] line : res.listResultLines()) { - StringJoiner valueJoiner = new StringJoiner("\t"); - valueJoiner.add(String.valueOf(res.getEntityId())); - valueJoiner.add(String.valueOf(res.getEntityId())); - for (int lIdx = 0; lIdx < line.length; lIdx++) { - Object val = line[lIdx]; - if(val == null) { - valueJoiner.add("null"); - continue; - } - ResultInfo info = resultInfos.get(lIdx); - joinValue(valueJoiner, val, info); - } - expected.add(valueJoiner.toString()); + results.stream().map(EntityResult.class::cast).forEach(res -> { + + for (Object[] line : res.listResultLines()) { + StringJoiner valueJoiner = new StringJoiner("\t"); + valueJoiner.add(String.valueOf(res.getEntityId())); + valueJoiner.add(String.valueOf(res.getEntityId())); + for (int lIdx = 0; lIdx < line.length; lIdx++) { + Object val = line[lIdx]; + if (val == null) { + valueJoiner.add("null"); + continue; } - }); + ResultInfo info = resultInfos.get(lIdx); + joinValue(valueJoiner, val, info); + } + expected.add(valueJoiner.toString()); + } + }); return expected; } @@ -172,7 +162,7 @@ private void joinValue(StringJoiner valueJoiner, Object val, ResultInfo info) { } if (info.getType().equals(ResultType.Primitive.MONEY)) { - printVal = printVal + " €"; + printVal += " €"; } valueJoiner.add(printVal); } diff --git a/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/ColumnStoreSerializationTests.java b/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/ColumnStoreSerializationTests.java index 06f41ef571..603a960f40 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/ColumnStoreSerializationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/ColumnStoreSerializationTests.java @@ -88,7 +88,7 @@ public static List createCTypes() { return Arrays.asList( new ScaledDecimalStore(13, IntArrayStore.create(10)), - new MoneyIntStore(IntArrayStore.create(10)), + new MoneyIntStore(IntArrayStore.create(10), 2), new DirectDateRangeStore(IntegerDateStore.create(10), IntegerDateStore.create(10)), new QuarterDateRangeStore(LongArrayStore.create(10)), new IntegerDateStore(LongArrayStore.create(10)), diff --git a/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java b/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java index 283255fe98..c103bfaea9 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java @@ -63,8 +63,8 @@ public static List testData() { .intValue()), "12.07.2013 - 12.07.2014"), Arguments.of(PRETTY, ResultType.Primitive.INTEGER, 51839274, "51,839,274"), Arguments.of(PRETTY_DE, ResultType.Primitive.INTEGER, 51839274, "51.839.274"), - Arguments.of(PRETTY, ResultType.Primitive.MONEY, 51839274L, "518,392.74"), - Arguments.of(PRETTY_DE, ResultType.Primitive.MONEY, 51839274L, "518.392,74"), + Arguments.of(PRETTY, ResultType.Primitive.MONEY, new BigDecimal("518392.74"), "518,392.74"), + Arguments.of(PRETTY_DE, ResultType.Primitive.MONEY, new BigDecimal("518392.74"), "518.392,74"), Arguments.of(PRETTY, ResultType.Primitive.NUMERIC, 0.2, "0.2"), Arguments.of(PRETTY_DE, ResultType.Primitive.NUMERIC, 0.2, "0,2"), Arguments.of(PRETTY, ResultType.Primitive.NUMERIC, new BigDecimal("716283712389817246892743124.12312"), "716,283,712,389,817,246,892,743,124.12312"), @@ -76,7 +76,7 @@ public static List testData() { Arguments.of(PLAIN, ResultType.Primitive.STRING, "test", "test"), Arguments.of(PLAIN, ResultType.Primitive.DATE, LocalDate.of(2013, 7, 12).toEpochDay(), "2013-07-12"), Arguments.of(PLAIN, ResultType.Primitive.INTEGER, 51839274, "51839274"), - Arguments.of(PLAIN, ResultType.Primitive.MONEY, 51839274L, "51839274"), + Arguments.of(PLAIN, ResultType.Primitive.MONEY, new BigDecimal(51839274L), "51839274"), Arguments.of(PLAIN, ResultType.Primitive.NUMERIC, 0.2, "0.2"), Arguments.of(PLAIN, ResultType.Primitive.NUMERIC, new BigDecimal("716283712389817246892743124.12312"), "716283712389817246892743124.12312"), Arguments.of(PLAIN, ResultType.Primitive.STRING, "test", "test"), From f6ff2239e9735717b036f09c57b6296859bccb92 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 22 Aug 2024 15:46:03 +0200 Subject: [PATCH 20/54] wip, trying to fix money filler --- .../com/bakdata/conquery/io/result/arrow/ArrowRenderer.java | 2 +- .../java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java index fde3a9ad80..051aab3119 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java @@ -248,7 +248,7 @@ public static RowConsumer[] generateWriterPipeline(VectorSchemaRoot root, int ve private static RowConsumer generateVectorFiller(int pos, ValueVector vector, final PrintSettings settings, ResultPrinters.Printer printer) { if (vector instanceof IntVector intVector) { - return intVectorFiller(intVector, (line) -> (Integer) line[pos]); + return intVectorFiller(intVector, (line) -> ((Number) line[pos]).intValue()); } if (vector instanceof VarCharVector varCharVector) { diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java index f5c755e020..a77839cff4 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java @@ -31,7 +31,7 @@ private BiFunction fieldFor(ResultType type) { return switch (((ResultType.Primitive) type)) { case BOOLEAN -> ArrowUtil::boolField; case INTEGER -> ArrowUtil::integerField; - case MONEY -> ArrowUtil::integerField; + case MONEY -> ArrowUtil::moneyField; case NUMERIC -> ArrowUtil::floatField; case DATE -> ArrowUtil::dateField; case DATE_RANGE -> ArrowUtil::dateRangeField; @@ -53,7 +53,6 @@ private static Field integerField(ResultInfo info, @NonNull String uniqueName) { } private static Field moneyField(ResultInfo info, @NonNull String uniqueName) { - //TODO shift left return new Field(uniqueName, FieldType.nullable(new ArrowType.Int(32, true)), null); } From 7d53595b4037fa6f02abe108da04c7b33da54d9c Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 28 Aug 2024 17:12:06 +0200 Subject: [PATCH 21/54] Reworks Printers: We inject the result specific Printers using the printerFactory. This untangles the burden of formatting. Now a Select claiming to produce a specific type only needs to guarantee they are in a format capable of being handled by the printers it elects to use. This allows us later to make a printer that alters the result before printing it, without having to inject into all renderers. --- .../com/bakdata/conquery/ResultHeaders.java | 35 +- .../conquery/apiv1/QueryProcessor.java | 3 +- .../apiv1/query/TableExportQuery.java | 11 +- .../apiv1/query/concept/specific/CQAnd.java | 4 +- .../apiv1/query/concept/specific/CQOr.java | 4 +- .../io/result/arrow/ArrowRenderer.java | 470 +++++++++--------- .../conquery/io/result/arrow/ArrowUtil.java | 82 ++- .../io/result/arrow/ResultArrowProcessor.java | 3 +- .../conquery/io/result/arrow/RowConsumer.java | 21 +- .../io/result/csv/ResultCsvProcessor.java | 3 +- .../io/result/excel/ExcelRenderer.java | 7 +- .../io/result/excel/ResultExcelProcessor.java | 3 +- .../parquet/EntityResultWriteSupport.java | 228 +++++---- .../io/result/parquet/ParquetRenderer.java | 15 +- .../parquet/ResultParquetProcessor.java | 3 +- .../models/config/ExcelResultProvider.java | 3 +- .../models/config/IdColumnConfig.java | 3 +- .../datasets/concepts/select/Select.java | 7 +- .../select/concept/ConceptColumnSelect.java | 16 +- .../select/connector/DistinctSelect.java | 11 +- .../specific/MappableSingleColumnSelect.java | 27 +- .../models/execution/ManagedExecution.java | 2 +- .../forms/managed/ManagedInternalForm.java | 2 +- .../conquery/models/query/ManagedQuery.java | 7 +- .../conquery/models/query/PrintSettings.java | 10 +- .../models/query/SingleTableResult.java | 9 +- .../query/preview/EntityPreviewExecution.java | 48 +- .../query/resultinfo/ColumnResultInfo.java | 6 +- .../query/resultinfo/ExternalResultInfo.java | 16 +- .../resultinfo/FixedLabelResultInfo.java | 11 +- .../models/query/resultinfo/ResultInfo.java | 10 +- .../query/resultinfo/SelectResultInfo.java | 6 +- .../printers/ArrowResultPrinters.java | 12 + .../resultinfo/printers/ChainingPrinter.java | 9 + .../resultinfo/printers/ConceptIdPrinter.java | 30 ++ ...ltPrinters.java => CsvResultPrinters.java} | 134 ++--- .../printers/JavaResultPrinters.java | 101 ++++ .../printers/JsonResultPrinters.java | 95 ++++ .../printers/LocalizedEnumPrinter.java | 20 + .../resultinfo/printers/MappedPrinter.java | 11 + .../query/resultinfo/printers/Printer.java | 6 + .../resultinfo/printers/PrinterFactory.java | 39 ++ .../printers/SecondaryIdResultInfo.java | 12 +- .../BooleanColumnStatsCollector.java | 9 +- .../statistics/ColumnStatsCollector.java | 8 +- .../statistics/DateColumnStatsCollector.java | 5 +- .../HistogramColumnDescription.java | 2 +- .../query/statistics/ResultStatistics.java | 14 +- .../StringColumnStatsCollector.java | 7 +- .../sql/conversion/NodeConversions.java | 2 +- .../DefaultSqlCDateSetParserTest.java | 7 +- .../json/AbstractQueryEngineTest.java | 3 +- .../conquery/integration/json/FormTest.java | 3 +- .../conquery/io/result/ResultTestUtil.java | 13 +- .../arrow/ArrowResultGenerationTest.java | 16 +- .../result/csv/CsvResultGenerationTest.java | 6 +- .../result/excel/ExcelResultRenderTest.java | 5 +- .../parquet/ParquetResultGenerationTest.java | 6 +- .../models/execution/DefaultLabelTest.java | 2 +- .../models/query/DefaultColumnNameTest.java | 2 +- .../conquery/models/query/UniqueNameTest.java | 3 +- .../conquery/models/types/ResultTypeTest.java | 13 +- 62 files changed, 996 insertions(+), 685 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ChainingPrinter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ConceptIdPrinter.java rename backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/{ResultPrinters.java => CsvResultPrinters.java} (52%) create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/LocalizedEnumPrinter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/MappedPrinter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java diff --git a/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java b/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java index 9d3eb527e3..2edd68e2ee 100644 --- a/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java +++ b/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java @@ -9,7 +9,8 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.FixedLabelResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.LocalizedEnumPrinter; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import lombok.experimental.UtilityClass; @@ -21,7 +22,7 @@ public static ResultInfo datesInfo(PrintSettings settings) { final ResultType.ListT type = new ResultType.ListT<>(ResultType.Primitive.DATE_RANGE); - return new FixedLabelResultInfo(label, label, type, Set.of(new SemanticType.EventDateT()), settings, ResultPrinters.printerFor(type, settings)); + return new FixedLabelResultInfo(label, label, type, Set.of(new SemanticType.EventDateT()), settings); } public static ResultInfo historyDatesInfo(PrintSettings settings) { @@ -29,44 +30,52 @@ public static ResultInfo historyDatesInfo(PrintSettings settings) { final ResultType.ListT type = new ResultType.ListT<>(ResultType.Primitive.DATE_RANGE); - return new FixedLabelResultInfo(label, label, type, Set.of(new SemanticType.EventDateT(), new SemanticType.GroupT()), settings, ResultPrinters.printerFor(type, settings)); + return new FixedLabelResultInfo(label, label, type, Set.of(new SemanticType.EventDateT(), new SemanticType.GroupT()), settings); } public static ResultInfo sourceInfo(PrintSettings settings) { final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()).source(); - return new FixedLabelResultInfo(label, label, ResultType.Primitive.STRING, Set.of(new SemanticType.SourcesT(), new SemanticType.CategoricalT(), new SemanticType.GroupT()), settings, ResultPrinters.printerFor(ResultType.Primitive.STRING, settings)); + return new FixedLabelResultInfo(label, label, ResultType.Primitive.STRING, Set.of(new SemanticType.SourcesT(), new SemanticType.CategoricalT(), new SemanticType.GroupT()), settings); } public static ResultInfo formContextInfo(PrintSettings settings) { final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()).index(); - return new FixedLabelResultInfo(label, label, ResultType.Primitive.INTEGER, Set.of(), settings, ResultPrinters.printerFor(ResultType.Primitive.INTEGER, settings)); + return new FixedLabelResultInfo(label, label, ResultType.Primitive.INTEGER, Set.of(), settings); } public static ResultInfo formDateRangeInfo(PrintSettings settings) { - final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()) - .dateRange(); + final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()).dateRange(); - return new FixedLabelResultInfo(label, label, ResultType.Primitive.DATE_RANGE, Set.of(), settings, ResultPrinters.printerFor(ResultType.Primitive.DATE_RANGE, settings)); + return new FixedLabelResultInfo(label, label, ResultType.Primitive.DATE_RANGE, Set.of(), settings); } public static ResultInfo formResolutionInfo(PrintSettings settings) { final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()).resolution(); - return new FixedLabelResultInfo(label, label, ResultType.Primitive.STRING, Set.of(), settings, new ResultPrinters.LocalizedEnumPrinter<>(settings, Resolution.class)); + return new FixedLabelResultInfo(label, label, ResultType.Primitive.STRING, Set.of(), settings) { + @Override + public Printer getPrinter() { + return new LocalizedEnumPrinter<>(settings, Resolution.class); + } + }; } public static ResultInfo formEventDateInfo(PrintSettings settings) { - final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()) - .eventDate(); + final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()).eventDate(); - return new FixedLabelResultInfo(label, label, ResultType.Primitive.DATE, Set.of(), settings, ResultPrinters.printerFor(ResultType.Primitive.DATE, settings)); + return new FixedLabelResultInfo(label, label, ResultType.Primitive.DATE, Set.of(), settings); } public static ResultInfo formObservationScopeInfo(PrintSettings settings) { final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()).observationScope(); - return new FixedLabelResultInfo(label, label, ResultType.Primitive.STRING, Set.of(), settings, new ResultPrinters.LocalizedEnumPrinter<>(settings, FeatureGroup.class)); + return new FixedLabelResultInfo(label, label, ResultType.Primitive.STRING, Set.of(), settings) { + @Override + public Printer getPrinter() { + return new LocalizedEnumPrinter<>(settings, FeatureGroup.class); + } + }; } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index 7bb95fa2f2..3d469d6271 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -74,6 +74,7 @@ import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; +import com.bakdata.conquery.models.query.resultinfo.printers.JavaResultPrinters; import com.bakdata.conquery.models.query.statistics.ResultStatistics; import com.bakdata.conquery.models.query.visitor.QueryVisitor; import com.bakdata.conquery.models.types.SemanticType; @@ -569,7 +570,7 @@ public ResultStatistics getResultStatistics(SingleTableResult managedQuery) { final PrintSettings printSettings = - new PrintSettings(true, locale, managedQuery.getNamespace(), config, null, null, decimalFormat, integerFormat); + new PrintSettings(true, locale, managedQuery.getNamespace(), config, null, null, decimalFormat, integerFormat, new JavaResultPrinters()); final UniqueNamer uniqueNamer = new UniqueNamer(printSettings); final List resultInfos = managedQuery.getResultInfos(printSettings); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java index c8f88d8dbb..1613833613 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java @@ -41,7 +41,9 @@ import com.bakdata.conquery.models.query.queryplan.TableExportQueryPlan; import com.bakdata.conquery.models.query.resultinfo.ColumnResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.ChainingPrinter; +import com.bakdata.conquery.models.query.resultinfo.printers.ConceptIdPrinter; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.query.resultinfo.printers.SecondaryIdResultInfo; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; @@ -266,7 +268,7 @@ private List createResultInfos(Set conceptColumns, PrintSett final Set semantics = new HashSet<>(); ResultType resultType = ResultType.resolveResultType(column.getType()); - ResultPrinters.Printer printer = ResultPrinters.printerFor(resultType, printSettings); + Printer printer = printSettings.getPrinterFactory().printerFor(resultType, printSettings); if (connectorColumns.containsKey(column)) { // Additionally, Concept Columns are returned as ConceptElementId, when rawConceptColumns is not set. @@ -278,7 +280,10 @@ private List createResultInfos(Set conceptColumns, PrintSett if (!isRawConceptValues()) { resultType = ResultType.Primitive.STRING; - printer = new ResultPrinters.ConceptIdPrinter(concept, printSettings); + printer = new ChainingPrinter( + new ConceptIdPrinter(concept, printSettings), + printSettings.getPrinterFactory().getStringPrinter(printSettings) + ); } } else { diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java index f49ff668f9..d721f57e2e 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java @@ -28,7 +28,6 @@ import com.bakdata.conquery.models.query.queryplan.specific.AndNode; import com.bakdata.conquery.models.query.resultinfo.FixedLabelResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.util.QueryUtils; import com.fasterxml.jackson.annotation.JsonView; @@ -120,11 +119,10 @@ public List getResultInfos(PrintSettings settings) { } if (createExists()) { - final ResultPrinters.BooleanPrinter printer = new ResultPrinters.BooleanPrinter(settings); final String userOrDefaultLabel = getUserOrDefaultLabel(settings.getLocale()); final String defaultLabel = defaultLabel(settings.getLocale()); - resultInfos.add(new FixedLabelResultInfo(userOrDefaultLabel, defaultLabel, ResultType.Primitive.BOOLEAN, Set.of(), settings, printer)); + resultInfos.add(new FixedLabelResultInfo(userOrDefaultLabel, defaultLabel, ResultType.Primitive.BOOLEAN, Set.of(), settings)); } return resultInfos; } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java index 6037200bea..427cae2107 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java @@ -28,7 +28,6 @@ import com.bakdata.conquery.models.query.queryplan.specific.OrNode; import com.bakdata.conquery.models.query.resultinfo.FixedLabelResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.util.QueryUtils; import com.fasterxml.jackson.annotation.JsonView; @@ -125,11 +124,10 @@ public List getResultInfos(PrintSettings settings) { } if (createExists()) { - final ResultPrinters.BooleanPrinter printer = new ResultPrinters.BooleanPrinter(settings); final String userOrDefaultLabel = getUserOrDefaultLabel(settings.getLocale()); final String defaultLabel = defaultLabel(settings.getLocale()); - resultInfos.add(new FixedLabelResultInfo(userOrDefaultLabel, defaultLabel, ResultType.Primitive.BOOLEAN, Set.of(), settings, printer)); + resultInfos.add(new FixedLabelResultInfo(userOrDefaultLabel, defaultLabel, ResultType.Primitive.BOOLEAN, Set.of(), settings)); } return resultInfos; diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java index 051aab3119..6218824d3a 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java @@ -3,6 +3,7 @@ import static com.bakdata.conquery.io.result.arrow.ArrowUtil.ROOT_ALLOCATOR; import java.io.IOException; +import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.function.Function; @@ -14,8 +15,9 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; -import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.query.results.EntityResult; +import com.bakdata.conquery.models.types.ResultType; import lombok.extern.slf4j.Slf4j; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.BitVector; @@ -37,7 +39,7 @@ @Slf4j public class ArrowRenderer { - public static void renderToStream( + public static void renderToStream( Function writerProducer, PrintSettings printSettings, ArrowConfig arrowConfig, @@ -45,265 +47,277 @@ public static void renderToStream( List resultInfo, Stream results) throws IOException { - List fields = ArrowUtil.generateFields(idHeaders, resultInfo, new UniqueNamer(printSettings)); - VectorSchemaRoot root = VectorSchemaRoot.create(new Schema(fields, null), ROOT_ALLOCATOR); + final List fields = ArrowUtil.generateFields(idHeaders, resultInfo, new UniqueNamer(printSettings)); + final VectorSchemaRoot root = VectorSchemaRoot.create(new Schema(fields, null), ROOT_ALLOCATOR); // Build separate pipelines for id and value, as they have different sources but the same target - RowConsumer[] idWriters = generateWriterPipeline(root, 0, idHeaders.size(), printSettings, idHeaders); - RowConsumer[] valueWriter = generateWriterPipeline(root, idHeaders.size(), resultInfo.size(), printSettings, resultInfo); + final RowConsumer[] idWriters = generateWriterPipeline(root, 0, idHeaders.size(), idHeaders); + final RowConsumer[] valueWriter = generateWriterPipeline(root, idHeaders.size(), resultInfo.size(), resultInfo); + + final List printers = new ArrayList<>(); + + for (ResultInfo header : idHeaders) { + printers.add(header.getPrinter()); + } + + for (ResultInfo info : resultInfo) { + printers.add(info.getPrinter()); + } + // Write the data try (ArrowWriter writer = writerProducer.apply(root)) { - write(writer, root, idWriters, valueWriter, printSettings.getIdMapper(), results, arrowConfig.getBatchSize()); + write(writer, root, idWriters, valueWriter, printSettings.getIdMapper(), printers, results, arrowConfig.getBatchSize()); } - } + } public static void write( ArrowWriter writer, VectorSchemaRoot root, - RowConsumer[] idWriter, - RowConsumer[] valueWriter, + RowConsumer[] idWriters, + RowConsumer[] valueWriters, PrintIdMapper idMapper, + List printers, Stream results, int batchSize) throws IOException { Preconditions.checkArgument(batchSize > 0, "Batch size needs be larger than 0."); // TODO add time metric for writing - log.trace("Starting result write"); - writer.start(); - int batchCount = 0; - int batchLineCount = 0; - Iterator resultIterator = results.iterator(); - while (resultIterator.hasNext()) { - EntityResult cer = resultIterator.next(); - for (Object[] line : cer.listResultLines()) { - if(line.length != valueWriter.length) { - throw new IllegalStateException("The number of value writers and values in a result line differs. Writers: " + valueWriter.length + " Line: " + line.length); - } - for (RowConsumer rowConsumer : idWriter) { - // Write id information - rowConsumer.accept(batchLineCount, idMapper.map(cer).getExternalId()); - } - for (RowConsumer rowConsumer : valueWriter) { - // Write values - rowConsumer.accept(batchLineCount, line); - } - batchLineCount++; - - if (batchLineCount >= batchSize) { - root.setRowCount(batchLineCount); - writer.writeBatch(); - root.clear(); - batchLineCount = 0; - } - } - } - if (batchLineCount > 0) { - root.setRowCount(batchLineCount); - writer.writeBatch(); - root.clear(); - batchCount++; - } - log.trace("Wrote {} batches of size {} (last batch might be smaller)", batchCount, batchSize); - writer.end(); - } - - private static RowConsumer intVectorFiller(IntVector vector, Function resultExtractor) { - return (rowNumber, line) -> { - Integer value = resultExtractor.apply(line); - if (value == null) { - vector.setNull(rowNumber); - return; - } - vector.setSafe(rowNumber, value); - }; - } - - private static RowConsumer bitVectorFiller(BitVector vector, Function resultExtractor) { - return (rowNumber, line) -> { - Boolean value = resultExtractor.apply(line); - if (value == null) { - vector.setNull(rowNumber); - return; - } - vector.setSafe(rowNumber, value ? 1 : 0); - }; - } - - private static RowConsumer float8VectorFiller(Float8Vector vector, Function resultExtractor) { - return (rowNumber, line) -> { - Number value = resultExtractor.apply(line); - if (value == null) { - vector.setNull(rowNumber); - return; - } - vector.setSafe(rowNumber, value.doubleValue()); - }; - } - - private static RowConsumer float4VectorFiller(Float4Vector vector, Function resultExtractor) { - return (rowNumber, line) -> { - Number value = resultExtractor.apply(line); - if (value == null) { - vector.setNull(rowNumber); - return; - } - vector.setSafe(rowNumber, value.floatValue()); - }; - } - - private static RowConsumer varCharVectorFiller(VarCharVector vector, Function resultExtractor) { - return (rowNumber, line) -> { - String value = resultExtractor.apply(line); - if (value == null) { - vector.setNull(rowNumber); - return; - } - vector.setSafe(rowNumber, new Text(value)); - }; - } - - private static RowConsumer dateDayVectorFiller(DateDayVector vector, Function resultExtractor) { - return (rowNumber, line) -> { - Number value = resultExtractor.apply(line); - if (value == null) { - vector.setNull(rowNumber); - return; - } - - // Treat our internal infinity dates (Interger.MIN and Integer.MAX) also as null - final int epochDay = value.intValue(); - if (CDate.isNegativeInfinity(epochDay) || CDate.isPositiveInfinity(epochDay)) { - vector.setNull(rowNumber); - return; - } - - vector.setSafe(rowNumber, epochDay); - }; - } - - private static RowConsumer structVectorFiller(StructVector vector, RowConsumer [] nestedConsumers, Function> resultExtractor) { - return (rowNumber, line) -> { - // Values is a horizontal list - List values = resultExtractor.apply(line); - if (values == null) { - vector.setNull(rowNumber); - return; - } - if(values.size() != nestedConsumers.length) { - throw new IllegalStateException("The number of the provided nested value differs from the number of consumer for the generated vectors. Provided values: " + values + "\t Available consumers: " + nestedConsumers.length); - } - for (RowConsumer nestedConsumer : nestedConsumers) { - nestedConsumer.accept(rowNumber, values.toArray()); - } - - // Finally mark that we populated the nested vectors - vector.setIndexDefined(rowNumber); - }; - } - - private static RowConsumer listVectorFiller(ListVector vector, RowConsumer nestedConsumer, Function> resultExtractor){ - return (rowNumber, line) -> { - // Values is a vertical list - List values = resultExtractor.apply(line); - if (values == null) { - vector.setNull(rowNumber); - return; - } - - int start = vector.startNewValue(rowNumber); - - for (int i = 0; i < values.size(); i++) { - // These short lived one value arrays are a workaround at the moment - nestedConsumer.accept(Math.addExact(start, i), new Object[] {values.get(i)}); - } - - vector.endValue(rowNumber, values.size()); - }; - } - - - public static RowConsumer[] generateWriterPipeline(VectorSchemaRoot root, int vectorOffset, int numVectors, final PrintSettings settings, List resultInfos) { - Preconditions.checkArgument(vectorOffset >= 0, "Offset was negative: %s", vectorOffset); - Preconditions.checkArgument(numVectors >= 0, "Number of vectors was negative: %s", numVectors); - - RowConsumer[] builder = new RowConsumer[numVectors]; - - for ( - int vecI = vectorOffset; - (vecI < root.getFieldVectors().size()) && (vecI < vectorOffset + numVectors); - vecI++ - ) { - final int pos = vecI - vectorOffset; - final FieldVector vector = root.getVector(vecI); - final ResultInfo resultInfo = resultInfos.get(pos); - builder[pos] = generateVectorFiller(pos, vector, settings, resultInfo.getPrinter()); + log.trace("Starting result write"); - } - return builder; + writer.start(); + int batchCount = 0; + int batchLineCount = 0; + final Iterator resultIterator = results.iterator(); + while (resultIterator.hasNext()) { + final EntityResult cer = resultIterator.next(); - } + final String[] externalId = idMapper.map(cer).getExternalId(); - private static RowConsumer generateVectorFiller(int pos, ValueVector vector, final PrintSettings settings, ResultPrinters.Printer printer) { - if (vector instanceof IntVector intVector) { - return intVectorFiller(intVector, (line) -> ((Number) line[pos]).intValue()); - } + final Object[] printedExternalId = new String[externalId.length]; + + for (int index = 0; index < idWriters.length; index++) { + printedExternalId[index] = printers.get(index).print(externalId[index]); + } + + for (Object[] line : cer.listResultLines()) { + Preconditions.checkState( + line.length == valueWriters.length, + "The number of value writers and values in a result line differs. Writers: %d Line: %d".formatted(valueWriters.length, line.length) + ); - if (vector instanceof VarCharVector varCharVector) { - return varCharVectorFiller( - varCharVector, - (line) -> { - // This is a bit clunky at the moment, since this lambda is executed for each textual value - // in the result, but it should be okay for now. This code moves as soon shards deliver themselves - // arrow as a result. + for (int index = 0; index < idWriters.length; index++) { + if(printedExternalId[index] == null){ + continue; + } - if (line[pos] == null) { - // If there is no value, we don't want to have it displayed as an empty string (see next if) - return null; - } - // We reference the printer directly, - return printer.print(line[pos]); - }); - } + idWriters[index].accept(batchLineCount, printedExternalId[index]); + } - if (vector instanceof BitVector bitVector) { - return bitVectorFiller(bitVector, (line) -> (Boolean) line[pos]); - } + for (int index = 0; index < valueWriters.length; index++) { + final int colId = index + idWriters.length; + // In this case, the printer normalizes and adjusts values. - if (vector instanceof Float4Vector float4Vector) { - return float4VectorFiller(float4Vector, (line) -> (Number) line[pos]); - } + final Object printed = printers.get(colId).print(line[index]); - if (vector instanceof Float8Vector float8Vector) { - return float8VectorFiller(float8Vector, (line) -> (Number) line[pos]); - } + if (printed == null) { + continue; + } + valueWriters[index].accept(batchLineCount, printed); + } - if (vector instanceof DateDayVector dateDayVector) { - return dateDayVectorFiller(dateDayVector, (line) -> (Number) line[pos]); - } + batchLineCount++; + + if (batchLineCount >= batchSize) { + root.setRowCount(batchLineCount); + writer.writeBatch(); + root.clear(); + batchLineCount = 0; + } + } + } + if (batchLineCount > 0) { + root.setRowCount(batchLineCount); + writer.writeBatch(); + root.clear(); + batchCount++; + } + log.trace("Wrote {} batches of size {} (last batch might be smaller)", batchCount, batchSize); + writer.end(); + } + + private static RowConsumer intVectorFiller(IntVector vector) { + return (rowNumber, valueRaw) -> { + final Integer value = (Integer) valueRaw; + if (value == null) { + vector.setNull(rowNumber); + return; + } + vector.setSafe(rowNumber, value); + }; + } + + private static RowConsumer bitVectorFiller(BitVector vector) { + return (rowNumber, valueRaw) -> { + final Boolean value = (Boolean) valueRaw; + if (value == null) { + vector.setNull(rowNumber); + return; + } + vector.setSafe(rowNumber, value ? 1 : 0); + }; + } + + private static RowConsumer float8VectorFiller(Float8Vector vector) { + return (rowNumber, valueRaw) -> { + final Number value = (Number) valueRaw; + if (value == null) { + vector.setNull(rowNumber); + return; + } + vector.setSafe(rowNumber, value.doubleValue()); + }; + } + + private static RowConsumer float4VectorFiller(Float4Vector vector) { + return (rowNumber, valueRaw) -> { + final Number value = (Number) valueRaw; + if (value == null) { + vector.setNull(rowNumber); + return; + } + vector.setSafe(rowNumber, value.floatValue()); + }; + } + + private static RowConsumer varCharVectorFiller(VarCharVector vector) { + return (rowNumber, valueRaw) -> { + final String value = (String) valueRaw; + if (value == null) { + vector.setNull(rowNumber); + return; + } + vector.setSafe(rowNumber, new Text(value)); + }; + } + + private static RowConsumer dateDayVectorFiller(DateDayVector vector) { + return (rowNumber, valueRaw) -> { + final Number value = (Number) valueRaw; + if (value == null) { + vector.setNull(rowNumber); + return; + } + + // Treat our internal infinity dates (Interger.MIN and Integer.MAX) also as null + final int epochDay = value.intValue(); + + if (CDate.isNegativeInfinity(epochDay) || CDate.isPositiveInfinity(epochDay)) { + vector.setNull(rowNumber); + return; + } + + vector.setSafe(rowNumber, epochDay); + }; + } + + private static RowConsumer structVectorFiller(StructVector vector, RowConsumer[] nestedConsumers) { + return (rowNumber, valueRaw) -> { + // Values is a horizontal list + final List values = (List) valueRaw; + if (values == null) { + vector.setNull(rowNumber); + return; + } + + Preconditions.checkState( + values.size() == nestedConsumers.length, + "The number of the provided nested value differs from the number of consumer for the generated vectors. Provided values: %s\t Available consumers: %d".formatted(values, nestedConsumers.length)); + + for (RowConsumer nestedConsumer : nestedConsumers) { + nestedConsumer.accept(rowNumber, values.toArray()); + } + + // Finally mark that we populated the nested vectors + vector.setIndexDefined(rowNumber); + }; + } + + private static RowConsumer listVectorFiller(ListVector vector, RowConsumer nestedConsumer) { + return (rowNumber, valueRaw) -> { + // Values is a vertical list + final List values = (List) valueRaw; + + if (values == null) { + vector.setNull(rowNumber); + return; + } + + final int start = vector.startNewValue(rowNumber); + + for (int i = 0; i < values.size(); i++) { + // These short lived one value arrays are a workaround at the moment + nestedConsumer.accept(Math.addExact(start, i), new Object[]{values.get(i)}); + } + + vector.endValue(rowNumber, values.size()); + }; + } + + + public static RowConsumer[] generateWriterPipeline(VectorSchemaRoot root, int vectorOffset, int numVectors, List resultInfos) { + Preconditions.checkArgument(vectorOffset >= 0, "Offset was negative: %s", vectorOffset); + Preconditions.checkArgument(numVectors >= 0, "Number of vectors was negative: %s", numVectors); + + final RowConsumer[] builder = new RowConsumer[numVectors]; + + for (int vecI = vectorOffset; (vecI < root.getFieldVectors().size()) && (vecI < vectorOffset + numVectors); vecI++) { + final int pos = vecI - vectorOffset; + final FieldVector vector = root.getVector(vecI); + final ResultInfo resultInfo = resultInfos.get(pos); + builder[pos] = generateVectorFiller(vector, resultInfo.getType()); + + } + return builder; + + } + + private static RowConsumer generateVectorFiller(ValueVector vector, ResultType type) { + if (type instanceof ResultType.ListT listT) { + final ValueVector nestedVector = ((ListVector) vector).getDataVector(); + + return listVectorFiller(((ListVector) vector), generateVectorFiller(nestedVector, listT.getElementType())); + } + //TODO who dis? + // if (vector instanceof Float4Vector float4Vector) { + // return float4VectorFiller(float4Vector, (line) -> (Number) line[pos]); + // } - if (vector instanceof StructVector structVector) { + return switch (((ResultType.Primitive) type)) { + case BOOLEAN -> bitVectorFiller(((BitVector) vector)); + case INTEGER -> intVectorFiller(((IntVector) vector)); + case MONEY -> float8VectorFiller(((Float8Vector) vector)); + case DATE -> dateDayVectorFiller(((DateDayVector) vector)); + case NUMERIC -> float8VectorFiller((Float8Vector) vector); + case STRING -> varCharVectorFiller(((VarCharVector) vector)); - List nestedVectors = structVector.getPrimitiveVectors(); - RowConsumer [] nestedConsumers = new RowConsumer[nestedVectors.size()]; + case DATE_RANGE -> { + final StructVector structVector = (StructVector) vector; + final List nestedVectors = structVector.getPrimitiveVectors(); + final RowConsumer[] nestedConsumers = new RowConsumer[nestedVectors.size()]; - for (int i = 0; i < nestedVectors.size(); i++) { - nestedConsumers[i] = generateVectorFiller(i, nestedVectors.get(i), settings, printer); - } - return structVectorFiller(structVector, nestedConsumers, (line) -> (List) line[pos]); - } + for (int i = 0; i < nestedVectors.size(); i++) { + nestedConsumers[i] = generateVectorFiller(nestedVectors.get(i), ResultType.Primitive.DATE); + } - if (vector instanceof ListVector listVector) { + yield structVectorFiller(structVector, nestedConsumers); + } - ValueVector nestedVector = listVector.getDataVector(); + }; - // pos = 0 is a workaround for now - return listVectorFiller(listVector, generateVectorFiller(0, nestedVector, settings, ((ResultPrinters.ListPrinter) printer).elementPrinter()), (line) -> (List) line[pos]); - } - throw new IllegalArgumentException("Unsupported vector type " + vector); - } + } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java index a77839cff4..e4c02a301f 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java @@ -2,8 +2,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.function.BiFunction; -import java.util.stream.Collectors; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; @@ -23,98 +21,80 @@ public class ArrowUtil { public static final RootAllocator ROOT_ALLOCATOR = new RootAllocator(); - private BiFunction fieldFor(ResultType type) { + private Field fieldFor(ResultType type, String name) { if (type instanceof ResultType.ListT) { - return ArrowUtil::listField; + return ArrowUtil.listField(name, type); } return switch (((ResultType.Primitive) type)) { - case BOOLEAN -> ArrowUtil::boolField; - case INTEGER -> ArrowUtil::integerField; - case MONEY -> ArrowUtil::moneyField; - case NUMERIC -> ArrowUtil::floatField; - case DATE -> ArrowUtil::dateField; - case DATE_RANGE -> ArrowUtil::dateRangeField; - case STRING -> ArrowUtil::stringField; + case BOOLEAN -> ArrowUtil.boolField(name); + case INTEGER -> ArrowUtil.integerField(name); + case MONEY -> ArrowUtil.moneyField(name); + case NUMERIC -> ArrowUtil.floatField(name); + case DATE -> ArrowUtil.dateField(name); + case DATE_RANGE -> ArrowUtil.dateRangeField(name); + case STRING -> ArrowUtil.stringField(name); }; } - private static Field stringField(ResultInfo info, @NonNull String uniqueName) { + private static Field stringField(@NonNull String uniqueName) { return new Field(uniqueName, FieldType.nullable(new ArrowType.Utf8()), null); } - private static Field boolField(ResultInfo info, @NonNull String uniqueName) { + private static Field boolField(@NonNull String uniqueName) { return new Field(uniqueName, FieldType.nullable(ArrowType.Bool.INSTANCE), null); } - private static Field integerField(ResultInfo info, @NonNull String uniqueName) { + private static Field integerField(@NonNull String uniqueName) { return new Field(uniqueName, FieldType.nullable(new ArrowType.Int(32, true)), null); } - private static Field moneyField(ResultInfo info, @NonNull String uniqueName) { - return new Field(uniqueName, FieldType.nullable(new ArrowType.Int(32, true)), null); + private static Field moneyField(@NonNull String uniqueName) { + //TODO We could use Decimal here? + return new Field(uniqueName, FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), null); } - private static Field floatField(ResultInfo info, @NonNull String uniqueName) { + private static Field floatField(@NonNull String uniqueName) { return new Field(uniqueName, FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), null); } - private static Field dateField(ResultInfo info, @NonNull String uniqueName) { + private static Field dateField(@NonNull String uniqueName) { return new Field(uniqueName, FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null); } - private static Field dateRangeField(ResultInfo info, @NonNull String uniqueName) { + private static Field dateRangeField(@NonNull String uniqueName) { return new Field( uniqueName, FieldType.nullable(ArrowType.Struct.INSTANCE), List.of( - dateField(info, "min"), - dateField(info, "max") + dateField("min"), + dateField("max") )); } - private static Field listField(ResultInfo info, @NonNull String uniqueName) { - if (!(info.getType() instanceof ResultType.ListT)) { - throw new IllegalStateException("Expected result type of " + ResultType.ListT.class.getName() + " but got " + info.getType().getClass().getName()); - } - - final ResultType elementType = ((ResultType.ListT) info.getType()).getElementType(); - BiFunction nestedFieldCreator = fieldFor(elementType); - final Field nestedField = nestedFieldCreator.apply(info, uniqueName); - return new Field( - uniqueName, - FieldType.nullable(ArrowType.List.INSTANCE), - List.of(nestedField) - ); - } - - /** - * Creates an arrow field vector (a column) corresponding to the internal conquery type and initializes the column with - * a localized header. - * @param info internal meta data for the result column - * @param collector to create unique names across the columns - * @return a Field (the arrow representation of the column) - */ - public Field createField(ResultInfo info, UniqueNamer collector) { - // Fallback to string field if type is not explicitly registered - BiFunction fieldCreator = fieldFor(info.getType()); - return fieldCreator.apply(info, collector.getUniqueName(info)); + private static Field listField(@NonNull String uniqueName, ResultType type) { + final ResultType elementType = ((ResultType.ListT) type).getElementType(); + final Field nestedField = fieldFor(elementType, uniqueName); + + return new Field(uniqueName, FieldType.nullable(ArrowType.List.INSTANCE), List.of(nestedField)); } public static List generateFields(@NonNull List info, UniqueNamer collector) { return info.stream() - .map(i -> createField(i, collector)) - .collect(Collectors.toUnmodifiableList()); + .map(i -> fieldFor(i.getType(), collector.getUniqueName(i))) + .toList(); } @NotNull public static List generateFields(List idHeaders, List resultInfo, UniqueNamer uniqueNamer) { // Combine id and value Fields to one vector to build a schema - final List idFields = generateFields(idHeaders, uniqueNamer); - List fields = new ArrayList<>(idFields); + List fields = new ArrayList<>(); + + fields.addAll(generateFields(idHeaders, uniqueNamer)); fields.addAll(generateFields(resultInfo, uniqueNamer)); + return fields; } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java index 7e2c02ada1..dd73ddb763 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java @@ -23,6 +23,7 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.models.query.resultinfo.printers.ArrowResultPrinters; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.util.io.ConqueryMDC; @@ -95,7 +96,7 @@ public static Response getArrow IdPrinter idPrinter = IdColumnUtil.getIdPrinter(subject, exec, namespace, config.getIdColumns().getIds()); final Locale locale = I18n.LOCALE.get(); - PrintSettings settings = new PrintSettings(pretty, locale, namespace, config, idPrinter::createId, null); + PrintSettings settings = new PrintSettings(pretty, locale, namespace, config, idPrinter::createId, null, new ArrowResultPrinters()); // Collect ResultInfos for id columns and result columns diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/RowConsumer.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/RowConsumer.java index 9c9d27a5af..f39381b47f 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/RowConsumer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/RowConsumer.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.io.result.arrow; -import java.util.Objects; - @FunctionalInterface public interface RowConsumer { @@ -10,23 +8,6 @@ public interface RowConsumer { * * @param t the input argument */ - void accept(int rowNumber, Object[] row); - - /** - * Returns a composed {@code Consumer} that performs, in sequence, this - * operation followed by the {@code after} operation. If performing either - * operation throws an exception, it is relayed to the caller of the - * composed operation. If performing this operation throws an exception, - * the {@code after} operation will not be performed. - * - * @param after the operation to perform after this operation - * @return a composed {@code Consumer} that performs in sequence this - * operation followed by the {@code after} operation - * @throws NullPointerException if {@code after} is null - */ - default RowConsumer andThen(RowConsumer after) { - Objects.requireNonNull(after); - return (n, r) -> { accept(n, r); after.accept(n, r); }; - } + void accept(int rowNumber, Object row); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java index 0eb59cca95..3def872f78 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java @@ -18,6 +18,7 @@ import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; +import com.bakdata.conquery.models.query.resultinfo.printers.CsvResultPrinters; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.resources.ResourceConstants; @@ -57,7 +58,7 @@ public Response createResult(Su // Get the locale extracted by the LocaleFilter final Locale locale = I18n.LOCALE.get(); - final PrintSettings settings = new PrintSettings(pretty, locale, namespace, config, idPrinter::createId, null); + final PrintSettings settings = new PrintSettings(pretty, locale, namespace, config, idPrinter::createId, null, new CsvResultPrinters()); final StreamingOutput out = os -> { try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(os, charset))) { diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java index 08ba06e063..8e0df41a8c 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java @@ -304,10 +304,7 @@ private void setColumnWidthsAndUntrack(SXSSFSheet sheet) { // Type specific cell writers private static void writeStringCell(ResultInfo info, Cell cell, Object value, Map styles) { - cell.setCellValue( - info.printNullable( - value - )); + cell.setCellValue((String) info.printNullable(value)); } /** @@ -319,7 +316,7 @@ private static void writeBooleanCell(ResultInfo info, Cell cell, Object value, M cell.setCellValue(aBoolean); return; } - cell.setCellValue(info.printNullable(value)); + cell.setCellValue((String) info.printNullable(value)); } private static void writeDateCell(ResultInfo info, PrintSettings settings, Cell cell, Object value, Map styles) { diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java index 89194e75c8..9141098eab 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java @@ -15,6 +15,7 @@ import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; +import com.bakdata.conquery.models.query.resultinfo.printers.CsvResultPrinters; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.resources.ResourceConstants; @@ -52,7 +53,7 @@ public Response createResult(Su final IdPrinter idPrinter = IdColumnUtil.getIdPrinter(subject, exec, namespace, conqueryConfig.getIdColumns().getIds()); final Locale locale = I18n.LOCALE.get(); - final PrintSettings settings = new PrintSettings(pretty, locale, namespace, conqueryConfig, idPrinter::createId, null); + final PrintSettings settings = new PrintSettings(pretty, locale, namespace, conqueryConfig, idPrinter::createId, null, new CsvResultPrinters()); final ExcelRenderer excelRenderer = new ExcelRenderer(excelConfig, settings); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java index 4208f5a772..a84aa9c6be 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java @@ -5,15 +5,14 @@ import java.util.Map; import com.bakdata.conquery.io.result.arrow.ArrowUtil; -import com.bakdata.conquery.models.common.CDate; +import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; -import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.MultilineEntityResult; import com.bakdata.conquery.models.types.ResultType; -import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.arrow.vector.types.pojo.Schema; @@ -40,14 +39,75 @@ public class EntityResultWriteSupport extends WriteSupport { private final PrintSettings printSettings; private MessageType schema; - private List columnConsumer; + private List columnConsumers; + private List columnPrinters; private RecordConsumer recordConsumer; + /** + * Generates the parquet schema format from the {@link ResultInfo}s of a query + * + * @param idHeaders {@link ResultInfo} for the Ids + * @param resultValueInfos {@link ResultInfo} for the result values + * @param uniqueNamer A column namer for the fields in the schema + * @return the parquet schema + */ + public static MessageType generateSchema(List idHeaders, List resultValueInfos, UniqueNamer uniqueNamer) { + /* + Because Parquet Schemas rely on primitive types with logical annotations + which are tedious to configure, we take the detour over the arrow schema. + */ + final SchemaMapping schemaMapping = new SchemaConverter().fromArrow(new Schema(ArrowUtil.generateFields(idHeaders, resultValueInfos, uniqueNamer))); + + return schemaMapping.getParquetSchema(); + + } + + private static List generateColumnConsumers(List idHeaders, List resultInfos) { + final List consumers = new ArrayList<>(); + for (ResultInfo idHeader : idHeaders) { + consumers.add(getForResultType(idHeader.getType())); + } + + for (ResultInfo resultInfo : resultInfos) { + consumers.add(getForResultType(resultInfo.getType())); + } + return consumers; + } + + private static List generateColumnPrinters(List idHeaders, List resultInfos) { + final List consumers = new ArrayList<>(); + for (ResultInfo idHeader : idHeaders) { + consumers.add(idHeader.getPrinter()); + } + + for (ResultInfo resultInfo : resultInfos) { + consumers.add(resultInfo.getPrinter()); + } + return consumers; + } + + private static ColumnConsumer getForResultType(ResultType resultType) { + + if (resultType instanceof ResultType.ListT listT) { + return new ListColumnConsumer(getForResultType(listT.getElementType())); + } + + return switch (((ResultType.Primitive) resultType)) { + case BOOLEAN -> new BooleanColumnConsumer(); + case INTEGER, DATE -> new IntegerColumnConsumer(); + case NUMERIC -> new NumericColumnConsumer(); + case MONEY -> new MoneyColumnConsumer(); + case DATE_RANGE -> new DateRangeColumnConsumer(); + case STRING -> new StringColumnConsumer(); + }; + } + @Override public WriteContext init(Configuration configuration) { schema = generateSchema(idHeaders, resultInfo, new UniqueNamer(printSettings)); - columnConsumer = generateColumnConsumers(idHeaders, resultInfo, printSettings); + columnConsumers = generateColumnConsumers(idHeaders, resultInfo); + columnPrinters = generateColumnPrinters(idHeaders, resultInfo); return new WriteContext(schema, Map.of()); } @@ -65,32 +125,49 @@ public void write(EntityResult record) { // This should not happen because of the workaround in ParquetRenderer log.warn("Processing a MultilineEntityResult is not working properly. Only the first line will be output"); } + + // Write ID fields + final String[] externalId = printSettings.getIdMapper().map(record).getExternalId(); + + final Object[] printedExternalId = new String[externalId.length]; + + for (int index = 0; index < externalId.length; index++) { + printedExternalId[index] = columnPrinters.get(index).print(externalId[index]); + } + for (Object[] listResultLine : listResultLines) { recordConsumer.startMessage(); - // Write ID fields - final String[] externalId = printSettings.getIdMapper().map(record).getExternalId(); - int cellIdx = 0; - for (int i = 0; i < externalId.length; i++, cellIdx++) { - final String subId = externalId[i]; - if (subId == null) { + + for (int index = 0; index < printedExternalId.length; index++) { + final Object printed = printedExternalId[index]; + if (printed == null) { continue; } - final String fieldName = schema.getFieldName(cellIdx); - recordConsumer.startField(fieldName, cellIdx); - columnConsumer.get(cellIdx).accept(recordConsumer, subId); - recordConsumer.endField(fieldName, cellIdx); + + final String fieldName = schema.getFieldName(index); + + recordConsumer.startField(fieldName, index); + columnConsumers.get(index).accept(recordConsumer, printed); + recordConsumer.endField(fieldName, index); } // Write Result fields - for (int i = 0; i < resultInfo.size(); i++, cellIdx++) { - final Object resultValue = listResultLine[i]; + for (int index = 0; index < listResultLine.length; index++) { + final int colId = index + printedExternalId.length; + + final Object resultValue = listResultLine[index]; + if (resultValue == null) { continue; } - final String fieldName = schema.getFieldName(cellIdx); - recordConsumer.startField(fieldName, cellIdx); - columnConsumer.get(cellIdx).accept(recordConsumer, resultValue); - recordConsumer.endField(fieldName, cellIdx); + + final Object printed = columnPrinters.get(colId).print(resultValue); + + final String fieldName = schema.getFieldName(colId); + + recordConsumer.startField(fieldName, colId); + columnConsumers.get(colId).accept(recordConsumer, printed); + recordConsumer.endField(fieldName, colId); } recordConsumer.endMessage(); @@ -98,102 +175,75 @@ public void write(EntityResult record) { } - /** - * Generates the parquet schema format from the {@link ResultInfo}s of a query - * - * @param idHeaders {@link ResultInfo} for the Ids - * @param resultValueInfos {@link ResultInfo} for the result values - * @param uniqueNamer A column namer for the fields in the schema - * @return the parquet schema - */ - public static MessageType generateSchema( - List idHeaders, - List resultValueInfos, UniqueNamer uniqueNamer) { - - /* - Because Parquet Schemas rely on primitive types with logical annotations - which are tedious to configure, we take the detour over the arrow schema. - */ - final SchemaMapping schemaMapping = new SchemaConverter().fromArrow(new Schema(ArrowUtil.generateFields(idHeaders, resultValueInfos, uniqueNamer))); - - return schemaMapping.getParquetSchema(); + private record StringColumnConsumer() implements ColumnConsumer { + @Override + public void accept(RecordConsumer recordConsumer, Object o) { + recordConsumer.addBinary(Binary.fromString((String) o)); + } } - @Data - private static class StringTColumnConsumer implements ColumnConsumer { - - private final ResultPrinters.Printer printer; - private final PrintSettings printSettings; + private record BooleanColumnConsumer() implements ColumnConsumer { @Override public void accept(RecordConsumer recordConsumer, Object o) { - final String printValue = getPrinter().print(o); - recordConsumer.addBinary(Binary.fromString(printValue)); + recordConsumer.addBoolean((boolean) o); } } - @RequiredArgsConstructor - private static class BooleanTColumnConsumer implements ColumnConsumer { + private record IntegerColumnConsumer() implements ColumnConsumer { @Override public void accept(RecordConsumer recordConsumer, Object o) { - recordConsumer.addBoolean((boolean) o); + recordConsumer.addInteger(((Number) o).intValue()); } } - @RequiredArgsConstructor - private static class IntegerTColumnConsumer implements ColumnConsumer { + + private record NumericColumnConsumer() implements ColumnConsumer { @Override public void accept(RecordConsumer recordConsumer, Object o) { - recordConsumer.addInteger((int) o); + recordConsumer.addDouble(((Number) o).doubleValue()); } } - @RequiredArgsConstructor - private static class NumericTColumnConsumer implements ColumnConsumer { + private record MoneyColumnConsumer() implements ColumnConsumer { @Override public void accept(RecordConsumer recordConsumer, Object o) { - recordConsumer.addDouble((double) o); + recordConsumer.addDouble(((Number) o).doubleValue()); } } - @RequiredArgsConstructor - private static class DateRangeTColumnConsumer implements ColumnConsumer { - private final static String MIN_FIELD_NAME = "min"; - private final static String MAX_FIELD_NAME = "max"; + + private record DateRangeColumnConsumer() implements ColumnConsumer { + private static final String MIN_FIELD_NAME = "min"; + private static final String MAX_FIELD_NAME = "max"; @Override public void accept(RecordConsumer recordConsumer, Object o) { - List dateRange = (List) o; - recordConsumer.startGroup(); - Integer min = dateRange.get(0); - + final CDateRange dateRange = (CDateRange) o; + recordConsumer.startGroup(); - if (min != null && !(CDate.isNegativeInfinity(min))) { + if (dateRange.hasLowerBound()) { recordConsumer.startField(MIN_FIELD_NAME, 0); - recordConsumer.addInteger(min); + recordConsumer.addInteger(dateRange.getMinValue()); recordConsumer.endField(MIN_FIELD_NAME, 0); } - Integer max = dateRange.get(1); - if (max != null && !(CDate.isPositiveInfinity(max))) { + if (dateRange.hasUpperBound()) { recordConsumer.startField(MAX_FIELD_NAME, 1); - recordConsumer.addInteger(max); + recordConsumer.addInteger(dateRange.getMaxValue()); recordConsumer.endField(MAX_FIELD_NAME, 1); } + recordConsumer.endGroup(); } } - @RequiredArgsConstructor - private static class ListTColumnConsumer implements ColumnConsumer { - - private final ColumnConsumer elementConsumer; - private final PrintSettings printSettings; + private record ListColumnConsumer(ColumnConsumer elementConsumer) implements ColumnConsumer { @Override public void accept(RecordConsumer recordConsumer, Object o) { @@ -207,7 +257,7 @@ public void accept(RecordConsumer recordConsumer, Object o) { return; } - // This nesting is wierd but documented https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists + // This nesting is weird but documented https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists recordConsumer.startGroup(); recordConsumer.startField("list", 0); for (Object elem : list) { @@ -221,32 +271,4 @@ public void accept(RecordConsumer recordConsumer, Object o) { recordConsumer.endGroup(); } } - - private static List generateColumnConsumers(List idHeaders, List resultInfos, PrintSettings printSettings) { - final List consumers = new ArrayList<>(); - for (ResultInfo idHeader : idHeaders) { - consumers.add(getForResultType(idHeader.getType(), idHeader.getPrinter(), printSettings)); - } - - for (ResultInfo resultInfo : resultInfos) { - consumers.add(getForResultType(resultInfo.getType(), resultInfo.getPrinter(), printSettings)); - } - return consumers; - } - - private static ColumnConsumer getForResultType(ResultType resultType, ResultPrinters.Printer printer, PrintSettings printSettings) { - - if (resultType instanceof ResultType.ListT listT) { - return new ListTColumnConsumer(getForResultType(listT.getElementType(), ((ResultPrinters.ListPrinter) printer).elementPrinter(), printSettings), printSettings); - } - - return switch (((ResultType.Primitive) resultType)) { - case BOOLEAN -> new BooleanTColumnConsumer(); - case INTEGER, DATE -> new IntegerTColumnConsumer(); - case MONEY -> new IntegerTColumnConsumer(); //TODO shift point left - case NUMERIC -> new NumericTColumnConsumer(); - case DATE_RANGE -> new DateRangeTColumnConsumer(); - case STRING -> new StringTColumnConsumer(printer, printSettings); - }; - } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ParquetRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ParquetRenderer.java index 12dcb2bbe2..1b057b0124 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ParquetRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ParquetRenderer.java @@ -75,14 +75,13 @@ public static void writeToStream( Stream results) throws IOException { // Wrap the request output stream in an output file, so the parquet writer can consume it - final OutputFile outputFile = new StreamOutputFile( - new PositionTrackingOutputStream( - new CountingOutputStream(outputStream))); - - final ConqueryParquetWriterBuilder conqueryParquetWriterBuilder = new ConqueryParquetWriterBuilder(outputFile) - .setIdHeaders(idHeaders) - .setResultInfo(resultInfo) - .setPrintSettings(printSettings); + final OutputFile outputFile = new StreamOutputFile(new PositionTrackingOutputStream(new CountingOutputStream(outputStream))); + + final ConqueryParquetWriterBuilder conqueryParquetWriterBuilder = + new ConqueryParquetWriterBuilder(outputFile) + .setIdHeaders(idHeaders) + .setResultInfo(resultInfo) + .setPrintSettings(printSettings); try (final ParquetWriter parquetWriter = conqueryParquetWriterBuilder.build()) { diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java index 6dfb0e61e0..1333d854f2 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java @@ -14,6 +14,7 @@ import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; +import com.bakdata.conquery.models.query.resultinfo.printers.ArrowResultPrinters; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.resources.ResourceConstants; @@ -52,7 +53,7 @@ public Response createResultFile(Subject subject, ManagedExecution exec, boolean final IdPrinter idPrinter = IdColumnUtil.getIdPrinter(subject, exec, namespace, config.getIdColumns().getIds()); final Locale locale = I18n.LOCALE.get(); - final PrintSettings settings = new PrintSettings(pretty, locale, namespace, config, idPrinter::createId, null); + final PrintSettings settings = new PrintSettings(pretty, locale, namespace, config, idPrinter::createId, null, new ArrowResultPrinters()); //TODO Parquet different printer? final StreamingOutput out = output -> { diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java index 633ffbc9f4..b7b08ef279 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java @@ -16,6 +16,7 @@ import com.bakdata.conquery.models.i18n.I18n; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; +import com.bakdata.conquery.models.query.resultinfo.printers.CsvResultPrinters; import com.bakdata.conquery.resources.api.ResultExcelResource; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.jersey.DropwizardResourceConfig; @@ -74,7 +75,7 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuil } - final PrintSettings printSettings = new PrintSettings(true, I18n.LOCALE.get(), exec.getNamespace(), exec.getConfig(), null, null); + final PrintSettings printSettings = new PrintSettings(true, I18n.LOCALE.get(), exec.getNamespace(), exec.getConfig(), null, null, new CsvResultPrinters()); //TODO excel different printer? // Save id column count to later check if xlsx dimensions are feasible idColumnsCount = exec.getConfig().getIdColumns().getIdResultInfos(printSettings).size(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java index ebac0b4a46..53fd1e9b13 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java @@ -15,7 +15,6 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.FixedLabelResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -136,7 +135,7 @@ public List getIdResultInfos(PrintSettings printSettings) { ), col.getField()); //TODO we can now hook our anonymizers into this - return new FixedLabelResultInfo(label, label, ResultType.Primitive.STRING, Set.of(new SemanticType.IdT(col.getName())), printSettings, ResultPrinters.printerFor(ResultType.Primitive.STRING, printSettings)); + return new FixedLabelResultInfo(label, label, ResultType.Primitive.STRING, Set.of(new SemanticType.IdT(col.getName())), printSettings); }).collect(Collectors.toUnmodifiableList()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java index c7c5b026d7..4789347738 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java @@ -18,7 +18,8 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.fasterxml.jackson.annotation.JsonBackReference; @@ -128,7 +129,7 @@ public boolean isEventDateSelect() { return false; } - public ResultPrinters.Printer createPrinter(PrintSettings printSettings) { - return ResultPrinters.printerFor(getResultType(), printSettings); + public Printer createPrinter(PrintSettings printSettings, PrinterFactory printerFactory) { + return printerFactory.printerFor(getResultType(), printSettings); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java index 47203e23a1..fbcb82c6be 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java @@ -13,7 +13,9 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.ConceptElementsAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.ConceptValuesAggregator; import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.ConceptIdPrinter; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import com.bakdata.conquery.sql.conversion.model.select.ConceptColumnSelectConverter; @@ -44,21 +46,21 @@ public Aggregator createAggregator() { } @Override - public ResultPrinters.Printer createPrinter(PrintSettings printSettings) { + public Printer createPrinter(PrintSettings printSettings, PrinterFactory printerFactory) { if (isAsIds()) { - return new ResultPrinters.ListPrinter(new ResultPrinters.ConceptIdPrinter(getHolder().findConcept(), printSettings), printSettings); + return printerFactory.getListPrinter(new ConceptIdPrinter(getHolder().findConcept(), printSettings), printSettings); } - return new ResultPrinters.ListPrinter(new ResultPrinters.StringPrinter(), printSettings); + return printerFactory.getListPrinter(printerFactory.getStringPrinter(printSettings), printSettings); } @Override public SelectResultInfo getResultInfo(CQConcept cqConcept, PrintSettings settings) { - if (!isAsIds()) { - return new SelectResultInfo(this, cqConcept, Collections.emptySet(), settings); + if (isAsIds()) { + return new SelectResultInfo(this, cqConcept, Set.of(new SemanticType.ConceptColumnT(cqConcept.getConcept())), settings); } + return new SelectResultInfo(this, cqConcept, Collections.emptySet(), settings); - return new SelectResultInfo(this, cqConcept, Set.of(new SemanticType.ConceptColumnT(cqConcept.getConcept())), settings); } @JsonIgnore diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java index 3a70cfc5cc..4bf617b31c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java @@ -9,7 +9,10 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.AllValuesAggregator; -import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.ChainingPrinter; +import com.bakdata.conquery.models.query.resultinfo.printers.MappedPrinter; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.sql.conversion.model.select.DistinctSelectConverter; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; @@ -40,11 +43,11 @@ public SelectConverter createConverter() { } @Override - public ResultPrinters.Printer createPrinter(PrintSettings printSettings) { + public Printer createPrinter(PrintSettings printSettings, PrinterFactory printerFactory) { if(getMapping() == null){ - return super.createPrinter(printSettings); + return super.createPrinter(printSettings, printerFactory); } - return new ResultPrinters.ListPrinter(new ResultPrinters.MappedPrinter(getMapping()), printSettings); + return printerFactory.getListPrinter(new ChainingPrinter(new MappedPrinter(getMapping()), printerFactory.getStringPrinter(printSettings)), printSettings); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java index 4d9ea4faa6..ce90fe1fda 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java @@ -1,6 +1,8 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector.specific; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.Set; import javax.annotation.Nullable; @@ -13,7 +15,10 @@ import com.bakdata.conquery.models.index.InternToExternMapper; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.ChainingPrinter; +import com.bakdata.conquery.models.query.resultinfo.printers.MappedPrinter; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import jakarta.validation.Valid; @@ -37,13 +42,27 @@ public MappableSingleColumnSelect(Column column, @Nullable InternToExternMapper this.mapping = mapping; } + public List print(List values) { + if(mapping == null){ + return values; + } + + final List out = new ArrayList<>(); + + for (String value : values) { + out.addAll(List.of(mapping.external(value))); + } + + return out; + } + @Override - public ResultPrinters.Printer createPrinter(PrintSettings printSettings) { + public Printer createPrinter(PrintSettings printSettings, PrinterFactory printerFactory) { if (mapping == null) { - return super.createPrinter(printSettings); + return super.createPrinter(printSettings, printerFactory); } - return new ResultPrinters.MappedPrinter(getMapping()); + return new ChainingPrinter(new MappedPrinter(getMapping()), printerFactory.getStringPrinter(printSettings)); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java index ae1887bef7..4400149f8a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java @@ -168,7 +168,7 @@ public final void initExecutable(Namespace namespace, ConqueryConfig config) { } if (label == null) { // IdMapper is not necessary here - label = makeAutoLabel(new PrintSettings(true, I18n.LOCALE.get(), namespace, config, null, null)); + label = makeAutoLabel(new PrintSettings(true, I18n.LOCALE.get(), namespace, config, null, null, null)); } this.namespace = namespace; diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java index 44247be5b8..ff35c34423 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java @@ -102,7 +102,7 @@ private Map createSubExecutions() { @Override public void start() { synchronized (this) { - subQueries.values().stream().forEach(flatSubQueries::add); + subQueries.values().forEach(flatSubQueries::add); } flatSubQueries.values().forEach(ManagedQuery::start); super.start(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java index ba40879c5e..3f5a844a5d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java @@ -51,8 +51,6 @@ public class ManagedQuery extends ManagedExecution implements SingleTableResult, */ private Long lastResultCount; - @JsonIgnore - private transient List columnDescriptions; protected ManagedQuery(@JacksonInject(useInput = OptBoolean.FALSE) MetaStorage storage) { @@ -115,10 +113,7 @@ public void setStatusBase(@NonNull Subject subject, @NonNull ExecutionStatus sta } protected void setAdditionalFieldsForStatusWithColumnDescription(Subject subject, FullExecutionStatus status) { - if (columnDescriptions == null) { - columnDescriptions = generateColumnDescriptions(isInitialized(), getConfig()); - } - status.setColumnDescriptions(columnDescriptions); + status.setColumnDescriptions(generateColumnDescriptions(isInitialized(), getConfig())); } @JsonIgnore diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java b/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java index 6379049aee..5fc4f5f2f6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java @@ -12,6 +12,7 @@ import com.bakdata.conquery.models.config.LocaleConfig; import com.bakdata.conquery.models.identifiable.mapping.PrintIdMapper; import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; +import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.worker.Namespace; import lombok.Getter; import lombok.ToString; @@ -58,11 +59,13 @@ public class PrintSettings { private final PrintIdMapper idMapper; - public PrintSettings(boolean prettyPrint, Locale locale, Namespace namespace, ConqueryConfig config, PrintIdMapper idMapper, Function columnNamer) { - this(prettyPrint, locale, namespace, config, idMapper, columnNamer, DECIMAL_FORMAT.apply(locale), NUMBER_FORMAT.apply(locale)); + private final PrinterFactory printerFactory; + + public PrintSettings(boolean prettyPrint, Locale locale, Namespace namespace, ConqueryConfig config, PrintIdMapper idMapper, Function columnNamer, PrinterFactory printerFactory) { + this(prettyPrint, locale, namespace, config, idMapper, columnNamer, DECIMAL_FORMAT.apply(locale), NUMBER_FORMAT.apply(locale), printerFactory); } - public PrintSettings(boolean prettyPrint, Locale locale, Namespace namespace, ConqueryConfig config, PrintIdMapper idMapper, Function columnNamer, NumberFormat decimalFormat, NumberFormat numberFormat) { + public PrintSettings(boolean prettyPrint, Locale locale, Namespace namespace, ConqueryConfig config, PrintIdMapper idMapper, Function columnNamer, NumberFormat decimalFormat, NumberFormat numberFormat, PrinterFactory printerFactory) { this.prettyPrint = prettyPrint; this.locale = locale; this.namespace = namespace; @@ -72,6 +75,7 @@ public PrintSettings(boolean prettyPrint, Locale locale, Namespace namespace, Co this.integerFormat = numberFormat; this.decimalFormat = decimalFormat; + this.printerFactory = printerFactory; this.listFormat = prettyPrint ? config.getLocale().getListFormats().get(0) : UNPRETTY_LIST_FORMAT; this.dateRangeSeparator = prettyPrint ? config.getLocale().findDateRangeSeparator(locale) : UNPRETTY_DATERANGE_SEPERATOR; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/SingleTableResult.java b/backend/src/main/java/com/bakdata/conquery/models/query/SingleTableResult.java index c647021640..0ae6fd61ef 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/SingleTableResult.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/SingleTableResult.java @@ -10,6 +10,7 @@ import com.bakdata.conquery.models.i18n.I18n; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; +import com.bakdata.conquery.models.query.resultinfo.printers.JavaResultPrinters; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.worker.Namespace; @@ -20,13 +21,13 @@ public interface SingleTableResult { default List generateColumnDescriptions(boolean isInitialized, ConqueryConfig config) { Preconditions.checkArgument(isInitialized, "The execution must have been initialized first"); - List columnDescriptions = new ArrayList<>(); + final List columnDescriptions = new ArrayList<>(); final Locale locale = I18n.LOCALE.get(); + // The printer is never used to generate results. But downstream code might touch them + final PrintSettings settings = new PrintSettings(true, locale, getNamespace(), config, null, null, new JavaResultPrinters()); - PrintSettings settings = new PrintSettings(true, locale, getNamespace(), config, null, null); - - UniqueNamer uniqNamer = new UniqueNamer(settings); + final UniqueNamer uniqNamer = new UniqueNamer(settings); // First add the id columns to the descriptor list. The are the first columns for (ResultInfo header : config.getIdColumns().getIdResultInfos(settings)) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java index 7ddfa024c8..01e4acbcf6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.models.query.preview; -import java.math.BigDecimal; import java.time.LocalDate; import java.util.ArrayList; import java.util.Collections; @@ -37,18 +36,14 @@ import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.JsonResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.MultilineEntityResult; -import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.OptBoolean; -import com.fasterxml.jackson.databind.node.BooleanNode; -import com.fasterxml.jackson.databind.node.DecimalNode; -import com.fasterxml.jackson.databind.node.IntNode; -import com.fasterxml.jackson.databind.node.TextNode; import com.google.common.collect.MoreCollectors; import lombok.ToString; import org.jetbrains.annotations.NotNull; @@ -132,8 +127,7 @@ private static Function> createLineToMapTransforme final int size = resultInfos.size(); final String[] columnNames = new String[size]; - - //TODO pull renderValue logic into outer loop, only use array as lookup + final Printer[] printers = new Printer[size]; for (int index = 0; index < size; index++) { final ResultInfo resultInfo = resultInfos.get(index); @@ -141,6 +135,8 @@ private static Function> createLineToMapTransforme if (resultInfo instanceof SelectResultInfo selectResultInfo) { columnNames[index] = select2desc.get(selectResultInfo.getSelect().getId()).label(); } + + printers[index] = resultInfo.getPrinter(); } return line -> { @@ -153,13 +149,12 @@ private static Function> createLineToMapTransforme continue; } - - final Object value = renderValue(line[column], resultInfos.get(column).getType(), printSettings); - - if (value == null) { + if (line[column] == null) { continue; } + final Object value = printers[column].print(line[column]); + out.put(columnName, value); } @@ -167,27 +162,7 @@ private static Function> createLineToMapTransforme }; } - /** - * Instead of outputting only String values, render to Json equivalents - */ - private static Object renderValue(Object value, ResultType type, PrintSettings printSettings) { - if (value == null) { - return null; - } - - if (type instanceof ResultType.ListT listT) { - return ((List) value).stream().map(entry -> renderValue(entry, listT.getElementType(), printSettings)).collect(Collectors.toList()); - } - return switch (((ResultType.Primitive) type)) { - case BOOLEAN -> BooleanNode.valueOf((Boolean) value); - case INTEGER -> new IntNode((Integer) value); - case NUMERIC,MONEY -> DecimalNode.valueOf((BigDecimal) value); - case DATE -> new TextNode(new ResultPrinters.DatePrinter(printSettings).print(value)); //TODO bind printers in outer loop - case DATE_RANGE -> new TextNode(new ResultPrinters.DateRangePrinter(printSettings).print(value)); //TODO bind printers in outer loop - case STRING -> new TextNode(value.toString()); //TODO mapping - }; - } /** * For the selects in result infos, build ColumnDescriptors using definitions (label and description) from PreviewConfig. @@ -243,9 +218,9 @@ public FullExecutionStatus buildStatusFull(Subject subject) { - status.setInfos(transformQueryResultToInfos(getInfoCardExecution(), new PrintSettings(true, I18n.LOCALE.get(), getNamespace(), getConfig(), null, previewConfig::resolveSelectLabel))); + status.setInfos(transformQueryResultToInfos(getInfoCardExecution(), new PrintSettings(true, I18n.LOCALE.get(), getNamespace(), getConfig(), null, previewConfig::resolveSelectLabel, new JsonResultPrinters()))); - status.setTimeStratifiedInfos(toChronoInfos(previewConfig, getSubQueries(), new PrintSettings(false, I18n.LOCALE.get(), getNamespace(), getConfig(), null, previewConfig::resolveSelectLabel))); + status.setTimeStratifiedInfos(toChronoInfos(previewConfig, getSubQueries(), new PrintSettings(false, I18n.LOCALE.get(), getNamespace(), getConfig(), null, previewConfig::resolveSelectLabel, new JsonResultPrinters()))); return status; } @@ -267,8 +242,7 @@ private List transformQueryResultToInfos(ManagedQuery for (int index = AbsoluteFormQuery.FEATURES_OFFSET; index < infoCardExecution.getResultInfos(printSettings).size(); index++) { final ResultInfo resultInfo = infoCardExecution.getResultInfos(printSettings).get(index); - - final Object printed = renderValue(values[index], resultInfo.getType(), printSettings); + final Object printed = resultInfo.printNullable(values[index]); extraInfos.add(new EntityPreviewStatus.Info( resultInfo.userColumnName(), diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java index b1d8abc4f5..9386f830a1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java @@ -4,7 +4,7 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.query.PrintSettings; -import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import lombok.EqualsAndHashCode; @@ -19,10 +19,10 @@ public class ColumnResultInfo extends ResultInfo { private final Column column; private final ResultType type; private final String description; - private final ResultPrinters.Printer printer; + private final Printer printer; - public ColumnResultInfo(Column column, ResultType type, Set semantics, ResultPrinters.Printer printer, String description, PrintSettings settings) { + public ColumnResultInfo(Column column, ResultType type, Set semantics, Printer printer, String description, PrintSettings settings) { super(semantics, settings); this.column = column; this.type = type; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ExternalResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ExternalResultInfo.java index 984840fd5c..b97babbab7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ExternalResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ExternalResultInfo.java @@ -1,12 +1,10 @@ package com.bakdata.conquery.models.query.resultinfo; import java.util.Collections; -import java.util.Set; import com.bakdata.conquery.models.query.PrintSettings; -import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.models.types.SemanticType; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -17,20 +15,16 @@ public class ExternalResultInfo extends ResultInfo { private final String name; private final ResultType type; private final String description; - private final ResultPrinters.Printer printer; + private final Printer printer; public ExternalResultInfo(String name, ResultType type, PrintSettings settings) { - this(name, type, null, ResultPrinters.printerFor(type, settings), Collections.emptySet(), settings); - } - public ExternalResultInfo(String name, ResultType type, String description, ResultPrinters.Printer printer, Set semantics, PrintSettings settings) { - super(semantics, settings); + super(Collections.emptySet(), settings); this.name = name; this.type = type; - this.description = description; - this.printer = printer; + this.description = null; + this.printer = settings.getPrinterFactory().printerFor(type, settings); } - @Override public String userColumnName() { return null; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/FixedLabelResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/FixedLabelResultInfo.java index 8a4b898115..244f6bcb65 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/FixedLabelResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/FixedLabelResultInfo.java @@ -4,7 +4,7 @@ import c10n.C10N; import com.bakdata.conquery.models.query.PrintSettings; -import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import lombok.EqualsAndHashCode; @@ -43,15 +43,16 @@ public class FixedLabelResultInfo extends ResultInfo { @Getter private final ResultType type; - @Getter - private final ResultPrinters.Printer printer; + @Override + public Printer getPrinter() { + return getSettings().getPrinterFactory().printerFor(getType(), getSettings()); + } - public FixedLabelResultInfo(String label, String defaultLabel, ResultType type, Set semantics, PrintSettings settings, ResultPrinters.Printer printer) { + public FixedLabelResultInfo(String label, String defaultLabel, ResultType type, Set semantics, PrintSettings settings) { super(semantics, settings); this.localizedLabel = label; this.localizedDefaultLabel = defaultLabel; this.type = type; - this.printer = printer; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java index 9122d20a92..584174470f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.models.query.ColumnDescriptor; import com.bakdata.conquery.models.query.PrintSettings; -import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import com.google.common.collect.ImmutableSet; @@ -64,17 +64,13 @@ public Set getSemantics() { public abstract String getDescription(); - public final String printNullable(Object f) { + public final Object printNullable(Object f) { if (f == null) { return ""; } - return print(f); - } - - protected String print(Object f) { return getPrinter().print(f); } - public abstract ResultPrinters.Printer getPrinter(); + public abstract Printer getPrinter(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java index b8873c64d4..4b24d379d5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java @@ -5,7 +5,7 @@ import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.query.PrintSettings; -import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import com.google.common.collect.Sets; @@ -34,8 +34,8 @@ public String getDescription() { } @Override - public ResultPrinters.Printer getPrinter() { - return select.createPrinter(getSettings()); + public Printer getPrinter() { + return select.createPrinter(getSettings(), getSettings().getPrinterFactory()); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java new file mode 100644 index 0000000000..496f24ed26 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java @@ -0,0 +1,12 @@ +package com.bakdata.conquery.models.query.resultinfo.printers; + +import com.bakdata.conquery.models.query.PrintSettings; + +public class ArrowResultPrinters extends JavaResultPrinters{ + + @Override + public Printer getDatePrinter(PrintSettings printSettings) { + return getIntegerPrinter(printSettings); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ChainingPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ChainingPrinter.java new file mode 100644 index 0000000000..998f289e07 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ChainingPrinter.java @@ -0,0 +1,9 @@ +package com.bakdata.conquery.models.query.resultinfo.printers; + +public record ChainingPrinter(Printer incoming, Printer outgoing) implements Printer { + + @Override + public Object print(Object value) { + return outgoing.print(incoming.print(value)); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ConceptIdPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ConceptIdPrinter.java new file mode 100644 index 0000000000..4c00a04ad9 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ConceptIdPrinter.java @@ -0,0 +1,30 @@ +package com.bakdata.conquery.models.query.resultinfo.printers; + +import com.bakdata.conquery.models.datasets.concepts.Concept; +import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeNode; +import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; +import com.bakdata.conquery.models.query.PrintSettings; + +public record ConceptIdPrinter(Concept concept, PrintSettings cfg) implements Printer { + + @Override + public String print(Object rawValue) { + if (rawValue == null) { + return null; + } + + final int localId = (int) rawValue; + + final ConceptTreeNode node = ((TreeConcept) concept).getElementByLocalId(localId); + + if (!cfg.isPrettyPrint()) { + return node.getId().toString(); + } + + if (node.getDescription() == null) { + return node.getLabel(); + } + + return node.getLabel() + " - " + node.getDescription(); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/CsvResultPrinters.java similarity index 52% rename from backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java rename to backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/CsvResultPrinters.java index 888339188e..ba00a06ccc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/CsvResultPrinters.java @@ -6,52 +6,65 @@ import com.bakdata.conquery.internationalization.Results; import com.bakdata.conquery.models.common.CDate; -import com.bakdata.conquery.models.common.LocalizedToString; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.config.LocaleConfig; -import com.bakdata.conquery.models.datasets.concepts.Concept; -import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeNode; -import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; -import com.bakdata.conquery.models.index.InternToExternMapper; import com.bakdata.conquery.models.query.C10nCache; import com.bakdata.conquery.models.query.PrintSettings; -import com.bakdata.conquery.models.types.ResultType; import com.google.common.base.Preconditions; -import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; -@UtilityClass @Slf4j -public class ResultPrinters { +public class CsvResultPrinters extends PrinterFactory { - public Printer printerFor(ResultType type, PrintSettings printSettings) { - if (type instanceof ResultType.ListT listT) { - return new ListPrinter(printerFor(listT.getElementType(), printSettings), printSettings); - } - return switch (((ResultType.Primitive) type)) { - case BOOLEAN -> new BooleanPrinter(printSettings); - case INTEGER -> new IntegerPrinter(printSettings); - case NUMERIC -> new NumericPrinter(printSettings); - case DATE -> new DatePrinter(printSettings); - case DATE_RANGE -> new DateRangePrinter(printSettings); - case STRING -> new StringPrinter(); - case MONEY -> new MoneyPrinter(printSettings); - }; + @Override + public Printer getListPrinter(Printer elementPrinter, PrintSettings printSettings) { + return new ListPrinter(elementPrinter, printSettings); + } + + @Override + public Printer getBooleanPrinter(PrintSettings printSettings) { + return new BooleanPrinter(printSettings); + } + + @Override + public Printer getIntegerPrinter(PrintSettings printSettings) { + return new IntegerPrinter(printSettings); + } + + @Override + public Printer getNumericPrinter(PrintSettings printSettings) { + return new NumericPrinter(printSettings); + } + + @Override + public Printer getDatePrinter(PrintSettings printSettings) { + return new DatePrinter(printSettings); + } + + @Override + public Printer getDateRangePrinter(PrintSettings printSettings) { + return new DateRangePrinter(printSettings); + } + + @Override + public Printer getStringPrinter(PrintSettings printSettings) { + return new StringPrinter(); } - public interface Printer { - String print(Object f); + @Override + public Printer getMoneyPrinter(PrintSettings printSettings) { + return new MoneyPrinter(printSettings); } - public record StringPrinter() implements Printer { + private record StringPrinter() implements Printer { @Override public String print(Object f) { return Objects.toString(f); } } - public record IntegerPrinter(PrintSettings cfg) implements Printer { + private record IntegerPrinter(PrintSettings cfg) implements Printer { @Override public String print(Object f) { @@ -63,7 +76,7 @@ public String print(Object f) { } } - public record NumericPrinter(PrintSettings cfg) implements Printer { + private record NumericPrinter(PrintSettings cfg) implements Printer { @Override public String print(Object f) { @@ -75,7 +88,7 @@ public String print(Object f) { } } - public record MoneyPrinter(PrintSettings cfg) implements Printer { + private record MoneyPrinter(PrintSettings cfg) implements Printer { @Override public String print(Object f) { @@ -88,7 +101,7 @@ public String print(Object f) { } } - public record DatePrinter(PrintSettings cfg) implements Printer { + private record DatePrinter(PrintSettings cfg) implements Printer { @Override public String print(Object f) { @@ -99,7 +112,7 @@ public String print(Object f) { } } - public record DateRangePrinter(DatePrinter datePrinter, PrintSettings cfg) implements Printer { + private record DateRangePrinter(DatePrinter datePrinter, PrintSettings cfg) implements Printer { public DateRangePrinter(PrintSettings printSettings) { this(new DatePrinter(printSettings), printSettings); @@ -130,14 +143,12 @@ public String print(Object f) { } } - public record BooleanPrinter(PrintSettings cfg, String trueVal, String falseVal) implements Printer { + private record BooleanPrinter(PrintSettings cfg, String trueVal, String falseVal) implements Printer { public BooleanPrinter(PrintSettings cfg) { - this( - cfg, - cfg.isPrettyPrint() ? C10nCache.getLocalized(Results.class, cfg.getLocale()).True() : "1", - cfg.isPrettyPrint() ? C10nCache.getLocalized(Results.class, cfg.getLocale()).False() : "0" - ); + this(cfg, cfg.isPrettyPrint() ? C10nCache.getLocalized(Results.class, cfg.getLocale()).True() : "1", cfg.isPrettyPrint() + ? C10nCache.getLocalized(Results.class, cfg.getLocale()).False() + : "0"); } @Override @@ -150,39 +161,7 @@ public String print(Object f) { } } - public record MappedPrinter(InternToExternMapper mapper) implements Printer { - - @Override - public String print(Object f) { - return mapper.external(((String) f)); - } - } - - public record ConceptIdPrinter(Concept concept, PrintSettings cfg) implements Printer { - - @Override - public String print(Object rawValue) { - if (rawValue == null) { - return null; - } - - final int localId = (int) rawValue; - - final ConceptTreeNode node = ((TreeConcept) concept).getElementByLocalId(localId); - - if (!cfg.isPrettyPrint()) { - return node.getId().toString(); - } - - if (node.getDescription() == null) { - return node.getLabel(); - } - - return node.getLabel() + " - " + node.getDescription(); - } - } - - public record ListPrinter(Printer elementPrinter, PrintSettings cfg, LocaleConfig.ListFormat listFormat) implements Printer { + private record ListPrinter(Printer elementPrinter, PrintSettings cfg, LocaleConfig.ListFormat listFormat) implements Printer { public ListPrinter(Printer elementPrinter, PrintSettings cfg) { this(elementPrinter, cfg, cfg.getListFormat()); @@ -197,25 +176,10 @@ public String print(Object f) { final StringJoiner joiner = listFormat.createListJoiner(); for (Object obj : (List) f) { - joiner.add(listFormat.escapeListElement(elementPrinter.print(obj))); + joiner.add(listFormat.escapeListElement(elementPrinter.print(obj).toString())); } return joiner.toString(); } } - public record LocalizedEnumPrinter & LocalizedToString>(PrintSettings cfg, Class clazz) implements Printer { - @Override - public String print(Object f) { - - if (clazz.isInstance(f)) { - return clazz.cast(f).toString(cfg.getLocale()); - } - try { - return Enum.valueOf(clazz, f.toString()).toString(cfg.getLocale()); - } - catch (Exception e) { - throw new IllegalArgumentException("%s is not a valid %s.".formatted(f, clazz), e); - } - } - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java new file mode 100644 index 0000000000..8851b1a912 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java @@ -0,0 +1,101 @@ +package com.bakdata.conquery.models.query.resultinfo.printers; + +import java.util.ArrayList; +import java.util.List; + +import com.bakdata.conquery.models.common.CDate; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.query.PrintSettings; +import com.google.common.base.Preconditions; + +public class JavaResultPrinters extends PrinterFactory { + + @Override + public Printer getListPrinter(Printer elementPrinter, PrintSettings printSettings) { + return new ListPrinter(elementPrinter); + } + + @Override + public Printer getBooleanPrinter(PrintSettings printSettings) { + return new IdentityPrinter(); + } + + @Override + public Printer getIntegerPrinter(PrintSettings printSettings) { + return new IdentityPrinter(); + } + + @Override + public Printer getNumericPrinter(PrintSettings printSettings) { + return new IdentityPrinter(); + } + + @Override + public Printer getDatePrinter(PrintSettings printSettings) { + return new DatePrinter(); + } + + @Override + public Printer getDateRangePrinter(PrintSettings printSettings) { + return new DateRangePrinter(); + } + + @Override + public Printer getStringPrinter(PrintSettings printSettings) { + return new IdentityPrinter(); + } + + + @Override + public Printer getMoneyPrinter(PrintSettings printSettings) { + return new IdentityPrinter(); + } + + private record ListPrinter(Printer elementPrinter) implements Printer { + + @Override + public Object print(Object value) { + final List inList = (List) value; + final List out = new ArrayList<>(inList.size()); + + for (Object elt : inList) { + out.add(elementPrinter.print(elt)); + } + + return out; + } + } + + private record IdentityPrinter() implements Printer { + + @Override + public Object print(Object value) { + return value; + } + } + + private record DatePrinter() implements Printer { + + @Override + public Object print(Object value) { + return CDate.toLocalDate(((Number) value).intValue()); + } + } + + private record DateRangePrinter() implements Printer { + + + @Override + public Object print(Object f) { + Preconditions.checkArgument(f instanceof List, "Expected a List got %s (Type: %s, as string: %s)", f, f.getClass().getName(), f); + Preconditions.checkArgument(((List) f).size() == 2, "Expected a list with 2 elements, one min, one max. The list was: %s ", f); + + final List list = (List) f; + final Integer min = (Integer) list.get(0); + final Integer max = (Integer) list.get(1); + + + return CDateRange.of(min, max); + } + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java new file mode 100644 index 0000000000..b495f5cc44 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java @@ -0,0 +1,95 @@ +package com.bakdata.conquery.models.query.resultinfo.printers; + +import java.math.BigDecimal; +import java.util.Objects; + +import com.bakdata.conquery.models.query.PrintSettings; +import com.fasterxml.jackson.databind.node.BooleanNode; +import com.fasterxml.jackson.databind.node.DecimalNode; +import com.fasterxml.jackson.databind.node.IntNode; +import com.fasterxml.jackson.databind.node.TextNode; + +public class JsonResultPrinters extends PrinterFactory { + + private final JavaResultPrinters delegate = new JavaResultPrinters(); + + @Override + public Printer getListPrinter(Printer elementPrinter, PrintSettings printSettings) { + return delegate.getListPrinter(elementPrinter, printSettings); + } + + public record BooleanPrinter() implements Printer { + + @Override + public Object print(Object value) { + return BooleanNode.valueOf(((Boolean) value)); + } + } + + @Override + public Printer getBooleanPrinter(PrintSettings printSettings) { + return new BooleanPrinter(); + } + + public record IntegerPrinter() implements Printer { + + @Override + public Object print(Object value) { + return IntNode.valueOf(((Integer) value)); + } + } + + @Override + public Printer getIntegerPrinter(PrintSettings printSettings) { + return new IntegerPrinter(); + } + + public record NumericPrinter() implements Printer { + + @Override + public Object print(Object value) { + return DecimalNode.valueOf(((BigDecimal) value)); + } + } + + @Override + public Printer getNumericPrinter(PrintSettings printSettings) { + return new NumericPrinter(); + } + + public record ToStringPrinter(Printer delegate) implements Printer { + + @Override + public Object print(Object value) { + return new TextNode(Objects.toString(delegate.print(value))); + } + } + + @Override + public Printer getDatePrinter(PrintSettings printSettings) { + //TODO compare with current impl + return new ToStringPrinter(delegate.getDatePrinter(printSettings)); + } + + @Override + public Printer getDateRangePrinter(PrintSettings printSettings) { + return new ToStringPrinter(delegate.getDateRangePrinter(printSettings)); + } + + public record StringPrinter() implements Printer { + + @Override + public Object print(Object value) { + return new TextNode((String) value); + } + } + @Override + public Printer getStringPrinter(PrintSettings printSettings) { + return new StringPrinter(); + } + + @Override + public Printer getMoneyPrinter(PrintSettings printSettings) { + return delegate.getMoneyPrinter(printSettings); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/LocalizedEnumPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/LocalizedEnumPrinter.java new file mode 100644 index 0000000000..aad7a38000 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/LocalizedEnumPrinter.java @@ -0,0 +1,20 @@ +package com.bakdata.conquery.models.query.resultinfo.printers; + +import com.bakdata.conquery.models.common.LocalizedToString; +import com.bakdata.conquery.models.query.PrintSettings; + +public record LocalizedEnumPrinter & LocalizedToString>(PrintSettings cfg, Class clazz) implements Printer { + @Override + public String print(Object f) { + + if (clazz.isInstance(f)) { + return clazz.cast(f).toString(cfg.getLocale()); + } + try { + return Enum.valueOf(clazz, f.toString()).toString(cfg.getLocale()); + } + catch (Exception e) { + throw new IllegalArgumentException("%s is not a valid %s.".formatted(f, clazz), e); + } + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/MappedPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/MappedPrinter.java new file mode 100644 index 0000000000..7f10733148 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/MappedPrinter.java @@ -0,0 +1,11 @@ +package com.bakdata.conquery.models.query.resultinfo.printers; + +import com.bakdata.conquery.models.index.InternToExternMapper; + +public record MappedPrinter(InternToExternMapper mapper) implements Printer { + + @Override + public String print(Object f) { + return mapper.external(((String) f)); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java new file mode 100644 index 0000000000..be61563ed7 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java @@ -0,0 +1,6 @@ +package com.bakdata.conquery.models.query.resultinfo.printers; + +@FunctionalInterface +public interface Printer { + Object print(Object value); +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java new file mode 100644 index 0000000000..8de0feb47b --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java @@ -0,0 +1,39 @@ +package com.bakdata.conquery.models.query.resultinfo.printers; + +import com.bakdata.conquery.models.query.PrintSettings; +import com.bakdata.conquery.models.types.ResultType; + +public abstract class PrinterFactory { + public final Printer printerFor(ResultType type, PrintSettings printSettings) { + if (type instanceof ResultType.ListT listT) { + final Printer elementPrinter = printerFor(listT.getElementType(), printSettings); + return getListPrinter(elementPrinter, printSettings); + } + + return switch (((ResultType.Primitive) type)) { + case BOOLEAN -> getBooleanPrinter(printSettings); + case INTEGER -> getIntegerPrinter(printSettings); + case NUMERIC -> getNumericPrinter(printSettings); + case DATE -> getDatePrinter(printSettings); + case DATE_RANGE -> getDateRangePrinter(printSettings); + case STRING -> getStringPrinter(printSettings); + case MONEY -> getMoneyPrinter(printSettings); + }; + } + + public abstract Printer getListPrinter(Printer elementPrinter, PrintSettings printSettings); + + public abstract Printer getBooleanPrinter(PrintSettings printSettings); + + public abstract Printer getIntegerPrinter(PrintSettings printSettings); + + public abstract Printer getNumericPrinter(PrintSettings printSettings); + + public abstract Printer getDatePrinter(PrintSettings printSettings); + + public abstract Printer getDateRangePrinter(PrintSettings printSettings); + + public abstract Printer getStringPrinter(PrintSettings printSettings); + + public abstract Printer getMoneyPrinter(PrintSettings printSettings); +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/SecondaryIdResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/SecondaryIdResultInfo.java index 6f33bbb124..346d236569 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/SecondaryIdResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/SecondaryIdResultInfo.java @@ -15,16 +15,20 @@ public class SecondaryIdResultInfo extends ResultInfo { private final SecondaryIdDescription secondaryId; private final ResultType type; - private final ResultPrinters.Printer printer; + private final Printer printer; public SecondaryIdResultInfo(SecondaryIdDescription secondaryId, PrintSettings settings) { super(Set.of(new SemanticType.SecondaryIdT(secondaryId)), settings); this.secondaryId = secondaryId; type = ResultType.Primitive.STRING; - printer = secondaryId.getMapping() == null - ? new ResultPrinters.StringPrinter() - : new ResultPrinters.MappedPrinter(secondaryId.getMapping()); + + if (secondaryId.getMapping() == null) { + printer = settings.getPrinterFactory().getStringPrinter(settings); + } + else { + printer = new ChainingPrinter(new MappedPrinter(secondaryId.getMapping()), settings.getPrinterFactory().getStringPrinter(settings)); + } } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/BooleanColumnStatsCollector.java b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/BooleanColumnStatsCollector.java index c805131751..fd4fb70237 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/BooleanColumnStatsCollector.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/BooleanColumnStatsCollector.java @@ -4,8 +4,9 @@ import java.util.Map; import c10n.C10N; +import com.bakdata.conquery.internationalization.Results; +import com.bakdata.conquery.models.query.C10nCache; import com.bakdata.conquery.models.query.PrintSettings; -import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; import lombok.Getter; @Getter @@ -36,13 +37,13 @@ public void consume(Object value) { @Override public ResultColumnStatistics describe() { - final ResultPrinters.BooleanPrinter printer = new ResultPrinters.BooleanPrinter(getPrintSettings()); + final Results results = C10nCache.getLocalized(Results.class, getPrintSettings().getLocale()); return new HistogramColumnDescription( getName(), getLabel(), getDescription(), List.of( - new HistogramColumnDescription.Entry(printer.print(true), trues), - new HistogramColumnDescription.Entry(printer.print(false), falses) + new HistogramColumnDescription.Entry(results.True(), trues), + new HistogramColumnDescription.Entry(results.False(), falses) ), Map.of( C10N.get(StatisticsLabels.class, getPrintSettings().getLocale()).missing(), diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ColumnStatsCollector.java b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ColumnStatsCollector.java index 28ccb85a5a..689f2e1129 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ColumnStatsCollector.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ColumnStatsCollector.java @@ -5,7 +5,6 @@ import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.models.config.FrontendConfig; import com.bakdata.conquery.models.query.PrintSettings; -import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; import com.bakdata.conquery.models.types.ResultType; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -19,11 +18,12 @@ public abstract class ColumnStatsCollector { @JsonIgnore private final PrintSettings printSettings; - public static ColumnStatsCollector getStatsCollector(String name, String description, ResultType type, ResultPrinters.Printer printer, PrintSettings printSettings, FrontendConfig config) { + public static ColumnStatsCollector getStatsCollector(String name, String description, ResultType type, PrintSettings printSettings, FrontendConfig config) { // List recursion must be done before assigning uniqueNames if (type instanceof ResultType.ListT listT) { - final ColumnStatsCollector columnStatsCollector = getStatsCollector(name, description, listT.getElementType(), ((ResultPrinters.ListPrinter) printer).elementPrinter(), printSettings, config); + + final ColumnStatsCollector columnStatsCollector = getStatsCollector(name, description, listT.getElementType(), printSettings, config); return new ListColumnStatsCollector(columnStatsCollector, printSettings); } @@ -31,7 +31,7 @@ public static ColumnStatsCollector getStatsCollector(String name, String descrip case BOOLEAN -> new BooleanColumnStatsCollector(name, name, description, printSettings); case INTEGER, MONEY, NUMERIC -> new NumberColumnStatsCollector<>(name, name, description, type, printSettings, config.getVisualisationsHistogramLimit(), config.getVisualisationPercentiles().lowerEndpoint(), config.getVisualisationPercentiles().upperEndpoint()); case DATE, DATE_RANGE -> new DateColumnStatsCollector(name, name, description, type, printSettings); - case STRING -> new StringColumnStatsCollector(name, name, description, printer, printSettings, config.getVisualisationsHistogramLimit()); + case STRING -> new StringColumnStatsCollector(name, name, description, printSettings, config.getVisualisationsHistogramLimit()); }; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/DateColumnStatsCollector.java b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/DateColumnStatsCollector.java index 2b56c99740..2c661d5f3d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/DateColumnStatsCollector.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/DateColumnStatsCollector.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.models.query.statistics; import java.time.LocalDate; -import java.util.List; import java.util.SortedMap; import java.util.TreeMap; import java.util.function.Function; @@ -35,8 +34,8 @@ public DateColumnStatsCollector(String name, String label, String description, R private static Function getDateExtractor(ResultType dateType) { return switch (((ResultType.Primitive) dateType)) { - case DATE_RANGE -> dateValue -> CDateRange.fromList((List) dateValue); - case DATE -> dateValue -> CDateRange.exactly((Integer) dateValue); + case DATE_RANGE -> dateValue -> (CDateRange) dateValue; + case DATE -> dateValue -> CDateRange.exactly((LocalDate) dateValue); default -> throw new IllegalStateException("Unexpected type %s".formatted(dateType)); }; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/HistogramColumnDescription.java b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/HistogramColumnDescription.java index 708550bee6..9106859f8a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/HistogramColumnDescription.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/HistogramColumnDescription.java @@ -12,7 +12,7 @@ @ToString(callSuper = true) public class HistogramColumnDescription extends ColumnStatsCollector.ResultColumnStatistics { - public static record Entry(String label, long value) {}; + public record Entry(String label, long value) {}; private final List entries; private final Map extras; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ResultStatistics.java b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ResultStatistics.java index 891caae97e..745d46afbf 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ResultStatistics.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ResultStatistics.java @@ -18,6 +18,7 @@ import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; @@ -72,15 +73,24 @@ public static ResultStatistics collectResultStatistics(SingleTableResult managed final StopWatch started = StopWatch.createStarted(); final ResultInfo info = resultInfos.get(col); + final Printer printer = info.getPrinter(); final ColumnStatsCollector statsCollector = - ColumnStatsCollector.getStatsCollector(uniqueNamer.getUniqueName(info), info.getDescription(), info.getType(), info.getPrinter(), printSettings, conqueryConfig.getFrontend()); + ColumnStatsCollector.getStatsCollector(uniqueNamer.getUniqueName(info), info.getDescription(), info.getType(), printSettings, conqueryConfig.getFrontend()); log.trace("BEGIN stats collection for {}", info); managedQuery.streamResults(OptionalLong.empty()) .map(EntityResult::listResultLines) .flatMap(List::stream) - .forEach(line -> statsCollector.consume(line[col])); + .forEach(line -> { + Object value = line[col]; + if (value == null) { + // Printers dont handle null + statsCollector.consume(null); + return; + } + statsCollector.consume(printer.print(value)); + }); log.trace("DONE collecting values for {}, in {}", info, started); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/StringColumnStatsCollector.java b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/StringColumnStatsCollector.java index 19babdfcb5..c66d368b65 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/StringColumnStatsCollector.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/StringColumnStatsCollector.java @@ -8,7 +8,6 @@ import c10n.C10N; import com.bakdata.conquery.models.query.PrintSettings; -import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.math3.stat.Frequency; @@ -19,15 +18,13 @@ public class StringColumnStatsCollector extends ColumnStatsCollector { private final Frequency frequencies = new Frequency(); private final long limit; - private final ResultPrinters.Printer printer; private int nulls = 0; - public StringColumnStatsCollector(String name, String label, String description, ResultPrinters.Printer printer, PrintSettings printSettings, long limit) { + public StringColumnStatsCollector(String name, String label, String description, PrintSettings printSettings, long limit) { super(name, label, description, printSettings); this.limit = limit; - this.printer = printer; } @Override @@ -38,7 +35,7 @@ public void consume(Object value) { } // In case there's a mapping, we need to map the value - final String printed = printer.print(value); + final String printed = (String) value; frequencies.addValue(printed); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java index be5564974b..c0a0904ee5 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java @@ -44,7 +44,7 @@ public NodeConversions( public ConversionContext convert(QueryDescription queryDescription, Namespace namespace, ConqueryConfig conqueryConfig) { ConversionContext initialCtx = ConversionContext.builder() .idColumns(idColumns) - .sqlPrintSettings(new PrintSettings(false, Locale.ROOT, namespace, conqueryConfig, null, null)) + .sqlPrintSettings(new PrintSettings(false, Locale.ROOT, namespace, conqueryConfig, null, null, null)) .config(config) .nameGenerator(nameGenerator) .nodeConversions(this) diff --git a/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java b/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java index d6643ae4bb..a57542cbe5 100644 --- a/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java +++ b/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java @@ -6,7 +6,7 @@ import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.query.PrintSettings; -import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.CsvResultPrinters; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.sql.execution.DefaultSqlCDateSetParser; import org.junit.jupiter.api.Assertions; @@ -17,14 +17,15 @@ class DefaultSqlCDateSetParserTest { private static final DefaultSqlCDateSetParser parser = new DefaultSqlCDateSetParser(); + private static final CsvResultPrinters csvResultPrinters = new CsvResultPrinters(); private static final ConqueryConfig CONFIG = new ConqueryConfig(); - private static final PrintSettings PLAIN = new PrintSettings(false, Locale.ENGLISH, null, CONFIG, null, null); + private static final PrintSettings PLAIN = new PrintSettings(false, Locale.ENGLISH, null, CONFIG, null, null, csvResultPrinters); @ParameterizedTest @MethodSource("testToEpochDayRangeListProvider") public void testToEpochDayRangeList(String input, String expected, String message) { List> epochDayRangeList = parser.toEpochDayRangeList(input); - String actual = ResultPrinters.printerFor(new ResultType.ListT(ResultType.Primitive.DATE_RANGE), PLAIN).print(epochDayRangeList); + final String actual = (String) csvResultPrinters.printerFor(new ResultType.ListT(ResultType.Primitive.DATE_RANGE), PLAIN).print(epochDayRangeList); Assertions.assertEquals(expected, actual, message); } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java index c80ac829cf..3cbbe2e758 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java @@ -23,6 +23,7 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.models.query.resultinfo.printers.CsvResultPrinters; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.MultilineEntityResult; import com.bakdata.conquery.resources.api.ResultCsvResource; @@ -57,7 +58,7 @@ public void executeTest(StandaloneSupport standaloneSupport) throws IOException SingleTableResult executionResult = (SingleTableResult) execution; //check result info size - PrintSettings printSettings = new PrintSettings(true, Locale.ROOT, standaloneSupport.getNamespace(), standaloneSupport.getConfig(), null, null); + PrintSettings printSettings = new PrintSettings(true, Locale.ROOT, standaloneSupport.getNamespace(), standaloneSupport.getConfig(), null, null, new CsvResultPrinters()); List resultInfos = executionResult.getResultInfos(printSettings); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java index cfa9728123..7524a585d5 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java @@ -31,6 +31,7 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.models.query.resultinfo.printers.CsvResultPrinters; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.util.io.IdColumnUtil; import com.bakdata.conquery.util.support.StandaloneSupport; @@ -129,7 +130,7 @@ private void checkResults(StandaloneSupport standaloneSupport, ManagedInternalFo final ConqueryConfig config = standaloneSupport.getConfig(); PrintSettings printSettings = - new PrintSettings(false, Locale.ENGLISH, standaloneSupport.getNamespace(), config, idPrinter::createId, null); + new PrintSettings(false, Locale.ENGLISH, standaloneSupport.getNamespace(), config, idPrinter::createId, null, new CsvResultPrinters()); checkSingleResult(managedForm, config, printSettings); diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java index 22778470c9..4218f0bcd5 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java @@ -5,7 +5,6 @@ import java.util.List; import java.util.Locale; import java.util.OptionalLong; -import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -24,7 +23,7 @@ import com.bakdata.conquery.models.query.resultinfo.ExternalResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.CsvResultPrinters; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.MultilineEntityResult; import com.bakdata.conquery.models.query.results.SinglelineEntityResult; @@ -49,12 +48,16 @@ public class ResultTestUtil { private static final PrintSettings PRINT_SETTINGS = - new PrintSettings(false, Locale.ROOT, null, new ConqueryConfig(), null, (selectInfo) -> selectInfo.getSelect().getLabel()); + new PrintSettings(false, Locale.ROOT, null, new ConqueryConfig(), null, (selectInfo) -> selectInfo.getSelect().getLabel(), new CsvResultPrinters()); public static List ID_FIELDS = Stream.of("id1", "id2") - .map(name -> new ExternalResultInfo(name, ResultType.Primitive.STRING, "", new ResultPrinters.StringPrinter(), Set.of(new SemanticType.IdT("ID")), PRINT_SETTINGS)) + .map(name -> { + ExternalResultInfo info = new ExternalResultInfo(name, ResultType.Primitive.STRING, PRINT_SETTINGS); + info.addSemantics(new SemanticType.IdT("ID")); + return info; + }) .collect(Collectors.toList()); @NotNull @@ -64,7 +67,7 @@ public static ManagedQuery getTestQuery() { public List getResultInfos(PrintSettings printSettings) { return getResultTypes().stream() .map(resultType -> new TypedSelectDummy(resultType)) - .map(select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet(), PRINT_SETTINGS)) + .map(select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet(), printSettings)) .collect(Collectors.toList()); } diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java index b7640e817b..2c9a22708f 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java @@ -33,6 +33,7 @@ import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; +import com.bakdata.conquery.models.query.resultinfo.printers.ArrowResultPrinters; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.types.ResultType; import lombok.NonNull; @@ -57,7 +58,7 @@ public class ArrowResultGenerationTest { public static final ConqueryConfig CONFIG = new ConqueryConfig(); private static final PrintSettings PRINT_SETTINGS = - new PrintSettings(false, Locale.ROOT, null, CONFIG, null, (selectInfo) -> selectInfo.getSelect().getLabel()); + new PrintSettings(false, Locale.ROOT, null, CONFIG, null, (selectInfo) -> selectInfo.getSelect().getLabel(), new ArrowResultPrinters()); @Test @@ -123,12 +124,13 @@ void writeAndRead() throws IOException { // Prepare every input data PrintSettings printSettings = new PrintSettings( - false, - Locale.ROOT, - null, - CONFIG, - (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), - (selectInfo) -> selectInfo.getSelect().getLabel()); + false, + Locale.ROOT, + null, + CONFIG, + (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), + (selectInfo) -> selectInfo.getSelect().getLabel(), new ArrowResultPrinters() + ); // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized List results = getTestEntityResults(); diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java index 770bb17270..8d5cf8932f 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java @@ -19,6 +19,7 @@ import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.models.query.resultinfo.printers.CsvResultPrinters; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.util.NonPersistentStoreFactory; @@ -47,7 +48,8 @@ void writeAndRead() throws IOException { null, CONFIG, (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), - (selectInfo) -> selectInfo.getSelect().getLabel()); + (selectInfo) -> selectInfo.getSelect().getLabel(), new CsvResultPrinters() + ); // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized List results = getTestEntityResults(); @@ -89,7 +91,7 @@ private String generateExpectedCSV(List results, List continue; } ResultInfo info = resultInfos.get(lIdx); - final String printVal = info.printNullable(val); + final String printVal = (String) info.printNullable(val); valueJoiner.add(printVal.contains(String.valueOf(CONFIG.getCsv().getDelimeter()))? "\""+printVal+"\"": printVal); } diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java index b68cdcbb51..18723d7260 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java @@ -27,6 +27,7 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; +import com.bakdata.conquery.models.query.resultinfo.printers.CsvResultPrinters; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.util.NonPersistentStoreFactory; @@ -58,7 +59,7 @@ void writeAndRead() throws IOException { PrintSettings printSettings = new PrintSettings(true, Locale.GERMAN, null, CONFIG, (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), (selectInfo) -> selectInfo.getSelect() - .getLabel()); + .getLabel(), new CsvResultPrinters()); // TODO ? // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized List results = getTestEntityResults(); @@ -152,7 +153,7 @@ private List generateExpectedTSV(List results, List selectInfo.getSelect().getLabel()); + new PrintSettings(false, Locale.ROOT, null, CONFIG, null, (selectInfo) -> selectInfo.getSelect().getLabel(), new ArrowResultPrinters()); @Test @@ -110,7 +111,8 @@ void writeAndRead() throws IOException { null, CONFIG, (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), - (selectInfo) -> selectInfo.getSelect().getLabel() + (selectInfo) -> selectInfo.getSelect().getLabel(), + new ArrowResultPrinters() ); // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized List results = getTestEntityResults(); diff --git a/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java b/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java index f2b1bece25..94218f0681 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java @@ -84,7 +84,7 @@ void autoLabelConceptQuery(Locale locale, String autoLabel) { @NotNull private PrintSettings getPrintSettings(Locale locale) { - return new PrintSettings(true, locale, NAMESPACE, CONFIG, null, null); + return new PrintSettings(true, locale, NAMESPACE, CONFIG, null, null, null); } private static CQConcept makeCQConcept(String label) { diff --git a/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java b/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java index 8782a45152..b7ab8535f6 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java @@ -46,7 +46,7 @@ @Slf4j public class DefaultColumnNameTest { private static final Namespace NAMESPACE = mock(LocalNamespace.class); - private static final PrintSettings SETTINGS = new PrintSettings(false, Locale.ENGLISH, NAMESPACE, new ConqueryConfig(), null, null); + private static final PrintSettings SETTINGS = new PrintSettings(false, Locale.ENGLISH, NAMESPACE, new ConqueryConfig(), null, null, null); private static final Validator VALIDATOR = Validators.newValidator(); private static final BiFunction CONCEPT_SELECT_SELECTOR = diff --git a/backend/src/test/java/com/bakdata/conquery/models/query/UniqueNameTest.java b/backend/src/test/java/com/bakdata/conquery/models/query/UniqueNameTest.java index 441b6ef2bf..3abde84190 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/query/UniqueNameTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/query/UniqueNameTest.java @@ -7,13 +7,14 @@ import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.query.resultinfo.ExternalResultInfo; import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; +import com.bakdata.conquery.models.query.resultinfo.printers.CsvResultPrinters; import com.bakdata.conquery.models.types.ResultType; import org.junit.jupiter.api.Test; public class UniqueNameTest { @Test void testNameCollision() { - PrintSettings settings = new PrintSettings(true, Locale.ROOT, null, new ConqueryConfig(), null, null); + PrintSettings settings = new PrintSettings(true, Locale.ROOT, null, new ConqueryConfig(), null, null, new CsvResultPrinters()); final UniqueNamer uniqueNamer = new UniqueNamer(settings); final ExternalResultInfo info1 = new ExternalResultInfo("test", ResultType.Primitive.STRING, settings); diff --git a/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java b/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java index c103bfaea9..b564b20f2c 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java @@ -17,7 +17,8 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.ExternalResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.CsvResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.google.common.collect.ImmutableMap; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -26,9 +27,9 @@ public class ResultTypeTest { public static final ConqueryConfig CONFIG = new ConqueryConfig(); - private static final PrintSettings PRETTY = new PrintSettings(true, Locale.ENGLISH, null, CONFIG, null, null); - private static final PrintSettings PRETTY_DE = new PrintSettings(true, Locale.GERMANY, null, CONFIG, null, null); - private static final PrintSettings PLAIN = new PrintSettings(false, Locale.ENGLISH, null, CONFIG, null, null); + private static final PrintSettings PRETTY = new PrintSettings(true, Locale.ENGLISH, null, CONFIG, null, null, new CsvResultPrinters()); + private static final PrintSettings PRETTY_DE = new PrintSettings(true, Locale.GERMANY, null, CONFIG, null, null, new CsvResultPrinters()); + private static final PrintSettings PLAIN = new PrintSettings(false, Locale.ENGLISH, null, CONFIG, null, null, new CsvResultPrinters()); static { // Initialization of the internationalization @@ -94,7 +95,7 @@ public static ResultInfo info(ResultType type, PrintSettings settings) { public void testPrinting(PrintSettings cfg, ResultType type, Object value, String expected) throws IOException { ResultInfo info = info(type, cfg); - final ResultPrinters.Printer printer = info.getPrinter(); + final Printer printer = info.getPrinter(); assertThat(printer.print(value)).isEqualTo(expected); @@ -109,7 +110,7 @@ public void testPrinting(PrintSettings cfg, ResultType type, Object value, Strin public void testBinaryPrinting(PrintSettings cfg, ResultType type, Object value, String expected) throws IOException { ResultInfo info = info(type, cfg); - final ResultPrinters.Printer printer = info.getPrinter(); + final Printer printer = info.getPrinter(); assertThat(printer.print(value)).isEqualTo(expected); final byte[] bytes = Jackson.BINARY_MAPPER.writeValueAsBytes(value); From 39608bea1f2aac601f02358a35c43f2a08528686 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 28 Aug 2024 17:46:38 +0200 Subject: [PATCH 22/54] use printers in ExcelRenderer --- .../io/result/excel/ExcelRenderer.java | 117 ++++++++---------- .../models/config/ExcelResultProvider.java | 5 +- .../printers/ExcelResultPrinters.java | 8 ++ .../resultinfo/printers/IdentityPrinter.java | 9 ++ .../printers/JavaResultPrinters.java | 8 -- 5 files changed, 73 insertions(+), 74 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/IdentityPrinter.java diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java index 8e0df41a8c..6b2a589101 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.io.OutputStream; import java.math.BigDecimal; +import java.time.LocalDate; import java.util.List; import java.util.Map; import java.util.OptionalLong; @@ -12,7 +13,6 @@ import c10n.C10N; import com.bakdata.conquery.internationalization.ExcelSheetNameC10n; import com.bakdata.conquery.models.auth.entities.User; -import com.bakdata.conquery.models.common.CDate; import com.bakdata.conquery.models.config.ExcelConfig; import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.i18n.I18n; @@ -20,6 +20,7 @@ import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.types.ResultType; import com.google.common.collect.ImmutableMap; @@ -43,19 +44,19 @@ public class ExcelRenderer { public static final int MAX_LINES = 1_048_576; - private static TypeWriter writer(ResultType type) { + private static TypeWriter writer(ResultType type, Printer printer, PrintSettings settings1) { if(!(type instanceof ResultType.Primitive)){ //Excel cannot handle complex types so we just toString them. - return (info, settings, cell, value, styles) -> writeStringCell(info, cell, value, styles); + return (value, cell, styles) -> writeStringCell(cell, value, printer); } return switch (((ResultType.Primitive) type)) { - case BOOLEAN -> (info, settings, cell, value, styles) -> writeBooleanCell(info, cell, value, styles); - case INTEGER -> ExcelRenderer::writeIntegerCell; - case MONEY -> ExcelRenderer::writeMoneyCell; - case NUMERIC -> ExcelRenderer::writeNumericCell; - case DATE -> ExcelRenderer::writeDateCell; - default -> (info, settings, cell, value, styles) -> writeStringCell(info, cell, value, styles); + case BOOLEAN -> (value, cell, styles) -> writeBooleanCell(value, cell, printer); + case INTEGER -> (value, cell, styles) -> writeIntegerCell(value, cell, printer, styles); + case MONEY -> (value, cell, styles1) -> writeMoneyCell(value, cell, settings1, styles1); + case NUMERIC -> (value, cell, styles) -> writeNumericCell(value, cell, printer, styles); + case DATE -> (value, cell, styles1) -> writeDateCell(value, cell, styles1, printer); + default -> (value, cell, styles) -> writeStringCell(cell, value, printer); }; } @@ -77,7 +78,7 @@ public ExcelRenderer(ExcelConfig config, PrintSettings cfg) { @FunctionalInterface private interface TypeWriter { - void writeCell(ResultInfo info, PrintSettings settings, Cell cell, Object value, Map styles); + void writeCell(Object value, Cell cell, Map styles); } public void renderToStream(List idHeaders, E exec, OutputStream outputStream, OptionalLong limit, PrintSettings printSettings) @@ -86,16 +87,16 @@ public void renderToStream(List setMetaData(exec); - SXSSFSheet sheet = workbook.createSheet(C10N.get(ExcelSheetNameC10n.class, I18n.LOCALE.get()).result()); + final SXSSFSheet sheet = workbook.createSheet(C10N.get(ExcelSheetNameC10n.class, I18n.LOCALE.get()).result()); try { sheet.setDefaultColumnWidth(config.getDefaultColumnWidth()); // Create a table environment inside the excel sheet - XSSFTable table = createTableEnvironment(exec, sheet); + final XSSFTable table = createTableEnvironment(exec, sheet); writeHeader(sheet, idHeaders, resultInfosExec, table); - int writtenLines = writeBody(sheet, resultInfosExec, exec.streamResults(OptionalLong.of(limit.orElse(MAX_LINES)))); + final int writtenLines = writeBody(sheet, resultInfosExec, exec.streamResults(OptionalLong.of(limit.orElse(MAX_LINES)))); postProcessTable(sheet, table, writtenLines, idHeaders.size()); @@ -129,11 +130,11 @@ private void setMetaData(E exec */ private void postProcessTable(SXSSFSheet sheet, XSSFTable table, int writtenLines, int size) { // Extend the table area to the added data - CellReference topLeft = new CellReference(0, 0); + final CellReference topLeft = new CellReference(0, 0); // The area must be at least a header row and a data row. If no line was written we include an empty data row so POI is happy - CellReference bottomRight = new CellReference(Math.max(1, writtenLines), table.getColumnCount() - 1); - AreaReference newArea = new AreaReference(topLeft, bottomRight, workbook.getSpreadsheetVersion()); + final CellReference bottomRight = new CellReference(Math.max(1, writtenLines), table.getColumnCount() - 1); + final AreaReference newArea = new AreaReference(topLeft, bottomRight, workbook.getSpreadsheetVersion()); table.setArea(newArea); // Add auto filters. This must be done on the lower level CTTable. Using SXSSFSheet::setAutoFilter will corrupt the table @@ -148,13 +149,13 @@ private void postProcessTable(SXSSFSheet sheet, XSSFTable table, int writtenLine */ @NotNull private XSSFTable createTableEnvironment(ManagedExecution exec, SXSSFSheet sheet) { - XSSFTable table = sheet.getWorkbook().getXSSFWorkbook().getSheet(sheet.getSheetName()).createTable(null); + final XSSFTable table = sheet.getWorkbook().getXSSFWorkbook().getSheet(sheet.getSheetName()).createTable(null); - CTTable cttable = table.getCTTable(); + final CTTable cttable = table.getCTTable(); table.setName(exec.getLabelWithoutAutoLabelSuffix()); cttable.setTotalsRowShown(false); - CTTableStyleInfo styleInfo = cttable.addNewTableStyleInfo(); + final CTTableStyleInfo styleInfo = cttable.addNewTableStyleInfo(); // Not sure how important this name is styleInfo.setName("TableStyleMedium2"); styleInfo.setShowColumnStripes(false); @@ -172,22 +173,22 @@ private void writeHeader( List infos, XSSFTable table) { - CTTableColumns columns = table.getCTTable().addNewTableColumns(); + final CTTableColumns columns = table.getCTTable().addNewTableColumns(); columns.setCount(idHeaders.size() + infos.size()); - UniqueNamer uniqueNamer = new UniqueNamer(cfg); + final UniqueNamer uniqueNamer = new UniqueNamer(cfg); { - Row header = sheet.createRow(0); + final Row header = sheet.createRow(0); // First to create the columns and track them for auto size before the first row is written int currentColumn = 0; for (ResultInfo idHeader : idHeaders) { - CTTableColumn column = columns.addNewTableColumn(); + final CTTableColumn column = columns.addNewTableColumn(); // Table column ids MUST be set and MUST start at 1, excel will fail otherwise column.setId(currentColumn + 1); final String uniqueName = uniqueNamer.getUniqueName(idHeader); column.setName(uniqueName); - Cell headerCell = header.createCell(currentColumn); + final Cell headerCell = header.createCell(currentColumn); headerCell.setCellValue(uniqueName); // Track column explicitly, because sheet.trackAllColumnsForAutoSizing() does not work with @@ -199,11 +200,11 @@ private void writeHeader( for (ResultInfo info : infos) { final String columnName = uniqueNamer.getUniqueName(info); - CTTableColumn column = columns.addNewTableColumn(); + final CTTableColumn column = columns.addNewTableColumn(); column.setId(currentColumn + 1); column.setName(columnName); - Cell headerCell = header.createCell(currentColumn); + final Cell headerCell = header.createCell(currentColumn); headerCell.setCellValue(columnName); sheet.trackColumnForAutoSizing(currentColumn); @@ -220,7 +221,7 @@ private int writeBody( // Row 0 is the Header the data starts at 1 final AtomicInteger currentRow = new AtomicInteger(1); - final int writtenLines = resultLines.mapToInt(l -> this.writeRowsForEntity(infos, l, currentRow, cfg, sheet)).sum(); + final int writtenLines = resultLines.mapToInt(l -> writeRowsForEntity(infos, l, currentRow, cfg, sheet)).sum(); // The result was shorter than the number of rows to track, so we auto size here explicitly if (writtenLines < config.getLastRowToAutosize()) { @@ -233,41 +234,41 @@ private int writeBody( /** * Writes the result lines for each entity. */ - private int writeRowsForEntity( - List infos, - EntityResult internalRow, - final AtomicInteger currentRow, - PrintSettings settings, - SXSSFSheet sheet) { - String[] ids = settings.getIdMapper().map(internalRow).getExternalId(); + private int writeRowsForEntity(List infos, EntityResult internalRow, final AtomicInteger currentRow, PrintSettings settings, SXSSFSheet sheet) { + + final String[] ids = settings.getIdMapper().map(internalRow).getExternalId(); + final TypeWriter[] writers = infos.stream().map(info -> writer(info.getType(), info.getPrinter(), settings)).toArray(TypeWriter[]::new); int writtenLines = 0; for (Object[] resultValues : internalRow.listResultLines()) { final int thisRow = currentRow.getAndIncrement(); - Row row = sheet.createRow(thisRow); + final Row row = sheet.createRow(thisRow); + // Write id cells int currentColumn = 0; + for (String id : ids) { - Cell idCell = row.createCell(currentColumn); + final Cell idCell = row.createCell(currentColumn); idCell.setCellValue(id); currentColumn++; } // Write data cells - for (int i = 0; i < infos.size(); i++) { - ResultInfo resultInfo = infos.get(i); - Object resultValue = resultValues[i]; - Cell dataCell = row.createCell(currentColumn); + for (int index = 0; index < infos.size(); index++) { + final ResultInfo resultInfo = infos.get(index); + final Object resultValue = resultValues[index]; + final Cell dataCell = row.createCell(currentColumn); currentColumn++; + if (resultValue == null) { continue; } // Fallback to string if type is not explicitly registered - TypeWriter typeWriter = writer(resultInfo.getType()); + final TypeWriter typeWriter = writers[index]; - typeWriter.writeCell(resultInfo, settings, dataCell, resultValue, styles); + typeWriter.writeCell(resultValue, dataCell, styles); } if (thisRow == config.getLastRowToAutosize()) { @@ -303,44 +304,34 @@ private void setColumnWidthsAndUntrack(SXSSFSheet sheet) { } // Type specific cell writers - private static void writeStringCell(ResultInfo info, Cell cell, Object value, Map styles) { - cell.setCellValue((String) info.printNullable(value)); + private static void writeStringCell(Cell cell, Object value, Printer printer) { + cell.setCellValue((String) printer.print(value)); } /** * This writer is only used on Columns with the result type {@link ResultType.Primitive#BOOLEAN}, not on complex types such as `LIST[BOOLEAN]`, * because MS Excel can only represent those as strings */ - private static void writeBooleanCell(ResultInfo info, Cell cell, Object value, Map styles) { - if (value instanceof Boolean aBoolean) { - cell.setCellValue(aBoolean); - return; - } - cell.setCellValue((String) info.printNullable(value)); + private static void writeBooleanCell(Object value, Cell cell, Printer printer) { + cell.setCellValue((Boolean) printer.print(value)); } - private static void writeDateCell(ResultInfo info, PrintSettings settings, Cell cell, Object value, Map styles) { - if (!(value instanceof Number)) { - throw new IllegalStateException(String.format("`%s` Expected an Number but got an '%s' with the value: %s", - info, - value != null ? value.getClass().getName() : "no type", value - )); - } - cell.setCellValue(CDate.toLocalDate(((Number) value).intValue())); + private static void writeDateCell(Object value, Cell cell, Map styles, Printer printer) { + cell.setCellValue((LocalDate) printer.print(value)); cell.setCellStyle(styles.get(ExcelConfig.DATE_STYLE)); } - public static void writeIntegerCell(ResultInfo info, PrintSettings settings, Cell cell, Object value, Map styles) { - cell.setCellValue(((Number) value).longValue()); + public static void writeIntegerCell(Object value, Cell cell, Printer printer, Map styles) { + cell.setCellValue(((Number) printer.print(value)).longValue()); cell.setCellStyle(styles.get(ExcelConfig.INTEGER_STYLE)); } - public static void writeNumericCell(ResultInfo info, PrintSettings settings, Cell cell, Object value, Map styles) { - cell.setCellValue(((Number) value).doubleValue()); + public static void writeNumericCell(Object value, Cell cell, Printer printer, Map styles) { + cell.setCellValue(((Number) printer.print(value)).doubleValue()); cell.setCellStyle(styles.get(ExcelConfig.NUMERIC_STYLE)); } - public static void writeMoneyCell(ResultInfo info, PrintSettings settings, Cell cell, Object value, Map styles) { + public static void writeMoneyCell(Object value, Cell cell, PrintSettings settings, Map styles) { final CellStyle currencyStyle = styles.get(ExcelConfig.CURRENCY_STYLE_PREFIX + settings.getCurrency().getCurrencyCode()); if (currencyStyle == null) { // Print as cents or what ever the minor currency unit is diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java index b7b08ef279..72d06eb641 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java @@ -16,7 +16,7 @@ import com.bakdata.conquery.models.i18n.I18n; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; -import com.bakdata.conquery.models.query.resultinfo.printers.CsvResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.ExcelResultPrinters; import com.bakdata.conquery.resources.api.ResultExcelResource; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.jersey.DropwizardResourceConfig; @@ -74,8 +74,7 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuil return Collections.emptyList(); } - - final PrintSettings printSettings = new PrintSettings(true, I18n.LOCALE.get(), exec.getNamespace(), exec.getConfig(), null, null, new CsvResultPrinters()); //TODO excel different printer? + final PrintSettings printSettings = new PrintSettings(true, I18n.LOCALE.get(), exec.getNamespace(), exec.getConfig(), null, null, new ExcelResultPrinters()); // Save id column count to later check if xlsx dimensions are feasible idColumnsCount = exec.getConfig().getIdColumns().getIdResultInfos(printSettings).size(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java new file mode 100644 index 0000000000..c833eef5c3 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java @@ -0,0 +1,8 @@ +package com.bakdata.conquery.models.query.resultinfo.printers; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class ExcelResultPrinters extends JavaResultPrinters { + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/IdentityPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/IdentityPrinter.java new file mode 100644 index 0000000000..30773978a6 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/IdentityPrinter.java @@ -0,0 +1,9 @@ +package com.bakdata.conquery.models.query.resultinfo.printers; + +record IdentityPrinter() implements Printer { + + @Override + public Object print(Object value) { + return value; + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java index 8851b1a912..0b7475a49c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java @@ -66,14 +66,6 @@ public Object print(Object value) { } } - private record IdentityPrinter() implements Printer { - - @Override - public Object print(Object value) { - return value; - } - } - private record DatePrinter() implements Printer { @Override From a39c4f494cdf86073dbc9a1848ab15214564cec0 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 29 Aug 2024 11:37:31 +0200 Subject: [PATCH 23/54] fixes MONEY resultType in ArrowResultGenerationTest.java --- .../conquery/io/result/arrow/ArrowResultGenerationTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java index 2c9a22708f..7cf154463d 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java @@ -102,7 +102,8 @@ void generateFieldsValue() { new Field("max", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null) )), new Field("STRING", FieldType.nullable(new ArrowType.Utf8()), null), - new Field("MONEY", FieldType.nullable(new ArrowType.Int(32, true)), null), + //TODO this is just an experiment + new Field("MONEY", FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), null), new Field("LIST[BOOLEAN]", FieldType.nullable(ArrowType.List.INSTANCE), List.of(new Field("LIST[BOOLEAN]", FieldType.nullable(ArrowType.Bool.INSTANCE), null))), new Field("LIST[DATE_RANGE]", FieldType.nullable(ArrowType.List.INSTANCE), List.of(new Field("LIST[DATE_RANGE]", FieldType.nullable(ArrowType.Struct.INSTANCE), From d92b0a9e96cfa5cf182ad6833551833ab4821fa5 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 29 Aug 2024 11:49:34 +0200 Subject: [PATCH 24/54] fixes MONEY resultType in ParquetResultGenerationTest.java --- .../conquery/io/result/parquet/ParquetResultGenerationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java index 17715c5f83..dcc4d75f22 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java @@ -74,7 +74,7 @@ void generateSchema() { .optional(INT32).as(LogicalTypeAnnotation.dateType()).named("max") .named("DATE_RANGE") .optional(BINARY).as(stringType()).named("STRING") - .optional(INT32).as(LogicalTypeAnnotation.intType(32, true)).named("MONEY") + .optional(DOUBLE).named("MONEY") .optionalGroup().as(LogicalTypeAnnotation.listType()) .repeatedGroup() .optional(BOOLEAN).named("element") From 4fdb5c23a2d898b954b012a1050802241650903a Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 29 Aug 2024 12:34:55 +0200 Subject: [PATCH 25/54] cleanup ResultTestUtil --- .../conquery/io/result/ResultTestUtil.java | 38 +++------ .../arrow/ArrowResultGenerationTest.java | 7 +- .../result/csv/CsvResultGenerationTest.java | 77 ++++++++++--------- .../result/excel/ExcelResultRenderTest.java | 36 ++++----- .../parquet/ParquetResultGenerationTest.java | 5 +- 5 files changed, 74 insertions(+), 89 deletions(-) diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java index 4218f0bcd5..7c51516ec4 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java @@ -3,13 +3,12 @@ import java.math.BigDecimal; import java.util.Collections; import java.util.List; -import java.util.Locale; import java.util.OptionalLong; import java.util.stream.Collectors; import java.util.stream.Stream; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; -import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.concepts.select.Select; @@ -23,7 +22,6 @@ import com.bakdata.conquery.models.query.resultinfo.ExternalResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.CsvResultPrinters; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.MultilineEntityResult; import com.bakdata.conquery.models.query.results.SinglelineEntityResult; @@ -37,7 +35,7 @@ @UtilityClass public class ResultTestUtil { - + private static User OWNER = new User("user", "User", null); private static TreeConcept concept; static { @@ -46,27 +44,21 @@ public class ResultTestUtil { concept.setDataset(new Dataset("dataset")); } - private static final PrintSettings - PRINT_SETTINGS = - new PrintSettings(false, Locale.ROOT, null, new ConqueryConfig(), null, (selectInfo) -> selectInfo.getSelect().getLabel(), new CsvResultPrinters()); - - public static List - ID_FIELDS = - Stream.of("id1", "id2") - .map(name -> { - ExternalResultInfo info = new ExternalResultInfo(name, ResultType.Primitive.STRING, PRINT_SETTINGS); - info.addSemantics(new SemanticType.IdT("ID")); - return info; - }) - .collect(Collectors.toList()); + public static List getIdFields(PrintSettings printSettings) { + return Stream.of("id1", "id2").map(name -> { + ExternalResultInfo info = new ExternalResultInfo(name, ResultType.Primitive.STRING, printSettings); + info.addSemantics(new SemanticType.IdT("ID")); + return info; + }).collect(Collectors.toList()); + } @NotNull public static ManagedQuery getTestQuery() { - return new ManagedQuery(null, null, null, null) { + return new ManagedQuery(null, OWNER, null, null) { @Override public List getResultInfos(PrintSettings printSettings) { return getResultTypes().stream() - .map(resultType -> new TypedSelectDummy(resultType)) + .map(TypedSelectDummy::new) .map(select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet(), printSettings)) .collect(Collectors.toList()); } @@ -85,13 +77,7 @@ public static List getResultTypes() { @NotNull public static List getTestEntityResults() { - return List.of(new SinglelineEntityResult("1", new Object[]{Boolean.TRUE, 2345634, 123423.34, 5646, List.of(345, 534), "test_string", new BigDecimal("45.21"), List.of(true, false), List.of(List.of(345, 534), List.of(1, 2)), List.of("fizz", "buzz")}), - new SinglelineEntityResult("2", new Object[]{Boolean.FALSE, null, null, null, null, null, null, List.of(), List.of(List.of(1234, Integer.MAX_VALUE)), List.of()}), - new SinglelineEntityResult("2", new Object[]{Boolean.TRUE, null, null, null, null, null, null, List.of(false, false), null, null}), - new MultilineEntityResult("3", - List.of(new Object[]{Boolean.FALSE, null, null, null, null, null, null, List.of(false), null, null}, - new Object[]{Boolean.TRUE, null, null, null, null, null, null, null, null, null}, - new Object[]{Boolean.TRUE, null, null, null, null, null, new BigDecimal(4), List.of(true, false, true, false), null, null}))); + return List.of(new SinglelineEntityResult("1", new Object[]{Boolean.TRUE, 2345634, 123423.34, 5646, List.of(345, 534), "test_string", new BigDecimal("45.21"), List.of(true, false), List.of(List.of(345, 534), List.of(1, 2)), List.of("fizz", "buzz")}), new SinglelineEntityResult("2", new Object[]{Boolean.FALSE, null, null, null, null, null, null, List.of(), List.of(List.of(1234, Integer.MAX_VALUE)), List.of()}), new SinglelineEntityResult("2", new Object[]{Boolean.TRUE, null, null, null, null, null, null, List.of(false, false), null, null}), new MultilineEntityResult("3", List.of(new Object[]{Boolean.FALSE, null, null, null, null, null, null, List.of(false), null, null}, new Object[]{Boolean.TRUE, null, null, null, null, null, null, null, null, null}, new Object[]{Boolean.TRUE, null, null, null, null, null, new BigDecimal(4), List.of(true, false, true, false), null, null}))); } public static class TypedSelectDummy extends Select { diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java index 7cf154463d..52cbdbe9de 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java @@ -22,7 +22,6 @@ import java.util.stream.Stream; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; -import com.bakdata.conquery.io.result.ResultTestUtil; import com.bakdata.conquery.models.common.CDate; import com.bakdata.conquery.models.config.ArrowConfig; import com.bakdata.conquery.models.config.ConqueryConfig; @@ -65,7 +64,7 @@ public class ArrowResultGenerationTest { void generateFieldsIdMapping() { final UniqueNamer uniqueNamer = new UniqueNamer(PRINT_SETTINGS); - List fields = generateFields(ResultTestUtil.ID_FIELDS, uniqueNamer); + List fields = generateFields(getIdFields(PRINT_SETTINGS), uniqueNamer); assertThat(fields).containsExactlyElementsOf( List.of( @@ -144,7 +143,7 @@ void writeAndRead() throws IOException { (root) -> new ArrowStreamWriter(root, new DictionaryProvider.MapDictionaryProvider(), output), printSettings, new ArrowConfig(BATCH_SIZE), - ResultTestUtil.ID_FIELDS, + getIdFields(PRINT_SETTINGS), mquery.getResultInfos(printSettings), mquery.streamResults(OptionalLong.empty()) ); @@ -204,7 +203,7 @@ public static String generateExpectedTSV(List results, List i.defaultColumnName()), + getIdFields(PRINT_SETTINGS).stream().map(i -> i.defaultColumnName()), // result column headers getResultTypes().stream().map(ResultType::typeInfo) ).collect(Collectors.joining("\t")) diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java index 8d5cf8932f..e1f3ceee7d 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java @@ -12,7 +12,6 @@ import java.util.StringJoiner; import java.util.stream.Collectors; -import com.bakdata.conquery.io.result.ResultTestUtil; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.i18n.I18n; import com.bakdata.conquery.models.identifiable.mapping.EntityPrintId; @@ -29,20 +28,19 @@ @Slf4j public class CsvResultGenerationTest { - static { - I18n.init(); - } - - public static final ConqueryConfig CONFIG = new ConqueryConfig(){{ + public static final ConqueryConfig CONFIG = new ConqueryConfig() {{ // Suppress java.lang.NoClassDefFoundError: com/bakdata/conquery/io/jackson/serializer/CurrencyUnitDeserializer setStorage(new NonPersistentStoreFactory()); }}; + static { + I18n.init(); + } @Test void writeAndRead() throws IOException { // Prepare every input data - PrintSettings printSettings = new PrintSettings( + final PrintSettings printSettings = new PrintSettings( true, Locale.GERMAN, null, @@ -51,20 +49,20 @@ void writeAndRead() throws IOException { (selectInfo) -> selectInfo.getSelect().getLabel(), new CsvResultPrinters() ); // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized - List results = getTestEntityResults(); + final List results = getTestEntityResults(); - ManagedQuery mquery = getTestQuery(); + final ManagedQuery mquery = getTestQuery(); // First we write to the buffer, than we read from it and parse it as TSV - StringWriter writer = new StringWriter(); + final StringWriter writer = new StringWriter(); - CsvRenderer renderer = new CsvRenderer(CONFIG.getCsv().createWriter(writer), printSettings); - renderer.toCSV(ResultTestUtil.ID_FIELDS, mquery.getResultInfos(printSettings), mquery.streamResults(OptionalLong.empty())); + final CsvRenderer renderer = new CsvRenderer(CONFIG.getCsv().createWriter(writer), printSettings); + renderer.toCSV(getIdFields(printSettings), mquery.getResultInfos(printSettings), mquery.streamResults(OptionalLong.empty())); - String computed = writer.toString(); + final String computed = writer.toString(); - String expected = generateExpectedCSV(results, mquery.getResultInfos(printSettings)); + final String expected = generateExpectedCSV(results, mquery.getResultInfos(printSettings), printSettings); log.info("Wrote and than read this csv data: {}", computed); @@ -73,31 +71,34 @@ void writeAndRead() throws IOException { } - private String generateExpectedCSV(List results, List resultInfos) { - List expected = new ArrayList<>(); - expected.add(ResultTestUtil.ID_FIELDS.stream().map(info -> info.defaultColumnName()).collect(Collectors.joining(",")) + "," + getResultTypes().stream().map(ResultType::typeInfo).collect(Collectors.joining(",")) + "\n"); + private String generateExpectedCSV(List results, List resultInfos, PrintSettings printSettings) { + final List expected = new ArrayList<>(); + expected.add(getIdFields(printSettings).stream().map(info -> info.defaultColumnName()).collect(Collectors.joining(",")) + + "," + + getResultTypes().stream().map(ResultType::typeInfo).collect(Collectors.joining(",")) + + "\n"); results.stream() - .map(EntityResult.class::cast) - .forEach(res -> { - - for (Object[] line : res.listResultLines()) { - StringJoiner valueJoiner = new StringJoiner(","); - valueJoiner.add(String.valueOf(res.getEntityId())); - valueJoiner.add(String.valueOf(res.getEntityId())); - for (int lIdx = 0; lIdx < line.length; lIdx++) { - Object val = line[lIdx]; - if(val == null) { - valueJoiner.add(""); - continue; - } - ResultInfo info = resultInfos.get(lIdx); - final String printVal = (String) info.printNullable(val); - valueJoiner.add(printVal.contains(String.valueOf(CONFIG.getCsv().getDelimeter()))? "\""+printVal+"\"": printVal); - } - - expected.add(valueJoiner + "\n"); - } - }); + .map(EntityResult.class::cast) + .forEach(res -> { + + for (Object[] line : res.listResultLines()) { + final StringJoiner valueJoiner = new StringJoiner(","); + valueJoiner.add(String.valueOf(res.getEntityId())); + valueJoiner.add(String.valueOf(res.getEntityId())); + for (int lIdx = 0; lIdx < line.length; lIdx++) { + final Object val = line[lIdx]; + if (val == null) { + valueJoiner.add(""); + continue; + } + final ResultInfo info = resultInfos.get(lIdx); + final String printVal = (String) info.printNullable(val); + valueJoiner.add(printVal.contains(String.valueOf(CONFIG.getCsv().getDelimeter())) ? "\"" + printVal + "\"" : printVal); + } + + expected.add(valueJoiner + "\n"); + } + }); return String.join("", expected); } diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java index 18723d7260..16f6f351c3 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java @@ -56,14 +56,14 @@ public class ExcelResultRenderTest { @Test void writeAndRead() throws IOException { // Prepare every input data - PrintSettings + final PrintSettings printSettings = new PrintSettings(true, Locale.GERMAN, null, CONFIG, (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), (selectInfo) -> selectInfo.getSelect() .getLabel(), new CsvResultPrinters()); // TODO ? // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized - List results = getTestEntityResults(); + final List results = getTestEntityResults(); - ManagedQuery mquery = new ManagedQuery(null, null, null, null) { + final ManagedQuery mquery = new ManagedQuery(null, null, null, null) { public List getResultInfos(PrintSettings printSettings) { return getResultTypes().stream() .map(ResultTestUtil.TypedSelectDummy::new) @@ -78,19 +78,19 @@ public Stream streamResults(OptionalLong maybeLimit) { }; // First we write to the buffer, than we read from it and parse it as TSV - ByteArrayOutputStream output = new ByteArrayOutputStream(); + final ByteArrayOutputStream output = new ByteArrayOutputStream(); - ExcelRenderer renderer = new ExcelRenderer(new ExcelConfig(), printSettings); + final ExcelRenderer renderer = new ExcelRenderer(new ExcelConfig(), printSettings); - renderer.renderToStream(ResultTestUtil.ID_FIELDS, mquery, output, OptionalLong.empty(), printSettings); + renderer.renderToStream(ResultTestUtil.getIdFields(printSettings), mquery, output, OptionalLong.empty(), printSettings); - InputStream inputStream = new ByteArrayInputStream(output.toByteArray()); + final InputStream inputStream = new ByteArrayInputStream(output.toByteArray()); - List computed = readComputed(inputStream, printSettings); + final List computed = readComputed(inputStream, printSettings); - List expected = generateExpectedTSV(results, mquery.getResultInfos(printSettings)); + final List expected = generateExpectedTSV(results, mquery.getResultInfos(printSettings)); log.info("Wrote and than read this excel data: {}", computed); @@ -101,14 +101,14 @@ public Stream streamResults(OptionalLong maybeLimit) { @NotNull private List readComputed(InputStream inputStream, PrintSettings settings) throws IOException { - XSSFWorkbook workbook = new XSSFWorkbook(inputStream); - XSSFSheet sheet = workbook.getSheetAt(0); + final XSSFWorkbook workbook = new XSSFWorkbook(inputStream); + final XSSFSheet sheet = workbook.getSheetAt(0); - List computed = new ArrayList<>(); + final List computed = new ArrayList<>(); int i = 0; for (Row row : sheet) { - StringJoiner sj = new StringJoiner("\t"); - DataFormatter formatter = new DataFormatter(settings.getLocale()); + final StringJoiner sj = new StringJoiner("\t"); + final DataFormatter formatter = new DataFormatter(settings.getLocale()); for (Cell cell : row) { final String formatted = switch (cell.getCellType()) { @@ -128,21 +128,21 @@ private List readComputed(InputStream inputStream, PrintSettings setting private List generateExpectedTSV(List results, List resultInfos) { - List expected = new ArrayList<>(); + final List expected = new ArrayList<>(); expected.add(String.join("\t", printIdFields) + "\t" + getResultTypes().stream().map(ResultType::typeInfo).collect(Collectors.joining("\t"))); results.stream().map(EntityResult.class::cast).forEach(res -> { for (Object[] line : res.listResultLines()) { - StringJoiner valueJoiner = new StringJoiner("\t"); + final StringJoiner valueJoiner = new StringJoiner("\t"); valueJoiner.add(String.valueOf(res.getEntityId())); valueJoiner.add(String.valueOf(res.getEntityId())); for (int lIdx = 0; lIdx < line.length; lIdx++) { - Object val = line[lIdx]; + final Object val = line[lIdx]; if (val == null) { valueJoiner.add("null"); continue; } - ResultInfo info = resultInfos.get(lIdx); + final ResultInfo info = resultInfos.get(lIdx); joinValue(valueJoiner, val, info); } expected.add(valueJoiner.toString()); diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java index dcc4d75f22..dc5bf6208f 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java @@ -16,7 +16,6 @@ import java.util.stream.Collectors; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; -import com.bakdata.conquery.io.result.ResultTestUtil; import com.bakdata.conquery.io.result.arrow.ArrowResultGenerationTest; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.i18n.I18n; @@ -59,7 +58,7 @@ void generateSchema() { List resultInfos = getResultTypes().stream().map(TypedSelectDummy::new) .map(select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet(), PRINT_SETTINGS)).collect(Collectors.toList()); - final MessageType messageType = EntityResultWriteSupport.generateSchema(ResultTestUtil.ID_FIELDS, resultInfos, uniqueNamer); + final MessageType messageType = EntityResultWriteSupport.generateSchema(getIdFields(PRINT_SETTINGS), resultInfos, uniqueNamer); assertThat(messageType).isEqualTo( Types.buildMessage() @@ -121,7 +120,7 @@ void writeAndRead() throws IOException { // First we write to the buffer, than we read from it and parse it as TSV ByteArrayOutputStream output = new ByteArrayOutputStream(); - ParquetRenderer.writeToStream(output, ResultTestUtil.ID_FIELDS, managedQuery.getResultInfos(printSettings), printSettings, managedQuery.streamResults(OptionalLong.empty())); + ParquetRenderer.writeToStream(output, getIdFields(PRINT_SETTINGS), managedQuery.getResultInfos(printSettings), printSettings, managedQuery.streamResults(OptionalLong.empty())); final byte[] buf = output.toByteArray(); From 84549dab5639b2a8ed558ddb830a7c275770e3d0 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 29 Aug 2024 12:39:25 +0200 Subject: [PATCH 26/54] adds missing Dataset --- .../conquery/io/result/ResultTestUtil.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java index 7c51516ec4..218822f456 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java @@ -35,13 +35,14 @@ @UtilityClass public class ResultTestUtil { - private static User OWNER = new User("user", "User", null); - private static TreeConcept concept; + private static final User OWNER = new User("user", "User", null); + private static final Dataset DATASET = new Dataset("dataset"); + private static final TreeConcept CONCEPT; static { - concept = new TreeConcept(); - concept.setName("concept"); - concept.setDataset(new Dataset("dataset")); + CONCEPT = new TreeConcept(); + CONCEPT.setName("concept"); + CONCEPT.setDataset(DATASET); } public static List getIdFields(PrintSettings printSettings) { @@ -54,7 +55,7 @@ public static List getIdFields(PrintSettings printSettings) { @NotNull public static ManagedQuery getTestQuery() { - return new ManagedQuery(null, OWNER, null, null) { + return new ManagedQuery(null, OWNER, DATASET, null) { @Override public List getResultInfos(PrintSettings printSettings) { return getResultTypes().stream() @@ -87,7 +88,7 @@ public static class TypedSelectDummy extends Select { public TypedSelectDummy(ResultType resultType) { setLabel(resultType.toString()); - setHolder(concept); + setHolder(CONCEPT); this.resultType = resultType; } From ab633f99f5e5e4a9fe652c1d77d67738274d1076 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 29 Aug 2024 15:17:59 +0200 Subject: [PATCH 27/54] cleanup DateRange vector Filler, annd fixes potential NPEs --- .../io/result/arrow/ArrowRenderer.java | 81 +++++++++---------- .../printers/ArrowResultPrinters.java | 2 +- 2 files changed, 41 insertions(+), 42 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java index 6218824d3a..d648144396 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java @@ -10,6 +10,7 @@ import java.util.stream.Stream; import com.bakdata.conquery.models.common.CDate; +import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.config.ArrowConfig; import com.bakdata.conquery.models.identifiable.mapping.PrintIdMapper; import com.bakdata.conquery.models.query.PrintSettings; @@ -109,7 +110,7 @@ public static void write( ); for (int index = 0; index < idWriters.length; index++) { - if(printedExternalId[index] == null){ + if (printedExternalId[index] == null) { continue; } @@ -150,67 +151,75 @@ public static void write( private static RowConsumer intVectorFiller(IntVector vector) { return (rowNumber, valueRaw) -> { - final Integer value = (Integer) valueRaw; - if (value == null) { + if (valueRaw == null) { vector.setNull(rowNumber); return; } + + final Integer value = (Integer) valueRaw; + vector.setSafe(rowNumber, value); }; } private static RowConsumer bitVectorFiller(BitVector vector) { return (rowNumber, valueRaw) -> { - final Boolean value = (Boolean) valueRaw; - if (value == null) { + if (valueRaw == null) { vector.setNull(rowNumber); return; } + + final Boolean value = (Boolean) valueRaw; + vector.setSafe(rowNumber, value ? 1 : 0); }; } private static RowConsumer float8VectorFiller(Float8Vector vector) { return (rowNumber, valueRaw) -> { - final Number value = (Number) valueRaw; - if (value == null) { + if (valueRaw == null) { vector.setNull(rowNumber); return; } + + final Number value = (Number) valueRaw; + vector.setSafe(rowNumber, value.doubleValue()); }; } private static RowConsumer float4VectorFiller(Float4Vector vector) { return (rowNumber, valueRaw) -> { - final Number value = (Number) valueRaw; - if (value == null) { + if (valueRaw == null) { vector.setNull(rowNumber); return; } + + final Number value = (Number) valueRaw; vector.setSafe(rowNumber, value.floatValue()); }; } private static RowConsumer varCharVectorFiller(VarCharVector vector) { return (rowNumber, valueRaw) -> { - final String value = (String) valueRaw; - if (value == null) { + if (valueRaw == null) { vector.setNull(rowNumber); return; } + final String value = (String) valueRaw; vector.setSafe(rowNumber, new Text(value)); }; } private static RowConsumer dateDayVectorFiller(DateDayVector vector) { return (rowNumber, valueRaw) -> { - final Number value = (Number) valueRaw; - if (value == null) { + if (valueRaw == null) { vector.setNull(rowNumber); return; } + final Number value = (Number) valueRaw; + // Treat our internal infinity dates (Interger.MIN and Integer.MAX) also as null final int epochDay = value.intValue(); @@ -223,42 +232,43 @@ private static RowConsumer dateDayVectorFiller(DateDayVector vector) { }; } - private static RowConsumer structVectorFiller(StructVector vector, RowConsumer[] nestedConsumers) { - return (rowNumber, valueRaw) -> { - // Values is a horizontal list - final List values = (List) valueRaw; - if (values == null) { + private static RowConsumer dateRangeVectorFiller(StructVector vector) { + final List nestedVectors = vector.getPrimitiveVectors(); + final RowConsumer minConsumer = generateVectorFiller(nestedVectors.get(0), ResultType.Primitive.DATE); + final RowConsumer maxConsumer = generateVectorFiller(nestedVectors.get(1), ResultType.Primitive.DATE); + + return ((rowNumber, valueRaw) -> { + if (valueRaw == null) { vector.setNull(rowNumber); return; } - Preconditions.checkState( - values.size() == nestedConsumers.length, - "The number of the provided nested value differs from the number of consumer for the generated vectors. Provided values: %s\t Available consumers: %d".formatted(values, nestedConsumers.length)); + final CDateRange value = (CDateRange) valueRaw; - for (RowConsumer nestedConsumer : nestedConsumers) { - nestedConsumer.accept(rowNumber, values.toArray()); - } + //TODO are we interested in infinities here? + minConsumer.accept(rowNumber, value.getMinValue()); + maxConsumer.accept(rowNumber, value.getMaxValue()); // Finally mark that we populated the nested vectors vector.setIndexDefined(rowNumber); - }; + }); } private static RowConsumer listVectorFiller(ListVector vector, RowConsumer nestedConsumer) { return (rowNumber, valueRaw) -> { - // Values is a vertical list - final List values = (List) valueRaw; - if (values == null) { + if (valueRaw == null) { vector.setNull(rowNumber); return; } + final List values = (List) valueRaw; + final int start = vector.startNewValue(rowNumber); for (int i = 0; i < values.size(); i++) { - // These short lived one value arrays are a workaround at the moment + // These short-lived one value arrays are a workaround at the moment + //TODO consider using only a single array and overwriting it every row - this works only if Arrow processes the values immediately nestedConsumer.accept(Math.addExact(start, i), new Object[]{values.get(i)}); } @@ -302,18 +312,7 @@ private static RowConsumer generateVectorFiller(ValueVector vector, ResultType t case DATE -> dateDayVectorFiller(((DateDayVector) vector)); case NUMERIC -> float8VectorFiller((Float8Vector) vector); case STRING -> varCharVectorFiller(((VarCharVector) vector)); - - case DATE_RANGE -> { - final StructVector structVector = (StructVector) vector; - final List nestedVectors = structVector.getPrimitiveVectors(); - final RowConsumer[] nestedConsumers = new RowConsumer[nestedVectors.size()]; - - for (int i = 0; i < nestedVectors.size(); i++) { - nestedConsumers[i] = generateVectorFiller(nestedVectors.get(i), ResultType.Primitive.DATE); - } - - yield structVectorFiller(structVector, nestedConsumers); - } + case DATE_RANGE -> dateRangeVectorFiller((StructVector) vector); }; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java index 496f24ed26..f76206e1bd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java @@ -2,7 +2,7 @@ import com.bakdata.conquery.models.query.PrintSettings; -public class ArrowResultPrinters extends JavaResultPrinters{ +public class ArrowResultPrinters extends JavaResultPrinters { @Override public Printer getDatePrinter(PrintSettings printSettings) { From a769b1619c607e886e8a183d4dcc41ed595387db Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 29 Aug 2024 15:44:07 +0200 Subject: [PATCH 28/54] cleanup usage of ChainingPrinter --- .../apiv1/query/TableExportQuery.java | 6 +---- .../io/result/arrow/ArrowRenderer.java | 4 ++-- .../conquery/io/result/arrow/RowConsumer.java | 2 +- .../io/result/excel/ExcelRenderer.java | 10 ++++----- .../parquet/EntityResultWriteSupport.java | 4 ++-- .../select/connector/DistinctSelect.java | 3 +-- .../specific/MappableSingleColumnSelect.java | 3 +-- .../query/preview/EntityPreviewExecution.java | 2 +- .../models/query/resultinfo/ResultInfo.java | 2 +- .../resultinfo/printers/ChainingPrinter.java | 9 -------- .../resultinfo/printers/ConceptIdPrinter.java | 2 +- .../printers/CsvResultPrinters.java | 22 +++++++++---------- .../resultinfo/printers/IdentityPrinter.java | 2 +- .../printers/JavaResultPrinters.java | 8 +++---- .../printers/JsonResultPrinters.java | 12 +++++----- .../printers/LocalizedEnumPrinter.java | 2 +- .../resultinfo/printers/MappedPrinter.java | 2 +- .../query/resultinfo/printers/Printer.java | 6 +++-- .../printers/SecondaryIdResultInfo.java | 2 +- .../query/statistics/ResultStatistics.java | 2 +- .../DefaultSqlCDateSetParserTest.java | 2 +- .../conquery/models/types/ResultTypeTest.java | 8 +++---- 22 files changed, 51 insertions(+), 64 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ChainingPrinter.java diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java index 1613833613..a01343d5d1 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java @@ -41,7 +41,6 @@ import com.bakdata.conquery.models.query.queryplan.TableExportQueryPlan; import com.bakdata.conquery.models.query.resultinfo.ColumnResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.ChainingPrinter; import com.bakdata.conquery.models.query.resultinfo.printers.ConceptIdPrinter; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.query.resultinfo.printers.SecondaryIdResultInfo; @@ -280,10 +279,7 @@ private List createResultInfos(Set conceptColumns, PrintSett if (!isRawConceptValues()) { resultType = ResultType.Primitive.STRING; - printer = new ChainingPrinter( - new ConceptIdPrinter(concept, printSettings), - printSettings.getPrinterFactory().getStringPrinter(printSettings) - ); + printer = new ConceptIdPrinter(concept, printSettings); } } else { diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java index d648144396..3734f3ce13 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java @@ -100,7 +100,7 @@ public static void write( final Object[] printedExternalId = new String[externalId.length]; for (int index = 0; index < idWriters.length; index++) { - printedExternalId[index] = printers.get(index).print(externalId[index]); + printedExternalId[index] = printers.get(index).apply(externalId[index]); } for (Object[] line : cer.listResultLines()) { @@ -121,7 +121,7 @@ public static void write( final int colId = index + idWriters.length; // In this case, the printer normalizes and adjusts values. - final Object printed = printers.get(colId).print(line[index]); + final Object printed = printers.get(colId).apply(line[index]); if (printed == null) { continue; diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/RowConsumer.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/RowConsumer.java index f39381b47f..057a98306b 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/RowConsumer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/RowConsumer.java @@ -8,6 +8,6 @@ public interface RowConsumer { * * @param t the input argument */ - void accept(int rowNumber, Object row); + void accept(int rowNumber, Object value); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java index 6b2a589101..f503ffba83 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java @@ -305,7 +305,7 @@ private void setColumnWidthsAndUntrack(SXSSFSheet sheet) { // Type specific cell writers private static void writeStringCell(Cell cell, Object value, Printer printer) { - cell.setCellValue((String) printer.print(value)); + cell.setCellValue((String) printer.apply(value)); } /** @@ -313,21 +313,21 @@ private static void writeStringCell(Cell cell, Object value, Printer printer) { * because MS Excel can only represent those as strings */ private static void writeBooleanCell(Object value, Cell cell, Printer printer) { - cell.setCellValue((Boolean) printer.print(value)); + cell.setCellValue((Boolean) printer.apply(value)); } private static void writeDateCell(Object value, Cell cell, Map styles, Printer printer) { - cell.setCellValue((LocalDate) printer.print(value)); + cell.setCellValue((LocalDate) printer.apply(value)); cell.setCellStyle(styles.get(ExcelConfig.DATE_STYLE)); } public static void writeIntegerCell(Object value, Cell cell, Printer printer, Map styles) { - cell.setCellValue(((Number) printer.print(value)).longValue()); + cell.setCellValue(((Number) printer.apply(value)).longValue()); cell.setCellStyle(styles.get(ExcelConfig.INTEGER_STYLE)); } public static void writeNumericCell(Object value, Cell cell, Printer printer, Map styles) { - cell.setCellValue(((Number) printer.print(value)).doubleValue()); + cell.setCellValue(((Number) printer.apply(value)).doubleValue()); cell.setCellStyle(styles.get(ExcelConfig.NUMERIC_STYLE)); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java index a84aa9c6be..6181231254 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java @@ -132,7 +132,7 @@ public void write(EntityResult record) { final Object[] printedExternalId = new String[externalId.length]; for (int index = 0; index < externalId.length; index++) { - printedExternalId[index] = columnPrinters.get(index).print(externalId[index]); + printedExternalId[index] = columnPrinters.get(index).apply(externalId[index]); } for (Object[] listResultLine : listResultLines) { @@ -161,7 +161,7 @@ public void write(EntityResult record) { continue; } - final Object printed = columnPrinters.get(colId).print(resultValue); + final Object printed = columnPrinters.get(colId).apply(resultValue); final String fieldName = schema.getFieldName(colId); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java index 4bf617b31c..b135160510 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java @@ -9,7 +9,6 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.AllValuesAggregator; -import com.bakdata.conquery.models.query.resultinfo.printers.ChainingPrinter; import com.bakdata.conquery.models.query.resultinfo.printers.MappedPrinter; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; @@ -48,6 +47,6 @@ public Printer createPrinter(PrintSettings printSettings, PrinterFactory printer return super.createPrinter(printSettings, printerFactory); } - return printerFactory.getListPrinter(new ChainingPrinter(new MappedPrinter(getMapping()), printerFactory.getStringPrinter(printSettings)), printSettings); + return printerFactory.getListPrinter(new MappedPrinter(getMapping()), printSettings); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java index ce90fe1fda..60ea6040f0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java @@ -15,7 +15,6 @@ import com.bakdata.conquery.models.index.InternToExternMapper; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.ChainingPrinter; import com.bakdata.conquery.models.query.resultinfo.printers.MappedPrinter; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; @@ -62,7 +61,7 @@ public Printer createPrinter(PrintSettings printSettings, PrinterFactory printer return super.createPrinter(printSettings, printerFactory); } - return new ChainingPrinter(new MappedPrinter(getMapping()), printerFactory.getStringPrinter(printSettings)); + return new MappedPrinter(getMapping()); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java index adf198d31d..2121ecfeba 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java @@ -145,7 +145,7 @@ private static Function> createLineToMapTransforme continue; } - final Object value = printers[column].print(line[column]); + final Object value = printers[column].apply(line[column]); out.put(columnName, value); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java index 584174470f..442c4a01dc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java @@ -69,7 +69,7 @@ public final Object printNullable(Object f) { return ""; } - return getPrinter().print(f); + return getPrinter().apply(f); } public abstract Printer getPrinter(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ChainingPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ChainingPrinter.java deleted file mode 100644 index 998f289e07..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ChainingPrinter.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.bakdata.conquery.models.query.resultinfo.printers; - -public record ChainingPrinter(Printer incoming, Printer outgoing) implements Printer { - - @Override - public Object print(Object value) { - return outgoing.print(incoming.print(value)); - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ConceptIdPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ConceptIdPrinter.java index 4c00a04ad9..fbba2b8607 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ConceptIdPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ConceptIdPrinter.java @@ -8,7 +8,7 @@ public record ConceptIdPrinter(Concept concept, PrintSettings cfg) implements Printer { @Override - public String print(Object rawValue) { + public String apply(Object rawValue) { if (rawValue == null) { return null; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/CsvResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/CsvResultPrinters.java index ba00a06ccc..62dbc36897 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/CsvResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/CsvResultPrinters.java @@ -59,7 +59,7 @@ public Printer getMoneyPrinter(PrintSettings printSettings) { private record StringPrinter() implements Printer { @Override - public String print(Object f) { + public String apply(Object f) { return Objects.toString(f); } } @@ -67,7 +67,7 @@ public String print(Object f) { private record IntegerPrinter(PrintSettings cfg) implements Printer { @Override - public String print(Object f) { + public String apply(Object f) { if (cfg.isPrettyPrint()) { return cfg.getIntegerFormat().format(((Number) f).longValue()); } @@ -79,7 +79,7 @@ public String print(Object f) { private record NumericPrinter(PrintSettings cfg) implements Printer { @Override - public String print(Object f) { + public String apply(Object f) { if (cfg.isPrettyPrint()) { return cfg.getDecimalFormat().format(f); } @@ -91,7 +91,7 @@ public String print(Object f) { private record MoneyPrinter(PrintSettings cfg) implements Printer { @Override - public String print(Object f) { + public String apply(Object f) { if (cfg.isPrettyPrint()) { return cfg.getDecimalFormat().format(f); @@ -104,7 +104,7 @@ public String print(Object f) { private record DatePrinter(PrintSettings cfg) implements Printer { @Override - public String print(Object f) { + public String apply(Object f) { Preconditions.checkArgument(f instanceof Number, "Expected an Number but got an '%s' with the value: %s".formatted(f.getClass().getName(), f)); final Number number = (Number) f; @@ -119,7 +119,7 @@ public DateRangePrinter(PrintSettings printSettings) { } @Override - public String print(Object f) { + public String apply(Object f) { Preconditions.checkArgument(f instanceof List, "Expected a List got %s (Type: %s, as string: %s)", f, f.getClass().getName(), f); Preconditions.checkArgument(((List) f).size() == 2, "Expected a list with 2 elements, one min, one max. The list was: %s ", f); @@ -131,13 +131,13 @@ public String print(Object f) { log.warn("Encountered incomplete range, treating it as an open range. Either min or max was null: {}", list); } // Compute minString first because we need it either way - final String minString = min == null || min == CDateRange.NEGATIVE_INFINITY ? "-∞" : datePrinter.print(min); + final String minString = min == null || min == CDateRange.NEGATIVE_INFINITY ? "-∞" : datePrinter.apply(min); if (cfg.isPrettyPrint() && min != null && min.equals(max)) { // If the min and max are the same we print it like a singe date, not a range (only in pretty printing) return minString; } - final String maxString = max == null || max == CDateRange.POSITIVE_INFINITY ? "+∞" : datePrinter.print(max); + final String maxString = max == null || max == CDateRange.POSITIVE_INFINITY ? "+∞" : datePrinter.apply(max); return minString + cfg.getDateRangeSeparator() + maxString; } @@ -152,7 +152,7 @@ public BooleanPrinter(PrintSettings cfg) { } @Override - public String print(Object f) { + public String apply(Object f) { if ((Boolean) f) { return trueVal; } @@ -168,7 +168,7 @@ public ListPrinter(Printer elementPrinter, PrintSettings cfg) { } @Override - public String print(Object f) { + public String apply(Object f) { // Jackson deserializes collections as lists instead of an array, if the type is not given Preconditions.checkArgument(f instanceof List, "Expected a List got %s (as String `%s` )".formatted(f.getClass().getName(), f)); @@ -176,7 +176,7 @@ public String print(Object f) { final StringJoiner joiner = listFormat.createListJoiner(); for (Object obj : (List) f) { - joiner.add(listFormat.escapeListElement(elementPrinter.print(obj).toString())); + joiner.add(listFormat.escapeListElement(elementPrinter.apply(obj).toString())); } return joiner.toString(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/IdentityPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/IdentityPrinter.java index 30773978a6..44b1c0208b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/IdentityPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/IdentityPrinter.java @@ -3,7 +3,7 @@ record IdentityPrinter() implements Printer { @Override - public Object print(Object value) { + public Object apply(Object value) { return value; } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java index 0b7475a49c..ddf070af7c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java @@ -54,12 +54,12 @@ public Printer getMoneyPrinter(PrintSettings printSettings) { private record ListPrinter(Printer elementPrinter) implements Printer { @Override - public Object print(Object value) { + public Object apply(Object value) { final List inList = (List) value; final List out = new ArrayList<>(inList.size()); for (Object elt : inList) { - out.add(elementPrinter.print(elt)); + out.add(elementPrinter.apply(elt)); } return out; @@ -69,7 +69,7 @@ public Object print(Object value) { private record DatePrinter() implements Printer { @Override - public Object print(Object value) { + public Object apply(Object value) { return CDate.toLocalDate(((Number) value).intValue()); } } @@ -78,7 +78,7 @@ private record DateRangePrinter() implements Printer { @Override - public Object print(Object f) { + public Object apply(Object f) { Preconditions.checkArgument(f instanceof List, "Expected a List got %s (Type: %s, as string: %s)", f, f.getClass().getName(), f); Preconditions.checkArgument(((List) f).size() == 2, "Expected a list with 2 elements, one min, one max. The list was: %s ", f); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java index b495f5cc44..d3aa344f8d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java @@ -21,7 +21,7 @@ public Printer getListPrinter(Printer elementPrinter, PrintSettings printSetting public record BooleanPrinter() implements Printer { @Override - public Object print(Object value) { + public Object apply(Object value) { return BooleanNode.valueOf(((Boolean) value)); } } @@ -34,7 +34,7 @@ public Printer getBooleanPrinter(PrintSettings printSettings) { public record IntegerPrinter() implements Printer { @Override - public Object print(Object value) { + public Object apply(Object value) { return IntNode.valueOf(((Integer) value)); } } @@ -47,7 +47,7 @@ public Printer getIntegerPrinter(PrintSettings printSettings) { public record NumericPrinter() implements Printer { @Override - public Object print(Object value) { + public Object apply(Object value) { return DecimalNode.valueOf(((BigDecimal) value)); } } @@ -60,8 +60,8 @@ public Printer getNumericPrinter(PrintSettings printSettings) { public record ToStringPrinter(Printer delegate) implements Printer { @Override - public Object print(Object value) { - return new TextNode(Objects.toString(delegate.print(value))); + public Object apply(Object value) { + return new TextNode(Objects.toString(delegate.apply(value))); } } @@ -79,7 +79,7 @@ public Printer getDateRangePrinter(PrintSettings printSettings) { public record StringPrinter() implements Printer { @Override - public Object print(Object value) { + public Object apply(Object value) { return new TextNode((String) value); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/LocalizedEnumPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/LocalizedEnumPrinter.java index aad7a38000..78b972494f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/LocalizedEnumPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/LocalizedEnumPrinter.java @@ -5,7 +5,7 @@ public record LocalizedEnumPrinter & LocalizedToString>(PrintSettings cfg, Class clazz) implements Printer { @Override - public String print(Object f) { + public String apply(Object f) { if (clazz.isInstance(f)) { return clazz.cast(f).toString(cfg.getLocale()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/MappedPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/MappedPrinter.java index 7f10733148..a87c80cdce 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/MappedPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/MappedPrinter.java @@ -5,7 +5,7 @@ public record MappedPrinter(InternToExternMapper mapper) implements Printer { @Override - public String print(Object f) { + public String apply(Object f) { return mapper.external(((String) f)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java index be61563ed7..01c6aadca5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java @@ -1,6 +1,8 @@ package com.bakdata.conquery.models.query.resultinfo.printers; +import java.util.function.Function; + @FunctionalInterface -public interface Printer { - Object print(Object value); +public interface Printer extends Function { + Object apply(Object value); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/SecondaryIdResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/SecondaryIdResultInfo.java index 346d236569..d0a5774aa8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/SecondaryIdResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/SecondaryIdResultInfo.java @@ -27,7 +27,7 @@ public SecondaryIdResultInfo(SecondaryIdDescription secondaryId, PrintSettings s printer = settings.getPrinterFactory().getStringPrinter(settings); } else { - printer = new ChainingPrinter(new MappedPrinter(secondaryId.getMapping()), settings.getPrinterFactory().getStringPrinter(settings)); + printer = new MappedPrinter(secondaryId.getMapping()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ResultStatistics.java b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ResultStatistics.java index 745d46afbf..a9648246b5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ResultStatistics.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ResultStatistics.java @@ -89,7 +89,7 @@ public static ResultStatistics collectResultStatistics(SingleTableResult managed statsCollector.consume(null); return; } - statsCollector.consume(printer.print(value)); + statsCollector.consume(printer.apply(value)); }); log.trace("DONE collecting values for {}, in {}", info, started); diff --git a/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java b/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java index a57542cbe5..55ecfca49a 100644 --- a/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java +++ b/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java @@ -25,7 +25,7 @@ class DefaultSqlCDateSetParserTest { @MethodSource("testToEpochDayRangeListProvider") public void testToEpochDayRangeList(String input, String expected, String message) { List> epochDayRangeList = parser.toEpochDayRangeList(input); - final String actual = (String) csvResultPrinters.printerFor(new ResultType.ListT(ResultType.Primitive.DATE_RANGE), PLAIN).print(epochDayRangeList); + final String actual = (String) csvResultPrinters.printerFor(new ResultType.ListT(ResultType.Primitive.DATE_RANGE), PLAIN).apply(epochDayRangeList); Assertions.assertEquals(expected, actual, message); } diff --git a/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java b/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java index b564b20f2c..c99a5d580c 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java @@ -97,12 +97,12 @@ public void testPrinting(PrintSettings cfg, ResultType type, Object value, Strin final Printer printer = info.getPrinter(); - assertThat(printer.print(value)).isEqualTo(expected); + assertThat(printer.apply(value)).isEqualTo(expected); final String str = Jackson.MAPPER.writeValueAsString(value); final Object copy = Jackson.MAPPER.readValue(str, Object.class); - assertThat(printer.print(copy)).isEqualTo(expected); + assertThat(printer.apply(copy)).isEqualTo(expected); } @ParameterizedTest(name = "{1}: {2}") @@ -111,11 +111,11 @@ public void testBinaryPrinting(PrintSettings cfg, ResultType type, Object value, ResultInfo info = info(type, cfg); final Printer printer = info.getPrinter(); - assertThat(printer.print(value)).isEqualTo(expected); + assertThat(printer.apply(value)).isEqualTo(expected); final byte[] bytes = Jackson.BINARY_MAPPER.writeValueAsBytes(value); final Object copy = Jackson.BINARY_MAPPER.readValue(bytes, Object.class); - assertThat(printer.print(copy)).isEqualTo(expected); + assertThat(printer.apply(copy)).isEqualTo(expected); } } From ce5f4073ee4f0382e47ea47539c90434966521b6 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 29 Aug 2024 17:14:55 +0200 Subject: [PATCH 29/54] delay binding of PrintSettings to createPrinter. Remove PrintSettings from ResultInfo, move all usages to usage sites. --- .../com/bakdata/conquery/ResultHeaders.java | 89 +++++++++++++------ .../conquery/apiv1/QueryProcessor.java | 14 +-- .../apiv1/query/ArrayConceptQuery.java | 7 +- .../conquery/apiv1/query/CQElement.java | 3 +- .../bakdata/conquery/apiv1/query/CQYes.java | 3 +- .../conquery/apiv1/query/ConceptQuery.java | 7 +- .../bakdata/conquery/apiv1/query/Query.java | 3 +- .../apiv1/query/SecondaryIdQuery.java | 7 +- .../apiv1/query/TableExportQuery.java | 41 ++++----- .../apiv1/query/concept/specific/CQAnd.java | 19 ++-- .../query/concept/specific/CQConcept.java | 7 +- .../concept/specific/CQDateRestriction.java | 5 +- .../query/concept/specific/CQNegation.java | 5 +- .../apiv1/query/concept/specific/CQOr.java | 19 ++-- .../query/concept/specific/CQReusedQuery.java | 5 +- .../concept/specific/external/CQExternal.java | 5 +- .../temporal/CQAbstractTemporalQuery.java | 7 +- .../io/result/arrow/ArrowRenderer.java | 28 +++--- .../conquery/io/result/arrow/ArrowUtil.java | 11 +-- .../io/result/arrow/ResultArrowProcessor.java | 4 +- .../conquery/io/result/csv/CsvRenderer.java | 28 ++++-- .../io/result/csv/ResultCsvProcessor.java | 2 +- .../io/result/excel/ExcelRenderer.java | 28 +++--- .../io/result/excel/ResultExcelProcessor.java | 2 +- .../parquet/EntityResultWriteSupport.java | 15 ++-- .../parquet/ResultParquetProcessor.java | 4 +- .../models/config/ExcelResultProvider.java | 4 +- .../models/config/IdColumnConfig.java | 26 +++--- .../datasets/concepts/select/Select.java | 4 +- .../select/concept/ConceptColumnSelect.java | 6 +- .../select/connector/SingleColumnSelect.java | 7 +- .../specific/MappableSingleColumnSelect.java | 6 +- .../forms/managed/AbsoluteFormQuery.java | 13 ++- .../models/forms/managed/EntityDateQuery.java | 11 ++- .../forms/managed/ManagedInternalForm.java | 5 +- .../forms/managed/RelativeFormQuery.java | 15 ++-- .../conquery/models/query/ManagedQuery.java | 4 +- .../models/query/SingleTableResult.java | 11 ++- .../query/preview/EntityPreviewExecution.java | 22 ++--- .../query/resultinfo/ColumnResultInfo.java | 27 ++++-- .../query/resultinfo/ExternalResultInfo.java | 23 +++-- .../resultinfo/FixedLabelResultInfo.java | 31 +++---- .../models/query/resultinfo/ResultInfo.java | 26 ++---- .../query/resultinfo/SelectResultInfo.java | 18 ++-- .../models/query/resultinfo/UniqueNamer.java | 4 +- .../printers/SecondaryIdResultInfo.java | 28 +++--- .../query/statistics/ResultStatistics.java | 4 +- .../query/AbsoluteFormQueryConverter.java | 2 +- .../query/ConceptQueryConverter.java | 2 +- .../query/EntityDateQueryConverter.java | 2 +- .../query/RelativFormQueryConverter.java | 2 +- .../query/SecondaryIdQueryConverter.java | 2 +- .../query/TableExportQueryConverter.java | 2 +- .../api/StoredQueriesProcessorTest.java | 9 +- .../json/AbstractQueryEngineTest.java | 2 +- .../conquery/integration/json/FormTest.java | 12 +-- .../conquery/io/result/ResultTestUtil.java | 9 +- .../arrow/ArrowResultGenerationTest.java | 26 +++--- .../result/csv/CsvResultGenerationTest.java | 8 +- .../result/excel/ExcelResultRenderTest.java | 16 ++-- .../parquet/ParquetResultGenerationTest.java | 8 +- .../models/query/DefaultColumnNameTest.java | 4 +- .../conquery/models/query/UniqueNameTest.java | 16 ++-- .../conquery/models/types/ResultTypeTest.java | 19 ++-- 64 files changed, 421 insertions(+), 383 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java b/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java index 2edd68e2ee..e71021431f 100644 --- a/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java +++ b/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java @@ -17,64 +17,95 @@ @UtilityClass public class ResultHeaders { - public static ResultInfo datesInfo(PrintSettings settings) { - final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()).dates(); + public static ResultInfo datesInfo() { final ResultType.ListT type = new ResultType.ListT<>(ResultType.Primitive.DATE_RANGE); - return new FixedLabelResultInfo(label, label, type, Set.of(new SemanticType.EventDateT()), settings); + return new FixedLabelResultInfo(type, Set.of(new SemanticType.EventDateT())){ + @Override + public String userColumnName(PrintSettings printSettings) { + return C10nCache.getLocalized(ResultHeadersC10n.class, printSettings.getLocale()).dates(); + } + }; } - public static ResultInfo historyDatesInfo(PrintSettings settings) { - final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()).dates(); + public static ResultInfo historyDatesInfo() { final ResultType.ListT type = new ResultType.ListT<>(ResultType.Primitive.DATE_RANGE); - return new FixedLabelResultInfo(label, label, type, Set.of(new SemanticType.EventDateT(), new SemanticType.GroupT()), settings); + return new FixedLabelResultInfo(type, Set.of(new SemanticType.EventDateT(), new SemanticType.GroupT())) { + @Override + public String userColumnName(PrintSettings printSettings) { + return C10nCache.getLocalized(ResultHeadersC10n.class, printSettings.getLocale()).dates(); + } + }; } - public static ResultInfo sourceInfo(PrintSettings settings) { - final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()).source(); - - return new FixedLabelResultInfo(label, label, ResultType.Primitive.STRING, Set.of(new SemanticType.SourcesT(), new SemanticType.CategoricalT(), new SemanticType.GroupT()), settings); + public static ResultInfo sourceInfo() { + return new FixedLabelResultInfo(ResultType.Primitive.STRING, Set.of(new SemanticType.SourcesT(), new SemanticType.CategoricalT(), new SemanticType.GroupT())) { + @Override + public String userColumnName(PrintSettings printSettings) { + return C10nCache.getLocalized(ResultHeadersC10n.class, printSettings.getLocale()).source(); + } + }; } - public static ResultInfo formContextInfo(PrintSettings settings) { - final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()).index(); + public static ResultInfo formContextInfo() { - return new FixedLabelResultInfo(label, label, ResultType.Primitive.INTEGER, Set.of(), settings); + return new FixedLabelResultInfo(ResultType.Primitive.INTEGER, Set.of()) { + @Override + public String userColumnName(PrintSettings printSettings) { + return C10nCache.getLocalized(ResultHeadersC10n.class, printSettings.getLocale()).index(); + } + }; } - public static ResultInfo formDateRangeInfo(PrintSettings settings) { - final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()).dateRange(); + public static ResultInfo formDateRangeInfo() { - return new FixedLabelResultInfo(label, label, ResultType.Primitive.DATE_RANGE, Set.of(), settings); + return new FixedLabelResultInfo(ResultType.Primitive.DATE_RANGE, Set.of()) { + @Override + public String userColumnName(PrintSettings printSettings) { + return C10nCache.getLocalized(ResultHeadersC10n.class, printSettings.getLocale()).dateRange(); + } + }; } - public static ResultInfo formResolutionInfo(PrintSettings settings) { - final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()).resolution(); + public static ResultInfo formResolutionInfo() { + + return new FixedLabelResultInfo(ResultType.Primitive.STRING, Set.of()) { + @Override + public Printer createPrinter(PrintSettings printSettings) { + return new LocalizedEnumPrinter<>(printSettings, Resolution.class); + } - return new FixedLabelResultInfo(label, label, ResultType.Primitive.STRING, Set.of(), settings) { @Override - public Printer getPrinter() { - return new LocalizedEnumPrinter<>(settings, Resolution.class); + public String userColumnName(PrintSettings printSettings) { + return C10nCache.getLocalized(ResultHeadersC10n.class, printSettings.getLocale()).resolution(); } }; } - public static ResultInfo formEventDateInfo(PrintSettings settings) { - final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()).eventDate(); + public static ResultInfo formEventDateInfo() { - return new FixedLabelResultInfo(label, label, ResultType.Primitive.DATE, Set.of(), settings); + return new FixedLabelResultInfo(ResultType.Primitive.DATE, Set.of()) { + @Override + public String userColumnName(PrintSettings printSettings) { + return C10nCache.getLocalized(ResultHeadersC10n.class, printSettings.getLocale()).eventDate(); + } + }; } - public static ResultInfo formObservationScopeInfo(PrintSettings settings) { - final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()).observationScope(); + public static ResultInfo formObservationScopeInfo() { + + return new FixedLabelResultInfo(ResultType.Primitive.STRING, Set.of()) { + @Override + public Printer createPrinter(PrintSettings printSettings) { + return new LocalizedEnumPrinter<>(printSettings, FeatureGroup.class); + } - return new FixedLabelResultInfo(label, label, ResultType.Primitive.STRING, Set.of(), settings) { @Override - public Printer getPrinter() { - return new LocalizedEnumPrinter<>(settings, FeatureGroup.class); + public String userColumnName(PrintSettings printSettings) { + return C10nCache.getLocalized(ResultHeadersC10n.class, printSettings.getLocale()).observationScope(); } }; } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index 0458166384..2a76bd91c5 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -21,12 +21,6 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; -import jakarta.inject.Inject; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.validation.Validator; -import jakarta.ws.rs.BadRequestException; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.apiv1.execution.ExecutionStatus; import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; @@ -90,6 +84,12 @@ import com.bakdata.conquery.util.QueryUtils; import com.bakdata.conquery.util.QueryUtils.NamespacedIdentifiableCollector; import com.bakdata.conquery.util.io.IdColumnUtil; +import jakarta.inject.Inject; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.validation.Validator; +import jakarta.ws.rs.BadRequestException; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.UriBuilder; import lombok.AllArgsConstructor; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -582,7 +582,7 @@ public ResultStatistics getResultStatistics(SingleTableResult managedQuery) { new PrintSettings(true, locale, managedQuery.getNamespace(), config, null, null, decimalFormat, integerFormat, new JavaResultPrinters()); final UniqueNamer uniqueNamer = new UniqueNamer(printSettings); - final List resultInfos = managedQuery.getResultInfos(printSettings); + final List resultInfos = managedQuery.getResultInfos(); final Optional dateInfo = diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/ArrayConceptQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/ArrayConceptQuery.java index 32e7871b23..057b7ac86c 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/ArrayConceptQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/ArrayConceptQuery.java @@ -11,7 +11,6 @@ import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.DateAggregationMode; -import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.Visitable; @@ -93,9 +92,9 @@ public void collectRequiredQueries(Set requiredQueries) { } @Override - public List getResultInfos(PrintSettings printSettings) { + public List getResultInfos() { final List resultInfos = new ArrayList<>(); - ResultInfo dateInfo = ResultHeaders.datesInfo(printSettings); + ResultInfo dateInfo = ResultHeaders.datesInfo(); if(getResolvedDateAggregationMode() != DateAggregationMode.NONE){ // Add one DateInfo for the whole Query @@ -103,7 +102,7 @@ public List getResultInfos(PrintSettings printSettings) { } int lastIndex = resultInfos.size(); - childQueries.forEach(q -> resultInfos.addAll(q.getResultInfos(printSettings))); + childQueries.forEach(q -> resultInfos.addAll(q.getResultInfos())); if(!resultInfos.isEmpty()) { // Remove DateInfo from each childQuery diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQElement.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQElement.java index cf309dcf90..61806adfb4 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQElement.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQElement.java @@ -11,7 +11,6 @@ import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; -import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -79,7 +78,7 @@ public final Set collectRequiredQueries() { public abstract void collectRequiredQueries(Set requiredQueries) ; @JsonIgnore - public abstract List getResultInfos(PrintSettings settings); + public abstract List getResultInfos(); public void visit(Consumer visitor) { visitor.accept(this); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQYes.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQYes.java index 2e3e34a727..8bdc6a00d6 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQYes.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQYes.java @@ -6,7 +6,6 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; -import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; @@ -33,7 +32,7 @@ public void collectRequiredQueries(Set requiredQueries) { } @Override - public List getResultInfos(PrintSettings settings) { + public List getResultInfos() { return Collections.emptyList(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/ConceptQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/ConceptQuery.java index 6f028f3684..0f3393f006 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/ConceptQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/ConceptQuery.java @@ -10,7 +10,6 @@ import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.DateAggregationMode; -import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -80,16 +79,16 @@ public void resolve(QueryResolveContext context) { } @Override - public List getResultInfos(PrintSettings printSettings) { + public List getResultInfos() { Preconditions.checkNotNull(resolvedDateAggregationMode); final List resultInfos = new ArrayList<>(); if (resolvedDateAggregationMode != DateAggregationMode.NONE) { - resultInfos.add(ResultHeaders.datesInfo(printSettings)); + resultInfos.add(ResultHeaders.datesInfo()); } - resultInfos.addAll(root.getResultInfos(printSettings)); + resultInfos.addAll(root.getResultInfos()); return resultInfos; } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/Query.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/Query.java index 483c71a545..cde81b3cc1 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/Query.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/Query.java @@ -12,7 +12,6 @@ import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.ManagedQuery; -import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.queryplan.QueryPlan; @@ -38,7 +37,7 @@ public Set collectRequiredQueries() { } @JsonIgnore - public abstract List getResultInfos(PrintSettings printSettings); + public abstract List getResultInfos(); @Override public ManagedQuery toManagedExecution(User user, Dataset submittedDataset, MetaStorage storage) { diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java index 85ba3ddc9a..b294f984ac 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java @@ -18,7 +18,6 @@ import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.DateAggregationMode; -import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -125,12 +124,12 @@ public void resolve(final QueryResolveContext context) { } @Override - public List getResultInfos(PrintSettings printSettings) { + public List getResultInfos() { final List resultInfos = new ArrayList<>(); - resultInfos.add(new SecondaryIdResultInfo(secondaryId, printSettings)); + resultInfos.add(new SecondaryIdResultInfo(secondaryId)); - resultInfos.addAll(query.getResultInfos(printSettings)); + resultInfos.addAll(query.getResultInfos()); return resultInfos; } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java index a01343d5d1..6ae8fef610 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java @@ -5,7 +5,6 @@ import java.util.Collection; import java.util.Comparator; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -31,7 +30,6 @@ import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.DateAggregationMode; -import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -41,8 +39,6 @@ import com.bakdata.conquery.models.query.queryplan.TableExportQueryPlan; import com.bakdata.conquery.models.query.resultinfo.ColumnResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.ConceptIdPrinter; -import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.query.resultinfo.printers.SecondaryIdResultInfo; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; @@ -169,8 +165,8 @@ public void resolve(QueryResolveContext context) { } @Override - public List getResultInfos(PrintSettings printSettings) { - return createResultInfos(conceptColumns, printSettings); + public List getResultInfos() { + return createResultInfos(conceptColumns); } private Map calculateSecondaryIdPositions(AtomicInteger currentPosition) { @@ -223,21 +219,21 @@ private static Map calculateColumnPositions(AtomicInteger curre return positions; } - private List createResultInfos(Set conceptColumns, PrintSettings printSettings) { + private List createResultInfos(Set conceptColumns) { final int size = positions.values().stream().mapToInt(i -> i).max().getAsInt() + 1; final ResultInfo[] infos = new ResultInfo[size]; - infos[0] = ResultHeaders.historyDatesInfo(printSettings); - infos[1] = ResultHeaders.sourceInfo(printSettings); + infos[0] = ResultHeaders.historyDatesInfo(); + infos[1] = ResultHeaders.sourceInfo(); for (Map.Entry e : secondaryIdPositions.entrySet()) { final SecondaryIdDescription desc = e.getKey(); final Integer pos = e.getValue(); - infos[pos] = new SecondaryIdResultInfo(desc, printSettings); + infos[pos] = new SecondaryIdResultInfo(desc); } @@ -264,30 +260,27 @@ private List createResultInfos(Set conceptColumns, PrintSett continue; } - final Set semantics = new HashSet<>(); - - ResultType resultType = ResultType.resolveResultType(column.getType()); - Printer printer = printSettings.getPrinterFactory().printerFor(resultType, printSettings); - + final ResultInfo columnResultInfo; if (connectorColumns.containsKey(column)) { - // Additionally, Concept Columns are returned as ConceptElementId, when rawConceptColumns is not set. - final Concept concept = connectorColumns.get(column).getConcept(); + // Additionally, Concept Columns are returned as ConceptElementId, when rawConceptColumns is not set. + columnResultInfo = new ColumnResultInfo(column, ResultType.Primitive.STRING, column.getDescription(), isRawConceptValues() ? null : concept); + // Columns that are used to build concepts are marked as ConceptColumn. - semantics.add(new SemanticType.ConceptColumnT(concept)); + columnResultInfo.addSemantics(new SemanticType.ConceptColumnT(concept)); - if (!isRawConceptValues()) { - resultType = ResultType.Primitive.STRING; - printer = new ConceptIdPrinter(concept, printSettings); - } + infos[position] = columnResultInfo; } else { // If it's not a connector column, we just link to the source column. - semantics.add(new SemanticType.ColumnT(column)); + columnResultInfo = new ColumnResultInfo(column, ResultType.resolveResultType(column.getType()), column.getDescription(), null); + columnResultInfo.addSemantics(new SemanticType.ColumnT(column)); } - infos[position] = new ColumnResultInfo(column, resultType, semantics, printer, column.getDescription(), printSettings); + infos[position] = columnResultInfo; + + } return List.of(infos); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java index d721f57e2e..bce7cdd346 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java @@ -112,17 +112,24 @@ private DateAggregationAction determineDateAction(QueryResolveContext context) { } @Override - public List getResultInfos(PrintSettings settings) { + public List getResultInfos() { final List resultInfos = new ArrayList<>(); for (CQElement c : children) { - resultInfos.addAll(c.getResultInfos(settings)); + resultInfos.addAll(c.getResultInfos()); } if (createExists()) { - final String userOrDefaultLabel = getUserOrDefaultLabel(settings.getLocale()); - final String defaultLabel = defaultLabel(settings.getLocale()); - - resultInfos.add(new FixedLabelResultInfo(userOrDefaultLabel, defaultLabel, ResultType.Primitive.BOOLEAN, Set.of(), settings)); + resultInfos.add(new FixedLabelResultInfo(ResultType.Primitive.BOOLEAN, Set.of()) { + @Override + public String userColumnName(PrintSettings printSettings) { + return getUserOrDefaultLabel(printSettings.getLocale()); + } + + @Override + public String defaultColumnName(PrintSettings printSettings) { + return defaultLabel(printSettings.getLocale()); + } + }); } return resultInfos; } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java index 538a987d93..be0e5814e9 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java @@ -24,7 +24,6 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.DateAggregationMode; import com.bakdata.conquery.models.query.NamespacedIdentifiableHolding; -import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -286,16 +285,16 @@ private ValidityDate selectValidityDate(CQTable table) { } @Override - public List getResultInfos(PrintSettings settings) { + public List getResultInfos() { final List resultInfos = new ArrayList<>(); for (Select select : selects) { - resultInfos.add(select.getResultInfo(this, settings)); + resultInfos.add(select.getResultInfo(this)); } for (CQTable table : tables) { for (Select sel : table.getSelects()) { - resultInfos.add(sel.getResultInfo(this, settings)); + resultInfos.add(sel.getResultInfo(this)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQDateRestriction.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQDateRestriction.java index 251f5c779b..6f78e0be8f 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQDateRestriction.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQDateRestriction.java @@ -12,7 +12,6 @@ import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; -import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -60,8 +59,8 @@ public void resolve(QueryResolveContext context) { } @Override - public List getResultInfos(PrintSettings settings) { - return child.getResultInfos(settings); + public List getResultInfos() { + return child.getResultInfos(); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQNegation.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQNegation.java index 2ae44af793..57fbef50f4 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQNegation.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQNegation.java @@ -8,7 +8,6 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; -import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.Visitable; @@ -67,8 +66,8 @@ private DateAggregationAction determineDateAction(QueryResolveContext context) { } @Override - public List getResultInfos(PrintSettings settings) { - return child.getResultInfos(settings); + public List getResultInfos() { + return child.getResultInfos(); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java index 427cae2107..882ead8a2b 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java @@ -117,17 +117,24 @@ private DateAggregationAction determineDateAction(QueryResolveContext context) { } @Override - public List getResultInfos(PrintSettings settings) { + public List getResultInfos() { List resultInfos = new ArrayList<>(); for (CQElement c : children) { - resultInfos.addAll(c.getResultInfos(settings)); + resultInfos.addAll(c.getResultInfos()); } if (createExists()) { - final String userOrDefaultLabel = getUserOrDefaultLabel(settings.getLocale()); - final String defaultLabel = defaultLabel(settings.getLocale()); - - resultInfos.add(new FixedLabelResultInfo(userOrDefaultLabel, defaultLabel, ResultType.Primitive.BOOLEAN, Set.of(), settings)); + resultInfos.add(new FixedLabelResultInfo(ResultType.Primitive.BOOLEAN, Set.of()) { + @Override + public String userColumnName(PrintSettings printSettings) { + return getUserOrDefaultLabel(printSettings.getLocale()); + } + + @Override + public String defaultColumnName(PrintSettings printSettings) { + return defaultLabel(printSettings.getLocale()); + } + }); } return resultInfos; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQReusedQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQReusedQuery.java index 7162f8f4fb..496fa51120 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQReusedQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQReusedQuery.java @@ -13,7 +13,6 @@ import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.ManagedQuery; -import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -93,8 +92,8 @@ public void visit(Consumer visitor) { } @Override - public List getResultInfos(PrintSettings settings) { - return resolvedQuery.getReusableComponents().getResultInfos(settings); + public List getResultInfos() { + return resolvedQuery.getReusableComponents().getResultInfos(); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java index d6d23de78d..7815d7fe03 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -19,7 +19,6 @@ import com.bakdata.conquery.models.config.IdColumnConfig; import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; -import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -215,7 +214,7 @@ public RequiredEntities collectRequiredEntities(QueryExecutionContext context) { } @Override - public List getResultInfos(PrintSettings settings) { + public List getResultInfos() { if (extra == null) { return Collections.emptyList(); } @@ -228,7 +227,7 @@ public List getResultInfos(PrintSettings settings) { final String column = headers[col]; final ResultType type = onlySingles ? ResultType.Primitive.STRING : new ResultType.ListT<>(ResultType.Primitive.STRING); - resultInfos.add(new ExternalResultInfo(column, type, settings)); + resultInfos.add(new ExternalResultInfo(column, type)); } return resultInfos; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/temporal/CQAbstractTemporalQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/temporal/CQAbstractTemporalQuery.java index 33a720c5ea..63e33a0616 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/temporal/CQAbstractTemporalQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/temporal/CQAbstractTemporalQuery.java @@ -8,7 +8,6 @@ import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.DateAggregationMode; -import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -81,10 +80,10 @@ public void resolve(QueryResolveContext context) { } @Override - public List getResultInfos(PrintSettings settings) { + public List getResultInfos() { List resultInfos = new ArrayList<>(); - resultInfos.addAll(index.getChild().getResultInfos(settings)); - resultInfos.addAll(preceding.getChild().getResultInfos(settings)); + resultInfos.addAll(index.getChild().getResultInfos()); + resultInfos.addAll(preceding.getChild().getResultInfos()); return resultInfos; } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java index 3734f3ce13..4be9e00e96 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java @@ -36,6 +36,7 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.Text; +import org.jetbrains.annotations.NotNull; @Slf4j public class ArrowRenderer { @@ -48,7 +49,7 @@ public static void renderToStream( List resultInfo, Stream results) throws IOException { - final List fields = ArrowUtil.generateFields(idHeaders, resultInfo, new UniqueNamer(printSettings)); + final List fields = ArrowUtil.generateFields(idHeaders, resultInfo, new UniqueNamer(printSettings), printSettings); final VectorSchemaRoot root = VectorSchemaRoot.create(new Schema(fields, null), ROOT_ALLOCATOR); // Build separate pipelines for id and value, as they have different sources but the same target @@ -58,14 +59,13 @@ public static void renderToStream( final List printers = new ArrayList<>(); for (ResultInfo header : idHeaders) { - printers.add(header.getPrinter()); + printers.add(header.createPrinter(printSettings)); } for (ResultInfo info : resultInfo) { - printers.add(info.getPrinter()); + printers.add(info.createPrinter(printSettings)); } - // Write the data try (ArrowWriter writer = writerProducer.apply(root)) { write(writer, root, idWriters, valueWriter, printSettings.getIdMapper(), printers, results, arrowConfig.getBatchSize()); @@ -95,13 +95,7 @@ public static void write( while (resultIterator.hasNext()) { final EntityResult cer = resultIterator.next(); - final String[] externalId = idMapper.map(cer).getExternalId(); - - final Object[] printedExternalId = new String[externalId.length]; - - for (int index = 0; index < idWriters.length; index++) { - printedExternalId[index] = printers.get(index).apply(externalId[index]); - } + final Object[] printedExternalId = getPrintedExternalId(idWriters, idMapper, printers, cer); for (Object[] line : cer.listResultLines()) { Preconditions.checkState( @@ -149,6 +143,18 @@ public static void write( writer.end(); } + @NotNull + private static Object[] getPrintedExternalId(RowConsumer[] idWriters, PrintIdMapper idMapper, List printers, EntityResult cer) { + final String[] externalId = idMapper.map(cer).getExternalId(); + + final Object[] printedExternalId = new String[externalId.length]; + + for (int index = 0; index < idWriters.length; index++) { + printedExternalId[index] = printers.get(index).apply(externalId[index]); + } + return printedExternalId; + } + private static RowConsumer intVectorFiller(IntVector vector) { return (rowNumber, valueRaw) -> { if (valueRaw == null) { diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java index e4c02a301f..67d1977cca 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java @@ -3,6 +3,7 @@ import java.util.ArrayList; import java.util.List; +import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; import com.bakdata.conquery.models.types.ResultType; @@ -80,20 +81,20 @@ private static Field listField(@NonNull String uniqueName, ResultType type) { return new Field(uniqueName, FieldType.nullable(ArrowType.List.INSTANCE), List.of(nestedField)); } - public static List generateFields(@NonNull List info, UniqueNamer collector) { + public static List generateFields(@NonNull List info, UniqueNamer collector, PrintSettings printSettings) { return info.stream() - .map(i -> fieldFor(i.getType(), collector.getUniqueName(i))) + .map(i -> fieldFor(i.getType(), collector.getUniqueName(i, printSettings))) .toList(); } @NotNull - public static List generateFields(List idHeaders, List resultInfo, UniqueNamer uniqueNamer) { + public static List generateFields(List idHeaders, List resultInfo, UniqueNamer uniqueNamer, PrintSettings printSettings) { // Combine id and value Fields to one vector to build a schema List fields = new ArrayList<>(); - fields.addAll(generateFields(idHeaders, uniqueNamer)); - fields.addAll(generateFields(resultInfo, uniqueNamer)); + fields.addAll(generateFields(idHeaders, uniqueNamer, printSettings)); + fields.addAll(generateFields(resultInfo, uniqueNamer, printSettings)); return fields; } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java index dd73ddb763..a00bbb3649 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java @@ -100,8 +100,8 @@ public static Response getArrow // Collect ResultInfos for id columns and result columns - final List resultInfosId = config.getIdColumns().getIdResultInfos(settings); - final List resultInfosExec = exec.getResultInfos(settings); + final List resultInfosId = config.getIdColumns().getIdResultInfos(); + final List resultInfosExec = exec.getResultInfos(); StreamingOutput out = output -> { try { diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java index 295df2b14a..82e8da8692 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java @@ -9,6 +9,7 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.query.results.EntityResult; import com.univocity.parsers.csv.CsvWriter; import lombok.RequiredArgsConstructor; @@ -22,36 +23,45 @@ public class CsvRenderer { private final CsvWriter writer; private final PrintSettings cfg; - public void toCSV(List idHeaders, List infos, Stream resultStream) { + public void toCSV(List idHeaders, List infos, Stream resultStream, PrintSettings printSettings) { UniqueNamer uniqNamer = new UniqueNamer(cfg); - final String[] headers = Stream.concat(idHeaders.stream(), infos.stream()).map(uniqNamer::getUniqueName).toArray(String[]::new); + final String[] headers = Stream.concat(idHeaders.stream(), infos.stream()).map(info -> uniqNamer.getUniqueName(info, printSettings)).toArray(String[]::new); writer.writeHeaders(headers); - createCSVBody(cfg, infos, resultStream); + createCSVBody(cfg, infos, resultStream, printSettings); } - private void createCSVBody(PrintSettings cfg, List infos, Stream results) { + private void createCSVBody(PrintSettings cfg, List infos, Stream results, PrintSettings printSettings) { + final Printer[] printers = infos.stream().map(info -> info.createPrinter(printSettings)).toArray(Printer[]::new); + results.map(result -> Pair.of(cfg.getIdMapper().map(result), result)) .sorted(Map.Entry.comparingByKey()) .forEach(res -> res .getValue() .streamValues() - .forEach(result -> printLine(res.getKey(), infos, result))); + .forEach(result -> printLine(res.getKey(), printers, result))); } - public void printLine(EntityPrintId entity, List infos, Object[] value) { + public void printLine(EntityPrintId entity, Printer[] printers, Object[] values) { // Cast here to Object[] so it is clear to intellij that the varargs call is intended writer.addValues((Object[]) entity.getExternalId()); try { - for (int i = 0; i < infos.size(); i++) { - writer.addValue(infos.get(i).printNullable(value[i])); + for (int i = 0; i < printers.length; i++) { + final Object value = values[i]; + + if (value == null) { + writer.addValue(""); + continue; + } + + writer.addValue(printers[i].apply(value)); } } catch (Exception e) { - throw new IllegalStateException("Unable to print line " + Arrays.deepToString(value), e); + throw new IllegalStateException("Unable to print line " + Arrays.deepToString(values), e); } writer.writeValuesToRow(); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java index 3def872f78..655061f79b 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java @@ -63,7 +63,7 @@ public Response createResult(Su final StreamingOutput out = os -> { try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(os, charset))) { final CsvRenderer renderer = new CsvRenderer(config.getCsv().createWriter(writer), settings); - renderer.toCSV(config.getIdColumns().getIdResultInfos(settings), exec.getResultInfos(settings), exec.streamResults(limit)); + renderer.toCSV(config.getIdColumns().getIdResultInfos(), exec.getResultInfos(), exec.streamResults(limit), settings); } catch (EofException e) { log.trace("User canceled download"); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java index f503ffba83..be4e868caa 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java @@ -44,7 +44,7 @@ public class ExcelRenderer { public static final int MAX_LINES = 1_048_576; - private static TypeWriter writer(ResultType type, Printer printer, PrintSettings settings1) { + private static TypeWriter writer(ResultType type, Printer printer, PrintSettings settings) { if(!(type instanceof ResultType.Primitive)){ //Excel cannot handle complex types so we just toString them. return (value, cell, styles) -> writeStringCell(cell, value, printer); @@ -53,7 +53,7 @@ private static TypeWriter writer(ResultType type, Printer printer, PrintSettings return switch (((ResultType.Primitive) type)) { case BOOLEAN -> (value, cell, styles) -> writeBooleanCell(value, cell, printer); case INTEGER -> (value, cell, styles) -> writeIntegerCell(value, cell, printer, styles); - case MONEY -> (value, cell, styles1) -> writeMoneyCell(value, cell, settings1, styles1); + case MONEY -> (value, cell, styles1) -> writeMoneyCell(value, cell, settings, styles1); case NUMERIC -> (value, cell, styles) -> writeNumericCell(value, cell, printer, styles); case DATE -> (value, cell, styles1) -> writeDateCell(value, cell, styles1, printer); default -> (value, cell, styles) -> writeStringCell(cell, value, printer); @@ -65,15 +65,15 @@ private static TypeWriter writer(ResultType type, Printer printer, PrintSettings private final SXSSFWorkbook workbook; private final ExcelConfig config; - private final PrintSettings cfg; + private final PrintSettings settings; private final ImmutableMap styles; - public ExcelRenderer(ExcelConfig config, PrintSettings cfg) { + public ExcelRenderer(ExcelConfig config, PrintSettings settings) { workbook = new SXSSFWorkbook(); this.config = config; - styles = config.generateStyles(workbook, cfg); - this.cfg = cfg; + styles = config.generateStyles(workbook, settings); + this.settings = settings; } @FunctionalInterface @@ -83,7 +83,7 @@ private interface TypeWriter { public void renderToStream(List idHeaders, E exec, OutputStream outputStream, OptionalLong limit, PrintSettings printSettings) throws IOException { - final List resultInfosExec = exec.getResultInfos(printSettings); + final List resultInfosExec = exec.getResultInfos(); setMetaData(exec); @@ -94,7 +94,7 @@ public void renderToStream(List // Create a table environment inside the excel sheet final XSSFTable table = createTableEnvironment(exec, sheet); - writeHeader(sheet, idHeaders, resultInfosExec, table); + writeHeader(sheet, idHeaders, resultInfosExec, table, printSettings); final int writtenLines = writeBody(sheet, resultInfosExec, exec.streamResults(OptionalLong.of(limit.orElse(MAX_LINES)))); @@ -171,11 +171,11 @@ private void writeHeader( SXSSFSheet sheet, List idHeaders, List infos, - XSSFTable table) { + XSSFTable table, PrintSettings printSettings) { final CTTableColumns columns = table.getCTTable().addNewTableColumns(); columns.setCount(idHeaders.size() + infos.size()); - final UniqueNamer uniqueNamer = new UniqueNamer(cfg); + final UniqueNamer uniqueNamer = new UniqueNamer(settings); { final Row header = sheet.createRow(0); @@ -185,7 +185,7 @@ private void writeHeader( final CTTableColumn column = columns.addNewTableColumn(); // Table column ids MUST be set and MUST start at 1, excel will fail otherwise column.setId(currentColumn + 1); - final String uniqueName = uniqueNamer.getUniqueName(idHeader); + final String uniqueName = uniqueNamer.getUniqueName(idHeader, printSettings); column.setName(uniqueName); final Cell headerCell = header.createCell(currentColumn); @@ -199,7 +199,7 @@ private void writeHeader( } for (ResultInfo info : infos) { - final String columnName = uniqueNamer.getUniqueName(info); + final String columnName = uniqueNamer.getUniqueName(info, printSettings); final CTTableColumn column = columns.addNewTableColumn(); column.setId(currentColumn + 1); column.setName(columnName); @@ -221,7 +221,7 @@ private int writeBody( // Row 0 is the Header the data starts at 1 final AtomicInteger currentRow = new AtomicInteger(1); - final int writtenLines = resultLines.mapToInt(l -> writeRowsForEntity(infos, l, currentRow, cfg, sheet)).sum(); + final int writtenLines = resultLines.mapToInt(l -> writeRowsForEntity(infos, l, currentRow, settings, sheet)).sum(); // The result was shorter than the number of rows to track, so we auto size here explicitly if (writtenLines < config.getLastRowToAutosize()) { @@ -237,7 +237,7 @@ private int writeBody( private int writeRowsForEntity(List infos, EntityResult internalRow, final AtomicInteger currentRow, PrintSettings settings, SXSSFSheet sheet) { final String[] ids = settings.getIdMapper().map(internalRow).getExternalId(); - final TypeWriter[] writers = infos.stream().map(info -> writer(info.getType(), info.getPrinter(), settings)).toArray(TypeWriter[]::new); + final TypeWriter[] writers = infos.stream().map(info -> writer(info.getType(), info.createPrinter(settings), settings)).toArray(TypeWriter[]::new); int writtenLines = 0; diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java index 9141098eab..d7d393d726 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java @@ -58,7 +58,7 @@ public Response createResult(Su final ExcelRenderer excelRenderer = new ExcelRenderer(excelConfig, settings); final StreamingOutput out = output -> { - excelRenderer.renderToStream(conqueryConfig.getIdColumns().getIdResultInfos(settings), exec, output, limit, settings); + excelRenderer.renderToStream(conqueryConfig.getIdColumns().getIdResultInfos(), exec, output, limit, settings); log.trace("FINISHED downloading {}", exec.getId()); }; diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java index 6181231254..a430e2deb7 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java @@ -50,14 +50,15 @@ public class EntityResultWriteSupport extends WriteSupport { * @param idHeaders {@link ResultInfo} for the Ids * @param resultValueInfos {@link ResultInfo} for the result values * @param uniqueNamer A column namer for the fields in the schema + * @param printSettings1 * @return the parquet schema */ - public static MessageType generateSchema(List idHeaders, List resultValueInfos, UniqueNamer uniqueNamer) { + public static MessageType generateSchema(List idHeaders, List resultValueInfos, UniqueNamer uniqueNamer, PrintSettings printSettings1) { /* Because Parquet Schemas rely on primitive types with logical annotations which are tedious to configure, we take the detour over the arrow schema. */ - final SchemaMapping schemaMapping = new SchemaConverter().fromArrow(new Schema(ArrowUtil.generateFields(idHeaders, resultValueInfos, uniqueNamer))); + final SchemaMapping schemaMapping = new SchemaConverter().fromArrow(new Schema(ArrowUtil.generateFields(idHeaders, resultValueInfos, uniqueNamer, printSettings1))); return schemaMapping.getParquetSchema(); @@ -75,14 +76,14 @@ private static List generateColumnConsumers(List idH return consumers; } - private static List generateColumnPrinters(List idHeaders, List resultInfos) { + private static List generateColumnPrinters(List idHeaders, List resultInfos, PrintSettings printSettings) { final List consumers = new ArrayList<>(); for (ResultInfo idHeader : idHeaders) { - consumers.add(idHeader.getPrinter()); + consumers.add(idHeader.createPrinter(printSettings)); } for (ResultInfo resultInfo : resultInfos) { - consumers.add(resultInfo.getPrinter()); + consumers.add(resultInfo.createPrinter(printSettings)); } return consumers; } @@ -105,9 +106,9 @@ private static ColumnConsumer getForResultType(ResultType resultType) { @Override public WriteContext init(Configuration configuration) { - schema = generateSchema(idHeaders, resultInfo, new UniqueNamer(printSettings)); + schema = generateSchema(idHeaders, resultInfo, new UniqueNamer(printSettings), printSettings); columnConsumers = generateColumnConsumers(idHeaders, resultInfo); - columnPrinters = generateColumnPrinters(idHeaders, resultInfo); + columnPrinters = generateColumnPrinters(idHeaders, resultInfo, printSettings); return new WriteContext(schema, Map.of()); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java index 1333d854f2..a6b03de025 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java @@ -60,8 +60,8 @@ public Response createResultFile(Subject subject, ManagedExecution exec, boolean final SingleTableResult singleTableResult = (SingleTableResult) exec; ParquetRenderer.writeToStream( output, - config.getIdColumns().getIdResultInfos(settings), - singleTableResult.getResultInfos(settings), + config.getIdColumns().getIdResultInfos(), + singleTableResult.getResultInfos(), settings, singleTableResult.streamResults(limit) ); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java index 72d06eb641..eb149cd3d0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java @@ -77,9 +77,9 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuil final PrintSettings printSettings = new PrintSettings(true, I18n.LOCALE.get(), exec.getNamespace(), exec.getConfig(), null, null, new ExcelResultPrinters()); // Save id column count to later check if xlsx dimensions are feasible - idColumnsCount = exec.getConfig().getIdColumns().getIdResultInfos(printSettings).size(); + idColumnsCount = exec.getConfig().getIdColumns().getIdResultInfos().size(); - final int columnCount = singleExecution.getResultInfos(printSettings).size() + idColumnsCount; + final int columnCount = singleExecution.getResultInfos().size() + idColumnsCount; final int maxColumnCount = SpreadsheetVersion.EXCEL2007.getMaxColumns(); if (columnCount > maxColumnCount) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java index 53fd1e9b13..c757481494 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java @@ -122,20 +122,24 @@ public boolean isExactlyOnePseudo() { * @return */ @JsonIgnore - public List getIdResultInfos(PrintSettings printSettings) { + public List getIdResultInfos() { return ids.stream().filter(ColumnConfig::isPrint).map(col -> { - final Map labels = col.getLabel(); - // Get the label for the locale, - // fall back to any label if there is exactly one defined, - // then fall back to the field name. - final String label = Objects.requireNonNullElse(labels.getOrDefault( - printSettings.getLocale(), - // fall backs - labels.size() == 1 ? labels.values().stream().collect(MoreCollectors.onlyElement()) : col.getField() - ), col.getField()); //TODO we can now hook our anonymizers into this - return new FixedLabelResultInfo(label, label, ResultType.Primitive.STRING, Set.of(new SemanticType.IdT(col.getName())), printSettings); + return new FixedLabelResultInfo(ResultType.Primitive.STRING, Set.of(new SemanticType.IdT(col.getName()))) { + @Override + public String userColumnName(PrintSettings printSettings) { + final Map labels = col.getLabel(); + // Get the label for the locale, + // fall back to any label if there is exactly one defined, + // then fall back to the field name. + return Objects.requireNonNullElse(labels.getOrDefault( + printSettings.getLocale(), + // fall backs + labels.size() == 1 ? labels.values().stream().collect(MoreCollectors.onlyElement()) : col.getField() + ), col.getField()); + } + }; }).collect(Collectors.toUnmodifiableList()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java index 4789347738..248dc83bc9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java @@ -88,8 +88,8 @@ public String getColumnName() { + getLabel(); } - public SelectResultInfo getResultInfo(CQConcept cqConcept, PrintSettings settings) { - return new SelectResultInfo(this, cqConcept, Collections.emptySet(), settings); + public SelectResultInfo getResultInfo(CQConcept cqConcept) { + return new SelectResultInfo(this, cqConcept, Collections.emptySet()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java index fbcb82c6be..13f920a2ce 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java @@ -55,11 +55,11 @@ public Printer createPrinter(PrintSettings printSettings, PrinterFactory printer } @Override - public SelectResultInfo getResultInfo(CQConcept cqConcept, PrintSettings settings) { + public SelectResultInfo getResultInfo(CQConcept cqConcept) { if (isAsIds()) { - return new SelectResultInfo(this, cqConcept, Set.of(new SemanticType.ConceptColumnT(cqConcept.getConcept())), settings); + return new SelectResultInfo(this, cqConcept, Set.of(new SemanticType.ConceptColumnT(cqConcept.getConcept()))); } - return new SelectResultInfo(this, cqConcept, Collections.emptySet(), settings); + return new SelectResultInfo(this, cqConcept, Collections.emptySet()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/SingleColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/SingleColumnSelect.java index a8a5e16ff0..98b0c82f81 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/SingleColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/SingleColumnSelect.java @@ -11,7 +11,6 @@ import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.events.MajorTypeId; -import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; import com.bakdata.conquery.models.types.SemanticType; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -50,13 +49,13 @@ public EnumSet getAcceptedColumnTypes() { } @Override - public SelectResultInfo getResultInfo(CQConcept cqConcept, PrintSettings settings) { + public SelectResultInfo getResultInfo(CQConcept cqConcept) { if(categorical){ - return new SelectResultInfo(this, cqConcept, Set.of(new SemanticType.CategoricalT()), settings); + return new SelectResultInfo(this, cqConcept, Set.of(new SemanticType.CategoricalT())); } - return new SelectResultInfo(this, cqConcept, Collections.emptySet(), settings); + return new SelectResultInfo(this, cqConcept, Collections.emptySet()); } @Nullable diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java index 60ea6040f0..08993a32ee 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java @@ -65,13 +65,13 @@ public Printer createPrinter(PrintSettings printSettings, PrinterFactory printer } @Override - public SelectResultInfo getResultInfo(CQConcept cqConcept, PrintSettings settings) { + public SelectResultInfo getResultInfo(CQConcept cqConcept) { if (!isCategorical()) { - return new SelectResultInfo(this, cqConcept, Collections.emptySet(), settings); + return new SelectResultInfo(this, cqConcept, Collections.emptySet()); } - return new SelectResultInfo(this, cqConcept, Set.of(new SemanticType.CategoricalT()), settings); + return new SelectResultInfo(this, cqConcept, Set.of(new SemanticType.CategoricalT())); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQuery.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQuery.java index 264d3d1a4c..4a678ef4eb 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQuery.java @@ -17,7 +17,6 @@ import com.bakdata.conquery.models.forms.util.DateContext; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.DateAggregationMode; -import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -43,7 +42,7 @@ public class AbsoluteFormQuery extends Query { public static final int TIME_INDEX = 2; /** - * see {@linkplain this#getResultInfos(PrintSettings)}. + * see {@linkplain #getResultInfos()}. */ public static final int FEATURES_OFFSET = 3; @@ -85,13 +84,13 @@ public AbsoluteFormQueryPlan createQueryPlan(QueryPlanContext context) { } @Override - public List getResultInfos(PrintSettings printSettings) { + public List getResultInfos() { final List resultInfos = new ArrayList<>(); - resultInfos.add(ResultHeaders.formResolutionInfo(printSettings)); - resultInfos.add(ResultHeaders.formContextInfo(printSettings)); - resultInfos.add(ResultHeaders.formDateRangeInfo(printSettings)); - resultInfos.addAll(features.getResultInfos(printSettings)); + resultInfos.add(ResultHeaders.formResolutionInfo()); + resultInfos.add(ResultHeaders.formContextInfo()); + resultInfos.add(ResultHeaders.formDateRangeInfo()); + resultInfos.addAll(features.getResultInfos()); return resultInfos; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQuery.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQuery.java index 68c6dce565..9b74f0830d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQuery.java @@ -16,7 +16,6 @@ import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.DateAggregationMode; -import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -89,13 +88,13 @@ public void resolve(QueryResolveContext context) { } @Override - public List getResultInfos(PrintSettings printSettings) { + public List getResultInfos() { List resultInfos = new ArrayList<>(); - resultInfos.add(ResultHeaders.formResolutionInfo(printSettings)); - resultInfos.add(ResultHeaders.formContextInfo(printSettings)); - resultInfos.add(ResultHeaders.formDateRangeInfo(printSettings)); + resultInfos.add(ResultHeaders.formResolutionInfo()); + resultInfos.add(ResultHeaders.formContextInfo()); + resultInfos.add(ResultHeaders.formDateRangeInfo()); - resultInfos.addAll(features.getResultInfos(printSettings)); + resultInfos.addAll(features.getResultInfos()); return resultInfos; diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java index d8ca727319..da1a753433 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java @@ -22,7 +22,6 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.ColumnDescriptor; import com.bakdata.conquery.models.query.ManagedQuery; -import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; @@ -134,11 +133,11 @@ public void cancel() { @Override @JsonIgnore - public List getResultInfos(PrintSettings printSettings) { + public List getResultInfos() { if (subQueries.size() != 1) { throw new UnsupportedOperationException("Cannot gather result info when multiple tables are generated"); } - return subQueries.values().iterator().next().getResultInfos(printSettings); + return subQueries.values().iterator().next().getResultInfos(); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQuery.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQuery.java index 6b912d0596..2668aa332a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQuery.java @@ -16,7 +16,6 @@ import com.bakdata.conquery.models.forms.util.CalendarUnit; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.DateAggregationMode; -import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -74,17 +73,17 @@ public void collectRequiredQueries(Set requiredQueries) { } @Override - public List getResultInfos(PrintSettings printSettings) { + public List getResultInfos() { List resultInfos = new ArrayList<>(); - resultInfos.add(ResultHeaders.formResolutionInfo(printSettings)); - resultInfos.add(ResultHeaders.formContextInfo(printSettings)); - resultInfos.add(ResultHeaders.formEventDateInfo(printSettings)); - resultInfos.add(ResultHeaders.formDateRangeInfo(printSettings)); + resultInfos.add(ResultHeaders.formResolutionInfo()); + resultInfos.add(ResultHeaders.formContextInfo()); + resultInfos.add(ResultHeaders.formEventDateInfo()); + resultInfos.add(ResultHeaders.formDateRangeInfo()); - final List featureInfos = features.getResultInfos(printSettings); + final List featureInfos = features.getResultInfos(); - resultInfos.add(ResultHeaders.formObservationScopeInfo(printSettings)); + resultInfos.add(ResultHeaders.formObservationScopeInfo()); resultInfos.addAll(featureInfos); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java index a1d2c7ddeb..4af084d4dd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java @@ -113,8 +113,8 @@ protected void setAdditionalFieldsForStatusWithColumnDescription(Subject subject } @JsonIgnore - public List getResultInfos(PrintSettings printSettings) { - return query.getResultInfos(printSettings); + public List getResultInfos() { + return query.getResultInfos(); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/SingleTableResult.java b/backend/src/main/java/com/bakdata/conquery/models/query/SingleTableResult.java index 0ae6fd61ef..e49ae92f0b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/SingleTableResult.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/SingleTableResult.java @@ -10,7 +10,6 @@ import com.bakdata.conquery.models.i18n.I18n; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; -import com.bakdata.conquery.models.query.resultinfo.printers.JavaResultPrinters; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.worker.Namespace; @@ -25,26 +24,26 @@ default List generateColumnDescriptions(boolean isInitialized, final Locale locale = I18n.LOCALE.get(); // The printer is never used to generate results. But downstream code might touch them - final PrintSettings settings = new PrintSettings(true, locale, getNamespace(), config, null, null, new JavaResultPrinters()); + final PrintSettings settings = new PrintSettings(true, locale, getNamespace(), config, null, null,null); final UniqueNamer uniqNamer = new UniqueNamer(settings); // First add the id columns to the descriptor list. The are the first columns - for (ResultInfo header : config.getIdColumns().getIdResultInfos(settings)) { + for (ResultInfo header : config.getIdColumns().getIdResultInfos()) { columnDescriptions.add(ColumnDescriptor.builder() - .label(uniqNamer.getUniqueName(header)) + .label(uniqNamer.getUniqueName(header, settings)) .type(ResultType.Primitive.STRING.typeInfo()) .semantics(header.getSemantics()) .build()); } final UniqueNamer collector = new UniqueNamer(settings); - getResultInfos(settings).forEach(info -> columnDescriptions.add(info.asColumnDescriptor(collector))); + getResultInfos().forEach(info -> columnDescriptions.add(info.asColumnDescriptor(collector, settings))); return columnDescriptions; } @JsonIgnore - List getResultInfos(PrintSettings printSettings); + List getResultInfos(); /** * @param limit Optionally limits how many lines are emitted. diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java index 2121ecfeba..cbd7357fc8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java @@ -128,7 +128,7 @@ private static Function> createLineToMapTransforme columnNames[index] = select2desc.get(selectResultInfo.getSelect().getId()).label(); } - printers[index] = resultInfo.getPrinter(); + printers[index] = resultInfo.createPrinter(printSettings); } return line -> { @@ -159,11 +159,11 @@ private static Function> createLineToMapTransforme /** * For the selects in result infos, build ColumnDescriptors using definitions (label and description) from PreviewConfig. */ - private static List createChronoColumnDescriptors(SingleTableResult query, Map select2desc, PrintSettings printSettings) { + private static List createChronoColumnDescriptors(SingleTableResult query, Map select2desc) { final List columnDescriptions = new ArrayList<>(); - for (ResultInfo info : query.getResultInfos(printSettings)) { + for (ResultInfo info : query.getResultInfos()) { if (info instanceof SelectResultInfo selectResultInfo) { final PreviewConfig.InfoCardSelect desc = select2desc.get(selectResultInfo.getSelect().getId()); @@ -232,13 +232,13 @@ private List transformQueryResultToInfos(ManagedQuery final List extraInfos = new ArrayList<>(values.length); // We are only interested in the Select results. - for (int index = AbsoluteFormQuery.FEATURES_OFFSET; index < infoCardExecution.getResultInfos(printSettings).size(); index++) { - final ResultInfo resultInfo = infoCardExecution.getResultInfos(printSettings).get(index); + for (int index = AbsoluteFormQuery.FEATURES_OFFSET; index < infoCardExecution.getResultInfos().size(); index++) { + final ResultInfo resultInfo = infoCardExecution.getResultInfos().get(index); - final Object printed = resultInfo.printNullable(values[index]); + final Object printed = resultInfo.createPrinter(printSettings).apply(values[index]); extraInfos.add(new EntityPreviewStatus.Info( - resultInfo.userColumnName(), + resultInfo.userColumnName(printSettings), printed, resultInfo.getType().typeInfo(), resultInfo.getDescription(), @@ -268,13 +268,13 @@ private List toChronoInfos(PreviewConfi .collect(Collectors.toMap(PreviewConfig.InfoCardSelect::select, Function.identity())); // Group lines by year and quarter. - final Function> lineTransformer = createLineToMapTransformer(query.getResultInfos(printSettings), select2desc, printSettings); + final Function> lineTransformer = createLineToMapTransformer(query.getResultInfos(), select2desc, printSettings); final List yearEntries = createYearEntries(entityResult, lineTransformer); final Object[] completeResult = getCompleteLine(entityResult); // get descriptions, but drop everything that isn't a select result as the rest is already structured - final List columnDescriptors = createChronoColumnDescriptors(query, select2desc, printSettings); + final List columnDescriptors = createChronoColumnDescriptors(query, select2desc); final EntityPreviewStatus.TimeStratifiedInfos @@ -361,8 +361,8 @@ protected void setAdditionalFieldsForStatusWithSource(Subject subject, FullExecu } @Override - public List getResultInfos(PrintSettings printSettings) { - return getValuesQuery().getResultInfos(printSettings); + public List getResultInfos() { + return getValuesQuery().getResultInfos(); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java index 9386f830a1..0a312dc9d7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java @@ -1,12 +1,13 @@ package com.bakdata.conquery.models.query.resultinfo; -import java.util.Set; +import java.util.Collections; import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.query.PrintSettings; +import com.bakdata.conquery.models.query.resultinfo.printers.ConceptIdPrinter; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.models.types.SemanticType; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; @@ -19,25 +20,33 @@ public class ColumnResultInfo extends ResultInfo { private final Column column; private final ResultType type; private final String description; - private final Printer printer; + private final Concept concept; - public ColumnResultInfo(Column column, ResultType type, Set semantics, Printer printer, String description, PrintSettings settings) { - super(semantics, settings); + public ColumnResultInfo(Column column, ResultType type, String description, Concept concept) { + super(Collections.emptySet()); this.column = column; this.type = type; this.description = description; - this.printer = printer; + this.concept = concept; } @Override - public String userColumnName() { + public String userColumnName(PrintSettings printSettings) { return column.getTable().getLabel() + " " + column.getLabel(); } @Override - public String defaultColumnName() { - return userColumnName(); + public String defaultColumnName(PrintSettings printSettings) { + return userColumnName(printSettings); + } + + @Override + public Printer createPrinter(PrintSettings printSettings) { + if(concept != null){ + return new ConceptIdPrinter(concept, printSettings); + } + return printSettings.getPrinterFactory().printerFor(type, printSettings); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ExternalResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ExternalResultInfo.java index b97babbab7..a6d2a05e55 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ExternalResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ExternalResultInfo.java @@ -14,24 +14,31 @@ public class ExternalResultInfo extends ResultInfo { private final String name; private final ResultType type; - private final String description; - private final Printer printer; - public ExternalResultInfo(String name, ResultType type, PrintSettings settings) { - super(Collections.emptySet(), settings); + + public ExternalResultInfo(String name, ResultType type) { + super(Collections.emptySet()); this.name = name; this.type = type; - this.description = null; - this.printer = settings.getPrinterFactory().printerFor(type, settings); } @Override - public String userColumnName() { + public String userColumnName(PrintSettings printSettings) { return null; } @Override - public String defaultColumnName() { + public String defaultColumnName(PrintSettings printSettings) { return name; } + + @Override + public String getDescription() { + return null; + } + + @Override + public Printer createPrinter(PrintSettings printSettings) { + return printSettings.getPrinterFactory().printerFor(type, printSettings); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/FixedLabelResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/FixedLabelResultInfo.java index 244f6bcb65..738f404abb 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/FixedLabelResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/FixedLabelResultInfo.java @@ -9,7 +9,6 @@ import com.bakdata.conquery.models.types.SemanticType; import lombok.EqualsAndHashCode; import lombok.Getter; -import lombok.NonNull; import lombok.ToString; /** @@ -34,36 +33,28 @@ */ @EqualsAndHashCode(callSuper = true) @ToString -public class FixedLabelResultInfo extends ResultInfo { +public abstract class FixedLabelResultInfo extends ResultInfo { + - @NonNull - private final String localizedLabel; - @NonNull - private final String localizedDefaultLabel; @Getter private final ResultType type; - @Override - public Printer getPrinter() { - return getSettings().getPrinterFactory().printerFor(getType(), getSettings()); - } - - public FixedLabelResultInfo(String label, String defaultLabel, ResultType type, Set semantics, PrintSettings settings) { - super(semantics, settings); - this.localizedLabel = label; - this.localizedDefaultLabel = defaultLabel; + public FixedLabelResultInfo(ResultType type, Set semantics) { + super(semantics); this.type = type; } - @Override - public String userColumnName() { - return localizedLabel; + public Printer createPrinter(PrintSettings printSettings) { + return printSettings.getPrinterFactory().printerFor(getType(), printSettings); } @Override - public String defaultColumnName() { - return localizedDefaultLabel; + public abstract String userColumnName(PrintSettings printSettings); + + @Override + public String defaultColumnName(PrintSettings printSettings) { + return userColumnName(printSettings); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java index 442c4a01dc..4c0059d35e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java @@ -24,13 +24,10 @@ @Slf4j public abstract class ResultInfo { - private final PrintSettings settings; - @ToString.Include private final Set semantics = new HashSet<>(); - protected ResultInfo(Collection semantics, PrintSettings settings) { - this.settings = settings; + protected ResultInfo(Collection semantics) { this.semantics.addAll(semantics); } @@ -38,12 +35,12 @@ public final void addSemantics(SemanticType... incoming) { semantics.addAll(Arrays.asList(incoming)); } - public abstract String userColumnName(); + public abstract String userColumnName(PrintSettings printSettings); - public final ColumnDescriptor asColumnDescriptor(UniqueNamer collector) { + public final ColumnDescriptor asColumnDescriptor(UniqueNamer collector, PrintSettings printSettings) { return ColumnDescriptor.builder() - .label(collector.getUniqueName(this)) - .defaultLabel(defaultColumnName()) + .label(collector.getUniqueName(this, printSettings)) + .defaultLabel(defaultColumnName(printSettings)) .type(getType().typeInfo()) .semantics(getSemantics()) .description(getDescription()) @@ -52,8 +49,9 @@ public final ColumnDescriptor asColumnDescriptor(UniqueNamer collector) { /** * Use default label schema which ignores user labels. + * @param printSettings */ - public abstract String defaultColumnName(); + public abstract String defaultColumnName(PrintSettings printSettings); @ToString.Include public abstract ResultType getType(); @@ -64,13 +62,5 @@ public Set getSemantics() { public abstract String getDescription(); - public final Object printNullable(Object f) { - if (f == null) { - return ""; - } - - return getPrinter().apply(f); - } - - public abstract Printer getPrinter(); + public abstract Printer createPrinter(PrintSettings printSettings); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java index 4b24d379d5..9ef5021b7a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java @@ -21,8 +21,8 @@ public class SelectResultInfo extends ResultInfo { @NonNull private final CQConcept cqConcept; - public SelectResultInfo(Select select, CQConcept cqConcept, Set semantics, PrintSettings settings) { - super(Sets.union(semantics, Set.of(new SemanticType.SelectResultT(select))), settings); + public SelectResultInfo(Select select, CQConcept cqConcept, Set semantics) { + super(Sets.union(semantics, Set.of(new SemanticType.SelectResultT(select)))); this.select = select; this.cqConcept = cqConcept; } @@ -34,8 +34,8 @@ public String getDescription() { } @Override - public Printer getPrinter() { - return select.createPrinter(getSettings(), getSettings().getPrinterFactory()); + public Printer createPrinter(PrintSettings printSettings) { + return select.createPrinter(printSettings, printSettings.getPrinterFactory()); } @Override @@ -44,11 +44,11 @@ public ResultType getType() { } @Override - public String userColumnName() { + public String userColumnName(PrintSettings printSettings) { - if (getSettings().getColumnNamer() != null) { + if (printSettings.getColumnNamer() != null) { // override user labels if column namer is set, TODO clean this up when userConceptLabel is removed - return getSettings().getColumnNamer().apply(this); + return printSettings.getColumnNamer().apply(this); } String label = getCqConcept().getLabel(); @@ -60,10 +60,10 @@ public String userColumnName() { } @Override - public String defaultColumnName() { + public String defaultColumnName(PrintSettings printSettings) { StringBuilder sb = new StringBuilder(); - String cqLabel = getCqConcept().defaultLabel(getSettings().getLocale()); + String cqLabel = getCqConcept().defaultLabel(printSettings.getLocale()); final String selectLabel = select.getColumnName(); if (selectLabel.equals(cqLabel)) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/UniqueNamer.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/UniqueNamer.java index 31f0eb85be..cf186272d5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/UniqueNamer.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/UniqueNamer.java @@ -33,9 +33,9 @@ public class UniqueNamer { @NonNull @JsonIgnore - public final String getUniqueName(ResultInfo info) { + public final String getUniqueName(ResultInfo info, PrintSettings printSettings) { @NonNull - String label = Objects.requireNonNullElse(info.userColumnName(), info.defaultColumnName()); + String label = Objects.requireNonNullElse(info.userColumnName(printSettings), info.defaultColumnName(printSettings)); // lookup if prefix is needed and computed it if necessary String uniqueName = label; synchronized (ocurrenceCounter) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/SecondaryIdResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/SecondaryIdResultInfo.java index d0a5774aa8..e3efcf70a2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/SecondaryIdResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/SecondaryIdResultInfo.java @@ -15,20 +15,14 @@ public class SecondaryIdResultInfo extends ResultInfo { private final SecondaryIdDescription secondaryId; private final ResultType type; - private final Printer printer; - public SecondaryIdResultInfo(SecondaryIdDescription secondaryId, PrintSettings settings) { - super(Set.of(new SemanticType.SecondaryIdT(secondaryId)), settings); + public SecondaryIdResultInfo(SecondaryIdDescription secondaryId) { + super(Set.of(new SemanticType.SecondaryIdT(secondaryId))); this.secondaryId = secondaryId; type = ResultType.Primitive.STRING; - if (secondaryId.getMapping() == null) { - printer = settings.getPrinterFactory().getStringPrinter(settings); - } - else { - printer = new MappedPrinter(secondaryId.getMapping()); - } + } @Override @@ -37,12 +31,22 @@ public String getDescription() { } @Override - public String userColumnName() { + public Printer createPrinter(PrintSettings printSettings) { + if (secondaryId.getMapping() == null) { + return printSettings.getPrinterFactory().getStringPrinter(printSettings); + } + else { + return new MappedPrinter(secondaryId.getMapping()); + } + } + + @Override + public String userColumnName(PrintSettings printSettings) { return secondaryId.getLabel(); } @Override - public String defaultColumnName() { - return userColumnName(); + public String defaultColumnName(PrintSettings printSettings) { + return userColumnName(printSettings); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ResultStatistics.java b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ResultStatistics.java index a9648246b5..8f0d3f9040 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ResultStatistics.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ResultStatistics.java @@ -73,9 +73,9 @@ public static ResultStatistics collectResultStatistics(SingleTableResult managed final StopWatch started = StopWatch.createStarted(); final ResultInfo info = resultInfos.get(col); - final Printer printer = info.getPrinter(); + final Printer printer = info.createPrinter(printSettings); final ColumnStatsCollector statsCollector = - ColumnStatsCollector.getStatsCollector(uniqueNamer.getUniqueName(info), info.getDescription(), info.getType(), printSettings, conqueryConfig.getFrontend()); + ColumnStatsCollector.getStatsCollector(uniqueNamer.getUniqueName(info, printSettings), info.getDescription(), info.getType(), printSettings, conqueryConfig.getFrontend()); log.trace("BEGIN stats collection for {}", info); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java index 573f2222b4..189a1748b6 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java @@ -40,7 +40,7 @@ public ConversionContext convert(AbsoluteFormQuery form, ConversionContext conte FormType.ABSOLUTE, stratificationTable, form.getFeatures(), - form.getResultInfos(context.getSqlPrintSettings()), + form.getResultInfos(), context ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java index 74f66d3e42..055bfcc262 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java @@ -46,7 +46,7 @@ public ConversionContext convert(ConceptQuery conceptQuery, ConversionContext co .build(); Select finalQuery = this.queryStepTransformer.toSelectQuery(finalStep); - return contextAfterConversion.withFinalQuery(new SqlQuery(finalQuery, conceptQuery.getResultInfos(context.getSqlPrintSettings()))); + return contextAfterConversion.withFinalQuery(new SqlQuery(finalQuery, conceptQuery.getResultInfos())); } private Selects getFinalSelects(ConceptQuery conceptQuery, Selects preFinalSelects, SqlFunctionProvider functionProvider) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java index 917f49f8f3..457eebe740 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java @@ -39,7 +39,7 @@ public ConversionContext convert(EntityDateQuery entityDateQuery, ConversionCont FormType.ENTITY_DATE, stratificationTable, entityDateQuery.getFeatures(), - entityDateQuery.getResultInfos(context.getSqlPrintSettings()), + entityDateQuery.getResultInfos(), context ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/RelativFormQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/RelativFormQueryConverter.java index b0062afbdc..544d1e0548 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/RelativFormQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/RelativFormQueryConverter.java @@ -29,7 +29,7 @@ public ConversionContext convert(RelativeFormQuery form, ConversionContext conte FormType.RELATIVE, stratificationTable, form.getFeatures(), - form.getResultInfos(context.getSqlPrintSettings()), + form.getResultInfos(), context ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/SecondaryIdQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/SecondaryIdQueryConverter.java index 34c3511f63..e5d9ee6197 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/SecondaryIdQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/SecondaryIdQueryConverter.java @@ -22,7 +22,7 @@ public ConversionContext convert(SecondaryIdQuery query, ConversionContext conte ); Preconditions.checkArgument(withConvertedQuery.getFinalQuery() != null, "The SecondaryIdQuery's query should be converted by now."); - SqlQuery secondaryIdSqlQuery = withConvertedQuery.getFinalQuery().overwriteResultInfos(query.getResultInfos(context.getSqlPrintSettings())); + SqlQuery secondaryIdSqlQuery = withConvertedQuery.getFinalQuery().overwriteResultInfos(query.getResultInfos()); return withConvertedQuery.withFinalQuery(secondaryIdSqlQuery); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java index 37489e79d0..a519d5e8a1 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java @@ -70,7 +70,7 @@ public ConversionContext convert(TableExportQuery tableExportQuery, ConversionCo ); Select selectQuery = queryStepTransformer.toSelectQuery(unionedTables); - return context.withFinalQuery(new SqlQuery(selectQuery, tableExportQuery.getResultInfos(context.getSqlPrintSettings()))); + return context.withFinalQuery(new SqlQuery(selectQuery, tableExportQuery.getResultInfos())); } /** diff --git a/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java b/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java index 0b5c173c20..01b2aa6338 100644 --- a/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java +++ b/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java @@ -10,8 +10,6 @@ import java.util.List; import java.util.UUID; import java.util.stream.Collectors; -import jakarta.validation.Validator; -import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.apiv1.QueryProcessor; import com.bakdata.conquery.apiv1.execution.ExecutionStatus; @@ -45,7 +43,6 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId; import com.bakdata.conquery.models.query.ManagedQuery; -import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.DistributedNamespace; @@ -53,6 +50,8 @@ import com.google.common.collect.ImmutableList; import io.dropwizard.core.setup.Environment; import io.dropwizard.jersey.validation.Validators; +import jakarta.validation.Validator; +import jakarta.ws.rs.core.UriBuilder; import lombok.SneakyThrows; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -202,7 +201,7 @@ private static ManagedQuery mockManagedQuery(Query queryDescription, User user, } @Override - public List getResultInfos(PrintSettings printSettings) { + public List getResultInfos() { // With method is mocked because the ExcelResultProvider needs some info to check dimensions, // but actually resolving the query here requires much more setup return Collections.emptyList(); @@ -222,7 +221,7 @@ private static ExecutionStatus makeState(ManagedExecutionId id, User owner, User } @Override - public List getResultInfos(PrintSettings printSettings) { + public List getResultInfos() { return Collections.emptyList(); } }; diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java index 3cbbe2e758..2cb6104c13 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java @@ -60,7 +60,7 @@ public void executeTest(StandaloneSupport standaloneSupport) throws IOException //check result info size PrintSettings printSettings = new PrintSettings(true, Locale.ROOT, standaloneSupport.getNamespace(), standaloneSupport.getConfig(), null, null, new CsvResultPrinters()); - List resultInfos = executionResult.getResultInfos(printSettings); + List resultInfos = executionResult.getResultInfos(); assertThat(executionResult.streamResults(OptionalLong.empty()).flatMap(EntityResult::streamValues)) .as("Should have same size as result infos") diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java index 67e68477cf..f0cb3b5e26 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java @@ -144,7 +144,7 @@ private void checkResults(StandaloneSupport standaloneSupport, ManagedInternalFo */ private void checkMultipleResult(Map> managedMapping, ConqueryConfig config, PrintSettings printSettings) throws IOException { for (Map.Entry> managed : managedMapping.entrySet()) { - List resultInfos = managed.getValue().get(0).getResultInfos(printSettings); + List resultInfos = managed.getValue().get(0).getResultInfos(); log.info("{} CSV TESTING: {}", getLabel(), managed.getKey()); ByteArrayOutputStream output = new ByteArrayOutputStream(); @@ -154,11 +154,11 @@ private void checkMultipleResult(Map> managedMapping, CsvRenderer renderer = new CsvRenderer(writer, printSettings); renderer.toCSV( - config.getIdColumns().getIdResultInfos(printSettings), + config.getIdColumns().getIdResultInfos(), resultInfos, managed.getValue() .stream() - .flatMap(managedQuery -> managedQuery.streamResults(OptionalLong.empty())) + .flatMap(managedQuery -> managedQuery.streamResults(OptionalLong.empty())), printSettings ); writer.close(); @@ -188,9 +188,9 @@ private void checkSingleResult(F man final CsvRenderer renderer = new CsvRenderer(writer, printSettings); renderer.toCSV( - config.getIdColumns().getIdResultInfos(printSettings), - managedForm.getResultInfos(printSettings), - managedForm.streamResults(OptionalLong.empty()) + config.getIdColumns().getIdResultInfos(), + managedForm.getResultInfos(), + managedForm.streamResults(OptionalLong.empty()), printSettings ); writer.close(); diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java index 218822f456..7af4476e2b 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java @@ -15,7 +15,6 @@ import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.query.ManagedQuery; -import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; @@ -45,9 +44,9 @@ public class ResultTestUtil { CONCEPT.setDataset(DATASET); } - public static List getIdFields(PrintSettings printSettings) { + public static List getIdFields() { return Stream.of("id1", "id2").map(name -> { - ExternalResultInfo info = new ExternalResultInfo(name, ResultType.Primitive.STRING, printSettings); + ExternalResultInfo info = new ExternalResultInfo(name, ResultType.Primitive.STRING); info.addSemantics(new SemanticType.IdT("ID")); return info; }).collect(Collectors.toList()); @@ -57,10 +56,10 @@ public static List getIdFields(PrintSettings printSettings) { public static ManagedQuery getTestQuery() { return new ManagedQuery(null, OWNER, DATASET, null) { @Override - public List getResultInfos(PrintSettings printSettings) { + public List getResultInfos() { return getResultTypes().stream() .map(TypedSelectDummy::new) - .map(select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet(), printSettings)) + .map(select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet())) .collect(Collectors.toList()); } diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java index 52cbdbe9de..afd0596ca0 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java @@ -64,7 +64,7 @@ public class ArrowResultGenerationTest { void generateFieldsIdMapping() { final UniqueNamer uniqueNamer = new UniqueNamer(PRINT_SETTINGS); - List fields = generateFields(getIdFields(PRINT_SETTINGS), uniqueNamer); + List fields = generateFields(getIdFields(), uniqueNamer, PRINT_SETTINGS); assertThat(fields).containsExactlyElementsOf( List.of( @@ -80,12 +80,12 @@ void generateFieldsValue() { List resultInfos = getResultTypes().stream().map(TypedSelectDummy::new) - .map(select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet(), PRINT_SETTINGS)).collect(Collectors.toList()); + .map(select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet())).collect(Collectors.toList()); List fields = generateFields( - resultInfos, - // Custom column namer so we don't require a dataset registry - uniqueNamer + resultInfos, + // Custom column namer so we don't require a dataset registry + uniqueNamer, PRINT_SETTINGS ); assertThat(fields).containsExactlyElementsOf( @@ -143,8 +143,8 @@ void writeAndRead() throws IOException { (root) -> new ArrowStreamWriter(root, new DictionaryProvider.MapDictionaryProvider(), output), printSettings, new ArrowConfig(BATCH_SIZE), - getIdFields(PRINT_SETTINGS), - mquery.getResultInfos(printSettings), + getIdFields(), + mquery.getResultInfos(), mquery.streamResults(OptionalLong.empty()) ); @@ -153,7 +153,7 @@ void writeAndRead() throws IOException { String computed = readTSV(inputStream); assertThat(computed).isNotBlank(); - assertThat(computed).isEqualTo(generateExpectedTSV(results, mquery.getResultInfos(printSettings), printSettings)); + assertThat(computed).isEqualTo(generateExpectedTSV(results, mquery.getResultInfos())); } @@ -180,7 +180,7 @@ public static String readTSV(InputStream inputStream) throws IOException { return stringJoiner.toString(); } - public static String generateExpectedTSV(List results, List resultInfos, PrintSettings settings) { + public static String generateExpectedTSV(List results, List resultInfos) { String expected = results.stream() .map(EntityResult.class::cast) .map(res -> { @@ -193,7 +193,7 @@ public static String generateExpectedTSV(List results, List results, List i.defaultColumnName()), + getIdFields().stream().map(i -> i.defaultColumnName(PRINT_SETTINGS)), // result column headers getResultTypes().stream().map(ResultType::typeInfo) ).collect(Collectors.joining("\t")) + "\n" + expected; } - private static String getPrintValue(Object obj, ResultType type, PrintSettings settings) { + private static String getPrintValue(Object obj, ResultType type) { if (obj == null) { return "null"; } @@ -238,7 +238,7 @@ private static String getPrintValue(Object obj, ResultType type, PrintSettings s Collection col = (Collection) obj; // Workaround: Arrow deserializes lists as a JsonStringArrayList which has a JSON String method @NonNull ResultType elemType = ((ResultType.ListT) type).getElementType(); - return col.stream().map(v -> getPrintValue(v, elemType, settings)).collect(Collectors.joining(",", "[", "]")); + return col.stream().map(v -> getPrintValue(v, elemType)).collect(Collectors.joining(",", "[", "]")); } return obj.toString(); } diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java index e1f3ceee7d..ee1f71b31f 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java @@ -57,12 +57,12 @@ void writeAndRead() throws IOException { final StringWriter writer = new StringWriter(); final CsvRenderer renderer = new CsvRenderer(CONFIG.getCsv().createWriter(writer), printSettings); - renderer.toCSV(getIdFields(printSettings), mquery.getResultInfos(printSettings), mquery.streamResults(OptionalLong.empty())); + renderer.toCSV(getIdFields(), mquery.getResultInfos(), mquery.streamResults(OptionalLong.empty()), printSettings); final String computed = writer.toString(); - final String expected = generateExpectedCSV(results, mquery.getResultInfos(printSettings), printSettings); + final String expected = generateExpectedCSV(results, mquery.getResultInfos(), printSettings); log.info("Wrote and than read this csv data: {}", computed); @@ -73,7 +73,7 @@ void writeAndRead() throws IOException { private String generateExpectedCSV(List results, List resultInfos, PrintSettings printSettings) { final List expected = new ArrayList<>(); - expected.add(getIdFields(printSettings).stream().map(info -> info.defaultColumnName()).collect(Collectors.joining(",")) + expected.add(getIdFields().stream().map(info -> info.defaultColumnName(printSettings)).collect(Collectors.joining(",")) + "," + getResultTypes().stream().map(ResultType::typeInfo).collect(Collectors.joining(",")) + "\n"); @@ -92,7 +92,7 @@ private String generateExpectedCSV(List results, List continue; } final ResultInfo info = resultInfos.get(lIdx); - final String printVal = (String) info.printNullable(val); + final String printVal = (String) info.createPrinter(printSettings).apply(val); valueJoiner.add(printVal.contains(String.valueOf(CONFIG.getCsv().getDelimeter())) ? "\"" + printVal + "\"" : printVal); } diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java index 16f6f351c3..2593e9b1c5 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java @@ -64,10 +64,10 @@ void writeAndRead() throws IOException { final List results = getTestEntityResults(); final ManagedQuery mquery = new ManagedQuery(null, null, null, null) { - public List getResultInfos(PrintSettings printSettings) { + public List getResultInfos() { return getResultTypes().stream() .map(ResultTestUtil.TypedSelectDummy::new) - .map(select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet(), printSettings)) + .map(select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet())) .collect(Collectors.toList()); } @@ -82,7 +82,7 @@ public Stream streamResults(OptionalLong maybeLimit) { final ExcelRenderer renderer = new ExcelRenderer(new ExcelConfig(), printSettings); - renderer.renderToStream(ResultTestUtil.getIdFields(printSettings), mquery, output, OptionalLong.empty(), printSettings); + renderer.renderToStream(ResultTestUtil.getIdFields(), mquery, output, OptionalLong.empty(), printSettings); final InputStream inputStream = new ByteArrayInputStream(output.toByteArray()); @@ -90,7 +90,7 @@ public Stream streamResults(OptionalLong maybeLimit) { final List computed = readComputed(inputStream, printSettings); - final List expected = generateExpectedTSV(results, mquery.getResultInfos(printSettings)); + final List expected = generateExpectedTSV(results, mquery.getResultInfos(), printSettings); log.info("Wrote and than read this excel data: {}", computed); @@ -127,7 +127,7 @@ private List readComputed(InputStream inputStream, PrintSettings setting } - private List generateExpectedTSV(List results, List resultInfos) { + private List generateExpectedTSV(List results, List resultInfos, PrintSettings printSettings) { final List expected = new ArrayList<>(); expected.add(String.join("\t", printIdFields) + "\t" + getResultTypes().stream().map(ResultType::typeInfo).collect(Collectors.joining("\t"))); results.stream().map(EntityResult.class::cast).forEach(res -> { @@ -143,7 +143,7 @@ private List generateExpectedTSV(List results, List generateExpectedTSV(List results, List resultInfos = getResultTypes().stream().map(TypedSelectDummy::new) - .map(select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet(), PRINT_SETTINGS)).collect(Collectors.toList()); + .map(select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet())).collect(Collectors.toList()); - final MessageType messageType = EntityResultWriteSupport.generateSchema(getIdFields(PRINT_SETTINGS), resultInfos, uniqueNamer); + final MessageType messageType = EntityResultWriteSupport.generateSchema(getIdFields(), resultInfos, uniqueNamer, PRINT_SETTINGS); assertThat(messageType).isEqualTo( Types.buildMessage() @@ -120,7 +120,7 @@ void writeAndRead() throws IOException { // First we write to the buffer, than we read from it and parse it as TSV ByteArrayOutputStream output = new ByteArrayOutputStream(); - ParquetRenderer.writeToStream(output, getIdFields(PRINT_SETTINGS), managedQuery.getResultInfos(printSettings), printSettings, managedQuery.streamResults(OptionalLong.empty())); + ParquetRenderer.writeToStream(output, getIdFields(), managedQuery.getResultInfos(), printSettings, managedQuery.streamResults(OptionalLong.empty())); final byte[] buf = output.toByteArray(); @@ -142,7 +142,7 @@ void writeAndRead() throws IOException { log.info("\n{}", actual); - assertThat(actual).isEqualTo(ArrowResultGenerationTest.generateExpectedTSV(results, managedQuery.getResultInfos(printSettings), printSettings)); + assertThat(actual).isEqualTo(ArrowResultGenerationTest.generateExpectedTSV(results, managedQuery.getResultInfos())); } diff --git a/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java b/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java index b7ab8535f6..5c5ecc96b2 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java @@ -169,9 +169,9 @@ void checkCombinations(TestConcept concept, boolean hasCQConceptLabel, String ex final CQConcept cqConcept = concept.createCQConcept(hasCQConceptLabel); final UniqueNamer uniqNamer = new UniqueNamer(SETTINGS); - SelectResultInfo info = new SelectResultInfo(concept.extractSelect(cqConcept), cqConcept, Collections.emptySet(), SETTINGS); + SelectResultInfo info = new SelectResultInfo(concept.extractSelect(cqConcept), cqConcept, Collections.emptySet()); - assertThat(uniqNamer.getUniqueName(info)).isEqualTo(expectedColumnName); + assertThat(uniqNamer.getUniqueName(info, SETTINGS)).isEqualTo(expectedColumnName); } diff --git a/backend/src/test/java/com/bakdata/conquery/models/query/UniqueNameTest.java b/backend/src/test/java/com/bakdata/conquery/models/query/UniqueNameTest.java index 3abde84190..4b28433098 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/query/UniqueNameTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/query/UniqueNameTest.java @@ -17,14 +17,14 @@ void testNameCollision() { PrintSettings settings = new PrintSettings(true, Locale.ROOT, null, new ConqueryConfig(), null, null, new CsvResultPrinters()); final UniqueNamer uniqueNamer = new UniqueNamer(settings); - final ExternalResultInfo info1 = new ExternalResultInfo("test", ResultType.Primitive.STRING, settings); - final ExternalResultInfo info2 = new ExternalResultInfo("test", ResultType.Primitive.STRING, settings); - final ExternalResultInfo info3 = new ExternalResultInfo("test_1", ResultType.Primitive.STRING, settings); - final ExternalResultInfo info4 = new ExternalResultInfo("test", ResultType.Primitive.STRING, settings); + final ExternalResultInfo info1 = new ExternalResultInfo("test", ResultType.Primitive.STRING); + final ExternalResultInfo info2 = new ExternalResultInfo("test", ResultType.Primitive.STRING); + final ExternalResultInfo info3 = new ExternalResultInfo("test_1", ResultType.Primitive.STRING); + final ExternalResultInfo info4 = new ExternalResultInfo("test", ResultType.Primitive.STRING); - assertThat(uniqueNamer.getUniqueName(info1)).isEqualTo("test"); - assertThat(uniqueNamer.getUniqueName(info2)).isEqualTo("test_1"); - assertThat(uniqueNamer.getUniqueName(info3)).isEqualTo("test_1_1"); - assertThat(uniqueNamer.getUniqueName(info4)).isEqualTo("test_2"); + assertThat(uniqueNamer.getUniqueName(info1, settings)).isEqualTo("test"); + assertThat(uniqueNamer.getUniqueName(info2, settings)).isEqualTo("test_1"); + assertThat(uniqueNamer.getUniqueName(info3, settings)).isEqualTo("test_1_1"); + assertThat(uniqueNamer.getUniqueName(info4, settings)).isEqualTo("test_2"); } } diff --git a/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java b/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java index c99a5d580c..19464e6afa 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java @@ -42,9 +42,6 @@ public class ResultTypeTest { @SuppressWarnings("unused") public static List testData() { return List.of( - //TODO Arguments.of(PRETTY, ConqueryConstants.formResolutionInfo(PLAIN), Resolution.COMPLETE.name(), "complete"), -//TODO Arguments.of(PRETTY_DE, ConqueryConstants.formResolutionInfo(PLAIN), Resolution.COMPLETE.name(), "Gesamt"), - Arguments.of(PRETTY, ResultType.Primitive.BOOLEAN, true, "Yes"), Arguments.of(PRETTY, ResultType.Primitive.BOOLEAN, false, "No"), Arguments.of(PRETTY, ResultType.Primitive.STRING, "test", "test"), @@ -86,16 +83,16 @@ public static List testData() { ); } - public static ResultInfo info(ResultType type, PrintSettings settings) { - return new ExternalResultInfo("col", type, settings); + public static ResultInfo info(ResultType type) { + return new ExternalResultInfo("col", type); } @ParameterizedTest(name = "{0} {1}: {2} -> {3}") @MethodSource("testData") - public void testPrinting(PrintSettings cfg, ResultType type, Object value, String expected) throws IOException { - ResultInfo info = info(type, cfg); + public void testPrinting(PrintSettings printSettings, ResultType type, Object value, String expected) throws IOException { + ResultInfo info = info(type); - final Printer printer = info.getPrinter(); + final Printer printer = info.createPrinter(printSettings); assertThat(printer.apply(value)).isEqualTo(expected); @@ -107,10 +104,10 @@ public void testPrinting(PrintSettings cfg, ResultType type, Object value, Strin @ParameterizedTest(name = "{1}: {2}") @MethodSource("testData") - public void testBinaryPrinting(PrintSettings cfg, ResultType type, Object value, String expected) throws IOException { - ResultInfo info = info(type, cfg); + public void testBinaryPrinting(PrintSettings printSettings, ResultType type, Object value, String expected) throws IOException { + ResultInfo info = info(type); - final Printer printer = info.getPrinter(); + final Printer printer = info.createPrinter(printSettings); assertThat(printer.apply(value)).isEqualTo(expected); final byte[] bytes = Jackson.BINARY_MAPPER.writeValueAsBytes(value); From b97fc49bb7e42b6d2d1b4ecc23d35483d8f6e7b9 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 29 Aug 2024 17:55:39 +0200 Subject: [PATCH 30/54] use DECIMAL in Arrow and Parquet for MONEY --- .../io/result/arrow/ArrowRenderer.java | 30 +++++++-------- .../conquery/io/result/arrow/ArrowUtil.java | 22 ++++++----- .../io/result/excel/ExcelRenderer.java | 17 +++++---- .../parquet/EntityResultWriteSupport.java | 37 ++++++++++++------- .../io/result/parquet/ParquetRenderer.java | 2 +- .../arrow/ArrowResultGenerationTest.java | 3 +- .../parquet/ParquetResultGenerationTest.java | 2 +- 7 files changed, 64 insertions(+), 49 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java index 4be9e00e96..8b9918084a 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java @@ -3,6 +3,7 @@ import static com.bakdata.conquery.io.result.arrow.ArrowUtil.ROOT_ALLOCATOR; import java.io.IOException; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -23,8 +24,8 @@ import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.DateDayVector; +import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.Float4Vector; import org.apache.arrow.vector.Float8Vector; import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.ValueVector; @@ -115,11 +116,14 @@ public static void write( final int colId = index + idWriters.length; // In this case, the printer normalizes and adjusts values. - final Object printed = printers.get(colId).apply(line[index]); + final Object value = line[index]; - if (printed == null) { - continue; + Object printed = null; + + if (value != null) { + printed = printers.get(colId).apply(value); } + valueWriters[index].accept(batchLineCount, printed); } @@ -181,20 +185,20 @@ private static RowConsumer bitVectorFiller(BitVector vector) { }; } - private static RowConsumer float8VectorFiller(Float8Vector vector) { + private static RowConsumer moneyVectorFiller(DecimalVector vector) { return (rowNumber, valueRaw) -> { if (valueRaw == null) { vector.setNull(rowNumber); return; } - final Number value = (Number) valueRaw; + final BigDecimal value = (BigDecimal) valueRaw; - vector.setSafe(rowNumber, value.doubleValue()); + vector.setSafe(rowNumber, value); }; } - private static RowConsumer float4VectorFiller(Float4Vector vector) { + private static RowConsumer float8VectorFiller(Float8Vector vector) { return (rowNumber, valueRaw) -> { if (valueRaw == null) { vector.setNull(rowNumber); @@ -202,7 +206,8 @@ private static RowConsumer float4VectorFiller(Float4Vector vector) { } final Number value = (Number) valueRaw; - vector.setSafe(rowNumber, value.floatValue()); + + vector.setSafe(rowNumber, value.doubleValue()); }; } @@ -251,7 +256,6 @@ private static RowConsumer dateRangeVectorFiller(StructVector vector) { final CDateRange value = (CDateRange) valueRaw; - //TODO are we interested in infinities here? minConsumer.accept(rowNumber, value.getMinValue()); maxConsumer.accept(rowNumber, value.getMaxValue()); @@ -306,15 +310,11 @@ private static RowConsumer generateVectorFiller(ValueVector vector, ResultType t return listVectorFiller(((ListVector) vector), generateVectorFiller(nestedVector, listT.getElementType())); } - //TODO who dis? - // if (vector instanceof Float4Vector float4Vector) { - // return float4VectorFiller(float4Vector, (line) -> (Number) line[pos]); - // } return switch (((ResultType.Primitive) type)) { case BOOLEAN -> bitVectorFiller(((BitVector) vector)); case INTEGER -> intVectorFiller(((IntVector) vector)); - case MONEY -> float8VectorFiller(((Float8Vector) vector)); + case MONEY -> moneyVectorFiller(((DecimalVector) vector)); case DATE -> dateDayVectorFiller(((DateDayVector) vector)); case NUMERIC -> float8VectorFiller((Float8Vector) vector); case STRING -> varCharVectorFiller(((VarCharVector) vector)); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java index 67d1977cca..2c00b09abc 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java @@ -22,15 +22,15 @@ public class ArrowUtil { public static final RootAllocator ROOT_ALLOCATOR = new RootAllocator(); - private Field fieldFor(ResultType type, String name) { + private Field fieldFor(ResultType type, String name, PrintSettings settings) { if (type instanceof ResultType.ListT) { - return ArrowUtil.listField(name, type); + return ArrowUtil.listField(name, type, settings); } return switch (((ResultType.Primitive) type)) { case BOOLEAN -> ArrowUtil.boolField(name); case INTEGER -> ArrowUtil.integerField(name); - case MONEY -> ArrowUtil.moneyField(name); + case MONEY -> ArrowUtil.moneyField(name, settings.getCurrency().getDefaultFractionDigits()); case NUMERIC -> ArrowUtil.floatField(name); case DATE -> ArrowUtil.dateField(name); case DATE_RANGE -> ArrowUtil.dateRangeField(name); @@ -51,9 +51,13 @@ private static Field integerField(@NonNull String uniqueName) { return new Field(uniqueName, FieldType.nullable(new ArrowType.Int(32, true)), null); } - private static Field moneyField(@NonNull String uniqueName) { - //TODO We could use Decimal here? - return new Field(uniqueName, FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), null); + private static Field moneyField(@NonNull String uniqueName, int scale) { + /* + * From https://arrow.apache.org/docs/python/generated/pyarrow.decimal128.html + * Maximum precision is 38 digits total, therefore we can assume to pack all digits, minus scale in there; + * assuming that no currency will exceed 28 digits, this should be more than fine as a heuristic. + */ + return new Field(uniqueName, FieldType.nullable(new ArrowType.Decimal(38 - scale, scale, 128)), null); } private static Field floatField(@NonNull String uniqueName) { @@ -74,16 +78,16 @@ private static Field dateRangeField(@NonNull String uniqueName) { )); } - private static Field listField(@NonNull String uniqueName, ResultType type) { + private static Field listField(@NonNull String uniqueName, ResultType type, PrintSettings printSettings) { final ResultType elementType = ((ResultType.ListT) type).getElementType(); - final Field nestedField = fieldFor(elementType, uniqueName); + final Field nestedField = fieldFor(elementType, uniqueName, printSettings); return new Field(uniqueName, FieldType.nullable(ArrowType.List.INSTANCE), List.of(nestedField)); } public static List generateFields(@NonNull List info, UniqueNamer collector, PrintSettings printSettings) { return info.stream() - .map(i -> fieldFor(i.getType(), collector.getUniqueName(i, printSettings))) + .map(i -> fieldFor(i.getType(), collector.getUniqueName(i, printSettings), printSettings)) .toList(); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java index be4e868caa..2e91ca598b 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java @@ -53,9 +53,9 @@ private static TypeWriter writer(ResultType type, Printer printer, PrintSettings return switch (((ResultType.Primitive) type)) { case BOOLEAN -> (value, cell, styles) -> writeBooleanCell(value, cell, printer); case INTEGER -> (value, cell, styles) -> writeIntegerCell(value, cell, printer, styles); - case MONEY -> (value, cell, styles1) -> writeMoneyCell(value, cell, settings, styles1); + case MONEY -> (value, cell, styles) -> writeMoneyCell(value, cell, printer, settings, styles); case NUMERIC -> (value, cell, styles) -> writeNumericCell(value, cell, printer, styles); - case DATE -> (value, cell, styles1) -> writeDateCell(value, cell, styles1, printer); + case DATE -> (value, cell, styles) -> writeDateCell(value, cell, printer, styles); default -> (value, cell, styles) -> writeStringCell(cell, value, printer); }; } @@ -316,7 +316,7 @@ private static void writeBooleanCell(Object value, Cell cell, Printer printer) { cell.setCellValue((Boolean) printer.apply(value)); } - private static void writeDateCell(Object value, Cell cell, Map styles, Printer printer) { + private static void writeDateCell(Object value, Cell cell, Printer printer, Map styles) { cell.setCellValue((LocalDate) printer.apply(value)); cell.setCellStyle(styles.get(ExcelConfig.DATE_STYLE)); } @@ -331,14 +331,17 @@ public static void writeNumericCell(Object value, Cell cell, Printer printer, Ma cell.setCellStyle(styles.get(ExcelConfig.NUMERIC_STYLE)); } - public static void writeMoneyCell(Object value, Cell cell, PrintSettings settings, Map styles) { + public static void writeMoneyCell(Object valueRaw, Cell cell, Printer printer, PrintSettings settings, Map styles) { + + final BigDecimal value = (BigDecimal) printer.apply(valueRaw); + final CellStyle currencyStyle = styles.get(ExcelConfig.CURRENCY_STYLE_PREFIX + settings.getCurrency().getCurrencyCode()); if (currencyStyle == null) { - // Print as cents or what ever the minor currency unit is - cell.setCellValue(((BigDecimal) value).movePointRight(settings.getCurrency().getDefaultFractionDigits()).intValue()); + // Print as cents or whatever the minor currency unit is + cell.setCellValue(value.movePointRight(settings.getCurrency().getDefaultFractionDigits()).intValue()); return; } cell.setCellStyle(currencyStyle); - cell.setCellValue(((BigDecimal) value).doubleValue()); + cell.setCellValue(value.doubleValue()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java index a430e2deb7..6f8cab4e31 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.io.result.parquet; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -23,6 +24,7 @@ import org.apache.parquet.io.api.Binary; import org.apache.parquet.io.api.RecordConsumer; import org.apache.parquet.schema.MessageType; +import org.jetbrains.annotations.NotNull; /** * {@link WriteSupport} for Conquery's {@link EntityResult} type. @@ -58,7 +60,9 @@ public static MessageType generateSchema(List idHeaders, List Date: Thu, 29 Aug 2024 18:00:01 +0200 Subject: [PATCH 31/54] Code Style issues from Code Review --- .../conquery/io/result/csv/ResultCsvProcessor.java | 4 ++-- .../io/result/excel/ResultExcelProcessor.java | 4 ++-- .../models/config/ExcelResultProvider.java | 4 ++-- .../query/resultinfo/ExternalResultInfo.java | 1 - .../resultinfo/printers/ExcelResultPrinters.java | 8 -------- .../resultinfo/printers/JsonResultPrinters.java | 14 +++----------- ...sultPrinters.java => StringResultPrinters.java} | 2 +- .../query/resultinfo/printers/ToStringPrinter.java | 13 +++++++++++++ .../execution/DefaultSqlCDateSetParserTest.java | 4 ++-- .../integration/json/AbstractQueryEngineTest.java | 4 ++-- .../conquery/integration/json/FormTest.java | 4 ++-- .../io/result/csv/CsvResultGenerationTest.java | 4 ++-- .../io/result/excel/ExcelResultRenderTest.java | 4 ++-- .../conquery/models/query/UniqueNameTest.java | 4 ++-- .../conquery/models/types/ResultTypeTest.java | 8 ++++---- 15 files changed, 39 insertions(+), 43 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java rename backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/{CsvResultPrinters.java => StringResultPrinters.java} (98%) create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ToStringPrinter.java diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java index 655061f79b..78cb604251 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java @@ -18,7 +18,7 @@ import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; -import com.bakdata.conquery.models.query.resultinfo.printers.CsvResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.StringResultPrinters; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.resources.ResourceConstants; @@ -58,7 +58,7 @@ public Response createResult(Su // Get the locale extracted by the LocaleFilter final Locale locale = I18n.LOCALE.get(); - final PrintSettings settings = new PrintSettings(pretty, locale, namespace, config, idPrinter::createId, null, new CsvResultPrinters()); + final PrintSettings settings = new PrintSettings(pretty, locale, namespace, config, idPrinter::createId, null, new StringResultPrinters()); final StreamingOutput out = os -> { try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(os, charset))) { diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java index d7d393d726..3acf25875e 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java @@ -15,7 +15,7 @@ import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; -import com.bakdata.conquery.models.query.resultinfo.printers.CsvResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.StringResultPrinters; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.resources.ResourceConstants; @@ -53,7 +53,7 @@ public Response createResult(Su final IdPrinter idPrinter = IdColumnUtil.getIdPrinter(subject, exec, namespace, conqueryConfig.getIdColumns().getIds()); final Locale locale = I18n.LOCALE.get(); - final PrintSettings settings = new PrintSettings(pretty, locale, namespace, conqueryConfig, idPrinter::createId, null, new CsvResultPrinters()); + final PrintSettings settings = new PrintSettings(pretty, locale, namespace, conqueryConfig, idPrinter::createId, null, new StringResultPrinters()); final ExcelRenderer excelRenderer = new ExcelRenderer(excelConfig, settings); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java index eb149cd3d0..560334d5bb 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java @@ -16,7 +16,7 @@ import com.bakdata.conquery.models.i18n.I18n; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; -import com.bakdata.conquery.models.query.resultinfo.printers.ExcelResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.JavaResultPrinters; import com.bakdata.conquery.resources.api.ResultExcelResource; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.jersey.DropwizardResourceConfig; @@ -74,7 +74,7 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuil return Collections.emptyList(); } - final PrintSettings printSettings = new PrintSettings(true, I18n.LOCALE.get(), exec.getNamespace(), exec.getConfig(), null, null, new ExcelResultPrinters()); + final PrintSettings printSettings = new PrintSettings(true, I18n.LOCALE.get(), exec.getNamespace(), exec.getConfig(), null, null, new JavaResultPrinters()); // Save id column count to later check if xlsx dimensions are feasible idColumnsCount = exec.getConfig().getIdColumns().getIdResultInfos().size(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ExternalResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ExternalResultInfo.java index a6d2a05e55..99f494d9dc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ExternalResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ExternalResultInfo.java @@ -15,7 +15,6 @@ public class ExternalResultInfo extends ResultInfo { private final String name; private final ResultType type; - public ExternalResultInfo(String name, ResultType type) { super(Collections.emptySet()); this.name = name; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java deleted file mode 100644 index c833eef5c3..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.bakdata.conquery.models.query.resultinfo.printers; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ExcelResultPrinters extends JavaResultPrinters { - -} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java index d3aa344f8d..7d29f8bbf7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java @@ -1,17 +1,18 @@ package com.bakdata.conquery.models.query.resultinfo.printers; import java.math.BigDecimal; -import java.util.Objects; import com.bakdata.conquery.models.query.PrintSettings; import com.fasterxml.jackson.databind.node.BooleanNode; import com.fasterxml.jackson.databind.node.DecimalNode; import com.fasterxml.jackson.databind.node.IntNode; import com.fasterxml.jackson.databind.node.TextNode; +import lombok.ToString; +@ToString public class JsonResultPrinters extends PrinterFactory { - private final JavaResultPrinters delegate = new JavaResultPrinters(); + private final PrinterFactory delegate = new JavaResultPrinters(); @Override public Printer getListPrinter(Printer elementPrinter, PrintSettings printSettings) { @@ -57,17 +58,8 @@ public Printer getNumericPrinter(PrintSettings printSettings) { return new NumericPrinter(); } - public record ToStringPrinter(Printer delegate) implements Printer { - - @Override - public Object apply(Object value) { - return new TextNode(Objects.toString(delegate.apply(value))); - } - } - @Override public Printer getDatePrinter(PrintSettings printSettings) { - //TODO compare with current impl return new ToStringPrinter(delegate.getDatePrinter(printSettings)); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/CsvResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java similarity index 98% rename from backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/CsvResultPrinters.java rename to backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java index 62dbc36897..6ca76dc394 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/CsvResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java @@ -14,7 +14,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public class CsvResultPrinters extends PrinterFactory { +public class StringResultPrinters extends PrinterFactory { @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ToStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ToStringPrinter.java new file mode 100644 index 0000000000..4b476a7fd0 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ToStringPrinter.java @@ -0,0 +1,13 @@ +package com.bakdata.conquery.models.query.resultinfo.printers; + +import java.util.Objects; + +import com.fasterxml.jackson.databind.node.TextNode; + +public record ToStringPrinter(Printer delegate) implements Printer { + + @Override + public Object apply(Object value) { + return new TextNode(Objects.toString(delegate.apply(value))); + } +} diff --git a/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java b/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java index 55ecfca49a..c4fdae0196 100644 --- a/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java +++ b/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java @@ -6,7 +6,7 @@ import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.query.PrintSettings; -import com.bakdata.conquery.models.query.resultinfo.printers.CsvResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.StringResultPrinters; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.sql.execution.DefaultSqlCDateSetParser; import org.junit.jupiter.api.Assertions; @@ -17,7 +17,7 @@ class DefaultSqlCDateSetParserTest { private static final DefaultSqlCDateSetParser parser = new DefaultSqlCDateSetParser(); - private static final CsvResultPrinters csvResultPrinters = new CsvResultPrinters(); + private static final StringResultPrinters csvResultPrinters = new StringResultPrinters(); private static final ConqueryConfig CONFIG = new ConqueryConfig(); private static final PrintSettings PLAIN = new PrintSettings(false, Locale.ENGLISH, null, CONFIG, null, null, csvResultPrinters); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java index 2cb6104c13..5f2a2b5d16 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java @@ -23,7 +23,7 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.CsvResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.StringResultPrinters; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.MultilineEntityResult; import com.bakdata.conquery.resources.api.ResultCsvResource; @@ -58,7 +58,7 @@ public void executeTest(StandaloneSupport standaloneSupport) throws IOException SingleTableResult executionResult = (SingleTableResult) execution; //check result info size - PrintSettings printSettings = new PrintSettings(true, Locale.ROOT, standaloneSupport.getNamespace(), standaloneSupport.getConfig(), null, null, new CsvResultPrinters()); + PrintSettings printSettings = new PrintSettings(true, Locale.ROOT, standaloneSupport.getNamespace(), standaloneSupport.getConfig(), null, null, new StringResultPrinters()); List resultInfos = executionResult.getResultInfos(); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java index f0cb3b5e26..0a1342e6d6 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java @@ -31,7 +31,7 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.CsvResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.StringResultPrinters; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.util.io.IdColumnUtil; import com.bakdata.conquery.util.support.StandaloneSupport; @@ -131,7 +131,7 @@ private void checkResults(StandaloneSupport standaloneSupport, ManagedInternalFo final ConqueryConfig config = standaloneSupport.getConfig(); PrintSettings printSettings = - new PrintSettings(false, Locale.ENGLISH, standaloneSupport.getNamespace(), config, idPrinter::createId, null, new CsvResultPrinters()); + new PrintSettings(false, Locale.ENGLISH, standaloneSupport.getNamespace(), config, idPrinter::createId, null, new StringResultPrinters()); checkSingleResult(managedForm, config, printSettings); diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java index ee1f71b31f..8239036e6b 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java @@ -18,7 +18,7 @@ import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.CsvResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.StringResultPrinters; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.util.NonPersistentStoreFactory; @@ -46,7 +46,7 @@ void writeAndRead() throws IOException { null, CONFIG, (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), - (selectInfo) -> selectInfo.getSelect().getLabel(), new CsvResultPrinters() + (selectInfo) -> selectInfo.getSelect().getLabel(), new StringResultPrinters() ); // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized final List results = getTestEntityResults(); diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java index 2593e9b1c5..a46903f421 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java @@ -27,7 +27,7 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.CsvResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.StringResultPrinters; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.util.NonPersistentStoreFactory; @@ -59,7 +59,7 @@ void writeAndRead() throws IOException { final PrintSettings printSettings = new PrintSettings(true, Locale.GERMAN, null, CONFIG, (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), (selectInfo) -> selectInfo.getSelect() - .getLabel(), new CsvResultPrinters()); // TODO ? + .getLabel(), new StringResultPrinters()); // TODO ? // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized final List results = getTestEntityResults(); diff --git a/backend/src/test/java/com/bakdata/conquery/models/query/UniqueNameTest.java b/backend/src/test/java/com/bakdata/conquery/models/query/UniqueNameTest.java index 4b28433098..251476c745 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/query/UniqueNameTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/query/UniqueNameTest.java @@ -7,14 +7,14 @@ import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.query.resultinfo.ExternalResultInfo; import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; -import com.bakdata.conquery.models.query.resultinfo.printers.CsvResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.StringResultPrinters; import com.bakdata.conquery.models.types.ResultType; import org.junit.jupiter.api.Test; public class UniqueNameTest { @Test void testNameCollision() { - PrintSettings settings = new PrintSettings(true, Locale.ROOT, null, new ConqueryConfig(), null, null, new CsvResultPrinters()); + PrintSettings settings = new PrintSettings(true, Locale.ROOT, null, new ConqueryConfig(), null, null, new StringResultPrinters()); final UniqueNamer uniqueNamer = new UniqueNamer(settings); final ExternalResultInfo info1 = new ExternalResultInfo("test", ResultType.Primitive.STRING); diff --git a/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java b/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java index 19464e6afa..b656d94114 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java @@ -17,8 +17,8 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.ExternalResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.CsvResultPrinters; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.bakdata.conquery.models.query.resultinfo.printers.StringResultPrinters; import com.google.common.collect.ImmutableMap; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -27,9 +27,9 @@ public class ResultTypeTest { public static final ConqueryConfig CONFIG = new ConqueryConfig(); - private static final PrintSettings PRETTY = new PrintSettings(true, Locale.ENGLISH, null, CONFIG, null, null, new CsvResultPrinters()); - private static final PrintSettings PRETTY_DE = new PrintSettings(true, Locale.GERMANY, null, CONFIG, null, null, new CsvResultPrinters()); - private static final PrintSettings PLAIN = new PrintSettings(false, Locale.ENGLISH, null, CONFIG, null, null, new CsvResultPrinters()); + private static final PrintSettings PRETTY = new PrintSettings(true, Locale.ENGLISH, null, CONFIG, null, null, new StringResultPrinters()); + private static final PrintSettings PRETTY_DE = new PrintSettings(true, Locale.GERMANY, null, CONFIG, null, null, new StringResultPrinters()); + private static final PrintSettings PLAIN = new PrintSettings(false, Locale.ENGLISH, null, CONFIG, null, null, new StringResultPrinters()); static { // Initialization of the internationalization From e95f8f7566b440d1f2e4c9225c12dc99f5fe304f Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 29 Aug 2024 18:31:21 +0200 Subject: [PATCH 32/54] fix test comparison of BigDecmial --- .../conquery/io/result/ResultTestUtil.java | 45 ++++++++++++++++++- .../arrow/ArrowResultGenerationTest.java | 4 ++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java index 7af4476e2b..5e626a1c47 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java @@ -72,12 +72,53 @@ public Stream streamResults(OptionalLong maybeLimit) { @NotNull public static List getResultTypes() { - return List.of(ResultType.Primitive.BOOLEAN, ResultType.Primitive.INTEGER, ResultType.Primitive.NUMERIC, ResultType.Primitive.DATE, ResultType.Primitive.DATE_RANGE, ResultType.Primitive.STRING, ResultType.Primitive.MONEY, new ResultType.ListT(ResultType.Primitive.BOOLEAN), new ResultType.ListT(ResultType.Primitive.DATE_RANGE), new ResultType.ListT(ResultType.Primitive.STRING)); + return List.of(ResultType.Primitive.BOOLEAN, + ResultType.Primitive.INTEGER, + ResultType.Primitive.NUMERIC, + ResultType.Primitive.DATE, + ResultType.Primitive.DATE_RANGE, + ResultType.Primitive.STRING, + ResultType.Primitive.MONEY, + new ResultType.ListT(ResultType.Primitive.BOOLEAN), + new ResultType.ListT(ResultType.Primitive.DATE_RANGE), + new ResultType.ListT(ResultType.Primitive.STRING) + ); } @NotNull public static List getTestEntityResults() { - return List.of(new SinglelineEntityResult("1", new Object[]{Boolean.TRUE, 2345634, 123423.34, 5646, List.of(345, 534), "test_string", new BigDecimal("45.21"), List.of(true, false), List.of(List.of(345, 534), List.of(1, 2)), List.of("fizz", "buzz")}), new SinglelineEntityResult("2", new Object[]{Boolean.FALSE, null, null, null, null, null, null, List.of(), List.of(List.of(1234, Integer.MAX_VALUE)), List.of()}), new SinglelineEntityResult("2", new Object[]{Boolean.TRUE, null, null, null, null, null, null, List.of(false, false), null, null}), new MultilineEntityResult("3", List.of(new Object[]{Boolean.FALSE, null, null, null, null, null, null, List.of(false), null, null}, new Object[]{Boolean.TRUE, null, null, null, null, null, null, null, null, null}, new Object[]{Boolean.TRUE, null, null, null, null, null, new BigDecimal(4), List.of(true, false, true, false), null, null}))); + return List.of( + new SinglelineEntityResult("1", + new Object[]{ + Boolean.TRUE, 2345634, 123423.34, 5646, List.of(345, + 534 + ), "test_string", new BigDecimal("45.21"), List.of(true, false), List.of(List.of(345, 534), List.of(1, 2)), List.of( + "fizz", + "buzz" + ) + } + ), + new SinglelineEntityResult("2", + new Object[]{ + Boolean.FALSE, null, null, null, null, null, null, List.of(), List.of(List.of(1234, + Integer.MAX_VALUE + )), List.of() + } + ), + new SinglelineEntityResult("2", new Object[]{Boolean.TRUE, null, null, null, null, null, null, List.of(false, false), null, null}), + new MultilineEntityResult("3", + List.of(new Object[]{Boolean.FALSE, null, null, null, null, null, null, List.of(false), null, null}, + new Object[]{Boolean.TRUE, null, null, null, null, null, null, null, null, null}, + new Object[]{ + Boolean.TRUE, null, null, null, null, null, new BigDecimal("4.00"), List.of(true, + false, + true, + false + ), null, null + } + ) + ) + ); } public static class TypedSelectDummy extends Select { diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java index a1cc47b3b0..5105e25a46 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java @@ -10,6 +10,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -213,6 +214,9 @@ private static String getPrintValue(Object obj, ResultType type) { if (obj == null) { return "null"; } + if (type.equals(ResultType.Primitive.MONEY)) { + return new String(((BigDecimal)obj).unscaledValue().toByteArray()); + } if (type.equals(ResultType.Primitive.DATE_RANGE)) { // Special case for daterange in this test because it uses a StructVector, we rebuild the structural information List dr = (List) obj; From 4239f39592e31cfec5c7521828488e37a2006f10 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 29 Aug 2024 18:41:44 +0200 Subject: [PATCH 33/54] fix handling of Money in ArrowResultGenerationTest separately to ParquetResultGenerationTest --- .../conquery/io/result/arrow/ArrowRenderer.java | 4 +--- .../io/result/arrow/ArrowResultGenerationTest.java | 13 +++++++------ .../result/parquet/ParquetResultGenerationTest.java | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java index 8b9918084a..5d2362d5c2 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java @@ -277,9 +277,7 @@ private static RowConsumer listVectorFiller(ListVector vector, RowConsumer neste final int start = vector.startNewValue(rowNumber); for (int i = 0; i < values.size(); i++) { - // These short-lived one value arrays are a workaround at the moment - //TODO consider using only a single array and overwriting it every row - this works only if Arrow processes the values immediately - nestedConsumer.accept(Math.addExact(start, i), new Object[]{values.get(i)}); + nestedConsumer.accept(Math.addExact(start, i), values.get(i)); } vector.endValue(rowNumber, values.size()); diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java index 5105e25a46..11239bc80e 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java @@ -153,7 +153,7 @@ void writeAndRead() throws IOException { String computed = readTSV(inputStream); assertThat(computed).isNotBlank(); - assertThat(computed).isEqualTo(generateExpectedTSV(results, mquery.getResultInfos())); + assertThat(computed).isEqualTo(generateExpectedTSV(results, mquery.getResultInfos(), false)); } @@ -180,7 +180,7 @@ public static String readTSV(InputStream inputStream) throws IOException { return stringJoiner.toString(); } - public static String generateExpectedTSV(List results, List resultInfos) { + public static String generateExpectedTSV(List results, List resultInfos, boolean isParquet) { String expected = results.stream() .map(EntityResult.class::cast) .map(res -> { @@ -193,7 +193,7 @@ public static String generateExpectedTSV(List results, List results, List col = (Collection) obj; // Workaround: Arrow deserializes lists as a JsonStringArrayList which has a JSON String method @NonNull ResultType elemType = ((ResultType.ListT) type).getElementType(); - return col.stream().map(v -> getPrintValue(v, elemType)).collect(Collectors.joining(",", "[", "]")); + return col.stream().map(v -> getPrintValue(v, elemType, isParquet)).collect(Collectors.joining(",", "[", "]")); } return obj.toString(); } diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java index 558c998ae0..cea9cf4d1d 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java @@ -142,7 +142,7 @@ void writeAndRead() throws IOException { log.info("\n{}", actual); - assertThat(actual).isEqualTo(ArrowResultGenerationTest.generateExpectedTSV(results, managedQuery.getResultInfos())); + assertThat(actual).isEqualTo(ArrowResultGenerationTest.generateExpectedTSV(results, managedQuery.getResultInfos(), true)); } From 8e9ed1ee29e793faf29df84b2f2129dfffdd6ffd Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Mon, 2 Sep 2024 18:42:34 +0200 Subject: [PATCH 34/54] cleanup structure of PrinterFactory. --- .../com/bakdata/conquery/ResultHeaders.java | 2 +- .../apiv1/query/SecondaryIdQuery.java | 2 +- .../apiv1/query/TableExportQuery.java | 2 +- .../io/result/excel/ExcelRenderer.java | 132 +++++----- .../io/result/excel/ResultExcelProcessor.java | 4 +- .../select/concept/ConceptColumnSelect.java | 2 +- .../select/connector/DistinctSelect.java | 2 +- .../specific/MappableSingleColumnSelect.java | 2 +- .../query/resultinfo/ColumnResultInfo.java | 2 +- .../{printers => }/SecondaryIdResultInfo.java | 5 +- .../printers/ExcelResultPrinters.java | 59 +++++ .../resultinfo/printers/IdentityPrinter.java | 9 - .../printers/JavaResultPrinters.java | 11 +- .../printers/JsonResultPrinters.java | 73 +----- .../resultinfo/printers/PrinterFactory.java | 2 +- .../resultinfo/printers/ResultPrinters.java | 227 ------------------ .../printers/StringResultPrinters.java | 157 ++---------- .../resultinfo/printers/ToStringPrinter.java | 13 - .../printers/common/BooleanStringPrinter.java | 28 +++ .../{ => common}/ConceptIdPrinter.java | 3 +- .../printers/common/DatePrinter.java | 12 + .../common/DateRangeStringPrinter.java | 41 ++++ .../printers/common/DateStringPrinter.java | 17 ++ .../printers/common/IdentityPrinter.java | 11 + .../printers/common/IntegerStringPrinter.java | 16 ++ .../printers/common/ListStringPrinter.java | 30 +++ .../{ => common}/LocalizedEnumPrinter.java | 3 +- .../printers/{ => common}/MappedPrinter.java | 3 +- .../printers/common/MoneyStringPrinter.java | 17 ++ .../printers/common/NumericStringPrinter.java | 16 ++ .../printers/common/StringPrinter.java | 12 + .../printers/common/ToStringPrinter.java | 13 + .../conquery/io/result/ResultTestUtil.java | 4 +- .../result/excel/ExcelResultRenderTest.java | 62 +++-- 34 files changed, 422 insertions(+), 572 deletions(-) rename backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/{printers => }/SecondaryIdResultInfo.java (85%) create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/IdentityPrinter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ToStringPrinter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/BooleanStringPrinter.java rename backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/{ => common}/ConceptIdPrinter.java (84%) create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DatePrinter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateRangeStringPrinter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateStringPrinter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/IdentityPrinter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/IntegerStringPrinter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java rename backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/{ => common}/LocalizedEnumPrinter.java (80%) rename backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/{ => common}/MappedPrinter.java (62%) create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MoneyStringPrinter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/NumericStringPrinter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/StringPrinter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ToStringPrinter.java diff --git a/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java b/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java index e71021431f..5fff3d083b 100644 --- a/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java +++ b/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java @@ -9,8 +9,8 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.FixedLabelResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.LocalizedEnumPrinter; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.bakdata.conquery.models.query.resultinfo.printers.common.LocalizedEnumPrinter; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import lombok.experimental.UtilityClass; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java index b294f984ac..acced1a092 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java @@ -26,7 +26,7 @@ import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; import com.bakdata.conquery.models.query.queryplan.SecondaryIdQueryPlan; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.SecondaryIdResultInfo; +import com.bakdata.conquery.models.query.resultinfo.SecondaryIdResultInfo; import com.fasterxml.jackson.annotation.JsonView; import jakarta.validation.constraints.NotNull; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java index 6ae8fef610..18176a428d 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java @@ -39,7 +39,7 @@ import com.bakdata.conquery.models.query.queryplan.TableExportQueryPlan; import com.bakdata.conquery.models.query.resultinfo.ColumnResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.SecondaryIdResultInfo; +import com.bakdata.conquery.models.query.resultinfo.SecondaryIdResultInfo; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java index 2e91ca598b..ccb2185817 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java @@ -16,6 +16,7 @@ import com.bakdata.conquery.models.config.ExcelConfig; import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.i18n.I18n; +import com.bakdata.conquery.models.identifiable.mapping.PrintIdMapper; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; @@ -43,32 +44,12 @@ public class ExcelRenderer { public static final int MAX_LINES = 1_048_576; - - private static TypeWriter writer(ResultType type, Printer printer, PrintSettings settings) { - if(!(type instanceof ResultType.Primitive)){ - //Excel cannot handle complex types so we just toString them. - return (value, cell, styles) -> writeStringCell(cell, value, printer); - } - - return switch (((ResultType.Primitive) type)) { - case BOOLEAN -> (value, cell, styles) -> writeBooleanCell(value, cell, printer); - case INTEGER -> (value, cell, styles) -> writeIntegerCell(value, cell, printer, styles); - case MONEY -> (value, cell, styles) -> writeMoneyCell(value, cell, printer, settings, styles); - case NUMERIC -> (value, cell, styles) -> writeNumericCell(value, cell, printer, styles); - case DATE -> (value, cell, styles) -> writeDateCell(value, cell, printer, styles); - default -> (value, cell, styles) -> writeStringCell(cell, value, printer); - }; - } - public static final int CHARACTER_WIDTH_DIVISOR = 256; public static final int AUTOFILTER_SPACE_WIDTH = 3; - private final SXSSFWorkbook workbook; private final ExcelConfig config; private final PrintSettings settings; private final ImmutableMap styles; - - public ExcelRenderer(ExcelConfig config, PrintSettings settings) { workbook = new SXSSFWorkbook(); this.config = config; @@ -76,11 +57,6 @@ public ExcelRenderer(ExcelConfig config, PrintSettings settings) { this.settings = settings; } - @FunctionalInterface - private interface TypeWriter { - void writeCell(Object value, Cell cell, Map styles); - } - public void renderToStream(List idHeaders, E exec, OutputStream outputStream, OptionalLong limit, PrintSettings printSettings) throws IOException { final List resultInfosExec = exec.getResultInfos(); @@ -122,28 +98,6 @@ private void setMetaData(E exec extendedProperties.setApplication(config.getApplicationName()); } - /** - * Do postprocessing on the result to improve the visuals: - * - Set the area of the table environment - * - Freeze the id columns - * - Add autofilters (not for now) - */ - private void postProcessTable(SXSSFSheet sheet, XSSFTable table, int writtenLines, int size) { - // Extend the table area to the added data - final CellReference topLeft = new CellReference(0, 0); - - // The area must be at least a header row and a data row. If no line was written we include an empty data row so POI is happy - final CellReference bottomRight = new CellReference(Math.max(1, writtenLines), table.getColumnCount() - 1); - final AreaReference newArea = new AreaReference(topLeft, bottomRight, workbook.getSpreadsheetVersion()); - table.setArea(newArea); - - // Add auto filters. This must be done on the lower level CTTable. Using SXSSFSheet::setAutoFilter will corrupt the table - table.getCTTable().addNewAutoFilter(); - - // Freeze Header and id columns - sheet.createFreezePane(size, 1); - } - /** * Create a table environment, which improves mainly the visuals of the produced table. */ @@ -221,7 +175,11 @@ private int writeBody( // Row 0 is the Header the data starts at 1 final AtomicInteger currentRow = new AtomicInteger(1); - final int writtenLines = resultLines.mapToInt(l -> writeRowsForEntity(infos, l, currentRow, settings, sheet)).sum(); + + final TypeWriter[] writers = infos.stream().map(info -> writer(info.getType(), info.createPrinter(settings), settings)).toArray(TypeWriter[]::new); + final PrintIdMapper idMapper = settings.getIdMapper(); + + final int writtenLines = resultLines.mapToInt(l -> writeRowsForEntity(infos, l, currentRow, sheet, writers, idMapper)).sum(); // The result was shorter than the number of rows to track, so we auto size here explicitly if (writtenLines < config.getLastRowToAutosize()) { @@ -231,17 +189,38 @@ private int writeBody( return writtenLines; } + /** + * Do postprocessing on the result to improve the visuals: + * - Set the area of the table environment + * - Freeze the id columns + * - Add autofilters (not for now) + */ + private void postProcessTable(SXSSFSheet sheet, XSSFTable table, int writtenLines, int size) { + // Extend the table area to the added data + final CellReference topLeft = new CellReference(0, 0); + + // The area must be at least a header row and a data row. If no line was written we include an empty data row so POI is happy + final CellReference bottomRight = new CellReference(Math.max(1, writtenLines), table.getColumnCount() - 1); + final AreaReference newArea = new AreaReference(topLeft, bottomRight, workbook.getSpreadsheetVersion()); + table.setArea(newArea); + + // Add auto filters. This must be done on the lower level CTTable. Using SXSSFSheet::setAutoFilter will corrupt the table + table.getCTTable().addNewAutoFilter(); + + // Freeze Header and id columns + sheet.createFreezePane(size, 1); + } + /** * Writes the result lines for each entity. */ - private int writeRowsForEntity(List infos, EntityResult internalRow, final AtomicInteger currentRow, PrintSettings settings, SXSSFSheet sheet) { + private int writeRowsForEntity(List infos, EntityResult internalRow, final AtomicInteger currentRow, SXSSFSheet sheet, TypeWriter[] writers, PrintIdMapper idMapper) { - final String[] ids = settings.getIdMapper().map(internalRow).getExternalId(); - final TypeWriter[] writers = infos.stream().map(info -> writer(info.getType(), info.createPrinter(settings), settings)).toArray(TypeWriter[]::new); + final String[] ids = idMapper.map(internalRow).getExternalId(); int writtenLines = 0; - for (Object[] resultValues : internalRow.listResultLines()) { + for (Object[] line : internalRow.listResultLines()) { final int thisRow = currentRow.getAndIncrement(); final Row row = sheet.createRow(thisRow); @@ -256,19 +235,19 @@ private int writeRowsForEntity(List infos, EntityResult internalRow, // Write data cells for (int index = 0; index < infos.size(); index++) { - final ResultInfo resultInfo = infos.get(index); - final Object resultValue = resultValues[index]; + final Object value = line[index]; final Cell dataCell = row.createCell(currentColumn); currentColumn++; - if (resultValue == null) { + if (value == null) { continue; } + // Fallback to string if type is not explicitly registered final TypeWriter typeWriter = writers[index]; - typeWriter.writeCell(resultValue, dataCell, styles); + typeWriter.writeCell(value, dataCell, styles); } if (thisRow == config.getLastRowToAutosize()) { @@ -303,6 +282,22 @@ private void setColumnWidthsAndUntrack(SXSSFSheet sheet) { } } + private static TypeWriter writer(ResultType type, Printer printer, PrintSettings settings) { + if (type instanceof ResultType.ListT) { + //Excel cannot handle LIST types so we just toString them. + return (value, cell, styles) -> writeStringCell(cell, value, printer); + } + + return switch (((ResultType.Primitive) type)) { + case BOOLEAN -> (value, cell, styles) -> writeBooleanCell(value, cell, printer); + case INTEGER -> (value, cell, styles) -> writeIntegerCell(value, cell, printer, styles); + case MONEY -> (value, cell, styles) -> writeMoneyCell(value, cell, printer, settings, styles); + case NUMERIC -> (value, cell, styles) -> writeNumericCell(value, cell, printer, styles); + case DATE -> (value, cell, styles) -> writeDateCell(value, cell, printer, styles); + default -> (value, cell, styles) -> writeStringCell(cell, value, printer); + }; + } + // Type specific cell writers private static void writeStringCell(Cell cell, Object value, Printer printer) { cell.setCellValue((String) printer.apply(value)); @@ -316,21 +311,11 @@ private static void writeBooleanCell(Object value, Cell cell, Printer printer) { cell.setCellValue((Boolean) printer.apply(value)); } - private static void writeDateCell(Object value, Cell cell, Printer printer, Map styles) { - cell.setCellValue((LocalDate) printer.apply(value)); - cell.setCellStyle(styles.get(ExcelConfig.DATE_STYLE)); - } - public static void writeIntegerCell(Object value, Cell cell, Printer printer, Map styles) { cell.setCellValue(((Number) printer.apply(value)).longValue()); cell.setCellStyle(styles.get(ExcelConfig.INTEGER_STYLE)); } - public static void writeNumericCell(Object value, Cell cell, Printer printer, Map styles) { - cell.setCellValue(((Number) printer.apply(value)).doubleValue()); - cell.setCellStyle(styles.get(ExcelConfig.NUMERIC_STYLE)); - } - public static void writeMoneyCell(Object valueRaw, Cell cell, Printer printer, PrintSettings settings, Map styles) { final BigDecimal value = (BigDecimal) printer.apply(valueRaw); @@ -344,4 +329,19 @@ public static void writeMoneyCell(Object valueRaw, Cell cell, Printer printer, P cell.setCellStyle(currencyStyle); cell.setCellValue(value.doubleValue()); } + + public static void writeNumericCell(Object value, Cell cell, Printer printer, Map styles) { + cell.setCellValue(((Number) printer.apply(value)).doubleValue()); + cell.setCellStyle(styles.get(ExcelConfig.NUMERIC_STYLE)); + } + + private static void writeDateCell(Object value, Cell cell, Printer printer, Map styles) { + cell.setCellValue((LocalDate) printer.apply(value)); + cell.setCellStyle(styles.get(ExcelConfig.DATE_STYLE)); + } + + @FunctionalInterface + private interface TypeWriter { + void writeCell(Object value, Cell cell, Map styles); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java index 3acf25875e..520f850aa2 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java @@ -15,7 +15,7 @@ import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; -import com.bakdata.conquery.models.query.resultinfo.printers.StringResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.ExcelResultPrinters; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.resources.ResourceConstants; @@ -53,7 +53,7 @@ public Response createResult(Su final IdPrinter idPrinter = IdColumnUtil.getIdPrinter(subject, exec, namespace, conqueryConfig.getIdColumns().getIds()); final Locale locale = I18n.LOCALE.get(); - final PrintSettings settings = new PrintSettings(pretty, locale, namespace, conqueryConfig, idPrinter::createId, null, new StringResultPrinters()); + final PrintSettings settings = new PrintSettings(pretty, locale, namespace, conqueryConfig, idPrinter::createId, null, new ExcelResultPrinters()); final ExcelRenderer excelRenderer = new ExcelRenderer(excelConfig, settings); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java index 13f920a2ce..c2d47992a7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java @@ -13,9 +13,9 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.ConceptElementsAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.ConceptValuesAggregator; import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.ConceptIdPrinter; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; +import com.bakdata.conquery.models.query.resultinfo.printers.common.ConceptIdPrinter; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import com.bakdata.conquery.sql.conversion.model.select.ConceptColumnSelectConverter; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java index b135160510..fe83b1c689 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java @@ -9,9 +9,9 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.AllValuesAggregator; -import com.bakdata.conquery.models.query.resultinfo.printers.MappedPrinter; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; +import com.bakdata.conquery.models.query.resultinfo.printers.common.MappedPrinter; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.sql.conversion.model.select.DistinctSelectConverter; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java index 08993a32ee..7412daaf3b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java @@ -15,9 +15,9 @@ import com.bakdata.conquery.models.index.InternToExternMapper; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.MappedPrinter; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; +import com.bakdata.conquery.models.query.resultinfo.printers.common.MappedPrinter; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import jakarta.validation.Valid; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java index 0a312dc9d7..8f80914cff 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java @@ -5,8 +5,8 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.query.PrintSettings; -import com.bakdata.conquery.models.query.resultinfo.printers.ConceptIdPrinter; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.bakdata.conquery.models.query.resultinfo.printers.common.ConceptIdPrinter; import com.bakdata.conquery.models.types.ResultType; import lombok.EqualsAndHashCode; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/SecondaryIdResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SecondaryIdResultInfo.java similarity index 85% rename from backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/SecondaryIdResultInfo.java rename to backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SecondaryIdResultInfo.java index e3efcf70a2..8c93e6cda6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/SecondaryIdResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SecondaryIdResultInfo.java @@ -1,10 +1,11 @@ -package com.bakdata.conquery.models.query.resultinfo.printers; +package com.bakdata.conquery.models.query.resultinfo; import java.util.Set; import com.bakdata.conquery.models.datasets.SecondaryIdDescription; import com.bakdata.conquery.models.query.PrintSettings; -import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.bakdata.conquery.models.query.resultinfo.printers.common.MappedPrinter; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java new file mode 100644 index 0000000000..ff709ec4b6 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java @@ -0,0 +1,59 @@ +package com.bakdata.conquery.models.query.resultinfo.printers; + +import com.bakdata.conquery.models.query.PrintSettings; +import com.bakdata.conquery.models.query.resultinfo.printers.common.DatePrinter; +import com.bakdata.conquery.models.query.resultinfo.printers.common.IdentityPrinter; +import com.bakdata.conquery.models.types.ResultType; + +/** + * This class is a mess because Excel supports some of our types natively. + * + * With LIST types we fall back onto the StringResultPrinter, as Excel does not support Lists. + */ +public class ExcelResultPrinters extends StringResultPrinters { + + private final PrinterFactory partialDelegate = new StringResultPrinters(); + + public Printer printerFor(ResultType type, PrintSettings printSettings) { + if (type instanceof ResultType.ListT listT) { + final Printer elementPrinter = partialDelegate.printerFor(listT.getElementType(), printSettings); + return getListPrinter(elementPrinter, printSettings); + } + + return switch (((ResultType.Primitive) type)) { + case BOOLEAN -> getBooleanPrinter(printSettings); + case INTEGER -> getIntegerPrinter(printSettings); + case NUMERIC -> getNumericPrinter(printSettings); + case DATE -> getDatePrinter(printSettings); + case DATE_RANGE -> getDateRangePrinter(printSettings); + case STRING -> getStringPrinter(printSettings); + case MONEY -> getMoneyPrinter(printSettings); + }; + } + + @Override + public Printer getBooleanPrinter(PrintSettings printSettings) { + return new IdentityPrinter(); + } + + @Override + public Printer getNumericPrinter(PrintSettings printSettings) { + return new IdentityPrinter(); + } + + @Override + public Printer getMoneyPrinter(PrintSettings printSettings) { + return new IdentityPrinter(); + } + + @Override + public Printer getDatePrinter(PrintSettings printSettings) { + return new DatePrinter(); + } + + @Override + public Printer getIntegerPrinter(PrintSettings printSettings) { + return new IdentityPrinter(); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/IdentityPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/IdentityPrinter.java deleted file mode 100644 index 44b1c0208b..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/IdentityPrinter.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.bakdata.conquery.models.query.resultinfo.printers; - -record IdentityPrinter() implements Printer { - - @Override - public Object apply(Object value) { - return value; - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java index ddf070af7c..1c2eb61f6b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java @@ -3,9 +3,10 @@ import java.util.ArrayList; import java.util.List; -import com.bakdata.conquery.models.common.CDate; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.query.PrintSettings; +import com.bakdata.conquery.models.query.resultinfo.printers.common.DatePrinter; +import com.bakdata.conquery.models.query.resultinfo.printers.common.IdentityPrinter; import com.google.common.base.Preconditions; public class JavaResultPrinters extends PrinterFactory { @@ -66,14 +67,6 @@ public Object apply(Object value) { } } - private record DatePrinter() implements Printer { - - @Override - public Object apply(Object value) { - return CDate.toLocalDate(((Number) value).intValue()); - } - } - private record DateRangePrinter() implements Printer { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java index 7d29f8bbf7..8aa2400a05 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java @@ -1,87 +1,22 @@ package com.bakdata.conquery.models.query.resultinfo.printers; -import java.math.BigDecimal; - import com.bakdata.conquery.models.query.PrintSettings; -import com.fasterxml.jackson.databind.node.BooleanNode; -import com.fasterxml.jackson.databind.node.DecimalNode; -import com.fasterxml.jackson.databind.node.IntNode; -import com.fasterxml.jackson.databind.node.TextNode; +import com.bakdata.conquery.models.query.resultinfo.printers.common.ToStringPrinter; import lombok.ToString; @ToString -public class JsonResultPrinters extends PrinterFactory { - - private final PrinterFactory delegate = new JavaResultPrinters(); - - @Override - public Printer getListPrinter(Printer elementPrinter, PrintSettings printSettings) { - return delegate.getListPrinter(elementPrinter, printSettings); - } - - public record BooleanPrinter() implements Printer { - - @Override - public Object apply(Object value) { - return BooleanNode.valueOf(((Boolean) value)); - } - } - - @Override - public Printer getBooleanPrinter(PrintSettings printSettings) { - return new BooleanPrinter(); - } - - public record IntegerPrinter() implements Printer { - - @Override - public Object apply(Object value) { - return IntNode.valueOf(((Integer) value)); - } - } +public class JsonResultPrinters extends JavaResultPrinters { - @Override - public Printer getIntegerPrinter(PrintSettings printSettings) { - return new IntegerPrinter(); - } - - public record NumericPrinter() implements Printer { - - @Override - public Object apply(Object value) { - return DecimalNode.valueOf(((BigDecimal) value)); - } - } - - @Override - public Printer getNumericPrinter(PrintSettings printSettings) { - return new NumericPrinter(); - } @Override public Printer getDatePrinter(PrintSettings printSettings) { - return new ToStringPrinter(delegate.getDatePrinter(printSettings)); + return new ToStringPrinter(super.getDatePrinter(printSettings)); } @Override public Printer getDateRangePrinter(PrintSettings printSettings) { - return new ToStringPrinter(delegate.getDateRangePrinter(printSettings)); + return new ToStringPrinter(super.getDateRangePrinter(printSettings)); } - public record StringPrinter() implements Printer { - @Override - public Object apply(Object value) { - return new TextNode((String) value); - } - } - @Override - public Printer getStringPrinter(PrintSettings printSettings) { - return new StringPrinter(); - } - - @Override - public Printer getMoneyPrinter(PrintSettings printSettings) { - return delegate.getMoneyPrinter(printSettings); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java index 8de0feb47b..2cf548e810 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java @@ -4,7 +4,7 @@ import com.bakdata.conquery.models.types.ResultType; public abstract class PrinterFactory { - public final Printer printerFor(ResultType type, PrintSettings printSettings) { + public Printer printerFor(ResultType type, PrintSettings printSettings) { if (type instanceof ResultType.ListT listT) { final Printer elementPrinter = printerFor(listT.getElementType(), printSettings); return getListPrinter(elementPrinter, printSettings); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java deleted file mode 100644 index 6f1a4b6d97..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java +++ /dev/null @@ -1,227 +0,0 @@ -package com.bakdata.conquery.models.query.resultinfo.printers; - -import java.math.BigDecimal; -import java.util.List; -import java.util.Objects; -import java.util.StringJoiner; - -import com.bakdata.conquery.internationalization.Results; -import com.bakdata.conquery.models.common.CDate; -import com.bakdata.conquery.models.common.LocalizedToString; -import com.bakdata.conquery.models.common.daterange.CDateRange; -import com.bakdata.conquery.models.config.LocaleConfig; -import com.bakdata.conquery.models.datasets.concepts.Concept; -import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeNode; -import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; -import com.bakdata.conquery.models.index.InternToExternMapper; -import com.bakdata.conquery.models.query.C10nCache; -import com.bakdata.conquery.models.query.PrintSettings; -import com.bakdata.conquery.models.types.ResultType; -import com.google.common.base.Preconditions; -import lombok.experimental.UtilityClass; -import lombok.extern.slf4j.Slf4j; - -@UtilityClass -@Slf4j -public class ResultPrinters { - - public Printer printerFor(ResultType type, PrintSettings printSettings) { - if (type instanceof ResultType.ListT listT) { - return new ListPrinter(printerFor(listT.getElementType(), printSettings), printSettings); - } - - return switch (((ResultType.Primitive) type)) { - case BOOLEAN -> new BooleanPrinter(printSettings); - case INTEGER -> new IntegerPrinter(printSettings); - case NUMERIC -> new NumericPrinter(printSettings); - case DATE -> new DatePrinter(printSettings); - case DATE_RANGE -> new DateRangePrinter(printSettings); - case STRING -> new StringPrinter(); - case MONEY -> new MoneyPrinter(printSettings); - }; - } - - public BigDecimal readMoney(PrintSettings cfg, Number value) { - return new BigDecimal(value.longValue()).movePointLeft(cfg.getCurrency().getDefaultFractionDigits()); - } - - public interface Printer { - String print(Object f); - } - - public record StringPrinter() implements Printer { - @Override - public String print(Object f) { - return Objects.toString(f); - } - } - - public record IntegerPrinter(PrintSettings cfg) implements Printer { - - @Override - public String print(Object f) { - if (cfg.isPrettyPrint()) { - return cfg.getIntegerFormat().format(((Number) f).longValue()); - } - - return f.toString(); - } - } - - public record NumericPrinter(PrintSettings cfg) implements Printer { - - @Override - public String print(Object f) { - if (cfg.isPrettyPrint()) { - return cfg.getDecimalFormat().format(f); - } - - return f.toString(); - } - } - - public record MoneyPrinter(PrintSettings cfg) implements Printer { - - @Override - public String print(Object f) { - - if (cfg.isPrettyPrint()) { - - return cfg.getDecimalFormat().format(readMoney(cfg, (Number) f)); - } - - return f.toString(); - } - } - - public record DatePrinter(PrintSettings cfg) implements Printer { - - @Override - public String print(Object f) { - Preconditions.checkArgument(f instanceof Number, "Expected an Number but got an '%s' with the value: %s".formatted(f.getClass().getName(), f)); - - final Number number = (Number) f; - return cfg.getDateFormatter().format(CDate.toLocalDate(number.intValue())); - } - } - - public record DateRangePrinter(DatePrinter datePrinter, PrintSettings cfg) implements Printer { - - public DateRangePrinter(PrintSettings printSettings) { - this(new DatePrinter(printSettings), printSettings); - } - - @Override - public String print(Object f) { - Preconditions.checkArgument(f instanceof List, "Expected a List got %s (Type: %s, as string: %s)", f, f.getClass().getName(), f); - Preconditions.checkArgument(((List) f).size() == 2, "Expected a list with 2 elements, one min, one max. The list was: %s ", f); - - final List list = (List) f; - final Integer min = (Integer) list.get(0); - final Integer max = (Integer) list.get(1); - - if (min == null || max == null) { - log.warn("Encountered incomplete range, treating it as an open range. Either min or max was null: {}", list); - } - // Compute minString first because we need it either way - final String minString = min == null || min == CDateRange.NEGATIVE_INFINITY ? "-∞" : datePrinter.print(min); - - if (cfg.isPrettyPrint() && min != null && min.equals(max)) { - // If the min and max are the same we print it like a singe date, not a range (only in pretty printing) - return minString; - } - final String maxString = max == null || max == CDateRange.POSITIVE_INFINITY ? "+∞" : datePrinter.print(max); - - return minString + cfg.getDateRangeSeparator() + maxString; - } - } - - public record BooleanPrinter(PrintSettings cfg, String trueVal, String falseVal) implements Printer { - - public BooleanPrinter(PrintSettings cfg) { - this( - cfg, - cfg.isPrettyPrint() ? C10nCache.getLocalized(Results.class, cfg.getLocale()).True() : "1", - cfg.isPrettyPrint() ? C10nCache.getLocalized(Results.class, cfg.getLocale()).False() : "0" - ); - } - - @Override - public String print(Object f) { - if ((Boolean) f) { - return trueVal; - } - return falseVal; - - } - } - - public record MappedPrinter(InternToExternMapper mapper) implements Printer { - - @Override - public String print(Object f) { - return mapper.external(((String) f)); - } - } - - public record ConceptIdPrinter(Concept concept, PrintSettings cfg) implements Printer { - - @Override - public String print(Object rawValue) { - if (rawValue == null) { - return null; - } - - final int localId = (int) rawValue; - - final ConceptTreeNode node = ((TreeConcept) concept).getElementByLocalId(localId); - - if (!cfg.isPrettyPrint()) { - return node.getId().toString(); - } - - if (node.getDescription() == null) { - return node.getLabel(); - } - - return node.getLabel() + " - " + node.getDescription(); - } - } - - public record ListPrinter(Printer elementPrinter, PrintSettings cfg, LocaleConfig.ListFormat listFormat) implements Printer { - - public ListPrinter(Printer elementPrinter, PrintSettings cfg) { - this(elementPrinter, cfg, cfg.getListFormat()); - } - - @Override - public String print(Object f) { - - // Jackson deserializes collections as lists instead of an array, if the type is not given - Preconditions.checkArgument(f instanceof List, "Expected a List got %s (as String `%s` )".formatted(f.getClass().getName(), f)); - - final StringJoiner joiner = listFormat.createListJoiner(); - - for (Object obj : (List) f) { - joiner.add(listFormat.escapeListElement(elementPrinter.print(obj))); - } - return joiner.toString(); - } - } - - public record LocalizedEnumPrinter & LocalizedToString>(PrintSettings cfg, Class clazz) implements Printer { - @Override - public String print(Object f) { - - if (clazz.isInstance(f)) { - return clazz.cast(f).toString(cfg.getLocale()); - } - try { - return Enum.valueOf(clazz, f.toString()).toString(cfg.getLocale()); - } - catch (Exception e) { - throw new IllegalArgumentException("%s is not a valid %s.".formatted(f, clazz), e); - } - } - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java index 6ca76dc394..e6ece1b5c5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java @@ -1,16 +1,14 @@ package com.bakdata.conquery.models.query.resultinfo.printers; -import java.util.List; -import java.util.Objects; -import java.util.StringJoiner; - -import com.bakdata.conquery.internationalization.Results; -import com.bakdata.conquery.models.common.CDate; -import com.bakdata.conquery.models.common.daterange.CDateRange; -import com.bakdata.conquery.models.config.LocaleConfig; -import com.bakdata.conquery.models.query.C10nCache; import com.bakdata.conquery.models.query.PrintSettings; -import com.google.common.base.Preconditions; +import com.bakdata.conquery.models.query.resultinfo.printers.common.BooleanStringPrinter; +import com.bakdata.conquery.models.query.resultinfo.printers.common.DateRangeStringPrinter; +import com.bakdata.conquery.models.query.resultinfo.printers.common.DateStringPrinter; +import com.bakdata.conquery.models.query.resultinfo.printers.common.IntegerStringPrinter; +import com.bakdata.conquery.models.query.resultinfo.printers.common.ListStringPrinter; +import com.bakdata.conquery.models.query.resultinfo.printers.common.MoneyStringPrinter; +import com.bakdata.conquery.models.query.resultinfo.printers.common.NumericStringPrinter; +import com.bakdata.conquery.models.query.resultinfo.printers.common.StringPrinter; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -19,32 +17,32 @@ public class StringResultPrinters extends PrinterFactory { @Override public Printer getListPrinter(Printer elementPrinter, PrintSettings printSettings) { - return new ListPrinter(elementPrinter, printSettings); + return new ListStringPrinter(elementPrinter, printSettings); } @Override public Printer getBooleanPrinter(PrintSettings printSettings) { - return new BooleanPrinter(printSettings); + return BooleanStringPrinter.create(printSettings); } @Override public Printer getIntegerPrinter(PrintSettings printSettings) { - return new IntegerPrinter(printSettings); + return new IntegerStringPrinter(printSettings); } @Override public Printer getNumericPrinter(PrintSettings printSettings) { - return new NumericPrinter(printSettings); + return new NumericStringPrinter(printSettings); } @Override public Printer getDatePrinter(PrintSettings printSettings) { - return new DatePrinter(printSettings); + return new DateStringPrinter(printSettings); } @Override public Printer getDateRangePrinter(PrintSettings printSettings) { - return new DateRangePrinter(printSettings); + return new DateRangeStringPrinter(printSettings); } @Override @@ -54,132 +52,7 @@ public Printer getStringPrinter(PrintSettings printSettings) { @Override public Printer getMoneyPrinter(PrintSettings printSettings) { - return new MoneyPrinter(printSettings); - } - - private record StringPrinter() implements Printer { - @Override - public String apply(Object f) { - return Objects.toString(f); - } - } - - private record IntegerPrinter(PrintSettings cfg) implements Printer { - - @Override - public String apply(Object f) { - if (cfg.isPrettyPrint()) { - return cfg.getIntegerFormat().format(((Number) f).longValue()); - } - - return f.toString(); - } - } - - private record NumericPrinter(PrintSettings cfg) implements Printer { - - @Override - public String apply(Object f) { - if (cfg.isPrettyPrint()) { - return cfg.getDecimalFormat().format(f); - } - - return f.toString(); - } - } - - private record MoneyPrinter(PrintSettings cfg) implements Printer { - - @Override - public String apply(Object f) { - - if (cfg.isPrettyPrint()) { - return cfg.getDecimalFormat().format(f); - } - - return f.toString(); - } - } - - private record DatePrinter(PrintSettings cfg) implements Printer { - - @Override - public String apply(Object f) { - Preconditions.checkArgument(f instanceof Number, "Expected an Number but got an '%s' with the value: %s".formatted(f.getClass().getName(), f)); - - final Number number = (Number) f; - return cfg.getDateFormatter().format(CDate.toLocalDate(number.intValue())); - } - } - - private record DateRangePrinter(DatePrinter datePrinter, PrintSettings cfg) implements Printer { - - public DateRangePrinter(PrintSettings printSettings) { - this(new DatePrinter(printSettings), printSettings); - } - - @Override - public String apply(Object f) { - Preconditions.checkArgument(f instanceof List, "Expected a List got %s (Type: %s, as string: %s)", f, f.getClass().getName(), f); - Preconditions.checkArgument(((List) f).size() == 2, "Expected a list with 2 elements, one min, one max. The list was: %s ", f); - - final List list = (List) f; - final Integer min = (Integer) list.get(0); - final Integer max = (Integer) list.get(1); - - if (min == null || max == null) { - log.warn("Encountered incomplete range, treating it as an open range. Either min or max was null: {}", list); - } - // Compute minString first because we need it either way - final String minString = min == null || min == CDateRange.NEGATIVE_INFINITY ? "-∞" : datePrinter.apply(min); - - if (cfg.isPrettyPrint() && min != null && min.equals(max)) { - // If the min and max are the same we print it like a singe date, not a range (only in pretty printing) - return minString; - } - final String maxString = max == null || max == CDateRange.POSITIVE_INFINITY ? "+∞" : datePrinter.apply(max); - - return minString + cfg.getDateRangeSeparator() + maxString; - } - } - - private record BooleanPrinter(PrintSettings cfg, String trueVal, String falseVal) implements Printer { - - public BooleanPrinter(PrintSettings cfg) { - this(cfg, cfg.isPrettyPrint() ? C10nCache.getLocalized(Results.class, cfg.getLocale()).True() : "1", cfg.isPrettyPrint() - ? C10nCache.getLocalized(Results.class, cfg.getLocale()).False() - : "0"); - } - - @Override - public String apply(Object f) { - if ((Boolean) f) { - return trueVal; - } - return falseVal; - - } - } - - private record ListPrinter(Printer elementPrinter, PrintSettings cfg, LocaleConfig.ListFormat listFormat) implements Printer { - - public ListPrinter(Printer elementPrinter, PrintSettings cfg) { - this(elementPrinter, cfg, cfg.getListFormat()); - } - - @Override - public String apply(Object f) { - - // Jackson deserializes collections as lists instead of an array, if the type is not given - Preconditions.checkArgument(f instanceof List, "Expected a List got %s (as String `%s` )".formatted(f.getClass().getName(), f)); - - final StringJoiner joiner = listFormat.createListJoiner(); - - for (Object obj : (List) f) { - joiner.add(listFormat.escapeListElement(elementPrinter.apply(obj).toString())); - } - return joiner.toString(); - } + return new MoneyStringPrinter(printSettings); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ToStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ToStringPrinter.java deleted file mode 100644 index 4b476a7fd0..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ToStringPrinter.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.bakdata.conquery.models.query.resultinfo.printers; - -import java.util.Objects; - -import com.fasterxml.jackson.databind.node.TextNode; - -public record ToStringPrinter(Printer delegate) implements Printer { - - @Override - public Object apply(Object value) { - return new TextNode(Objects.toString(delegate.apply(value))); - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/BooleanStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/BooleanStringPrinter.java new file mode 100644 index 0000000000..e35036483f --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/BooleanStringPrinter.java @@ -0,0 +1,28 @@ +package com.bakdata.conquery.models.query.resultinfo.printers.common; + +import com.bakdata.conquery.internationalization.Results; +import com.bakdata.conquery.models.query.C10nCache; +import com.bakdata.conquery.models.query.PrintSettings; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; + +public record BooleanStringPrinter(PrintSettings cfg, String trueVal, String falseVal) implements Printer { + + public static BooleanStringPrinter create(PrintSettings settings) { + if (!settings.isPrettyPrint()) { + return new BooleanStringPrinter(settings, "1", "0"); + } + + final Results localized = C10nCache.getLocalized(Results.class, settings.getLocale()); + return new BooleanStringPrinter(settings, localized.True(), localized.False()); + } + + + @Override + public String apply(Object f) { + if ((Boolean) f) { + return trueVal; + } + return falseVal; + + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ConceptIdPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ConceptIdPrinter.java similarity index 84% rename from backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ConceptIdPrinter.java rename to backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ConceptIdPrinter.java index fbba2b8607..35a6f317d0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ConceptIdPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ConceptIdPrinter.java @@ -1,9 +1,10 @@ -package com.bakdata.conquery.models.query.resultinfo.printers; +package com.bakdata.conquery.models.query.resultinfo.printers.common; import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeNode; import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.models.query.PrintSettings; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; public record ConceptIdPrinter(Concept concept, PrintSettings cfg) implements Printer { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DatePrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DatePrinter.java new file mode 100644 index 0000000000..2e8fbf8a1c --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DatePrinter.java @@ -0,0 +1,12 @@ +package com.bakdata.conquery.models.query.resultinfo.printers.common; + +import com.bakdata.conquery.models.common.CDate; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; + +public record DatePrinter() implements Printer { + + @Override + public Object apply(Object value) { + return CDate.toLocalDate(((Number) value).intValue()); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateRangeStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateRangeStringPrinter.java new file mode 100644 index 0000000000..e9e1b3c7f3 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateRangeStringPrinter.java @@ -0,0 +1,41 @@ +package com.bakdata.conquery.models.query.resultinfo.printers.common; + +import java.util.List; + +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.query.PrintSettings; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.google.common.base.Preconditions; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public record DateRangeStringPrinter(DateStringPrinter datePrinter, PrintSettings cfg) implements Printer { + + public DateRangeStringPrinter(PrintSettings printSettings) { + this(new DateStringPrinter(printSettings), printSettings); + } + + @Override + public String apply(Object f) { + Preconditions.checkArgument(f instanceof List, "Expected a List got %s (Type: %s, as string: %s)", f, f.getClass().getName(), f); + Preconditions.checkArgument(((List) f).size() == 2, "Expected a list with 2 elements, one min, one max. The list was: %s ", f); + + final List list = (List) f; + final Integer min = (Integer) list.get(0); + final Integer max = (Integer) list.get(1); + + if (min == null || max == null) { + log.warn("Encountered incomplete range, treating it as an open range. Either min or max was null: {}", list); + } + // Compute minString first because we need it either way + final String minString = min == null || min == CDateRange.NEGATIVE_INFINITY ? "-∞" : datePrinter.apply(min); + + if (cfg.isPrettyPrint() && min != null && min.equals(max)) { + // If the min and max are the same we print it like a singe date, not a range (only in pretty printing) + return minString; + } + final String maxString = max == null || max == CDateRange.POSITIVE_INFINITY ? "+∞" : datePrinter.apply(max); + + return minString + cfg.getDateRangeSeparator() + maxString; + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateStringPrinter.java new file mode 100644 index 0000000000..7d912c5d09 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateStringPrinter.java @@ -0,0 +1,17 @@ +package com.bakdata.conquery.models.query.resultinfo.printers.common; + +import com.bakdata.conquery.models.common.CDate; +import com.bakdata.conquery.models.query.PrintSettings; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.google.common.base.Preconditions; + +public record DateStringPrinter(PrintSettings cfg) implements Printer { + + @Override + public String apply(Object f) { + Preconditions.checkArgument(f instanceof Number, "Expected an Number but got an '%s' with the value: %s".formatted(f.getClass().getName(), f)); + + final Number number = (Number) f; + return cfg.getDateFormatter().format(CDate.toLocalDate(number.intValue())); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/IdentityPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/IdentityPrinter.java new file mode 100644 index 0000000000..a1fcc83bcb --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/IdentityPrinter.java @@ -0,0 +1,11 @@ +package com.bakdata.conquery.models.query.resultinfo.printers.common; + +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; + +public record IdentityPrinter() implements Printer { + + @Override + public Object apply(Object value) { + return value; + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/IntegerStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/IntegerStringPrinter.java new file mode 100644 index 0000000000..1345b718a3 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/IntegerStringPrinter.java @@ -0,0 +1,16 @@ +package com.bakdata.conquery.models.query.resultinfo.printers.common; + +import com.bakdata.conquery.models.query.PrintSettings; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; + +public record IntegerStringPrinter(PrintSettings cfg) implements Printer { + + @Override + public String apply(Object f) { + if (cfg.isPrettyPrint()) { + return cfg.getIntegerFormat().format(((Number) f).longValue()); + } + + return f.toString(); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java new file mode 100644 index 0000000000..e25ccc7d7f --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java @@ -0,0 +1,30 @@ +package com.bakdata.conquery.models.query.resultinfo.printers.common; + +import java.util.List; +import java.util.StringJoiner; + +import com.bakdata.conquery.models.config.LocaleConfig; +import com.bakdata.conquery.models.query.PrintSettings; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.google.common.base.Preconditions; + +public record ListStringPrinter(Printer elementPrinter, PrintSettings cfg, LocaleConfig.ListFormat listFormat) implements Printer { + + public ListStringPrinter(Printer elementPrinter, PrintSettings cfg) { + this(elementPrinter, cfg, cfg.getListFormat()); + } + + @Override + public String apply(Object f) { + + // Jackson deserializes collections as lists instead of an array, if the type is not given + Preconditions.checkArgument(f instanceof List, "Expected a List got %s (as String `%s` )".formatted(f.getClass().getName(), f)); + + final StringJoiner joiner = listFormat.createListJoiner(); + + for (Object obj : (List) f) { + joiner.add(listFormat.escapeListElement(elementPrinter.apply(obj).toString())); + } + return joiner.toString(); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/LocalizedEnumPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/LocalizedEnumPrinter.java similarity index 80% rename from backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/LocalizedEnumPrinter.java rename to backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/LocalizedEnumPrinter.java index 78b972494f..b94a6ece63 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/LocalizedEnumPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/LocalizedEnumPrinter.java @@ -1,7 +1,8 @@ -package com.bakdata.conquery.models.query.resultinfo.printers; +package com.bakdata.conquery.models.query.resultinfo.printers.common; import com.bakdata.conquery.models.common.LocalizedToString; import com.bakdata.conquery.models.query.PrintSettings; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; public record LocalizedEnumPrinter & LocalizedToString>(PrintSettings cfg, Class clazz) implements Printer { @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/MappedPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MappedPrinter.java similarity index 62% rename from backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/MappedPrinter.java rename to backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MappedPrinter.java index a87c80cdce..58b32eda8e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/MappedPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MappedPrinter.java @@ -1,6 +1,7 @@ -package com.bakdata.conquery.models.query.resultinfo.printers; +package com.bakdata.conquery.models.query.resultinfo.printers.common; import com.bakdata.conquery.models.index.InternToExternMapper; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; public record MappedPrinter(InternToExternMapper mapper) implements Printer { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MoneyStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MoneyStringPrinter.java new file mode 100644 index 0000000000..cfc760cd98 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MoneyStringPrinter.java @@ -0,0 +1,17 @@ +package com.bakdata.conquery.models.query.resultinfo.printers.common; + +import com.bakdata.conquery.models.query.PrintSettings; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; + +public record MoneyStringPrinter(PrintSettings cfg) implements Printer { + + @Override + public String apply(Object f) { + + if (cfg.isPrettyPrint()) { + return cfg.getDecimalFormat().format(f); + } + + return f.toString(); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/NumericStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/NumericStringPrinter.java new file mode 100644 index 0000000000..d5a9a95661 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/NumericStringPrinter.java @@ -0,0 +1,16 @@ +package com.bakdata.conquery.models.query.resultinfo.printers.common; + +import com.bakdata.conquery.models.query.PrintSettings; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; + +public record NumericStringPrinter(PrintSettings cfg) implements Printer { + + @Override + public String apply(Object f) { + if (cfg.isPrettyPrint()) { + return cfg.getDecimalFormat().format(f); + } + + return f.toString(); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/StringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/StringPrinter.java new file mode 100644 index 0000000000..322b6907a0 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/StringPrinter.java @@ -0,0 +1,12 @@ +package com.bakdata.conquery.models.query.resultinfo.printers.common; + +import java.util.Objects; + +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; + +public record StringPrinter() implements Printer { + @Override + public String apply(Object f) { + return Objects.toString(f); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ToStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ToStringPrinter.java new file mode 100644 index 0000000000..affefd797a --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ToStringPrinter.java @@ -0,0 +1,13 @@ +package com.bakdata.conquery.models.query.resultinfo.printers.common; + +import java.util.Objects; + +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; + +public record ToStringPrinter(Printer delegate) implements Printer { + + @Override + public Object apply(Object value) { + return Objects.toString(delegate.apply(value)); + } +} diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java index 5e626a1c47..b2061b1ac2 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java @@ -34,8 +34,8 @@ @UtilityClass public class ResultTestUtil { - private static final User OWNER = new User("user", "User", null); - private static final Dataset DATASET = new Dataset("dataset"); + public static final User OWNER = new User("user", "User", null); + public static final Dataset DATASET = new Dataset("dataset"); private static final TreeConcept CONCEPT; static { diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java index a46903f421..3e27f7d66c 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.io.result.excel; -import static com.bakdata.conquery.io.result.ResultTestUtil.getResultTypes; -import static com.bakdata.conquery.io.result.ResultTestUtil.getTestEntityResults; +import static com.bakdata.conquery.io.result.ResultTestUtil.*; import static org.assertj.core.api.Assertions.assertThat; import java.io.ByteArrayInputStream; @@ -12,6 +11,7 @@ import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.Objects; import java.util.OptionalLong; import java.util.StringJoiner; import java.util.stream.Collectors; @@ -27,6 +27,8 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; +import com.bakdata.conquery.models.query.resultinfo.printers.ExcelResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.query.resultinfo.printers.StringResultPrinters; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.types.ResultType; @@ -56,14 +58,18 @@ public class ExcelResultRenderTest { @Test void writeAndRead() throws IOException { // Prepare every input data - final PrintSettings - printSettings = - new PrintSettings(true, Locale.GERMAN, null, CONFIG, (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), (selectInfo) -> selectInfo.getSelect() - .getLabel(), new StringResultPrinters()); // TODO ? + final PrintSettings printSettings = new PrintSettings(true, + Locale.GERMAN, + null, + CONFIG, + (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), + (selectInfo) -> selectInfo.getSelect().getLabel(), + new ExcelResultPrinters() + ); // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized final List results = getTestEntityResults(); - final ManagedQuery mquery = new ManagedQuery(null, null, null, null) { + final ManagedQuery mquery = new ManagedQuery(null, OWNER, DATASET, null) { public List getResultInfos() { return getResultTypes().stream() .map(ResultTestUtil.TypedSelectDummy::new) @@ -89,8 +95,16 @@ public Stream streamResults(OptionalLong maybeLimit) { final List computed = readComputed(inputStream, printSettings); - - final List expected = generateExpectedTSV(results, mquery.getResultInfos(), printSettings); + // We have to do some magic here to emulate the excel printed results. + PrintSettings tsvPrintSettings = new PrintSettings(true, + Locale.GERMAN, + null, + CONFIG, + (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), + (selectInfo) -> selectInfo.getSelect().getLabel(), + new StringResultPrinters() + ); + final List expected = generateExpectedTSV(results, mquery.getResultInfos(), tsvPrintSettings); log.info("Wrote and than read this excel data: {}", computed); @@ -134,16 +148,17 @@ private List generateExpectedTSV(List results, List generateExpectedTSV(List results, List Date: Tue, 3 Sep 2024 08:57:35 +0200 Subject: [PATCH 35/54] fix excel money cell style and test render format --- .../conquery/models/config/ExcelConfig.java | 18 ++++++++---------- .../printers/common/MoneyStringPrinter.java | 2 +- .../io/result/excel/ExcelResultRenderTest.java | 10 +--------- 3 files changed, 10 insertions(+), 20 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelConfig.java index 0aa040d8c3..bdf0cc42d4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelConfig.java @@ -1,5 +1,11 @@ package com.bakdata.conquery.models.config; +import java.util.Collections; +import java.util.Map; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; + import com.bakdata.conquery.models.query.PrintSettings; import com.google.common.collect.ImmutableMap; import lombok.AllArgsConstructor; @@ -12,15 +18,6 @@ import org.apache.poi.ss.usermodel.Font; import org.apache.poi.xssf.streaming.SXSSFSheet; import org.apache.poi.xssf.streaming.SXSSFWorkbook; -import org.apache.poi.xssf.usermodel.XSSFDataFormat; -import org.apache.poi.xssf.usermodel.XSSFFont; -import org.apache.poi.xssf.usermodel.XSSFWorkbook; - -import jakarta.validation.constraints.Min; -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; -import java.util.Collections; -import java.util.Map; @Data public class ExcelConfig { @@ -33,7 +30,8 @@ public class ExcelConfig { private static final Map FALLBACK_STYLES = Map.of( BASIC_STYLE, new CellStyler(), - CURRENCY_STYLE_PREFIX + "EUR", new CellStyler().withDataFormatString("#,##0.00 €"), + // \u00A0 is the non breakable space + CURRENCY_STYLE_PREFIX + "EUR", new CellStyler().withDataFormatString("#,##0.00\u00A0€"), NUMERIC_STYLE, new CellStyler().withDataFormatString("#,##0.00"), INTEGER_STYLE, new CellStyler().withDataFormatString("#,##0") ); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MoneyStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MoneyStringPrinter.java index cfc760cd98..4e5a04bfe8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MoneyStringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MoneyStringPrinter.java @@ -9,7 +9,7 @@ public record MoneyStringPrinter(PrintSettings cfg) implements Printer { public String apply(Object f) { if (cfg.isPrettyPrint()) { - return cfg.getDecimalFormat().format(f); + return cfg.getCurrencyFormat().format(f); } return f.toString(); diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java index 3e27f7d66c..8287ea5b4d 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java @@ -119,7 +119,6 @@ private List readComputed(InputStream inputStream, PrintSettings setting final XSSFSheet sheet = workbook.getSheetAt(0); final List computed = new ArrayList<>(); - int i = 0; for (Row row : sheet) { final StringJoiner sj = new StringJoiner("\t"); final DataFormatter formatter = new DataFormatter(settings.getLocale()); @@ -135,7 +134,6 @@ private List readComputed(InputStream inputStream, PrintSettings setting sj.add(formatted); } computed.add(sj.toString()); - i++; } return computed; } @@ -175,16 +173,10 @@ private String printValue(Object val, ResultInfo info, PrintSettings printSettin final Printer printer = info.createPrinter(printSettings); if (info.getType().equals(ResultType.Primitive.BOOLEAN)) { - /* - * Even though we set the locale to GERMAN, poi's {@link DataFormatter#formatCellValue(Cell)} hardcoded english booleans - */ + // Even though we set the locale to GERMAN, poi's {@link DataFormatter#formatCellValue(Cell)} hardcoded english booleans return (Boolean) val ? "TRUE" : "FALSE"; } - if (info.getType().equals(ResultType.Primitive.MONEY)) { - return printer.apply(val) + " €"; - } - return Objects.toString(printer.apply(val)); From 88ce4fb2a6f881e6e741499f45b0ac53b82df92b Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 3 Sep 2024 09:02:29 +0200 Subject: [PATCH 36/54] fix ResultTypeTest --- .../com/bakdata/conquery/models/types/ResultTypeTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java b/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java index b656d94114..89b2520975 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java @@ -61,8 +61,8 @@ public static List testData() { .intValue()), "12.07.2013 - 12.07.2014"), Arguments.of(PRETTY, ResultType.Primitive.INTEGER, 51839274, "51,839,274"), Arguments.of(PRETTY_DE, ResultType.Primitive.INTEGER, 51839274, "51.839.274"), - Arguments.of(PRETTY, ResultType.Primitive.MONEY, new BigDecimal("518392.74"), "518,392.74"), - Arguments.of(PRETTY_DE, ResultType.Primitive.MONEY, new BigDecimal("518392.74"), "518.392,74"), + Arguments.of(PRETTY, ResultType.Primitive.MONEY, new BigDecimal("518392.74"), "€518,392.74"), + Arguments.of(PRETTY_DE, ResultType.Primitive.MONEY, new BigDecimal("518392.74"), "518.392,74\u00A0€"), Arguments.of(PRETTY, ResultType.Primitive.NUMERIC, 0.2, "0.2"), Arguments.of(PRETTY_DE, ResultType.Primitive.NUMERIC, 0.2, "0,2"), Arguments.of(PRETTY, ResultType.Primitive.NUMERIC, new BigDecimal("716283712389817246892743124.12312"), "716,283,712,389,817,246,892,743,124.12312"), From b939bef0ba2cc84b661ce7d4c131eb5a44301d18 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 3 Sep 2024 10:56:34 +0200 Subject: [PATCH 37/54] NumberFilterConverter now also fixes FilterValue --- .../filters/specific/NumberFilter.java | 26 ++++++++++++++----- .../model/filter/NumberFilterConverter.java | 14 +++++----- .../integration/common/IntegrationUtils.java | 12 ++++----- 3 files changed, 32 insertions(+), 20 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java index 356c74fd73..b655734ac0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java @@ -10,6 +10,7 @@ import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.concepts.filters.Filter; import com.bakdata.conquery.models.datasets.concepts.filters.SingleColumnFilter; +import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.exceptions.ConceptConfigurationException; import com.bakdata.conquery.models.query.filter.event.number.DecimalFilterNode; import com.bakdata.conquery.models.query.filter.event.number.IntegerFilterNode; @@ -54,19 +55,32 @@ public void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig f.setType(type); } - @Override public FilterNode createFilterNode(RANGE value) { - return switch (getColumn().getType()) { - case MONEY -> new MoneyFilterNode(getColumn(), Range.MoneyRange.from(value, config.getFrontend().getCurrency())); - case INTEGER -> new IntegerFilterNode(getColumn(), (Range.LongRange) value); - case DECIMAL -> new DecimalFilterNode(getColumn(), ((Range) value)); - case REAL -> new RealFilterNode(getColumn(), Range.DoubleRange.fromNumberRange(value)); + final IRange range = readFilterValue(value, getColumn().getType(), config); + return switch (getColumn().getType()) { + case MONEY -> new MoneyFilterNode(getColumn(), (Range.MoneyRange) range); + case INTEGER -> new IntegerFilterNode(getColumn(), (Range.LongRange) range); + case DECIMAL -> new DecimalFilterNode(getColumn(), (Range) range); + case REAL -> new RealFilterNode(getColumn(), (Range.DoubleRange) range); default -> throw new IllegalStateException(String.format("Column type %s may not be used (Assignment should not have been possible)", getColumn())); }; } + /** + * This method only exists because we messed up and never implemented a DECIMAL_RANGE, otherwise it could be embedded in the FilterValues themselves. + */ + public static IRange readFilterValue(IRange value, @NotNull MajorTypeId type, @NotNull ConqueryConfig config) { + return switch (type) { + case MONEY -> Range.MoneyRange.from(value, config.getFrontend().getCurrency()); + case INTEGER -> (Range.LongRange) value; + case DECIMAL -> ((Range) value); + case REAL -> Range.DoubleRange.fromNumberRange(value); + default -> throw new IllegalStateException(String.format("Column type %s may not be used (Assignment should not have been possible)", type)); + }; + } + @Override public FilterConverter, RANGE> createConverter() { return new NumberFilterConverter<>(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java index 6903e2dfd4..16fe976764 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java @@ -27,16 +27,12 @@ public SqlFilters convertToSqlFilter(NumberFilter filter, FilterContext rootSelect = new ExtractingSqlSelect<>(tables.getRootTable(), column.getName(), numberClass); Field eventFilterCtePredecessor = rootSelect.qualify(tables.getPredecessor(ConceptCteStep.EVENT_FILTER)).select(); - IRange filterValue = filterContext.getValue(); + IRange filterValue = NumberFilter.readFilterValue(filterContext.getValue(), column.getType(), filter.getConfig()); NumberCondition condition = new NumberCondition(eventFilterCtePredecessor, filterValue); - ConnectorSqlSelects selects = ConnectorSqlSelects.builder() - .preprocessingSelects(List.of(rootSelect)) - .build(); + ConnectorSqlSelects selects = ConnectorSqlSelects.builder().preprocessingSelects(List.of(rootSelect)).build(); - WhereClauses whereClauses = WhereClauses.builder() - .eventFilter(condition) - .build(); + WhereClauses whereClauses = WhereClauses.builder().eventFilter(condition).build(); return new SqlFilters(selects, whereClauses); } @@ -47,6 +43,8 @@ public Condition convertForTableExport(NumberFilter filter, FilterContext String tableName = column.getTable().getName(); String columnName = column.getName(); Field field = DSL.field(DSL.name(tableName, columnName), Number.class); - return new NumberCondition(field, filterContext.getValue()).condition(); + IRange range = NumberFilter.readFilterValue(filterContext.getValue(), column.getType(), filter.getConfig()); + + return new NumberCondition(field, range).condition(); } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java b/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java index 8295740eab..fccb6f78d7 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java @@ -6,10 +6,6 @@ import java.net.URI; import java.util.Map; -import jakarta.ws.rs.client.Entity; -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.Response; - import com.bakdata.conquery.apiv1.execution.ExecutionStatus; import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.apiv1.query.Query; @@ -26,6 +22,9 @@ import com.bakdata.conquery.resources.hierarchies.HierarchyHelper; import com.bakdata.conquery.util.support.StandaloneSupport; import com.fasterxml.jackson.databind.JsonNode; +import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; @@ -80,8 +79,9 @@ public static ManagedExecutionId assertQueryResult(StandaloneSupport conquery, O .post(Entity.entity(query, MediaType.APPLICATION_JSON_TYPE)); - assertThat(response.getStatusInfo().getStatusCode()).as("Result of %s", postQueryURI) - .isEqualTo(expectedResponseCode); + assertThat(response.getStatusInfo().getStatusCode()) + .as(() -> response.readEntity(String.class)) + .isEqualTo(expectedResponseCode); if (expectedState == ExecutionState.FAILED && !response.getStatusInfo().getFamily().equals(Response.Status.Family.SUCCESSFUL)) { return null; From ca94db881221b2613e271605531d61d97dd485a8 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 3 Sep 2024 12:09:59 +0200 Subject: [PATCH 38/54] removes problematic MONEY NUMBER_RANGE test --- .../bakdata/conquery/models/common/Range.java | 12 +-- .../filters/specific/NumberFilter.java | 2 +- .../number_money/real_range/content.csv | 19 ----- .../number_money/real_range/expected.csv | 6 -- .../real_range/number_money.spec.json | 73 ------------------- 5 files changed, 8 insertions(+), 104 deletions(-) delete mode 100644 backend/src/test/resources/tests/sql/filter/number_money/real_range/content.csv delete mode 100644 backend/src/test/resources/tests/sql/filter/number_money/real_range/expected.csv delete mode 100644 backend/src/test/resources/tests/sql/filter/number_money/real_range/number_money.spec.json diff --git a/backend/src/main/java/com/bakdata/conquery/models/common/Range.java b/backend/src/main/java/com/bakdata/conquery/models/common/Range.java index 930507ca07..37ed1cc2a8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/common/Range.java +++ b/backend/src/main/java/com/bakdata/conquery/models/common/Range.java @@ -296,13 +296,15 @@ public MoneyRange(BigDecimal min, BigDecimal max) { super(min, max); } - public static MoneyRange from(IRange orig, FrontendConfig.CurrencyConfig currency) { + public static MoneyRange fromNumberRange(IRange orig, FrontendConfig.CurrencyConfig currency) { BigDecimal mappedMin = Optional.ofNullable(orig.getMin()) - .map(val -> new BigDecimal(val.longValue()).movePointLeft(currency.getDecimalScale())) - .orElse(null); + .map(val -> new BigDecimal(val.longValue()).movePointLeft(currency.getDecimalScale())) + .orElse(null); + BigDecimal mappedMax = Optional.ofNullable(orig.getMax()) - .map(val -> new BigDecimal(val.longValue()).movePointLeft(currency.getDecimalScale())) - .orElse(null); + .map(val -> new BigDecimal(val.longValue()).movePointLeft(currency.getDecimalScale())) + .orElse(null); + return new Range.MoneyRange(mappedMin, mappedMax); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java index b655734ac0..37e250de19 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java @@ -73,7 +73,7 @@ public FilterNode createFilterNode(RANGE value) { */ public static IRange readFilterValue(IRange value, @NotNull MajorTypeId type, @NotNull ConqueryConfig config) { return switch (type) { - case MONEY -> Range.MoneyRange.from(value, config.getFrontend().getCurrency()); + case MONEY -> Range.MoneyRange.fromNumberRange(value, config.getFrontend().getCurrency()); case INTEGER -> (Range.LongRange) value; case DECIMAL -> ((Range) value); case REAL -> Range.DoubleRange.fromNumberRange(value); diff --git a/backend/src/test/resources/tests/sql/filter/number_money/real_range/content.csv b/backend/src/test/resources/tests/sql/filter/number_money/real_range/content.csv deleted file mode 100644 index 1cb67fbcf4..0000000000 --- a/backend/src/test/resources/tests/sql/filter/number_money/real_range/content.csv +++ /dev/null @@ -1,19 +0,0 @@ -pid,nr -1,50 -2,250 -3,150 -3,150 -4,50 -4,150 -5,250 -5,150 -6,50 -6,250 -7,150.01 -8,200.01 -9,99.99 -10, -11, -11,300 -12,150 -12, diff --git a/backend/src/test/resources/tests/sql/filter/number_money/real_range/expected.csv b/backend/src/test/resources/tests/sql/filter/number_money/real_range/expected.csv deleted file mode 100644 index 235c68aa46..0000000000 --- a/backend/src/test/resources/tests/sql/filter/number_money/real_range/expected.csv +++ /dev/null @@ -1,6 +0,0 @@ -result,dates -3,{} -4,{} -5,{} -9,{} -12,{} diff --git a/backend/src/test/resources/tests/sql/filter/number_money/real_range/number_money.spec.json b/backend/src/test/resources/tests/sql/filter/number_money/real_range/number_money.spec.json deleted file mode 100644 index 9c869eaf1b..0000000000 --- a/backend/src/test/resources/tests/sql/filter/number_money/real_range/number_money.spec.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "label": "Single Number-Real-Range Filter Query with REAL RANGE on column with MONEY type", - "type": "QUERY_TEST", - "sqlSpec": { - "isEnabled": true - }, - "expectedCsv": "tests/sql/filter/number_money/real_range/expected.csv", - "query": { - "type": "CONCEPT_QUERY", - "root": { - "type": "AND", - "children": [ - { - "ids": [ - "number" - ], - "type": "CONCEPT", - "label": "vs", - "tables": [ - { - "id": "number.number_connector", - "filters": [ - { - "filter": "number.number_connector.money_number_filter", - "type": "REAL_RANGE", - "value": { - "min": 99.00, - "max": 150.00 - } - } - ] - } - ] - } - ] - } - }, - "concepts": [ - { - "label": "number", - "type": "TREE", - "connectors": [ - { - "label": "number_connector", - "table": "table1", - "filters": { - "name": "money_number_filter", - "column": "table1.money", - "type": "NUMBER" - } - } - ] - } - ], - "content": { - "tables": [ - { - "csv": "tests/sql/filter/number_money/real_range/content.csv", - "name": "table1", - "primaryColumn": { - "name": "pid", - "type": "STRING" - }, - "columns": [ - { - "name": "money", - "type": "MONEY" - } - ] - } - ] - } -} From d5ae999d14263b151c1291b3fb247fea57a501df Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 3 Sep 2024 13:20:10 +0200 Subject: [PATCH 39/54] deletes problematic Aggregator --- .../specific/MultiSelectAggregator.java | 80 ------------------- .../specific/SelectAggregator.java | 60 -------------- .../SIMPLE_VIRTUAL_CONCEPT_Query.test.json | 69 ---------------- .../MULTI_SELECT_AGGREGATOR/content.csv | 10 --- .../MULTI_SELECT_AGGREGATOR/expected.csv | 7 -- .../SIMPLE_VIRTUAL_CONCEPT_Query.test.json | 66 --------------- .../aggregator/SELECT_AGGREGATOR/content.csv | 6 -- .../aggregator/SELECT_AGGREGATOR/expected.csv | 5 -- 8 files changed, 303 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/MultiSelectAggregator.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SelectAggregator.java delete mode 100644 backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json delete mode 100644 backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/content.csv delete mode 100644 backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/expected.csv delete mode 100644 backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json delete mode 100644 backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/content.csv delete mode 100644 backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/expected.csv diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/MultiSelectAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/MultiSelectAggregator.java deleted file mode 100644 index f428a0167c..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/MultiSelectAggregator.java +++ /dev/null @@ -1,80 +0,0 @@ -package com.bakdata.conquery.models.query.queryplan.aggregators.specific; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; - -import com.bakdata.conquery.models.datasets.Column; -import com.bakdata.conquery.models.events.Bucket; -import com.bakdata.conquery.models.query.QueryExecutionContext; -import com.bakdata.conquery.models.query.entity.Entity; -import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; -import lombok.ToString; - -/** - * Aggregator counting the occurrence of multiple values. - */ -@ToString(callSuper = true, of = "selection") -public class MultiSelectAggregator extends SingleColumnAggregator> { - - private final String[] selection; - private final int[] hits; - - public MultiSelectAggregator(Column column, String[] selection) { - super(column); - this.selection = selection; - this.hits = new int[selection.length]; - } - - @Override - public void init(Entity entity, QueryExecutionContext context) { - Arrays.fill(hits, 0); - } - - @Override - public void nextBlock(Bucket bucket) { - } - - @Override - public void consumeEvent(Bucket bucket, int event) { - if (!bucket.has(event, getColumn())) { - return; - } - - String stringToken = bucket.getString(event, getColumn()); - - for (int index = 0; index < selection.length; index++) { - if (Objects.equals(selection[index], stringToken)) { - hits[index]++; - return; - } - } - } - - @Override - public Map createAggregationResult() { - Map out = new HashMap<>(); - - for (int i = 0; i < hits.length; i++) { - int hit = hits[i]; - if (hit > 0) { - out.merge(selection[i], hit, Integer::sum); - } - } - - return out.isEmpty() ? null : out; - } - - @Override - public boolean isOfInterest(Bucket bucket) { -//TODO - // for (String selected : selection) { -// if (((StringStore) bucket.getStores()[getColumn().getPosition()]).getId(selected) == -1) { -// return false; -// } -// } - - return super.isOfInterest(bucket); - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SelectAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SelectAggregator.java deleted file mode 100644 index d654d03ad2..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SelectAggregator.java +++ /dev/null @@ -1,60 +0,0 @@ -package com.bakdata.conquery.models.query.queryplan.aggregators.specific; - -import java.util.Objects; - -import com.bakdata.conquery.models.datasets.Column; -import com.bakdata.conquery.models.events.Bucket; -import com.bakdata.conquery.models.query.QueryExecutionContext; -import com.bakdata.conquery.models.query.entity.Entity; -import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; -import lombok.ToString; - - -/** - * Aggregator counting the number of occurrences of a selected value in a column. - */ -@ToString(callSuper = true, of = {"selected"}) -public class SelectAggregator extends SingleColumnAggregator { - - private final String selected; - private long hits = 0; - - public SelectAggregator(Column column, String selected) { - super(column); - this.selected = selected; - } - - @Override - public void init(Entity entity, QueryExecutionContext context) { - hits = 0; - } - - @Override - public void nextBlock(Bucket bucket) { - } - - @Override - public void consumeEvent(Bucket bucket, int event) { - - if (!bucket.has(event, getColumn())) { - return; - } - - final String value = bucket.getString(event, getColumn()); - - if (Objects.equals(value, selected)) { - hits++; - } - } - - @Override - public Long createAggregationResult() { - return hits > 0 ? hits : null; - } - - @Override - public boolean isOfInterest(Bucket bucket) { - return super.isOfInterest(bucket); - //TODO && ((StringStore) bucket.getStores()[getColumn().getPosition()]).getId(selected) != -1; - } -} diff --git a/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json b/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json deleted file mode 100644 index 8c624cf0e2..0000000000 --- a/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "type": "QUERY_TEST", - "label": "MULTI_SELECT_AGGREGATOR Test", - "expectedCsv": "tests/aggregator/MULTI_SELECT_AGGREGATOR/expected.csv", - "query": { - "type": "CONCEPT_QUERY", - "root": { - "ids": [ - "concept" - ], - "type": "CONCEPT", - "tables": [ - { - "id": "concept.connector", - "selects": [ - "concept.connector.select" - ] - } - ] - } - }, - "concepts": [ - { - "name": "concept", - "type": "TREE", - "connectors": [ - { - "label": "connector", - "table": "table", - "validityDates": { - "label": "datum", - "column": "table.datum" - }, - "selects": { - "type": "COUNT_OCCURENCES", - "name" : "select", - "column": "table.value", - "selection": [ - "f", - "m" - ] - } - } - ] - } - ], - "content": { - "tables": [ - { - "csv": "tests/aggregator/MULTI_SELECT_AGGREGATOR/content.csv", - "name": "table", - "primaryColumn": { - "name": "pid", - "type": "STRING" - }, - "columns": [ - { - "name": "datum", - "type": "DATE" - }, - { - "name": "value", - "type": "STRING" - } - ] - } - ] - } -} diff --git a/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/content.csv b/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/content.csv deleted file mode 100644 index 5358e7b61a..0000000000 --- a/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/content.csv +++ /dev/null @@ -1,10 +0,0 @@ -pid,datum,value -1,2012-01-01,"f" -1,2012-01-02,"f" -2,2010-07-15, -3,2013-11-10,"f" -4,2012-11-11,"m" -5,2012-11-11,"x" -5,2012-11-11,"f" -6,2012-11-11,"m" -6,2012-11-11,"f" diff --git a/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/expected.csv deleted file mode 100644 index 15ec609372..0000000000 --- a/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/expected.csv +++ /dev/null @@ -1,7 +0,0 @@ -result,dates,concept select -2,{2010-07-15/2010-07-15}, -3,{2013-11-10/2013-11-10},{f=1} -6,{2012-11-11/2012-11-11},"{f=1, m=1}" -1,{2012-01-01/2012-01-02},{f=2} -4,{2012-11-11/2012-11-11},{m=1} -5,{2012-11-11/2012-11-11},{f=1} \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json b/backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json deleted file mode 100644 index 59a83c0f6e..0000000000 --- a/backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "type": "QUERY_TEST", - "label": "SELECT_AGGREGATOR Test", - "expectedCsv": "tests/aggregator/SELECT_AGGREGATOR/expected.csv", - "query": { - "type": "CONCEPT_QUERY", - "root": { - "ids": [ - "concept" - ], - "type": "CONCEPT", - "tables": [ - { - "id": "concept.connector", - "selects": [ - "concept.connector.select" - ] - } - ] - } - }, - "concepts": [ - { - "name": "concept", - "type": "TREE", - "connectors": [ - { - "label": "connector", - "table": "table", - "validityDates": { - "label": "datum", - "column": "table.datum" - }, - "selects": { - "type": "COUNT_OCCURENCES", - "name": "select", - "column": "table.value", - "selection": "f" - } - } - ] - } - ], - "content": { - "tables": [ - { - "csv": "tests/aggregator/SELECT_AGGREGATOR/content.csv", - "name": "table", - "primaryColumn": { - "name": "pid", - "type": "STRING" - }, - "columns": [ - { - "name": "datum", - "type": "DATE" - }, - { - "name": "value", - "type": "STRING" - } - ] - } - ] - } -} diff --git a/backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/content.csv b/backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/content.csv deleted file mode 100644 index cadb60f0f0..0000000000 --- a/backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/content.csv +++ /dev/null @@ -1,6 +0,0 @@ -pid,datum,value -1,2012-01-01,"f" -1,2012-01-02,"f" -2,2010-07-15, -3,2013-11-10,"f" -4,2012-11-11,"m" diff --git a/backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/expected.csv deleted file mode 100644 index f01e356de2..0000000000 --- a/backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/expected.csv +++ /dev/null @@ -1,5 +0,0 @@ -result,dates,concept select -1,{2012-01-01/2012-01-02},2 -2,{2010-07-15/2010-07-15}, -3,{2013-11-10/2013-11-10},1 -4,{2012-11-11/2012-11-11}, \ No newline at end of file From 9ee0f68e380e204113d27398338305f54570ebf0 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 3 Sep 2024 13:20:25 +0200 Subject: [PATCH 40/54] introduce types for Printers --- .../specific/CountOccurencesSelect.java | 53 --------- .../printers/ArrowResultPrinters.java | 5 +- .../printers/ExcelResultPrinters.java | 20 ++-- .../printers/JavaResultPrinters.java | 48 ++++---- .../printers/JsonResultPrinters.java | 10 +- .../query/resultinfo/printers/Printer.java | 4 +- .../resultinfo/printers/PrinterFactory.java | 27 +++-- .../printers/StringResultPrinters.java | 20 ++-- .../printers/common/BooleanStringPrinter.java | 10 +- .../printers/common/ConceptIdPrinter.java | 8 +- .../printers/common/DatePrinter.java | 6 +- .../common/DateRangeStringPrinter.java | 15 +-- .../printers/common/DateStringPrinter.java | 10 +- .../printers/common/IdentityPrinter.java | 4 +- .../printers/common/IntegerStringPrinter.java | 6 +- .../printers/common/ListStringPrinter.java | 12 +- .../printers/common/LocalizedEnumPrinter.java | 10 +- .../printers/common/MappedPrinter.java | 4 +- .../printers/common/MoneyStringPrinter.java | 4 +- .../printers/common/NumericStringPrinter.java | 4 +- .../printers/common/StringPrinter.java | 4 +- .../printers/common/ToStringPrinter.java | 4 +- .../DefaultSqlCDateSetParserTest.java | 2 +- .../conquery/models/types/ResultTypeTest.java | 106 ++++++++++-------- 24 files changed, 170 insertions(+), 226 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountOccurencesSelect.java diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountOccurencesSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountOccurencesSelect.java deleted file mode 100644 index c06d8cf943..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountOccurencesSelect.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.bakdata.conquery.models.datasets.concepts.select.connector.specific; - -import java.util.EnumSet; - -import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.io.jackson.serializer.NsIdRef; -import com.bakdata.conquery.models.datasets.Column; -import com.bakdata.conquery.models.datasets.concepts.select.Select; -import com.bakdata.conquery.models.datasets.concepts.select.connector.SingleColumnSelect; -import com.bakdata.conquery.models.events.MajorTypeId; -import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.aggregators.specific.MultiSelectAggregator; -import com.bakdata.conquery.models.query.queryplan.aggregators.specific.SelectAggregator; -import com.bakdata.conquery.models.types.ResultType; -import com.fasterxml.jackson.annotation.JsonCreator; -import jakarta.validation.constraints.NotNull; -import lombok.Getter; -import lombok.Setter; - -//TODO delete? -@CPSType(id = "COUNT_OCCURENCES", base = Select.class) -public class CountOccurencesSelect extends SingleColumnSelect { - - @Override - public EnumSet getAcceptedColumnTypes() { - return EnumSet.of(MajorTypeId.STRING); - } - - @Getter - @Setter - @NotNull - private String[] selection; - - @JsonCreator - public CountOccurencesSelect(@NsIdRef Column column, String[] selection) { - super(column); - this.selection = selection; - } - - @Override - public Aggregator createAggregator() { - if (selection.length == 1) { - return new SelectAggregator(getColumn(), selection[0]); - } - - return new MultiSelectAggregator(getColumn(), selection); - } - - @Override - public ResultType getResultType() { - return ResultType.Primitive.INTEGER; - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java index f76206e1bd..caf887b15e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java @@ -1,12 +1,13 @@ package com.bakdata.conquery.models.query.resultinfo.printers; import com.bakdata.conquery.models.query.PrintSettings; +import com.bakdata.conquery.models.query.resultinfo.printers.common.IdentityPrinter; public class ArrowResultPrinters extends JavaResultPrinters { @Override - public Printer getDatePrinter(PrintSettings printSettings) { - return getIntegerPrinter(printSettings); + public Printer getDatePrinter(PrintSettings printSettings) { + return new IdentityPrinter<>(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java index ff709ec4b6..e995a576ea 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java @@ -14,7 +14,7 @@ public class ExcelResultPrinters extends StringResultPrinters { private final PrinterFactory partialDelegate = new StringResultPrinters(); - public Printer printerFor(ResultType type, PrintSettings printSettings) { + public Printer printerFor(ResultType type, PrintSettings printSettings) { if (type instanceof ResultType.ListT listT) { final Printer elementPrinter = partialDelegate.printerFor(listT.getElementType(), printSettings); return getListPrinter(elementPrinter, printSettings); @@ -32,28 +32,28 @@ public Printer printerFor(ResultType type, PrintSettings printSettings) { } @Override - public Printer getBooleanPrinter(PrintSettings printSettings) { - return new IdentityPrinter(); + public Printer getBooleanPrinter(PrintSettings printSettings) { + return new IdentityPrinter<>(); } @Override - public Printer getNumericPrinter(PrintSettings printSettings) { - return new IdentityPrinter(); + public Printer getNumericPrinter(PrintSettings printSettings) { + return new IdentityPrinter<>(); } @Override - public Printer getMoneyPrinter(PrintSettings printSettings) { - return new IdentityPrinter(); + public Printer getMoneyPrinter(PrintSettings printSettings) { + return new IdentityPrinter<>(); } @Override - public Printer getDatePrinter(PrintSettings printSettings) { + public Printer getDatePrinter(PrintSettings printSettings) { return new DatePrinter(); } @Override - public Printer getIntegerPrinter(PrintSettings printSettings) { - return new IdentityPrinter(); + public Printer getIntegerPrinter(PrintSettings printSettings) { + return new IdentityPrinter<>(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java index 1c2eb61f6b..c560520e1f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java @@ -7,59 +7,57 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.common.DatePrinter; import com.bakdata.conquery.models.query.resultinfo.printers.common.IdentityPrinter; -import com.google.common.base.Preconditions; public class JavaResultPrinters extends PrinterFactory { @Override - public Printer getListPrinter(Printer elementPrinter, PrintSettings printSettings) { - return new ListPrinter(elementPrinter); + public Printer> getListPrinter(Printer elementPrinter, PrintSettings printSettings) { + return new ListPrinter<>(elementPrinter); } @Override - public Printer getBooleanPrinter(PrintSettings printSettings) { + public Printer getBooleanPrinter(PrintSettings printSettings) { return new IdentityPrinter(); } @Override - public Printer getIntegerPrinter(PrintSettings printSettings) { - return new IdentityPrinter(); + public Printer getIntegerPrinter(PrintSettings printSettings) { + return new IdentityPrinter<>(); } @Override - public Printer getNumericPrinter(PrintSettings printSettings) { - return new IdentityPrinter(); + public Printer getNumericPrinter(PrintSettings printSettings) { + return new IdentityPrinter<>(); } @Override - public Printer getDatePrinter(PrintSettings printSettings) { + public Printer getDatePrinter(PrintSettings printSettings) { return new DatePrinter(); } @Override - public Printer getDateRangePrinter(PrintSettings printSettings) { + public Printer> getDateRangePrinter(PrintSettings printSettings) { return new DateRangePrinter(); } @Override - public Printer getStringPrinter(PrintSettings printSettings) { - return new IdentityPrinter(); + public Printer getStringPrinter(PrintSettings printSettings) { + return new IdentityPrinter<>(); } @Override - public Printer getMoneyPrinter(PrintSettings printSettings) { - return new IdentityPrinter(); + public Printer getMoneyPrinter(PrintSettings printSettings) { + return new IdentityPrinter<>(); } - private record ListPrinter(Printer elementPrinter) implements Printer { + private record ListPrinter(Printer elementPrinter) implements Printer> { @Override - public Object apply(Object value) { - final List inList = (List) value; - final List out = new ArrayList<>(inList.size()); + public Object apply(List value) { + final List out = new ArrayList<>(value.size()); - for (Object elt : inList) { + for (T elt : value) { out.add(elementPrinter.apply(elt)); } @@ -67,17 +65,13 @@ public Object apply(Object value) { } } - private record DateRangePrinter() implements Printer { + private record DateRangePrinter() implements Printer> { @Override - public Object apply(Object f) { - Preconditions.checkArgument(f instanceof List, "Expected a List got %s (Type: %s, as string: %s)", f, f.getClass().getName(), f); - Preconditions.checkArgument(((List) f).size() == 2, "Expected a list with 2 elements, one min, one max. The list was: %s ", f); - - final List list = (List) f; - final Integer min = (Integer) list.get(0); - final Integer max = (Integer) list.get(1); + public Object apply(List f) { + final Integer min = (Integer) ((List) f).get(0); + final Integer max = (Integer) ((List) f).get(1); return CDateRange.of(min, max); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java index 8aa2400a05..c45bb0ca5a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.query.resultinfo.printers; +import java.util.List; + import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.common.ToStringPrinter; import lombok.ToString; @@ -9,13 +11,13 @@ public class JsonResultPrinters extends JavaResultPrinters { @Override - public Printer getDatePrinter(PrintSettings printSettings) { - return new ToStringPrinter(super.getDatePrinter(printSettings)); + public Printer getDatePrinter(PrintSettings printSettings) { + return new ToStringPrinter<>(super.getDatePrinter(printSettings)); } @Override - public Printer getDateRangePrinter(PrintSettings printSettings) { - return new ToStringPrinter(super.getDateRangePrinter(printSettings)); + public Printer> getDateRangePrinter(PrintSettings printSettings) { + return new ToStringPrinter<>(super.getDateRangePrinter(printSettings)); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java index 01c6aadca5..1a78023393 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java @@ -3,6 +3,6 @@ import java.util.function.Function; @FunctionalInterface -public interface Printer extends Function { - Object apply(Object value); +public interface Printer extends Function { + Object apply(T value); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java index 2cf548e810..beb868a420 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java @@ -1,16 +1,19 @@ package com.bakdata.conquery.models.query.resultinfo.printers; +import java.util.List; + import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.types.ResultType; + public abstract class PrinterFactory { - public Printer printerFor(ResultType type, PrintSettings printSettings) { + public Printer printerFor(ResultType type, PrintSettings printSettings) { if (type instanceof ResultType.ListT listT) { - final Printer elementPrinter = printerFor(listT.getElementType(), printSettings); - return getListPrinter(elementPrinter, printSettings); + final Printer elementPrinter = printerFor(listT.getElementType(), printSettings); + return (Printer) getListPrinter(elementPrinter, printSettings); } - return switch (((ResultType.Primitive) type)) { + return (Printer) switch (((ResultType.Primitive) type)) { case BOOLEAN -> getBooleanPrinter(printSettings); case INTEGER -> getIntegerPrinter(printSettings); case NUMERIC -> getNumericPrinter(printSettings); @@ -21,19 +24,19 @@ public Printer printerFor(ResultType type, PrintSettings printSettings) { }; } - public abstract Printer getListPrinter(Printer elementPrinter, PrintSettings printSettings); + public abstract Printer> getListPrinter(Printer elementPrinter, PrintSettings printSettings); - public abstract Printer getBooleanPrinter(PrintSettings printSettings); + public abstract Printer getBooleanPrinter(PrintSettings printSettings); - public abstract Printer getIntegerPrinter(PrintSettings printSettings); + public abstract Printer getIntegerPrinter(PrintSettings printSettings); - public abstract Printer getNumericPrinter(PrintSettings printSettings); + public abstract Printer getNumericPrinter(PrintSettings printSettings); - public abstract Printer getDatePrinter(PrintSettings printSettings); + public abstract Printer getDatePrinter(PrintSettings printSettings); - public abstract Printer getDateRangePrinter(PrintSettings printSettings); + public abstract Printer> getDateRangePrinter(PrintSettings printSettings); - public abstract Printer getStringPrinter(PrintSettings printSettings); + public abstract Printer getStringPrinter(PrintSettings printSettings); - public abstract Printer getMoneyPrinter(PrintSettings printSettings); + public abstract Printer getMoneyPrinter(PrintSettings printSettings); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java index e6ece1b5c5..574cd7f43a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.query.resultinfo.printers; +import java.util.List; + import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.common.BooleanStringPrinter; import com.bakdata.conquery.models.query.resultinfo.printers.common.DateRangeStringPrinter; @@ -16,42 +18,42 @@ public class StringResultPrinters extends PrinterFactory { @Override - public Printer getListPrinter(Printer elementPrinter, PrintSettings printSettings) { - return new ListStringPrinter(elementPrinter, printSettings); + public Printer> getListPrinter(Printer elementPrinter, PrintSettings printSettings) { + return new ListStringPrinter<>(elementPrinter, printSettings); } @Override - public Printer getBooleanPrinter(PrintSettings printSettings) { + public Printer getBooleanPrinter(PrintSettings printSettings) { return BooleanStringPrinter.create(printSettings); } @Override - public Printer getIntegerPrinter(PrintSettings printSettings) { + public Printer getIntegerPrinter(PrintSettings printSettings) { return new IntegerStringPrinter(printSettings); } @Override - public Printer getNumericPrinter(PrintSettings printSettings) { + public Printer getNumericPrinter(PrintSettings printSettings) { return new NumericStringPrinter(printSettings); } @Override - public Printer getDatePrinter(PrintSettings printSettings) { + public Printer getDatePrinter(PrintSettings printSettings) { return new DateStringPrinter(printSettings); } @Override - public Printer getDateRangePrinter(PrintSettings printSettings) { + public Printer> getDateRangePrinter(PrintSettings printSettings) { return new DateRangeStringPrinter(printSettings); } @Override - public Printer getStringPrinter(PrintSettings printSettings) { + public Printer getStringPrinter(PrintSettings printSettings) { return new StringPrinter(); } @Override - public Printer getMoneyPrinter(PrintSettings printSettings) { + public Printer getMoneyPrinter(PrintSettings printSettings) { return new MoneyStringPrinter(printSettings); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/BooleanStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/BooleanStringPrinter.java index e35036483f..d7d3879d21 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/BooleanStringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/BooleanStringPrinter.java @@ -5,7 +5,7 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; -public record BooleanStringPrinter(PrintSettings cfg, String trueVal, String falseVal) implements Printer { +public record BooleanStringPrinter(PrintSettings cfg, String trueVal, String falseVal) implements Printer { public static BooleanStringPrinter create(PrintSettings settings) { if (!settings.isPrettyPrint()) { @@ -18,11 +18,7 @@ public static BooleanStringPrinter create(PrintSettings settings) { @Override - public String apply(Object f) { - if ((Boolean) f) { - return trueVal; - } - return falseVal; - + public String apply(Boolean f) { + return f ? trueVal : falseVal; } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ConceptIdPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ConceptIdPrinter.java index 35a6f317d0..51fb158a13 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ConceptIdPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ConceptIdPrinter.java @@ -6,16 +6,14 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; -public record ConceptIdPrinter(Concept concept, PrintSettings cfg) implements Printer { +public record ConceptIdPrinter(Concept concept, PrintSettings cfg) implements Printer { @Override - public String apply(Object rawValue) { - if (rawValue == null) { + public String apply(Integer localId) { + if (localId == null) { return null; } - final int localId = (int) rawValue; - final ConceptTreeNode node = ((TreeConcept) concept).getElementByLocalId(localId); if (!cfg.isPrettyPrint()) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DatePrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DatePrinter.java index 2e8fbf8a1c..c03d72f55b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DatePrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DatePrinter.java @@ -3,10 +3,10 @@ import com.bakdata.conquery.models.common.CDate; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; -public record DatePrinter() implements Printer { +public record DatePrinter() implements Printer { @Override - public Object apply(Object value) { - return CDate.toLocalDate(((Number) value).intValue()); + public Object apply(Number value) { + return CDate.toLocalDate(value.intValue()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateRangeStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateRangeStringPrinter.java index e9e1b3c7f3..f1f623cff1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateRangeStringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateRangeStringPrinter.java @@ -9,24 +9,19 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public record DateRangeStringPrinter(DateStringPrinter datePrinter, PrintSettings cfg) implements Printer { +public record DateRangeStringPrinter(DateStringPrinter datePrinter, PrintSettings cfg) implements Printer> { public DateRangeStringPrinter(PrintSettings printSettings) { this(new DateStringPrinter(printSettings), printSettings); } @Override - public String apply(Object f) { - Preconditions.checkArgument(f instanceof List, "Expected a List got %s (Type: %s, as string: %s)", f, f.getClass().getName(), f); - Preconditions.checkArgument(((List) f).size() == 2, "Expected a list with 2 elements, one min, one max. The list was: %s ", f); + public String apply(List f) { + Preconditions.checkArgument(f.size() == 2, "Expected a list with 2 elements, one min, one max. The list was: %s ", f); - final List list = (List) f; - final Integer min = (Integer) list.get(0); - final Integer max = (Integer) list.get(1); + final Integer min = f.get(0); + final Integer max = f.get(1); - if (min == null || max == null) { - log.warn("Encountered incomplete range, treating it as an open range. Either min or max was null: {}", list); - } // Compute minString first because we need it either way final String minString = min == null || min == CDateRange.NEGATIVE_INFINITY ? "-∞" : datePrinter.apply(min); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateStringPrinter.java index 7d912c5d09..5240159d19 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateStringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateStringPrinter.java @@ -3,15 +3,11 @@ import com.bakdata.conquery.models.common.CDate; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; -import com.google.common.base.Preconditions; -public record DateStringPrinter(PrintSettings cfg) implements Printer { +public record DateStringPrinter(PrintSettings cfg) implements Printer { @Override - public String apply(Object f) { - Preconditions.checkArgument(f instanceof Number, "Expected an Number but got an '%s' with the value: %s".formatted(f.getClass().getName(), f)); - - final Number number = (Number) f; - return cfg.getDateFormatter().format(CDate.toLocalDate(number.intValue())); + public String apply(Number f) { + return cfg.getDateFormatter().format(CDate.toLocalDate(f.intValue())); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/IdentityPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/IdentityPrinter.java index a1fcc83bcb..134135d727 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/IdentityPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/IdentityPrinter.java @@ -2,10 +2,10 @@ import com.bakdata.conquery.models.query.resultinfo.printers.Printer; -public record IdentityPrinter() implements Printer { +public record IdentityPrinter() implements Printer { @Override - public Object apply(Object value) { + public Object apply(T value) { return value; } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/IntegerStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/IntegerStringPrinter.java index 1345b718a3..47a765d83a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/IntegerStringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/IntegerStringPrinter.java @@ -3,12 +3,12 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; -public record IntegerStringPrinter(PrintSettings cfg) implements Printer { +public record IntegerStringPrinter(PrintSettings cfg) implements Printer { @Override - public String apply(Object f) { + public String apply(Number f) { if (cfg.isPrettyPrint()) { - return cfg.getIntegerFormat().format(((Number) f).longValue()); + return cfg.getIntegerFormat().format(f.longValue()); } return f.toString(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java index e25ccc7d7f..4128b217d9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java @@ -6,23 +6,19 @@ import com.bakdata.conquery.models.config.LocaleConfig; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; -import com.google.common.base.Preconditions; -public record ListStringPrinter(Printer elementPrinter, PrintSettings cfg, LocaleConfig.ListFormat listFormat) implements Printer { +public record ListStringPrinter(Printer elementPrinter, PrintSettings cfg, LocaleConfig.ListFormat listFormat) implements Printer> { - public ListStringPrinter(Printer elementPrinter, PrintSettings cfg) { + public ListStringPrinter(Printer elementPrinter, PrintSettings cfg) { this(elementPrinter, cfg, cfg.getListFormat()); } @Override - public String apply(Object f) { - - // Jackson deserializes collections as lists instead of an array, if the type is not given - Preconditions.checkArgument(f instanceof List, "Expected a List got %s (as String `%s` )".formatted(f.getClass().getName(), f)); + public String apply(List f) { final StringJoiner joiner = listFormat.createListJoiner(); - for (Object obj : (List) f) { + for (T obj : f) { joiner.add(listFormat.escapeListElement(elementPrinter.apply(obj).toString())); } return joiner.toString(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/LocalizedEnumPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/LocalizedEnumPrinter.java index b94a6ece63..cf7e37b24c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/LocalizedEnumPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/LocalizedEnumPrinter.java @@ -4,15 +4,11 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; -public record LocalizedEnumPrinter & LocalizedToString>(PrintSettings cfg, Class clazz) implements Printer { +public record LocalizedEnumPrinter & LocalizedToString>(PrintSettings cfg, Class clazz) implements Printer { @Override - public String apply(Object f) { - - if (clazz.isInstance(f)) { - return clazz.cast(f).toString(cfg.getLocale()); - } + public String apply(String f) { try { - return Enum.valueOf(clazz, f.toString()).toString(cfg.getLocale()); + return Enum.valueOf(clazz, f).toString(cfg.getLocale()); } catch (Exception e) { throw new IllegalArgumentException("%s is not a valid %s.".formatted(f, clazz), e); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MappedPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MappedPrinter.java index 58b32eda8e..91f3f45333 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MappedPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MappedPrinter.java @@ -3,10 +3,10 @@ import com.bakdata.conquery.models.index.InternToExternMapper; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; -public record MappedPrinter(InternToExternMapper mapper) implements Printer { +public record MappedPrinter(InternToExternMapper mapper) implements Printer { @Override - public String apply(Object f) { + public String apply(String f) { return mapper.external(((String) f)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MoneyStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MoneyStringPrinter.java index 4e5a04bfe8..23621fb959 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MoneyStringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MoneyStringPrinter.java @@ -3,10 +3,10 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; -public record MoneyStringPrinter(PrintSettings cfg) implements Printer { +public record MoneyStringPrinter(PrintSettings cfg) implements Printer { @Override - public String apply(Object f) { + public String apply(Number f) { if (cfg.isPrettyPrint()) { return cfg.getCurrencyFormat().format(f); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/NumericStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/NumericStringPrinter.java index d5a9a95661..b2b6bd40ee 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/NumericStringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/NumericStringPrinter.java @@ -3,10 +3,10 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; -public record NumericStringPrinter(PrintSettings cfg) implements Printer { +public record NumericStringPrinter(PrintSettings cfg) implements Printer { @Override - public String apply(Object f) { + public String apply(Number f) { if (cfg.isPrettyPrint()) { return cfg.getDecimalFormat().format(f); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/StringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/StringPrinter.java index 322b6907a0..50e34f33c1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/StringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/StringPrinter.java @@ -4,9 +4,9 @@ import com.bakdata.conquery.models.query.resultinfo.printers.Printer; -public record StringPrinter() implements Printer { +public record StringPrinter() implements Printer { @Override - public String apply(Object f) { + public String apply(String f) { return Objects.toString(f); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ToStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ToStringPrinter.java index affefd797a..2a06847e3a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ToStringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ToStringPrinter.java @@ -4,10 +4,10 @@ import com.bakdata.conquery.models.query.resultinfo.printers.Printer; -public record ToStringPrinter(Printer delegate) implements Printer { +public record ToStringPrinter(Printer delegate) implements Printer { @Override - public Object apply(Object value) { + public Object apply(T value) { return Objects.toString(delegate.apply(value)); } } diff --git a/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java b/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java index c4fdae0196..16eb417ec0 100644 --- a/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java +++ b/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java @@ -25,7 +25,7 @@ class DefaultSqlCDateSetParserTest { @MethodSource("testToEpochDayRangeListProvider") public void testToEpochDayRangeList(String input, String expected, String message) { List> epochDayRangeList = parser.toEpochDayRangeList(input); - final String actual = (String) csvResultPrinters.printerFor(new ResultType.ListT(ResultType.Primitive.DATE_RANGE), PLAIN).apply(epochDayRangeList); + final String actual = (String) csvResultPrinters.>>printerFor(new ResultType.ListT<>(ResultType.Primitive.DATE_RANGE), PLAIN).apply(epochDayRangeList); Assertions.assertEquals(expected, actual, message); } diff --git a/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java b/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java index 89b2520975..921726a48b 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java @@ -11,6 +11,7 @@ import java.util.Map; import com.bakdata.conquery.io.jackson.Jackson; +import com.bakdata.conquery.models.common.CDate; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.forms.util.Resolution; import com.bakdata.conquery.models.i18n.I18n; @@ -19,7 +20,6 @@ import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.query.resultinfo.printers.StringResultPrinters; -import com.google.common.collect.ImmutableMap; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -41,52 +41,66 @@ public class ResultTypeTest { @SuppressWarnings("unused") public static List testData() { - return List.of( - Arguments.of(PRETTY, ResultType.Primitive.BOOLEAN, true, "Yes"), - Arguments.of(PRETTY, ResultType.Primitive.BOOLEAN, false, "No"), - Arguments.of(PRETTY, ResultType.Primitive.STRING, "test", "test"), - Arguments.of(PRETTY, ResultType.Primitive.DATE, LocalDate.of(2013, 7, 12).toEpochDay(), "2013-07-12"), - Arguments.of(PRETTY_DE, ResultType.Primitive.DATE, LocalDate.of(2013, 7, 12).toEpochDay(), "12.07.2013"), - Arguments.of(PRETTY, ResultType.Primitive.DATE_RANGE, List.of(Long.valueOf(LocalDate.of(2013, 7, 12).toEpochDay()) - .intValue(), Long.valueOf(LocalDate.of(2013, 7, 12).toEpochDay()) - .intValue()), "2013-07-12"), - Arguments.of(PRETTY_DE, ResultType.Primitive.DATE_RANGE, List.of(Long.valueOf(LocalDate.of(2013, 7, 12).toEpochDay()) - .intValue(), Long.valueOf(LocalDate.of(2013, 7, 12).toEpochDay()) - .intValue()), "12.07.2013"), - Arguments.of(PRETTY, ResultType.Primitive.DATE_RANGE, List.of(Long.valueOf(LocalDate.of(2013, 7, 12).toEpochDay()) - .intValue(), Long.valueOf(LocalDate.of(2014, 7, 12).toEpochDay()) - .intValue()), "2013-07-12/2014-07-12"), - Arguments.of(PRETTY_DE, ResultType.Primitive.DATE_RANGE, List.of(Long.valueOf(LocalDate.of(2013, 7, 12).toEpochDay()) - .intValue(), Long.valueOf(LocalDate.of(2014, 7, 12).toEpochDay()) - .intValue()), "12.07.2013 - 12.07.2014"), - Arguments.of(PRETTY, ResultType.Primitive.INTEGER, 51839274, "51,839,274"), - Arguments.of(PRETTY_DE, ResultType.Primitive.INTEGER, 51839274, "51.839.274"), - Arguments.of(PRETTY, ResultType.Primitive.MONEY, new BigDecimal("518392.74"), "€518,392.74"), - Arguments.of(PRETTY_DE, ResultType.Primitive.MONEY, new BigDecimal("518392.74"), "518.392,74\u00A0€"), - Arguments.of(PRETTY, ResultType.Primitive.NUMERIC, 0.2, "0.2"), - Arguments.of(PRETTY_DE, ResultType.Primitive.NUMERIC, 0.2, "0,2"), - Arguments.of(PRETTY, ResultType.Primitive.NUMERIC, new BigDecimal("716283712389817246892743124.12312"), "716,283,712,389,817,246,892,743,124.12312"), - Arguments.of(PRETTY_DE, ResultType.Primitive.NUMERIC, new BigDecimal("716283712389817246892743124.12312"), "716.283.712.389.817.246.892.743.124,12312"), - Arguments.of(PRETTY, ResultType.Primitive.STRING, "test", "test"), - - Arguments.of(PLAIN, ResultType.Primitive.BOOLEAN, true, "1"), - Arguments.of(PLAIN, ResultType.Primitive.BOOLEAN, false, "0"), - Arguments.of(PLAIN, ResultType.Primitive.STRING, "test", "test"), - Arguments.of(PLAIN, ResultType.Primitive.DATE, LocalDate.of(2013, 7, 12).toEpochDay(), "2013-07-12"), - Arguments.of(PLAIN, ResultType.Primitive.INTEGER, 51839274, "51839274"), - Arguments.of(PLAIN, ResultType.Primitive.MONEY, new BigDecimal(51839274L), "51839274"), - Arguments.of(PLAIN, ResultType.Primitive.NUMERIC, 0.2, "0.2"), - Arguments.of(PLAIN, ResultType.Primitive.NUMERIC, new BigDecimal("716283712389817246892743124.12312"), "716283712389817246892743124.12312"), - Arguments.of(PLAIN, ResultType.Primitive.STRING, "test", "test"), - Arguments.of(PLAIN, ResultType.Primitive.STRING, Resolution.COMPLETE.name(), "COMPLETE"), // TODO fk: is supposed not to test the mapping? - Arguments.of(PLAIN, ResultType.Primitive.STRING, ImmutableMap.of("a", 2, "c", 1), "{a=2, c=1}") + return List.of(Arguments.of(PRETTY, ResultType.Primitive.BOOLEAN, true, "Yes"), + Arguments.of(PRETTY, ResultType.Primitive.BOOLEAN, false, "No"), + Arguments.of(PRETTY, ResultType.Primitive.STRING, "test", "test"), + Arguments.of(PRETTY, ResultType.Primitive.DATE, CDate.ofLocalDate(LocalDate.of(2013, 7, 12)), "2013-07-12"), + Arguments.of(PRETTY_DE, ResultType.Primitive.DATE, CDate.ofLocalDate(LocalDate.of(2013, 7, 12)), "12.07.2013"), + Arguments.of(PRETTY, + ResultType.Primitive.DATE_RANGE, + List.of(CDate.ofLocalDate(LocalDate.of(2013, 7, 12)), CDate.ofLocalDate(LocalDate.of(2013, 7, 12))), + "2013-07-12" + ), + Arguments.of(PRETTY_DE, + ResultType.Primitive.DATE_RANGE, + List.of(CDate.ofLocalDate(LocalDate.of(2013, 7, 12)), CDate.ofLocalDate(LocalDate.of(2013, 7, 12))), + "12.07.2013" + ), + Arguments.of(PRETTY, + ResultType.Primitive.DATE_RANGE, + List.of(CDate.ofLocalDate(LocalDate.of(2013, 7, 12)), CDate.ofLocalDate(LocalDate.of(2014, 7, 12))), + "2013-07-12/2014-07-12" + ), + Arguments.of(PRETTY_DE, + ResultType.Primitive.DATE_RANGE, + List.of(CDate.ofLocalDate(LocalDate.of(2013, 7, 12)), CDate.ofLocalDate(LocalDate.of(2014, 7, 12))), + "12.07.2013 - 12.07.2014" + ), + Arguments.of(PRETTY, ResultType.Primitive.INTEGER, 51839274L, "51,839,274"), + Arguments.of(PRETTY_DE, ResultType.Primitive.INTEGER, 51839274L, "51.839.274"), + Arguments.of(PRETTY, ResultType.Primitive.MONEY, new BigDecimal("518392.74"), "€518,392.74"), + Arguments.of(PRETTY_DE, ResultType.Primitive.MONEY, new BigDecimal("518392.74"), "518.392,74\u00A0€"), + Arguments.of(PRETTY, ResultType.Primitive.NUMERIC, 0.2, "0.2"), + Arguments.of(PRETTY_DE, ResultType.Primitive.NUMERIC, 0.2, "0,2"), + Arguments.of(PRETTY, + ResultType.Primitive.NUMERIC, + new BigDecimal("716283712389817246892743124.12312"), + "716,283,712,389,817,246,892,743,124.12312" + ), + Arguments.of(PRETTY_DE, + ResultType.Primitive.NUMERIC, + new BigDecimal("716283712389817246892743124.12312"), + "716.283.712.389.817.246.892.743.124,12312" + ), + Arguments.of(PRETTY, ResultType.Primitive.STRING, "test", "test"), + + Arguments.of(PLAIN, ResultType.Primitive.BOOLEAN, true, "1"), + Arguments.of(PLAIN, ResultType.Primitive.BOOLEAN, false, "0"), + Arguments.of(PLAIN, ResultType.Primitive.STRING, "test", "test"), + Arguments.of(PLAIN, ResultType.Primitive.DATE, LocalDate.of(2013, 7, 12).toEpochDay(), "2013-07-12"), + Arguments.of(PLAIN, ResultType.Primitive.INTEGER, 51839274L, "51839274"), + Arguments.of(PLAIN, ResultType.Primitive.MONEY, new BigDecimal(51839274L), "51839274"), + Arguments.of(PLAIN, ResultType.Primitive.NUMERIC, 0.2, "0.2"), + Arguments.of(PLAIN, + ResultType.Primitive.NUMERIC, + new BigDecimal("716283712389817246892743124.12312"), + "716283712389817246892743124.12312" + ), + Arguments.of(PLAIN, ResultType.Primitive.STRING, "test", "test"), + Arguments.of(PLAIN, ResultType.Primitive.STRING, Resolution.COMPLETE.name(), "COMPLETE") ); } - public static ResultInfo info(ResultType type) { - return new ExternalResultInfo("col", type); - } - @ParameterizedTest(name = "{0} {1}: {2} -> {3}") @MethodSource("testData") public void testPrinting(PrintSettings printSettings, ResultType type, Object value, String expected) throws IOException { @@ -102,6 +116,10 @@ public void testPrinting(PrintSettings printSettings, ResultType type, Object va assertThat(printer.apply(copy)).isEqualTo(expected); } + public static ResultInfo info(ResultType type) { + return new ExternalResultInfo("col", type); + } + @ParameterizedTest(name = "{1}: {2}") @MethodSource("testData") public void testBinaryPrinting(PrintSettings printSettings, ResultType type, Object value, String expected) throws IOException { From b90891a9c9bcf40237a7d21bd23a39af7f91fe8c Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 3 Sep 2024 14:25:29 +0200 Subject: [PATCH 41/54] pull PrinterFactory out of PrintSettings, pass manually into createPrinter --- .../com/bakdata/conquery/ResultHeaders.java | 5 +- .../conquery/apiv1/QueryProcessor.java | 12 +- .../io/result/arrow/ArrowRenderer.java | 7 +- .../io/result/arrow/ResultArrowProcessor.java | 4 +- .../conquery/io/result/csv/CsvRenderer.java | 9 +- .../io/result/csv/ResultCsvProcessor.java | 3 +- .../io/result/excel/ExcelRenderer.java | 8 +- .../io/result/excel/ResultExcelProcessor.java | 3 +- .../parquet/EntityResultWriteSupport.java | 11 +- .../parquet/ResultParquetProcessor.java | 3 +- .../models/config/ExcelResultProvider.java | 5 - .../datasets/concepts/select/Select.java | 2 +- .../select/concept/ConceptColumnSelect.java | 2 +- .../select/connector/DistinctSelect.java | 4 +- .../specific/MappableSingleColumnSelect.java | 4 +- .../models/execution/ManagedExecution.java | 6 +- .../conquery/models/query/PrintSettings.java | 10 +- .../models/query/SingleTableResult.java | 2 +- .../query/preview/EntityPreviewExecution.java | 286 ++++++++-------- .../query/resultinfo/ColumnResultInfo.java | 5 +- .../query/resultinfo/ExternalResultInfo.java | 5 +- .../resultinfo/FixedLabelResultInfo.java | 5 +- .../models/query/resultinfo/ResultInfo.java | 3 +- .../resultinfo/SecondaryIdResultInfo.java | 5 +- .../query/resultinfo/SelectResultInfo.java | 5 +- .../query/statistics/ResultStatistics.java | 6 +- .../sql/conversion/NodeConversions.java | 2 +- .../DefaultSqlCDateSetParserTest.java | 2 +- .../json/AbstractQueryEngineTest.java | 5 - .../conquery/integration/json/FormTest.java | 106 +++--- .../arrow/ArrowResultGenerationTest.java | 324 ++++++++++-------- .../result/csv/CsvResultGenerationTest.java | 10 +- .../result/excel/ExcelResultRenderTest.java | 19 +- .../parquet/ParquetResultGenerationTest.java | 6 +- .../models/execution/DefaultLabelTest.java | 2 +- .../models/query/DefaultColumnNameTest.java | 2 +- .../conquery/models/query/UniqueNameTest.java | 3 +- .../conquery/models/types/ResultTypeTest.java | 15 +- 38 files changed, 483 insertions(+), 433 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java b/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java index 5fff3d083b..e42347476b 100644 --- a/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java +++ b/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java @@ -10,6 +10,7 @@ import com.bakdata.conquery.models.query.resultinfo.FixedLabelResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.query.resultinfo.printers.common.LocalizedEnumPrinter; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; @@ -74,7 +75,7 @@ public static ResultInfo formResolutionInfo() { return new FixedLabelResultInfo(ResultType.Primitive.STRING, Set.of()) { @Override - public Printer createPrinter(PrintSettings printSettings) { + public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printSettings) { return new LocalizedEnumPrinter<>(printSettings, Resolution.class); } @@ -99,7 +100,7 @@ public static ResultInfo formObservationScopeInfo() { return new FixedLabelResultInfo(ResultType.Primitive.STRING, Set.of()) { @Override - public Printer createPrinter(PrintSettings printSettings) { + public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printSettings) { return new LocalizedEnumPrinter<>(printSettings, FeatureGroup.class); } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index 2a76bd91c5..91c1bbe372 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -579,7 +579,7 @@ public ResultStatistics getResultStatistics(SingleTableResult managedQuery) { final PrintSettings printSettings = - new PrintSettings(true, locale, managedQuery.getNamespace(), config, null, null, decimalFormat, integerFormat, new JavaResultPrinters()); + new PrintSettings(true, locale, managedQuery.getNamespace(), config, null, null, decimalFormat, integerFormat); final UniqueNamer uniqueNamer = new UniqueNamer(printSettings); final List resultInfos = managedQuery.getResultInfos(); @@ -590,7 +590,15 @@ public ResultStatistics getResultStatistics(SingleTableResult managedQuery) { final Optional dateIndex = dateInfo.map(resultInfos::indexOf); - return ResultStatistics.collectResultStatistics(managedQuery, resultInfos, dateInfo, dateIndex, printSettings, uniqueNamer, config); + return ResultStatistics.collectResultStatistics(managedQuery, + resultInfos, + dateInfo, + dateIndex, + printSettings, + uniqueNamer, + config, + new JavaResultPrinters() + ); } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java index 5d2362d5c2..852035bb84 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java @@ -18,6 +18,7 @@ import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.types.ResultType; import lombok.extern.slf4j.Slf4j; @@ -48,7 +49,7 @@ public static void renderToStream( ArrowConfig arrowConfig, List idHeaders, List resultInfo, - Stream results) throws IOException { + Stream results, PrinterFactory printerFactory) throws IOException { final List fields = ArrowUtil.generateFields(idHeaders, resultInfo, new UniqueNamer(printSettings), printSettings); final VectorSchemaRoot root = VectorSchemaRoot.create(new Schema(fields, null), ROOT_ALLOCATOR); @@ -60,11 +61,11 @@ public static void renderToStream( final List printers = new ArrayList<>(); for (ResultInfo header : idHeaders) { - printers.add(header.createPrinter(printSettings)); + printers.add(header.createPrinter(printerFactory, printSettings)); } for (ResultInfo info : resultInfo) { - printers.add(info.createPrinter(printSettings)); + printers.add(info.createPrinter(printerFactory, printSettings)); } // Write the data diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java index a00bbb3649..376d1dcb17 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java @@ -96,7 +96,7 @@ public static Response getArrow IdPrinter idPrinter = IdColumnUtil.getIdPrinter(subject, exec, namespace, config.getIdColumns().getIds()); final Locale locale = I18n.LOCALE.get(); - PrintSettings settings = new PrintSettings(pretty, locale, namespace, config, idPrinter::createId, null, new ArrowResultPrinters()); + PrintSettings settings = new PrintSettings(pretty, locale, namespace, config, idPrinter::createId, null); // Collect ResultInfos for id columns and result columns @@ -111,7 +111,7 @@ public static Response getArrow arrowConfig, resultInfosId, resultInfosExec, - exec.streamResults(limit) + exec.streamResults(limit), new ArrowResultPrinters() ); } finally { diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java index 82e8da8692..a9eac207ff 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java @@ -10,6 +10,8 @@ import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; +import com.bakdata.conquery.models.query.resultinfo.printers.StringResultPrinters; import com.bakdata.conquery.models.query.results.EntityResult; import com.univocity.parsers.csv.CsvWriter; import lombok.RequiredArgsConstructor; @@ -30,11 +32,12 @@ public void toCSV(List idHeaders, List infos, Stream infos, Stream results, PrintSettings printSettings) { - final Printer[] printers = infos.stream().map(info -> info.createPrinter(printSettings)).toArray(Printer[]::new); + private void createCSVBody(PrintSettings cfg, List infos, Stream results, PrintSettings printSettings, + PrinterFactory printerFactory) { + final Printer[] printers = infos.stream().map(info -> info.createPrinter(printerFactory, printSettings)).toArray(Printer[]::new); results.map(result -> Pair.of(cfg.getIdMapper().map(result), result)) .sorted(Map.Entry.comparingByKey()) diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java index 78cb604251..261ca45f1e 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java @@ -18,7 +18,6 @@ import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; -import com.bakdata.conquery.models.query.resultinfo.printers.StringResultPrinters; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.resources.ResourceConstants; @@ -58,7 +57,7 @@ public Response createResult(Su // Get the locale extracted by the LocaleFilter final Locale locale = I18n.LOCALE.get(); - final PrintSettings settings = new PrintSettings(pretty, locale, namespace, config, idPrinter::createId, null, new StringResultPrinters()); + final PrintSettings settings = new PrintSettings(pretty, locale, namespace, config, idPrinter::createId, null); final StreamingOutput out = os -> { try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(os, charset))) { diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java index ccb2185817..ec29adb030 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java @@ -21,7 +21,9 @@ import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; +import com.bakdata.conquery.models.query.resultinfo.printers.ExcelResultPrinters; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.types.ResultType; import com.google.common.collect.ImmutableMap; @@ -72,7 +74,7 @@ public void renderToStream(List writeHeader(sheet, idHeaders, resultInfosExec, table, printSettings); - final int writtenLines = writeBody(sheet, resultInfosExec, exec.streamResults(OptionalLong.of(limit.orElse(MAX_LINES)))); + final int writtenLines = writeBody(sheet, resultInfosExec, exec.streamResults(OptionalLong.of(limit.orElse(MAX_LINES))), new ExcelResultPrinters()); postProcessTable(sheet, table, writtenLines, idHeaders.size()); @@ -171,12 +173,12 @@ private void writeHeader( private int writeBody( SXSSFSheet sheet, List infos, - Stream resultLines) { + Stream resultLines, PrinterFactory printerFactory) { // Row 0 is the Header the data starts at 1 final AtomicInteger currentRow = new AtomicInteger(1); - final TypeWriter[] writers = infos.stream().map(info -> writer(info.getType(), info.createPrinter(settings), settings)).toArray(TypeWriter[]::new); + final TypeWriter[] writers = infos.stream().map(info -> writer(info.getType(), info.createPrinter(printerFactory, settings), settings)).toArray(TypeWriter[]::new); final PrintIdMapper idMapper = settings.getIdMapper(); final int writtenLines = resultLines.mapToInt(l -> writeRowsForEntity(infos, l, currentRow, sheet, writers, idMapper)).sum(); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java index 520f850aa2..cb0c79d25c 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java @@ -15,7 +15,6 @@ import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; -import com.bakdata.conquery.models.query.resultinfo.printers.ExcelResultPrinters; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.resources.ResourceConstants; @@ -53,7 +52,7 @@ public Response createResult(Su final IdPrinter idPrinter = IdColumnUtil.getIdPrinter(subject, exec, namespace, conqueryConfig.getIdColumns().getIds()); final Locale locale = I18n.LOCALE.get(); - final PrintSettings settings = new PrintSettings(pretty, locale, namespace, conqueryConfig, idPrinter::createId, null, new ExcelResultPrinters()); + final PrintSettings settings = new PrintSettings(pretty, locale, namespace, conqueryConfig, idPrinter::createId, null); final ExcelRenderer excelRenderer = new ExcelRenderer(excelConfig, settings); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java index 6f8cab4e31..693de95fd4 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java @@ -10,7 +10,9 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; +import com.bakdata.conquery.models.query.resultinfo.printers.ArrowResultPrinters; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.MultilineEntityResult; import com.bakdata.conquery.models.types.ResultType; @@ -80,14 +82,15 @@ private static List generateColumnConsumers(List idH return consumers; } - private static List generateColumnPrinters(List idHeaders, List resultInfos, PrintSettings printSettings) { + private static List generateColumnPrinters(List idHeaders, List resultInfos, PrintSettings printSettings, + PrinterFactory printerFactory) { final List consumers = new ArrayList<>(); for (ResultInfo idHeader : idHeaders) { - consumers.add(idHeader.createPrinter(printSettings)); + consumers.add(idHeader.createPrinter(printerFactory, printSettings)); } for (ResultInfo resultInfo : resultInfos) { - consumers.add(resultInfo.createPrinter(printSettings)); + consumers.add(resultInfo.createPrinter(printerFactory, printSettings)); } return consumers; } @@ -112,7 +115,7 @@ private static ColumnConsumer getForResultType(ResultType resultType) { public WriteContext init(Configuration configuration) { schema = generateSchema(idHeaders, resultInfo, new UniqueNamer(printSettings), printSettings); columnConsumers = generateColumnConsumers(idHeaders, resultInfo); - columnPrinters = generateColumnPrinters(idHeaders, resultInfo, printSettings); + columnPrinters = generateColumnPrinters(idHeaders, resultInfo, printSettings, new ArrowResultPrinters()); return new WriteContext(schema, Map.of()); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java index a6b03de025..33ce54324a 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java @@ -14,7 +14,6 @@ import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; -import com.bakdata.conquery.models.query.resultinfo.printers.ArrowResultPrinters; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.resources.ResourceConstants; @@ -53,7 +52,7 @@ public Response createResultFile(Subject subject, ManagedExecution exec, boolean final IdPrinter idPrinter = IdColumnUtil.getIdPrinter(subject, exec, namespace, config.getIdColumns().getIds()); final Locale locale = I18n.LOCALE.get(); - final PrintSettings settings = new PrintSettings(pretty, locale, namespace, config, idPrinter::createId, null, new ArrowResultPrinters()); //TODO Parquet different printer? + final PrintSettings settings = new PrintSettings(pretty, locale, namespace, config, idPrinter::createId, null); final StreamingOutput out = output -> { diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java index 560334d5bb..5f2e713f22 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java @@ -13,10 +13,7 @@ import com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider; import com.bakdata.conquery.io.result.excel.ResultExcelProcessor; import com.bakdata.conquery.models.execution.ManagedExecution; -import com.bakdata.conquery.models.i18n.I18n; -import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; -import com.bakdata.conquery.models.query.resultinfo.printers.JavaResultPrinters; import com.bakdata.conquery.resources.api.ResultExcelResource; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.jersey.DropwizardResourceConfig; @@ -74,8 +71,6 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuil return Collections.emptyList(); } - final PrintSettings printSettings = new PrintSettings(true, I18n.LOCALE.get(), exec.getNamespace(), exec.getConfig(), null, null, new JavaResultPrinters()); - // Save id column count to later check if xlsx dimensions are feasible idColumnsCount = exec.getConfig().getIdColumns().getIdResultInfos().size(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java index 248dc83bc9..465581e2de 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java @@ -129,7 +129,7 @@ public boolean isEventDateSelect() { return false; } - public Printer createPrinter(PrintSettings printSettings, PrinterFactory printerFactory) { + public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printSettings) { return printerFactory.printerFor(getResultType(), printSettings); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java index c2d47992a7..5cf893887f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java @@ -46,7 +46,7 @@ public Aggregator createAggregator() { } @Override - public Printer createPrinter(PrintSettings printSettings, PrinterFactory printerFactory) { + public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printSettings) { if (isAsIds()) { return printerFactory.getListPrinter(new ConceptIdPrinter(getHolder().findConcept(), printSettings), printSettings); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java index fe83b1c689..5f1ad1b075 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java @@ -42,9 +42,9 @@ public SelectConverter createConverter() { } @Override - public Printer createPrinter(PrintSettings printSettings, PrinterFactory printerFactory) { + public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printSettings) { if(getMapping() == null){ - return super.createPrinter(printSettings, printerFactory); + return super.createPrinter(printerFactory, printSettings); } return printerFactory.getListPrinter(new MappedPrinter(getMapping()), printSettings); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java index 7412daaf3b..6768f07bdc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java @@ -56,9 +56,9 @@ public List print(List values) { } @Override - public Printer createPrinter(PrintSettings printSettings, PrinterFactory printerFactory) { + public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printSettings) { if (mapping == null) { - return super.createPrinter(printSettings, printerFactory); + return super.createPrinter(printerFactory, printSettings); } return new MappedPrinter(getMapping()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java index 7e27b95b6f..34ddafcc69 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java @@ -9,8 +9,6 @@ import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; -import jakarta.validation.constraints.NotNull; -import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.apiv1.execution.ExecutionStatus; import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; @@ -50,6 +48,8 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.annotation.OptBoolean; import com.google.common.base.Preconditions; +import jakarta.validation.constraints.NotNull; +import jakarta.ws.rs.core.UriBuilder; import lombok.AccessLevel; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -160,7 +160,7 @@ public final void initExecutable(Namespace namespace, ConqueryConfig config) { } if (label == null) { // IdMapper is not necessary here - label = makeAutoLabel(new PrintSettings(true, I18n.LOCALE.get(), namespace, config, null, null, null)); + label = makeAutoLabel(new PrintSettings(true, I18n.LOCALE.get(), namespace, config, null, null)); } this.config = config; this.namespace = namespace; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java b/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java index 5fc4f5f2f6..6379049aee 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java @@ -12,7 +12,6 @@ import com.bakdata.conquery.models.config.LocaleConfig; import com.bakdata.conquery.models.identifiable.mapping.PrintIdMapper; import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.worker.Namespace; import lombok.Getter; import lombok.ToString; @@ -59,13 +58,11 @@ public class PrintSettings { private final PrintIdMapper idMapper; - private final PrinterFactory printerFactory; - - public PrintSettings(boolean prettyPrint, Locale locale, Namespace namespace, ConqueryConfig config, PrintIdMapper idMapper, Function columnNamer, PrinterFactory printerFactory) { - this(prettyPrint, locale, namespace, config, idMapper, columnNamer, DECIMAL_FORMAT.apply(locale), NUMBER_FORMAT.apply(locale), printerFactory); + public PrintSettings(boolean prettyPrint, Locale locale, Namespace namespace, ConqueryConfig config, PrintIdMapper idMapper, Function columnNamer) { + this(prettyPrint, locale, namespace, config, idMapper, columnNamer, DECIMAL_FORMAT.apply(locale), NUMBER_FORMAT.apply(locale)); } - public PrintSettings(boolean prettyPrint, Locale locale, Namespace namespace, ConqueryConfig config, PrintIdMapper idMapper, Function columnNamer, NumberFormat decimalFormat, NumberFormat numberFormat, PrinterFactory printerFactory) { + public PrintSettings(boolean prettyPrint, Locale locale, Namespace namespace, ConqueryConfig config, PrintIdMapper idMapper, Function columnNamer, NumberFormat decimalFormat, NumberFormat numberFormat) { this.prettyPrint = prettyPrint; this.locale = locale; this.namespace = namespace; @@ -75,7 +72,6 @@ public PrintSettings(boolean prettyPrint, Locale locale, Namespace namespace, Co this.integerFormat = numberFormat; this.decimalFormat = decimalFormat; - this.printerFactory = printerFactory; this.listFormat = prettyPrint ? config.getLocale().getListFormats().get(0) : UNPRETTY_LIST_FORMAT; this.dateRangeSeparator = prettyPrint ? config.getLocale().findDateRangeSeparator(locale) : UNPRETTY_DATERANGE_SEPERATOR; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/SingleTableResult.java b/backend/src/main/java/com/bakdata/conquery/models/query/SingleTableResult.java index e49ae92f0b..06ca375724 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/SingleTableResult.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/SingleTableResult.java @@ -24,7 +24,7 @@ default List generateColumnDescriptions(boolean isInitialized, final Locale locale = I18n.LOCALE.get(); // The printer is never used to generate results. But downstream code might touch them - final PrintSettings settings = new PrintSettings(true, locale, getNamespace(), config, null, null,null); + final PrintSettings settings = new PrintSettings(true, locale, getNamespace(), config, null, null); final UniqueNamer uniqNamer = new UniqueNamer(settings); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java index cbd7357fc8..1c6bf88494 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java @@ -36,6 +36,7 @@ import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; import com.bakdata.conquery.models.query.resultinfo.printers.JsonResultPrinters; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.MultilineEntityResult; import com.bakdata.conquery.models.types.SemanticType; @@ -63,125 +64,6 @@ public EntityPreviewExecution(EntityPreviewForm entityPreviewQuery, User user, D super(entityPreviewQuery, user, submittedDataset, storage); } - /** - * Query contains both YEARS and QUARTERS lines: Group them. - */ - private static Map> getQuarterLines(EntityResult entityResult) { - final Map> quarterLines = new HashMap<>(); - - for (Object[] line : entityResult.listResultLines()) { - if (Resolution.valueOf((String) line[AbsoluteFormQuery.RESOLUTION_INDEX]) != Resolution.QUARTERS) { - continue; - } - - // Since we know the dates are always aligned we need to only respect their starts. - final LocalDate date = CDate.toLocalDate(((List) line[AbsoluteFormQuery.TIME_INDEX]).get(0)); - - final int year = date.getYear(); - final int quarter = QuarterUtils.getQuarter(date); - - quarterLines.computeIfAbsent(year, (ignored) -> new HashMap<>(4)).put(quarter, line); - } - - return quarterLines; - } - - /** - * Query contains both YEARS and QUARTERS lines: Group them. - */ - private static Map getYearLines(EntityResult entityResult) { - - final Map yearLines = new HashMap<>(); - - for (Object[] line : entityResult.listResultLines()) { - - if (Resolution.valueOf((String) line[AbsoluteFormQuery.RESOLUTION_INDEX]) != Resolution.YEARS) { - continue; - } - - // Since we know the dates are always aligned we need to only respect their starts. - final LocalDate date = CDate.toLocalDate(((List) line[AbsoluteFormQuery.TIME_INDEX]).get(0)); - - final int year = date.getYear(); - - yearLines.put(year, line); - } - - return yearLines; - } - - /** - * Creates a transformer printing lines, transformed into a Map of label->value. - * Null values are omitted. - */ - private static Function> createLineToMapTransformer(List resultInfos, Map select2desc, PrintSettings printSettings) { - - - final int size = resultInfos.size(); - final String[] columnNames = new String[size]; - final Printer[] printers = new Printer[size]; - - for (int index = 0; index < size; index++) { - final ResultInfo resultInfo = resultInfos.get(index); - - if (resultInfo instanceof SelectResultInfo selectResultInfo) { - columnNames[index] = select2desc.get(selectResultInfo.getSelect().getId()).label(); - } - - printers[index] = resultInfo.createPrinter(printSettings); - } - - return line -> { - final Map out = new HashMap<>(size); - - for (int column = 0; column < size; column++) { - final String columnName = columnNames[column]; - - if (columnName == null) { - continue; - } - - if (line[column] == null) { - continue; - } - - final Object value = printers[column].apply(line[column]); - - out.put(columnName, value); - } - - return out; - }; - } - - - - /** - * For the selects in result infos, build ColumnDescriptors using definitions (label and description) from PreviewConfig. - */ - private static List createChronoColumnDescriptors(SingleTableResult query, Map select2desc) { - - final List columnDescriptions = new ArrayList<>(); - - for (ResultInfo info : query.getResultInfos()) { - if (info instanceof SelectResultInfo selectResultInfo) { - final PreviewConfig.InfoCardSelect desc = select2desc.get(selectResultInfo.getSelect().getId()); - - // We build these by hand because they are labeled and described by config. - columnDescriptions.add(ColumnDescriptor.builder() - .label(desc.label()) - .defaultLabel(desc.label()) - .type(info.getType().typeInfo()) - .semantics(info.getSemantics()) - .description((desc.description() != null) ? desc.description() : selectResultInfo.getDescription()) // both might be null - .build()); - } - } - - - return columnDescriptions; - } - @Override public boolean isSystem() { // This Form should NEVER be started manually. Nor persisted @@ -209,24 +91,35 @@ public FullExecutionStatus buildStatusFull(Subject subject, Namespace namespace) status.setQuery(getValuesQuery().getQuery()); - final PrintSettings infoSettings = new PrintSettings(true, I18n.LOCALE.get(), getNamespace(), getConfig(), null, previewConfig::resolveSelectLabel, new JsonResultPrinters()); - status.setInfos(transformQueryResultToInfos(getInfoCardExecution(), infoSettings)); + JsonResultPrinters printers = new JsonResultPrinters(); + final PrintSettings infoSettings = + new PrintSettings(true, I18n.LOCALE.get(), getNamespace(), getConfig(), null, previewConfig::resolveSelectLabel); + status.setInfos(transformQueryResultToInfos(getInfoCardExecution(), infoSettings, printers)); - final PrintSettings stratifiedSettings = new PrintSettings(false, I18n.LOCALE.get(), getNamespace(), getConfig(), null, previewConfig::resolveSelectLabel, new JsonResultPrinters()); - status.setTimeStratifiedInfos(toChronoInfos(previewConfig, getSubQueries(), stratifiedSettings)); + final PrintSettings stratifiedSettings = + new PrintSettings(false, I18n.LOCALE.get(), getNamespace(), getConfig(), null, previewConfig::resolveSelectLabel); + status.setTimeStratifiedInfos(toChronoInfos(previewConfig, getSubQueries(), stratifiedSettings, printers)); return status; } + @JsonIgnore + private ManagedQuery getValuesQuery() { + return getSubQueries().get(EntityPreviewForm.VALUES_QUERY_NAME); + } + /** * Takes a ManagedQuery, and transforms its result into a List of {@link EntityPreviewStatus.Info}. * The format of the query is an {@link AbsoluteFormQuery} containing a single line for one person. This should correspond to {@link EntityPreviewForm#VALUES_QUERY_NAME}. */ - private List transformQueryResultToInfos(ManagedQuery infoCardExecution, PrintSettings printSettings) { + private List transformQueryResultToInfos( + ManagedQuery infoCardExecution, PrintSettings printSettings, + PrinterFactory printerFactory) { // Submitted Query is a single line of an AbsoluteFormQuery => MultilineEntityResult with a single line. - final MultilineEntityResult result = (MultilineEntityResult) infoCardExecution.streamResults(OptionalLong.empty()).collect(MoreCollectors.onlyElement()); + final MultilineEntityResult result = + (MultilineEntityResult) infoCardExecution.streamResults(OptionalLong.empty()).collect(MoreCollectors.onlyElement()); final Object[] values = result.getValues().get(0); final List extraInfos = new ArrayList<>(values.length); @@ -235,7 +128,7 @@ private List transformQueryResultToInfos(ManagedQuery for (int index = AbsoluteFormQuery.FEATURES_OFFSET; index < infoCardExecution.getResultInfos().size(); index++) { final ResultInfo resultInfo = infoCardExecution.getResultInfos().get(index); - final Object printed = resultInfo.createPrinter(printSettings).apply(values[index]); + final Object printed = resultInfo.createPrinter(printerFactory, printSettings).apply(values[index]); extraInfos.add(new EntityPreviewStatus.Info( resultInfo.userColumnName(printSettings), @@ -255,7 +148,10 @@ private ManagedQuery getInfoCardExecution() { } @NotNull - private List toChronoInfos(PreviewConfig previewConfig, Map subQueries, PrintSettings printSettings) { + private List toChronoInfos( + PreviewConfig previewConfig, + Map subQueries, + PrintSettings printSettings, PrinterFactory printers) { final List timeStratifiedInfos = new ArrayList<>(); for (PreviewConfig.TimeStratifiedSelects description : previewConfig.getTimeStratifiedSelects()) { @@ -268,7 +164,8 @@ private List toChronoInfos(PreviewConfi .collect(Collectors.toMap(PreviewConfig.InfoCardSelect::select, Function.identity())); // Group lines by year and quarter. - final Function> lineTransformer = createLineToMapTransformer(query.getResultInfos(), select2desc, printSettings); + final Function> lineTransformer = + createLineToMapTransformer(query.getResultInfos(), select2desc, printSettings, printers); final List yearEntries = createYearEntries(entityResult, lineTransformer); final Object[] completeResult = getCompleteLine(entityResult); @@ -279,7 +176,12 @@ private List toChronoInfos(PreviewConfi final EntityPreviewStatus.TimeStratifiedInfos infos = - new EntityPreviewStatus.TimeStratifiedInfos(description.label(), description.description(), columnDescriptors, lineTransformer.apply(completeResult), yearEntries); + new EntityPreviewStatus.TimeStratifiedInfos(description.label(), + description.description(), + columnDescriptors, + lineTransformer.apply(completeResult), + yearEntries + ); timeStratifiedInfos.add(infos); } @@ -287,6 +189,52 @@ private List toChronoInfos(PreviewConfi return timeStratifiedInfos; } + /** + * Creates a transformer printing lines, transformed into a Map of label->value. + * Null values are omitted. + */ + private static Function> createLineToMapTransformer( + List resultInfos, Map select2desc, PrintSettings printSettings, + PrinterFactory printerFactory) { + + + final int size = resultInfos.size(); + final String[] columnNames = new String[size]; + final Printer[] printers = new Printer[size]; + + for (int index = 0; index < size; index++) { + final ResultInfo resultInfo = resultInfos.get(index); + + if (resultInfo instanceof SelectResultInfo selectResultInfo) { + columnNames[index] = select2desc.get(selectResultInfo.getSelect().getId()).label(); + } + + printers[index] = resultInfo.createPrinter(printerFactory, printSettings); + } + + return line -> { + final Map out = new HashMap<>(size); + + for (int column = 0; column < size; column++) { + final String columnName = columnNames[column]; + + if (columnName == null) { + continue; + } + + if (line[column] == null) { + continue; + } + + final Object value = printers[column].apply(line[column]); + + out.put(columnName, value); + } + + return out; + }; + } + @NotNull private List createYearEntries(EntityResult entityResult, Function> lineTransformer) { final Map yearLines = getYearLines(entityResult); @@ -321,6 +269,81 @@ private Object[] getCompleteLine(EntityResult entityResult) { throw new IllegalStateException("Result has no row for COMPLETE"); } + /** + * For the selects in result infos, build ColumnDescriptors using definitions (label and description) from PreviewConfig. + */ + private static List createChronoColumnDescriptors(SingleTableResult query, Map select2desc) { + + final List columnDescriptions = new ArrayList<>(); + + for (ResultInfo info : query.getResultInfos()) { + if (info instanceof SelectResultInfo selectResultInfo) { + final PreviewConfig.InfoCardSelect desc = select2desc.get(selectResultInfo.getSelect().getId()); + + // We build these by hand because they are labeled and described by config. + columnDescriptions.add(ColumnDescriptor.builder() + .label(desc.label()) + .defaultLabel(desc.label()) + .type(info.getType().typeInfo()) + .semantics(info.getSemantics()) + .description((desc.description() != null) + ? desc.description() + : selectResultInfo.getDescription()) // both might be null + .build()); + } + } + + + return columnDescriptions; + } + + /** + * Query contains both YEARS and QUARTERS lines: Group them. + */ + private static Map getYearLines(EntityResult entityResult) { + + final Map yearLines = new HashMap<>(); + + for (Object[] line : entityResult.listResultLines()) { + + if (Resolution.valueOf((String) line[AbsoluteFormQuery.RESOLUTION_INDEX]) != Resolution.YEARS) { + continue; + } + + // Since we know the dates are always aligned we need to only respect their starts. + final LocalDate date = CDate.toLocalDate(((List) line[AbsoluteFormQuery.TIME_INDEX]).get(0)); + + final int year = date.getYear(); + + yearLines.put(year, line); + } + + return yearLines; + } + + /** + * Query contains both YEARS and QUARTERS lines: Group them. + */ + private static Map> getQuarterLines(EntityResult entityResult) { + final Map> quarterLines = new HashMap<>(); + + for (Object[] line : entityResult.listResultLines()) { + if (Resolution.valueOf((String) line[AbsoluteFormQuery.RESOLUTION_INDEX]) != Resolution.QUARTERS) { + continue; + } + + // Since we know the dates are always aligned we need to only respect their starts. + final LocalDate date = CDate.toLocalDate(((List) line[AbsoluteFormQuery.TIME_INDEX]).get(0)); + + final int year = date.getYear(); + final int quarter = QuarterUtils.getQuarter(date); + + quarterLines.computeIfAbsent(year, (ignored) -> new HashMap<>(4)).put(quarter, line); + } + + return quarterLines; + } + protected void setAdditionalFieldsForStatusWithColumnDescription(Subject subject, FullExecutionStatus status) { status.setColumnDescriptions(generateColumnDescriptions(isInitialized(), getConfig())); } @@ -350,11 +373,6 @@ public List generateColumnDescriptions(boolean isInitialized, return descriptors; } - @JsonIgnore - private ManagedQuery getValuesQuery() { - return getSubQueries().get(EntityPreviewForm.VALUES_QUERY_NAME); - } - @Override protected void setAdditionalFieldsForStatusWithSource(Subject subject, FullExecutionStatus status, Namespace namespace) { status.setColumnDescriptions(generateColumnDescriptions(isInitialized(), getConfig())); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java index 8f80914cff..96faaa22f3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java @@ -6,6 +6,7 @@ import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.query.resultinfo.printers.common.ConceptIdPrinter; import com.bakdata.conquery.models.types.ResultType; import lombok.EqualsAndHashCode; @@ -42,11 +43,11 @@ public String defaultColumnName(PrintSettings printSettings) { } @Override - public Printer createPrinter(PrintSettings printSettings) { + public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printSettings) { if(concept != null){ return new ConceptIdPrinter(concept, printSettings); } - return printSettings.getPrinterFactory().printerFor(type, printSettings); + return printerFactory.printerFor(type, printSettings); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ExternalResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ExternalResultInfo.java index 99f494d9dc..f5900aae5a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ExternalResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ExternalResultInfo.java @@ -4,6 +4,7 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.types.ResultType; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -37,7 +38,7 @@ public String getDescription() { } @Override - public Printer createPrinter(PrintSettings printSettings) { - return printSettings.getPrinterFactory().printerFor(type, printSettings); + public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printSettings) { + return printerFactory.printerFor(type, printSettings); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/FixedLabelResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/FixedLabelResultInfo.java index 738f404abb..8e09dd8d79 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/FixedLabelResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/FixedLabelResultInfo.java @@ -5,6 +5,7 @@ import c10n.C10N; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import lombok.EqualsAndHashCode; @@ -45,8 +46,8 @@ public FixedLabelResultInfo(ResultType type, Set semantics) { } @Override - public Printer createPrinter(PrintSettings printSettings) { - return printSettings.getPrinterFactory().printerFor(getType(), printSettings); + public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printSettings) { + return printerFactory.printerFor(getType(), printSettings); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java index 4c0059d35e..10f581f767 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java @@ -8,6 +8,7 @@ import com.bakdata.conquery.models.query.ColumnDescriptor; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import com.google.common.collect.ImmutableSet; @@ -62,5 +63,5 @@ public Set getSemantics() { public abstract String getDescription(); - public abstract Printer createPrinter(PrintSettings printSettings); + public abstract Printer createPrinter(PrinterFactory printerFactory, PrintSettings printSettings); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SecondaryIdResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SecondaryIdResultInfo.java index 8c93e6cda6..a50a4efea7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SecondaryIdResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SecondaryIdResultInfo.java @@ -5,6 +5,7 @@ import com.bakdata.conquery.models.datasets.SecondaryIdDescription; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.query.resultinfo.printers.common.MappedPrinter; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; @@ -32,9 +33,9 @@ public String getDescription() { } @Override - public Printer createPrinter(PrintSettings printSettings) { + public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printSettings) { if (secondaryId.getMapping() == null) { - return printSettings.getPrinterFactory().getStringPrinter(printSettings); + return printerFactory.getStringPrinter(printSettings); } else { return new MappedPrinter(secondaryId.getMapping()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java index 9ef5021b7a..33db8e3e8c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java @@ -6,6 +6,7 @@ import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import com.google.common.collect.Sets; @@ -34,8 +35,8 @@ public String getDescription() { } @Override - public Printer createPrinter(PrintSettings printSettings) { - return select.createPrinter(printSettings, printSettings.getPrinterFactory()); + public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printSettings) { + return select.createPrinter(printerFactory, printSettings); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ResultStatistics.java b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ResultStatistics.java index 8f0d3f9040..01ab5a17ef 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ResultStatistics.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ResultStatistics.java @@ -19,6 +19,7 @@ import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; @@ -35,7 +36,8 @@ public record ResultStatistics(int entities, int total, List statistics, Range dateRange) { @SneakyThrows @NotNull - public static ResultStatistics collectResultStatistics(SingleTableResult managedQuery, List resultInfos, Optional dateInfo, Optional dateIndex, PrintSettings printSettings, UniqueNamer uniqueNamer, ConqueryConfig conqueryConfig) { + public static ResultStatistics collectResultStatistics(SingleTableResult managedQuery, List resultInfos, Optional dateInfo, Optional dateIndex, PrintSettings printSettings, UniqueNamer uniqueNamer, ConqueryConfig conqueryConfig, + PrinterFactory printerFactory) { //TODO pull inner executor service from ManagerNode @@ -73,7 +75,7 @@ public static ResultStatistics collectResultStatistics(SingleTableResult managed final StopWatch started = StopWatch.createStarted(); final ResultInfo info = resultInfos.get(col); - final Printer printer = info.createPrinter(printSettings); + final Printer printer = info.createPrinter(printerFactory, printSettings); final ColumnStatsCollector statsCollector = ColumnStatsCollector.getStatsCollector(uniqueNamer.getUniqueName(info, printSettings), info.getDescription(), info.getType(), printSettings, conqueryConfig.getFrontend()); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java index c0a0904ee5..be5564974b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java @@ -44,7 +44,7 @@ public NodeConversions( public ConversionContext convert(QueryDescription queryDescription, Namespace namespace, ConqueryConfig conqueryConfig) { ConversionContext initialCtx = ConversionContext.builder() .idColumns(idColumns) - .sqlPrintSettings(new PrintSettings(false, Locale.ROOT, namespace, conqueryConfig, null, null, null)) + .sqlPrintSettings(new PrintSettings(false, Locale.ROOT, namespace, conqueryConfig, null, null)) .config(config) .nameGenerator(nameGenerator) .nodeConversions(this) diff --git a/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java b/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java index 16eb417ec0..6225b7ef06 100644 --- a/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java +++ b/backend/src/test/java/com/bakdata/conquery/execution/DefaultSqlCDateSetParserTest.java @@ -19,7 +19,7 @@ class DefaultSqlCDateSetParserTest { private static final DefaultSqlCDateSetParser parser = new DefaultSqlCDateSetParser(); private static final StringResultPrinters csvResultPrinters = new StringResultPrinters(); private static final ConqueryConfig CONFIG = new ConqueryConfig(); - private static final PrintSettings PLAIN = new PrintSettings(false, Locale.ENGLISH, null, CONFIG, null, null, csvResultPrinters); + private static final PrintSettings PLAIN = new PrintSettings(false, Locale.ENGLISH, null, CONFIG, null, null); @ParameterizedTest @MethodSource("testToEpochDayRangeListProvider") diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java index 5f2a2b5d16..a5bf248eab 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java @@ -20,10 +20,8 @@ import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.ManagedQuery; -import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.StringResultPrinters; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.MultilineEntityResult; import com.bakdata.conquery.resources.api.ResultCsvResource; @@ -57,9 +55,6 @@ public void executeTest(StandaloneSupport standaloneSupport) throws IOException final ManagedExecution execution = standaloneSupport.getMetaStorage().getExecution(executionId); SingleTableResult executionResult = (SingleTableResult) execution; - //check result info size - PrintSettings printSettings = new PrintSettings(true, Locale.ROOT, standaloneSupport.getNamespace(), standaloneSupport.getConfig(), null, null, new StringResultPrinters()); - List resultInfos = executionResult.getResultInfos(); assertThat(executionResult.streamResults(OptionalLong.empty()).flatMap(EntityResult::streamValues)) diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java index 0a1342e6d6..34dcdae3a7 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java @@ -31,7 +31,6 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.StringResultPrinters; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.util.io.IdColumnUtil; import com.bakdata.conquery.util.support.StandaloneSupport; @@ -77,6 +76,25 @@ public class FormTest extends ConqueryTestSpec { @JsonIgnore private Form form; + private static void importConcepts(StandaloneSupport support, ArrayNode rawConcepts) throws JSONException, IOException { + if (rawConcepts == null) { + return; + } + + Dataset dataset = support.getDataset(); + + List> concepts = parseSubTreeList( + support, + rawConcepts, + Concept.class, + c -> c.setDataset(support.getDataset()) + ); + + for (Concept concept : concepts) { + LoadingUtil.uploadConcept(support, dataset, concept); + } + } + @ValidationMethod(message = "Form test defines no concepts. Neither explicit nor automatic concepts") public boolean isWithConcepts() { return rawConcepts != null || content.isAutoConcept(); @@ -89,6 +107,10 @@ public void importRequiredData(StandaloneSupport support) throws Exception { form = parseForm(support); } + private Form parseForm(StandaloneSupport support) throws JSONException, IOException { + return parseSubTree(support, rawForm, Form.class); + } + @Override public void executeTest(StandaloneSupport support) throws Exception { Namespace namespace = support.getNamespace(); @@ -131,103 +153,79 @@ private void checkResults(StandaloneSupport standaloneSupport, ManagedInternalFo final ConqueryConfig config = standaloneSupport.getConfig(); PrintSettings printSettings = - new PrintSettings(false, Locale.ENGLISH, standaloneSupport.getNamespace(), config, idPrinter::createId, null, new StringResultPrinters()); + new PrintSettings(false, Locale.ENGLISH, standaloneSupport.getNamespace(), config, idPrinter::createId, null); checkSingleResult(managedForm, config, printSettings); } /** - * Checks result of subqueries instead of form result. + * The form produces only one result, so the result is directly requested. * - * @see FormTest#checkSingleResult(ManagedForm, ConqueryConfig, PrintSettings) + * @see FormTest#checkMultipleResult(Map, ConqueryConfig, PrintSettings) */ - private void checkMultipleResult(Map> managedMapping, ConqueryConfig config, PrintSettings printSettings) throws IOException { - for (Map.Entry> managed : managedMapping.entrySet()) { - List resultInfos = managed.getValue().get(0).getResultInfos(); - log.info("{} CSV TESTING: {}", getLabel(), managed.getKey()); + private void checkSingleResult(F managedForm, ConqueryConfig config, PrintSettings printSettings) + throws IOException { - ByteArrayOutputStream output = new ByteArrayOutputStream(); + try (ByteArrayOutputStream output = new ByteArrayOutputStream()) { final CsvWriter writer = config.getCsv().createWriter(output); - - CsvRenderer renderer = new CsvRenderer(writer, printSettings); + final CsvRenderer renderer = new CsvRenderer(writer, printSettings); renderer.toCSV( config.getIdColumns().getIdResultInfos(), - resultInfos, - managed.getValue() - .stream() - .flatMap(managedQuery -> managedQuery.streamResults(OptionalLong.empty())), printSettings + managedForm.getResultInfos(), + managedForm.streamResults(OptionalLong.empty()), printSettings ); writer.close(); - output.close(); assertThat(In.stream(new ByteArrayInputStream(output.toByteArray())).withUTF8().readLines()) - .as("Checking result " + managed.getKey()) + .as("Checking result " + managedForm.getLabelWithoutAutoLabelSuffix()) .containsExactlyInAnyOrderElementsOf( - In.stream(expectedCsv.get(managed.getKey()).stream()) + In.stream(expectedCsv.values().iterator().next().stream()) .withUTF8() .readLines() ); } + + } /** - * The form produces only one result, so the result is directly requested. + * Checks result of subqueries instead of form result. * - * @see FormTest#checkMultipleResult(Map, ConqueryConfig, PrintSettings) + * @see FormTest#checkSingleResult(ManagedForm, ConqueryConfig, PrintSettings) */ - private void checkSingleResult(F managedForm, ConqueryConfig config, PrintSettings printSettings) - throws IOException { + private void checkMultipleResult(Map> managedMapping, ConqueryConfig config, PrintSettings printSettings) throws IOException { + for (Map.Entry> managed : managedMapping.entrySet()) { + List resultInfos = managed.getValue().get(0).getResultInfos(); + log.info("{} CSV TESTING: {}", getLabel(), managed.getKey()); + ByteArrayOutputStream output = new ByteArrayOutputStream(); - try (ByteArrayOutputStream output = new ByteArrayOutputStream()) { final CsvWriter writer = config.getCsv().createWriter(output); - final CsvRenderer renderer = new CsvRenderer(writer, printSettings); + + CsvRenderer renderer = new CsvRenderer(writer, printSettings); renderer.toCSV( config.getIdColumns().getIdResultInfos(), - managedForm.getResultInfos(), - managedForm.streamResults(OptionalLong.empty()), printSettings + resultInfos, + managed.getValue() + .stream() + .flatMap(managedQuery -> managedQuery.streamResults(OptionalLong.empty())), printSettings ); writer.close(); + output.close(); assertThat(In.stream(new ByteArrayInputStream(output.toByteArray())).withUTF8().readLines()) - .as("Checking result " + managedForm.getLabelWithoutAutoLabelSuffix()) + .as("Checking result " + managed.getKey()) .containsExactlyInAnyOrderElementsOf( - In.stream(expectedCsv.values().iterator().next().stream()) + In.stream(expectedCsv.get(managed.getKey()).stream()) .withUTF8() .readLines() ); } - - - } - - private static void importConcepts(StandaloneSupport support, ArrayNode rawConcepts) throws JSONException, IOException { - if (rawConcepts == null) { - return; - } - - Dataset dataset = support.getDataset(); - - List> concepts = parseSubTreeList( - support, - rawConcepts, - Concept.class, - c -> c.setDataset(support.getDataset()) - ); - - for (Concept concept : concepts) { - LoadingUtil.uploadConcept(support, dataset, concept); - } - } - - - private Form parseForm(StandaloneSupport support) throws JSONException, IOException { - return parseSubTree(support, rawForm, Form.class); } } diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java index 11239bc80e..465f7c586e 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java @@ -54,110 +54,13 @@ @Slf4j public class ArrowResultGenerationTest { - private static final int BATCH_SIZE = 2; - public static final ConqueryConfig CONFIG = new ConqueryConfig(); + public static final ConqueryConfig CONFIG = new ConqueryConfig(); + private static final int BATCH_SIZE = 2; private static final PrintSettings PRINT_SETTINGS = - new PrintSettings(false, Locale.ROOT, null, CONFIG, null, (selectInfo) -> selectInfo.getSelect().getLabel(), new ArrowResultPrinters()); + new PrintSettings(false, Locale.ROOT, null, CONFIG, null, (selectInfo) -> selectInfo.getSelect().getLabel()); - - @Test - void generateFieldsIdMapping() { - final UniqueNamer uniqueNamer = new UniqueNamer(PRINT_SETTINGS); - - List fields = generateFields(getIdFields(), uniqueNamer, PRINT_SETTINGS); - - assertThat(fields).containsExactlyElementsOf( - List.of( - new Field("id1", FieldType.nullable(new ArrowType.Utf8()), null), - new Field("id2", FieldType.nullable(new ArrowType.Utf8()), null))); - - } - - @Test - void generateFieldsValue() { - final UniqueNamer uniqueNamer = new UniqueNamer(PRINT_SETTINGS); - - - - List resultInfos = getResultTypes().stream().map(TypedSelectDummy::new) - .map(select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet())).collect(Collectors.toList()); - - List fields = generateFields( - resultInfos, - // Custom column namer so we don't require a dataset registry - uniqueNamer, PRINT_SETTINGS - ); - - assertThat(fields).containsExactlyElementsOf( - List.of( - new Field("BOOLEAN", FieldType.nullable(ArrowType.Bool.INSTANCE), null), - new Field("INTEGER", FieldType.nullable(new ArrowType.Int(32, true)), null), - new Field("NUMERIC", FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), null), - new Field("DATE", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null), - new Field("DATE_RANGE", - FieldType.nullable(ArrowType.Struct.INSTANCE), - List.of( - new Field("min", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null), - new Field("max", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null) - )), - new Field("STRING", FieldType.nullable(new ArrowType.Utf8()), null), - new Field("MONEY", FieldType.nullable(new ArrowType.Decimal(38 - PRINT_SETTINGS.getCurrency().getDefaultFractionDigits(), PRINT_SETTINGS.getCurrency().getDefaultFractionDigits(), 128)), null), - new Field("LIST[BOOLEAN]", FieldType.nullable(ArrowType.List.INSTANCE), List.of(new Field("LIST[BOOLEAN]", FieldType.nullable(ArrowType.Bool.INSTANCE), null))), - new Field("LIST[DATE_RANGE]", FieldType.nullable(ArrowType.List.INSTANCE), List.of(new Field("LIST[DATE_RANGE]", - FieldType.nullable(ArrowType.Struct.INSTANCE), - List.of( - new Field("min", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null), - new Field("max", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null) - )))), - new Field("LIST[STRING]", FieldType.nullable(ArrowType.List.INSTANCE), List.of(new Field("LIST[STRING]", FieldType.nullable(new ArrowType.Utf8()), null))) - ) - ); - - } - - @Test - void writeAndRead() throws IOException { - - // Initialize internationalization - I18n.init(); - - // Prepare every input data - PrintSettings printSettings = new PrintSettings( - false, - Locale.ROOT, - null, - CONFIG, - (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), - (selectInfo) -> selectInfo.getSelect().getLabel(), new ArrowResultPrinters() - ); - // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized - List results = getTestEntityResults(); - - ManagedQuery mquery = getTestQuery(); - - // First we write to the buffer, than we read from it and parse it as TSV - ByteArrayOutputStream output = new ByteArrayOutputStream(); - - renderToStream( - (root) -> new ArrowStreamWriter(root, new DictionaryProvider.MapDictionaryProvider(), output), - printSettings, - new ArrowConfig(BATCH_SIZE), - getIdFields(), - mquery.getResultInfos(), - mquery.streamResults(OptionalLong.empty()) - ); - - InputStream inputStream = new ByteArrayInputStream(output.toByteArray()); - - String computed = readTSV(inputStream); - - assertThat(computed).isNotBlank(); - assertThat(computed).isEqualTo(generateExpectedTSV(results, mquery.getResultInfos(), false)); - - } - - public static String readTSV(InputStream inputStream) throws IOException { + public static String readTSV(InputStream inputStream) throws IOException { StringJoiner stringJoiner = new StringJoiner("\n"); try (ArrowStreamReader arrowReader = new ArrowStreamReader(inputStream, ROOT_ALLOCATOR)) { log.info("Reading the produced arrow data."); @@ -174,11 +77,11 @@ public static String readTSV(InputStream inputStream) throws IOException { .map(vec -> vec.getObject(currentRow)) .map(ArrowResultGenerationTest::getPrintValue) .collect(Collectors.joining("\t"))); - } - } - } + } + } + } return stringJoiner.toString(); - } + } public static String generateExpectedTSV(List results, List resultInfos, boolean isParquet) { String expected = results.stream() @@ -191,15 +94,15 @@ public static String generateExpectedTSV(List results, List results, List dr = (List) obj; - StringBuilder sb = new StringBuilder(); - sb.append("{"); - final int min = (int) dr.get(0); - final int max = (int) dr.get(1); - // Handle cases where one of the limits is infinity - if (!CDate.isNegativeInfinity(min)) { - sb.append("\"min\":").append(min); - } - if (!CDate.isNegativeInfinity(min) && !CDate.isPositiveInfinity(max)) { - sb.append(","); - } - if (!CDate.isPositiveInfinity(max)) { - sb.append("\"max\":").append(max); - } - sb.append("}"); - return sb.toString(); - } - if(obj instanceof Collection) { - Collection col = (Collection) obj; - // Workaround: Arrow deserializes lists as a JsonStringArrayList which has a JSON String method - @NonNull ResultType elemType = ((ResultType.ListT) type).getElementType(); + if (type.equals(ResultType.Primitive.DATE_RANGE)) { + // Special case for daterange in this test because it uses a StructVector, we rebuild the structural information + List dr = (List) obj; + StringBuilder sb = new StringBuilder(); + sb.append("{"); + final int min = (int) dr.get(0); + final int max = (int) dr.get(1); + // Handle cases where one of the limits is infinity + if (!CDate.isNegativeInfinity(min)) { + sb.append("\"min\":").append(min); + } + if (!CDate.isNegativeInfinity(min) && !CDate.isPositiveInfinity(max)) { + sb.append(","); + } + if (!CDate.isPositiveInfinity(max)) { + sb.append("\"max\":").append(max); + } + sb.append("}"); + return sb.toString(); + } + if (obj instanceof Collection) { + Collection col = (Collection) obj; + // Workaround: Arrow deserializes lists as a JsonStringArrayList which has a JSON String method + @NonNull ResultType elemType = ((ResultType.ListT) type).getElementType(); return col.stream().map(v -> getPrintValue(v, elemType, isParquet)).collect(Collectors.joining(",", "[", "]")); - } - return obj.toString(); - } + } + return obj.toString(); + } - private static String getPrintValue(Object obj) { - if(obj instanceof JsonStringArrayList) { + private static String getPrintValue(Object obj) { + if (obj instanceof JsonStringArrayList) { // Workaround: Arrow deserializes lists as a JsonStringArrayList which has a JSON String method return new ArrayList<>((JsonStringArrayList) obj).stream() .map(ArrowResultGenerationTest::getPrintValue) .collect(Collectors.joining(",", "[", "]")); } - return Objects.toString(obj); - } + return Objects.toString(obj); + } + + @Test + void generateFieldsIdMapping() { + final UniqueNamer uniqueNamer = new UniqueNamer(PRINT_SETTINGS); + + List fields = generateFields(getIdFields(), uniqueNamer, PRINT_SETTINGS); + + assertThat(fields).containsExactlyElementsOf( + List.of( + new Field("id1", FieldType.nullable(new ArrowType.Utf8()), null), + new Field("id2", FieldType.nullable(new ArrowType.Utf8()), null) + )); + + } + + @Test + void generateFieldsValue() { + final UniqueNamer uniqueNamer = new UniqueNamer(PRINT_SETTINGS); + + + List resultInfos = getResultTypes().stream() + .map(TypedSelectDummy::new) + .map(select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet())) + .collect(Collectors.toList()); + + List fields = generateFields( + resultInfos, + // Custom column namer so we don't require a dataset registry + uniqueNamer, PRINT_SETTINGS + ); + + assertThat(fields).containsExactlyElementsOf( + List.of( + new Field("BOOLEAN", FieldType.nullable(ArrowType.Bool.INSTANCE), null), + new Field("INTEGER", FieldType.nullable(new ArrowType.Int(32, true)), null), + new Field("NUMERIC", FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), null), + new Field("DATE", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null), + new Field("DATE_RANGE", + FieldType.nullable(ArrowType.Struct.INSTANCE), + List.of( + new Field("min", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null), + new Field("max", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null) + ) + ), + new Field("STRING", FieldType.nullable(new ArrowType.Utf8()), null), + new Field("MONEY", + FieldType.nullable(new ArrowType.Decimal(38 - PRINT_SETTINGS.getCurrency().getDefaultFractionDigits(), + PRINT_SETTINGS.getCurrency().getDefaultFractionDigits(), + 128 + )), + null + ), + new Field("LIST[BOOLEAN]", + FieldType.nullable(ArrowType.List.INSTANCE), + List.of(new Field("LIST[BOOLEAN]", FieldType.nullable(ArrowType.Bool.INSTANCE), null)) + ), + new Field("LIST[DATE_RANGE]", FieldType.nullable(ArrowType.List.INSTANCE), List.of(new Field("LIST[DATE_RANGE]", + FieldType.nullable(ArrowType.Struct.INSTANCE), + List.of( + new Field("min", + FieldType.nullable(new ArrowType.Date( + DateUnit.DAY)), + null + ), + new Field("max", + FieldType.nullable(new ArrowType.Date( + DateUnit.DAY)), + null + ) + ) + ))), + new Field("LIST[STRING]", + FieldType.nullable(ArrowType.List.INSTANCE), + List.of(new Field("LIST[STRING]", FieldType.nullable(new ArrowType.Utf8()), null)) + ) + ) + ); + + } + + @Test + void writeAndRead() throws IOException { + + // Initialize internationalization + I18n.init(); + + // Prepare every input data + PrintSettings printSettings = new PrintSettings(false, + Locale.ROOT, + null, + CONFIG, + (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), + (selectInfo) -> selectInfo.getSelect().getLabel() + ); + // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized + List results = getTestEntityResults(); + + ManagedQuery mquery = getTestQuery(); + + // First we write to the buffer, than we read from it and parse it as TSV + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + renderToStream((root) -> new ArrowStreamWriter(root, new DictionaryProvider.MapDictionaryProvider(), output), + printSettings, + new ArrowConfig(BATCH_SIZE), + getIdFields(), + mquery.getResultInfos(), + mquery.streamResults(OptionalLong.empty()), + new ArrowResultPrinters() + ); + + InputStream inputStream = new ByteArrayInputStream(output.toByteArray()); + + String computed = readTSV(inputStream); + + assertThat(computed).isNotBlank(); + assertThat(computed).isEqualTo(generateExpectedTSV(results, mquery.getResultInfos(), false)); + + } } diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java index 8239036e6b..dd301d348d 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java @@ -18,6 +18,7 @@ import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.query.resultinfo.printers.StringResultPrinters; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.types.ResultType; @@ -40,13 +41,14 @@ public class CsvResultGenerationTest { @Test void writeAndRead() throws IOException { // Prepare every input data + StringResultPrinters printers = new StringResultPrinters(); final PrintSettings printSettings = new PrintSettings( true, Locale.GERMAN, null, CONFIG, (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), - (selectInfo) -> selectInfo.getSelect().getLabel(), new StringResultPrinters() + (selectInfo) -> selectInfo.getSelect().getLabel() ); // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized final List results = getTestEntityResults(); @@ -62,7 +64,7 @@ void writeAndRead() throws IOException { final String computed = writer.toString(); - final String expected = generateExpectedCSV(results, mquery.getResultInfos(), printSettings); + final String expected = generateExpectedCSV(results, mquery.getResultInfos(), printSettings, printers); log.info("Wrote and than read this csv data: {}", computed); @@ -71,7 +73,7 @@ void writeAndRead() throws IOException { } - private String generateExpectedCSV(List results, List resultInfos, PrintSettings printSettings) { + private String generateExpectedCSV(List results, List resultInfos, PrintSettings printSettings, PrinterFactory printerFactory) { final List expected = new ArrayList<>(); expected.add(getIdFields().stream().map(info -> info.defaultColumnName(printSettings)).collect(Collectors.joining(",")) + "," @@ -92,7 +94,7 @@ private String generateExpectedCSV(List results, List continue; } final ResultInfo info = resultInfos.get(lIdx); - final String printVal = (String) info.createPrinter(printSettings).apply(val); + final String printVal = (String) info.createPrinter(printerFactory, printSettings).apply(val); valueJoiner.add(printVal.contains(String.valueOf(CONFIG.getCsv().getDelimeter())) ? "\"" + printVal + "\"" : printVal); } diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java index 8287ea5b4d..d974b4b9e8 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java @@ -27,8 +27,8 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; -import com.bakdata.conquery.models.query.resultinfo.printers.ExcelResultPrinters; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.query.resultinfo.printers.StringResultPrinters; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.types.ResultType; @@ -63,8 +63,7 @@ void writeAndRead() throws IOException { null, CONFIG, (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), - (selectInfo) -> selectInfo.getSelect().getLabel(), - new ExcelResultPrinters() + (selectInfo) -> selectInfo.getSelect().getLabel() ); // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized final List results = getTestEntityResults(); @@ -101,10 +100,9 @@ public Stream streamResults(OptionalLong maybeLimit) { null, CONFIG, (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), - (selectInfo) -> selectInfo.getSelect().getLabel(), - new StringResultPrinters() + (selectInfo) -> selectInfo.getSelect().getLabel() ); - final List expected = generateExpectedTSV(results, mquery.getResultInfos(), tsvPrintSettings); + final List expected = generateExpectedTSV(results, mquery.getResultInfos(), tsvPrintSettings, new StringResultPrinters()); log.info("Wrote and than read this excel data: {}", computed); @@ -139,7 +137,8 @@ private List readComputed(InputStream inputStream, PrintSettings setting } - private List generateExpectedTSV(List results, List resultInfos, PrintSettings printSettings) { + private List generateExpectedTSV(List results, List resultInfos, PrintSettings printSettings, + PrinterFactory printerFactory) { final List expected = new ArrayList<>(); expected.add(String.join("\t", printIdFields) + "\t" + getResultTypes().stream().map(ResultType::typeInfo).collect(Collectors.joining("\t"))); results.stream().map(EntityResult.class::cast).forEach(res -> { @@ -154,7 +153,7 @@ private List generateExpectedTSV(List results, List generateExpectedTSV(List results, List selectInfo.getSelect().getLabel(), new ArrowResultPrinters()); + new PrintSettings(false, Locale.ROOT, null, CONFIG, null, (selectInfo) -> selectInfo.getSelect().getLabel()); @Test @@ -110,8 +109,7 @@ void writeAndRead() throws IOException { null, CONFIG, (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), - (selectInfo) -> selectInfo.getSelect().getLabel(), - new ArrowResultPrinters() + (selectInfo) -> selectInfo.getSelect().getLabel() ); // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized List results = getTestEntityResults(); diff --git a/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java b/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java index 6db32e95ee..71c2be7212 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java @@ -85,7 +85,7 @@ void autoLabelConceptQuery(Locale locale, String autoLabel) { @NotNull private PrintSettings getPrintSettings(Locale locale) { - return new PrintSettings(true, locale, NAMESPACE, CONFIG, null, null, null); + return new PrintSettings(true, locale, NAMESPACE, CONFIG, null, null); } private static CQConcept makeCQConcept(String label) { diff --git a/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java b/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java index 5c5ecc96b2..bc54baed36 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java @@ -46,7 +46,7 @@ @Slf4j public class DefaultColumnNameTest { private static final Namespace NAMESPACE = mock(LocalNamespace.class); - private static final PrintSettings SETTINGS = new PrintSettings(false, Locale.ENGLISH, NAMESPACE, new ConqueryConfig(), null, null, null); + private static final PrintSettings SETTINGS = new PrintSettings(false, Locale.ENGLISH, NAMESPACE, new ConqueryConfig(), null, null); private static final Validator VALIDATOR = Validators.newValidator(); private static final BiFunction CONCEPT_SELECT_SELECTOR = diff --git a/backend/src/test/java/com/bakdata/conquery/models/query/UniqueNameTest.java b/backend/src/test/java/com/bakdata/conquery/models/query/UniqueNameTest.java index 251476c745..c95a6d3a01 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/query/UniqueNameTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/query/UniqueNameTest.java @@ -7,14 +7,13 @@ import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.query.resultinfo.ExternalResultInfo; import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; -import com.bakdata.conquery.models.query.resultinfo.printers.StringResultPrinters; import com.bakdata.conquery.models.types.ResultType; import org.junit.jupiter.api.Test; public class UniqueNameTest { @Test void testNameCollision() { - PrintSettings settings = new PrintSettings(true, Locale.ROOT, null, new ConqueryConfig(), null, null, new StringResultPrinters()); + PrintSettings settings = new PrintSettings(true, Locale.ROOT, null, new ConqueryConfig(), null, null); final UniqueNamer uniqueNamer = new UniqueNamer(settings); final ExternalResultInfo info1 = new ExternalResultInfo("test", ResultType.Primitive.STRING); diff --git a/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java b/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java index 921726a48b..f66f7fff5b 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java @@ -19,6 +19,7 @@ import com.bakdata.conquery.models.query.resultinfo.ExternalResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; import com.bakdata.conquery.models.query.resultinfo.printers.StringResultPrinters; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -27,9 +28,11 @@ public class ResultTypeTest { public static final ConqueryConfig CONFIG = new ConqueryConfig(); - private static final PrintSettings PRETTY = new PrintSettings(true, Locale.ENGLISH, null, CONFIG, null, null, new StringResultPrinters()); - private static final PrintSettings PRETTY_DE = new PrintSettings(true, Locale.GERMANY, null, CONFIG, null, null, new StringResultPrinters()); - private static final PrintSettings PLAIN = new PrintSettings(false, Locale.ENGLISH, null, CONFIG, null, null, new StringResultPrinters()); + private static final PrintSettings PRETTY = new PrintSettings(true, Locale.ENGLISH, null, CONFIG, null, null); + private static final PrintSettings PRETTY_DE = new PrintSettings(true, Locale.GERMANY, null, CONFIG, null, null); + private static final PrintSettings PLAIN = new PrintSettings(false, Locale.ENGLISH, null, CONFIG, null, null); + + private static PrinterFactory PRINTERS = new StringResultPrinters(); static { // Initialization of the internationalization @@ -39,6 +42,7 @@ public class ResultTypeTest { CONFIG.getLocale().setDateFormatMapping(Map.of(Locale.GERMAN, "dd.MM.yyyy")); } + @SuppressWarnings("unused") public static List testData() { return List.of(Arguments.of(PRETTY, ResultType.Primitive.BOOLEAN, true, "Yes"), @@ -106,7 +110,8 @@ public static List testData() { public void testPrinting(PrintSettings printSettings, ResultType type, Object value, String expected) throws IOException { ResultInfo info = info(type); - final Printer printer = info.createPrinter(printSettings); + + final Printer printer = info.createPrinter(PRINTERS, printSettings); assertThat(printer.apply(value)).isEqualTo(expected); @@ -125,7 +130,7 @@ public static ResultInfo info(ResultType type) { public void testBinaryPrinting(PrintSettings printSettings, ResultType type, Object value, String expected) throws IOException { ResultInfo info = info(type); - final Printer printer = info.createPrinter(printSettings); + final Printer printer = info.createPrinter(PRINTERS, printSettings); assertThat(printer.apply(value)).isEqualTo(expected); final byte[] bytes = Jackson.BINARY_MAPPER.writeValueAsBytes(value); From 23a6b1412ed1af43225925e95b6494bc1528be8b Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 3 Sep 2024 14:46:05 +0200 Subject: [PATCH 42/54] removes deleted Select --- cypress/e2e/backend-admin-ui/test_2_dataset.cy.js | 2 +- cypress/support/test_data/all_types.concept.json | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/cypress/e2e/backend-admin-ui/test_2_dataset.cy.js b/cypress/e2e/backend-admin-ui/test_2_dataset.cy.js index a57f95ea41..746335e4f4 100644 --- a/cypress/e2e/backend-admin-ui/test_2_dataset.cy.js +++ b/cypress/e2e/backend-admin-ui/test_2_dataset.cy.js @@ -139,7 +139,7 @@ context("Admin UI Single Dataset", () => { it("Counts are right", () => { visitAdminUI(`datasets/${testDSID}/connectors/${testDSID}.concept1.column`); cy.get('[data-test-id="accordion-Filters"] > .card-header').contains("20 entries"); - cy.get('[data-test-id="accordion-Selects"] > .card-header').contains("17 entries"); + cy.get('[data-test-id="accordion-Selects"] > .card-header').contains("16 entries"); }); }); diff --git a/cypress/support/test_data/all_types.concept.json b/cypress/support/test_data/all_types.concept.json index 36e7d64ab8..c629b0c7d7 100644 --- a/cypress/support/test_data/all_types.concept.json +++ b/cypress/support/test_data/all_types.concept.json @@ -159,12 +159,6 @@ "type": "RANDOM", "column": "table.INTEGER" }, - { - "label": "COUNT_OCCURENCES", - "type": "COUNT_OCCURENCES", - "column": "table.STRING", - "selection": [] - }, { "label": "FIRST", "type": "FIRST", From 38e7833ccdb70982f67607ab46c1f0db506f4ec0 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 3 Sep 2024 16:23:55 +0200 Subject: [PATCH 43/54] Code Style issues --- .../bakdata/conquery/models/common/Range.java | 24 ++-- .../specific/MappableSingleColumnSelect.java | 18 +-- .../query/preview/EntityPreviewExecution.java | 53 +++----- .../printers/ExcelResultPrinters.java | 2 +- .../printers/JavaResultPrinters.java | 7 +- .../printers/JsonResultPrinters.java | 3 + .../query/resultinfo/printers/Printer.java | 5 + .../resultinfo/printers/PrinterFactory.java | 16 ++- .../printers/StringResultPrinters.java | 7 +- .../printers/common/MappedPrinter.java | 2 +- .../query/statistics/ResultStatistics.java | 53 +++++--- .../conquery/io/result/ResultTestUtil.java | 64 +++++----- .../arrow/ArrowResultGenerationTest.java | 119 ++++++++---------- .../result/csv/CsvResultGenerationTest.java | 27 ++-- .../result/excel/ExcelResultRenderTest.java | 35 +++--- .../parquet/ParquetResultGenerationTest.java | 13 +- 16 files changed, 227 insertions(+), 221 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/common/Range.java b/backend/src/main/java/com/bakdata/conquery/models/common/Range.java index 37ed1cc2a8..c085c4aec2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/common/Range.java +++ b/backend/src/main/java/com/bakdata/conquery/models/common/Range.java @@ -167,9 +167,9 @@ public IntegerRange(Integer min, Integer max) { } public static IntegerRange fromNumberRange(IRange orig) { - return new Range.IntegerRange(Optional.ofNullable(orig.getMin()).map(Number::intValue).orElse(null), Optional.ofNullable(orig.getMax()) - .map(Number::intValue) - .orElse(null)); + return new Range.IntegerRange(Optional.ofNullable(orig.getMin()).map(Number::intValue).orElse(null), + Optional.ofNullable(orig.getMax()).map(Number::intValue).orElse(null) + ); } @Override @@ -198,9 +198,9 @@ public LongRange(Long min, Long max) { } public static LongRange fromNumberRange(IRange orig) { - return new Range.LongRange(Optional.ofNullable(orig.getMin()).map(Number::longValue).orElse(null), Optional.ofNullable(orig.getMax()) - .map(Number::longValue) - .orElse(null)); + return new Range.LongRange(Optional.ofNullable(orig.getMin()).map(Number::longValue).orElse(null), + Optional.ofNullable(orig.getMax()).map(Number::longValue).orElse(null) + ); } @Override @@ -229,9 +229,9 @@ public FloatRange(Float min, Float max) { } public static FloatRange fromNumberRange(IRange orig) { - return new Range.FloatRange(Optional.ofNullable(orig.getMin()).map(Number::floatValue).orElse(null), Optional.ofNullable(orig.getMax()) - .map(Number::floatValue) - .orElse(null)); + return new Range.FloatRange(Optional.ofNullable(orig.getMin()).map(Number::floatValue).orElse(null), + Optional.ofNullable(orig.getMax()).map(Number::floatValue).orElse(null) + ); } @Override @@ -263,9 +263,9 @@ public DoubleRange(Double min, Double max) { } public static DoubleRange fromNumberRange(IRange orig) { - return new Range.DoubleRange(Optional.ofNullable(orig.getMin()).map(Number::doubleValue).orElse(null), Optional.ofNullable(orig.getMax()) - .map(Number::doubleValue) - .orElse(null)); + return new Range.DoubleRange(Optional.ofNullable(orig.getMin()).map(Number::doubleValue).orElse(null), + Optional.ofNullable(orig.getMax()).map(Number::doubleValue).orElse(null) + ); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java index 6768f07bdc..af5d6ebe2d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java @@ -1,8 +1,6 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector.specific; -import java.util.ArrayList; import java.util.Collections; -import java.util.List; import java.util.Set; import javax.annotation.Nullable; @@ -41,20 +39,6 @@ public MappableSingleColumnSelect(Column column, @Nullable InternToExternMapper this.mapping = mapping; } - public List print(List values) { - if(mapping == null){ - return values; - } - - final List out = new ArrayList<>(); - - for (String value : values) { - out.addAll(List.of(mapping.external(value))); - } - - return out; - } - @Override public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printSettings) { if (mapping == null) { @@ -76,7 +60,7 @@ public SelectResultInfo getResultInfo(CQConcept cqConcept) { @Override public ResultType getResultType() { - if(mapping == null){ + if (mapping == null) { return ResultType.resolveResultType(getColumn().getType()); } return ResultType.Primitive.STRING; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java index 1c6bf88494..091368c350 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java @@ -92,12 +92,10 @@ public FullExecutionStatus buildStatusFull(Subject subject, Namespace namespace) JsonResultPrinters printers = new JsonResultPrinters(); - final PrintSettings infoSettings = - new PrintSettings(true, I18n.LOCALE.get(), getNamespace(), getConfig(), null, previewConfig::resolveSelectLabel); + final PrintSettings infoSettings = new PrintSettings(true, I18n.LOCALE.get(), getNamespace(), getConfig(), null, previewConfig::resolveSelectLabel); status.setInfos(transformQueryResultToInfos(getInfoCardExecution(), infoSettings, printers)); - final PrintSettings stratifiedSettings = - new PrintSettings(false, I18n.LOCALE.get(), getNamespace(), getConfig(), null, previewConfig::resolveSelectLabel); + final PrintSettings stratifiedSettings = new PrintSettings(false, I18n.LOCALE.get(), getNamespace(), getConfig(), null, previewConfig::resolveSelectLabel); status.setTimeStratifiedInfos(toChronoInfos(previewConfig, getSubQueries(), stratifiedSettings, printers)); return status; @@ -113,13 +111,11 @@ private ManagedQuery getValuesQuery() { * The format of the query is an {@link AbsoluteFormQuery} containing a single line for one person. This should correspond to {@link EntityPreviewForm#VALUES_QUERY_NAME}. */ private List transformQueryResultToInfos( - ManagedQuery infoCardExecution, PrintSettings printSettings, - PrinterFactory printerFactory) { + ManagedQuery infoCardExecution, PrintSettings printSettings, PrinterFactory printerFactory) { // Submitted Query is a single line of an AbsoluteFormQuery => MultilineEntityResult with a single line. - final MultilineEntityResult result = - (MultilineEntityResult) infoCardExecution.streamResults(OptionalLong.empty()).collect(MoreCollectors.onlyElement()); + final MultilineEntityResult result = (MultilineEntityResult) infoCardExecution.streamResults(OptionalLong.empty()).collect(MoreCollectors.onlyElement()); final Object[] values = result.getValues().get(0); final List extraInfos = new ArrayList<>(values.length); @@ -130,12 +126,11 @@ private List transformQueryResultToInfos( final Object printed = resultInfo.createPrinter(printerFactory, printSettings).apply(values[index]); - extraInfos.add(new EntityPreviewStatus.Info( - resultInfo.userColumnName(printSettings), - printed, - resultInfo.getType().typeInfo(), - resultInfo.getDescription(), - resultInfo.getSemantics() + extraInfos.add(new EntityPreviewStatus.Info(resultInfo.userColumnName(printSettings), + printed, + resultInfo.getType().typeInfo(), + resultInfo.getDescription(), + resultInfo.getSemantics() )); } @@ -149,9 +144,7 @@ private ManagedQuery getInfoCardExecution() { @NotNull private List toChronoInfos( - PreviewConfig previewConfig, - Map subQueries, - PrintSettings printSettings, PrinterFactory printers) { + PreviewConfig previewConfig, Map subQueries, PrintSettings printSettings, PrinterFactory printers) { final List timeStratifiedInfos = new ArrayList<>(); for (PreviewConfig.TimeStratifiedSelects description : previewConfig.getTimeStratifiedSelects()) { @@ -160,12 +153,10 @@ private List toChronoInfos( final EntityResult entityResult = query.streamResults(OptionalLong.empty()).collect(MoreCollectors.onlyElement()); final Map select2desc = - description.selects().stream() - .collect(Collectors.toMap(PreviewConfig.InfoCardSelect::select, Function.identity())); + description.selects().stream().collect(Collectors.toMap(PreviewConfig.InfoCardSelect::select, Function.identity())); // Group lines by year and quarter. - final Function> lineTransformer = - createLineToMapTransformer(query.getResultInfos(), select2desc, printSettings, printers); + final Function> lineTransformer = createLineToMapTransformer(query.getResultInfos(), select2desc, printSettings, printers); final List yearEntries = createYearEntries(entityResult, lineTransformer); final Object[] completeResult = getCompleteLine(entityResult); @@ -174,14 +165,12 @@ private List toChronoInfos( final List columnDescriptors = createChronoColumnDescriptors(query, select2desc); - final EntityPreviewStatus.TimeStratifiedInfos - infos = - new EntityPreviewStatus.TimeStratifiedInfos(description.label(), - description.description(), - columnDescriptors, - lineTransformer.apply(completeResult), - yearEntries - ); + final EntityPreviewStatus.TimeStratifiedInfos infos = new EntityPreviewStatus.TimeStratifiedInfos(description.label(), + description.description(), + columnDescriptors, + lineTransformer.apply(completeResult), + yearEntries + ); timeStratifiedInfos.add(infos); } @@ -194,8 +183,7 @@ private List toChronoInfos( * Null values are omitted. */ private static Function> createLineToMapTransformer( - List resultInfos, Map select2desc, PrintSettings printSettings, - PrinterFactory printerFactory) { + List resultInfos, Map select2desc, PrintSettings printSettings, PrinterFactory printerFactory) { final int size = resultInfos.size(); @@ -356,8 +344,7 @@ public List generateColumnDescriptions(boolean isInitialized, // Add grouping semantics to secondaryIds to group by if (descriptor.getSemantics() .stream() - .anyMatch(semanticType -> semanticType instanceof SemanticType.SecondaryIdT desc - && previewConfig.isGroupingColumn(desc.getSecondaryId()))) { + .anyMatch(semanticType -> semanticType instanceof SemanticType.SecondaryIdT desc && previewConfig.isGroupingColumn(desc.getSecondaryId()))) { descriptor.getSemantics().add(new SemanticType.GroupT()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java index e995a576ea..4a9e05a0ad 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java @@ -8,7 +8,7 @@ /** * This class is a mess because Excel supports some of our types natively. * - * With LIST types we fall back onto the StringResultPrinter, as Excel does not support Lists. + * With LIST types we fall back onto the StringResultPrinter, as Excel does not support Lists, BUT we also cannot use the {@link IdentityPrinter} inside the list, as some of our printers are native types. */ public class ExcelResultPrinters extends StringResultPrinters { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java index c560520e1f..a0972526cb 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java @@ -67,14 +67,9 @@ public Object apply(List value) { private record DateRangePrinter() implements Printer> { - @Override public Object apply(List f) { - final Integer min = (Integer) ((List) f).get(0); - final Integer max = (Integer) ((List) f).get(1); - - - return CDateRange.of(min, max); + return CDateRange.of(f.get(0), f.get(1)); } } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java index c45bb0ca5a..69dd30b38c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java @@ -6,6 +6,9 @@ import com.bakdata.conquery.models.query.resultinfo.printers.common.ToStringPrinter; import lombok.ToString; +/** + * This class simply put's out native types where possible to let Jackson handle the Serialization, except for Date and DateRange, where the Frontend cannot ensure proper handling. + */ @ToString public class JsonResultPrinters extends JavaResultPrinters { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java index 1a78023393..56b0a4e3aa 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java @@ -2,6 +2,11 @@ import java.util.function.Function; +/** + * Printers handle transformation from {@link com.bakdata.conquery.models.query.results.EntityResult} to the respective renderers "native" representation. + * + * @param The intermediate representation of the type we are printing. + */ @FunctionalInterface public interface Printer extends Function { Object apply(T value); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java index beb868a420..2d46e73c68 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java @@ -5,8 +5,16 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.types.ResultType; - +/** + * This class allows {@link com.bakdata.conquery.models.datasets.concepts.select.Select}s to abstractly define printing, for all our renderers. + * + * The primary thing this class solves is {@link List} printing interacting with special handling like {@link com.bakdata.conquery.models.datasets.concepts.select.concept.ConceptColumnSelect} and {@link com.bakdata.conquery.models.datasets.concepts.select.connector.specific.MappableSingleColumnSelect}. + */ public abstract class PrinterFactory { + /** + * Default implementation of determining the printer for a {@link ResultType}. + * Generally, this method should not be overriden and preferably be final, but {@link ExcelResultPrinters} makes this problematic. + */ public Printer printerFor(ResultType type, PrintSettings printSettings) { if (type instanceof ResultType.ListT listT) { final Printer elementPrinter = printerFor(listT.getElementType(), printSettings); @@ -28,10 +36,16 @@ public Printer printerFor(ResultType type, PrintSettings printSettings) { public abstract Printer getBooleanPrinter(PrintSettings printSettings); + /** + * Jackson will opportunistically read {@link Long} and {@link Integer} hence our usage of Number. + */ public abstract Printer getIntegerPrinter(PrintSettings printSettings); public abstract Printer getNumericPrinter(PrintSettings printSettings); + /** + * Jackson will opportunistically read {@link Long} and {@link Integer} hence our usage of Number. + */ public abstract Printer getDatePrinter(PrintSettings printSettings); public abstract Printer> getDateRangePrinter(PrintSettings printSettings); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java index 574cd7f43a..4bcd1c0b8d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java @@ -11,9 +11,12 @@ import com.bakdata.conquery.models.query.resultinfo.printers.common.MoneyStringPrinter; import com.bakdata.conquery.models.query.resultinfo.printers.common.NumericStringPrinter; import com.bakdata.conquery.models.query.resultinfo.printers.common.StringPrinter; -import lombok.extern.slf4j.Slf4j; +import lombok.ToString; -@Slf4j +/** + * All printers in this factory should be assumed to return {@link String}, this is useful for CSV or HTML printing. + */ +@ToString public class StringResultPrinters extends PrinterFactory { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MappedPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MappedPrinter.java index 91f3f45333..34c5d8daab 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MappedPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MappedPrinter.java @@ -7,6 +7,6 @@ public record MappedPrinter(InternToExternMapper mapper) implements Printer statistics, Range dateRange) { @SneakyThrows @NotNull - public static ResultStatistics collectResultStatistics(SingleTableResult managedQuery, List resultInfos, Optional dateInfo, Optional dateIndex, PrintSettings printSettings, UniqueNamer uniqueNamer, ConqueryConfig conqueryConfig, - PrinterFactory printerFactory) { + public static ResultStatistics collectResultStatistics( + SingleTableResult managedQuery, + List resultInfos, + Optional dateInfo, + Optional dateIndex, + PrintSettings printSettings, + UniqueNamer uniqueNamer, + ConqueryConfig conqueryConfig, + PrinterFactory printerFactory) { //TODO pull inner executor service from ManagerNode - final ListeningExecutorService - executorService = - MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() - 1)); + final ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() - 1)); // Yes, we are actually iterating the result for every job. @@ -53,7 +59,7 @@ public static ResultStatistics collectResultStatistics(SingleTableResult managed final boolean containsDates = dateInfo.isPresent(); if (containsDates) { - futureSpan = executorService.submit(() -> calculateDateSpan(managedQuery, dateInfo, dateIndex.get())); + futureSpan = executorService.submit(() -> calculateDateSpan(managedQuery, dateInfo, dateIndex.get(), printSettings)); } else { futureSpan = Futures.immediateFuture(CDateRange.all().toSimpleRange()); @@ -62,12 +68,10 @@ public static ResultStatistics collectResultStatistics(SingleTableResult managed // Count result lines and entities (may differ in case of form or SecondaryIdQuery) final ListenableFuture futureLines = executorService.submit(() -> (int) managedQuery.resultRowCount()); - final ListenableFuture futureEntities = - executorService.submit(() -> (int) managedQuery.streamResults(OptionalLong.empty()).count()); + final ListenableFuture futureEntities = executorService.submit(() -> (int) managedQuery.streamResults(OptionalLong.empty()).count()); // compute ResultColumnStatistics for each column - final List> - futureDescriptions = + final List> futureDescriptions = IntStream.range(0, resultInfos.size()) // If the query doesn't contain dates, we can skip the dates-column. .filter(col -> !resultInfos.get(col).getSemantics().contains(new SemanticType.EventDateT()) || containsDates) @@ -77,7 +81,12 @@ public static ResultStatistics collectResultStatistics(SingleTableResult managed final ResultInfo info = resultInfos.get(col); final Printer printer = info.createPrinter(printerFactory, printSettings); final ColumnStatsCollector statsCollector = - ColumnStatsCollector.getStatsCollector(uniqueNamer.getUniqueName(info, printSettings), info.getDescription(), info.getType(), printSettings, conqueryConfig.getFrontend()); + ColumnStatsCollector.getStatsCollector(uniqueNamer.getUniqueName(info, printSettings), + info.getDescription(), + info.getType(), + printSettings, + conqueryConfig.getFrontend() + ); log.trace("BEGIN stats collection for {}", info); @@ -85,7 +94,7 @@ public static ResultStatistics collectResultStatistics(SingleTableResult managed .map(EntityResult::listResultLines) .flatMap(List::stream) .forEach(line -> { - Object value = line[col]; + final Object value = line[col]; if (value == null) { // Printers dont handle null statsCollector.consume(null); @@ -115,13 +124,13 @@ public static ResultStatistics collectResultStatistics(SingleTableResult managed return new ResultStatistics(entities, lines, descriptions, span); } - private static Range calculateDateSpan(SingleTableResult managedQuery, Optional dateInfo, int dateIndex) { + private static Range calculateDateSpan(SingleTableResult managedQuery, Optional dateInfo, int dateIndex, PrintSettings printSettings) { if (dateInfo.isEmpty()) { return CDateRange.all().toSimpleRange(); } final AtomicReference spanRef = new AtomicReference<>(null); - final Consumer dateAggregator = getDateSpanner(dateInfo.get(), dateIndex, spanRef); + final Consumer dateAggregator = getDateSpanner(dateInfo.get(), dateIndex, spanRef, printSettings); managedQuery.streamResults(OptionalLong.empty()).flatMap(EntityResult::streamValues).forEach(dateAggregator); @@ -137,28 +146,32 @@ private static Range calculateDateSpan(SingleTableResult managedQuery /** * If not dateInfo is given, don't try to span values. otherwise takes values from line at dateIndex, and handles them according to dateInfo. */ - private static Consumer getDateSpanner(ResultInfo dateInfo, int dateIndex, AtomicReference spanRef) { + private static Consumer getDateSpanner(ResultInfo dateInfo, int dateIndex, AtomicReference spanRef, PrintSettings printSettings) { final Consumer spanner = date -> spanRef.getAndAccumulate(date, (old, incoming) -> incoming.spanClosed(old)); - final BiConsumer> extractor = validityDateExtractor(dateInfo.getType()); + final JavaResultPrinters printers = new JavaResultPrinters(); + final BiConsumer> extractor = validityDateExtractor(dateInfo.getType(), printSettings, printers); return line -> extractor.accept(line[dateIndex], spanner); } - public static BiConsumer> validityDateExtractor(ResultType dateType) { + public static BiConsumer> validityDateExtractor(ResultType dateType, PrintSettings printSettings, JavaResultPrinters printers) { + if (dateType.equals(ResultType.Primitive.DATE_RANGE)) { - return (obj, con) -> con.accept(CDateRange.fromList((List) obj)); + final Printer printer = printers.getDateRangePrinter(printSettings); + return (obj, con) -> con.accept((CDateRange) printer.apply(obj)); } if (dateType.equals(ResultType.Primitive.DATE)) { - return (obj, con) -> con.accept(CDateRange.exactly((Integer) obj)); + final Printer printer = printers.getDatePrinter(printSettings); + return (obj, con) -> con.accept(CDateRange.exactly((LocalDate) printer.apply(obj))); } if (dateType instanceof ResultType.ListT listT) { - final BiConsumer> extractor = validityDateExtractor(listT.getElementType()); + final BiConsumer> extractor = validityDateExtractor(listT.getElementType(), printSettings, printers); return (obj, con) -> ((List) obj).forEach(date -> extractor.accept(date, con)); } diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java index b2061b1ac2..78e5c9bd5e 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java @@ -87,37 +87,39 @@ public static List getResultTypes() { @NotNull public static List getTestEntityResults() { - return List.of( - new SinglelineEntityResult("1", - new Object[]{ - Boolean.TRUE, 2345634, 123423.34, 5646, List.of(345, - 534 - ), "test_string", new BigDecimal("45.21"), List.of(true, false), List.of(List.of(345, 534), List.of(1, 2)), List.of( - "fizz", - "buzz" - ) - } - ), - new SinglelineEntityResult("2", - new Object[]{ - Boolean.FALSE, null, null, null, null, null, null, List.of(), List.of(List.of(1234, - Integer.MAX_VALUE - )), List.of() - } - ), - new SinglelineEntityResult("2", new Object[]{Boolean.TRUE, null, null, null, null, null, null, List.of(false, false), null, null}), - new MultilineEntityResult("3", - List.of(new Object[]{Boolean.FALSE, null, null, null, null, null, null, List.of(false), null, null}, - new Object[]{Boolean.TRUE, null, null, null, null, null, null, null, null, null}, - new Object[]{ - Boolean.TRUE, null, null, null, null, null, new BigDecimal("4.00"), List.of(true, - false, - true, - false - ), null, null - } - ) - ) + return List.of(new SinglelineEntityResult("1", new Object[]{ + Boolean.TRUE, + 2345634, + 123423.34, + 5646, + List.of(345, 534), + "test_string", + new BigDecimal("45.21"), + List.of(true, false), + List.of(List.of(345, 534), List.of(1, 2)), + List.of("fizz", "buzz") + }), + new SinglelineEntityResult("2", new Object[]{ + Boolean.FALSE, null, null, null, null, null, null, List.of(), List.of(List.of(1234, Integer.MAX_VALUE)), List.of() + }), + new SinglelineEntityResult("2", new Object[]{Boolean.TRUE, null, null, null, null, null, null, List.of(false, false), null, null}), + new MultilineEntityResult("3", + List.of(new Object[]{Boolean.FALSE, null, null, null, null, null, null, List.of(false), null, null}, + new Object[]{Boolean.TRUE, null, null, null, null, null, null, null, null, null}, + new Object[]{ + Boolean.TRUE, + null, + null, + null, + null, + null, + new BigDecimal("4.00"), + List.of(true, false, true, false), + null, + null + } + ) + ) ); } diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java index 465f7c586e..da5cc444f2 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java @@ -36,7 +36,6 @@ import com.bakdata.conquery.models.query.resultinfo.printers.ArrowResultPrinters; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.types.ResultType; -import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.VectorSchemaRoot; @@ -56,9 +55,7 @@ public class ArrowResultGenerationTest { public static final ConqueryConfig CONFIG = new ConqueryConfig(); private static final int BATCH_SIZE = 2; - private static final PrintSettings - PRINT_SETTINGS = - new PrintSettings(false, Locale.ROOT, null, CONFIG, null, (selectInfo) -> selectInfo.getSelect().getLabel()); + private static final PrintSettings PRINT_SETTINGS = new PrintSettings(false, Locale.ROOT, null, CONFIG, null, (selectInfo) -> selectInfo.getSelect().getLabel()); public static String readTSV(InputStream inputStream) throws IOException { StringJoiner stringJoiner = new StringJoiner("\n"); @@ -72,11 +69,10 @@ public static String readTSV(InputStream inputStream) throws IOException { for (int rowI = 0; rowI < readRoot.getRowCount(); rowI++) { final int currentRow = rowI; - stringJoiner.add( - vectors.stream() - .map(vec -> vec.getObject(currentRow)) - .map(ArrowResultGenerationTest::getPrintValue) - .collect(Collectors.joining("\t"))); + stringJoiner.add(vectors.stream() + .map(vec -> vec.getObject(currentRow)) + .map(ArrowResultGenerationTest::getPrintValue) + .collect(Collectors.joining("\t"))); } } } @@ -84,33 +80,39 @@ public static String readTSV(InputStream inputStream) throws IOException { } public static String generateExpectedTSV(List results, List resultInfos, boolean isParquet) { - String expected = results.stream() - .map(EntityResult.class::cast) - .map(res -> { - StringJoiner lineJoiner = new StringJoiner("\n"); - - for (Object[] line : res.listResultLines()) { - StringJoiner valueJoiner = new StringJoiner("\t"); - valueJoiner.add(String.valueOf(res.getEntityId())); - valueJoiner.add(String.valueOf(res.getEntityId())); - for (int lIdx = 0; lIdx < line.length; lIdx++) { - Object val = line[lIdx]; - ResultInfo info = resultInfos.get(lIdx); - valueJoiner.add(getPrintValue(val, info.getType(), isParquet)); - } - lineJoiner.add(valueJoiner.toString()); - } - return lineJoiner.toString(); - }) - .collect(Collectors.joining("\n")); + String expected = + results.stream() + .map(EntityResult.class::cast) + .map(res -> { + StringJoiner lineJoiner = new StringJoiner("\n"); + + for (Object[] line : res.listResultLines()) { + StringJoiner valueJoiner = new StringJoiner("\t"); + + valueJoiner.add(String.valueOf(res.getEntityId())); + valueJoiner.add(String.valueOf(res.getEntityId())); + + for (int lIdx = 0; lIdx < line.length; lIdx++) { + Object val = line[lIdx]; + ResultInfo info = resultInfos.get(lIdx); + + valueJoiner.add(getPrintValue(val, info.getType(), isParquet)); + } + + lineJoiner.add(valueJoiner.toString()); + } + return lineJoiner.toString(); + }).collect(Collectors.joining("\n")); return Stream.concat( - // Id column headers - getIdFields().stream().map(i -> i.defaultColumnName(PRINT_SETTINGS)), - // result column headers - getResultTypes().stream().map(ResultType::typeInfo) - ).collect(Collectors.joining("\t")) - + "\n" + expected; + // Id column headers + getIdFields().stream().map(i -> i.defaultColumnName(PRINT_SETTINGS)), + // result column headers + getResultTypes().stream().map(ResultType::typeInfo) + ) + .collect(Collectors.joining("\t")) + + "\n" + + expected; } private static String getPrintValue(Object obj, ResultType type, boolean isParquet) { @@ -144,7 +146,7 @@ private static String getPrintValue(Object obj, ResultType type, boolean isParqu if (obj instanceof Collection) { Collection col = (Collection) obj; // Workaround: Arrow deserializes lists as a JsonStringArrayList which has a JSON String method - @NonNull ResultType elemType = ((ResultType.ListT) type).getElementType(); + ResultType elemType = ((ResultType.ListT) type).getElementType(); return col.stream().map(v -> getPrintValue(v, elemType, isParquet)).collect(Collectors.joining(",", "[", "]")); } return obj.toString(); @@ -153,9 +155,7 @@ private static String getPrintValue(Object obj, ResultType type, boolean isParqu private static String getPrintValue(Object obj) { if (obj instanceof JsonStringArrayList) { // Workaround: Arrow deserializes lists as a JsonStringArrayList which has a JSON String method - return new ArrayList<>((JsonStringArrayList) obj).stream() - .map(ArrowResultGenerationTest::getPrintValue) - .collect(Collectors.joining(",", "[", "]")); + return new ArrayList<>((JsonStringArrayList) obj).stream().map(ArrowResultGenerationTest::getPrintValue).collect(Collectors.joining(",", "[", "]")); } return Objects.toString(obj); } @@ -167,8 +167,7 @@ void generateFieldsIdMapping() { List fields = generateFields(getIdFields(), uniqueNamer, PRINT_SETTINGS); assertThat(fields).containsExactlyElementsOf( - List.of( - new Field("id1", FieldType.nullable(new ArrowType.Utf8()), null), + List.of(new Field("id1", FieldType.nullable(new ArrowType.Utf8()), null), new Field("id2", FieldType.nullable(new ArrowType.Utf8()), null) )); @@ -184,22 +183,19 @@ void generateFieldsValue() { .map(select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet())) .collect(Collectors.toList()); - List fields = generateFields( - resultInfos, - // Custom column namer so we don't require a dataset registry - uniqueNamer, PRINT_SETTINGS + List fields = generateFields(resultInfos, + // Custom column namer so we don't require a dataset registry + uniqueNamer, PRINT_SETTINGS ); assertThat(fields).containsExactlyElementsOf( - List.of( - new Field("BOOLEAN", FieldType.nullable(ArrowType.Bool.INSTANCE), null), + List.of(new Field("BOOLEAN", FieldType.nullable(ArrowType.Bool.INSTANCE), null), new Field("INTEGER", FieldType.nullable(new ArrowType.Int(32, true)), null), new Field("NUMERIC", FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), null), new Field("DATE", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null), new Field("DATE_RANGE", FieldType.nullable(ArrowType.Struct.INSTANCE), - List.of( - new Field("min", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null), + List.of(new Field("min", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null), new Field("max", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null) ) ), @@ -215,27 +211,20 @@ void generateFieldsValue() { FieldType.nullable(ArrowType.List.INSTANCE), List.of(new Field("LIST[BOOLEAN]", FieldType.nullable(ArrowType.Bool.INSTANCE), null)) ), - new Field("LIST[DATE_RANGE]", FieldType.nullable(ArrowType.List.INSTANCE), List.of(new Field("LIST[DATE_RANGE]", - FieldType.nullable(ArrowType.Struct.INSTANCE), - List.of( - new Field("min", - FieldType.nullable(new ArrowType.Date( - DateUnit.DAY)), - null - ), - new Field("max", - FieldType.nullable(new ArrowType.Date( - DateUnit.DAY)), - null - ) - ) - ))), + new Field("LIST[DATE_RANGE]", + FieldType.nullable(ArrowType.List.INSTANCE), + List.of(new Field("LIST[DATE_RANGE]", + FieldType.nullable(ArrowType.Struct.INSTANCE), + List.of(new Field("min", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null), + new Field("max", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null) + ) + )) + ), new Field("LIST[STRING]", FieldType.nullable(ArrowType.List.INSTANCE), List.of(new Field("LIST[STRING]", FieldType.nullable(new ArrowType.Utf8()), null)) ) - ) - ); + )); } diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java index dd301d348d..54ae964fed 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java @@ -42,13 +42,12 @@ public class CsvResultGenerationTest { void writeAndRead() throws IOException { // Prepare every input data StringResultPrinters printers = new StringResultPrinters(); - final PrintSettings printSettings = new PrintSettings( - true, - Locale.GERMAN, - null, - CONFIG, - (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), - (selectInfo) -> selectInfo.getSelect().getLabel() + final PrintSettings printSettings = new PrintSettings(true, + Locale.GERMAN, + null, + CONFIG, + (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), + (selectInfo) -> selectInfo.getSelect().getLabel() ); // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized final List results = getTestEntityResults(); @@ -77,25 +76,35 @@ private String generateExpectedCSV(List results, List final List expected = new ArrayList<>(); expected.add(getIdFields().stream().map(info -> info.defaultColumnName(printSettings)).collect(Collectors.joining(",")) + "," - + getResultTypes().stream().map(ResultType::typeInfo).collect(Collectors.joining(",")) + + getResultTypes().stream() + .map(ResultType::typeInfo) + .collect(Collectors.joining(",")) + "\n"); + + final String delimiter = String.valueOf(CONFIG.getCsv().getDelimeter()); + results.stream() .map(EntityResult.class::cast) .forEach(res -> { for (Object[] line : res.listResultLines()) { final StringJoiner valueJoiner = new StringJoiner(","); + valueJoiner.add(String.valueOf(res.getEntityId())); valueJoiner.add(String.valueOf(res.getEntityId())); + for (int lIdx = 0; lIdx < line.length; lIdx++) { final Object val = line[lIdx]; + if (val == null) { valueJoiner.add(""); continue; } + final ResultInfo info = resultInfos.get(lIdx); final String printVal = (String) info.createPrinter(printerFactory, printSettings).apply(val); - valueJoiner.add(printVal.contains(String.valueOf(CONFIG.getCsv().getDelimeter())) ? "\"" + printVal + "\"" : printVal); + + valueJoiner.add(printVal.contains(delimiter) ? "\"" + printVal + "\"" : printVal); } expected.add(valueJoiner + "\n"); diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java index d974b4b9e8..4d7b3cde9f 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java @@ -102,6 +102,7 @@ public Stream streamResults(OptionalLong maybeLimit) { (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), (selectInfo) -> selectInfo.getSelect().getLabel() ); + final List expected = generateExpectedTSV(results, mquery.getResultInfos(), tsvPrintSettings, new StringResultPrinters()); log.info("Wrote and than read this excel data: {}", computed); @@ -137,29 +138,31 @@ private List readComputed(InputStream inputStream, PrintSettings setting } - private List generateExpectedTSV(List results, List resultInfos, PrintSettings printSettings, - PrinterFactory printerFactory) { + private List generateExpectedTSV( + List results, List resultInfos, PrintSettings printSettings, PrinterFactory printerFactory) { final List expected = new ArrayList<>(); expected.add(String.join("\t", printIdFields) + "\t" + getResultTypes().stream().map(ResultType::typeInfo).collect(Collectors.joining("\t"))); - results.stream().map(EntityResult.class::cast).forEach(res -> { + results.stream() + .map(EntityResult.class::cast) + .forEach(res -> { - for (Object[] line : res.listResultLines()) { - final StringJoiner valueJoiner = new StringJoiner("\t"); + for (Object[] line : res.listResultLines()) { + final StringJoiner valueJoiner = new StringJoiner("\t"); - valueJoiner.add(String.valueOf(res.getEntityId())); - valueJoiner.add(String.valueOf(res.getEntityId())); + valueJoiner.add(String.valueOf(res.getEntityId())); + valueJoiner.add(String.valueOf(res.getEntityId())); - for (int lIdx = 0; lIdx < line.length; lIdx++) { - final Object val = line[lIdx]; + for (int lIdx = 0; lIdx < line.length; lIdx++) { + final Object val = line[lIdx]; - final ResultInfo info = resultInfos.get(lIdx); - final String printed = printValue(val, info, printSettings, printerFactory); + final ResultInfo info = resultInfos.get(lIdx); + final String printed = printValue(val, info, printSettings, printerFactory); - valueJoiner.add(printed); - } - expected.add(valueJoiner.toString()); - } - }); + valueJoiner.add(printed); + } + expected.add(valueJoiner.toString()); + } + }); return expected; } diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java index b23c806bf2..361593b117 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java @@ -103,13 +103,12 @@ void writeAndRead() throws IOException { I18n.init(); // Prepare every input data - PrintSettings printSettings = new PrintSettings( - false, - Locale.ROOT, - null, - CONFIG, - (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), - (selectInfo) -> selectInfo.getSelect().getLabel() + PrintSettings printSettings = new PrintSettings(false, + Locale.ROOT, + null, + CONFIG, + (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), + (selectInfo) -> selectInfo.getSelect().getLabel() ); // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized List results = getTestEntityResults(); From 03434fb7fbbe1bda0f1327fb7c00bd0d4473f579 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 4 Sep 2024 13:45:04 +0200 Subject: [PATCH 44/54] pull default for decimalShift from CurrencyConfig --- .../events/stores/specific/MoneyIntStore.java | 45 +++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java index 8a589b80bf..b67ae8758a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java @@ -3,21 +3,48 @@ import java.math.BigDecimal; import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.io.jackson.Initializing; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.events.stores.root.ColumnStore; import com.bakdata.conquery.models.events.stores.root.IntegerStore; import com.bakdata.conquery.models.events.stores.root.MoneyStore; +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.OptBoolean; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import jakarta.validation.constraints.NotNull; import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; import lombok.ToString; @CPSType(base = ColumnStore.class, id = "MONEY_VARINT") @Data @ToString(of = "numberType") -public class MoneyIntStore implements MoneyStore { +@NoArgsConstructor(onConstructor_ = {@JsonCreator}) +@JsonDeserialize(converter = MoneyIntStore.MoneyIntStoreInitializer.class) +public class MoneyIntStore implements MoneyStore, Initializing { - private final IntegerStore numberType; - private final int decimalShift; //TODO this might require preprocessing, consider injecting this. + @JsonIgnore + @JacksonInject(useInput = OptBoolean.FALSE) + @NotNull + @EqualsAndHashCode.Exclude + private ConqueryConfig config; + private IntegerStore numberType; + + @JsonProperty(required = false) + private int decimalShift = Integer.MIN_VALUE; + + + public MoneyIntStore(IntegerStore store, int decimalShift){ + this(); + this.numberType = store; + this.decimalShift = decimalShift; + } @Override public int getLines() { @@ -62,4 +89,16 @@ public final boolean has(int event) { public void setParent(Bucket bucket) { // not used } + + @Override + public void init() { + if (decimalShift != Integer.MIN_VALUE){ + return; + } + + decimalShift = config.getFrontend().getCurrency().getDecimalScale(); + } + + public static class MoneyIntStoreInitializer extends Initializing.Converter {} + } From 672cb4256761ecba08957637dc7f9a95eff038c8 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 4 Sep 2024 13:55:04 +0200 Subject: [PATCH 45/54] fixes injection and deserialization test --- .../conquery/mode/cluster/InternalMapperFactory.java | 5 +++-- .../models/events/stores/specific/MoneyIntStore.java | 4 ++-- .../events/stores/types/ColumnStoreSerializationTests.java | 6 ++++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/InternalMapperFactory.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/InternalMapperFactory.java index e4ab7af4bd..832d7bdef4 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/InternalMapperFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/InternalMapperFactory.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.mode.cluster; -import jakarta.validation.Validator; - import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.io.jackson.View; @@ -12,6 +10,7 @@ import com.fasterxml.jackson.databind.DeserializationConfig; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationConfig; +import jakarta.validation.Validator; public record InternalMapperFactory(ConqueryConfig config, Validator validator) { @@ -31,6 +30,7 @@ public ObjectMapper createWorkerPersistenceMapper(Workers workers) { final ObjectMapper objectMapper = createInternalObjectMapper(View.Persistence.Shard.class); workers.injectInto(objectMapper); + config.injectInto(objectMapper); return objectMapper; } @@ -40,6 +40,7 @@ public ObjectMapper createNamespacePersistenceMapper(DatasetRegistry datasetR datasetRegistry.injectInto(objectMapper); + return objectMapper; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java index b67ae8758a..1784c7f7c5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java @@ -15,11 +15,11 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.OptBoolean; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import jakarta.validation.constraints.NotNull; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import lombok.ToString; +import lombok.experimental.Accessors; @CPSType(base = ColumnStore.class, id = "MONEY_VARINT") @Data @@ -30,8 +30,8 @@ public class MoneyIntStore implements MoneyStore, Initializing { @JsonIgnore @JacksonInject(useInput = OptBoolean.FALSE) - @NotNull @EqualsAndHashCode.Exclude + @Accessors(fluent = true) private ConqueryConfig config; private IntegerStore numberType; diff --git a/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/ColumnStoreSerializationTests.java b/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/ColumnStoreSerializationTests.java index 603a960f40..7fec1b9d7c 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/ColumnStoreSerializationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/ColumnStoreSerializationTests.java @@ -54,6 +54,7 @@ public class ColumnStoreSerializationTests { private static final CentralRegistry CENTRAL_REGISTRY = new CentralRegistry(); private static ObjectMapper shardInternalMapper; + private static ConqueryConfig config; @BeforeAll public static void setupRegistry() { @@ -61,7 +62,8 @@ public static void setupRegistry() { // Prepare shard node internal mapper - InternalMapperFactory internalMapperFactory = new InternalMapperFactory(new ConqueryConfig(), Validators.newValidator()); + config = new ConqueryConfig(); + InternalMapperFactory internalMapperFactory = new InternalMapperFactory(config, Validators.newValidator()); shardInternalMapper = internalMapperFactory.createWorkerPersistenceMapper(mock(Workers.class)); } @@ -88,7 +90,7 @@ public static List createCTypes() { return Arrays.asList( new ScaledDecimalStore(13, IntArrayStore.create(10)), - new MoneyIntStore(IntArrayStore.create(10), 2), + new MoneyIntStore(IntArrayStore.create(10), 2).config(config), new DirectDateRangeStore(IntegerDateStore.create(10), IntegerDateStore.create(10)), new QuarterDateRangeStore(LongArrayStore.create(10)), new IntegerDateStore(LongArrayStore.create(10)), From f8ffea94adad20982d5cb949f490301576ff987c Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 4 Sep 2024 14:47:38 +0200 Subject: [PATCH 46/54] semantics are immutable --- .../models/query/preview/EntityPreviewExecution.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java index 091368c350..f4bc36c626 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java @@ -8,6 +8,7 @@ import java.util.List; import java.util.Map; import java.util.OptionalLong; +import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -43,6 +44,7 @@ import com.bakdata.conquery.models.worker.Namespace; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.MoreCollectors; +import com.google.common.collect.Sets; import lombok.AccessLevel; import lombok.NoArgsConstructor; import lombok.ToString; @@ -345,14 +347,14 @@ public List generateColumnDescriptions(boolean isInitialized, if (descriptor.getSemantics() .stream() .anyMatch(semanticType -> semanticType instanceof SemanticType.SecondaryIdT desc && previewConfig.isGroupingColumn(desc.getSecondaryId()))) { - descriptor.getSemantics().add(new SemanticType.GroupT()); + descriptor.setSemantics(Sets.union(Set.of(new SemanticType.GroupT()), descriptor.getSemantics())); } // Add hidden semantics to fields flagged for hiding. if (descriptor.getSemantics() .stream() .anyMatch(semanticType -> semanticType instanceof SemanticType.ColumnT desc && previewConfig.isHidden(desc.getColumn()))) { - descriptor.getSemantics().add(new SemanticType.HiddenT()); + descriptor.setSemantics(Sets.union(Set.of(new SemanticType.HiddenT()), descriptor.getSemantics())); } } From 8bdfdd15f336b751b581226f93b1fcf2157f014e Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 4 Sep 2024 16:23:10 +0200 Subject: [PATCH 47/54] fixes missing null-Check --- .../models/query/preview/EntityPreviewExecution.java | 11 ++++++++++- .../query/resultinfo/printers/JavaResultPrinters.java | 5 +++-- .../models/query/resultinfo/printers/Printer.java | 4 +++- .../printers/common/BooleanStringPrinter.java | 3 ++- .../resultinfo/printers/common/ConceptIdPrinter.java | 3 ++- .../query/resultinfo/printers/common/DatePrinter.java | 3 ++- .../printers/common/DateRangeStringPrinter.java | 3 ++- .../resultinfo/printers/common/DateStringPrinter.java | 3 ++- .../resultinfo/printers/common/IdentityPrinter.java | 3 ++- .../printers/common/IntegerStringPrinter.java | 3 ++- .../resultinfo/printers/common/ListStringPrinter.java | 7 ++++++- .../printers/common/LocalizedEnumPrinter.java | 3 ++- .../resultinfo/printers/common/MappedPrinter.java | 3 ++- .../printers/common/MoneyStringPrinter.java | 3 ++- .../printers/common/NumericStringPrinter.java | 3 ++- .../resultinfo/printers/common/StringPrinter.java | 3 ++- .../resultinfo/printers/common/ToStringPrinter.java | 3 ++- 17 files changed, 48 insertions(+), 18 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java index f4bc36c626..4994b53e3a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java @@ -126,7 +126,16 @@ private List transformQueryResultToInfos( for (int index = AbsoluteFormQuery.FEATURES_OFFSET; index < infoCardExecution.getResultInfos().size(); index++) { final ResultInfo resultInfo = infoCardExecution.getResultInfos().get(index); - final Object printed = resultInfo.createPrinter(printerFactory, printSettings).apply(values[index]); + final Object value = values[index]; + final Object printed; + + if (value == null) { + printed = null; + } + else { + Printer printer = resultInfo.createPrinter(printerFactory, printSettings); + printed = printer.apply(value); + } extraInfos.add(new EntityPreviewStatus.Info(resultInfo.userColumnName(printSettings), printed, diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java index a0972526cb..bab9fc5153 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java @@ -7,6 +7,7 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.common.DatePrinter; import com.bakdata.conquery.models.query.resultinfo.printers.common.IdentityPrinter; +import org.jetbrains.annotations.NotNull; public class JavaResultPrinters extends PrinterFactory { @@ -54,7 +55,7 @@ public Printer getMoneyPrinter(PrintSettings printSettings) { private record ListPrinter(Printer elementPrinter) implements Printer> { @Override - public Object apply(List value) { + public Object apply(@NotNull List value) { final List out = new ArrayList<>(value.size()); for (T elt : value) { @@ -68,7 +69,7 @@ public Object apply(List value) { private record DateRangePrinter() implements Printer> { @Override - public Object apply(List f) { + public Object apply(@NotNull List f) { return CDateRange.of(f.get(0), f.get(1)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java index 56b0a4e3aa..b165df89ea 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java @@ -2,6 +2,8 @@ import java.util.function.Function; +import org.jetbrains.annotations.NotNull; + /** * Printers handle transformation from {@link com.bakdata.conquery.models.query.results.EntityResult} to the respective renderers "native" representation. * @@ -9,5 +11,5 @@ */ @FunctionalInterface public interface Printer extends Function { - Object apply(T value); + Object apply(@NotNull T value); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/BooleanStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/BooleanStringPrinter.java index d7d3879d21..d54fea8853 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/BooleanStringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/BooleanStringPrinter.java @@ -4,6 +4,7 @@ import com.bakdata.conquery.models.query.C10nCache; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import org.jetbrains.annotations.NotNull; public record BooleanStringPrinter(PrintSettings cfg, String trueVal, String falseVal) implements Printer { @@ -18,7 +19,7 @@ public static BooleanStringPrinter create(PrintSettings settings) { @Override - public String apply(Boolean f) { + public String apply(@NotNull Boolean f) { return f ? trueVal : falseVal; } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ConceptIdPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ConceptIdPrinter.java index 51fb158a13..e495a5110a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ConceptIdPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ConceptIdPrinter.java @@ -5,11 +5,12 @@ import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import org.jetbrains.annotations.NotNull; public record ConceptIdPrinter(Concept concept, PrintSettings cfg) implements Printer { @Override - public String apply(Integer localId) { + public String apply(@NotNull Integer localId) { if (localId == null) { return null; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DatePrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DatePrinter.java index c03d72f55b..d55fac0a44 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DatePrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DatePrinter.java @@ -2,11 +2,12 @@ import com.bakdata.conquery.models.common.CDate; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import org.jetbrains.annotations.NotNull; public record DatePrinter() implements Printer { @Override - public Object apply(Number value) { + public Object apply(@NotNull Number value) { return CDate.toLocalDate(value.intValue()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateRangeStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateRangeStringPrinter.java index f1f623cff1..bbc8cc9d9f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateRangeStringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateRangeStringPrinter.java @@ -7,6 +7,7 @@ import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.google.common.base.Preconditions; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; @Slf4j public record DateRangeStringPrinter(DateStringPrinter datePrinter, PrintSettings cfg) implements Printer> { @@ -16,7 +17,7 @@ public DateRangeStringPrinter(PrintSettings printSettings) { } @Override - public String apply(List f) { + public String apply(@NotNull List f) { Preconditions.checkArgument(f.size() == 2, "Expected a list with 2 elements, one min, one max. The list was: %s ", f); final Integer min = f.get(0); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateStringPrinter.java index 5240159d19..f9a57f05b5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateStringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateStringPrinter.java @@ -3,11 +3,12 @@ import com.bakdata.conquery.models.common.CDate; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import org.jetbrains.annotations.NotNull; public record DateStringPrinter(PrintSettings cfg) implements Printer { @Override - public String apply(Number f) { + public String apply(@NotNull Number f) { return cfg.getDateFormatter().format(CDate.toLocalDate(f.intValue())); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/IdentityPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/IdentityPrinter.java index 134135d727..30c05117e4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/IdentityPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/IdentityPrinter.java @@ -1,11 +1,12 @@ package com.bakdata.conquery.models.query.resultinfo.printers.common; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import org.jetbrains.annotations.NotNull; public record IdentityPrinter() implements Printer { @Override - public Object apply(T value) { + public Object apply(@NotNull T value) { return value; } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/IntegerStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/IntegerStringPrinter.java index 47a765d83a..81418aa87a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/IntegerStringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/IntegerStringPrinter.java @@ -2,11 +2,12 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import org.jetbrains.annotations.NotNull; public record IntegerStringPrinter(PrintSettings cfg) implements Printer { @Override - public String apply(Number f) { + public String apply(@NotNull Number f) { if (cfg.isPrettyPrint()) { return cfg.getIntegerFormat().format(f.longValue()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java index 4128b217d9..f311075acb 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java @@ -6,6 +6,7 @@ import com.bakdata.conquery.models.config.LocaleConfig; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import org.jetbrains.annotations.NotNull; public record ListStringPrinter(Printer elementPrinter, PrintSettings cfg, LocaleConfig.ListFormat listFormat) implements Printer> { @@ -14,11 +15,15 @@ public ListStringPrinter(Printer elementPrinter, PrintSettings cfg) { } @Override - public String apply(List f) { + public String apply(@NotNull List f) { final StringJoiner joiner = listFormat.createListJoiner(); for (T obj : f) { + if (obj == null){ + continue; + } + joiner.add(listFormat.escapeListElement(elementPrinter.apply(obj).toString())); } return joiner.toString(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/LocalizedEnumPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/LocalizedEnumPrinter.java index cf7e37b24c..61cd8ccd60 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/LocalizedEnumPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/LocalizedEnumPrinter.java @@ -3,10 +3,11 @@ import com.bakdata.conquery.models.common.LocalizedToString; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import org.jetbrains.annotations.NotNull; public record LocalizedEnumPrinter & LocalizedToString>(PrintSettings cfg, Class clazz) implements Printer { @Override - public String apply(String f) { + public String apply(@NotNull String f) { try { return Enum.valueOf(clazz, f).toString(cfg.getLocale()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MappedPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MappedPrinter.java index 34c5d8daab..d388ceb689 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MappedPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MappedPrinter.java @@ -2,11 +2,12 @@ import com.bakdata.conquery.models.index.InternToExternMapper; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import org.jetbrains.annotations.NotNull; public record MappedPrinter(InternToExternMapper mapper) implements Printer { @Override - public String apply(String f) { + public String apply(@NotNull String f) { return mapper.external(f); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MoneyStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MoneyStringPrinter.java index 23621fb959..3579dabbbe 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MoneyStringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MoneyStringPrinter.java @@ -2,11 +2,12 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import org.jetbrains.annotations.NotNull; public record MoneyStringPrinter(PrintSettings cfg) implements Printer { @Override - public String apply(Number f) { + public String apply(@NotNull Number f) { if (cfg.isPrettyPrint()) { return cfg.getCurrencyFormat().format(f); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/NumericStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/NumericStringPrinter.java index b2b6bd40ee..0f6d008d05 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/NumericStringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/NumericStringPrinter.java @@ -2,11 +2,12 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import org.jetbrains.annotations.NotNull; public record NumericStringPrinter(PrintSettings cfg) implements Printer { @Override - public String apply(Number f) { + public String apply(@NotNull Number f) { if (cfg.isPrettyPrint()) { return cfg.getDecimalFormat().format(f); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/StringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/StringPrinter.java index 50e34f33c1..335fbce2f1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/StringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/StringPrinter.java @@ -3,10 +3,11 @@ import java.util.Objects; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import org.jetbrains.annotations.NotNull; public record StringPrinter() implements Printer { @Override - public String apply(String f) { + public String apply(@NotNull String f) { return Objects.toString(f); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ToStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ToStringPrinter.java index 2a06847e3a..5b5c204b7b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ToStringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ToStringPrinter.java @@ -3,11 +3,12 @@ import java.util.Objects; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import org.jetbrains.annotations.NotNull; public record ToStringPrinter(Printer delegate) implements Printer { @Override - public Object apply(T value) { + public Object apply(@NotNull T value) { return Objects.toString(delegate.apply(value)); } } From 6358f6400ec1ddbec7cc93a3a7c7a10b5d0bbfaa Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 4 Sep 2024 17:34:48 +0200 Subject: [PATCH 48/54] try fixing Money handling for preview --- frontend/src/js/preview/tableUtils.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/frontend/src/js/preview/tableUtils.ts b/frontend/src/js/preview/tableUtils.ts index 2ee65d8888..18c4b4b075 100644 --- a/frontend/src/js/preview/tableUtils.ts +++ b/frontend/src/js/preview/tableUtils.ts @@ -70,10 +70,16 @@ export function useCustomTableRenderers(queryData: GetQueryResponseDoneT) { }; } else if (cellType === "MONEY") { return (value) => { - if (value && !isNaN(value as unknown as number)) { - return currencyFormatter.format((value as unknown as number) / 100); // MONEY is sent as cent + if (!value) { + return ""; } - return ""; + + const numVal = Number(value); + if(isNan(numVal)) { + return ""; + } + + return currencyFormatter.format(value); }; } else if (cellType === "BOOLEAN") { return (value) => (value ? t("common.true") : t("common.false")); From 65e701c13129269fe979959ac14083881cf352f8 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 5 Sep 2024 11:29:55 +0200 Subject: [PATCH 49/54] Revert "try fixing Money handling for preview" This reverts commit 6358f6400ec1ddbec7cc93a3a7c7a10b5d0bbfaa. --- frontend/src/js/preview/tableUtils.ts | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/frontend/src/js/preview/tableUtils.ts b/frontend/src/js/preview/tableUtils.ts index 18c4b4b075..2ee65d8888 100644 --- a/frontend/src/js/preview/tableUtils.ts +++ b/frontend/src/js/preview/tableUtils.ts @@ -70,16 +70,10 @@ export function useCustomTableRenderers(queryData: GetQueryResponseDoneT) { }; } else if (cellType === "MONEY") { return (value) => { - if (!value) { - return ""; - } - - const numVal = Number(value); - if(isNan(numVal)) { - return ""; + if (value && !isNaN(value as unknown as number)) { + return currencyFormatter.format((value as unknown as number) / 100); // MONEY is sent as cent } - - return currencyFormatter.format(value); + return ""; }; } else if (cellType === "BOOLEAN") { return (value) => (value ? t("common.true") : t("common.false")); From e9203fa346b98ffe0f2878da88c7e77fa0b084b8 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 5 Sep 2024 11:30:36 +0200 Subject: [PATCH 50/54] revert using Decimal for Arrow/Parquet: causes issues with jsArrow and other dependencies --- .../io/result/arrow/ArrowRenderer.java | 8 +++--- .../conquery/io/result/arrow/ArrowUtil.java | 25 ++++++++++--------- .../parquet/EntityResultWriteSupport.java | 3 +-- .../printers/ArrowResultPrinters.java | 16 +++++++++++- .../printers/ExcelResultPrinters.java | 10 +++++--- .../printers/JavaResultPrinters.java | 9 ++++--- .../printers/JsonResultPrinters.java | 2 +- .../resultinfo/printers/PrinterFactory.java | 9 ++++--- .../printers/StringResultPrinters.java | 9 ++++--- .../printers/common/MoneyStringPrinter.java | 6 +++-- .../arrow/ArrowResultGenerationTest.java | 23 ++++++----------- .../parquet/ParquetResultGenerationTest.java | 4 +-- 12 files changed, 68 insertions(+), 56 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java index 852035bb84..f0caac037b 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java @@ -3,7 +3,6 @@ import static com.bakdata.conquery.io.result.arrow.ArrowUtil.ROOT_ALLOCATOR; import java.io.IOException; -import java.math.BigDecimal; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -25,7 +24,6 @@ import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.DateDayVector; -import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.Float8Vector; import org.apache.arrow.vector.IntVector; @@ -186,14 +184,14 @@ private static RowConsumer bitVectorFiller(BitVector vector) { }; } - private static RowConsumer moneyVectorFiller(DecimalVector vector) { + private static RowConsumer moneyVectorFiller(IntVector vector) { return (rowNumber, valueRaw) -> { if (valueRaw == null) { vector.setNull(rowNumber); return; } - final BigDecimal value = (BigDecimal) valueRaw; + final int value = (int) valueRaw; vector.setSafe(rowNumber, value); }; @@ -313,7 +311,7 @@ private static RowConsumer generateVectorFiller(ValueVector vector, ResultType t return switch (((ResultType.Primitive) type)) { case BOOLEAN -> bitVectorFiller(((BitVector) vector)); case INTEGER -> intVectorFiller(((IntVector) vector)); - case MONEY -> moneyVectorFiller(((DecimalVector) vector)); + case MONEY -> moneyVectorFiller(((IntVector) vector)); case DATE -> dateDayVectorFiller(((DateDayVector) vector)); case NUMERIC -> float8VectorFiller((Float8Vector) vector); case STRING -> varCharVectorFiller(((VarCharVector) vector)); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java index 2c00b09abc..741885e961 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java @@ -22,15 +22,15 @@ public class ArrowUtil { public static final RootAllocator ROOT_ALLOCATOR = new RootAllocator(); - private Field fieldFor(ResultType type, String name, PrintSettings settings) { + private Field fieldFor(ResultType type, String name) { if (type instanceof ResultType.ListT) { - return ArrowUtil.listField(name, type, settings); + return ArrowUtil.listField(name, type); } return switch (((ResultType.Primitive) type)) { case BOOLEAN -> ArrowUtil.boolField(name); case INTEGER -> ArrowUtil.integerField(name); - case MONEY -> ArrowUtil.moneyField(name, settings.getCurrency().getDefaultFractionDigits()); + case MONEY -> ArrowUtil.moneyField(name); case NUMERIC -> ArrowUtil.floatField(name); case DATE -> ArrowUtil.dateField(name); case DATE_RANGE -> ArrowUtil.dateRangeField(name); @@ -51,13 +51,14 @@ private static Field integerField(@NonNull String uniqueName) { return new Field(uniqueName, FieldType.nullable(new ArrowType.Int(32, true)), null); } - private static Field moneyField(@NonNull String uniqueName, int scale) { - /* - * From https://arrow.apache.org/docs/python/generated/pyarrow.decimal128.html - * Maximum precision is 38 digits total, therefore we can assume to pack all digits, minus scale in there; - * assuming that no currency will exceed 28 digits, this should be more than fine as a heuristic. + private static Field moneyField(@NonNull String uniqueName) { + /*TODO FK + use decimal: new Field(uniqueName, FieldType.nullable(new ArrowType.Decimal(38 - scale, scale, 128)), null); + This will also impact Frontend preview, and ExternalFormBackends, needs planning. + Note: I suspect jsArrow has a bug, where it reads Decimal as BigInt */ - return new Field(uniqueName, FieldType.nullable(new ArrowType.Decimal(38 - scale, scale, 128)), null); + return new Field(uniqueName, FieldType.nullable(new ArrowType.Int(32, true)), null); + } private static Field floatField(@NonNull String uniqueName) { @@ -78,16 +79,16 @@ private static Field dateRangeField(@NonNull String uniqueName) { )); } - private static Field listField(@NonNull String uniqueName, ResultType type, PrintSettings printSettings) { + private static Field listField(@NonNull String uniqueName, ResultType type) { final ResultType elementType = ((ResultType.ListT) type).getElementType(); - final Field nestedField = fieldFor(elementType, uniqueName, printSettings); + final Field nestedField = fieldFor(elementType, uniqueName); return new Field(uniqueName, FieldType.nullable(ArrowType.List.INSTANCE), List.of(nestedField)); } public static List generateFields(@NonNull List info, UniqueNamer collector, PrintSettings printSettings) { return info.stream() - .map(i -> fieldFor(i.getType(), collector.getUniqueName(i, printSettings), printSettings)) + .map(i -> fieldFor(i.getType(), collector.getUniqueName(i, printSettings))) .toList(); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java index 693de95fd4..bc103231c5 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.io.result.parquet; -import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -225,7 +224,7 @@ public void accept(RecordConsumer recordConsumer, Object o) { private record MoneyColumnConsumer() implements ColumnConsumer { @Override public void accept(RecordConsumer recordConsumer, Object o) { - recordConsumer.addBinary(Binary.fromConstantByteArray(((BigDecimal) o).unscaledValue().toByteArray())); + recordConsumer.addInteger(((Integer) o)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java index caf887b15e..1fd3723f17 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java @@ -1,13 +1,27 @@ package com.bakdata.conquery.models.query.resultinfo.printers; +import java.math.BigDecimal; + import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.common.IdentityPrinter; +import org.jetbrains.annotations.NotNull; public class ArrowResultPrinters extends JavaResultPrinters { @Override - public Printer getDatePrinter(PrintSettings printSettings) { + public Printer getDatePrinter(PrintSettings printSettings) { return new IdentityPrinter<>(); } + @Override + public Printer getMoneyPrinter(PrintSettings printSettings) { + return new MoneyPrinter(); + } + + private record MoneyPrinter() implements Printer { + @Override + public Object apply(@NotNull BigDecimal value) { + return value.unscaledValue().intValueExact(); + } + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java index 4a9e05a0ad..6ba25a5246 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.query.resultinfo.printers; +import java.math.BigDecimal; + import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.common.DatePrinter; import com.bakdata.conquery.models.query.resultinfo.printers.common.IdentityPrinter; @@ -37,22 +39,22 @@ public Printer getBooleanPrinter(PrintSettings printSettings) { } @Override - public Printer getNumericPrinter(PrintSettings printSettings) { + public Printer getNumericPrinter(PrintSettings printSettings) { return new IdentityPrinter<>(); } @Override - public Printer getMoneyPrinter(PrintSettings printSettings) { + public Printer getMoneyPrinter(PrintSettings printSettings) { return new IdentityPrinter<>(); } @Override - public Printer getDatePrinter(PrintSettings printSettings) { + public Printer getDatePrinter(PrintSettings printSettings) { return new DatePrinter(); } @Override - public Printer getIntegerPrinter(PrintSettings printSettings) { + public Printer getIntegerPrinter(PrintSettings printSettings) { return new IdentityPrinter<>(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java index bab9fc5153..eca3268284 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.query.resultinfo.printers; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; @@ -22,17 +23,17 @@ public Printer getBooleanPrinter(PrintSettings printSettings) { } @Override - public Printer getIntegerPrinter(PrintSettings printSettings) { + public Printer getIntegerPrinter(PrintSettings printSettings) { return new IdentityPrinter<>(); } @Override - public Printer getNumericPrinter(PrintSettings printSettings) { + public Printer getNumericPrinter(PrintSettings printSettings) { return new IdentityPrinter<>(); } @Override - public Printer getDatePrinter(PrintSettings printSettings) { + public Printer getDatePrinter(PrintSettings printSettings) { return new DatePrinter(); } @@ -48,7 +49,7 @@ public Printer getStringPrinter(PrintSettings printSettings) { @Override - public Printer getMoneyPrinter(PrintSettings printSettings) { + public Printer getMoneyPrinter(PrintSettings printSettings) { return new IdentityPrinter<>(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java index 69dd30b38c..aa325571e0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java @@ -14,7 +14,7 @@ public class JsonResultPrinters extends JavaResultPrinters { @Override - public Printer getDatePrinter(PrintSettings printSettings) { + public Printer getDatePrinter(PrintSettings printSettings) { return new ToStringPrinter<>(super.getDatePrinter(printSettings)); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java index 2d46e73c68..3fdb1d72fc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.query.resultinfo.printers; +import java.math.BigDecimal; import java.util.List; import com.bakdata.conquery.models.query.PrintSettings; @@ -39,18 +40,18 @@ public Printer printerFor(ResultType type, PrintSettings printSettings) { /** * Jackson will opportunistically read {@link Long} and {@link Integer} hence our usage of Number. */ - public abstract Printer getIntegerPrinter(PrintSettings printSettings); + public abstract Printer getIntegerPrinter(PrintSettings printSettings); - public abstract Printer getNumericPrinter(PrintSettings printSettings); + public abstract Printer getNumericPrinter(PrintSettings printSettings); /** * Jackson will opportunistically read {@link Long} and {@link Integer} hence our usage of Number. */ - public abstract Printer getDatePrinter(PrintSettings printSettings); + public abstract Printer getDatePrinter(PrintSettings printSettings); public abstract Printer> getDateRangePrinter(PrintSettings printSettings); public abstract Printer getStringPrinter(PrintSettings printSettings); - public abstract Printer getMoneyPrinter(PrintSettings printSettings); + public abstract Printer getMoneyPrinter(PrintSettings printSettings); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java index 4bcd1c0b8d..5578934a57 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.query.resultinfo.printers; +import java.math.BigDecimal; import java.util.List; import com.bakdata.conquery.models.query.PrintSettings; @@ -31,17 +32,17 @@ public Printer getBooleanPrinter(PrintSettings printSettings) { } @Override - public Printer getIntegerPrinter(PrintSettings printSettings) { + public Printer getIntegerPrinter(PrintSettings printSettings) { return new IntegerStringPrinter(printSettings); } @Override - public Printer getNumericPrinter(PrintSettings printSettings) { + public Printer getNumericPrinter(PrintSettings printSettings) { return new NumericStringPrinter(printSettings); } @Override - public Printer getDatePrinter(PrintSettings printSettings) { + public Printer getDatePrinter(PrintSettings printSettings) { return new DateStringPrinter(printSettings); } @@ -56,7 +57,7 @@ public Printer getStringPrinter(PrintSettings printSettings) { } @Override - public Printer getMoneyPrinter(PrintSettings printSettings) { + public Printer getMoneyPrinter(PrintSettings printSettings) { return new MoneyStringPrinter(printSettings); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MoneyStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MoneyStringPrinter.java index 3579dabbbe..072ccad752 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MoneyStringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MoneyStringPrinter.java @@ -1,13 +1,15 @@ package com.bakdata.conquery.models.query.resultinfo.printers.common; +import java.math.BigDecimal; + import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import org.jetbrains.annotations.NotNull; -public record MoneyStringPrinter(PrintSettings cfg) implements Printer { +public record MoneyStringPrinter(PrintSettings cfg) implements Printer { @Override - public String apply(@NotNull Number f) { + public String apply(@NotNull BigDecimal f) { if (cfg.isPrettyPrint()) { return cfg.getCurrencyFormat().format(f); diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java index da5cc444f2..497fa526b4 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java @@ -79,7 +79,7 @@ public static String readTSV(InputStream inputStream) throws IOException { return stringJoiner.toString(); } - public static String generateExpectedTSV(List results, List resultInfos, boolean isParquet) { + public static String generateExpectedTSV(List results, List resultInfos) { String expected = results.stream() .map(EntityResult.class::cast) @@ -96,7 +96,7 @@ public static String generateExpectedTSV(List results, List results, List col = (Collection) obj; // Workaround: Arrow deserializes lists as a JsonStringArrayList which has a JSON String method ResultType elemType = ((ResultType.ListT) type).getElementType(); - return col.stream().map(v -> getPrintValue(v, elemType, isParquet)).collect(Collectors.joining(",", "[", "]")); + return col.stream().map(v -> getPrintValue(v, elemType)).collect(Collectors.joining(",", "[", "]")); } return obj.toString(); } @@ -200,13 +199,7 @@ void generateFieldsValue() { ) ), new Field("STRING", FieldType.nullable(new ArrowType.Utf8()), null), - new Field("MONEY", - FieldType.nullable(new ArrowType.Decimal(38 - PRINT_SETTINGS.getCurrency().getDefaultFractionDigits(), - PRINT_SETTINGS.getCurrency().getDefaultFractionDigits(), - 128 - )), - null - ), + new Field("MONEY", FieldType.nullable(new ArrowType.Int(32, true)), null), new Field("LIST[BOOLEAN]", FieldType.nullable(ArrowType.List.INSTANCE), List.of(new Field("LIST[BOOLEAN]", FieldType.nullable(ArrowType.Bool.INSTANCE), null)) @@ -264,7 +257,7 @@ void writeAndRead() throws IOException { String computed = readTSV(inputStream); assertThat(computed).isNotBlank(); - assertThat(computed).isEqualTo(generateExpectedTSV(results, mquery.getResultInfos(), false)); + assertThat(computed).isEqualTo(generateExpectedTSV(results, mquery.getResultInfos())); } diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java index 361593b117..0b6a5e4f85 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java @@ -72,7 +72,7 @@ void generateSchema() { .optional(INT32).as(LogicalTypeAnnotation.dateType()).named("max") .named("DATE_RANGE") .optional(BINARY).as(stringType()).named("STRING") - .optional(BINARY).as(LogicalTypeAnnotation.decimalType(PRINT_SETTINGS.getCurrency().getDefaultFractionDigits(), 38 - PRINT_SETTINGS.getCurrency().getDefaultFractionDigits())).named("MONEY") + .optional(INT32).as(LogicalTypeAnnotation.intType(32, true)).named("MONEY") .optionalGroup().as(LogicalTypeAnnotation.listType()) .repeatedGroup() .optional(BOOLEAN).named("element") @@ -139,7 +139,7 @@ void writeAndRead() throws IOException { log.info("\n{}", actual); - assertThat(actual).isEqualTo(ArrowResultGenerationTest.generateExpectedTSV(results, managedQuery.getResultInfos(), true)); + assertThat(actual).isEqualTo(ArrowResultGenerationTest.generateExpectedTSV(results, managedQuery.getResultInfos())); } From 88224e58b77a0f3f79771d22830e4c6f2cd1350c Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 5 Sep 2024 11:53:12 +0200 Subject: [PATCH 51/54] fixes MoneyPrinter being fixed to BigDecimal causing issues with deserialization --- .../resultinfo/printers/ArrowResultPrinters.java | 12 +++++------- .../resultinfo/printers/ExcelResultPrinters.java | 10 ++++------ .../resultinfo/printers/JavaResultPrinters.java | 9 ++++----- .../resultinfo/printers/JsonResultPrinters.java | 2 +- .../query/resultinfo/printers/PrinterFactory.java | 9 ++++----- .../resultinfo/printers/StringResultPrinters.java | 9 ++++----- .../printers/common/MoneyStringPrinter.java | 6 ++---- 7 files changed, 24 insertions(+), 33 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java index 1fd3723f17..e535e910e3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.models.query.resultinfo.printers; -import java.math.BigDecimal; - import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.common.IdentityPrinter; import org.jetbrains.annotations.NotNull; @@ -9,19 +7,19 @@ public class ArrowResultPrinters extends JavaResultPrinters { @Override - public Printer getDatePrinter(PrintSettings printSettings) { + public Printer getDatePrinter(PrintSettings printSettings) { return new IdentityPrinter<>(); } @Override - public Printer getMoneyPrinter(PrintSettings printSettings) { + public Printer getMoneyPrinter(PrintSettings printSettings) { return new MoneyPrinter(); } - private record MoneyPrinter() implements Printer { + private record MoneyPrinter() implements Printer { @Override - public Object apply(@NotNull BigDecimal value) { - return value.unscaledValue().intValueExact(); + public Object apply(@NotNull Number value) { + return value.intValue(); } } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java index 6ba25a5246..4a9e05a0ad 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ExcelResultPrinters.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.models.query.resultinfo.printers; -import java.math.BigDecimal; - import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.common.DatePrinter; import com.bakdata.conquery.models.query.resultinfo.printers.common.IdentityPrinter; @@ -39,22 +37,22 @@ public Printer getBooleanPrinter(PrintSettings printSettings) { } @Override - public Printer getNumericPrinter(PrintSettings printSettings) { + public Printer getNumericPrinter(PrintSettings printSettings) { return new IdentityPrinter<>(); } @Override - public Printer getMoneyPrinter(PrintSettings printSettings) { + public Printer getMoneyPrinter(PrintSettings printSettings) { return new IdentityPrinter<>(); } @Override - public Printer getDatePrinter(PrintSettings printSettings) { + public Printer getDatePrinter(PrintSettings printSettings) { return new DatePrinter(); } @Override - public Printer getIntegerPrinter(PrintSettings printSettings) { + public Printer getIntegerPrinter(PrintSettings printSettings) { return new IdentityPrinter<>(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java index eca3268284..bab9fc5153 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.models.query.resultinfo.printers; -import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; @@ -23,17 +22,17 @@ public Printer getBooleanPrinter(PrintSettings printSettings) { } @Override - public Printer getIntegerPrinter(PrintSettings printSettings) { + public Printer getIntegerPrinter(PrintSettings printSettings) { return new IdentityPrinter<>(); } @Override - public Printer getNumericPrinter(PrintSettings printSettings) { + public Printer getNumericPrinter(PrintSettings printSettings) { return new IdentityPrinter<>(); } @Override - public Printer getDatePrinter(PrintSettings printSettings) { + public Printer getDatePrinter(PrintSettings printSettings) { return new DatePrinter(); } @@ -49,7 +48,7 @@ public Printer getStringPrinter(PrintSettings printSettings) { @Override - public Printer getMoneyPrinter(PrintSettings printSettings) { + public Printer getMoneyPrinter(PrintSettings printSettings) { return new IdentityPrinter<>(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java index aa325571e0..69dd30b38c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java @@ -14,7 +14,7 @@ public class JsonResultPrinters extends JavaResultPrinters { @Override - public Printer getDatePrinter(PrintSettings printSettings) { + public Printer getDatePrinter(PrintSettings printSettings) { return new ToStringPrinter<>(super.getDatePrinter(printSettings)); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java index 3fdb1d72fc..2d46e73c68 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.models.query.resultinfo.printers; -import java.math.BigDecimal; import java.util.List; import com.bakdata.conquery.models.query.PrintSettings; @@ -40,18 +39,18 @@ public Printer printerFor(ResultType type, PrintSettings printSettings) { /** * Jackson will opportunistically read {@link Long} and {@link Integer} hence our usage of Number. */ - public abstract Printer getIntegerPrinter(PrintSettings printSettings); + public abstract Printer getIntegerPrinter(PrintSettings printSettings); - public abstract Printer getNumericPrinter(PrintSettings printSettings); + public abstract Printer getNumericPrinter(PrintSettings printSettings); /** * Jackson will opportunistically read {@link Long} and {@link Integer} hence our usage of Number. */ - public abstract Printer getDatePrinter(PrintSettings printSettings); + public abstract Printer getDatePrinter(PrintSettings printSettings); public abstract Printer> getDateRangePrinter(PrintSettings printSettings); public abstract Printer getStringPrinter(PrintSettings printSettings); - public abstract Printer getMoneyPrinter(PrintSettings printSettings); + public abstract Printer getMoneyPrinter(PrintSettings printSettings); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java index 5578934a57..4bcd1c0b8d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.models.query.resultinfo.printers; -import java.math.BigDecimal; import java.util.List; import com.bakdata.conquery.models.query.PrintSettings; @@ -32,17 +31,17 @@ public Printer getBooleanPrinter(PrintSettings printSettings) { } @Override - public Printer getIntegerPrinter(PrintSettings printSettings) { + public Printer getIntegerPrinter(PrintSettings printSettings) { return new IntegerStringPrinter(printSettings); } @Override - public Printer getNumericPrinter(PrintSettings printSettings) { + public Printer getNumericPrinter(PrintSettings printSettings) { return new NumericStringPrinter(printSettings); } @Override - public Printer getDatePrinter(PrintSettings printSettings) { + public Printer getDatePrinter(PrintSettings printSettings) { return new DateStringPrinter(printSettings); } @@ -57,7 +56,7 @@ public Printer getStringPrinter(PrintSettings printSettings) { } @Override - public Printer getMoneyPrinter(PrintSettings printSettings) { + public Printer getMoneyPrinter(PrintSettings printSettings) { return new MoneyStringPrinter(printSettings); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MoneyStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MoneyStringPrinter.java index 072ccad752..3579dabbbe 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MoneyStringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MoneyStringPrinter.java @@ -1,15 +1,13 @@ package com.bakdata.conquery.models.query.resultinfo.printers.common; -import java.math.BigDecimal; - import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import org.jetbrains.annotations.NotNull; -public record MoneyStringPrinter(PrintSettings cfg) implements Printer { +public record MoneyStringPrinter(PrintSettings cfg) implements Printer { @Override - public String apply(@NotNull BigDecimal f) { + public String apply(@NotNull Number f) { if (cfg.isPrettyPrint()) { return cfg.getCurrencyFormat().format(f); From efa20bad53fdb8b301f0c8e4f8c74a7fb0400f78 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 5 Sep 2024 12:28:57 +0200 Subject: [PATCH 52/54] skip rescaling of money values --- frontend/src/js/entity-history/timeline/EventCard.tsx | 2 +- frontend/src/js/entity-history/timeline/GroupedContent.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/js/entity-history/timeline/EventCard.tsx b/frontend/src/js/entity-history/timeline/EventCard.tsx index ce0d50c8b9..9fed38d023 100644 --- a/frontend/src/js/entity-history/timeline/EventCard.tsx +++ b/frontend/src/js/entity-history/timeline/EventCard.tsx @@ -161,7 +161,7 @@ const EventCard = ({ {...currencyConfig} suffix={" " + currencyConfig.unit} displayType="text" - value={parseInt(row[column.label] as string) / 100} + value={parseFloat(row[column.label] as string)} /> diff --git a/frontend/src/js/entity-history/timeline/GroupedContent.tsx b/frontend/src/js/entity-history/timeline/GroupedContent.tsx index 9f16b1368a..2c1758ba94 100644 --- a/frontend/src/js/entity-history/timeline/GroupedContent.tsx +++ b/frontend/src/js/entity-history/timeline/GroupedContent.tsx @@ -172,7 +172,7 @@ const Cell = memo( ); } From c65d0fe60ef0fbaf449ff95d02749d8a599ce11d Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Mon, 16 Sep 2024 17:00:50 +0200 Subject: [PATCH 53/54] fixes rescaling for Money --- .../parquet/EntityResultWriteSupport.java | 7 +- .../io/result/parquet/ParquetRenderer.java | 85 +++++++++---------- .../printers/ArrowResultPrinters.java | 6 ++ 3 files changed, 51 insertions(+), 47 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java index bc103231c5..a0480e7dfd 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java @@ -61,8 +61,7 @@ public static MessageType generateSchema(List idHeaders, List generateColumnConsumers(List idH return consumers; } - private static List generateColumnPrinters(List idHeaders, List resultInfos, PrintSettings printSettings, - PrinterFactory printerFactory) { + private static List generateColumnPrinters( + List idHeaders, List resultInfos, PrintSettings printSettings, PrinterFactory printerFactory) { final List consumers = new ArrayList<>(); for (ResultInfo idHeader : idHeaders) { consumers.add(idHeader.createPrinter(printerFactory, printSettings)); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ParquetRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ParquetRenderer.java index e5f7a32a82..a903133ff6 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ParquetRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ParquetRenderer.java @@ -24,51 +24,8 @@ @UtilityClass public class ParquetRenderer { - @RequiredArgsConstructor - public static class StreamOutputFile implements OutputFile { - - private final PositionTrackingOutputStream outputStream; - - @Override - public PositionOutputStream create(long blockSizeHint) throws IOException { - return outputStream; - } - - @Override - public PositionOutputStream createOrOverwrite(long blockSizeHint) throws IOException { - return outputStream; - } - - @Override - public boolean supportsBlockSize() { - return false; - } - - @Override - public long defaultBlockSize() { - return 0; - } - } - - @RequiredArgsConstructor - public static class PositionTrackingOutputStream extends PositionOutputStream { - - private final CountingOutputStream stream; - - @Override - public long getPos() throws IOException { - return stream.getCount(); - } - - @Override - public void write(int b) throws IOException { - stream.write(b); - } - } - public static void writeToStream( OutputStream outputStream, - List idHeaders, List resultInfo, PrintSettings printSettings, @@ -111,5 +68,47 @@ private static Stream convertToSingleLine(EntityResult e return entityResult.streamValues().map(line -> new SinglelineEntityResult(entityResult.getEntityId(), line)); } + @RequiredArgsConstructor + public static class StreamOutputFile implements OutputFile { + + private final PositionTrackingOutputStream outputStream; + + @Override + public PositionOutputStream create(long blockSizeHint) throws IOException { + return outputStream; + } + + @Override + public PositionOutputStream createOrOverwrite(long blockSizeHint) { + return outputStream; + } + + @Override + public boolean supportsBlockSize() { + return false; + } + + @Override + public long defaultBlockSize() { + return 0; + } + } + + @RequiredArgsConstructor + public static class PositionTrackingOutputStream extends PositionOutputStream { + + private final CountingOutputStream stream; + + @Override + public long getPos() { + return stream.getCount(); + } + + @Override + public void write(int b) throws IOException { + stream.write(b); + } + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java index e535e910e3..9e1a4ecffe 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.query.resultinfo.printers; +import java.math.BigDecimal; + import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.common.IdentityPrinter; import org.jetbrains.annotations.NotNull; @@ -19,6 +21,10 @@ public Printer getMoneyPrinter(PrintSettings printSettings) { private record MoneyPrinter() implements Printer { @Override public Object apply(@NotNull Number value) { + if (value instanceof BigDecimal bigDecimal){ + return bigDecimal.unscaledValue().intValueExact(); + } + return value.intValue(); } } From 484acf2b6bff6028d0d9cd01869641d40ed51ab7 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Mon, 16 Sep 2024 17:20:12 +0200 Subject: [PATCH 54/54] simplify printer usage --- .../io/result/arrow/ArrowRenderer.java | 21 +++++----- .../parquet/EntityResultWriteSupport.java | 39 ++++++++----------- 2 files changed, 25 insertions(+), 35 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java index f0caac037b..d1f84adf31 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java @@ -3,7 +3,6 @@ import static com.bakdata.conquery.io.result.arrow.ArrowUtil.ROOT_ALLOCATOR; import java.io.IOException; -import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.function.Function; @@ -56,15 +55,10 @@ public static void renderToStream( final RowConsumer[] idWriters = generateWriterPipeline(root, 0, idHeaders.size(), idHeaders); final RowConsumer[] valueWriter = generateWriterPipeline(root, idHeaders.size(), resultInfo.size(), resultInfo); - final List printers = new ArrayList<>(); - - for (ResultInfo header : idHeaders) { - printers.add(header.createPrinter(printerFactory, printSettings)); - } - - for (ResultInfo info : resultInfo) { - printers.add(info.createPrinter(printerFactory, printSettings)); - } + final List printers = + Stream.concat(idHeaders.stream(), resultInfo.stream()) + .map(info -> info.createPrinter(printerFactory, printSettings)) + .toList(); // Write the data try (ArrowWriter writer = writerProducer.apply(root)) { @@ -92,6 +86,7 @@ public static void write( int batchCount = 0; int batchLineCount = 0; final Iterator resultIterator = results.iterator(); + while (resultIterator.hasNext()) { final EntityResult cer = resultIterator.next(); @@ -120,7 +115,8 @@ public static void write( Object printed = null; if (value != null) { - printed = printers.get(colId).apply(value); + Printer printer = printers.get(colId); + printed = printer.apply(value); } valueWriters[index].accept(batchLineCount, printed); @@ -153,7 +149,8 @@ private static Object[] getPrintedExternalId(RowConsumer[] idWriters, PrintIdMap final Object[] printedExternalId = new String[externalId.length]; for (int index = 0; index < idWriters.length; index++) { - printedExternalId[index] = printers.get(index).apply(externalId[index]); + Printer printer = printers.get(index); + printedExternalId[index] = printer.apply(externalId[index]); } return printedExternalId; } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java index a0480e7dfd..387eadbb09 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java @@ -1,8 +1,8 @@ package com.bakdata.conquery.io.result.parquet; -import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.Stream; import com.bakdata.conquery.io.result.arrow.ArrowUtil; import com.bakdata.conquery.models.common.daterange.CDateRange; @@ -69,34 +69,25 @@ public static MessageType generateSchema(List idHeaders, List generateColumnConsumers(List idHeaders, List resultInfos) { - final List consumers = new ArrayList<>(); - for (ResultInfo idHeader : idHeaders) { - consumers.add(getForResultType(idHeader.getType())); - } + return Stream.concat(idHeaders.stream(), resultInfos.stream()) + .map(ResultInfo::getType) + .map(EntityResultWriteSupport::columnConsumerForType) + .toList(); - for (ResultInfo resultInfo : resultInfos) { - consumers.add(getForResultType(resultInfo.getType())); - } - return consumers; } - private static List generateColumnPrinters( - List idHeaders, List resultInfos, PrintSettings printSettings, PrinterFactory printerFactory) { - final List consumers = new ArrayList<>(); - for (ResultInfo idHeader : idHeaders) { - consumers.add(idHeader.createPrinter(printerFactory, printSettings)); - } + private static List generateColumnPrinters(List idHeaders, List resultInfos, PrintSettings printSettings, PrinterFactory printerFactory) { + + return Stream.concat(idHeaders.stream(), resultInfos.stream()) + .map(info -> info.createPrinter(printerFactory, printSettings)) + .toList(); - for (ResultInfo resultInfo : resultInfos) { - consumers.add(resultInfo.createPrinter(printerFactory, printSettings)); - } - return consumers; } - private static ColumnConsumer getForResultType(ResultType resultType) { + private static ColumnConsumer columnConsumerForType(ResultType resultType) { if (resultType instanceof ResultType.ListT listT) { - return new ListColumnConsumer(getForResultType(listT.getElementType())); + return new ListColumnConsumer(columnConsumerForType(listT.getElementType())); } return switch (((ResultType.Primitive) resultType)) { @@ -162,7 +153,8 @@ public void write(EntityResult record) { continue; } - final Object printed = columnPrinters.get(colId).apply(value); + Printer printer = columnPrinters.get(colId); + final Object printed = printer.apply(value); final String fieldName = schema.getFieldName(colId); @@ -183,7 +175,8 @@ private Object[] getPrintedExternalId(EntityResult record) { final Object[] printedExternalId = new String[externalId.length]; for (int index = 0; index < externalId.length; index++) { - printedExternalId[index] = columnPrinters.get(index).apply(externalId[index]); + Printer printer = columnPrinters.get(index); + printedExternalId[index] = printer.apply(externalId[index]); } return printedExternalId; }