diff --git a/backend/pom.xml b/backend/pom.xml index 3df1683371..ab20155260 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -155,7 +155,7 @@ commons-io commons-io - 2.14.0 + 2.18.0 com.univocity diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java index 3c034c2d1e..f931b94d03 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java @@ -90,7 +90,7 @@ public boolean isSearchDisabled() { public TrieSearch createTrieSearch(IndexConfig config) throws IndexCreationException { final URI resolvedURI = FileUtil.getResolvedUri(config.getBaseUrl(), getFilePath()); - log.trace("Resolved filter template reference url for search '{}': {}", this.getId(), resolvedURI); + log.trace("Resolved filter template reference url for search '{}': {}", getId(), resolvedURI); final FrontendValueIndex search = indexService.getIndex(new FrontendValueIndexKey( resolvedURI, @@ -101,7 +101,7 @@ public TrieSearch createTrieSearch(IndexConfig config) throws Ind config.getSearchSplitChars() )); - return search; + return search.getDelegate(); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/Jackson.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/Jackson.java index da9b1a45a1..5ba5177574 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/Jackson.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/Jackson.java @@ -33,48 +33,46 @@ public class Jackson { /** * Helper method that also creates a copy of the injected values to reduce side effects. + * * @param om the {@link ObjectMapper} which is copied. Its {@link com.fasterxml.jackson.databind.InjectableValues} must be {@link MutableInjectableValues} * @return A copy of the {@link ObjectMapper} along with a copy of its {@link MutableInjectableValues}. */ public static ObjectMapper copyMapperAndInjectables(ObjectMapper om) { final ObjectMapper copy = om.copy(); - copy.setInjectableValues(((MutableInjectableValues)copy.getInjectableValues()).copy()); + copy.setInjectableValues(((MutableInjectableValues) copy.getInjectableValues()).copy()); return copy; } - public static T configure(T objectMapper){ + public static T configure(T objectMapper) { - objectMapper - .enable(MapperFeature.PROPAGATE_TRANSIENT_MARKER) - .enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY) - .enable(Feature.ALLOW_UNQUOTED_FIELD_NAMES) - .enable(Feature.ALLOW_COMMENTS) - .enable(Feature.ALLOW_UNQUOTED_CONTROL_CHARS) - //TODO this is just a hotfix to avoid reimports - // .enable(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES) - .enable(DeserializationFeature.FAIL_ON_INVALID_SUBTYPE) - .enable(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES) - .enable(DeserializationFeature.FAIL_ON_NUMBERS_FOR_ENUMS) - .enable(DeserializationFeature.FAIL_ON_READING_DUP_TREE_KEY) - .enable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) - .enable(DeserializationFeature.FAIL_ON_UNRESOLVED_OBJECT_IDS) - .enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS) - .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) - .setLocale(Locale.ROOT) - .disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET) - .enable(SerializationFeature.WRITE_EMPTY_JSON_ARRAYS) - .enable(SerializationFeature.WRITE_NULL_MAP_VALUES) - .registerModule(new JavaTimeModule()) - .registerModule(new ParameterNamesModule()) - .registerModule(new GuavaModule()) - .registerModule(new BlackbirdModule()) - .registerModule(ConquerySerializersModule.INSTANCE) - .setSerializationInclusion(Include.ALWAYS) - .setDefaultPropertyInclusion(Include.ALWAYS) - //.setAnnotationIntrospector(new RestrictingAnnotationIntrospector()) - .setInjectableValues(new MutableInjectableValues()) - .addMixIn(Permission.class, ConqueryPermission.class) - .addMixIn(Object2IntMap.class, Object2IntMapMixin.class); + objectMapper.enable(MapperFeature.PROPAGATE_TRANSIENT_MARKER) + .enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY) + .enable(Feature.ALLOW_UNQUOTED_FIELD_NAMES) + .enable(Feature.ALLOW_COMMENTS) + .enable(Feature.ALLOW_UNQUOTED_CONTROL_CHARS) + .enable(DeserializationFeature.FAIL_ON_INVALID_SUBTYPE) + .enable(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES) + .enable(DeserializationFeature.FAIL_ON_NUMBERS_FOR_ENUMS) + .enable(DeserializationFeature.FAIL_ON_READING_DUP_TREE_KEY) + .enable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) + .enable(DeserializationFeature.FAIL_ON_UNRESOLVED_OBJECT_IDS) + .enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS) + .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) + .setLocale(Locale.ROOT) + .disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET) + .enable(SerializationFeature.WRITE_EMPTY_JSON_ARRAYS) + .enable(SerializationFeature.WRITE_NULL_MAP_VALUES) + .registerModule(new JavaTimeModule()) + .registerModule(new ParameterNamesModule()) + .registerModule(new GuavaModule()) + .registerModule(new BlackbirdModule()) + .registerModule(ConquerySerializersModule.INSTANCE) + .setSerializationInclusion(Include.ALWAYS) + .setDefaultPropertyInclusion(Include.ALWAYS) + //.setAnnotationIntrospector(new RestrictingAnnotationIntrospector()) + .setInjectableValues(new MutableInjectableValues()) + .addMixIn(Permission.class, ConqueryPermission.class) + .addMixIn(Object2IntMap.class, Object2IntMapMixin.class); return objectMapper; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java index 9d32279997..2014febe7f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java @@ -1,5 +1,9 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; + import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.MappableSingleColumnSelect; @@ -10,7 +14,7 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.AllValuesAggregator; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; -import com.bakdata.conquery.models.query.resultinfo.printers.common.MappedPrinter; +import com.bakdata.conquery.models.query.resultinfo.printers.common.OneToManyMappingPrinter; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.sql.conversion.model.select.DistinctSelectConverter; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; @@ -20,8 +24,7 @@ public class DistinctSelect extends MappableSingleColumnSelect { @JsonCreator - public DistinctSelect(ColumnId column, - InternToExternMapperId mapping) { + public DistinctSelect(ColumnId column, InternToExternMapperId mapping) { super(column, mapping); } @@ -37,15 +40,33 @@ public SelectConverter createConverter() { @Override public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printSettings) { - if(getMapping() == null){ + if (getMapping() == null) { return super.createPrinter(printerFactory, printSettings); } - return printerFactory.getListPrinter(new MappedPrinter(getMapping().resolve()), printSettings); + return new FlatMappingPrinter(new OneToManyMappingPrinter(getMapping().resolve())) + .andThen(printerFactory.getListPrinter(printerFactory.getStringPrinter(printSettings), printSettings)); } @Override public ResultType getResultType() { return new ResultType.ListT<>(super.getResultType()); } + + /** + * Ensures that mapped values are still distinct. + */ + private record FlatMappingPrinter(OneToManyMappingPrinter mapper) implements Printer> { + + @Override + public Collection apply(Collection values) { + final Set out = new HashSet<>(); + + for (String value : values) { + mapper.apply(value).forEach(out::add); + } + + return out; + } + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/FirstValueSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/FirstValueSelect.java index 0ca305e6aa..2ece9f3861 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/FirstValueSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/FirstValueSelect.java @@ -31,4 +31,5 @@ public Aggregator createAggregator() { public SelectConverter createConverter() { return new FirstValueSelectConverter(); } + } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java index d6eab22730..a67905f781 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java @@ -10,11 +10,11 @@ import com.bakdata.conquery.models.datasets.concepts.select.connector.SingleColumnSelect; import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; import com.bakdata.conquery.models.identifiable.ids.specific.InternToExternMapperId; +import com.bakdata.conquery.models.index.InternToExternMapper; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; -import com.bakdata.conquery.models.query.resultinfo.printers.common.MappedPrinter; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import lombok.Getter; @@ -42,15 +42,9 @@ public Printer createPrinter(PrinterFactory printerFactory, PrintSettings pri return super.createPrinter(printerFactory, printSettings); } - return new MappedPrinter(mapping.resolve()); - } + final InternToExternMapper resolvedMapping = mapping.resolve(); - @Override - public ResultType getResultType() { - if(mapping == null){ - return ResultType.resolveResultType(getColumn().resolve().getType()); - } - return ResultType.Primitive.STRING; + return resolvedMapping.createPrinter(printerFactory, printSettings); } @Override @@ -63,6 +57,14 @@ public SelectResultInfo getResultInfo(CQConcept cqConcept) { return new SelectResultInfo(this, cqConcept, Set.of(new SemanticType.CategoricalT())); } + @Override + public ResultType getResultType() { + if(mapping == null){ + return ResultType.resolveResultType(getColumn().resolve().getType()); + } + return ResultType.Primitive.STRING; + } + public void loadMapping() { if (mapping != null) { mapping.resolve().init(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/AbstractIndexKey.java b/backend/src/main/java/com/bakdata/conquery/models/index/AbstractIndexKey.java deleted file mode 100644 index df6b9cee45..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/index/AbstractIndexKey.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.bakdata.conquery.models.index; - -import java.net.URI; - -import lombok.Data; - -/** - * Abstract base class for {@link IndexKey} which is used by {@link IndexService} to create and cache a new {@link Index}. - * - * @param The type of Index that is indexed by this key - */ -@Data -public abstract class AbstractIndexKey>> implements IndexKey { - private final URI csv; - private final String internalColumn; -} diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java b/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java index 80c797e39b..25ad46e191 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java @@ -1,19 +1,22 @@ package com.bakdata.conquery.models.index; +import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Set; import com.bakdata.conquery.apiv1.FilterTemplate; import com.bakdata.conquery.apiv1.frontend.FrontendValue; import com.bakdata.conquery.models.query.FilterSearch; import com.bakdata.conquery.util.search.TrieSearch; +import lombok.Getter; import lombok.ToString; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.time.StopWatch; @Slf4j @ToString -public class FrontendValueIndex extends TrieSearch implements Index { +public class FrontendValueIndex implements Index { /** @@ -28,11 +31,15 @@ public class FrontendValueIndex extends TrieSearch implements Ind private final String optionValueTemplate; private final String defaultEmptyLabel; - public FrontendValueIndex(int suffixCutoff, String split, String valueTemplate, String optionValueTemplate, String defaultEmptyLabel1) { - super(suffixCutoff, split); + @Getter + private final TrieSearch delegate; + + public FrontendValueIndex(int suffixCutoff, String split, String valueTemplate, String optionValueTemplate, String defaultEmptyLabel) { this.valueTemplate = valueTemplate; this.optionValueTemplate = optionValueTemplate; - this.defaultEmptyLabel = defaultEmptyLabel1; + this.defaultEmptyLabel = defaultEmptyLabel; + + delegate = new TrieSearch<>(suffixCutoff, split); } @Override @@ -43,12 +50,12 @@ public void put(String internalValue, Map templateToConcrete) { templateToConcrete.get(optionValueTemplate) ); - addItem(feValue, FilterSearch.extractKeywords(feValue)); + delegate.addItem(feValue, FilterSearch.extractKeywords(feValue)); } @Override public int size() { - final long longSize = calculateSize(); + final long longSize = delegate.calculateSize(); if (longSize > Integer.MAX_VALUE) { log.trace("Trie size was larger than an int. Reporting Integer.MAX_VALUE. Was actually: {}", longSize); return Integer.MAX_VALUE; @@ -56,15 +63,34 @@ public int size() { return (int) longSize; } + @Override + public Collection externalMultiple(String key) { + final List matches = delegate.findExact(Set.of(key), Integer.MAX_VALUE); + if (matches.isEmpty()) { + return null; + } + return matches; + } + + @Override + public FrontendValue external(String key) { + final List matches = delegate.findExact(Set.of(key), 1); + + if (matches.isEmpty()) { + return null; + } + + return matches.iterator().next(); + } @Override public void finalizer() { - StopWatch timer = StopWatch.createStarted(); + final StopWatch timer = StopWatch.createStarted(); // If no empty label was provided by the mapping, we insert the configured default-label - if (findExact(List.of(""), 1).isEmpty()) { - addItem(new FrontendValue("", defaultEmptyLabel), List.of(defaultEmptyLabel)); + if (delegate.findExact(List.of(""), 1).isEmpty()) { + delegate.addItem(new FrontendValue("", defaultEmptyLabel), List.of(defaultEmptyLabel)); } log.trace("DONE-FINALIZER ADDING_ITEMS in {}", timer); @@ -72,7 +98,7 @@ public void finalizer() { timer.reset(); log.trace("START-FV-FIN SHRINKING"); - shrinkToFit(); + delegate.shrinkToFit(); log.trace("DONE-FV-FIN SHRINKING in {}", timer); diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndexKey.java b/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndexKey.java index c8b5e7e400..edac023503 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndexKey.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndexKey.java @@ -5,11 +5,12 @@ import com.bakdata.conquery.apiv1.FilterTemplate; import lombok.EqualsAndHashCode; +import lombok.Getter; import lombok.ToString; -@EqualsAndHashCode(callSuper = true) +@EqualsAndHashCode @ToString -public class FrontendValueIndexKey extends AbstractIndexKey { +public class FrontendValueIndexKey implements IndexKey { private final int suffixCutoff; @@ -27,15 +28,20 @@ public class FrontendValueIndexKey extends AbstractIndexKey * @see FilterTemplate#getOptionValue() */ private final String optionValueTemplate; + @Getter + private final URI csv; + @Getter + private final String internalColumn; public FrontendValueIndexKey(URI csv, String internalColumn, String valueTemplate, String optionValueTemplate, int suffixCutoff, String splitPattern) { - super(csv, internalColumn); this.suffixCutoff = suffixCutoff; this.splitPattern = splitPattern; this.valueTemplate = valueTemplate; this.optionValueTemplate = optionValueTemplate; + this.csv = csv; + this.internalColumn = internalColumn; } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/Index.java b/backend/src/main/java/com/bakdata/conquery/models/index/Index.java index a8cc959bb7..55a7aa6219 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/Index.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/Index.java @@ -1,8 +1,10 @@ package com.bakdata.conquery.models.index; +import java.util.Collection; import java.util.Map; +import javax.annotation.CheckForNull; -public interface Index>> { +public interface Index { void put(String key, Map templateToConcrete); @@ -10,4 +12,10 @@ public interface Index>> { void finalizer(); + @CheckForNull + V external(String key); + + @CheckForNull + Collection externalMultiple(String key); + } diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/IndexCreationException.java b/backend/src/main/java/com/bakdata/conquery/models/index/IndexCreationException.java index f8b85731d1..277c8d5226 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/IndexCreationException.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/IndexCreationException.java @@ -2,7 +2,7 @@ public class IndexCreationException extends Exception { - public IndexCreationException(IndexKey key, Throwable cause) { + public IndexCreationException(IndexKey key, Throwable cause) { super(String.format("Unable to build index from index configuration: %s", key), cause); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/IndexKey.java b/backend/src/main/java/com/bakdata/conquery/models/index/IndexKey.java index dbc570269d..00920a8e4d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/IndexKey.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/IndexKey.java @@ -5,12 +5,10 @@ /** * Interface which is used by {@link IndexService} to create and cache a new {@link Index}. - * For concrete implementations please use {@link AbstractIndexKey} and look out for correct * {@link Object#equals(Object)} and {@link Object#hashCode()} functions. * - * @param The type of Index that is indexed by this key */ -public interface IndexKey>> { +public interface IndexKey { /** * An url, or a part of it, that points to the referenced csv file. @@ -24,6 +22,6 @@ public interface IndexKey>> { List getExternalTemplates(); - I createIndex(String defaultEmptyLabel); + Index createIndex(String defaultEmptyLabel); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java b/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java index 5bc17b61a1..b8fd18ce82 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java @@ -38,10 +38,10 @@ public class IndexService implements Injectable { private final CsvParserSettings csvParserSettings; private final String emptyDefaultLabel; - private final LoadingCache, Index> mappings = CacheBuilder.newBuilder().recordStats().build(new CacheLoader<>() { + private final LoadingCache> mappings = CacheBuilder.newBuilder().recordStats().build(new CacheLoader<>() { @NotNull @Override - public Index load(@NotNull IndexKey key) throws Exception { + public Index load(@NotNull IndexKey key) throws Exception { final StopWatch timer = StopWatch.createStarted(); @@ -103,7 +103,7 @@ public IndexService(CsvParserSettings csvParserSettings, String emptyDefaultLabe } @Nullable - private Pair> computeInternalExternal(@NotNull IndexKey key, CsvParser csvParser, Record row) { + private Pair> computeInternalExternal(@NotNull IndexKey key, CsvParser csvParser, Record row) { final StringSubstitutor substitutor = new StringSubstitutor(row::getString, "{{", "}}", StringSubstitutor.DEFAULT_ESCAPE); final String internalValue = row.getString(key.getInternalColumn()); @@ -133,7 +133,7 @@ private Map computeTemplates(StringSubstitutor substitutor, List .collect(Collectors.toMap(Functions.identity(), value -> whitespaceMatcher.trimAndCollapseFrom(substitutor.replace(value), ' '))); } - private Map computeEmptyDefaults(IndexKey key) { + private Map computeEmptyDefaults(IndexKey key) { final StringSubstitutor substitutor = new StringSubstitutor((ignored) -> "", "{{", "}}", StringSubstitutor.DEFAULT_ESCAPE); final List externalTemplates = key.getExternalTemplates(); @@ -149,13 +149,14 @@ public void evictCache() { /** * Returns an index mapping from the information in the given key. * If the index is not yet present, it is loaded. + *

* @param key the key describing the requested index * @return the index mapping * @throws IndexCreationException if the index mapping could not be loaded. */ @SuppressWarnings("unchecked") @NotNull - public , I extends Index> I getIndex(@NotNull K key) throws IndexCreationException { + public > I getIndex(@NotNull IndexKey key) throws IndexCreationException { try { return (I) mappings.get(key); } @@ -168,7 +169,7 @@ public CacheStats getStatistics() { return mappings.stats(); } - public Set> getLoadedIndexes() { + public Set getLoadedIndexes() { return mappings.asMap().keySet(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/InternToExternMapper.java b/backend/src/main/java/com/bakdata/conquery/models/index/InternToExternMapper.java index 7a7572830b..5d7d79a357 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/InternToExternMapper.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/InternToExternMapper.java @@ -1,9 +1,16 @@ package com.bakdata.conquery.models.index; +import java.util.Collection; + import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.models.identifiable.Named; import com.bakdata.conquery.models.identifiable.ids.NamespacedIdentifiable; import com.bakdata.conquery.models.identifiable.ids.specific.InternToExternMapperId; +import com.bakdata.conquery.models.query.PrintSettings; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; +import com.bakdata.conquery.models.query.resultinfo.printers.common.OneToManyMappingPrinter; +import com.bakdata.conquery.models.query.resultinfo.printers.common.OneToOneMappingPrinter; import com.fasterxml.jackson.annotation.JsonTypeInfo; @CPSBase @@ -14,8 +21,21 @@ public interface InternToExternMapper extends NamespacedIdentifiable externalMultiple(String key); @Override InternToExternMapperId getId(); + + default Printer createPrinter(PrinterFactory printerFactory, PrintSettings printSettings) { + if (isAllowMultiple()) { + return new OneToManyMappingPrinter(this) + .andThen(printerFactory.getListPrinter(printerFactory.getStringPrinter(printSettings), printSettings)); + } + + return new OneToOneMappingPrinter(this, printerFactory.getStringPrinter(printSettings)); + } + + boolean isAllowMultiple(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/MapIndex.java b/backend/src/main/java/com/bakdata/conquery/models/index/MapIndex.java index f6874f5f36..599275e7e9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/MapIndex.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/MapIndex.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.index; +import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -8,20 +10,40 @@ @Slf4j @RequiredArgsConstructor -public class MapIndex extends HashMap implements Index { +public class MapIndex implements Index { private final String externalTemplate; + private final HashMap delegate = new HashMap<>(); @Override public void put(String key, Map templateToConcrete) { - if (containsKey(key)) { - throw new IllegalArgumentException("The key '" + key + "' already exists in the index. Cannot map '" + key + "' -> '" + templateToConcrete + "'."); + final String prior = delegate.putIfAbsent(key, templateToConcrete.get(externalTemplate)); + if (prior != null) { + throw new IllegalArgumentException("Duplicate entry for key %s".formatted(key)); } - super.put(key, templateToConcrete.get(externalTemplate)); + } + + @Override + public int size() { + return delegate.size(); } @Override public void finalizer() { // Nothing to finalize } + + @Override + public Collection externalMultiple(String key) { + if (delegate.containsKey(key)) { + return Collections.singleton(delegate.get(key)); + } + + return null; + } + + @Override + public String external(String key) { + return delegate.get(key); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/MapIndexKey.java b/backend/src/main/java/com/bakdata/conquery/models/index/MapIndexKey.java index 83809c4905..906bf17f58 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/MapIndexKey.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/MapIndexKey.java @@ -3,20 +3,17 @@ import java.net.URI; import java.util.List; -import lombok.EqualsAndHashCode; -import lombok.ToString; +import lombok.Data; -@EqualsAndHashCode(callSuper = true) -@ToString(callSuper = true) -public class MapIndexKey extends AbstractIndexKey { +@Data +public class MapIndexKey implements IndexKey { + private final URI csv; + private final String internalColumn; private final String externalTemplate; + private final boolean allowMultiple; - public MapIndexKey(URI csv, String internalColumn, String externalTemplate) { - super(csv, internalColumn); - this.externalTemplate = externalTemplate; - } @Override public List getExternalTemplates() { @@ -24,7 +21,10 @@ public List getExternalTemplates() { } @Override - public MapIndex createIndex(String defaultEmptyLabel) { + public Index createIndex(String defaultEmptyLabel) { + if (allowMultiple){ + return new MapMultiIndex(externalTemplate); + } return new MapIndex(externalTemplate); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java b/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java index b60769e303..3ae82c2361 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java @@ -2,6 +2,8 @@ import java.net.URI; +import java.util.Collection; +import java.util.Collections; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import jakarta.validation.constraints.NotEmpty; @@ -12,17 +14,18 @@ import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.identifiable.NamedImpl; -import com.bakdata.conquery.models.identifiable.ids.NamespacedIdentifiable; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.bakdata.conquery.models.identifiable.ids.specific.InternToExternMapperId; import com.bakdata.conquery.util.io.FileUtil; import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.OptBoolean; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import lombok.AccessLevel; import lombok.EqualsAndHashCode; import lombok.Getter; -import lombok.RequiredArgsConstructor; +import lombok.NoArgsConstructor; import lombok.Setter; import lombok.ToString; import lombok.experimental.FieldNameConstants; @@ -31,15 +34,29 @@ @Slf4j @CPSType(id = "CSV_MAP", base = InternToExternMapper.class) -@RequiredArgsConstructor @ToString(onlyExplicitlyIncluded = true) @FieldNameConstants @Getter -@JsonDeserialize(converter = MapInternToExternMapper.Initializer.class ) +@JsonDeserialize(converter = MapInternToExternMapper.Initializer.class) @EqualsAndHashCode(callSuper = true) -public class MapInternToExternMapper extends NamedImpl implements InternToExternMapper, NamespacedIdentifiable, Initializing { +@NoArgsConstructor(access = AccessLevel.PRIVATE, onConstructor_ = {@JsonCreator}) +@Setter +public class MapInternToExternMapper extends NamedImpl implements InternToExternMapper, Initializing { + @ToString.Include + @NotEmpty + private String name; + @ToString.Include + @NotNull + private URI csv; + @ToString.Include + @NotEmpty + private String internalColumn; + @ToString.Include + @NotEmpty + private String externalTemplate; + private boolean allowMultiple; // We inject the service as a non-final property so, jackson will never try to create a serializer for it (in contrast to constructor injection) @JsonIgnore @JacksonInject(useInput = OptBoolean.FALSE) @@ -47,44 +64,34 @@ public class MapInternToExternMapper extends NamedImpl i @Setter(onMethod_ = @TestOnly) @EqualsAndHashCode.Exclude private IndexService mapIndex; - @JsonIgnore @JacksonInject(useInput = OptBoolean.FALSE) @NotNull @Setter(onMethod_ = @TestOnly) @EqualsAndHashCode.Exclude private ConqueryConfig config; - @JsonIgnore @JacksonInject(useInput = OptBoolean.FALSE) @NotNull @Setter(onMethod_ = @TestOnly) @EqualsAndHashCode.Exclude private NamespaceStorage storage; - @JsonIgnore @NotNull private DatasetId dataset; - - @ToString.Include - @NotEmpty - private final String name; - @ToString.Include - @NotNull - private final URI csv; - @ToString.Include - @NotEmpty - private final String internalColumn; - @ToString.Include - @NotEmpty - private final String externalTemplate; - - //Manager only @JsonIgnore @Getter(onMethod_ = {@TestOnly}) @EqualsAndHashCode.Exclude - private CompletableFuture int2ext = null; + private CompletableFuture> int2ext; + + public MapInternToExternMapper(@NotEmpty String name, @NotNull URI csv, @NotEmpty String internalColumn, @NotEmpty String externalTemplate, boolean allowMultiple) { + this.name = name; + this.csv = csv; + this.internalColumn = internalColumn; + this.externalTemplate = externalTemplate; + this.allowMultiple = allowMultiple; + } @Override @@ -98,24 +105,58 @@ public synchronized void init() { dataset = storage.getDataset().getId(); final URI resolvedURI = FileUtil.getResolvedUri(config.getIndex().getBaseUrl(), csv); - log.trace("Resolved mapping reference csv url '{}': {}", this.getId(), resolvedURI); + log.trace("Resolved mapping reference csv url '{}': {}", getId(), resolvedURI); - MapIndexKey key = new MapIndexKey(resolvedURI, internalColumn, externalTemplate); + final IndexKey key = new MapIndexKey(resolvedURI, internalColumn, externalTemplate, allowMultiple); int2ext = CompletableFuture.supplyAsync(() -> { + try { + return mapIndex.>getIndex(key); + } + catch (IndexCreationException e) { + throw new IllegalStateException(e); + } + }) + .whenComplete((m, e) -> { + if (e != null) { + log.warn("Unable to get index: {} (enable TRACE for exception)", key, log.isTraceEnabled() ? e : null); + } + }); + } + + @Override + public Collection externalMultiple(String internalValue) { + if (indexAvailable()) { try { - return mapIndex.getIndex(key); - } - catch (IndexCreationException e) { - throw new IllegalStateException(e); + final Collection mapped = int2ext.get().externalMultiple(internalValue); + + if (mapped == null) { + return Collections.singleton(internalValue); + } + + return mapped; } - }).whenComplete((m, e) -> { - if (e != null) { - log.warn("Unable to get index: {} (enable TRACE for exception)", key, (Exception) (log.isTraceEnabled() ? e : null)); + catch (InterruptedException | ExecutionException e) { + // Should never be reached + log.warn("Unable to resolve mapping for internal value {} (enable TRACE for exception)", internalValue, log.isTraceEnabled() ? e : null); } - }); + } + + return Collections.singleton(internalValue); } + private boolean indexAvailable() { + if (!initialized()) { + log.trace("Mapping {} not available, because mapper is not yet initialized", getId()); + return false; + } + + if (int2ext.isCompletedExceptionally() || int2ext.isCancelled()) { + log.trace("Mapping {} not available, because mapper could not be initialized", getId()); + return false; + } + return true; + } @Override public boolean initialized() { @@ -124,23 +165,17 @@ public boolean initialized() { @Override public String external(String internalValue) { - if(!initialized()){ - log.trace("Skip mapping for value '{}', because mapper is not initialized", internalValue); - return internalValue; - } - - if (int2ext.isCompletedExceptionally() || int2ext.isCancelled()) { - log.trace("Skip mapping for value '{}', because mapper could not be initialized", internalValue); - return internalValue; + if (indexAvailable()) { + try { + return int2ext.get().external(internalValue); + } + catch (InterruptedException | ExecutionException e) { + // Should never be reached + log.warn("Unable to resolve mapping for internal value {} (enable TRACE for exception)", internalValue, log.isTraceEnabled() ? e : null); + } } - try { - return int2ext.get().getOrDefault(internalValue, internalValue); - } catch (InterruptedException | ExecutionException e) { - // Should never be reached - log.warn("Unable to resolve mapping for internal value {} (enable TRACE for exception)", internalValue, (Exception) (log.isTraceEnabled() ? e : null)); - return internalValue; - } + return internalValue; } @Override @@ -148,5 +183,6 @@ public InternToExternMapperId createId() { return new InternToExternMapperId(getDataset(), getName()); } - public static class Initializer extends Initializing.Converter {} + public static class Initializer extends Initializing.Converter { + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/MapMultiIndex.java b/backend/src/main/java/com/bakdata/conquery/models/index/MapMultiIndex.java new file mode 100644 index 0000000000..35b01bfff4 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/index/MapMultiIndex.java @@ -0,0 +1,52 @@ +package com.bakdata.conquery.models.index; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@RequiredArgsConstructor +public class MapMultiIndex implements Index { + + private final String externalTemplate; + private final Map> delegate = new HashMap<>(); + + @Override + public void put(String key, Map templateToConcrete) { + delegate.computeIfAbsent(key, (ignored) -> new HashSet<>()) + .add(templateToConcrete.get(externalTemplate)); + } + + @Override + public int size() { + return delegate.size(); + } + + @Override + public void finalizer() { + // Nothing to finalize + } + + @Override + public String external(String key) { + if (!delegate.containsKey(key)) { + return null; + } + + return delegate.get(key).iterator().next(); + } + + @Override + public Collection externalMultiple(String key) { + if (!delegate.containsKey(key)) { + return null; + } + + return delegate.get(key); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SecondaryIdResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SecondaryIdResultInfo.java index 63d1812332..e38aa8a941 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SecondaryIdResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SecondaryIdResultInfo.java @@ -6,7 +6,6 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import com.bakdata.conquery.models.query.resultinfo.printers.PrinterFactory; -import com.bakdata.conquery.models.query.resultinfo.printers.common.MappedPrinter; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import lombok.Getter; @@ -23,8 +22,6 @@ public SecondaryIdResultInfo(SecondaryIdDescription secondaryId) { super(Set.of(new SemanticType.SecondaryIdT(secondaryId.getId()))); this.secondaryId = secondaryId; type = ResultType.Primitive.STRING; - - } @Override @@ -37,9 +34,8 @@ public Printer createPrinter(PrinterFactory printerFactory, PrintSettings pri if (secondaryId.getMapping() == null) { return printerFactory.getStringPrinter(printSettings); } - else { - return new MappedPrinter(secondaryId.getMapping().resolve()); - } + + return secondaryId.getMapping().resolve().createPrinter(printerFactory, printSettings); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java index bab9fc5153..5e9275df3a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.models.query.resultinfo.printers; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import com.bakdata.conquery.models.common.daterange.CDateRange; @@ -12,7 +13,7 @@ public class JavaResultPrinters extends PrinterFactory { @Override - public Printer> getListPrinter(Printer elementPrinter, PrintSettings printSettings) { + public Printer> getListPrinter(Printer elementPrinter, PrintSettings printSettings) { return new ListPrinter<>(elementPrinter); } @@ -52,13 +53,18 @@ public Printer getMoneyPrinter(PrintSettings printSettings) { return new IdentityPrinter<>(); } - private record ListPrinter(Printer elementPrinter) implements Printer> { + private record ListPrinter(Printer elementPrinter) implements Printer> { @Override - public Object apply(@NotNull List value) { + public Object apply(@NotNull Collection value) { final List out = new ArrayList<>(value.size()); for (T elt : value) { + if (elt == null){ + // Printers do not handle null. + continue; + } + out.add(elementPrinter.apply(elt)); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java index b165df89ea..9b539646df 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java @@ -12,4 +12,9 @@ @FunctionalInterface public interface Printer extends Function { Object apply(@NotNull T value); + + @NotNull + default Printer andThen(@NotNull Printer after) { + return (T t) -> after.apply((V) apply(t)); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java index 2d46e73c68..2cb91531c2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/PrinterFactory.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.query.resultinfo.printers; +import java.util.Collection; import java.util.List; import com.bakdata.conquery.models.query.PrintSettings; @@ -32,7 +33,7 @@ public Printer printerFor(ResultType type, PrintSettings printSettings) { }; } - public abstract Printer> getListPrinter(Printer elementPrinter, PrintSettings printSettings); + public abstract Printer> getListPrinter(Printer elementPrinter, PrintSettings printSettings); public abstract Printer getBooleanPrinter(PrintSettings printSettings); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java index cbd74ddfaf..5b4889922c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/StringResultPrinters.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.query.resultinfo.printers; +import java.util.Collection; import java.util.List; import com.bakdata.conquery.models.query.PrintSettings; @@ -19,7 +20,7 @@ public class StringResultPrinters extends PrinterFactory { @Override - public Printer> getListPrinter(Printer elementPrinter, PrintSettings printSettings) { + public Printer> getListPrinter(Printer elementPrinter, PrintSettings printSettings) { return new ListStringPrinter<>(elementPrinter, printSettings); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java index f311075acb..218a37d71f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java @@ -1,6 +1,6 @@ package com.bakdata.conquery.models.query.resultinfo.printers.common; -import java.util.List; +import java.util.Collection; import java.util.StringJoiner; import com.bakdata.conquery.models.config.LocaleConfig; @@ -8,14 +8,14 @@ import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import org.jetbrains.annotations.NotNull; -public record ListStringPrinter(Printer elementPrinter, PrintSettings cfg, LocaleConfig.ListFormat listFormat) implements Printer> { +public record ListStringPrinter(Printer elementPrinter, PrintSettings cfg, LocaleConfig.ListFormat listFormat) implements Printer> { public ListStringPrinter(Printer elementPrinter, PrintSettings cfg) { this(elementPrinter, cfg, cfg.getListFormat()); } @Override - public String apply(@NotNull List f) { + public String apply(@NotNull Collection f) { final StringJoiner joiner = listFormat.createListJoiner(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MappedPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/OneToManyMappingPrinter.java similarity index 55% rename from backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MappedPrinter.java rename to backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/OneToManyMappingPrinter.java index d388ceb689..9786235227 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/MappedPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/OneToManyMappingPrinter.java @@ -1,13 +1,14 @@ package com.bakdata.conquery.models.query.resultinfo.printers.common; +import java.util.Collection; + import com.bakdata.conquery.models.index.InternToExternMapper; import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import org.jetbrains.annotations.NotNull; -public record MappedPrinter(InternToExternMapper mapper) implements Printer { - +public record OneToManyMappingPrinter(InternToExternMapper mapper) implements Printer { @Override - public String apply(@NotNull String f) { - return mapper.external(f); + public Collection apply(@NotNull String f) { + return mapper.externalMultiple(f); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/OneToOneMappingPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/OneToOneMappingPrinter.java new file mode 100644 index 0000000000..4495e3608f --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/OneToOneMappingPrinter.java @@ -0,0 +1,17 @@ +package com.bakdata.conquery.models.query.resultinfo.printers.common; + +import com.bakdata.conquery.models.index.InternToExternMapper; +import com.bakdata.conquery.models.query.resultinfo.printers.Printer; +import org.jetbrains.annotations.NotNull; + +public record OneToOneMappingPrinter(InternToExternMapper mapper, Printer andThen) implements Printer { + + @Override + public Object apply(@NotNull String f) { + String external = mapper.external(f); + if (external == null){ + return f; + } + return andThen.apply(external); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java b/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java index 08e353f19a..00ca78f71e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java @@ -99,7 +99,7 @@ public Collection getDatasets() { return datasets.values(); } - public Set> getLoadedIndexes() { + public Set getLoadedIndexes() { return indexService.getLoadedIndexes(); } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminProcessor.java index a84e2b6c0e..999182e671 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminProcessor.java @@ -307,7 +307,7 @@ public Object executeScript(String script) { } } - public Set> getLoadedIndexes() { + public Set getLoadedIndexes() { return datasetRegistry.getLoadedIndexes(); } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UIProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UIProcessor.java index 5325de3b01..d2ab8df297 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UIProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UIProcessor.java @@ -66,7 +66,7 @@ public UIContext getUIContext(String csrfToken) { return new UIContext(adminProcessor.getNodeProvider(), csrfToken); } - public Set> getLoadedIndexes() { + public Set getLoadedIndexes() { return getAdminProcessor().getLoadedIndexes(); } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/IndexServiceUIResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/IndexServiceUIResource.java index dd1ee509fd..1243d40526 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/IndexServiceUIResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/IndexServiceUIResource.java @@ -3,13 +3,6 @@ import static com.bakdata.conquery.resources.ResourceConstants.INDEX_SERVICE_PATH_ELEMENT; import java.util.Set; - -import com.bakdata.conquery.models.auth.web.csrf.CsrfTokenSetFilter; -import com.bakdata.conquery.models.index.IndexKey; -import com.bakdata.conquery.resources.admin.rest.UIProcessor; -import com.bakdata.conquery.resources.admin.ui.model.UIView; -import com.google.common.cache.CacheStats; -import io.dropwizard.views.common.View; import jakarta.inject.Inject; import jakarta.ws.rs.GET; import jakarta.ws.rs.Path; @@ -17,6 +10,13 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.MediaType; + +import com.bakdata.conquery.models.auth.web.csrf.CsrfTokenSetFilter; +import com.bakdata.conquery.models.index.IndexKey; +import com.bakdata.conquery.resources.admin.rest.UIProcessor; +import com.bakdata.conquery.resources.admin.ui.model.UIView; +import com.google.common.cache.CacheStats; +import io.dropwizard.views.common.View; import lombok.Data; import lombok.RequiredArgsConstructor; @@ -42,6 +42,6 @@ public View getIndexService() { @Data public static class IndexServiceUIContent { private final CacheStats stats; - private final Set> indexes; + private final Set indexes; } } diff --git a/backend/src/test/java/com/bakdata/conquery/service/IndexServiceTest.java b/backend/src/test/java/com/bakdata/conquery/service/IndexServiceTest.java index 50c301a5c3..04efdb93d7 100644 --- a/backend/src/test/java/com/bakdata/conquery/service/IndexServiceTest.java +++ b/backend/src/test/java/com/bakdata/conquery/service/IndexServiceTest.java @@ -15,8 +15,8 @@ import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.index.Index; import com.bakdata.conquery.models.index.IndexService; -import com.bakdata.conquery.models.index.MapIndex; import com.bakdata.conquery.models.index.MapInternToExternMapper; import com.bakdata.conquery.util.NonPersistentStoreFactory; import com.bakdata.conquery.util.extensions.MockServerExtension; @@ -40,6 +40,8 @@ public class IndexServiceTest { @RegisterExtension private static final MockServerExtension REF_SERVER = new MockServerExtension(ClientAndServer.startClientAndServer(), IndexServiceTest::initRefServer); + public static final String MAPPING_PATH = "/tests/aggregator/MAPPED/mapping.csv"; + private static final NamespaceStorage NAMESPACE_STORAGE = new NamespaceStorage(new NonPersistentStoreFactory(), IndexServiceTest.class.getName()); private static final Dataset DATASET = new Dataset("dataset"); private static final ConqueryConfig CONFIG = new ConqueryConfig(); @@ -49,7 +51,7 @@ public class IndexServiceTest { private static void initRefServer(ClientAndServer mockServer) { log.info("Test loading of mapping"); - try (InputStream inputStream = In.resource("/tests/aggregator/FIRST_MAPPED_AGGREGATOR/mapping.csv").asStream()) { + try (InputStream inputStream = In.resource(MAPPING_PATH).asStream()) { mockServer.when(request().withPath("/mapping.csv")) .respond(HttpResponse.response().withContentType(new MediaType("text", "csv")).withBody(inputStream.readAllBytes())); } @@ -71,13 +73,20 @@ public static void beforeAll() { @Test @Order(0) - void testLoading() throws NoSuchFieldException, IllegalAccessException, URISyntaxException, IOException { + void testLoading() throws NoSuchFieldException, IllegalAccessException, URISyntaxException, IOException, ExecutionException, InterruptedException { + log.info("Test loading of mapping"); + + try (InputStream inputStream = In.resource(MAPPING_PATH).asStream()) { + REF_SERVER.when(request().withPath("/mapping.csv")) + .respond(HttpResponse.response().withContentType(new MediaType("text", "csv")).withBody(inputStream.readAllBytes())); + } final MapInternToExternMapper mapper = new MapInternToExternMapper( "test1", - new URI("classpath:/tests/aggregator/FIRST_MAPPED_AGGREGATOR/mapping.csv"), + new URI("classpath:"+MAPPING_PATH), "internal", - "{{external}}" + "{{external}}", + false ); @@ -85,14 +94,16 @@ void testLoading() throws NoSuchFieldException, IllegalAccessException, URISynta "testUrlAbsolute", new URI(String.format("http://localhost:%d/mapping.csv", REF_SERVER.getPort())), "internal", - "{{external}}" + "{{external}}", + false ); final MapInternToExternMapper mapperUrlRelative = new MapInternToExternMapper( "testUrlRelative", new URI("./mapping.csv"), "internal", - "{{external}}" + "{{external}}", + false ); @@ -140,9 +151,10 @@ void testEvictOnMapper() log.info("Test evicting of mapping on mapper"); final MapInternToExternMapper mapInternToExternMapper = new MapInternToExternMapper( "test1", - new URI("classpath:/tests/aggregator/FIRST_MAPPED_AGGREGATOR/mapping.csv"), + new URI("classpath:"+MAPPING_PATH), "internal", - "{{external}}" + "{{external}}", + false ); injectComponents(mapInternToExternMapper, indexService); @@ -153,7 +165,7 @@ void testEvictOnMapper() assertThat(mapInternToExternMapper.external("int1")).as("Internal Value").isEqualTo("hello"); - final MapIndex mappingBeforeEvict = mapInternToExternMapper.getInt2ext().get(); + final Index mappingBeforeEvict = mapInternToExternMapper.getInt2ext().get(); indexService.evictCache(); @@ -162,7 +174,7 @@ void testEvictOnMapper() mapInternToExternMapper.init(); - final MapIndex mappingAfterEvict = mapInternToExternMapper.getInt2ext().get(); + final Index mappingAfterEvict = mapInternToExternMapper.getInt2ext().get(); // Check that the mapping reinitialized assertThat(mappingBeforeEvict).as("Mapping before and after eviction") diff --git a/backend/src/test/resources/tests/aggregator/DISTINCT_MAPPED_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json b/backend/src/test/resources/tests/aggregator/DISTINCT_MAPPED_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json deleted file mode 100644 index 2be66b2d7a..0000000000 --- a/backend/src/test/resources/tests/aggregator/DISTINCT_MAPPED_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "type": "QUERY_TEST", - "label": "DISTINCT_MAPPED_AGGREGATOR Test", - "expectedCsv": "tests/aggregator/DISTINCT_MAPPED_AGGREGATOR/expected.csv", - "query": { - "type": "CONCEPT_QUERY", - "root": { - "ids": [ - "concept" - ], - "type": "CONCEPT", - "tables": [ - { - "id": "concept.connector", - "selects": [ - "concept.connector.select" - ] - } - ] - } - }, - "internToExternMappings": { - "name": "test_map", - "type": "CSV_MAP", - "internalColumn": "internal", - "externalTemplate": "{{external}}", - "csv": "classpath:/tests/aggregator/DISTINCT_MAPPED_AGGREGATOR/mapping.csv" - }, - "concepts": [ - { - "label": "concept", - "type": "TREE", - "connectors": [ - { - "label": "connector", - "table": "table", - "validityDates": { - "label": "datum", - "column": "table.datum" - }, - "selects": { - "name": "select", - "type": "DISTINCT", - "column": "table.value", - "mapping": "test_map" - } - } - ] - } - ], - "content": { - "tables": [ - { - "csv": "tests/aggregator/DISTINCT_MAPPED_AGGREGATOR/content.csv", - "name": "table", - "primaryColumn": { - "name": "pid", - "type": "STRING" - }, - "columns": [ - { - "name": "datum", - "type": "DATE" - }, - { - "name": "value", - "type": "STRING" - } - ] - } - ] - } -} diff --git a/backend/src/test/resources/tests/aggregator/DISTINCT_MAPPED_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/DISTINCT_MAPPED_AGGREGATOR/expected.csv deleted file mode 100644 index 68cb3e7cb9..0000000000 --- a/backend/src/test/resources/tests/aggregator/DISTINCT_MAPPED_AGGREGATOR/expected.csv +++ /dev/null @@ -1,6 +0,0 @@ -result,dates,concept select -1,{2012-01-01/2012-01-02},"{ext2,ext1}" -2,{2010-07-15/2010-07-15}, -3,{2012-01-01/2012-01-02},{ext1} -5,{2012-01-01/2012-01-01},{ext2} -6,{2012-01-01/2012-01-01},"{ext2,ext1}" \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/DISTINCT_MAPPED_AGGREGATOR/mapping.csv b/backend/src/test/resources/tests/aggregator/DISTINCT_MAPPED_AGGREGATOR/mapping.csv deleted file mode 100644 index 5f8077a511..0000000000 --- a/backend/src/test/resources/tests/aggregator/DISTINCT_MAPPED_AGGREGATOR/mapping.csv +++ /dev/null @@ -1,3 +0,0 @@ -internal,unused,external -int1,unused,ext1 -int2,unused,ext2 diff --git a/backend/src/test/resources/tests/aggregator/FIRST_MAPPED_AGGREGATOR/content.csv b/backend/src/test/resources/tests/aggregator/FIRST_MAPPED_AGGREGATOR/content.csv deleted file mode 100644 index 7c4c553b55..0000000000 --- a/backend/src/test/resources/tests/aggregator/FIRST_MAPPED_AGGREGATOR/content.csv +++ /dev/null @@ -1,13 +0,0 @@ -pid,datum,value -1,2012-01-01,int1 -1,2012-01-02,int2 - -2,2010-07-15, - -3,2012-01-01, -3,2012-01-02,int1 -4,,int1 -5,2012-01-01,int2 -5,,int1 -6,2012-01-01,int2 -6,2012-01-01,int1 \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/MAPPED/DISTINCT/SIMPLE_VIRTUAL_CONCEPT_Query.test.json b/backend/src/test/resources/tests/aggregator/MAPPED/DISTINCT/SIMPLE_VIRTUAL_CONCEPT_Query.test.json new file mode 100644 index 0000000000..b1ad5d40c6 --- /dev/null +++ b/backend/src/test/resources/tests/aggregator/MAPPED/DISTINCT/SIMPLE_VIRTUAL_CONCEPT_Query.test.json @@ -0,0 +1,74 @@ +{ + "type": "QUERY_TEST", + "label": "DISTINCT_MAPPED_AGGREGATOR Test", + "expectedCsv": "tests/aggregator/MAPPED/DISTINCT/expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "ids": [ + "concept" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "concept.connector", + "selects": [ + "concept.connector.select" + ] + } + ] + } + }, + "internToExternMappings": { + "name": "test_map", + "type": "CSV_MAP", + "internalColumn": "internal", + "externalTemplate": "{{external}}", + "csv": "classpath:/tests/aggregator/MAPPED/mapping.csv", + "allowMultiple": false + }, + "concepts": [ + { + "label": "concept", + "type": "TREE", + "connectors": [ + { + "label": "connector", + "table": "table", + "validityDates": { + "label": "datum", + "column": "table.datum" + }, + "selects": { + "name": "select", + "type": "DISTINCT", + "column": "table.value", + "mapping": "test_map" + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/aggregator/MAPPED/content.csv", + "name": "table", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE" + }, + { + "name": "value", + "type": "STRING" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/aggregator/MAPPED/DISTINCT/expected.csv b/backend/src/test/resources/tests/aggregator/MAPPED/DISTINCT/expected.csv new file mode 100644 index 0000000000..c8cd2026db --- /dev/null +++ b/backend/src/test/resources/tests/aggregator/MAPPED/DISTINCT/expected.csv @@ -0,0 +1,6 @@ +result,dates,concept select +1,{2012-01-01/2012-01-02},"{int2,hello}" +2,{2010-07-15/2010-07-15}, +3,{2012-01-01/2012-01-02},{hello} +5,{2012-01-01/2012-01-01},{int2} +6,{2012-01-01/2012-01-01},"{int2,hello}" \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/MAPPED/DISTINCT_MULTI/SIMPLE_VIRTUAL_CONCEPT_Query.test.json b/backend/src/test/resources/tests/aggregator/MAPPED/DISTINCT_MULTI/SIMPLE_VIRTUAL_CONCEPT_Query.test.json new file mode 100644 index 0000000000..29667a4ef3 --- /dev/null +++ b/backend/src/test/resources/tests/aggregator/MAPPED/DISTINCT_MULTI/SIMPLE_VIRTUAL_CONCEPT_Query.test.json @@ -0,0 +1,74 @@ +{ + "type": "QUERY_TEST", + "label": "DISTINCT MULTI MAPPED AGGREGATOR Test", + "expectedCsv": "tests/aggregator/MAPPED/DISTINCT_MULTI/expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "ids": [ + "concept" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "concept.connector", + "selects": [ + "concept.connector.select" + ] + } + ] + } + }, + "internToExternMappings": { + "name": "test_map", + "type": "CSV_MAP", + "internalColumn": "internal", + "externalTemplate": "{{external}}", + "allowMultiple" : true, + "csv": "classpath:/tests/aggregator/MAPPED/mapping.csv" + }, + "concepts": [ + { + "label": "concept", + "type": "TREE", + "connectors": [ + { + "label": "connector", + "table": "table", + "validityDates": { + "label": "datum", + "column": "table.datum" + }, + "selects": { + "name": "select", + "type": "DISTINCT", + "column": "table.value", + "mapping": "test_map" + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/aggregator/MAPPED/content.csv", + "name": "table", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE" + }, + { + "name": "value", + "type": "STRING" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/aggregator/MAPPED/DISTINCT_MULTI/expected.csv b/backend/src/test/resources/tests/aggregator/MAPPED/DISTINCT_MULTI/expected.csv new file mode 100644 index 0000000000..85c148ff89 --- /dev/null +++ b/backend/src/test/resources/tests/aggregator/MAPPED/DISTINCT_MULTI/expected.csv @@ -0,0 +1,6 @@ +result,dates,concept select +1,{2012-01-01/2012-01-02},"{int2,goodbye,hello}" +2,{2010-07-15/2010-07-15}, +3,{2012-01-01/2012-01-02},"{goodbye,hello}" +5,{2012-01-01/2012-01-01},{int2} +6,{2012-01-01/2012-01-01},"{int2,goodbye,hello}" \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/FIRST_MAPPED_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json b/backend/src/test/resources/tests/aggregator/MAPPED/FIRST/SIMPLE_VIRTUAL_CONCEPT_Query.test.json similarity index 88% rename from backend/src/test/resources/tests/aggregator/FIRST_MAPPED_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json rename to backend/src/test/resources/tests/aggregator/MAPPED/FIRST/SIMPLE_VIRTUAL_CONCEPT_Query.test.json index e032833ead..354bc8a395 100644 --- a/backend/src/test/resources/tests/aggregator/FIRST_MAPPED_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json +++ b/backend/src/test/resources/tests/aggregator/MAPPED/FIRST/SIMPLE_VIRTUAL_CONCEPT_Query.test.json @@ -1,7 +1,7 @@ { "type": "QUERY_TEST", "label": "FIRST_MAPPED_AGGREGATOR Test", - "expectedCsv": "tests/aggregator/FIRST_MAPPED_AGGREGATOR/expected.csv", + "expectedCsv": "tests/aggregator/MAPPED/FIRST/expected.csv", "query": { "type": "CONCEPT_QUERY", "root": { @@ -24,7 +24,7 @@ "type": "CSV_MAP", "internalColumn": "internal", "externalTemplate": "External: {{external}} {{external2}}", - "csv": "classpath:/tests/aggregator/FIRST_MAPPED_AGGREGATOR/mapping.csv" + "csv": "classpath:/tests/aggregator/MAPPED/mapping.csv" }, "concepts": [ { @@ -51,7 +51,7 @@ "content": { "tables": [ { - "csv": "tests/aggregator/FIRST_MAPPED_AGGREGATOR/content.csv", + "csv": "tests/aggregator/MAPPED/content.csv", "name": "table", "primaryColumn": { "name": "pid", diff --git a/backend/src/test/resources/tests/aggregator/FIRST_MAPPED_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/MAPPED/FIRST/expected.csv similarity index 100% rename from backend/src/test/resources/tests/aggregator/FIRST_MAPPED_AGGREGATOR/expected.csv rename to backend/src/test/resources/tests/aggregator/MAPPED/FIRST/expected.csv diff --git a/backend/src/test/resources/tests/aggregator/MAPPED/FIRST_MULTI/SIMPLE_VIRTUAL_CONCEPT_Query.test.json b/backend/src/test/resources/tests/aggregator/MAPPED/FIRST_MULTI/SIMPLE_VIRTUAL_CONCEPT_Query.test.json new file mode 100644 index 0000000000..424c9e855e --- /dev/null +++ b/backend/src/test/resources/tests/aggregator/MAPPED/FIRST_MULTI/SIMPLE_VIRTUAL_CONCEPT_Query.test.json @@ -0,0 +1,74 @@ +{ + "type": "QUERY_TEST", + "label": "FIRST_MULTI_MAPPED_AGGREGATOR Test", + "expectedCsv": "tests/aggregator/MAPPED/FIRST_MULTI/expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "ids": [ + "concept" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "concept.connector", + "selects": [ + "concept.connector.select" + ] + } + ] + } + }, + "internToExternMappings": { + "name": "test_map", + "type": "CSV_MAP", + "internalColumn": "internal", + "externalTemplate": "External: {{external}} {{external2}}", + "csv": "classpath:/tests/aggregator/MAPPED/mapping.csv", + "allowMultiple": true + }, + "concepts": [ + { + "label": "concept", + "type": "TREE", + "connectors": [ + { + "label": "connector", + "table": "table", + "validityDates": { + "label": "datum", + "column": "table.datum" + }, + "selects": { + "name": "select", + "type": "FIRST", + "column": "table.value", + "mapping": "test_map" + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/aggregator/MAPPED/content.csv", + "name": "table", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE" + }, + { + "name": "value", + "type": "STRING" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/aggregator/MAPPED/FIRST_MULTI/expected.csv b/backend/src/test/resources/tests/aggregator/MAPPED/FIRST_MULTI/expected.csv new file mode 100644 index 0000000000..e7b1b0541d --- /dev/null +++ b/backend/src/test/resources/tests/aggregator/MAPPED/FIRST_MULTI/expected.csv @@ -0,0 +1,6 @@ +result,dates,concept select +1,{2012-01-01/2012-01-02},"{External: hello world,External: goodbye moon}" +2,{2010-07-15/2010-07-15}, +3,{2012-01-01/2012-01-02},"{External: hello world,External: goodbye moon}" +5,{2012-01-01/2012-01-01},{External: ext2} +6,{2012-01-01/2012-01-01},{External: ext2} \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/DISTINCT_MAPPED_AGGREGATOR/content.csv b/backend/src/test/resources/tests/aggregator/MAPPED/content.csv similarity index 100% rename from backend/src/test/resources/tests/aggregator/DISTINCT_MAPPED_AGGREGATOR/content.csv rename to backend/src/test/resources/tests/aggregator/MAPPED/content.csv diff --git a/backend/src/test/resources/tests/aggregator/FIRST_MAPPED_AGGREGATOR/mapping.csv b/backend/src/test/resources/tests/aggregator/MAPPED/mapping.csv similarity index 76% rename from backend/src/test/resources/tests/aggregator/FIRST_MAPPED_AGGREGATOR/mapping.csv rename to backend/src/test/resources/tests/aggregator/MAPPED/mapping.csv index 07ca6491a6..11062f3a34 100644 --- a/backend/src/test/resources/tests/aggregator/FIRST_MAPPED_AGGREGATOR/mapping.csv +++ b/backend/src/test/resources/tests/aggregator/MAPPED/mapping.csv @@ -1,3 +1,4 @@ internal,unused,external,external2 int1,unused,hello,world int2,unused,,ext2 +int1,unused,goodbye,moon \ No newline at end of file