Merge pull request #3396 from ingef/reintegrate-master

Reintegrate master

backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java

@@ -8,7 +8,6 @@
 import com.bakdata.conquery.apiv1.frontend.FrontendValue;
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.io.jackson.serializer.NsIdRef;
-import com.bakdata.conquery.io.storage.NamespaceStorage;
 import com.bakdata.conquery.models.config.IndexConfig;
 import com.bakdata.conquery.models.datasets.Dataset;
 import com.bakdata.conquery.models.datasets.concepts.Searchable;
@@ -40,7 +39,7 @@
 @ToString
 @Slf4j
 @CPSType(id = "CSV_TEMPLATE", base = SearchIndex.class)
-public class FilterTemplate extends IdentifiableImpl<SearchIndexId> implements Searchable<SearchIndexId>, SearchIndex {
+public class FilterTemplate extends IdentifiableImpl<SearchIndexId> implements Searchable, SearchIndex {
 
 	private static final long serialVersionUID = 1L;
 
@@ -90,7 +89,7 @@ public boolean isSearchDisabled() {
 		return false;
 	}
 
-	public TrieSearch<FrontendValue> createTrieSearch(IndexConfig config, NamespaceStorage storage) {
+	public TrieSearch<FrontendValue> createTrieSearch(IndexConfig config) {
 
 		final URI resolvedURI = FileUtil.getResolvedUri(config.getBaseUrl(), getFilePath());
 		log.trace("Resolved filter template reference url for search '{}': {}", this.getId(), resolvedURI);

backend/src/main/java/com/bakdata/conquery/apiv1/LabelMap.java

@@ -0,0 +1,71 @@
+package com.bakdata.conquery.apiv1;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+import com.bakdata.conquery.apiv1.frontend.FrontendValue;
+import com.bakdata.conquery.models.config.IndexConfig;
+import com.bakdata.conquery.models.datasets.concepts.Searchable;
+import com.bakdata.conquery.models.identifiable.ids.specific.FilterId;
+import com.bakdata.conquery.models.query.FilterSearch;
+import com.bakdata.conquery.util.search.TrieSearch;
+import com.google.common.collect.BiMap;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.Delegate;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.time.StopWatch;
+
+@Getter
+@RequiredArgsConstructor
+@Slf4j
+@EqualsAndHashCode
+public class LabelMap implements Searchable {
+
+	private final FilterId id;
+	@Delegate
+	private final BiMap<String, String> delegate;
+	private final int minSuffixLength;
+	private final boolean generateSearchSuffixes;
+
+	@Override
+	public TrieSearch<FrontendValue> createTrieSearch(IndexConfig config) {
+
+		final TrieSearch<FrontendValue> search = config.createTrieSearch(true);
+
+		final List<FrontendValue> collected = delegate.entrySet().stream()
+													  .map(entry -> new FrontendValue(entry.getKey(), entry.getValue()))
+													  .collect(Collectors.toList());
+
+		if (log.isTraceEnabled()) {
+			log.trace("Labels for {}: `{}`", getId(), collected.stream().map(FrontendValue::toString).collect(Collectors.toList()));
+		}
+
+		StopWatch timer = StopWatch.createStarted();
+		log.trace("START-SELECT ADDING_ITEMS for {}", getId());
+
+		collected.forEach(feValue -> search.addItem(feValue, FilterSearch.extractKeywords(feValue)));
+
+		log.trace("DONE-SELECT ADDING_ITEMS for {} in {}", getId(), timer);
+
+		timer.reset();
+		log.trace("START-SELECT SHRINKING for {}", getId());
+
+		search.shrinkToFit();
+
+		log.trace("DONE-SELECT SHRINKING for {} in {}", getId(), timer);
+
+		return search;
+	}
+
+	@Override
+	public boolean isGenerateSuffixes() {
+		return generateSearchSuffixes;
+	}
+
+	@Override
+	public boolean isSearchDisabled() {
+		return false;
+	}
+}

backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java

@@ -554,15 +554,14 @@ public Stream<Map<String, String>> resolveEntities(Subject subject, List<FilterV
 					 .filter(Predicate.not(Map::isEmpty));
 	}
 
-	public ResultStatistics getResultStatistics(ManagedQuery managedQuery) {
-		final Query query = managedQuery.getQuery();
-		final List<ResultInfo> resultInfos = query.getResultInfos();
+	public ResultStatistics getResultStatistics(SingleTableResult managedQuery) {
+		final List<ResultInfo> resultInfos = managedQuery.getResultInfos();
 
 		final Optional<ResultInfo>
 				dateInfo =
-				query.getResultInfos().stream().filter(info -> info.getSemantics().contains(new SemanticType.EventDateT())).findFirst();
+				resultInfos.stream().filter(info -> info.getSemantics().contains(new SemanticType.EventDateT())).findFirst();
 
-		final int dateIndex = dateInfo.map(resultInfos::indexOf).orElse(0 /*Discarded if dateInfo is not present*/);
+		final Optional<Integer> dateIndex = dateInfo.map(resultInfos::indexOf);
 
 		final Locale locale = I18n.LOCALE.get();
 		final NumberFormat decimalFormat = NumberFormat.getNumberInstance(locale);

backend/src/main/java/com/bakdata/conquery/io/mina/NetworkSession.java

@@ -13,25 +13,29 @@
 import lombok.extern.slf4j.Slf4j;
 import org.apache.mina.core.future.WriteFuture;
 import org.apache.mina.core.session.IoSession;
+import org.jetbrains.annotations.NotNull;
 
 @RequiredArgsConstructor
 @Slf4j
 public class NetworkSession implements MessageSender<NetworkMessage<?>> {
+	public static final int MAX_MESSAGE_LENGTH = 30;
+	public static final int MAX_QUEUE_LENGTH = 20;
 	@Getter
 	private final IoSession session;
-	private final LinkedBlockingQueue<NetworkMessage<?>> queuedMessages = new LinkedBlockingQueue<>(20);
+	private final LinkedBlockingQueue<NetworkMessage<?>> queuedMessages = new LinkedBlockingQueue<>(MAX_QUEUE_LENGTH);
 
 	@Override
 	public WriteFuture send(final NetworkMessage<?> message) {
 		try {
 			while (!queuedMessages.offer(message, 2, TimeUnit.MINUTES)) {
-				log.debug(
-						"Waiting for full writing queue for {}\n\tcurrently filled by: {}",
-						message,
-						new ArrayList<>(queuedMessages)
-								.stream()
-								.map(Objects::toString)
-								.collect(Collectors.joining("\n\t\t"))
+				log.debug("Waiting for full writing queue for {} currently filled by:\n\t- {}",
+						  message,
+						  log.isTraceEnabled()
+						  ? new ArrayList<>(queuedMessages).stream()
+														   .map(Objects::toString)
+														   .map(NetworkSession::shorten)
+														   .collect(Collectors.joining("\n\t\t- "))
+						  : "%s messages".formatted(queuedMessages.size())
 				);
 			}
 		}
@@ -45,6 +49,16 @@ public WriteFuture send(final NetworkMessage<?> message) {
 		return future;
 	}
 
+	@NotNull
+	private static String shorten(String desc) {
+		if (desc.length() <= MAX_MESSAGE_LENGTH) {
+			return desc;
+		}
+
+		return desc.substring(0, MAX_MESSAGE_LENGTH) + "…";
+
+	}
+
 	@Override
 	public void trySend(final NetworkMessage<?> message) {
 		if (isConnected()) {

backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java

@@ -46,7 +46,7 @@ static NamespaceSetupData createNamespaceSetup(NamespaceStorage storage, final C
 
 		JobManager jobManager = new JobManager(storage.getDataset().getName(), config.isFailOnError());
 
-		FilterSearch filterSearch = new FilterSearch(storage, jobManager, config.getCsv(), config.getIndex());
+		FilterSearch filterSearch = new FilterSearch(config.getIndex());
 		return new NamespaceSetupData(injectables, indexService, communicationMapper, preprocessMapper, jobManager, filterSearch);
 	}
 

backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java

@@ -4,7 +4,6 @@
 
 import com.bakdata.conquery.apiv1.frontend.FrontendValue;
 import com.bakdata.conquery.io.jackson.serializer.NsIdRef;
-import com.bakdata.conquery.io.storage.NamespaceStorage;
 import com.bakdata.conquery.models.config.IndexConfig;
 import com.bakdata.conquery.models.datasets.concepts.Searchable;
 import com.bakdata.conquery.models.events.MajorTypeId;
@@ -26,7 +25,7 @@
 @Setter
 @NoArgsConstructor
 @Slf4j
-public class Column extends Labeled<ColumnId> implements NamespacedIdentifiable<ColumnId>, Searchable<ColumnId> {
+public class Column extends Labeled<ColumnId> implements NamespacedIdentifiable<ColumnId>, Searchable {
 
 	public static final int UNKNOWN_POSITION = -1;
 
@@ -75,7 +74,7 @@ public Dataset getDataset() {
 	 * We create only an empty search here, because the content is provided through {@link com.bakdata.conquery.models.messages.namespaces.specific.RegisterColumnValues} and filled by the caller.
 	 */
 	@Override
-	public TrieSearch<FrontendValue> createTrieSearch(IndexConfig config, NamespaceStorage storage) {
+	public TrieSearch<FrontendValue> createTrieSearch(IndexConfig config) {
 		return config.createTrieSearch(isGenerateSuffixes());
 	}
 

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java

@@ -3,11 +3,7 @@
 import java.util.List;
 
 import com.bakdata.conquery.apiv1.frontend.FrontendValue;
-import com.bakdata.conquery.io.storage.NamespaceStorage;
 import com.bakdata.conquery.models.config.IndexConfig;
-import com.bakdata.conquery.models.datasets.Dataset;
-import com.bakdata.conquery.models.identifiable.Identifiable;
-import com.bakdata.conquery.models.identifiable.ids.Id;
 import com.bakdata.conquery.models.query.FilterSearch;
 import com.bakdata.conquery.util.search.TrieSearch;
 import com.fasterxml.jackson.annotation.JsonIgnore;
@@ -18,25 +14,12 @@
  * <p>
  * Searchable classes describe how a search should be constructed, and provide the values with getSearchValues.
  */
-public interface Searchable<ID extends Id<? extends Identifiable<? extends ID>>> extends Identifiable<ID> {
-
-	public Dataset getDataset();
+public interface Searchable {
 
 	/**
 	 * All available {@link FrontendValue}s for searching in a {@link TrieSearch}.
 	 */
-	TrieSearch<FrontendValue> createTrieSearch(IndexConfig config, NamespaceStorage storage);
-
-	/**
-	 * The actual Searchables to use, if there is potential for deduplication/pooling.
-	 *
-	 * @implSpec The order of objects returned is used to also sort search results from different sources.
-	 */
-	@JsonIgnore
-	default List<Searchable<?>> getSearchReferences() {
-		//Hopefully the only candidate will be Column
-		return List.of(this);
-	}
+	TrieSearch<FrontendValue> createTrieSearch(IndexConfig config);
 
 	/**
 	 * Parameter used in the construction of {@link com.bakdata.conquery.util.search.TrieSearch}, defining the shortest suffix to create.

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java

@@ -7,20 +7,16 @@
 import java.util.stream.Collectors;
 
 import com.bakdata.conquery.apiv1.FilterTemplate;
+import com.bakdata.conquery.apiv1.LabelMap;
 import com.bakdata.conquery.apiv1.frontend.FrontendFilterConfiguration;
 import com.bakdata.conquery.apiv1.frontend.FrontendValue;
 import com.bakdata.conquery.io.jackson.View;
 import com.bakdata.conquery.io.jackson.serializer.NsIdRef;
-import com.bakdata.conquery.io.storage.NamespaceStorage;
 import com.bakdata.conquery.models.config.ConqueryConfig;
-import com.bakdata.conquery.models.config.IndexConfig;
 import com.bakdata.conquery.models.datasets.concepts.Searchable;
 import com.bakdata.conquery.models.datasets.concepts.filters.SingleColumnFilter;
 import com.bakdata.conquery.models.events.MajorTypeId;
 import com.bakdata.conquery.models.exceptions.ConceptConfigurationException;
-import com.bakdata.conquery.models.identifiable.ids.specific.FilterId;
-import com.bakdata.conquery.models.query.FilterSearch;
-import com.bakdata.conquery.util.search.TrieSearch;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.google.common.collect.BiMap;
@@ -30,15 +26,14 @@
 import lombok.NoArgsConstructor;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.time.StopWatch;
 import org.jetbrains.annotations.NotNull;
 
 @Setter
 @Getter
 @NoArgsConstructor
 @Slf4j
 @JsonIgnoreProperties({"searchType"})
-public abstract class SelectFilter<FE_TYPE> extends SingleColumnFilter<FE_TYPE> implements Searchable<FilterId> {
+public abstract class SelectFilter<FE_TYPE> extends SingleColumnFilter<FE_TYPE> {
 
 	/**
 	 * user given mapping from the values in the columns to shown labels
@@ -71,19 +66,27 @@ public void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig
 	@JsonIgnore
 	public abstract String getFilterType();
 
-	@Override
-	public List<Searchable<?>> getSearchReferences() {
-		final List<Searchable<?>> out = new ArrayList<>();
 
-		if (getTemplate() != null) {
+	/**
+	 * The actual Searchables to use, if there is potential for deduplication/pooling.
+	 *
+	 * @implSpec The order of objects returned is used to also sort search results from different sources.
+	 */
+	@JsonIgnore
+	public List<Searchable> getSearchReferences() {
+		final List<Searchable> out = new ArrayList<>();
+
+		if (getTemplate() != null && !getTemplate().isSearchDisabled()) {
 			out.add(getTemplate());
 		}
 
 		if (!labels.isEmpty()) {
-			out.add(this);
+			out.add(new LabelMap(getId(), labels, searchMinSuffixLength, generateSearchSuffixes));
 		}
 
-		out.addAll(getColumn().getSearchReferences());
+		if (!getColumn().isSearchDisabled()) {
+			out.add(getColumn());
+		}
 
 		return out;
 	}
@@ -105,51 +108,4 @@ public boolean isNotUsingTemplateAndLabels() {
 
 		return (getTemplate() == null) != labels.isEmpty();
 	}
-
-	@Override
-	@JsonIgnore
-	public boolean isGenerateSuffixes() {
-		return generateSearchSuffixes;
-	}
-
-	@Override
-	@JsonIgnore
-	public int getMinSuffixLength() {
-		return searchMinSuffixLength;
-	}
-
-	/**
-	 * Does not make sense to distinguish at Filter level since it's only referenced when labels are set.
-	 */
-	@Override
-	@JsonIgnore
-	public boolean isSearchDisabled() {
-		return false;
-	}
-
-	@Override
-	public TrieSearch<FrontendValue> createTrieSearch(IndexConfig config, NamespaceStorage storage) {
-
-		final TrieSearch<FrontendValue> search = config.createTrieSearch(true);
-
-		if(log.isTraceEnabled()) {
-			log.trace("Labels for {}: `{}`", getId(), collectLabels().stream().map(FrontendValue::toString).collect(Collectors.toList()));
-		}
-
-		StopWatch timer = StopWatch.createStarted();
-		log.trace("START-SELECT ADDING_ITEMS for {}", getId());
-
-		collectLabels().forEach(feValue -> search.addItem(feValue, FilterSearch.extractKeywords(feValue)));
-
-		log.trace("DONE-SELECT ADDING_ITEMS for {} in {}", getId(), timer);
-
-		timer.reset();
-		log.trace("START-SELECT SHRINKING for {}", getId());
-
-		search.shrinkToFit();
-
-		log.trace("DONE-SELECT SHRINKING for {} in {}", getId(), timer);
-
-		return search;
-	}
 }

backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/StringStoreString.java

@@ -27,13 +27,20 @@ public static StringStoreString create(int size) {
 
 	@JsonCreator
 	public static StringStoreString withInternedStrings(String[] values) {
-		for (int index = 0; index < values.length; index++) {
-			values[index] = values[index] != null ? values[index].intern() : null;
+		if(shouldIntern()) {
+			for (int index = 0; index < values.length; index++) {
+				values[index] = values[index] != null ? values[index].intern() : null;
+			}
 		}
 
 		return new StringStoreString(values);
 	}
 
+	private static boolean shouldIntern() {
+		//TODO use mixin or properly wire this property
+		return "yes".equals(System.getProperty("cq.intern", "no"));
+	}
+
 	@Override
 	public boolean has(int event) {
 		return values[event] != null;