Merge Release

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java

@@ -76,15 +76,17 @@ public void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig
 	public List<Searchable> getSearchReferences() {
 		final List<Searchable> out = new ArrayList<>();
 
-		if (getTemplate() != null) {
+		if (getTemplate() != null && !getTemplate().isSearchDisabled()) {
 			out.add(getTemplate());
 		}
 
 		if (!labels.isEmpty()) {
 			out.add(new LabelMap(getId(), labels, searchMinSuffixLength, generateSearchSuffixes));
 		}
 
-		out.add(getColumn());
+		if (!getColumn().isSearchDisabled()) {
+			out.add(getColumn());
+		}
 
 		return out;
 	}

backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/StringStoreString.java

@@ -27,13 +27,20 @@ public static StringStoreString create(int size) {
 
 	@JsonCreator
 	public static StringStoreString withInternedStrings(String[] values) {
-		for (int index = 0; index < values.length; index++) {
-			values[index] = values[index] != null ? values[index].intern() : null;
+		if(shouldIntern()) {
+			for (int index = 0; index < values.length; index++) {
+				values[index] = values[index] != null ? values[index].intern() : null;
+			}
 		}
 
 		return new StringStoreString(values);
 	}
 
+	private static boolean shouldIntern() {
+		//TODO use mixin or properly wire this property
+		return "yes".equals(System.getProperty("cq.intern", "no"));
+	}
+
 	@Override
 	public boolean has(int event) {
 		return values[event] != null;

backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java

@@ -11,7 +11,6 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
-import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
 import com.bakdata.conquery.apiv1.frontend.FrontendValue;
@@ -70,9 +69,6 @@ public void execute() throws Exception {
 				allSelectFilters.stream()
 								.map(SelectFilter::getSearchReferences)
 								.flatMap(Collection::stream)
-								// Disabling search is only a last resort for when columns are too big to store in memory or process for indexing.
-								// TODO FK: We want no Searchable to be disabled, better scaling searches or mechanisms to fill search.
-								.filter(Predicate.not(Searchable::isSearchDisabled))
 								// Group Searchables into "Columns" and other "Searchables"
 								.collect(Collectors.groupingBy(s -> s instanceof Column ? Column.class : Searchable.class, Collectors.toSet()));
 

backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/CollectColumnValuesJob.java

@@ -14,6 +14,7 @@
 import com.bakdata.conquery.io.jackson.serializer.NsIdRefCollection;
 import com.bakdata.conquery.models.datasets.Column;
 import com.bakdata.conquery.models.datasets.Table;
+import com.bakdata.conquery.models.datasets.concepts.filters.specific.SelectFilter;
 import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.StringStore;
 import com.bakdata.conquery.models.jobs.SimpleJob;
@@ -113,9 +114,16 @@ public void afterAllReaction() {
 							columns.forEach(filterSearch::shrinkSearch);
 
 
-							log.info("BEGIN counting Search totals.");
-							UpdateFilterSearchJob.getAllSelectFilters(namespace.getStorage()).forEach(namespace.getFilterSearch()::getTotal);
-							log.debug("FINISHED counting Search totals.");
+							log.info("BEGIN counting search totals on {}", namespace.getDataset().getId());
+							for (SelectFilter<?> filter : UpdateFilterSearchJob.getAllSelectFilters(namespace.getStorage())) {
+								try {
+									namespace.getFilterSearch().getTotal(filter);
+								}
+								catch (Exception e) {
+									log.warn("Unable to calculate totals for filter '{}'", filter.getId(), e);
+								}
+							}
+							log.debug("FINISHED counting search totals on {}", namespace.getDataset().getId());
 						}
 				)
 		);