Merge pull request #3380 from ingef/hotfix/skip-disabled-searchables

Hotfix: move sorting out of disabled searchables from the job to the filter
ingef · Apr 9, 2024 · b2b6e47 · b2b6e47
2 parents 32d05c4 + 50a392a
commit b2b6e47
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 9 deletions.
diff --git a/...ain/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java b/...ain/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java
@@ -76,15 +76,17 @@ public void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig
 	public List<Searchable> getSearchReferences() {
 		final List<Searchable> out = new ArrayList<>();
 
-		if (getTemplate() != null) {
+		if (getTemplate() != null && !getTemplate().isSearchDisabled()) {
 			out.add(getTemplate());
 		}
 
 		if (!labels.isEmpty()) {
 			out.add(new LabelMap(getId(), labels, searchMinSuffixLength, generateSearchSuffixes));
 		}
 
-		out.add(getColumn());
+		if (!getColumn().isSearchDisabled()) {
+			out.add(getColumn());
+		}
 
 		return out;
 	}

diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java
@@ -11,7 +11,6 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
-import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
 import com.bakdata.conquery.apiv1.frontend.FrontendValue;
@@ -70,9 +69,6 @@ public void execute() throws Exception {
 				allSelectFilters.stream()
 								.map(SelectFilter::getSearchReferences)
 								.flatMap(Collection::stream)
-								// Disabling search is only a last resort for when columns are too big to store in memory or process for indexing.
-								// TODO FK: We want no Searchable to be disabled, better scaling searches or mechanisms to fill search.
-								.filter(Predicate.not(Searchable::isSearchDisabled))
 								// Group Searchables into "Columns" and other "Searchables"
 								.collect(Collectors.groupingBy(s -> s instanceof Column ? Column.class : Searchable.class, Collectors.toSet()));
 

diff --git a/...java/com/bakdata/conquery/models/messages/namespaces/specific/CollectColumnValuesJob.java b/...java/com/bakdata/conquery/models/messages/namespaces/specific/CollectColumnValuesJob.java
@@ -14,6 +14,7 @@
 import com.bakdata.conquery.io.jackson.serializer.NsIdRefCollection;
 import com.bakdata.conquery.models.datasets.Column;
 import com.bakdata.conquery.models.datasets.Table;
+import com.bakdata.conquery.models.datasets.concepts.filters.specific.SelectFilter;
 import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.StringStore;
 import com.bakdata.conquery.models.jobs.SimpleJob;
@@ -113,9 +114,16 @@ public void afterAllReaction() {
 							columns.forEach(filterSearch::shrinkSearch);
 
 
-							log.info("BEGIN counting Search totals.");
-							UpdateFilterSearchJob.getAllSelectFilters(namespace.getStorage()).forEach(namespace.getFilterSearch()::getTotal);
-							log.debug("FINISHED counting Search totals.");
+							log.info("BEGIN counting search totals on {}", namespace.getDataset().getId());
+							for (SelectFilter<?> filter : UpdateFilterSearchJob.getAllSelectFilters(namespace.getStorage())) {
+								try {
+									namespace.getFilterSearch().getTotal(filter);
+								}
+								catch (Exception e) {
+									log.warn("Unable to calculate totals for filter '{}'", filter.getId(), e);
+								}
+							}
+							log.debug("FINISHED counting search totals on {}", namespace.getDataset().getId());
 						}
 				)
 		);