Merge pull request #3498 from ingef/feature/more-http-import-for-tests

use http client where possible to import test data
ingef · Jul 25, 2024 · f458e65 · f458e65
2 parents 7d74b1a + 276eb62
commit f458e65
Show file tree

Hide file tree

Showing 4 changed files with 53 additions and 92 deletions.
diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java b/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java
@@ -1,21 +1,12 @@
 package com.bakdata.conquery.integration.json;
 
-import java.io.IOException;
 import java.util.Collection;
-import java.util.List;
 
-import com.bakdata.conquery.integration.common.LoadingUtil;
 import com.bakdata.conquery.integration.common.RequiredData;
-import com.bakdata.conquery.integration.common.RequiredSecondaryId;
 import com.bakdata.conquery.integration.common.RequiredTable;
 import com.bakdata.conquery.integration.json.filter.FilterTest;
 import com.bakdata.conquery.integration.sql.CsvTableImporter;
-import com.bakdata.conquery.models.datasets.SecondaryIdDescription;
-import com.bakdata.conquery.models.datasets.Table;
-import com.bakdata.conquery.models.datasets.concepts.Concept;
-import com.bakdata.conquery.models.exceptions.JSONException;
 import com.bakdata.conquery.util.support.StandaloneSupport;
-import com.fasterxml.jackson.databind.node.ArrayNode;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
@@ -36,6 +27,9 @@ public void importQueryTestData(StandaloneSupport support, QueryTest test) throw
 		importTables(support, content.getTables(), true);
 		importConcepts(support, test.getRawConcepts());
 		importTableContents(support, content.getTables());
+
+		importSearchIndexes(support, test.getSearchIndexes());
+		importIdMapping(support, content);
 	}
 
 	@Override
@@ -46,41 +40,15 @@ public void importFormTestData(StandaloneSupport support, FormTest test) throws
 		importTables(support, content.getTables(), true);
 		importConcepts(support, test.getRawConcepts());
 		importTableContents(support, content.getTables());
-		waitUntilDone(support, () -> LoadingUtil.importIdMapping(support, content));
-		waitUntilDone(support, () -> LoadingUtil.importPreviousQueries(support, content, support.getTestUser()));
+		importIdMapping(support, content);
+		importPreviousQueries(support, content);
 	}
 
 	@Override
-	public void importFilterTestData(StandaloneSupport support, FilterTest filterTest) throws Exception {
+	public void importFilterTestData(StandaloneSupport support, FilterTest filterTest) {
 		throw new UnsupportedOperationException("Not implemented yet.");
 	}
 
-	@Override
-	public void importSecondaryIds(StandaloneSupport support, List<RequiredSecondaryId> secondaryIds) {
-		for (RequiredSecondaryId required : secondaryIds) {
-			final SecondaryIdDescription description =
-					required.toSecondaryId(support.getDataset(), support.getDatasetRegistry().findRegistry(support.getDataset().getId()));
-			support.getDatasetsProcessor().addSecondaryId(support.getNamespace(), description);
-		}
-	}
-
-	@Override
-	public void importTables(StandaloneSupport support, List<RequiredTable> tables, boolean autoConcept) throws JSONException {
-		for (RequiredTable requiredTable : tables) {
-			final Table table = requiredTable.toTable(support.getDataset(), support.getNamespaceStorage().getCentralRegistry());
-			support.getNamespaceStorage().addTable(table);
-		}
-	}
-
-	@Override
-	public void importConcepts(StandaloneSupport support, ArrayNode rawConcepts) throws JSONException, IOException {
-		List<Concept<?>> concepts =
-				ConqueryTestSpec.parseSubTreeList(support, rawConcepts, Concept.class, concept -> concept.setDataset(support.getDataset()));
-		for (Concept<?> concept : concepts) {
-			support.getNamespaceStorage().updateConcept(concept);
-		}
-	}
-
 	@Override
 	public void importTableContents(StandaloneSupport support, Collection<RequiredTable> tables) throws Exception {
 		for (RequiredTable table : tables) {

diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/TestDataImporter.java b/backend/src/test/java/com/bakdata/conquery/integration/json/TestDataImporter.java
@@ -4,10 +4,13 @@
 import java.util.Collection;
 import java.util.List;
 
+import com.bakdata.conquery.integration.common.LoadingUtil;
+import com.bakdata.conquery.integration.common.RequiredData;
 import com.bakdata.conquery.integration.common.RequiredSecondaryId;
 import com.bakdata.conquery.integration.common.RequiredTable;
 import com.bakdata.conquery.integration.json.filter.FilterTest;
 import com.bakdata.conquery.models.exceptions.JSONException;
+import com.bakdata.conquery.models.index.search.SearchIndex;
 import com.bakdata.conquery.util.support.StandaloneSupport;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 
@@ -19,13 +22,31 @@ public interface TestDataImporter {
 
 	void importFilterTestData(StandaloneSupport support, FilterTest filterTest) throws Exception;
 
-	void importSecondaryIds(StandaloneSupport support, List<RequiredSecondaryId> secondaryIds);
+	void importTableContents(StandaloneSupport support, Collection<RequiredTable> tables) throws Exception;
+
+	default void importSearchIndexes(StandaloneSupport support, List<SearchIndex> searchIndices) {
+		waitUntilDone(support, () -> LoadingUtil.importSearchIndexes(support, searchIndices));
+	}
 
-	void importTables(StandaloneSupport support, List<RequiredTable> tables, boolean autoConcept) throws JSONException;
+	default void importIdMapping(StandaloneSupport support, RequiredData content) {
+		waitUntilDone(support, () -> LoadingUtil.importIdMapping(support, content));
+	}
 
-	void importConcepts(StandaloneSupport support, ArrayNode rawConcepts) throws JSONException, IOException;
+	default void importSecondaryIds(StandaloneSupport support, List<RequiredSecondaryId> secondaryIds) {
+		waitUntilDone(support, () -> LoadingUtil.importSecondaryIds(support, secondaryIds));
+	}
 
-	void importTableContents(StandaloneSupport support, Collection<RequiredTable> tables) throws Exception;
+	default void importTables(StandaloneSupport support, List<RequiredTable> tables, boolean autoConcept) throws JSONException {
+		waitUntilDone(support, () -> LoadingUtil.importTables(support, tables, autoConcept));
+	}
+
+	default void importConcepts(StandaloneSupport support, ArrayNode rawConcepts) throws JSONException, IOException {
+		waitUntilDone(support, () -> LoadingUtil.importConcepts(support, rawConcepts));
+	}
+
+	default void importPreviousQueries(StandaloneSupport support, RequiredData content) {
+		waitUntilDone(support, () -> LoadingUtil.importPreviousQueries(support, content, support.getTestUser()));
+	}
 
 	default void waitUntilDone(StandaloneSupport support, CheckedRunnable<?> runnable) {
 		runnable.run();

diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/WorkerTestDataImporter.java b/backend/src/test/java/com/bakdata/conquery/integration/json/WorkerTestDataImporter.java
@@ -2,22 +2,15 @@
 
 import static com.bakdata.conquery.integration.common.LoadingUtil.importInternToExternMappers;
 
-import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.List;
 
 import com.bakdata.conquery.integration.common.LoadingUtil;
 import com.bakdata.conquery.integration.common.RequiredData;
-import com.bakdata.conquery.integration.common.RequiredSecondaryId;
 import com.bakdata.conquery.integration.common.RequiredTable;
 import com.bakdata.conquery.integration.json.filter.FilterTest;
 import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeConnector;
-import com.bakdata.conquery.models.exceptions.JSONException;
-import com.bakdata.conquery.models.messages.namespaces.specific.UpdateMatchingStatsMessage;
-import com.bakdata.conquery.models.worker.DistributedNamespace;
 import com.bakdata.conquery.util.support.StandaloneSupport;
-import com.fasterxml.jackson.databind.node.ArrayNode;
 
 public class WorkerTestDataImporter implements TestDataImporter {
 
@@ -29,77 +22,54 @@ public void importQueryTestData(StandaloneSupport support, QueryTest test) throw
 		importSecondaryIds(support, content.getSecondaryIds());
 		importInternToExternMappers(support, test.getInternToExternMappings());
 
-		waitUntilDone(support, () -> LoadingUtil.importSearchIndexes(support, test.getSearchIndexes()));
-		waitUntilDone(support, () -> LoadingUtil.importTables(support, content.getTables(), content.isAutoConcept()));
-		waitUntilDone(support, () -> LoadingUtil.importConcepts(support, test.getRawConcepts()));
+		importSearchIndexes(support, test.getSearchIndexes());
+		importTables(support, content.getTables(), content.isAutoConcept());
+		importConcepts(support, test.getRawConcepts());
 		waitUntilDone(support, () -> LoadingUtil.importTableContents(support, content.getTables()));
-		waitUntilDone(support, () -> LoadingUtil.importIdMapping(support, content));
-		waitUntilDone(support, () -> LoadingUtil.importPreviousQueries(support, content, support.getTestUser()));
+		importIdMapping(support, content);
+		importPreviousQueries(support, content);
 		waitUntilDone(support, () -> LoadingUtil.updateMatchingStats(support));
-
-		sendUpdateMatchingStatsMessage(support);
 	}
 
 	@Override
 	public void importFormTestData(StandaloneSupport support, FormTest test) throws Exception {
 
 		RequiredData content = test.getContent();
 
-		waitUntilDone(support, () -> LoadingUtil.importSecondaryIds(support, content.getSecondaryIds()));
-		waitUntilDone(support, () -> LoadingUtil.importTables(support, content.getTables(), content.isAutoConcept()));
-		waitUntilDone(support, () -> LoadingUtil.importConcepts(support, test.getRawConcepts()));
+		importSecondaryIds(support, content.getSecondaryIds());
+		importTables(support, content.getTables(), content.isAutoConcept());
+		importConcepts(support, test.getRawConcepts());
 		waitUntilDone(support, () -> LoadingUtil.importTableContents(support, content.getTables()));
-		waitUntilDone(support, () -> LoadingUtil.importIdMapping(support, content));
-		waitUntilDone(support, () -> LoadingUtil.importPreviousQueries(support, content, support.getTestUser()));
+		importIdMapping(support, content);
+		importPreviousQueries(support, content);
 	}
 
 	@Override
 	public void importFilterTestData(StandaloneSupport support, FilterTest test) throws Exception {
 
 		RequiredData content = test.getContent();
 
-		LoadingUtil.importInternToExternMappers(support, test.getInternToExternMappings());
-		LoadingUtil.importSearchIndexes(support, test.getSearchIndices());
-		waitUntilDone(support, () -> LoadingUtil.importTables(support, content.getTables(), content.isAutoConcept()));
+		importInternToExternMappers(support, test.getInternToExternMappings());
+		importSearchIndexes(support, test.getSearchIndices());
+		importTables(support, content.getTables(), content.isAutoConcept());
 
 		test.setConnector(ConqueryTestSpec.parseSubTree(
-								  support,
-								  test.getRawConnector(),
-								  ConceptTreeConnector.class,
-								  conn -> conn.setConcept(test.getConcept())
-						  )
-		);
+				support,
+				test.getRawConnector(),
+				ConceptTreeConnector.class,
+				conn -> conn.setConcept(test.getConcept())
+		));
 		test.getConcept().setConnectors(Collections.singletonList((ConceptTreeConnector) test.getConnector()));
 
 		waitUntilDone(support, () -> LoadingUtil.uploadConcept(support, support.getDataset(), test.getConcept()));
 		waitUntilDone(support, () -> LoadingUtil.importTableContents(support, content.getTables()));
 		waitUntilDone(support, () -> LoadingUtil.updateMatchingStats(support));
 	}
 
-	@Override
-	public void importSecondaryIds(StandaloneSupport support, List<RequiredSecondaryId> secondaryIds) {
-		waitUntilDone(support, () -> LoadingUtil.importSecondaryIds(support, secondaryIds));
-	}
-
-	@Override
-	public void importTables(StandaloneSupport support, List<RequiredTable> tables, boolean autoConcept) throws JSONException {
-		waitUntilDone(support, () -> LoadingUtil.importTables(support, tables, autoConcept));
-	}
-
-	@Override
-	public void importConcepts(StandaloneSupport support, ArrayNode rawConcepts) throws JSONException, IOException {
-		waitUntilDone(support, () -> LoadingUtil.importConcepts(support, rawConcepts));
-	}
 
 	@Override
 	public void importTableContents(StandaloneSupport support, Collection<RequiredTable> tables) throws Exception {
 		waitUntilDone(support, () -> LoadingUtil.importTableContents(support, tables));
 	}
 
-	private static void sendUpdateMatchingStatsMessage(StandaloneSupport support) {
-		DistributedNamespace namespace = (DistributedNamespace) support.getNamespace();
-		namespace.getWorkerHandler().sendToAll(new UpdateMatchingStatsMessage(support.getNamespace().getStorage().getAllConcepts()));
-		support.waitUntilWorkDone();
-	}
-
-}
+}
diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java
@@ -17,6 +17,7 @@
 import com.bakdata.conquery.integration.IntegrationTest;
 import com.bakdata.conquery.integration.json.ConqueryTestSpec;
 import com.bakdata.conquery.integration.json.JsonIntegrationTest;
+import com.bakdata.conquery.io.storage.NamespaceStorage;
 import com.bakdata.conquery.models.config.CSVConfig;
 import com.bakdata.conquery.models.datasets.concepts.Concept;
 import com.bakdata.conquery.models.datasets.concepts.Connector;
@@ -151,7 +152,8 @@ private static SelectFilter<?> setupSearch(StandaloneSupport conquery) throws Ex
 
 		final CSVConfig csvConf = conquery.getConfig().getCsv();
 
-		final Concept<?> concept = conquery.getNamespace().getStorage().getAllConcepts().iterator().next();
+		NamespaceStorage namespaceStorage = conquery.getNamespace().getStorage();
+		final Concept<?> concept = namespaceStorage.getAllConcepts().stream().filter(c -> c.getName().equals("geschlecht_select")).findFirst().orElseThrow();
 		final Connector connector = concept.getConnectors().iterator().next();
 		final SelectFilter<?> filter = (SelectFilter<?>) connector.getFilters().iterator().next();