From 65f0b0d691e9713fde0dd484eac738ac3103c089 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 24 May 2022 07:48:32 +0000
Subject: [PATCH 01/41] Bump lombok from 1.18.22 to 1.18.24

Bumps [lombok](https://github.com/projectlombok/lombok) from 1.18.22 to 1.18.24.
- [Release notes](https://github.com/projectlombok/lombok/releases)
- [Changelog](https://github.com/projectlombok/lombok/blob/master/doc/changelog.markdown)
- [Commits](https://github.com/projectlombok/lombok/compare/v1.18.22...v1.18.24)

---
updated-dependencies:
- dependency-name: org.projectlombok:lombok
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 8d48b79044..aa8d82a891 100644
--- a/pom.xml
+++ b/pom.xml
@@ -18,7 +18,7 @@
 
 	<properties>
 		<java.required>11</java.required>
-		<lombok.version>1.18.22</lombok.version>
+		<lombok.version>1.18.24</lombok.version>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<maven.compiler.source>${java.required}</maven.compiler.source>
 		<maven.compiler.target>${java.required}</maven.compiler.target>

From 41cd1eb732551a9d57003242a6f3f49388d80fbd Mon Sep 17 00:00:00 2001
From: awildturtok <1553491+awildturtok@users.noreply.github.com>
Date: Wed, 15 Mar 2023 15:19:24 +0100
Subject: [PATCH 02/41] enforce non-null

---
 .../models/datasets/PreviewConfig.java        | 21 ++++++++++---------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/PreviewConfig.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/PreviewConfig.java
index 5a682b7892..a99d877a47 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/datasets/PreviewConfig.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/PreviewConfig.java
@@ -51,16 +51,19 @@ public class PreviewConfig {
 	 * @implSpec the order of selects is the order of the output fields.
 	 */
 	@Valid
+	@NotNull
 	private List<InfoCardSelect> infoCardSelects = List.of();
 
 	/**
 	 * Columns that should not be displayed to users in entity preview.
 	 */
+	@NotNull
 	private Set<ColumnId> hidden = Collections.emptySet();
 
 	/**
 	 * SecondaryIds where the columns should be grouped together.
 	 */
+	@NotNull
 	private Set<SecondaryIdDescriptionId> grouping = Collections.emptySet();
 
 	/**
@@ -68,11 +71,13 @@ public class PreviewConfig {
 	 *
 	 * @implNote This is purely for the frontend, the backend can theoretically be queried for all Connectors.
 	 */
+	@NotNull
 	private Set<ConnectorId> allConnectors = Collections.emptySet();
 
 	/**
 	 * Connectors that shall be selected by default by the frontend.
 	 */
+	@NotNull
 	private Set<ConnectorId> defaultConnectors = Collections.emptySet();
 
 	/**
@@ -82,6 +87,7 @@ public class PreviewConfig {
 	 * <p>
 	 * The Frontend will use the concepts filters to render a search for entity preview.
 	 */
+	@NotNull
 	private Set<FilterId> searchFilters = Collections.emptySet();
 
 	@JacksonInject(useInput = OptBoolean.FALSE)
@@ -147,24 +153,19 @@ public List<Select> getSelects() {
 	}
 
 	public List<Filter<?>> resolveSearchFilters() {
-		if (searchFilters == null) {
-			return Collections.emptyList();
-		}
-
-		return searchFilters.stream()
-							.map(filterId -> datasetRegistry.findRegistry(filterId.getDataset()).getOptional(filterId))
+		return getSearchFilters().stream()
+							.map(filterId -> getDatasetRegistry().findRegistry(filterId.getDataset()).getOptional(filterId))
 							.flatMap(Optional::stream)
 							.toList();
 	}
 
 	public Concept<?> resolveSearchConcept() {
-		if (searchFilters == null) {
+		if (getSearchFilters().isEmpty()) {
 			return null;
 		}
 
-
-		return searchFilters.stream()
-							.map(filterId -> datasetRegistry.findRegistry(filterId.getDataset()).getOptional(filterId))
+		return getSearchFilters().stream()
+							.map(filterId -> getDatasetRegistry().findRegistry(filterId.getDataset()).getOptional(filterId))
 							.flatMap(Optional::stream)
 							.map(filter -> filter.getConnector().getConcept())
 							.distinct()

From 368c12d970ac2313267133f0c1c8d02f6a079575 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 18 Jun 2024 19:22:23 +0000
Subject: [PATCH 03/41] Bump ws from 6.2.2 to 6.2.3 in /frontend

Bumps [ws](https://github.com/websockets/ws) from 6.2.2 to 6.2.3.
- [Release notes](https://github.com/websockets/ws/releases)
- [Commits](https://github.com/websockets/ws/compare/6.2.2...6.2.3)

---
updated-dependencies:
- dependency-name: ws
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 frontend/package-lock.json | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/frontend/package-lock.json b/frontend/package-lock.json
index 4e5a1879be..24e8a89e3f 100644
--- a/frontend/package-lock.json
+++ b/frontend/package-lock.json
@@ -13317,9 +13317,10 @@
       }
     },
     "node_modules/puppeteer-core/node_modules/ws": {
-      "version": "6.2.2",
+      "version": "6.2.3",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.3.tgz",
+      "integrity": "sha512-jmTjYU0j60B+vHey6TfR3Z7RD61z/hmxBS3VMSGIrroOWXQEneK1zNuotOUrGyBHQj0yrpsLHPWtigEFd13ndA==",
       "dev": true,
-      "license": "MIT",
       "dependencies": {
         "async-limiter": "~1.0.0"
       }
@@ -16572,9 +16573,10 @@
       }
     },
     "node_modules/ws": {
-      "version": "8.14.2",
+      "version": "8.17.1",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz",
+      "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==",
       "dev": true,
-      "license": "MIT",
       "engines": {
         "node": ">=10.0.0"
       },

From ce4b91be5bf1372f6e5b88967138b1df223b677e Mon Sep 17 00:00:00 2001
From: awildturtok <1553491+awildturtok@users.noreply.github.com>
Date: Tue, 2 Jul 2024 13:07:07 +0200
Subject: [PATCH 04/41] drop CBlockLock, as it is not needed and causes severe
 performance degradation

---
 .../conquery/models/events/BucketManager.java |  7 -----
 .../models/jobs/CalculateCBlocksJob.java      | 27 +++++++++----------
 2 files changed, 12 insertions(+), 22 deletions(-)

diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/BucketManager.java b/backend/src/main/java/com/bakdata/conquery/models/events/BucketManager.java
index 2135951817..fe91ec2ba8 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/BucketManager.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/BucketManager.java
@@ -18,10 +18,7 @@
 import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeConnector;
 import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept;
-import com.bakdata.conquery.models.identifiable.IdMutex;
-import com.bakdata.conquery.models.identifiable.IdMutex.Locked;
 import com.bakdata.conquery.models.identifiable.ids.specific.CBlockId;
-import com.bakdata.conquery.models.identifiable.ids.specific.ConnectorId;
 import com.bakdata.conquery.models.jobs.CalculateCBlocksJob;
 import com.bakdata.conquery.models.jobs.JobManager;
 import com.bakdata.conquery.models.query.entity.Entity;
@@ -47,7 +44,6 @@
 @RequiredArgsConstructor
 public class BucketManager {
 
-	private final IdMutex<ConnectorId> cBlockLocks = new IdMutex<>();
 	private final JobManager jobManager;
 	private final WorkerStorage storage;
 
@@ -123,9 +119,6 @@ private static void registerCBlock(CBlock cBlock, Map<Connector, Int2ObjectMap<M
 						.put(cBlock.getBucket(), cBlock);
 	}
 
-	public Locked acquireLock(Connector connector) {
-		return cBlockLocks.acquire(connector.getId());
-	}
 
 	@SneakyThrows
 	public void fullUpdate() {
diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/CalculateCBlocksJob.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/CalculateCBlocksJob.java
index a401694d00..afad884d47 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/jobs/CalculateCBlocksJob.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/CalculateCBlocksJob.java
@@ -10,7 +10,6 @@
 import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.BucketManager;
 import com.bakdata.conquery.models.events.CBlock;
-import com.bakdata.conquery.models.identifiable.IdMutex;
 import com.bakdata.conquery.models.identifiable.ids.specific.CBlockId;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
@@ -47,7 +46,7 @@ public void addCBlock(Bucket bucket, ConceptTreeConnector connector) {
 
 	@Override
 	public void execute() throws Exception {
-		if(infos.isEmpty()){
+		if (infos.isEmpty()) {
 			return;
 		}
 
@@ -56,10 +55,10 @@ public void execute() throws Exception {
 		final ListeningExecutorService executorService = MoreExecutors.listeningDecorator(this.executorService);
 
 		final List<? extends ListenableFuture<?>> futures = infos.stream()
-				.map(this::createInformationProcessor)
-				.map(executorService::submit)
-				.peek(f -> f.addListener(this::incrementProgressReporter, MoreExecutors.directExecutor()))
-				.collect(Collectors.toList());
+																 .map(this::createInformationProcessor)
+																 .map(executorService::submit)
+																 .peek(f -> f.addListener(this::incrementProgressReporter, MoreExecutors.directExecutor()))
+																 .collect(Collectors.toList());
 
 		Futures.allAsList(futures).get();
 	}
@@ -98,17 +97,15 @@ private static class CalculationInformationProcessor implements Runnable {
 		@Override
 		public void run() {
 			try {
-				try(IdMutex.Locked ignored = bucketManager.acquireLock(info.connector)) {
-					if (bucketManager.hasCBlock(info.getCBlockId())) {
-						log.trace("Skipping calculation of CBlock[{}] because its already present in the BucketManager.", info.getCBlockId());
-						return;
-					}
+				if (bucketManager.hasCBlock(info.getCBlockId())) {
+					log.trace("Skipping calculation of CBlock[{}] because its already present in the BucketManager.", info.getCBlockId());
+					return;
+				}
 
-					CBlock cBlock = CBlock.createCBlock(info.getConnector(), info.getBucket(), bucketManager.getEntityBucketSize());
+				CBlock cBlock = CBlock.createCBlock(info.getConnector(), info.getBucket(), bucketManager.getEntityBucketSize());
 
-					bucketManager.addCalculatedCBlock(cBlock);
-					storage.addCBlock(cBlock);
-				}
+				bucketManager.addCalculatedCBlock(cBlock);
+				storage.addCBlock(cBlock);
 			}
 			catch (Exception e) {
 				throw new RuntimeException(

From 4a0e2006392979ea20013f2d3bb1c573d5563f08 Mon Sep 17 00:00:00 2001
From: awildturtok <1553491+awildturtok@users.noreply.github.com>
Date: Tue, 9 Jul 2024 18:01:33 +0200
Subject: [PATCH 05/41] adds error message when async SQL-query fails

---
 .../com/bakdata/conquery/models/error/ConqueryError.java     | 5 +++--
 .../com/bakdata/conquery/models/error/ErrorMessages.java     | 2 +-
 .../bakdata/conquery/models/execution/ManagedExecution.java  | 2 +-
 .../bakdata/conquery/sql/conquery/SqlExecutionManager.java   | 3 ++-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/backend/src/main/java/com/bakdata/conquery/models/error/ConqueryError.java b/backend/src/main/java/com/bakdata/conquery/models/error/ConqueryError.java
index 3630d0de4d..ca21fc24d5 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/error/ConqueryError.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/error/ConqueryError.java
@@ -248,7 +248,7 @@ public String getMessageTemplate(ErrorMessages errorMessages) {
 	@CPSType(base = ConqueryError.class, id = "CQ_EXECUTION_NO_SECONDARY_ID")
 	@RequiredArgsConstructor(onConstructor_ = {@JsonCreator})
 	public static class NoSecondaryIdSelectedError extends ConqueryError {
-				@Override
+		@Override
 		public String getMessageTemplate(ErrorMessages errorMessages) {
 			return errorMessages.noSecondaryIdSelected();
 		}
@@ -257,7 +257,8 @@ public String getMessageTemplate(ErrorMessages errorMessages) {
 	@CPSType(base = ConqueryError.class, id = "CQ_SQL_ERROR")
 	@RequiredArgsConstructor(onConstructor_ = {@JsonCreator})
 	public static class SqlError extends ConqueryError {
-		private final Exception error;
+		@ToString.Include
+		private final Throwable error;
 
 		@Override
 		public String getMessageTemplate(ErrorMessages errorMessages) {
diff --git a/backend/src/main/java/com/bakdata/conquery/models/error/ErrorMessages.java b/backend/src/main/java/com/bakdata/conquery/models/error/ErrorMessages.java
index 027c05368c..a5f9f733c8 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/error/ErrorMessages.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/error/ErrorMessages.java
@@ -58,5 +58,5 @@ public interface ErrorMessages {
 
 	@En("Something went wrong while querying the database: ${0}.")
 	@De("Etwas ist beim Anfragen des Servers fehlgeschlagen: ${0}.")
-	String sqlError(Exception error);
+	String sqlError(Throwable error);
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java
index 52b1688e48..03b9a0d1fd 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java
@@ -205,7 +205,7 @@ public void fail(ConqueryErrorInfo error) {
 		else {
 			this.error = error;
 			// Log the error, so its id is atleast once in the logs
-			log.warn("The execution [{}] failed with:\n\t{}", this.getId(), this.error);
+			log.warn("The execution [{}] failed with:\n\t{}", getId(), getError());
 		}
 
 		finish(ExecutionState.FAILED);
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java
index aa785bb5f3..7d9c88f6d9 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java
@@ -7,6 +7,7 @@
 
 import com.bakdata.conquery.io.storage.MetaStorage;
 import com.bakdata.conquery.models.datasets.Dataset;
+import com.bakdata.conquery.models.error.ConqueryError;
 import com.bakdata.conquery.models.execution.ExecutionState;
 import com.bakdata.conquery.models.execution.InternalExecution;
 import com.bakdata.conquery.models.execution.ManagedExecution;
@@ -80,7 +81,7 @@ private CompletableFuture<Void> executeAsync(ManagedQuery managedQuery) {
 									runningExecutions.remove(managedQuery.getId());
 								})
 								.exceptionally(e -> {
-									managedQuery.finish(ExecutionState.FAILED);
+									managedQuery.fail(new ConqueryError.SqlError(e));
 									runningExecutions.remove(managedQuery.getId());
 									return null;
 								});

From d3b0c3a79ea45511b3158e248e16f55978b68a48 Mon Sep 17 00:00:00 2001
From: awildturtok <1553491+awildturtok@users.noreply.github.com>
Date: Thu, 11 Jul 2024 15:58:36 +0200
Subject: [PATCH 06/41] catch and log exceptions for
 registerColumnValuesInSearch, such errors only cause issues for a single
 Column

---
 .../bakdata/conquery/models/worker/LocalNamespace.java    | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java
index e5a2c8f5aa..b21aabf5bd 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java
@@ -52,8 +52,12 @@ void updateMatchingStats() {
 	@Override
 	void registerColumnValuesInSearch(Set<Column> columns) {
 		for (Column column : columns) {
-			final Stream<String> stringStream = storageHandler.lookupColumnValues(getStorage(), column);
-			getFilterSearch().registerValues(column, stringStream.collect(Collectors.toSet()));
+			try {
+				final Stream<String> stringStream = storageHandler.lookupColumnValues(getStorage(), column);
+				getFilterSearch().registerValues(column, stringStream.collect(Collectors.toSet()));
+			}catch (Exception e) {
+				log.error("Problem collecting column values for {}", column, e);
+			}
 		}
 	}
 

From f6dac561448df3f31e3c6e4830f59ec77092ab9f Mon Sep 17 00:00:00 2001
From: Jonas Arnhold <jonas.arnhold@bakdata.com>
Date: Fri, 12 Jul 2024 13:07:46 +0200
Subject: [PATCH 07/41] Fix SqlConnectorConfig serialization (#3485)

---
 .../conquery/apiv1/QueryProcessor.java        |  3 +-
 .../concept/specific/external/CQExternal.java | 22 ++++-----
 .../mode/local/LocalNamespaceHandler.java     |  6 ++-
 .../conquery/models/config/ColumnConfig.java  | 28 ++++++-----
 .../models/config/DatabaseConfig.java         |  4 ++
 .../models/config/IdColumnConfig.java         | 25 ++++++----
 .../models/config/SqlConnectorConfig.java     |  5 +-
 .../identifiable/mapping/EntityIdMap.java     |  9 +---
 .../resources/api/ConfigResource.java         | 11 +----
 .../sql/conversion/NodeConversions.java       | 22 +++++++--
 .../conquery/sql/conversion/SqlConverter.java |  7 +--
 .../conversion/cqelement/CQYesConverter.java  | 43 +++++++++++++++++
 .../cqelement/ConversionContext.java          |  6 +++
 .../sql/conversion/dialect/SqlDialect.java    |  2 +
 .../com/bakdata/conquery/util/io/Cloner.java  |  1 -
 .../conquery/util/io/IdColumnUtil.java        |  2 +-
 .../integration/IntegrationTests.java         |  5 +-
 .../integration/json/ConqueryTestSpec.java    | 32 +++++++------
 .../sql/dialect/HanaSqlIntegrationTests.java  |  3 +-
 .../sql/dialect/TestSqlConnectorConfig.java   | 11 +++--
 .../resources/tests/sql/yes/CQYES.test.json   | 48 +++++++++++++++++++
 .../test/resources/tests/sql/yes/expected.csv |  7 +++
 .../test/resources/tests/sql/yes/persons.csv  |  7 +++
 23 files changed, 223 insertions(+), 86 deletions(-)
 create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java
 create mode 100644 backend/src/test/resources/tests/sql/yes/CQYES.test.json
 create mode 100644 backend/src/test/resources/tests/sql/yes/expected.csv
 create mode 100644 backend/src/test/resources/tests/sql/yes/persons.csv

diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java
index 52f3294e08..1dbb9753d1 100644
--- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java
+++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java
@@ -526,9 +526,8 @@ public Stream<Map<String, String>> resolveEntities(Subject subject, List<FilterV
 
 		final List<ColumnConfig> ids = config.getIdColumns()
 											 .getIds().stream()
-											 // We're only interested in returning printable AND resolvable ids
+											 // We're only interested in returning printable ids
 											 .filter(ColumnConfig::isPrint)
-											 .filter(ColumnConfig::isResolvable)
 											 .collect(Collectors.toList());
 
 
diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java
index 20dbbdd034..48466e2dd8 100644
--- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java
+++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java
@@ -138,17 +138,17 @@ public void collectRequiredQueries(Set<ManagedExecutionId> requiredQueries) {
 	private ExternalNode<String> createExternalNodeOnlySingle(QueryPlanContext context, ConceptQueryPlan plan, String[] extraHeaders) {
 		// Remove zero element Lists and substitute one element Lists by containing String
 		final Map<String, Map<String, String>> extraFlat = extra.entrySet().stream()
-																 .collect(Collectors.toMap(
-																		 Map.Entry::getKey,
-																		 entityToRowMap -> entityToRowMap.getValue().entrySet().stream()
-																										 .filter(headerToValue -> !headerToValue.getValue()
-																																				.isEmpty())
-																										 .collect(Collectors.toMap(
-																												 Map.Entry::getKey,
-																												 headerToValue -> headerToValue.getValue()
-																																			   .get(0)
-																										 ))
-																 ));
+																.collect(Collectors.toMap(
+																		Map.Entry::getKey,
+																		entityToRowMap -> entityToRowMap.getValue().entrySet().stream()
+																										.filter(headerToValue -> !headerToValue.getValue()
+																																			   .isEmpty())
+																										.collect(Collectors.toMap(
+																												Map.Entry::getKey,
+																												headerToValue -> headerToValue.getValue()
+																																			  .get(0)
+																										))
+																));
 
 		final Map<String, ConstantValueAggregator<String>> extraAggregators = new HashMap<>(extraHeaders.length);
 		for (String extraHeader : extraHeaders) {
diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java
index 52909c50ef..b437423e30 100644
--- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java
+++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java
@@ -7,6 +7,7 @@
 import com.bakdata.conquery.mode.NamespaceSetupData;
 import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.config.DatabaseConfig;
+import com.bakdata.conquery.models.config.IdColumnConfig;
 import com.bakdata.conquery.models.config.SqlConnectorConfig;
 import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId;
 import com.bakdata.conquery.models.index.IndexService;
@@ -15,6 +16,7 @@
 import com.bakdata.conquery.sql.DSLContextWrapper;
 import com.bakdata.conquery.sql.DslContextFactory;
 import com.bakdata.conquery.sql.conquery.SqlExecutionManager;
+import com.bakdata.conquery.sql.conversion.NodeConversions;
 import com.bakdata.conquery.sql.conversion.SqlConverter;
 import com.bakdata.conquery.sql.conversion.dialect.SqlDialect;
 import com.bakdata.conquery.sql.conversion.dialect.SqlDialectFactory;
@@ -38,6 +40,7 @@ public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaSto
 
 		NamespaceSetupData namespaceData = NamespaceHandler.createNamespaceSetup(namespaceStorage, config, mapperCreator, indexService);
 
+		IdColumnConfig idColumns = config.getIdColumns();
 		SqlConnectorConfig sqlConnectorConfig = config.getSqlConnectorConfig();
 		DatabaseConfig databaseConfig = sqlConnectorConfig.getDatabaseConfig(namespaceStorage.getDataset());
 
@@ -45,8 +48,9 @@ public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaSto
 		DSLContext dslContext = dslContextWrapper.getDslContext();
 		SqlDialect sqlDialect = dialectFactory.createSqlDialect(databaseConfig.getDialect());
 
-		SqlConverter sqlConverter = new SqlConverter(sqlDialect, dslContext, databaseConfig);
 		SqlExecutionService sqlExecutionService = new SqlExecutionService(dslContext, ResultSetProcessorFactory.create(sqlDialect));
+		NodeConversions nodeConversions = new NodeConversions(idColumns, sqlDialect, dslContext, databaseConfig, sqlExecutionService);
+		SqlConverter sqlConverter = new SqlConverter(nodeConversions);
 		ExecutionManager<SqlExecutionResult> executionManager = new SqlExecutionManager(sqlConverter, sqlExecutionService, metaStorage);
 		SqlStorageHandler sqlStorageHandler = new SqlStorageHandler(sqlExecutionService);
 
diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java
index 80a98a320b..6c74ce0cc1 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java
@@ -7,6 +7,8 @@
 import com.bakdata.conquery.io.jackson.View;
 import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap;
 import com.bakdata.conquery.resources.admin.rest.AdminDatasetProcessor;
+import com.fasterxml.jackson.annotation.JsonAlias;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonView;
 import com.google.common.base.Strings;
 import jakarta.validation.constraints.NotEmpty;
@@ -20,7 +22,7 @@
 
 /**
  * Configuration class for QueryUpload and IdMapping.
- *
+ * <p>
  * Describes how rows are mapped for {@link EntityIdMap}/{@link AdminDatasetProcessor#setIdMapping(java.io.InputStream, com.bakdata.conquery.models.worker.Namespace)}.
  */
 @Builder
@@ -29,13 +31,10 @@
 @NoArgsConstructor
 @Setter
 @Getter
+@JsonIgnoreProperties(value = {"resolvable"}) // for backwards compatibility
 public class ColumnConfig {
 
 	public EntityIdMap.ExternalId read(String value) {
-		if (!isResolvable()) {
-			return null;
-		}
-
 		if (Strings.isNullOrEmpty(value)) {
 			return null;
 		}
@@ -87,12 +86,6 @@ public EntityIdMap.ExternalId read(String value) {
 	@Builder.Default
 	private int length = -1;
 
-	/**
-	 * Set to true, if the column should be resolvable in upload. This can be used to add supplemental information to an entity, for example it's data-source, which would not be unique among entities.
-	 */
-	@Builder.Default
-	private boolean resolvable = false;
-
 	/**
 	 * Set to true, if the Column should be printed to output. This can be used to have resolvable but not printable fields in mapping.
 	 */
@@ -100,10 +93,19 @@ public EntityIdMap.ExternalId read(String value) {
 	@JsonView(View.Persistence.class)
 	private boolean print = true;
 
+	/*
+	 * Set to true, if the column should be resolvable in upload. This can be used to add supplemental information to an entity, for example it's data-source, which would not be unique among entities.
+	 */
+	@Builder.Default
+	private boolean resolvable = false;
+
 	/**
-	 * Used in conjunction with {@link com.bakdata.conquery.models.identifiable.mapping.AutoIncrementingPseudomizer}: One column is required to have fillAnon true, which will be filled with pseudomized data.
+	 * Used for CQYes to select all entities. And CQExternal as primaryId for decoding. And for IdMapping for outputting additional Ids.
+	 * <p>
+	 * Additionally, used in conjunction with {@link com.bakdata.conquery.models.identifiable.mapping.AutoIncrementingPseudomizer}: One column is required to have fillAnon true, which will be filled with pseudomized data.
 	 */
 	@Builder.Default
 	@JsonView(View.Persistence.class)
-	private boolean fillAnon = false;
+	@JsonAlias("fillAnon")
+	private boolean primaryId = false;
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConfig.java
index 86a6aaa039..db23de6553 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConfig.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConfig.java
@@ -1,12 +1,16 @@
 package com.bakdata.conquery.models.config;
 
+import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 import lombok.extern.jackson.Jacksonized;
 
 @Data
 @Builder
 @Jacksonized
+@NoArgsConstructor
+@AllArgsConstructor
 public class DatabaseConfig {
 
 	private static final String DEFAULT_PRIMARY_COLUMN = "pid";
diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java
index 25d6c050e5..d49d6fcbb7 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java
@@ -10,9 +10,6 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import jakarta.validation.Valid;
-import jakarta.validation.constraints.NotEmpty;
-
 import com.bakdata.conquery.apiv1.query.concept.specific.external.DateFormat;
 import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap;
 import com.bakdata.conquery.models.query.resultinfo.LocalizedDefaultResultInfo;
@@ -23,6 +20,8 @@
 import com.google.common.base.Functions;
 import com.google.common.collect.MoreCollectors;
 import io.dropwizard.validation.ValidationMethod;
+import jakarta.validation.Valid;
+import jakarta.validation.constraints.NotEmpty;
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
@@ -42,6 +41,11 @@
 @Slf4j
 public class IdColumnConfig {
 
+	/**
+	 * Relevant in SQL-Mode, used as AllIdsTable for CQExternal and CQYes.
+	 */
+	private String table;
+
 	/**
 	 * List of resolvable and printable ids.
 	 *
@@ -54,8 +58,7 @@ public class IdColumnConfig {
 						.name("ID")
 						.field("result")
 						.label(Map.of(Locale.ROOT, "result"))
-						.resolvable(true)
-						.fillAnon(true)
+						.primaryId(true)
 						.print(true)
 						.build()
 	);
@@ -64,9 +67,15 @@ public class IdColumnConfig {
 	@JsonIgnore
 	@Setter(AccessLevel.NONE)
 	@Getter(lazy = true, value = AccessLevel.PUBLIC)
-	private final Map<String, ColumnConfig> idMappers = ids.stream().filter(ColumnConfig::isResolvable)
-														   .collect(Collectors.toMap(ColumnConfig::getName, Functions.identity()));
+	private final Map<String, ColumnConfig> idMappers = ids.stream().collect(Collectors.toMap(ColumnConfig::getName, Functions.identity()));
 
+	@JsonIgnore
+	public ColumnConfig findPrimaryIdColumn() {
+		return ids.stream()
+				  .filter(ColumnConfig::isPrimaryId)
+				  .findFirst()
+				  .orElseThrow(() -> new IllegalStateException("Requiring at least 1 primary key column in IdColumnConfig"));
+	}
 
 	@ValidationMethod(message = "Duplicate Claims for Mapping Columns.")
 	@JsonIgnore
@@ -101,7 +110,7 @@ public boolean isIdColsHaveMapping() {
 	public boolean isExactlyOnePseudo() {
 		return ids.stream()
 				  .filter(conf -> conf.getField() != null)
-				  .filter(ColumnConfig::isFillAnon)
+				  .filter(ColumnConfig::isPrimaryId)
 				  .count() == 1;
 	}
 
diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java
index cfce88537b..c8140ef205 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java
@@ -6,11 +6,9 @@
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import io.dropwizard.validation.ValidationMethod;
 import jakarta.validation.Valid;
-import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
-import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.extern.jackson.Jacksonized;
 
@@ -21,7 +19,7 @@
 @AllArgsConstructor
 public class SqlConnectorConfig {
 
-	boolean enabled;
+	private boolean enabled;
 
 	/**
 	 * Determines if generated SQL should be formatted.
@@ -31,7 +29,6 @@ public class SqlConnectorConfig {
 	/**
 	 * Keys must match the name of existing {@link Dataset}s.
 	 */
-	@Getter(AccessLevel.PRIVATE)
 	private Map<String, @Valid DatabaseConfig> databaseConfigs;
 
 	public DatabaseConfig getDatabaseConfig(Dataset dataset) {
diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java
index 4bbb70ce92..83723c0b4b 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java
@@ -22,7 +22,6 @@
 
 /**
  * Mapping from uploaded {@link ExternalId} for resolving in {@link com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal}, and also for printing with {@link EntityPrintId}.
- *
  */
 @Getter
 @EqualsAndHashCode
@@ -67,7 +66,7 @@ public static EntityIdMap generateIdMapping(CsvParser parser, List<ColumnConfig>
 				final String otherId = record.getString(columnConfig.getField());
 
 				// Collect printable parts into id
-				if(columnConfig.isPrint()) {
+				if (columnConfig.isPrint()) {
 					idParts.add(otherId);
 				}
 
@@ -75,10 +74,6 @@ public static EntityIdMap generateIdMapping(CsvParser parser, List<ColumnConfig>
 					continue;
 				}
 
-				if (!columnConfig.isResolvable()) {
-					continue;
-				}
-
 				final ExternalId transformed = columnConfig.read(otherId);
 
 				mapping.addInputMapping(id, transformed);
@@ -141,7 +136,7 @@ public EntityPrintId toExternal(String internal) {
 
 	/**
 	 * Resolve external ID to Entity Id.
-	 *
+	 * <p>
 	 * Return -1 when not resolved.
 	 */
 	public String resolve(ExternalId key) {
diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java
index c641784154..c07e3fbc67 100644
--- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java
+++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java
@@ -4,7 +4,6 @@
 
 import com.bakdata.conquery.apiv1.AdditionalMediaTypes;
 import com.bakdata.conquery.apiv1.frontend.FrontendConfiguration;
-import com.bakdata.conquery.models.config.ColumnConfig;
 import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.config.FrontendConfig;
 import com.bakdata.conquery.models.config.IdColumnConfig;
@@ -25,16 +24,10 @@ public class ConfigResource {
 	@GET
 	@Path("frontend")
 	public FrontendConfiguration getFrontendConfig() {
-		// Filter Ids that are not resolvable
-		final IdColumnConfig idColumns = config.getIdColumns().withIds(config.getIdColumns()
-																			 .getIds()
-																			 .stream()
-																			 .filter(ColumnConfig::isResolvable)
-																			 .toList());
 
+		final IdColumnConfig idColumns = config.getIdColumns().withIds(config.getIdColumns().getIds());
 		final FrontendConfig frontendConfig = config.getFrontend();
 
-
 		return new FrontendConfiguration(
 				VersionInfo.INSTANCE.getVersions(),
 				frontendConfig.getCurrency(),
@@ -45,4 +38,4 @@ public FrontendConfiguration getFrontendConfig() {
 		);
 	}
 
-}
\ No newline at end of file
+}
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java
index 769fe973ed..bbff47b0cd 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java
@@ -2,10 +2,12 @@
 
 import com.bakdata.conquery.apiv1.query.QueryDescription;
 import com.bakdata.conquery.models.config.DatabaseConfig;
+import com.bakdata.conquery.models.config.IdColumnConfig;
 import com.bakdata.conquery.models.query.Visitable;
 import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext;
 import com.bakdata.conquery.sql.conversion.dialect.SqlDialect;
 import com.bakdata.conquery.sql.conversion.model.NameGenerator;
+import com.bakdata.conquery.sql.execution.SqlExecutionService;
 import org.jooq.DSLContext;
 
 /**
@@ -13,21 +15,35 @@
  */
 public class NodeConversions extends Conversions<Visitable, ConversionContext, ConversionContext> {
 
+	private final IdColumnConfig idColumns;
 	private final SqlDialect dialect;
 	private final DatabaseConfig config;
+	private final NameGenerator nameGenerator;
+	private final SqlExecutionService executionService;
 
-	public NodeConversions(SqlDialect dialect, DSLContext dslContext, DatabaseConfig config) {
+	public NodeConversions(
+			IdColumnConfig idColumns,
+			SqlDialect dialect,
+			DSLContext dslContext,
+			DatabaseConfig config,
+			SqlExecutionService executionService
+	) {
 		super(dialect.getNodeConverters(dslContext));
+		this.idColumns = idColumns;
 		this.dialect = dialect;
 		this.config = config;
+		this.nameGenerator = new NameGenerator(config.getDialect().getNameMaxLength());
+		this.executionService = executionService;
 	}
 
 	public ConversionContext convert(QueryDescription queryDescription) {
 		ConversionContext initialCtx = ConversionContext.builder()
+														.idColumns(idColumns)
 														.config(config)
-														.nameGenerator(new NameGenerator(config.getDialect().getNameMaxLength()))
+														.nameGenerator(nameGenerator)
 														.nodeConversions(this)
-														.sqlDialect(this.dialect)
+														.sqlDialect(dialect)
+														.executionService(executionService)
 														.build();
 		return convert(queryDescription, initialCtx);
 	}
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java
index af2a6fa681..a365417ad2 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java
@@ -1,18 +1,15 @@
 package com.bakdata.conquery.sql.conversion;
 
 import com.bakdata.conquery.apiv1.query.QueryDescription;
-import com.bakdata.conquery.models.config.DatabaseConfig;
 import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext;
-import com.bakdata.conquery.sql.conversion.dialect.SqlDialect;
 import com.bakdata.conquery.sql.conversion.model.SqlQuery;
-import org.jooq.DSLContext;
 
 public class SqlConverter {
 
 	private final NodeConversions nodeConversions;
 
-	public SqlConverter(SqlDialect dialect, DSLContext dslContext, DatabaseConfig config) {
-		this.nodeConversions = new NodeConversions(dialect, dslContext, config);
+	public SqlConverter(NodeConversions nodeConversions) {
+		this.nodeConversions = nodeConversions;
 	}
 
 	public SqlQuery convert(QueryDescription queryDescription) {
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java
new file mode 100644
index 0000000000..a1a3186946
--- /dev/null
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java
@@ -0,0 +1,43 @@
+package com.bakdata.conquery.sql.conversion.cqelement;
+
+import com.bakdata.conquery.apiv1.query.CQYes;
+import com.bakdata.conquery.models.config.ColumnConfig;
+import com.bakdata.conquery.sql.conversion.NodeConverter;
+import com.bakdata.conquery.sql.conversion.model.QueryStep;
+import com.bakdata.conquery.sql.conversion.model.Selects;
+import com.bakdata.conquery.sql.conversion.model.SqlIdColumns;
+import org.jooq.Field;
+import org.jooq.Record;
+import org.jooq.impl.DSL;
+
+public class CQYesConverter implements NodeConverter<CQYes> {
+
+	private static final String ALL_IDS_CTE = "all_ids";
+
+	@Override
+	public Class<? extends CQYes> getConversionClass() {
+		return CQYes.class;
+	}
+
+	@Override
+	public ConversionContext convert(CQYes cqYes, ConversionContext context) {
+
+		ColumnConfig primaryColumnConfig = context.getIdColumns().getIds().stream()
+												  .filter(ColumnConfig::isPrimaryId)
+												  .findFirst()
+												  .orElseThrow(() -> new IllegalStateException("SQL mode requires a primary key column in IdColumnConfig"));
+
+		Field<Object> primaryColumn = DSL.field(DSL.name(primaryColumnConfig.getField()));
+		SqlIdColumns ids = new SqlIdColumns(primaryColumn);
+
+		Selects selects = Selects.builder().ids(ids).build();
+		org.jooq.Table<Record> fromTable = DSL.table(DSL.name(context.getIdColumns().getTable()));
+
+		QueryStep cqYesTep = QueryStep.builder()
+									  .cteName(ALL_IDS_CTE)
+									  .selects(selects)
+									  .fromTable(fromTable)
+									  .build();
+		return context.withQueryStep(cqYesTep);
+	}
+}
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java
index f55754e70f..0801e4f504 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java
@@ -8,6 +8,7 @@
 import com.bakdata.conquery.apiv1.query.concept.specific.CQDateRestriction;
 import com.bakdata.conquery.models.common.daterange.CDateRange;
 import com.bakdata.conquery.models.config.DatabaseConfig;
+import com.bakdata.conquery.models.config.IdColumnConfig;
 import com.bakdata.conquery.models.datasets.SecondaryIdDescription;
 import com.bakdata.conquery.sql.conversion.Context;
 import com.bakdata.conquery.sql.conversion.NodeConversions;
@@ -15,6 +16,7 @@
 import com.bakdata.conquery.sql.conversion.model.NameGenerator;
 import com.bakdata.conquery.sql.conversion.model.QueryStep;
 import com.bakdata.conquery.sql.conversion.model.SqlQuery;
+import com.bakdata.conquery.sql.execution.SqlExecutionService;
 import lombok.Builder;
 import lombok.Singular;
 import lombok.Value;
@@ -25,6 +27,8 @@
 @Builder(toBuilder = true)
 public class ConversionContext implements Context {
 
+	IdColumnConfig idColumns;
+
 	DatabaseConfig config;
 
 	NodeConversions nodeConversions;
@@ -33,6 +37,8 @@ public class ConversionContext implements Context {
 
 	NameGenerator nameGenerator;
 
+	SqlExecutionService executionService;
+
 	@Singular
 	List<QueryStep> querySteps;
 
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java
index d201bdf720..1d1a8cd086 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java
@@ -13,6 +13,7 @@
 import com.bakdata.conquery.sql.conversion.cqelement.CQExternalConverter;
 import com.bakdata.conquery.sql.conversion.cqelement.CQNegationConverter;
 import com.bakdata.conquery.sql.conversion.cqelement.CQOrConverter;
+import com.bakdata.conquery.sql.conversion.cqelement.CQYesConverter;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.CQConceptConverter;
 import com.bakdata.conquery.sql.conversion.model.QueryStepTransformer;
 import com.bakdata.conquery.sql.conversion.query.AbsoluteFormQueryConverter;
@@ -60,6 +61,7 @@ default List<NodeConverter<? extends Visitable>> getDefaultNodeConverters(DSLCon
 				new CQAndConverter(),
 				new CQOrConverter(),
 				new CQNegationConverter(),
+				new CQYesConverter(),
 				new CQConceptConverter(),
 				new CQExternalConverter(),
 				new CQReusedQueryConverter(),
diff --git a/backend/src/main/java/com/bakdata/conquery/util/io/Cloner.java b/backend/src/main/java/com/bakdata/conquery/util/io/Cloner.java
index 9c217e30d3..84cf314b16 100644
--- a/backend/src/main/java/com/bakdata/conquery/util/io/Cloner.java
+++ b/backend/src/main/java/com/bakdata/conquery/util/io/Cloner.java
@@ -25,7 +25,6 @@ public static <T> ConqueryConfig clone(ConqueryConfig config, Map<Class<T>, T> i
 					ConqueryConfig.class
 			);
 			clone.setLoggingFactory(config.getLoggingFactory());
-			clone.setSqlConnectorConfig(config.getSqlConnectorConfig());
 			return clone;
 		}
 		catch (IOException e) {
diff --git a/backend/src/main/java/com/bakdata/conquery/util/io/IdColumnUtil.java b/backend/src/main/java/com/bakdata/conquery/util/io/IdColumnUtil.java
index 8c2832477c..3db0a88d50 100644
--- a/backend/src/main/java/com/bakdata/conquery/util/io/IdColumnUtil.java
+++ b/backend/src/main/java/com/bakdata/conquery/util/io/IdColumnUtil.java
@@ -48,7 +48,7 @@ public static IdPrinter getIdPrinter(Subject owner, ManagedExecution execution,
 		final int size = (int) ids.stream().filter(ColumnConfig::isPrint).count();
 
 		final int pos = IntStream.range(0, ids.size())
-								 .filter(idx -> ids.get(idx).isFillAnon())
+								 .filter(idx -> ids.get(idx).isPrimaryId())
 								 .findFirst()
 								 .orElseThrow();
 
diff --git a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java
index d48f99401c..29ace90e72 100644
--- a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java
+++ b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java
@@ -31,6 +31,7 @@
 import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.config.DatabaseConfig;
 import com.bakdata.conquery.models.config.Dialect;
+import com.bakdata.conquery.models.config.SqlConnectorConfig;
 import com.bakdata.conquery.util.support.ConfigOverride;
 import com.bakdata.conquery.util.support.StandaloneSupport;
 import com.bakdata.conquery.util.support.TestConquery;
@@ -63,7 +64,9 @@ public class IntegrationTests {
 		final ObjectMapper mapper = Jackson.MAPPER.copy();
 
 		MAPPER = mapper.setConfig(mapper.getDeserializationConfig().withView(View.Persistence.class))
-					   .setConfig(mapper.getSerializationConfig().withView(View.Persistence.class));
+					   .setConfig(mapper.getSerializationConfig().withView(View.Persistence.class))
+					   // to always deserialize into TestSqlConnectorConfig for our tests
+					   .addMixIn(SqlConnectorConfig.class, TestSqlConnectorConfig.class);
 
 		CONFIG_WRITER = MAPPER.writerFor(ConqueryConfig.class);
 	}
diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java b/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java
index 353fa2db06..07e8aea28a 100644
--- a/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java
+++ b/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java
@@ -15,6 +15,7 @@
 import com.bakdata.conquery.io.storage.MetaStorage;
 import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.config.Dialect;
+import com.bakdata.conquery.models.config.IdColumnConfig;
 import com.bakdata.conquery.models.exceptions.JSONException;
 import com.bakdata.conquery.models.exceptions.ValidatorHelper;
 import com.bakdata.conquery.models.identifiable.Identifiable;
@@ -36,29 +37,27 @@
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
 
+@Setter
+@Getter
 @JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, include = JsonTypeInfo.As.PROPERTY, property = "type")
-@Slf4j @CPSBase
+@Slf4j
+@CPSBase
 public abstract class ConqueryTestSpec {
 
-	@Getter
-	@Setter
 	private String label;
 
-	@Setter
-	@Getter
 	@Nullable
 	private String description;
 
-	@Setter
-	@Getter
 	@Nullable
 	private ConqueryConfig config;
 
-	@Setter
-	@Getter
 	@Nullable
 	SqlSpec sqlSpec;
 
+	@Nullable
+	private IdColumnConfig idColumns;
+
 	public ConqueryConfig overrideConfig(ConqueryConfig config) {
 
 		if (getConfig() != null) {
@@ -67,7 +66,9 @@ public ConqueryConfig overrideConfig(ConqueryConfig config) {
 			return conqueryConfig;
 		}
 
-		return config.withStorage(new NonPersistentStoreFactory());
+		final IdColumnConfig idColumnConfig = idColumns != null ? idColumns : config.getIdColumns();
+		return config.withIdColumns(idColumnConfig)
+					 .withStorage(new NonPersistentStoreFactory());
 	}
 
 	public abstract void executeTest(StandaloneSupport support) throws Exception;
@@ -84,7 +85,7 @@ public static <T> T parseSubTree(TestSupport support, JsonNode node, Class<T> ex
 		return parseSubTree(support, node, expectedClass, null);
 	}
 
-	public static <T> T parseSubTree(TestSupport support, JsonNode node, Class<T> expectedClass, Consumer<T> modifierBeforeValidation) throws IOException, JSONException {
+	public static <T> T parseSubTree(TestSupport support, JsonNode node, Class<T> expectedClass, Consumer<T> modifierBeforeValidation) throws IOException {
 		return parseSubTree(support, node, Jackson.MAPPER.getTypeFactory().constructParametricType(expectedClass, new JavaType[0]), modifierBeforeValidation);
 	}
 
@@ -92,7 +93,7 @@ public static <T> T parseSubTree(TestSupport support, JsonNode node, JavaType ex
 		return parseSubTree(support, node, expectedType, null);
 	}
 
-	public static  <T> T parseSubTree(TestSupport support, JsonNode node, JavaType expectedType, Consumer<T> modifierBeforeValidation) throws IOException, JSONException {
+	public static <T> T parseSubTree(TestSupport support, JsonNode node, JavaType expectedType, Consumer<T> modifierBeforeValidation) throws IOException {
 		final ObjectMapper om = Jackson.MAPPER.copy();
 		ObjectMapper mapper = support.getDataset().injectIntoNew(
 				new SingletonNamespaceCollection(support.getNamespace().getStorage().getCentralRegistry(), support.getMetaStorage().getCentralRegistry())
@@ -114,7 +115,8 @@ public static  <T> T parseSubTree(TestSupport support, JsonNode node, JavaType e
 		return result;
 	}
 
-	public static <T> List<T> parseSubTreeList(TestSupport support, ArrayNode node, Class<?> expectedType, Consumer<T> modifierBeforeValidation) throws IOException, JSONException {
+	public static <T> List<T> parseSubTreeList(TestSupport support, ArrayNode node, Class<?> expectedType, Consumer<T> modifierBeforeValidation)
+			throws IOException {
 		final ObjectMapper om = Jackson.MAPPER.copy();
 		ObjectMapper mapper = support.getDataset().injectInto(
 				new SingletonNamespaceCollection(support.getNamespace().getStorage().getCentralRegistry()).injectIntoNew(
@@ -137,8 +139,8 @@ public static <T> List<T> parseSubTreeList(TestSupport support, ArrayNode node,
 					try {
 						value = mapper.readerFor(expectedType).readValue(IntegrationTest.class.getResource(potentialPath));
 					}
-					catch(Exception e2) {
-						throw new RuntimeException("Could not parse value "+potentialPath, e2);
+					catch (Exception e2) {
+						throw new RuntimeException("Could not parse value " + potentialPath, e2);
 					}
 				}
 				else {
diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/HanaSqlIntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/HanaSqlIntegrationTests.java
index 9ec5ea4d3e..e2ca8cf4c1 100644
--- a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/HanaSqlIntegrationTests.java
+++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/HanaSqlIntegrationTests.java
@@ -48,7 +48,8 @@
 @Slf4j
 public class HanaSqlIntegrationTests extends IntegrationTests {
 
-	private final static DockerImageName HANA_IMAGE = DockerImageName.parse("saplabs/hanaexpress:2.00.072.00.20231123.1");
+	// SAP does not provide more than 1 image and on an update, the earlier image is deleted from dockerhub, thus latest tag is fine
+	private final static DockerImageName HANA_IMAGE = DockerImageName.parse("saplabs/hanaexpress:latest");
 	private static final Path TMP_HANA_MOUNT_DIR = Paths.get("/tmp/data/hana");
 	private static boolean useLocalHanaDb = true;
 	private static DSLContextWrapper dslContextWrapper;
diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlConnectorConfig.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlConnectorConfig.java
index ec5dccddc8..5eeddd3500 100644
--- a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlConnectorConfig.java
+++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlConnectorConfig.java
@@ -5,19 +5,22 @@
 import com.bakdata.conquery.models.config.DatabaseConfig;
 import com.bakdata.conquery.models.config.SqlConnectorConfig;
 import com.bakdata.conquery.models.datasets.Dataset;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import lombok.NoArgsConstructor;
 
+@NoArgsConstructor
+@JsonDeserialize(as = TestSqlConnectorConfig.class)
 public class TestSqlConnectorConfig extends SqlConnectorConfig {
 
-	private final DatabaseConfig databaseConfig;
+	private static final String TEST_DATASET = "test";
 
 	public TestSqlConnectorConfig(DatabaseConfig databaseConfig) {
-		super(true, true, Map.of());
-		this.databaseConfig = databaseConfig;
+		super(true, true, Map.of(TEST_DATASET, databaseConfig));
 	}
 
 	@Override
 	public DatabaseConfig getDatabaseConfig(Dataset dataset) {
-		return databaseConfig;
+		return getDatabaseConfigs().get(TEST_DATASET);
 	}
 
 }
diff --git a/backend/src/test/resources/tests/sql/yes/CQYES.test.json b/backend/src/test/resources/tests/sql/yes/CQYES.test.json
new file mode 100644
index 0000000000..e5b9e5fe3b
--- /dev/null
+++ b/backend/src/test/resources/tests/sql/yes/CQYES.test.json
@@ -0,0 +1,48 @@
+{
+  "type": "QUERY_TEST",
+  "label": "CQYES Test",
+  "expectedCsv": "tests/sql/yes/expected.csv",
+  "query": {
+    "type": "CONCEPT_QUERY",
+    "root": {
+      "type": "YES"
+    }
+  },
+  "concepts": [
+  ],
+  "idColumns": {
+    "table": "persons",
+    "ids": [
+      {
+        "name": "PID",
+        "field": "pid",
+        "primaryId": true,
+        "label": {
+          "de": "PID"
+        },
+        "description": {
+          "en": "Pid of an Entity.",
+          "de": "Pid einer Entität."
+        }
+      }
+    ]
+  },
+  "content": {
+    "tables": [
+      {
+        "csv": "tests/sql/yes/persons.csv",
+        "name": "persons",
+        "primaryColumn": {
+          "name": "pid",
+          "type": "STRING"
+        },
+        "columns": [
+          {
+            "name": "datum",
+            "type": "DATE"
+          }
+        ]
+      }
+    ]
+  }
+}
diff --git a/backend/src/test/resources/tests/sql/yes/expected.csv b/backend/src/test/resources/tests/sql/yes/expected.csv
new file mode 100644
index 0000000000..edf163f57b
--- /dev/null
+++ b/backend/src/test/resources/tests/sql/yes/expected.csv
@@ -0,0 +1,7 @@
+PID,dates
+4,{}
+5,{}
+6,{}
+1,{}
+2,{}
+3,{}
diff --git a/backend/src/test/resources/tests/sql/yes/persons.csv b/backend/src/test/resources/tests/sql/yes/persons.csv
new file mode 100644
index 0000000000..5499484b3d
--- /dev/null
+++ b/backend/src/test/resources/tests/sql/yes/persons.csv
@@ -0,0 +1,7 @@
+pid,datum
+4,2013-11-10
+5,2013-11-10
+6,2013-11-10
+1,2012-01-01
+2,2013-11-10
+3,2013-11-10

From 105a7e3b686944150196c969462cb5d3aaaba566 Mon Sep 17 00:00:00 2001
From: awildturtok <1553491+awildturtok@users.noreply.github.com>
Date: Mon, 15 Jul 2024 14:36:30 +0200
Subject: [PATCH 08/41] use ConcurrentHashMap in ConceptTreeCache.java as the
 prior impl can suffer race-conditions.

---
 .../concepts/tree/ConceptTreeCache.java       | 23 +++++++++----------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeCache.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeCache.java
index c23469d152..5875e89264 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeCache.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeCache.java
@@ -1,27 +1,29 @@
 package com.bakdata.conquery.models.datasets.concepts.tree;
 
-import java.util.HashMap;
 import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
 
 import com.bakdata.conquery.models.exceptions.ConceptConfigurationException;
 import com.bakdata.conquery.util.CalculatedValue;
 import com.fasterxml.jackson.annotation.JsonIgnore;
-import lombok.Getter;
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
 
 /**
  * Cache for ConceptTree index searches.
  */
+@RequiredArgsConstructor
+@Data
 public class ConceptTreeCache {
 
 	/**
 	 * Statistics for Cache.
 	 */
-	@Getter
 	private int hits;
 	/**
 	 * Statistics for Cache.
 	 */
-	@Getter
 	private int misses;
 
 	@JsonIgnore
@@ -29,15 +31,12 @@ public class ConceptTreeCache {
 
 	/**
 	 * Store of all cached values.
+	 *
+	 * @implNote ConcurrentHashMap does not allow null values, but we want to have null values in the map. So we wrap the values in Optional.
 	 */
-
 	@JsonIgnore
-	private final Map<String, ConceptTreeChild> cached;
+	private final Map<String, Optional<ConceptTreeChild>> cached = new ConcurrentHashMap<>();;
 
-	public ConceptTreeCache(TreeConcept treeConcept) {
-		this.treeConcept = treeConcept;
-		cached = new HashMap<>();
-	}
 
 	/**
 	 * If id is already in cache, use that. If not calculate it by querying treeConcept. If rowMap was not used to query, cache the response.
@@ -48,7 +47,7 @@ public ConceptTreeChild findMostSpecificChild(String value, CalculatedValue<Map<
 
 		if(cached.containsKey(value)) {
 			hits++;
-			return cached.get(value);
+			return cached.get(value).orElse(null);
 		}
 
 		misses++;
@@ -56,7 +55,7 @@ public ConceptTreeChild findMostSpecificChild(String value, CalculatedValue<Map<
 		final ConceptTreeChild child = treeConcept.findMostSpecificChild(value, rowMap);
 
 		if(!rowMap.isCalculated()) {
-			cached.put(value, child);
+			cached.put(value, Optional.ofNullable(child));
 		}
 
 		return child;

From a845aef8852d01b0ea37eb5ad2ad5358cc007d63 Mon Sep 17 00:00:00 2001
From: Jonas Arnhold <jonas.arnhold@bakdata.com>
Date: Mon, 15 Jul 2024 16:28:45 +0200
Subject: [PATCH 09/41] Remove order statement from SQL query (#3491)

---
 .../sql/conversion/model/QueryStepTransformer.java     | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java
index 8690587890..1c80762f63 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java
@@ -5,12 +5,10 @@
 
 import org.jooq.CommonTableExpression;
 import org.jooq.DSLContext;
-import org.jooq.Field;
 import org.jooq.Record;
 import org.jooq.Select;
 import org.jooq.SelectConditionStep;
 import org.jooq.SelectHavingStep;
-import org.jooq.SelectSeekStepN;
 import org.jooq.impl.DSL;
 
 /**
@@ -40,15 +38,11 @@ public Select<Record> toSelectQuery(QueryStep queryStep) {
 			grouped = queryBase.groupBy(queryStep.getGroupBy());
 		}
 
-		// ordering
-		List<Field<?>> orderByFields = queryStep.getSelects().getIds().toFields();
-		SelectSeekStepN<Record> ordered = grouped.orderBy(orderByFields);
-
 		// union
 		if (!queryStep.isUnion()) {
-			return ordered;
+			return grouped;
 		}
-		return union(queryStep, ordered);
+		return union(queryStep, grouped);
 	}
 
 	private List<CommonTableExpression<Record>> constructPredecessorCteList(QueryStep queryStep) {

From edc3bf1cb07a0b76348db9d4ccd5345301337761 Mon Sep 17 00:00:00 2001
From: Jonas Arnhold <jonas.arnhold@bakdata.com>
Date: Mon, 15 Jul 2024 17:46:40 +0200
Subject: [PATCH 10/41] Fix EXISTS select conversion on connector level (#3493)

---
 .../concept/AggregationFilterCte.java         | 10 ++-
 .../tests/sql/combined/combined.json          |  5 ++
 .../resources/tests/sql/combined/expected.csv | 12 ++--
 .../exists/exists_on_connector/content.csv    |  8 +++
 .../exists_on_connector/exists.spec.json      | 61 +++++++++++++++++++
 .../exists/exists_on_connector/expected.csv   |  6 ++
 6 files changed, 93 insertions(+), 9 deletions(-)
 create mode 100644 backend/src/test/resources/tests/sql/selects/exists/exists_on_connector/content.csv
 create mode 100644 backend/src/test/resources/tests/sql/selects/exists/exists_on_connector/exists.spec.json
 create mode 100644 backend/src/test/resources/tests/sql/selects/exists/exists_on_connector/expected.csv

diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java
index 0ada55f076..85179b7043 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java
@@ -1,7 +1,6 @@
 package com.bakdata.conquery.sql.conversion.cqelement.concept;
 
 import java.util.List;
-import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
 import com.bakdata.conquery.sql.conversion.model.QueryStep;
@@ -39,8 +38,13 @@ private Selects collectSelects(CQTableContext tableContext) {
 		List<SqlSelect> forAggregationFilterStep =
 				tableContext.allSqlSelects().stream()
 							.flatMap(sqlSelects -> sqlSelects.getFinalSelects().stream())
-							.filter(Predicate.not(SqlSelect::isUniversal))
-							.map(sqlSelect -> sqlSelect.qualify(previous.getCteName()))
+							.map(sqlSelect -> {
+								// universal selects like an ExistsSelect have no predecessor in preceding CTE
+								if (sqlSelect.isUniversal()) {
+									return sqlSelect;
+								}
+								return sqlSelect.qualify(previous.getCteName());
+							})
 							.collect(Collectors.toList());
 
 		return Selects.builder()
diff --git a/backend/src/test/resources/tests/sql/combined/combined.json b/backend/src/test/resources/tests/sql/combined/combined.json
index 3e9b99ce63..e7a04869d9 100644
--- a/backend/src/test/resources/tests/sql/combined/combined.json
+++ b/backend/src/test/resources/tests/sql/combined/combined.json
@@ -16,6 +16,7 @@
         {
           "id": "concept.connector",
           "selects": [
+            "concept.connector.exists",
             "concept.connector.event-date",
             "concept.connector.event_duration_sum",
             "concept.connector.first_value",
@@ -83,6 +84,10 @@
             }
           ],
           "selects": [
+            {
+              "type": "EXISTS",
+              "name": "exists"
+            },
             {
               "type": "EVENT_DATE_UNION",
               "name": "event-date"
diff --git a/backend/src/test/resources/tests/sql/combined/expected.csv b/backend/src/test/resources/tests/sql/combined/expected.csv
index a87c0031ab..4902ce95be 100644
--- a/backend/src/test/resources/tests/sql/combined/expected.csv
+++ b/backend/src/test/resources/tests/sql/combined/expected.csv
@@ -1,6 +1,6 @@
-result,dates,concept exists,concept event-date,concept event_duration_sum,concept connector event-date,concept connector event_duration_sum,concept connector first_value,concept connector sum_distinct,concept tree_label first_test_column
-1,"{2012-01-01/2012-01-01,2013-01-01/2013-12-31,2015-01-01/2015-01-01}",1,"{2012-01-01/2012-01-01,2013-01-01/2013-12-31,2015-01-01/2015-01-01}",367,"{2012-01-01/2012-01-01,2013-01-01/2013-12-31}",366,1.0,2.0,A1
-2,"{2010-07-15/2010-07-15,2012-01-01/2012-01-02,2012-01-05/2012-01-10}",1,"{2010-07-15/2010-07-15,2012-01-01/2012-01-02,2012-01-05/2012-01-10}",9,"{2012-01-01/2012-01-02,2012-01-05/2012-01-10}",8,1.01,2.01,B2
-3,"{2012-01-01/2012-01-03,2013-11-10/2013-11-10}",1,"{2012-01-01/2012-01-03,2013-11-10/2013-11-10}",4,{2012-01-01/2012-01-03},3,0.5,0.5,A1
-4,"{2012-01-01/2012-01-04,2012-11-11/2012-11-11}",1,"{2012-01-01/2012-01-04,2012-11-11/2012-11-11}",5,{2012-01-01/2012-01-04},4,0.5,0.5,B2
-5,{2012-01-01/2012-01-05},1,{2012-01-01/2012-01-05},5,{2012-01-01/2012-01-05},5,1.0,1.5,
+result,dates,concept exists,concept event-date,concept event_duration_sum,concept connector exists,concept connector event-date,concept connector event_duration_sum,concept connector first_value,concept connector sum_distinct,concept tree_label first_test_column
+1,"{2012-01-01/2012-01-01,2013-01-01/2013-12-31,2015-01-01/2015-01-01}",1,"{2012-01-01/2012-01-01,2013-01-01/2013-12-31,2015-01-01/2015-01-01}",367,1,"{2012-01-01/2012-01-01,2013-01-01/2013-12-31}",366,1.0,2.0,A1
+2,"{2010-07-15/2010-07-15,2012-01-01/2012-01-02,2012-01-05/2012-01-10}",1,"{2010-07-15/2010-07-15,2012-01-01/2012-01-02,2012-01-05/2012-01-10}",9,1,"{2012-01-01/2012-01-02,2012-01-05/2012-01-10}",8,1.01,2.01,B2
+3,"{2012-01-01/2012-01-03,2013-11-10/2013-11-10}",1,"{2012-01-01/2012-01-03,2013-11-10/2013-11-10}",4,1,{2012-01-01/2012-01-03},3,0.5,0.5,A1
+4,"{2012-01-01/2012-01-04,2012-11-11/2012-11-11}",1,"{2012-01-01/2012-01-04,2012-11-11/2012-11-11}",5,1,{2012-01-01/2012-01-04},4,0.5,0.5,B2
+5,{2012-01-01/2012-01-05},1,{2012-01-01/2012-01-05},5,1,{2012-01-01/2012-01-05},5,1.0,1.5,
diff --git a/backend/src/test/resources/tests/sql/selects/exists/exists_on_connector/content.csv b/backend/src/test/resources/tests/sql/selects/exists/exists_on_connector/content.csv
new file mode 100644
index 0000000000..f3fe352607
--- /dev/null
+++ b/backend/src/test/resources/tests/sql/selects/exists/exists_on_connector/content.csv
@@ -0,0 +1,8 @@
+pid,datum_start,datum_end
+1,2014-06-30,2015-06-30
+2,2014-06-30,2015-06-30
+2,2014-06-30,2015-06-30
+3,2014-06-30,2015-06-30
+4,2014-06-30,2015-06-30
+5,2014-06-30,2015-06-30
+5,2014-06-30,2015-06-30
diff --git a/backend/src/test/resources/tests/sql/selects/exists/exists_on_connector/exists.spec.json b/backend/src/test/resources/tests/sql/selects/exists/exists_on_connector/exists.spec.json
new file mode 100644
index 0000000000..c3592c4365
--- /dev/null
+++ b/backend/src/test/resources/tests/sql/selects/exists/exists_on_connector/exists.spec.json
@@ -0,0 +1,61 @@
+{
+  "type": "QUERY_TEST",
+  "sqlSpec": {
+    "isEnabled": true
+  },
+  "label": "EXISTS select on connector",
+  "expectedCsv": "tests/sql/selects/exists/exists_on_connector/expected.csv",
+  "query": {
+    "type": "CONCEPT_QUERY",
+    "root": {
+      "ids": [
+        "concept"
+      ],
+      "type": "CONCEPT",
+      "tables": [
+        {
+          "id": "concept.connector",
+          "selects": "concept.connector.exists"
+        }
+      ]
+    }
+  },
+  "concepts": [
+    {
+      "name": "concept",
+      "type": "TREE",
+      "connectors": [
+        {
+          "label": "connector",
+          "table": "exists_table",
+          "selects": {
+            "type": "EXISTS",
+            "name": "exists"
+          }
+        }
+      ]
+    }
+  ],
+  "content": {
+    "tables": [
+      {
+        "csv": "tests/sql/selects/exists/exists_on_connector/content.csv",
+        "name": "exists_table",
+        "primaryColumn": {
+          "name": "pid",
+          "type": "STRING"
+        },
+        "columns": [
+          {
+            "name": "datum_start",
+            "type": "DATE"
+          },
+          {
+            "name": "datum_end",
+            "type": "DATE"
+          }
+        ]
+      }
+    ]
+  }
+}
diff --git a/backend/src/test/resources/tests/sql/selects/exists/exists_on_connector/expected.csv b/backend/src/test/resources/tests/sql/selects/exists/exists_on_connector/expected.csv
new file mode 100644
index 0000000000..2760828d6b
--- /dev/null
+++ b/backend/src/test/resources/tests/sql/selects/exists/exists_on_connector/expected.csv
@@ -0,0 +1,6 @@
+result,dates,concept exists
+2,{},1
+4,{},1
+3,{},1
+5,{},1
+1,{},1

From 86daceb1fe1fa8ea34987235f9c26383e7a3655c Mon Sep 17 00:00:00 2001
From: Jonas Arnhold <jonas.arnhold@bakdata.com>
Date: Tue, 16 Jul 2024 16:08:41 +0200
Subject: [PATCH 11/41] Implement DistinctSelect conversion (#3494)

---
 .../select/connector/DistinctSelect.java      |   7 +
 .../conquery/models/types/ResultType.java     |  40 ++++--
 .../dialect/HanaSqlFunctionProvider.java      |  12 --
 .../dialect/PostgreSqlFunctionProvider.java   |  12 --
 .../dialect/SqlFunctionProvider.java          |  13 +-
 .../sql/conversion/model/QueryStep.java       |   4 +
 .../model/QueryStepTransformer.java           |  14 +-
 .../model/select/DistinctSelectConverter.java | 135 ++++++++++++++++++
 .../execution/DefaultResultSetProcessor.java  |  39 ++++-
 .../sql/execution/ResultSetProcessor.java     |  10 ++
 .../tests/sql/combined/combined.json          |   8 +-
 .../resources/tests/sql/combined/expected.csv |  12 +-
 .../tests/sql/selects/distinct/content.csv    |  14 ++
 .../tests/sql/selects/distinct/distinct.json  |  65 +++++++++
 .../tests/sql/selects/distinct/expected.csv   |   6 +
 15 files changed, 344 insertions(+), 47 deletions(-)
 create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java
 create mode 100644 backend/src/test/resources/tests/sql/selects/distinct/content.csv
 create mode 100644 backend/src/test/resources/tests/sql/selects/distinct/distinct.json
 create mode 100644 backend/src/test/resources/tests/sql/selects/distinct/expected.csv

diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java
index 4cd7c460a1..50a1316eff 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java
@@ -9,6 +9,8 @@
 import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator;
 import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.AllValuesAggregator;
 import com.bakdata.conquery.models.types.ResultType;
+import com.bakdata.conquery.sql.conversion.model.select.DistinctSelectConverter;
+import com.bakdata.conquery.sql.conversion.model.select.SelectConverter;
 import com.fasterxml.jackson.annotation.JsonCreator;
 
 @CPSType(id = "DISTINCT", base = Select.class)
@@ -29,4 +31,9 @@ public Aggregator<?> createAggregator() {
 	public ResultType getResultType() {
 		return new ResultType.ListT(super.getResultType());
 	}
+
+	@Override
+	public SelectConverter<DistinctSelect> createConverter() {
+		return new DistinctSelectConverter();
+	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java b/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java
index f4ede6725c..02c82abfe1 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java
@@ -49,9 +49,7 @@ protected String print(PrintSettings cfg, @NonNull Object f) {
 
 	public abstract T getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException;
 
-	protected List<T> getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
-		throw new UnsupportedOperationException("ResultType list of type %s not supported for now.".formatted(getClass().getSimpleName()));
-	}
+	protected abstract List<T> getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException;
 
 	public static ResultType<?> resolveResultType(MajorTypeId majorTypeId) {
 		return switch (majorTypeId) {
@@ -99,6 +97,11 @@ public String print(PrintSettings cfg, Object f) {
 		public Boolean getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
 			return resultSetProcessor.getBoolean(resultSet, columnIndex);
 		}
+
+		@Override
+		protected List<Boolean> getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
+			return resultSetProcessor.getBooleanList(resultSet, columnIndex);
+		}
 	}
 
 
@@ -120,6 +123,11 @@ public String print(PrintSettings cfg, Object f) {
 		public Integer getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
 			return resultSetProcessor.getInteger(resultSet, columnIndex);
 		}
+
+		@Override
+		protected List<Integer> getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
+			return resultSetProcessor.getIntegerList(resultSet, columnIndex);
+		}
 	}
 
 	@CPSType(id = "NUMERIC", base = ResultType.class)
@@ -140,6 +148,11 @@ public String print(PrintSettings cfg, Object f) {
 		public Double getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
 			return resultSetProcessor.getDouble(resultSet, columnIndex);
 		}
+
+		@Override
+		protected List<Double> getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
+			return resultSetProcessor.getDoubleList(resultSet, columnIndex);
+		}
 	}
 
 	@CPSType(id = "DATE", base = ResultType.class)
@@ -162,6 +175,11 @@ public Number getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetPr
 			return resultSetProcessor.getDate(resultSet, columnIndex);
 		}
 
+		@Override
+		protected List<Number> getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
+			return resultSetProcessor.getDateList(resultSet, columnIndex);
+		}
+
 		public static String print(Number num, DateTimeFormatter formatter) {
 			return formatter.format(LocalDate.ofEpochDay(num.intValue()));
 		}
@@ -276,6 +294,11 @@ public BigDecimal getFromResultSet(ResultSet resultSet, int columnIndex, ResultS
 		public BigDecimal readIntermediateValue(PrintSettings cfg, Number f) {
 			return new BigDecimal(f.longValue()).movePointLeft(cfg.getCurrency().getDefaultFractionDigits());
 		}
+
+		@Override
+		protected List<BigDecimal> getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
+			return resultSetProcessor.getMoneyList(resultSet, columnIndex);
+		}
 	}
 
 	@CPSType(id = "LIST", base = ResultType.class)
@@ -313,11 +336,12 @@ public String typeInfo() {
 
 		@Override
 		public List<T> getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
-			if (elementType.getClass() == DateRangeT.class || elementType.getClass() == StringT.class) {
-				return elementType.getFromResultSetAsList(resultSet, columnIndex, resultSetProcessor);
-			}
-			// TODO handle all other list types properly
-			throw new UnsupportedOperationException("Other result type lists not supported for now.");
+			return elementType.getFromResultSetAsList(resultSet, columnIndex, resultSetProcessor);
+		}
+
+		@Override
+		protected List<List<T>> getFromResultSetAsList(final ResultSet resultSet, final int columnIndex, final ResultSetProcessor resultSetProcessor) {
+			throw new UnsupportedOperationException("Nested lists not supported in SQL mode");
 		}
 
 		@Override
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java
index 26a9c4eb57..a4c40c0d86 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java
@@ -155,18 +155,6 @@ public QueryStep unnestValidityDate(QueryStep predecessor, String cteName) {
 		return predecessor;
 	}
 
-	@Override
-	public Field<String> stringAggregation(Field<String> stringField, Field<String> delimiter, List<Field<?>> orderByFields) {
-		return DSL.field(
-				"{0}({1}, {2} {3})",
-				String.class,
-				DSL.keyword("STRING_AGG"),
-				stringField,
-				delimiter,
-				DSL.orderBy(orderByFields)
-		);
-	}
-
 	@Override
 	public Field<String> daterangeStringAggregation(ColumnDateRange columnDateRange) {
 
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java
index 23128c0fc4..5d23751f56 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java
@@ -164,18 +164,6 @@ public QueryStep unnestValidityDate(QueryStep predecessor, String cteName) {
 						.build();
 	}
 
-	@Override
-	public Field<String> stringAggregation(Field<String> stringField, Field<String> delimiter, List<Field<?>> orderByFields) {
-		return DSL.field(
-				"{0}({1}, {2} {3})",
-				String.class,
-				DSL.keyword("string_agg"),
-				stringField,
-				delimiter,
-				DSL.orderBy(orderByFields)
-		);
-	}
-
 	@Override
 	public Field<String> daterangeStringAggregation(ColumnDateRange columnDateRange) {
 		Field<Object> asMultirange = rangeAgg(columnDateRange);
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java
index 1d7e09644e..9a0de8834b 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java
@@ -98,8 +98,6 @@ public interface SqlFunctionProvider {
 	 */
 	QueryStep unnestValidityDate(QueryStep predecessor, String cteName);
 
-	Field<String> stringAggregation(Field<String> stringField, Field<String> delimiter, List<Field<?>> orderByFields);
-
 	/**
 	 * Aggregates the start and end columns of the validity date of entries into one compound string expression.
 	 * <p>
@@ -140,6 +138,17 @@ public interface SqlFunctionProvider {
 	 */
 	Field<String> yearQuarter(Field<Date> dateField);
 
+	default Field<String> stringAggregation(Field<String> stringField, Field<String> delimiter, List<Field<?>> orderByFields) {
+		return DSL.field(
+				"{0}({1}, {2} {3})",
+				String.class,
+				DSL.keyword("string_agg"),
+				stringField,
+				delimiter,
+				DSL.orderBy(orderByFields)
+		);
+	}
+
 	default Field<String> concat(List<Field<String>> fields) {
 		String concatenated = fields.stream()
 									// if a field is null, the whole concatenation would be null - but we just want to skip this field in this case,
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java
index f2f9b80cac..204bdd59e4 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java
@@ -41,6 +41,10 @@ public class QueryStep {
 	 */
 	@Builder.Default
 	boolean unionAll = true;
+	/**
+	 * Determines if the select should be distinct.
+	 */
+	boolean selectDistinct;
 	/**
 	 * All {@link QueryStep}'s that shall be converted before this {@link QueryStep}.
 	 */
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java
index 1c80762f63..04dcf0cd84 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java
@@ -9,6 +9,7 @@
 import org.jooq.Select;
 import org.jooq.SelectConditionStep;
 import org.jooq.SelectHavingStep;
+import org.jooq.SelectSelectStep;
 import org.jooq.impl.DSL;
 
 /**
@@ -70,10 +71,15 @@ private CommonTableExpression<Record> toCte(QueryStep queryStep) {
 
 	private Select<Record> toSelectStep(QueryStep queryStep) {
 
-		Select<Record> selectStep = this.dslContext
-				.select(queryStep.getSelects().all())
-				.from(queryStep.getFromTables())
-				.where(queryStep.getConditions());
+		SelectSelectStep<Record> selectClause;
+		if (queryStep.isSelectDistinct()) {
+			selectClause = dslContext.selectDistinct(queryStep.getSelects().all());
+		}
+		else {
+			selectClause = dslContext.select(queryStep.getSelects().all());
+		}
+
+		Select<Record> selectStep = selectClause.from(queryStep.getFromTables()).where(queryStep.getConditions());
 
 		if (queryStep.isGroupBy()) {
 			selectStep = ((SelectConditionStep<Record>) selectStep).groupBy(queryStep.getGroupBy());
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java
new file mode 100644
index 0000000000..ec2bf25a5f
--- /dev/null
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java
@@ -0,0 +1,135 @@
+package com.bakdata.conquery.sql.conversion.model.select;
+
+import static org.jooq.impl.DSL.field;
+import static org.jooq.impl.DSL.name;
+
+import java.util.List;
+import java.util.Optional;
+
+import com.bakdata.conquery.models.datasets.concepts.Connector;
+import com.bakdata.conquery.models.datasets.concepts.select.connector.DistinctSelect;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables;
+import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider;
+import com.bakdata.conquery.sql.conversion.model.CteStep;
+import com.bakdata.conquery.sql.conversion.model.QueryStep;
+import com.bakdata.conquery.sql.conversion.model.Selects;
+import com.bakdata.conquery.sql.conversion.model.SqlIdColumns;
+import com.bakdata.conquery.sql.execution.ResultSetProcessor;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import org.jooq.Field;
+import org.jooq.impl.DSL;
+import org.jooq.impl.SQLDataType;
+
+/**
+ * <pre>
+ *  The two additional CTEs this aggregator creates:
+ * 	<ol>
+ * 	    <li>
+ * 	        Select distinct values of a column.
+ *            {@code
+ * 	        	"distinct" as (
+ *     				select distinct "pid", "column"
+ *     				from "event_filter"
+ *  			)
+ *            }
+ * 	    </li>
+ * 	    <li>
+ * 	        String agg all distinct values of the column.
+ *            {@code
+ * 	        "aggregated" as (
+ *    			 select
+ *    			   "select-1-distinct"."pid",
+ *    			   string_agg(cast("column" as varchar), cast(' ' as varchar) ) as "select-1"
+ *    			 from "distinct"
+ *    			 group by "pid"
+ *   			)
+ *            }
+ * 	    </li>
+ * 	</ol>
+ * </pre>
+ */
+public class DistinctSelectConverter implements SelectConverter<DistinctSelect> {
+
+	@Getter
+	@RequiredArgsConstructor
+	private enum DistinctSelectCteStep implements CteStep {
+
+		DISTINCT_SELECT("distinct", null),
+		STRING_AGG("aggregated", DISTINCT_SELECT);
+
+		private final String suffix;
+		private final DistinctSelectCteStep predecessor;
+	}
+
+	@Override
+	public ConnectorSqlSelects connectorSelect(DistinctSelect distinctSelect, SelectContext<Connector, ConnectorSqlTables> selectContext) {
+
+		String alias = selectContext.getNameGenerator().selectName(distinctSelect);
+
+		ConnectorSqlTables tables = selectContext.getTables();
+		FieldWrapper<Object> preprocessingSelect = new FieldWrapper<>(field(name(tables.getRootTable(), distinctSelect.getColumn().getName())).as(alias));
+
+		QueryStep distinctSelectCte = createDistinctSelectCte(preprocessingSelect, alias, selectContext);
+		QueryStep aggregatedCte = createAggregationCte(selectContext, preprocessingSelect, distinctSelectCte, alias);
+
+		ExtractingSqlSelect<Object> finalSelect = preprocessingSelect.qualify(tables.cteName(ConceptCteStep.AGGREGATION_FILTER));
+
+		return ConnectorSqlSelects.builder()
+								  .preprocessingSelect(preprocessingSelect)
+								  .additionalPredecessor(Optional.of(aggregatedCte))
+								  .finalSelect(finalSelect)
+								  .build();
+	}
+
+	private static QueryStep createAggregationCte(
+			SelectContext<Connector, ConnectorSqlTables> selectContext,
+			FieldWrapper<Object> preprocessingSelect,
+			QueryStep distinctSelectCte,
+			String alias
+	) {
+		SqlFunctionProvider functionProvider = selectContext.getFunctionProvider();
+		Field<String> castedColumn = functionProvider.cast(preprocessingSelect.qualify(distinctSelectCte.getCteName()).select(), SQLDataType.VARCHAR);
+		Field<String> aggregatedColumn = functionProvider.stringAggregation(castedColumn, DSL.toChar(ResultSetProcessor.UNIT_SEPARATOR), List.of(castedColumn))
+														 .as(alias);
+
+		SqlIdColumns ids = distinctSelectCte.getQualifiedSelects().getIds();
+
+		Selects selects = Selects.builder()
+								 .ids(ids)
+								 .sqlSelect(new FieldWrapper<>(aggregatedColumn))
+								 .build();
+
+		return QueryStep.builder()
+						.cteName(selectContext.getNameGenerator().cteStepName(DistinctSelectCteStep.STRING_AGG, alias))
+						.selects(selects)
+						.fromTable(QueryStep.toTableLike(distinctSelectCte.getCteName()))
+						.groupBy(ids.toFields())
+						.predecessor(distinctSelectCte)
+						.build();
+	}
+
+	private static QueryStep createDistinctSelectCte(
+			FieldWrapper<Object> preprocessingSelect,
+			String alias,
+			SelectContext<Connector, ConnectorSqlTables> selectContext
+	) {
+		// values to aggregate must be event-filtered first
+		String eventFilterTable = selectContext.getTables().cteName(ConceptCteStep.EVENT_FILTER);
+		ExtractingSqlSelect<Object> qualified = preprocessingSelect.qualify(eventFilterTable);
+		SqlIdColumns ids = selectContext.getIds().qualify(eventFilterTable);
+
+		Selects selects = Selects.builder()
+								 .ids(ids)
+								 .sqlSelect(qualified)
+								 .build();
+
+		return QueryStep.builder()
+						.cteName(selectContext.getNameGenerator().cteStepName(DistinctSelectCteStep.DISTINCT_SELECT, alias))
+						.selectDistinct(true)
+						.selects(selects)
+						.fromTable(QueryStep.toTableLike(eventFilterTable))
+						.build();
+	}
+}
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java
index 62b430d9ed..415087b74b 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java
@@ -6,8 +6,8 @@
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
+import java.util.function.Function;
 import java.util.function.Predicate;
 
 import lombok.RequiredArgsConstructor;
@@ -67,12 +67,47 @@ public List<List<Integer>> getDateRangeList(ResultSet resultSet, int columnIndex
 
 	@Override
 	public List<String> getStringList(ResultSet resultSet, int columnIndex) throws SQLException {
+		return fromString(resultSet, columnIndex, (string) -> string);
+	}
+
+	@Override
+	public List<Boolean> getBooleanList(ResultSet resultSet, int columnIndex) throws SQLException {
+		return fromString(resultSet, columnIndex, Boolean::valueOf);
+	}
+
+	@Override
+	public List<Integer> getIntegerList(ResultSet resultSet, int columnIndex) throws SQLException {
+		return fromString(resultSet, columnIndex, Integer::valueOf);
+	}
+
+	@Override
+	public List<Double> getDoubleList(ResultSet resultSet, int columnIndex) throws SQLException {
+		return fromString(resultSet, columnIndex, Double::valueOf);
+	}
+
+	@Override
+	public List<BigDecimal> getMoneyList(ResultSet resultSet, int columnIndex) throws SQLException {
+		return fromString(
+				resultSet,
+				columnIndex,
+				(string) -> BigDecimal.valueOf(Double.parseDouble(string))
+									  .setScale(2, RoundingMode.HALF_EVEN)
+		);
+	}
+
+	@Override
+	public List<Number> getDateList(ResultSet resultSet, int columnIndex) throws SQLException {
+		return fromString(resultSet, columnIndex, (string) -> Date.valueOf(string).toLocalDate().toEpochDay());
+	}
+
+	private <T> List<T> fromString(ResultSet resultSet, int columnIndex, Function<String, T> parseFunction) throws SQLException {
 		String arrayExpression = resultSet.getString(columnIndex);
 		if (arrayExpression == null) {
-			return Collections.emptyList();
+			return null;
 		}
 		return Arrays.stream(arrayExpression.split(String.valueOf(ResultSetProcessor.UNIT_SEPARATOR)))
 					 .filter(Predicate.not(String::isBlank))
+					 .map(parseFunction)
 					 .toList();
 	}
 
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessor.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessor.java
index 9f63f58c36..6581ec8b21 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessor.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessor.java
@@ -26,4 +26,14 @@ public interface ResultSetProcessor {
 	List<List<Integer>> getDateRangeList(ResultSet resultSet, int columnIndex) throws SQLException;
 
 	List<String> getStringList(ResultSet resultSet, int columnIndex) throws SQLException;
+
+	List<Boolean> getBooleanList(ResultSet resultSet, int columnIndex) throws SQLException;
+
+	List<Integer> getIntegerList(ResultSet resultSet, int columnIndex) throws SQLException;
+
+	List<Double> getDoubleList(ResultSet resultSet, int columnIndex) throws SQLException;
+
+	List<BigDecimal> getMoneyList(ResultSet resultSet, int columnIndex) throws SQLException;
+
+	List<Number> getDateList(ResultSet resultSet, int columnIndex) throws SQLException;
 }
diff --git a/backend/src/test/resources/tests/sql/combined/combined.json b/backend/src/test/resources/tests/sql/combined/combined.json
index e7a04869d9..119b3c3f58 100644
--- a/backend/src/test/resources/tests/sql/combined/combined.json
+++ b/backend/src/test/resources/tests/sql/combined/combined.json
@@ -20,7 +20,8 @@
             "concept.connector.event-date",
             "concept.connector.event_duration_sum",
             "concept.connector.first_value",
-            "concept.connector.sum_distinct"
+            "concept.connector.sum_distinct",
+            "concept.connector.distinct_select"
           ],
           "filters": [
             {
@@ -101,6 +102,11 @@
               "column": "table.value",
               "type": "FIRST"
             },
+            {
+              "label": "distinct_select",
+              "column": "table.value",
+              "type": "DISTINCT"
+            },
             {
               "name": "sum_distinct",
               "type": "SUM",
diff --git a/backend/src/test/resources/tests/sql/combined/expected.csv b/backend/src/test/resources/tests/sql/combined/expected.csv
index 4902ce95be..f1c7b15856 100644
--- a/backend/src/test/resources/tests/sql/combined/expected.csv
+++ b/backend/src/test/resources/tests/sql/combined/expected.csv
@@ -1,6 +1,6 @@
-result,dates,concept exists,concept event-date,concept event_duration_sum,concept connector exists,concept connector event-date,concept connector event_duration_sum,concept connector first_value,concept connector sum_distinct,concept tree_label first_test_column
-1,"{2012-01-01/2012-01-01,2013-01-01/2013-12-31,2015-01-01/2015-01-01}",1,"{2012-01-01/2012-01-01,2013-01-01/2013-12-31,2015-01-01/2015-01-01}",367,1,"{2012-01-01/2012-01-01,2013-01-01/2013-12-31}",366,1.0,2.0,A1
-2,"{2010-07-15/2010-07-15,2012-01-01/2012-01-02,2012-01-05/2012-01-10}",1,"{2010-07-15/2010-07-15,2012-01-01/2012-01-02,2012-01-05/2012-01-10}",9,1,"{2012-01-01/2012-01-02,2012-01-05/2012-01-10}",8,1.01,2.01,B2
-3,"{2012-01-01/2012-01-03,2013-11-10/2013-11-10}",1,"{2012-01-01/2012-01-03,2013-11-10/2013-11-10}",4,1,{2012-01-01/2012-01-03},3,0.5,0.5,A1
-4,"{2012-01-01/2012-01-04,2012-11-11/2012-11-11}",1,"{2012-01-01/2012-01-04,2012-11-11/2012-11-11}",5,1,{2012-01-01/2012-01-04},4,0.5,0.5,B2
-5,{2012-01-01/2012-01-05},1,{2012-01-01/2012-01-05},5,1,{2012-01-01/2012-01-05},5,1.0,1.5,
+result,dates,concept exists,concept event-date,concept event_duration_sum,concept connector exists,concept connector event-date,concept connector event_duration_sum,concept connector first_value,concept connector sum_distinct,concept connector distinct_select,concept tree_label first_test_column
+1,"{2012-01-01/2012-01-01,2013-01-01/2013-12-31,2015-01-01/2015-01-01}",1,"{2012-01-01/2012-01-01,2013-01-01/2013-12-31,2015-01-01/2015-01-01}",367,1,"{2012-01-01/2012-01-01,2013-01-01/2013-12-31}",366,1.0,2.0,{1.0},A1
+2,"{2010-07-15/2010-07-15,2012-01-01/2012-01-02,2012-01-05/2012-01-10}",1,"{2010-07-15/2010-07-15,2012-01-01/2012-01-02,2012-01-05/2012-01-10}",9,1,"{2012-01-01/2012-01-02,2012-01-05/2012-01-10}",8,1.01,2.01,"{1.0,1.01}",B2
+3,"{2012-01-01/2012-01-03,2013-11-10/2013-11-10}",1,"{2012-01-01/2012-01-03,2013-11-10/2013-11-10}",4,1,{2012-01-01/2012-01-03},3,0.5,0.5,{0.5},A1
+4,"{2012-01-01/2012-01-04,2012-11-11/2012-11-11}",1,"{2012-01-01/2012-01-04,2012-11-11/2012-11-11}",5,1,{2012-01-01/2012-01-04},4,0.5,0.5,{0.5},B2
+5,{2012-01-01/2012-01-05},1,{2012-01-01/2012-01-05},5,1,{2012-01-01/2012-01-05},5,1.0,1.5,"{0.5,1.0}",
diff --git a/backend/src/test/resources/tests/sql/selects/distinct/content.csv b/backend/src/test/resources/tests/sql/selects/distinct/content.csv
new file mode 100644
index 0000000000..03265056db
--- /dev/null
+++ b/backend/src/test/resources/tests/sql/selects/distinct/content.csv
@@ -0,0 +1,14 @@
+pid,datum,value
+1,2012-01-01,"f"
+1,2012-01-02,"f"
+2,2010-07-15,
+3,2012-01-01,"f"
+3,2012-01-02,"m"
+4,2012-01-01,"f"
+4,2012-01-02,"m"
+4,2012-01-03,"f"
+4,2012-01-04,"m"
+5,2012-01-01,"f "
+5,2012-01-02,"m "
+5,2012-01-03,"f"
+5,2012-01-04,"m"
diff --git a/backend/src/test/resources/tests/sql/selects/distinct/distinct.json b/backend/src/test/resources/tests/sql/selects/distinct/distinct.json
new file mode 100644
index 0000000000..4ff17cd780
--- /dev/null
+++ b/backend/src/test/resources/tests/sql/selects/distinct/distinct.json
@@ -0,0 +1,65 @@
+{
+  "type": "QUERY_TEST",
+  "label": "DISTINCT select",
+  "expectedCsv": "tests/sql/selects/distinct/expected.csv",
+  "query": {
+    "type": "CONCEPT_QUERY",
+    "root": {
+      "ids": [
+        "concept"
+      ],
+      "type": "CONCEPT",
+      "tables": [
+        {
+          "id": "concept.connector",
+          "selects": [
+            "concept.connector.select"
+          ]
+        }
+      ]
+    }
+  },
+  "concepts": [
+    {
+      "label": "concept",
+      "type": "TREE",
+      "connectors": [
+        {
+          "label": "connector",
+          "table": "table",
+          "validityDates": {
+            "label": "datum",
+            "column": "table.datum"
+          },
+          "selects": {
+            "name": "select",
+            "type": "DISTINCT",
+            "column": "table.value"
+          }
+        }
+      ]
+    }
+  ],
+  "content": {
+    "tables": [
+      {
+        "csv": "tests/sql/selects/distinct/content.csv",
+        "name": "table",
+        "primaryColumn": {
+          "name": "pid",
+          "type": "STRING"
+        },
+        "columns": [
+          {
+            "name": "datum",
+            "type": "DATE"
+          },
+          {
+            "name": "value",
+            "type": "STRING"
+          }
+        ]
+      }
+    ]
+  }
+}
diff --git a/backend/src/test/resources/tests/sql/selects/distinct/expected.csv b/backend/src/test/resources/tests/sql/selects/distinct/expected.csv
new file mode 100644
index 0000000000..6b7033299b
--- /dev/null
+++ b/backend/src/test/resources/tests/sql/selects/distinct/expected.csv
@@ -0,0 +1,6 @@
+result,dates,concept select
+1,{2012-01-01/2012-01-02},{f}
+2,{2010-07-15/2010-07-15},
+3,{2012-01-01/2012-01-02},"{f,m}"
+4,{2012-01-01/2012-01-04},"{f,m}"
+5,{2012-01-01/2012-01-04},"{f,f ,m,m }"

From 6797c0b22c47a8af21d238c1091f8a3887cbe3db Mon Sep 17 00:00:00 2001
From: Jonas Arnhold <jonas.arnhold@bakdata.com>
Date: Thu, 18 Jul 2024 10:36:44 +0200
Subject: [PATCH 12/41] Implement arbitrary date select conversion (#3375)

---
 .../connector/specific/DateUnionSelect.java   |   7 +
 .../connector/specific/DurationSumSelect.java |   8 +-
 .../concept/IntervalPackingSelectsCte.java    |  61 +++++++--
 .../AnsiSqlIntervalPacker.java                |   2 +-
 .../IntervalPackingCteStep.java               |  13 ++
 .../dialect/HanaSqlFunctionProvider.java      |  17 ++-
 .../dialect/PostgreSqlFunctionProvider.java   |  73 +++++-----
 .../dialect/SqlFunctionProvider.java          |   9 +-
 .../forms/RelativeStratification.java         |   4 +-
 .../sql/conversion/model/NameGenerator.java   |   4 +-
 .../select/DateUnionSelectConverter.java      |  18 +++
 .../model/select/DaterangeSelectUtil.java     | 127 ++++++++++++++++++
 .../select/DurationSumSelectConverter.java    |  22 +++
 .../query/EntityDateQueryConverter.java       |   3 +-
 .../tests/sql/selects/date_union/content.csv  |  11 ++
 .../sql/selects/date_union/date_union.json    |  67 +++++++++
 .../tests/sql/selects/date_union/expected.csv |  11 ++
 .../sum/duration_sum/duration_sum.json        |   9 +-
 .../sql/selects/sum/duration_sum/expected.csv |  10 +-
 .../sum/event_duration_sum/content.csv        |  25 ++++
 .../sum/event_duration_sum/duration_sum.json  |  99 ++++++++++++++
 .../sum/event_duration_sum/expected.csv       |   6 +
 22 files changed, 540 insertions(+), 66 deletions(-)
 create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java
 create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java
 create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DurationSumSelectConverter.java
 create mode 100644 backend/src/test/resources/tests/sql/selects/date_union/content.csv
 create mode 100644 backend/src/test/resources/tests/sql/selects/date_union/date_union.json
 create mode 100644 backend/src/test/resources/tests/sql/selects/date_union/expected.csv
 create mode 100644 backend/src/test/resources/tests/sql/selects/sum/event_duration_sum/content.csv
 create mode 100644 backend/src/test/resources/tests/sql/selects/sum/event_duration_sum/duration_sum.json
 create mode 100644 backend/src/test/resources/tests/sql/selects/sum/event_duration_sum/expected.csv

diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java
index 2d1ff6246c..05b694ebb9 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java
@@ -12,6 +12,8 @@
 import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator;
 import com.bakdata.conquery.models.query.queryplan.aggregators.specific.DateUnionAggregator;
 import com.bakdata.conquery.models.types.ResultType;
+import com.bakdata.conquery.sql.conversion.model.select.DateUnionSelectConverter;
+import com.bakdata.conquery.sql.conversion.model.select.SelectConverter;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import lombok.Getter;
@@ -53,4 +55,9 @@ public Aggregator<?> createAggregator() {
 	public ResultType<?> getResultType() {
 		return new ResultType.ListT<>(ResultType.DateRangeT.INSTANCE);
 	}
+
+	@Override
+	public SelectConverter<DateUnionSelect> createConverter() {
+		return new DateUnionSelectConverter();
+	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java
index 8e32141534..e82f3311c3 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java
@@ -12,6 +12,8 @@
 import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator;
 import com.bakdata.conquery.models.query.queryplan.aggregators.specific.DurationSumAggregator;
 import com.bakdata.conquery.models.types.ResultType;
+import com.bakdata.conquery.sql.conversion.model.select.DurationSumSelectConverter;
+import com.bakdata.conquery.sql.conversion.model.select.SelectConverter;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import lombok.Getter;
@@ -45,7 +47,6 @@ public List<Column> getRequiredColumns() {
 
 	@Override
 	public Aggregator<?> createAggregator() {
-		// TODO fix this for 2 columns
 		return new DurationSumAggregator(getColumn());
 	}
 
@@ -53,4 +54,9 @@ public Aggregator<?> createAggregator() {
 	public ResultType<?> getResultType() {
 		return ResultType.IntegerT.INSTANCE;
 	}
+
+	@Override
+	public SelectConverter<DurationSumSelect> createConverter() {
+		return new DurationSumSelectConverter();
+	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java
index 973381cae3..71339400cd 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java
@@ -1,12 +1,14 @@
 package com.bakdata.conquery.sql.conversion.cqelement.concept;
 
 import java.util.List;
+import java.util.Optional;
 
-import com.bakdata.conquery.models.datasets.concepts.select.Select;
 import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDateUnionSelect;
 import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDurationSumSelect;
 import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext;
+import com.bakdata.conquery.sql.conversion.dialect.SqlDialect;
 import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider;
+import com.bakdata.conquery.sql.conversion.model.ColumnDateRange;
 import com.bakdata.conquery.sql.conversion.model.QueryStep;
 import com.bakdata.conquery.sql.conversion.model.Selects;
 import com.bakdata.conquery.sql.conversion.model.SqlTables;
@@ -14,7 +16,39 @@
 import com.bakdata.conquery.sql.conversion.model.select.SqlSelect;
 import com.google.common.base.Preconditions;
 
-class IntervalPackingSelectsCte {
+public class IntervalPackingSelectsCte {
+
+	public static QueryStep forSelect(
+			QueryStep withAggregatedDaterange,
+			ColumnDateRange daterange,
+			SqlSelect select,
+			SqlTables tables,
+			SqlDialect sqlDialect
+	) {
+		List<QueryStep> predecessors = List.of(withAggregatedDaterange);
+		QueryStep directPredecessor = withAggregatedDaterange;
+
+		// we need an additional predecessor to unnest the validity date if it is a single column range
+		if (sqlDialect.supportsSingleColumnRanges()) {
+			String unnestCteName = tables.cteName(ConceptCteStep.UNNEST_DATE);
+			directPredecessor = sqlDialect.getFunctionProvider().unnestDaterange(daterange, withAggregatedDaterange, unnestCteName);
+			predecessors = List.of(withAggregatedDaterange, directPredecessor);
+		}
+
+		Selects predecessorSelects = directPredecessor.getQualifiedSelects();
+		Selects selects = Selects.builder()
+								 .ids(predecessorSelects.getIds())
+								 .sqlSelect(select)
+								 .build();
+
+		return QueryStep.builder()
+						.cteName(tables.cteName(ConceptCteStep.INTERVAL_PACKING_SELECTS))
+						.selects(selects)
+						.fromTable(QueryStep.toTableLike(directPredecessor.getCteName()))
+						.groupBy(predecessorSelects.getIds().toFields())
+						.predecessors(predecessors)
+						.build();
+	}
 
 	public static QueryStep forConnector(QueryStep predecessor, CQTableContext cqTableContext) {
 		return create(
@@ -40,8 +74,8 @@ public static QueryStep forConcept(
 	}
 
 	/**
-	 * @param predecessor The preceding query step which must contain an aggregated validity date.
-	 * @param intervalPackingSelects {@link Select}s that require an interval-packed date.
+	 * @param predecessor            The preceding query step which must contain an aggregated validity date.
+	 * @param intervalPackingSelects {@link SqlSelect}s which will be part of the returned {@link QueryStep}.
 	 * @return A {@link QueryStep} containing converted interval packing selects, like {@link EventDurationSumSelect}, {@link EventDateUnionSelect}, etc.
 	 * Returns the given predecessor as is if the given list of interval packing selects is empty.
 	 */
@@ -55,20 +89,19 @@ private static QueryStep create(
 			return predecessor;
 		}
 
-		Preconditions.checkArgument(
-				predecessor.getQualifiedSelects().getValidityDate().isPresent(),
-				"Can't create a IntervalPackingSelectsCte without a validity date present."
-		);
+		Optional<ColumnDateRange> validityDate = predecessor.getQualifiedSelects().getValidityDate();
+		Preconditions.checkArgument(validityDate.isPresent(), "Can't create a IntervalPackingSelectsCte without a validity date present.");
 
 		// we need an additional predecessor to unnest the validity date if it is a single column range
 		List<QueryStep> predecessors = List.of();
-		QueryStep actualPredecessor = predecessor;
-		if (predecessor.getQualifiedSelects().getValidityDate().get().isSingleColumnRange()) {
-			actualPredecessor = functionProvider.unnestValidityDate(predecessor, tables.cteName(ConceptCteStep.UNNEST_DATE));
-			predecessors = List.of(actualPredecessor);
+		QueryStep directPredecessor = predecessor;
+		if (validityDate.get().isSingleColumnRange()) {
+			String unnestCteName = tables.cteName(ConceptCteStep.UNNEST_DATE);
+			directPredecessor = functionProvider.unnestDaterange(validityDate.get(), predecessor, unnestCteName);
+			predecessors = List.of(directPredecessor);
 		}
 
-		Selects predecessorSelects = actualPredecessor.getQualifiedSelects();
+		Selects predecessorSelects = directPredecessor.getQualifiedSelects();
 		Selects selects = Selects.builder()
 								 .ids(predecessorSelects.getIds())
 								 .sqlSelects(intervalPackingSelects)
@@ -77,7 +110,7 @@ private static QueryStep create(
 		return QueryStep.builder()
 						.cteName(tables.cteName(ConceptCteStep.INTERVAL_PACKING_SELECTS))
 						.selects(selects)
-						.fromTable(QueryStep.toTableLike(actualPredecessor.getCteName()))
+						.fromTable(QueryStep.toTableLike(directPredecessor.getCteName()))
 						.groupBy(predecessorSelects.getIds().toFields())
 						.predecessors(predecessors)
 						.build();
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java
index bfe46e4623..aa18433443 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java
@@ -52,7 +52,7 @@ private QueryStep createPreviousEndStep(IntervalPackingContext context, Aggregat
 									 .as(IntervalPacker.PREVIOUS_END_FIELD_NAME);
 
 		List<SqlSelect> qualifiedSelects = new ArrayList<>(QualifyingUtil.qualify(context.getCarryThroughSelects(), sourceTableName));
-		qualifiedSelects.add(new FieldWrapper<>(previousEnd));
+		qualifiedSelects.add(new FieldWrapper<>(previousEnd, daterange.getStart().getName(), daterange.getEnd().getName()));
 
 		Selects previousEndSelects = buildSelects(ids, daterange, qualifiedSelects, aggregationMode);
 
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java
index 84f7f150f7..ef07bd02b1 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java
@@ -1,5 +1,6 @@
 package com.bakdata.conquery.sql.conversion.cqelement.intervalpacking;
 
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
 
@@ -53,4 +54,16 @@ public static Map<CteStep, CteStep> getMappings(CteStep root, SqlDialect dialect
 		return mappings;
 	}
 
+	/**
+	 * Create predecessor mappings for these interval packing {@link CteStep}s based on the default mapping.
+	 */
+	public static Map<CteStep, CteStep> getMappings(SqlDialect dialect) {
+		if (dialect.supportsSingleColumnRanges()) {
+			Map<CteStep, CteStep> mappings = new HashMap<>();
+			mappings.put(INTERVAL_COMPLETE, null); // final step directly mapped onto root table
+			return mappings;
+		}
+		return CteStep.getDefaultPredecessorMap(Set.of(values()));
+	}
+
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java
index a4c40c0d86..f765f7cd84 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java
@@ -9,6 +9,7 @@
 import com.bakdata.conquery.models.common.CDateSet;
 import com.bakdata.conquery.models.common.daterange.CDateRange;
 import com.bakdata.conquery.models.datasets.Column;
+import com.bakdata.conquery.models.datasets.concepts.DaterangeSelectOrFilter;
 import com.bakdata.conquery.models.datasets.concepts.ValidityDate;
 import com.bakdata.conquery.sql.conversion.SharedAliases;
 import com.bakdata.conquery.sql.conversion.model.ColumnDateRange;
@@ -127,6 +128,15 @@ public ColumnDateRange forValidityDate(ValidityDate validityDate, CDateRange dat
 		return ColumnDateRange.of(lowerBound, upperBound);
 	}
 
+	@Override
+	public ColumnDateRange forArbitraryDateRange(DaterangeSelectOrFilter daterangeSelectOrFilter) {
+		String tableName = daterangeSelectOrFilter.getTable().getName();
+		if (daterangeSelectOrFilter.getEndColumn() != null) {
+			return ofStartAndEnd(tableName, daterangeSelectOrFilter.getStartColumn(), daterangeSelectOrFilter.getEndColumn());
+		}
+		return ofStartAndEnd(tableName, daterangeSelectOrFilter.getColumn(), daterangeSelectOrFilter.getColumn());
+	}
+
 	@Override
 	public ColumnDateRange aggregated(ColumnDateRange columnDateRange) {
 		return ColumnDateRange.of(
@@ -150,7 +160,7 @@ public ColumnDateRange intersection(ColumnDateRange left, ColumnDateRange right)
 	}
 
 	@Override
-	public QueryStep unnestValidityDate(QueryStep predecessor, String cteName) {
+	public QueryStep unnestDaterange(ColumnDateRange nested, QueryStep predecessor, String cteName) {
 		// HANA does not support single column datemultiranges
 		return predecessor;
 	}
@@ -318,6 +328,11 @@ private ColumnDateRange toColumnDateRange(ValidityDate validityDate) {
 			endColumn = validityDate.getEndColumn();
 		}
 
+		return ofStartAndEnd(tableName, startColumn, endColumn);
+	}
+
+	private ColumnDateRange ofStartAndEnd(String tableName, Column startColumn, Column endColumn) {
+
 		Field<Date> rangeStart = DSL.coalesce(
 				DSL.field(DSL.name(tableName, startColumn.getName()), Date.class),
 				toDateField(MIN_DATE_VALUE)
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java
index 5d23751f56..96ed3e3f7e 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java
@@ -9,12 +9,12 @@
 import com.bakdata.conquery.models.common.CDateSet;
 import com.bakdata.conquery.models.common.daterange.CDateRange;
 import com.bakdata.conquery.models.datasets.Column;
+import com.bakdata.conquery.models.datasets.concepts.DaterangeSelectOrFilter;
 import com.bakdata.conquery.models.datasets.concepts.ValidityDate;
 import com.bakdata.conquery.sql.conversion.SharedAliases;
 import com.bakdata.conquery.sql.conversion.model.ColumnDateRange;
 import com.bakdata.conquery.sql.conversion.model.QueryStep;
 import com.bakdata.conquery.sql.conversion.model.Selects;
-import com.google.common.base.Preconditions;
 import org.jooq.ArrayAggOrderByStep;
 import org.jooq.Condition;
 import org.jooq.DataType;
@@ -30,7 +30,8 @@
  *
  * @see <a href="https://www.postgresql.org/docs/15/functions.html">PostgreSQL Documentation</a>
  */
-public class PostgreSqlFunctionProvider implements SqlFunctionProvider {
+public class
+PostgreSqlFunctionProvider implements SqlFunctionProvider {
 
 	private static final String INFINITY_DATE_VALUE = "infinity";
 	private static final String MINUS_INFINITY_DATE_VALUE = "-infinity";
@@ -118,6 +119,15 @@ public ColumnDateRange forValidityDate(ValidityDate validityDate, CDateRange dat
 		return intersection(validityDateRange, restriction);
 	}
 
+	@Override
+	public ColumnDateRange forArbitraryDateRange(DaterangeSelectOrFilter daterangeSelectOrFilter) {
+		String tableName = daterangeSelectOrFilter.getTable().getName();
+		if (daterangeSelectOrFilter.getEndColumn() != null) {
+			return ofStartAndEnd(tableName, daterangeSelectOrFilter.getStartColumn(), daterangeSelectOrFilter.getEndColumn());
+		}
+		return ofSingleColumn(tableName, daterangeSelectOrFilter.getColumn());
+	}
+
 	@Override
 	public ColumnDateRange aggregated(ColumnDateRange columnDateRange) {
 		return ColumnDateRange.of(rangeAgg(columnDateRange)).as(columnDateRange.getAlias());
@@ -141,16 +151,10 @@ public ColumnDateRange intersection(ColumnDateRange left, ColumnDateRange right)
 	}
 
 	@Override
-	public QueryStep unnestValidityDate(QueryStep predecessor, String cteName) {
-
-		Preconditions.checkArgument(
-				predecessor.getSelects().getValidityDate().isPresent(),
-				"Can't create a unnest-CTE without a validity date present."
-		);
+	public QueryStep unnestDaterange(ColumnDateRange nested, QueryStep predecessor, String cteName) {
 
-		Selects predecessorSelects = predecessor.getQualifiedSelects();
-		ColumnDateRange validityDate = predecessorSelects.getValidityDate().get();
-		ColumnDateRange unnested = ColumnDateRange.of(unnest(validityDate.getRange()).as(validityDate.getAlias()));
+		ColumnDateRange qualifiedRange = nested.qualify(predecessor.getCteName());
+		ColumnDateRange unnested = ColumnDateRange.of(unnest(qualifiedRange.getRange()).as(qualifiedRange.getAlias()));
 
 		Selects selects = Selects.builder()
 								 .ids(predecessor.getQualifiedSelects().getIds())
@@ -306,30 +310,21 @@ private ColumnDateRange toColumnDateRange(CDateRange dateRestriction) {
 	}
 
 	private ColumnDateRange toColumnDateRange(ValidityDate validityDate) {
-
 		String tableName = validityDate.getConnector().getTable().getName();
-
-		Field<?> dateRange;
-
 		if (validityDate.getEndColumn() != null) {
+			return ofStartAndEnd(tableName, validityDate.getStartColumn(), validityDate.getEndColumn());
+		}
+		return ofSingleColumn(tableName, validityDate.getColumn());
+	}
 
-			Field<?> startColumn = DSL.coalesce(
-					DSL.field(DSL.name(tableName, validityDate.getStartColumn().getName())),
-					toDateField(MINUS_INFINITY_DATE_VALUE)
-			);
-			Field<?> endColumn = DSL.coalesce(
-					DSL.field(DSL.name(tableName, validityDate.getEndColumn().getName())),
-					toDateField(INFINITY_DATE_VALUE)
-			);
+	private ColumnDateRange ofSingleColumn(String tableName, Column column) {
 
-			return ColumnDateRange.of(daterange(startColumn, endColumn, "[]"));
-		}
+		Field<?> dateRange;
 
-		Column validityDateColumn = validityDate.getColumn();
-		dateRange = switch (validityDateColumn.getType()) {
+		dateRange = switch (column.getType()) {
 			// if validityDateColumn is a DATE_RANGE we can make use of Postgres' integrated daterange type, but the upper bound is exclusive by default
 			case DATE_RANGE -> {
-				Field<Object> daterange = DSL.field(DSL.name(validityDateColumn.getName()));
+				Field<Object> daterange = DSL.field(DSL.name(column.getName()));
 				Field<Date> startColumn = DSL.coalesce(
 						DSL.function("lower", Date.class, daterange),
 						toDateField(MINUS_INFINITY_DATE_VALUE)
@@ -342,19 +337,33 @@ private ColumnDateRange toColumnDateRange(ValidityDate validityDate) {
 			}
 			// if the validity date column is not of daterange type, we construct it manually
 			case DATE -> {
-				Field<Date> column = DSL.field(DSL.name(tableName, validityDate.getColumn().getName()), Date.class);
-				Field<Date> startColumn = DSL.coalesce(column, toDateField(MINUS_INFINITY_DATE_VALUE));
-				Field<Date> endColumn = DSL.coalesce(column, toDateField(INFINITY_DATE_VALUE));
+				Field<Date> singleDate = DSL.field(DSL.name(tableName, column.getName()), Date.class);
+				Field<Date> startColumn = DSL.coalesce(singleDate, toDateField(MINUS_INFINITY_DATE_VALUE));
+				Field<Date> endColumn = DSL.coalesce(singleDate, toDateField(INFINITY_DATE_VALUE));
 				yield daterange(startColumn, endColumn, "[]");
 			}
 			default -> throw new IllegalArgumentException(
-					"Given column type '%s' can't be converted to a proper date restriction.".formatted(validityDateColumn.getType())
+					"Given column type '%s' can't be converted to a proper date restriction.".formatted(column.getType())
 			);
 		};
 
 		return ColumnDateRange.of(dateRange);
 	}
 
+	private ColumnDateRange ofStartAndEnd(String tableName, Column startColumn, Column endColumn) {
+
+		Field<?> start = DSL.coalesce(
+				DSL.field(DSL.name(tableName, startColumn.getName())),
+				toDateField(MINUS_INFINITY_DATE_VALUE)
+		);
+		Field<?> end = DSL.coalesce(
+				DSL.field(DSL.name(tableName, endColumn.getName())),
+				toDateField(INFINITY_DATE_VALUE)
+		);
+
+		return ColumnDateRange.of(daterange(start, end, "[]"));
+	}
+
 	private ColumnDateRange ensureIsSingleColumnRange(ColumnDateRange daterange) {
 		return daterange.isSingleColumnRange()
 			   ? daterange
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java
index 9a0de8834b..68dfc0b099 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java
@@ -8,6 +8,7 @@
 import com.bakdata.conquery.apiv1.query.concept.filter.CQTable;
 import com.bakdata.conquery.models.common.CDateSet;
 import com.bakdata.conquery.models.common.daterange.CDateRange;
+import com.bakdata.conquery.models.datasets.concepts.DaterangeSelectOrFilter;
 import com.bakdata.conquery.models.datasets.concepts.ValidityDate;
 import com.bakdata.conquery.sql.conversion.SharedAliases;
 import com.bakdata.conquery.sql.conversion.model.ColumnDateRange;
@@ -79,6 +80,8 @@ public interface SqlFunctionProvider {
 	 */
 	ColumnDateRange forValidityDate(ValidityDate validityDate, CDateRange dateRestriction);
 
+	ColumnDateRange forArbitraryDateRange(DaterangeSelectOrFilter daterangeSelectOrFilter);
+
 	ColumnDateRange aggregated(ColumnDateRange columnDateRange);
 
 	/**
@@ -92,11 +95,13 @@ public interface SqlFunctionProvider {
 	ColumnDateRange intersection(ColumnDateRange left, ColumnDateRange right);
 
 	/**
-	 * @param predecessor The predeceasing step containing an aggregated validity date.
+	 * @param predecessor The predeceasing step containing the aggregated {@link ColumnDateRange}.
+	 * @param nested      The {@link ColumnDateRange} you want to unnest.
+	 * @param cteName     The CTE name of the returned {@link QueryStep}.
 	 * @return A QueryStep containing an unnested validity date with 1 row per single daterange for each id. For dialects that don't support single column
 	 * multiranges, the given predecessor will be returned as is.
 	 */
-	QueryStep unnestValidityDate(QueryStep predecessor, String cteName);
+	QueryStep unnestDaterange(ColumnDateRange nested, QueryStep predecessor, String cteName);
 
 	/**
 	 * Aggregates the start and end columns of the validity date of entries into one compound string expression.
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java
index 404d5a030e..913f7fbad9 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java
@@ -44,7 +44,9 @@ public QueryStep createRelativeStratificationTable(RelativeFormQuery form) {
 
 		// we want to create the stratification for each distinct validity date range of an entity,
 		// so we first need to unnest the validity date in case it is a multirange
-		QueryStep withUnnestedValidityDate = functionProvider.unnestValidityDate(baseStep, FormCteStep.UNNEST_DATES.getSuffix());
+		Preconditions.checkArgument(baseStep.getSelects().getValidityDate().isPresent(), "Base step must contain a validity date");
+		String unnestCteName = FormCteStep.UNNEST_DATES.getSuffix();
+		QueryStep withUnnestedValidityDate = functionProvider.unnestDaterange(baseStep.getSelects().getValidityDate().get(), baseStep, unnestCteName);
 
 		QueryStep indexSelectorStep = createIndexSelectorStep(form, withUnnestedValidityDate);
 		QueryStep indexStartStep = createIndexStartStep(form, indexSelectorStep);
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java
index af6923cbcf..1bb312e979 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java
@@ -7,7 +7,7 @@
 
 import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept;
 import com.bakdata.conquery.models.datasets.concepts.Connector;
-import com.bakdata.conquery.models.identifiable.Labeled;
+import com.bakdata.conquery.models.identifiable.Named;
 import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
@@ -47,7 +47,7 @@ public String cteStepName(CteStep cteStep, String nodeLabel) {
 		return ensureValidLength(cteStep.cteName(nodeLabel));
 	}
 
-	public String selectName(Labeled<?> selectOrFilter) {
+	public String selectName(Named<?> selectOrFilter) {
 		int selectCount = this.selectCountMap.merge(selectOrFilter.getName(), 1, Integer::sum);
 		String name = lowerAndReplaceWhitespace(selectOrFilter.getName());
 		return ensureValidLength("%s-%d".formatted(name, selectCount));
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java
new file mode 100644
index 0000000000..d2c04e9c34
--- /dev/null
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java
@@ -0,0 +1,18 @@
+package com.bakdata.conquery.sql.conversion.model.select;
+
+import com.bakdata.conquery.models.datasets.concepts.Connector;
+import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.DateUnionSelect;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables;
+
+public class DateUnionSelectConverter implements SelectConverter<DateUnionSelect> {
+
+	@Override
+	public ConnectorSqlSelects connectorSelect(DateUnionSelect select, SelectContext<Connector, ConnectorSqlTables> selectContext) {
+		return DaterangeSelectUtil.createConnectorSqlSelects(
+				select,
+				(daterange, alias, functionProvider) -> new FieldWrapper<>(functionProvider.daterangeStringAggregation(daterange).as(alias)),
+				selectContext
+		);
+	}
+
+}
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java
new file mode 100644
index 0000000000..e44f03b12b
--- /dev/null
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java
@@ -0,0 +1,127 @@
+package com.bakdata.conquery.sql.conversion.model.select;
+
+import static com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep.EVENT_FILTER;
+import static com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep.INTERVAL_PACKING_SELECTS;
+import static com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep.UNNEST_DATE;
+import static com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep.INTERVAL_COMPLETE;
+
+import java.math.BigDecimal;
+import java.sql.Date;
+import java.time.temporal.ChronoUnit;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import com.bakdata.conquery.models.datasets.concepts.Connector;
+import com.bakdata.conquery.models.datasets.concepts.DaterangeSelectOrFilter;
+import com.bakdata.conquery.models.identifiable.Named;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.IntervalPackingSelectsCte;
+import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingContext;
+import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep;
+import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider;
+import com.bakdata.conquery.sql.conversion.model.ColumnDateRange;
+import com.bakdata.conquery.sql.conversion.model.CteStep;
+import com.bakdata.conquery.sql.conversion.model.QueryStep;
+import com.bakdata.conquery.sql.conversion.model.SqlTables;
+import org.jooq.Condition;
+import org.jooq.Field;
+import org.jooq.impl.DSL;
+
+class DaterangeSelectUtil {
+
+	@FunctionalInterface
+	public interface AggregationFunction {
+		FieldWrapper<?> apply(ColumnDateRange daterange, String alias, SqlFunctionProvider functionProvider);
+	}
+
+	/**
+	 * Aggregates the daterange of a corresponding {@link DaterangeSelectOrFilter} and applies the respective converted aggregation via
+	 * {@link IntervalPackingSelectsCte}s using additional predecessor tables.
+	 */
+	public static <S extends DaterangeSelectOrFilter & Named<?>> ConnectorSqlSelects createConnectorSqlSelects(
+			S select,
+			AggregationFunction aggregationFunction,
+			SelectContext<Connector, ConnectorSqlTables> context
+	) {
+		String alias = context.getNameGenerator().selectName(select);
+		ConnectorSqlTables tables = context.getTables();
+		SqlFunctionProvider functionProvider = context.getSqlDialect().getFunctionProvider();
+
+		ColumnDateRange daterange = functionProvider.forArbitraryDateRange(select).as(alias);
+		List<SqlSelect> rootSelects = daterange.toFields().stream()
+											   .map(FieldWrapper::new)
+											   .collect(Collectors.toList());
+
+		SqlTables daterangeSelectTables = createTables(alias, context);
+		QueryStep lastIntervalPackingStep = applyIntervalPacking(daterange, daterangeSelectTables, context);
+
+		ColumnDateRange qualified = daterange.qualify(daterangeSelectTables.getPredecessor(INTERVAL_PACKING_SELECTS));
+		FieldWrapper<?> aggregationField = aggregationFunction.apply(qualified, alias, functionProvider);
+
+		QueryStep intervalPackingSelectsStep = IntervalPackingSelectsCte.forSelect(
+				lastIntervalPackingStep,
+				qualified,
+				aggregationField,
+				daterangeSelectTables,
+				context.getSqlDialect()
+		);
+
+		ExtractingSqlSelect<?> finalSelect = aggregationField.qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER));
+
+		return ConnectorSqlSelects.builder()
+								  .preprocessingSelects(rootSelects)
+								  .additionalPredecessor(Optional.of(intervalPackingSelectsStep))
+								  .finalSelect(finalSelect)
+								  .build();
+	}
+
+	public static FieldWrapper<BigDecimal> createDurationSumSqlSelect(String alias, ColumnDateRange validityDate, SqlFunctionProvider functionProvider) {
+		Field<Integer> dateDistanceInDays = functionProvider.dateDistance(ChronoUnit.DAYS, validityDate.getStart(), validityDate.getEnd());
+		Field<BigDecimal> durationSum = DSL.sum(
+												   DSL.when(containsInfinityDate(validityDate, functionProvider), DSL.val(null, Integer.class))
+													  .otherwise(dateDistanceInDays)
+										   )
+										   .as(alias);
+		return new FieldWrapper<>(durationSum);
+	}
+
+	private static Condition containsInfinityDate(ColumnDateRange validityDate, SqlFunctionProvider functionProvider) {
+		Field<Date> negativeInfinity = functionProvider.toDateField(functionProvider.getMinDateExpression());
+		Field<Date> positiveInfinity = functionProvider.toDateField(functionProvider.getMaxDateExpression());
+		return validityDate.getStart().eq(negativeInfinity).or(validityDate.getEnd().eq(positiveInfinity));
+	}
+
+	private static SqlTables createTables(String alias, SelectContext<Connector, ConnectorSqlTables> context) {
+		Map<CteStep, CteStep> predecessorMapping = new HashMap<>();
+		String eventFilterCteName = context.getTables().cteName(EVENT_FILTER);
+		predecessorMapping.putAll(IntervalPackingCteStep.getMappings(context.getSqlDialect()));
+		if (context.getSqlDialect().supportsSingleColumnRanges()) {
+			predecessorMapping.put(UNNEST_DATE, INTERVAL_COMPLETE);
+			predecessorMapping.put(INTERVAL_PACKING_SELECTS, UNNEST_DATE);
+		}
+		else {
+			predecessorMapping.put(INTERVAL_PACKING_SELECTS, INTERVAL_COMPLETE);
+		}
+		Map<CteStep, String> cteNameMap = CteStep.createCteNameMap(predecessorMapping.keySet(), alias, context.getNameGenerator());
+		return new SqlTables(eventFilterCteName, cteNameMap, predecessorMapping);
+	}
+
+	private static QueryStep applyIntervalPacking(ColumnDateRange daterange, SqlTables dateUnionTables, SelectContext<?, ?> context) {
+
+		String eventFilterCteName = context.getTables().cteName(EVENT_FILTER);
+		IntervalPackingContext intervalPackingContext = IntervalPackingContext.builder()
+																			  .ids(context.getIds().qualify(eventFilterCteName))
+																			  .daterange(daterange.qualify(eventFilterCteName))
+																			  .tables(dateUnionTables)
+																			  .build();
+
+		return context.getSqlDialect()
+					  .getIntervalPacker()
+					  .aggregateAsArbitrarySelect(intervalPackingContext);
+	}
+
+}
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DurationSumSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DurationSumSelectConverter.java
new file mode 100644
index 0000000000..01800ea36a
--- /dev/null
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DurationSumSelectConverter.java
@@ -0,0 +1,22 @@
+package com.bakdata.conquery.sql.conversion.model.select;
+
+import com.bakdata.conquery.models.datasets.concepts.Connector;
+import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.DurationSumSelect;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables;
+import com.bakdata.conquery.sql.conversion.model.ColumnDateRange;
+
+public class DurationSumSelectConverter implements SelectConverter<DurationSumSelect> {
+
+	@Override
+	public ConnectorSqlSelects connectorSelect(DurationSumSelect select, SelectContext<Connector, ConnectorSqlTables> selectContext) {
+		return DaterangeSelectUtil.createConnectorSqlSelects(
+				select,
+				(daterange, alias, functionProvider) -> {
+					ColumnDateRange asDualColumn = functionProvider.toDualColumn(daterange);
+					return DaterangeSelectUtil.createDurationSumSqlSelect(alias, asDualColumn, functionProvider);
+				},
+				selectContext
+		);
+	}
+
+}
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java
index 808c672a63..457eebe740 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java
@@ -57,7 +57,8 @@ private static QueryStep overwriteBounds(QueryStep prerequisite, EntityDateQuery
 		SqlFunctionProvider functionProvider = context.getSqlDialect().getFunctionProvider();
 		// we want to create a stratification for each distinct validity date range of an entity,
 		// so we first need to unnest the validity date in case it is a multirange
-		QueryStep unnestedEntityDate = functionProvider.unnestValidityDate(prerequisite, FormCteStep.UNNEST_ENTITY_DATE_CTE.getSuffix());
+		String unnestCteName = FormCteStep.UNNEST_ENTITY_DATE_CTE.getSuffix();
+		QueryStep unnestedEntityDate = functionProvider.unnestDaterange(prerequisite.getSelects().getValidityDate().get(), prerequisite, unnestCteName);
 		Selects unnestedSelects = unnestedEntityDate.getQualifiedSelects();
 
 		ColumnDateRange withOverwrittenBounds;
diff --git a/backend/src/test/resources/tests/sql/selects/date_union/content.csv b/backend/src/test/resources/tests/sql/selects/date_union/content.csv
new file mode 100644
index 0000000000..edb160e384
--- /dev/null
+++ b/backend/src/test/resources/tests/sql/selects/date_union/content.csv
@@ -0,0 +1,11 @@
+pid,indexdatum_start,indexdatum_end,geburtsdatum
+1,2010-01-01,2010-03-31,2010-01-31
+2,2010-01-01,2010-03-31,1998-01-01
+3,2010-01-01,2010-03-31,1997-12-31
+4,2010-01-01,2010-03-31,
+5,2010-01-01,2010-03-31,2011-01-01
+6,2010-01-01,2010-03-31,1998-03-01
+7,2010-01-01,2010-03-31,2010-01-31
+8,2010-01-01,2010-03-31,1998-03-01
+9,2010-01-01,2010-03-31,1998-04-01
+10,2010-01-01,2010-03-31,2012-01-01
diff --git a/backend/src/test/resources/tests/sql/selects/date_union/date_union.json b/backend/src/test/resources/tests/sql/selects/date_union/date_union.json
new file mode 100644
index 0000000000..d2ffff5374
--- /dev/null
+++ b/backend/src/test/resources/tests/sql/selects/date_union/date_union.json
@@ -0,0 +1,67 @@
+{
+  "type": "QUERY_TEST",
+  "sqlSpec": {
+    "isEnabled": true
+  },
+  "label": "DATE_UNION select test",
+  "expectedCsv": "tests/sql/selects/date_union/expected.csv",
+  "query": {
+    "type": "CONCEPT_QUERY",
+    "root": {
+      "ids": [
+        "concept"
+      ],
+      "type": "CONCEPT",
+      "tables": [
+        {
+          "id": "concept.connector",
+          "selects": "concept.connector.date-union"
+        }
+      ]
+    }
+  },
+  "concepts": [
+    {
+      "name": "concept",
+      "type": "TREE",
+      "connectors": [
+        {
+          "label": "connector",
+          "table": "table",
+          "selects": {
+            "type": "DATE_UNION",
+            "name": "date-union",
+            "startColumn": "table.indexdatum_start",
+            "endColumn": "table.indexdatum_end"
+          }
+        }
+      ]
+    }
+  ],
+  "content": {
+    "tables": [
+      {
+        "csv": "tests/sql/selects/date_union/content.csv",
+        "name": "table",
+        "primaryColumn": {
+          "name": "pid",
+          "type": "STRING"
+        },
+        "columns": [
+          {
+            "name": "indexdatum_start",
+            "type": "DATE"
+          },
+          {
+            "name": "indexdatum_end",
+            "type": "DATE"
+          },
+          {
+            "name": "geburtsdatum",
+            "type": "DATE"
+          }
+        ]
+      }
+    ]
+  }
+}
diff --git a/backend/src/test/resources/tests/sql/selects/date_union/expected.csv b/backend/src/test/resources/tests/sql/selects/date_union/expected.csv
new file mode 100644
index 0000000000..8fd8203bcf
--- /dev/null
+++ b/backend/src/test/resources/tests/sql/selects/date_union/expected.csv
@@ -0,0 +1,11 @@
+result,dates,concept date-union
+1,{},{2010-01-01/2010-03-31}
+10,{},{2010-01-01/2010-03-31}
+2,{},{2010-01-01/2010-03-31}
+3,{},{2010-01-01/2010-03-31}
+4,{},{2010-01-01/2010-03-31}
+5,{},{2010-01-01/2010-03-31}
+6,{},{2010-01-01/2010-03-31}
+7,{},{2010-01-01/2010-03-31}
+8,{},{2010-01-01/2010-03-31}
+9,{},{2010-01-01/2010-03-31}
diff --git a/backend/src/test/resources/tests/sql/selects/sum/duration_sum/duration_sum.json b/backend/src/test/resources/tests/sql/selects/sum/duration_sum/duration_sum.json
index 088565c6c6..b6862fc9e0 100644
--- a/backend/src/test/resources/tests/sql/selects/sum/duration_sum/duration_sum.json
+++ b/backend/src/test/resources/tests/sql/selects/sum/duration_sum/duration_sum.json
@@ -45,13 +45,10 @@
       "connectors": {
         "name": "connector",
         "column": "table.column",
-        "validityDates": {
-          "name": "datum",
-          "column": "table.datum"
-        },
         "selects": {
-          "type": "EVENT_DURATION_SUM",
-          "name": "event_duration_sum"
+          "type": "DURATION_SUM",
+          "name": "event_duration_sum",
+          "column": "table.datum"
         }
       },
       "children": [
diff --git a/backend/src/test/resources/tests/sql/selects/sum/duration_sum/expected.csv b/backend/src/test/resources/tests/sql/selects/sum/duration_sum/expected.csv
index 9d58b233f6..50cc9ee49a 100644
--- a/backend/src/test/resources/tests/sql/selects/sum/duration_sum/expected.csv
+++ b/backend/src/test/resources/tests/sql/selects/sum/duration_sum/expected.csv
@@ -1,6 +1,6 @@
 result,dates,tree a event_duration_sum
-2,{2012-01-01/2012-01-01},1
-3,{2012-01-01/2012-01-02},1
-4,{2012-01-01/2012-01-02},2
-5,{2012-01-01/2012-01-05},4
-6,{-∞/+∞},
+2,{},1
+3,{},1
+4,{},2
+5,{},4
+6,{},
diff --git a/backend/src/test/resources/tests/sql/selects/sum/event_duration_sum/content.csv b/backend/src/test/resources/tests/sql/selects/sum/event_duration_sum/content.csv
new file mode 100644
index 0000000000..f70ad54470
--- /dev/null
+++ b/backend/src/test/resources/tests/sql/selects/sum/event_duration_sum/content.csv
@@ -0,0 +1,25 @@
+pid,datum,column
+1,2012-01-01,"A"
+1,2012-01-01,
+
+2,2012-01-01,"A"
+2,2012-01-01,"B"
+
+3,2012-01-01,"A"
+3,2012-01-02,"B"
+
+4,2012-01-01,"A"
+4,2012-01-02,"A"
+4,2012-01-02,"B"
+
+5,2012-01-01,"A"
+5,2012-01-02,"A"
+5,2012-01-03,"A"
+5,2012-01-04,"A"
+5,2012-01-02,"B"
+5,2012-01-03,"B"
+5,2012-01-04,"B"
+5,2012-01-05,"B"
+
+6,,"A"
+6,,"B"
diff --git a/backend/src/test/resources/tests/sql/selects/sum/event_duration_sum/duration_sum.json b/backend/src/test/resources/tests/sql/selects/sum/event_duration_sum/duration_sum.json
new file mode 100644
index 0000000000..8e9a14690d
--- /dev/null
+++ b/backend/src/test/resources/tests/sql/selects/sum/event_duration_sum/duration_sum.json
@@ -0,0 +1,99 @@
+{
+  "type": "QUERY_TEST",
+  "sqlSpec": {
+    "isEnabled": true
+  },
+  "label": "AND DURATION SUM on event date Test",
+  "expectedCsv": "tests/sql/selects/sum/event_duration_sum/expected.csv",
+  "query": {
+    "type": "CONCEPT_QUERY",
+    "root": {
+      "type": "AND",
+      "children": [
+        {
+          "type": "CONCEPT",
+          "ids": [
+            "tree.a"
+          ],
+          "tables": [
+            {
+              "id": "tree.connector",
+              "selects": [
+                "tree.connector.event_duration_sum"
+              ]
+            }
+          ]
+        },
+        {
+          "type": "CONCEPT",
+          "ids": [
+            "tree.b"
+          ],
+          "tables": [
+            {
+              "id": "tree.connector"
+            }
+          ]
+        }
+      ]
+    }
+  },
+  "concepts": [
+    {
+      "name": "tree",
+      "type": "TREE",
+      "connectors": {
+        "name": "connector",
+        "column": "table.column",
+        "validityDates": {
+          "name": "datum",
+          "column": "table.datum"
+        },
+        "selects": {
+          "type": "EVENT_DURATION_SUM",
+          "name": "event_duration_sum"
+        }
+      },
+      "children": [
+        {
+          "name": "a",
+          "condition": {
+            "type": "PREFIX_LIST",
+            "prefixes": "A"
+          },
+          "children": []
+        },
+        {
+          "name": "b",
+          "condition": {
+            "type": "PREFIX_LIST",
+            "prefixes": "B"
+          },
+          "children": []
+        }
+      ]
+    }
+  ],
+  "content": {
+    "tables": [
+      {
+        "csv": "tests/sql/selects/sum/event_duration_sum/content.csv",
+        "name": "table",
+        "primaryColumn": {
+          "name": "pid",
+          "type": "STRING"
+        },
+        "columns": [
+          {
+            "name": "datum",
+            "type": "DATE"
+          },
+          {
+            "name": "column",
+            "type": "STRING"
+          }
+        ]
+      }
+    ]
+  }
+}
diff --git a/backend/src/test/resources/tests/sql/selects/sum/event_duration_sum/expected.csv b/backend/src/test/resources/tests/sql/selects/sum/event_duration_sum/expected.csv
new file mode 100644
index 0000000000..9d58b233f6
--- /dev/null
+++ b/backend/src/test/resources/tests/sql/selects/sum/event_duration_sum/expected.csv
@@ -0,0 +1,6 @@
+result,dates,tree a event_duration_sum
+2,{2012-01-01/2012-01-01},1
+3,{2012-01-01/2012-01-02},1
+4,{2012-01-01/2012-01-02},2
+5,{2012-01-01/2012-01-05},4
+6,{-∞/+∞},

From 0d850cb29d578ab9795f786cd761c9963779955d Mon Sep 17 00:00:00 2001
From: Jonas Arnhold <jonas.arnhold@bakdata.com>
Date: Thu, 18 Jul 2024 13:29:59 +0200
Subject: [PATCH 13/41] Use DateReader in DefaultResultSetProcessor (#3497)

---
 .../mode/local/LocalNamespaceHandler.java       |  4 +++-
 .../execution/DefaultResultSetProcessor.java    | 17 ++++++++++++-----
 .../execution/ResultSetProcessorFactory.java    |  5 +++--
 .../sql/dialect/PostgreSqlIntegrationTests.java |  4 +++-
 4 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java
index b437423e30..52a389c5fe 100644
--- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java
+++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java
@@ -20,6 +20,7 @@
 import com.bakdata.conquery.sql.conversion.SqlConverter;
 import com.bakdata.conquery.sql.conversion.dialect.SqlDialect;
 import com.bakdata.conquery.sql.conversion.dialect.SqlDialectFactory;
+import com.bakdata.conquery.sql.execution.ResultSetProcessor;
 import com.bakdata.conquery.sql.execution.ResultSetProcessorFactory;
 import com.bakdata.conquery.sql.execution.SqlExecutionResult;
 import com.bakdata.conquery.sql.execution.SqlExecutionService;
@@ -48,7 +49,8 @@ public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaSto
 		DSLContext dslContext = dslContextWrapper.getDslContext();
 		SqlDialect sqlDialect = dialectFactory.createSqlDialect(databaseConfig.getDialect());
 
-		SqlExecutionService sqlExecutionService = new SqlExecutionService(dslContext, ResultSetProcessorFactory.create(sqlDialect));
+		ResultSetProcessor resultSetProcessor = ResultSetProcessorFactory.create(config, sqlDialect);
+		SqlExecutionService sqlExecutionService = new SqlExecutionService(dslContext, resultSetProcessor);
 		NodeConversions nodeConversions = new NodeConversions(idColumns, sqlDialect, dslContext, databaseConfig, sqlExecutionService);
 		SqlConverter sqlConverter = new SqlConverter(nodeConversions);
 		ExecutionManager<SqlExecutionResult> executionManager = new SqlExecutionManager(sqlConverter, sqlExecutionService, metaStorage);
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java
index 415087b74b..f14fc852b9 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java
@@ -2,7 +2,6 @@
 
 import java.math.BigDecimal;
 import java.math.RoundingMode;
-import java.sql.Date;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.Arrays;
@@ -10,11 +9,14 @@
 import java.util.function.Function;
 import java.util.function.Predicate;
 
+import com.bakdata.conquery.models.config.ConqueryConfig;
+import com.bakdata.conquery.util.DateReader;
 import lombok.RequiredArgsConstructor;
 
 @RequiredArgsConstructor
 class DefaultResultSetProcessor implements ResultSetProcessor {
 
+	private final ConqueryConfig config;
 	private final SqlCDateSetParser sqlCDateSetParser;
 
 	@Override
@@ -48,11 +50,12 @@ public Boolean getBoolean(ResultSet resultSet, int columnIndex) throws SQLExcept
 
 	@Override
 	public Number getDate(ResultSet resultSet, int columnIndex) throws SQLException {
-		Date date = resultSet.getDate(columnIndex);
-		if (date == null) {
+		String dateString = resultSet.getString(columnIndex);
+		if (dateString == null) {
 			return null;
 		}
-		return date.toLocalDate().toEpochDay();
+		DateReader dateReader = config.getLocale().getDateReader();
+		return dateReader.parseToLocalDate(dateString).toEpochDay();
 	}
 
 	@Override
@@ -97,7 +100,11 @@ public List<BigDecimal> getMoneyList(ResultSet resultSet, int columnIndex) throw
 
 	@Override
 	public List<Number> getDateList(ResultSet resultSet, int columnIndex) throws SQLException {
-		return fromString(resultSet, columnIndex, (string) -> Date.valueOf(string).toLocalDate().toEpochDay());
+		return fromString(resultSet, columnIndex, this::parseWithDateReader);
+	}
+
+	private Number parseWithDateReader(String string) {
+		return config.getLocale().getDateReader().parseToLocalDate(string).toEpochDay();
 	}
 
 	private <T> List<T> fromString(ResultSet resultSet, int columnIndex, Function<String, T> parseFunction) throws SQLException {
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessorFactory.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessorFactory.java
index 1ab5e99be0..a9165b1ba1 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessorFactory.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessorFactory.java
@@ -1,11 +1,12 @@
 package com.bakdata.conquery.sql.execution;
 
+import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.sql.conversion.dialect.SqlDialect;
 
 public class ResultSetProcessorFactory {
 
-	public static ResultSetProcessor create(SqlDialect sqlDialect) {
-		return new DefaultResultSetProcessor(sqlDialect.getCDateSetParser());
+	public static ResultSetProcessor create(ConqueryConfig config, SqlDialect sqlDialect) {
+		return new DefaultResultSetProcessor(config, sqlDialect.getCDateSetParser());
 	}
 
 }
diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java
index 3a31b73d7c..019e1cb5f7 100644
--- a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java
+++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java
@@ -17,6 +17,7 @@
 import com.bakdata.conquery.sql.conversion.dialect.PostgreSqlDialect;
 import com.bakdata.conquery.sql.conversion.model.SqlQuery;
 import com.bakdata.conquery.sql.conversion.supplier.DateNowSupplier;
+import com.bakdata.conquery.sql.execution.ResultSetProcessor;
 import com.bakdata.conquery.sql.execution.ResultSetProcessorFactory;
 import com.bakdata.conquery.sql.execution.SqlExecutionService;
 import lombok.Getter;
@@ -84,7 +85,8 @@ static void after() throws IOException {
 	public void shouldThrowException() {
 		// This can be removed as soon as we switch to a full integration test including the REST API
 		I18n.init();
-		SqlExecutionService executionService = new SqlExecutionService(dslContextWrapper.getDslContext(), ResultSetProcessorFactory.create(testSqlDialect));
+		ResultSetProcessor resultSetProcessor = ResultSetProcessorFactory.create(config, testSqlDialect);
+		SqlExecutionService executionService = new SqlExecutionService(dslContextWrapper.getDslContext(), resultSetProcessor);
 		SqlQuery validQuery = new TestSqlQuery("SELECT 1");
 		Assertions.assertThatNoException().isThrownBy(() -> executionService.execute(validQuery));
 

From b64580131715b17c09f3733ecdc9d4de037b7947 Mon Sep 17 00:00:00 2001
From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com>
Date: Tue, 23 Jul 2024 09:57:49 +0200
Subject: [PATCH 14/41] use http client where possible to import test data

---
 .../integration/json/SqlTestDataImporter.java | 44 ++----------
 .../integration/json/TestDataImporter.java    | 29 ++++++--
 .../json/WorkerTestDataImporter.java          | 68 ++++++-------------
 3 files changed, 50 insertions(+), 91 deletions(-)

diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java b/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java
index 21a37e7267..b356786602 100644
--- a/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java
+++ b/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java
@@ -1,21 +1,12 @@
 package com.bakdata.conquery.integration.json;
 
-import java.io.IOException;
 import java.util.Collection;
-import java.util.List;
 
-import com.bakdata.conquery.integration.common.LoadingUtil;
 import com.bakdata.conquery.integration.common.RequiredData;
-import com.bakdata.conquery.integration.common.RequiredSecondaryId;
 import com.bakdata.conquery.integration.common.RequiredTable;
 import com.bakdata.conquery.integration.json.filter.FilterTest;
 import com.bakdata.conquery.integration.sql.CsvTableImporter;
-import com.bakdata.conquery.models.datasets.SecondaryIdDescription;
-import com.bakdata.conquery.models.datasets.Table;
-import com.bakdata.conquery.models.datasets.concepts.Concept;
-import com.bakdata.conquery.models.exceptions.JSONException;
 import com.bakdata.conquery.util.support.StandaloneSupport;
-import com.fasterxml.jackson.databind.node.ArrayNode;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
@@ -32,6 +23,9 @@ public void importQueryTestData(StandaloneSupport support, QueryTest test) throw
 		importTables(support, content.getTables(), true);
 		importConcepts(support, test.getRawConcepts());
 		importTableContents(support, content.getTables());
+
+		importSearchIndexes(support, test.getSearchIndexes());
+		importIdMapping(support, content);
 	}
 
 	@Override
@@ -41,41 +35,15 @@ public void importFormTestData(StandaloneSupport support, FormTest test) throws
 		importTables(support, content.getTables(), true);
 		importConcepts(support, test.getRawConcepts());
 		importTableContents(support, content.getTables());
-		waitUntilDone(support, () -> LoadingUtil.importIdMapping(support, content));
-		waitUntilDone(support, () -> LoadingUtil.importPreviousQueries(support, content, support.getTestUser()));
+		importIdMapping(support, content);
+		importPreviousQueries(support, content);
 	}
 
 	@Override
-	public void importFilterTestData(StandaloneSupport support, FilterTest filterTest) throws Exception {
+	public void importFilterTestData(StandaloneSupport support, FilterTest filterTest) {
 		throw new UnsupportedOperationException("Not implemented yet.");
 	}
 
-	@Override
-	public void importSecondaryIds(StandaloneSupport support, List<RequiredSecondaryId> secondaryIds) {
-		for (RequiredSecondaryId required : secondaryIds) {
-			final SecondaryIdDescription description =
-					required.toSecondaryId(support.getDataset(), support.getDatasetRegistry().findRegistry(support.getDataset().getId()));
-			support.getDatasetsProcessor().addSecondaryId(support.getNamespace(), description);
-		}
-	}
-
-	@Override
-	public void importTables(StandaloneSupport support, List<RequiredTable> tables, boolean autoConcept) throws JSONException {
-		for (RequiredTable requiredTable : tables) {
-			final Table table = requiredTable.toTable(support.getDataset(), support.getNamespaceStorage().getCentralRegistry());
-			support.getNamespaceStorage().addTable(table);
-		}
-	}
-
-	@Override
-	public void importConcepts(StandaloneSupport support, ArrayNode rawConcepts) throws JSONException, IOException {
-		List<Concept<?>> concepts =
-				ConqueryTestSpec.parseSubTreeList(support, rawConcepts, Concept.class, concept -> concept.setDataset(support.getDataset()));
-		for (Concept<?> concept : concepts) {
-			support.getNamespaceStorage().updateConcept(concept);
-		}
-	}
-
 	@Override
 	public void importTableContents(StandaloneSupport support, Collection<RequiredTable> tables) throws Exception {
 		for (RequiredTable table : tables) {
diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/TestDataImporter.java b/backend/src/test/java/com/bakdata/conquery/integration/json/TestDataImporter.java
index 5d29d2bbd6..6fef8c0dad 100644
--- a/backend/src/test/java/com/bakdata/conquery/integration/json/TestDataImporter.java
+++ b/backend/src/test/java/com/bakdata/conquery/integration/json/TestDataImporter.java
@@ -4,10 +4,13 @@
 import java.util.Collection;
 import java.util.List;
 
+import com.bakdata.conquery.integration.common.LoadingUtil;
+import com.bakdata.conquery.integration.common.RequiredData;
 import com.bakdata.conquery.integration.common.RequiredSecondaryId;
 import com.bakdata.conquery.integration.common.RequiredTable;
 import com.bakdata.conquery.integration.json.filter.FilterTest;
 import com.bakdata.conquery.models.exceptions.JSONException;
+import com.bakdata.conquery.models.index.search.SearchIndex;
 import com.bakdata.conquery.util.support.StandaloneSupport;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 
@@ -19,13 +22,31 @@ public interface TestDataImporter {
 
 	void importFilterTestData(StandaloneSupport support, FilterTest filterTest) throws Exception;
 
-	void importSecondaryIds(StandaloneSupport support, List<RequiredSecondaryId> secondaryIds);
+	void importTableContents(StandaloneSupport support, Collection<RequiredTable> tables) throws Exception;
+
+	default void importSearchIndexes(StandaloneSupport support, List<SearchIndex> searchIndices) {
+		waitUntilDone(support, () -> LoadingUtil.importSearchIndexes(support, searchIndices));
+	}
 
-	void importTables(StandaloneSupport support, List<RequiredTable> tables, boolean autoConcept) throws JSONException;
+	default void importIdMapping(StandaloneSupport support, RequiredData content) {
+		waitUntilDone(support, () -> LoadingUtil.importIdMapping(support, content));
+	}
 
-	void importConcepts(StandaloneSupport support, ArrayNode rawConcepts) throws JSONException, IOException;
+	default void importSecondaryIds(StandaloneSupport support, List<RequiredSecondaryId> secondaryIds) {
+		waitUntilDone(support, () -> LoadingUtil.importSecondaryIds(support, secondaryIds));
+	}
 
-	void importTableContents(StandaloneSupport support, Collection<RequiredTable> tables) throws Exception;
+	default void importTables(StandaloneSupport support, List<RequiredTable> tables, boolean autoConcept) throws JSONException {
+		waitUntilDone(support, () -> LoadingUtil.importTables(support, tables, autoConcept));
+	}
+
+	default void importConcepts(StandaloneSupport support, ArrayNode rawConcepts) throws JSONException, IOException {
+		waitUntilDone(support, () -> LoadingUtil.importConcepts(support, rawConcepts));
+	}
+
+	default void importPreviousQueries(StandaloneSupport support, RequiredData content) {
+		waitUntilDone(support, () -> LoadingUtil.importPreviousQueries(support, content, support.getTestUser()));
+	}
 
 	default void waitUntilDone(StandaloneSupport support, CheckedRunnable<?> runnable) {
 		runnable.run();
diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/WorkerTestDataImporter.java b/backend/src/test/java/com/bakdata/conquery/integration/json/WorkerTestDataImporter.java
index 21a537e54b..ba050711b1 100644
--- a/backend/src/test/java/com/bakdata/conquery/integration/json/WorkerTestDataImporter.java
+++ b/backend/src/test/java/com/bakdata/conquery/integration/json/WorkerTestDataImporter.java
@@ -2,22 +2,15 @@
 
 import static com.bakdata.conquery.integration.common.LoadingUtil.importInternToExternMappers;
 
-import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.List;
 
 import com.bakdata.conquery.integration.common.LoadingUtil;
 import com.bakdata.conquery.integration.common.RequiredData;
-import com.bakdata.conquery.integration.common.RequiredSecondaryId;
 import com.bakdata.conquery.integration.common.RequiredTable;
 import com.bakdata.conquery.integration.json.filter.FilterTest;
 import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeConnector;
-import com.bakdata.conquery.models.exceptions.JSONException;
-import com.bakdata.conquery.models.messages.namespaces.specific.UpdateMatchingStatsMessage;
-import com.bakdata.conquery.models.worker.DistributedNamespace;
 import com.bakdata.conquery.util.support.StandaloneSupport;
-import com.fasterxml.jackson.databind.node.ArrayNode;
 
 public class WorkerTestDataImporter implements TestDataImporter {
 
@@ -29,15 +22,13 @@ public void importQueryTestData(StandaloneSupport support, QueryTest test) throw
 		importSecondaryIds(support, content.getSecondaryIds());
 		importInternToExternMappers(support, test.getInternToExternMappings());
 
-		waitUntilDone(support, () -> LoadingUtil.importSearchIndexes(support, test.getSearchIndexes()));
-		waitUntilDone(support, () -> LoadingUtil.importTables(support, content.getTables(), content.isAutoConcept()));
-		waitUntilDone(support, () -> LoadingUtil.importConcepts(support, test.getRawConcepts()));
+		importSearchIndexes(support, test.getSearchIndexes());
+		importTables(support, content.getTables(), content.isAutoConcept());
+		importConcepts(support, test.getRawConcepts());
 		waitUntilDone(support, () -> LoadingUtil.importTableContents(support, content.getTables()));
-		waitUntilDone(support, () -> LoadingUtil.importIdMapping(support, content));
-		waitUntilDone(support, () -> LoadingUtil.importPreviousQueries(support, content, support.getTestUser()));
+		importIdMapping(support, content);
+		importPreviousQueries(support, content);
 		waitUntilDone(support, () -> LoadingUtil.updateMatchingStats(support));
-
-		sendUpdateMatchingStatsMessage(support);
 	}
 
 	@Override
@@ -45,12 +36,12 @@ public void importFormTestData(StandaloneSupport support, FormTest test) throws
 
 		RequiredData content = test.getContent();
 
-		waitUntilDone(support, () -> LoadingUtil.importSecondaryIds(support, content.getSecondaryIds()));
-		waitUntilDone(support, () -> LoadingUtil.importTables(support, content.getTables(), content.isAutoConcept()));
-		waitUntilDone(support, () -> LoadingUtil.importConcepts(support, test.getRawConcepts()));
+		importSecondaryIds(support, content.getSecondaryIds());
+		importTables(support, content.getTables(), content.isAutoConcept());
+		importConcepts(support, test.getRawConcepts());
 		waitUntilDone(support, () -> LoadingUtil.importTableContents(support, content.getTables()));
-		waitUntilDone(support, () -> LoadingUtil.importIdMapping(support, content));
-		waitUntilDone(support, () -> LoadingUtil.importPreviousQueries(support, content, support.getTestUser()));
+		importIdMapping(support, content);
+		importPreviousQueries(support, content);
 	}
 
 	@Override
@@ -58,17 +49,16 @@ public void importFilterTestData(StandaloneSupport support, FilterTest test) thr
 
 		RequiredData content = test.getContent();
 
-		LoadingUtil.importInternToExternMappers(support, test.getInternToExternMappings());
-		LoadingUtil.importSearchIndexes(support, test.getSearchIndices());
-		waitUntilDone(support, () -> LoadingUtil.importTables(support, content.getTables(), content.isAutoConcept()));
+		importInternToExternMappers(support, test.getInternToExternMappings());
+		importSearchIndexes(support, test.getSearchIndices());
+		importTables(support, content.getTables(), content.isAutoConcept());
 
 		test.setConnector(ConqueryTestSpec.parseSubTree(
-								  support,
-								  test.getRawConnector(),
-								  ConceptTreeConnector.class,
-								  conn -> conn.setConcept(test.getConcept())
-						  )
-		);
+				support,
+				test.getRawConnector(),
+				ConceptTreeConnector.class,
+				conn -> conn.setConcept(test.getConcept())
+		));
 		test.getConcept().setConnectors(Collections.singletonList((ConceptTreeConnector) test.getConnector()));
 
 		waitUntilDone(support, () -> LoadingUtil.uploadConcept(support, support.getDataset(), test.getConcept()));
@@ -76,30 +66,10 @@ public void importFilterTestData(StandaloneSupport support, FilterTest test) thr
 		waitUntilDone(support, () -> LoadingUtil.updateMatchingStats(support));
 	}
 
-	@Override
-	public void importSecondaryIds(StandaloneSupport support, List<RequiredSecondaryId> secondaryIds) {
-		waitUntilDone(support, () -> LoadingUtil.importSecondaryIds(support, secondaryIds));
-	}
-
-	@Override
-	public void importTables(StandaloneSupport support, List<RequiredTable> tables, boolean autoConcept) throws JSONException {
-		waitUntilDone(support, () -> LoadingUtil.importTables(support, tables, autoConcept));
-	}
-
-	@Override
-	public void importConcepts(StandaloneSupport support, ArrayNode rawConcepts) throws JSONException, IOException {
-		waitUntilDone(support, () -> LoadingUtil.importConcepts(support, rawConcepts));
-	}
 
 	@Override
 	public void importTableContents(StandaloneSupport support, Collection<RequiredTable> tables) throws Exception {
 		waitUntilDone(support, () -> LoadingUtil.importTableContents(support, tables));
 	}
 
-	private static void sendUpdateMatchingStatsMessage(StandaloneSupport support) {
-		DistributedNamespace namespace = (DistributedNamespace) support.getNamespace();
-		namespace.getWorkerHandler().sendToAll(new UpdateMatchingStatsMessage(support.getNamespace().getStorage().getAllConcepts()));
-		support.waitUntilWorkDone();
-	}
-
-}
+}
\ No newline at end of file

From bfe907b70917ed65bf8847f328b2336d2542f5fc Mon Sep 17 00:00:00 2001
From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com>
Date: Tue, 23 Jul 2024 11:00:51 +0200
Subject: [PATCH 15/41] fix FilterAutocompleteTest to be independent of concept
 order

---
 .../conquery/integration/tests/FilterAutocompleteTest.java    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java
index e05a0b4acd..3846fb4286 100644
--- a/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java
+++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java
@@ -17,6 +17,7 @@
 import com.bakdata.conquery.integration.IntegrationTest;
 import com.bakdata.conquery.integration.json.ConqueryTestSpec;
 import com.bakdata.conquery.integration.json.JsonIntegrationTest;
+import com.bakdata.conquery.io.storage.NamespaceStorage;
 import com.bakdata.conquery.models.config.CSVConfig;
 import com.bakdata.conquery.models.datasets.concepts.Concept;
 import com.bakdata.conquery.models.datasets.concepts.Connector;
@@ -151,7 +152,8 @@ private static SelectFilter<?> setupSearch(StandaloneSupport conquery) throws Ex
 
 		final CSVConfig csvConf = conquery.getConfig().getCsv();
 
-		final Concept<?> concept = conquery.getNamespace().getStorage().getAllConcepts().iterator().next();
+		NamespaceStorage namespaceStorage = conquery.getNamespace().getStorage();
+		final Concept<?> concept = namespaceStorage.getAllConcepts().stream().filter(c -> c.getName().equals("geschlecht_select")).findFirst().orElseThrow();
 		final Connector connector = concept.getConnectors().iterator().next();
 		final SelectFilter<?> filter = (SelectFilter<?>) connector.getFilters().iterator().next();
 

From ef33e097424d9a84bcc267af527be399534a5294 Mon Sep 17 00:00:00 2001
From: awildturtok <1553491+awildturtok@users.noreply.github.com>
Date: Wed, 24 Jul 2024 16:24:24 +0200
Subject: [PATCH 16/41] remove ResultType CPSType usage. Simply inline the
 type.

---
 .../conquery/models/types/ResultType.java     | 152 +++++++++---------
 1 file changed, 79 insertions(+), 73 deletions(-)

diff --git a/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java b/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java
index 02c82abfe1..52ccbfbea9 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java
@@ -11,15 +11,12 @@
 
 import c10n.C10N;
 import com.bakdata.conquery.internationalization.Results;
-import com.bakdata.conquery.io.cps.CPSBase;
-import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.models.common.daterange.CDateRange;
 import com.bakdata.conquery.models.config.LocaleConfig;
 import com.bakdata.conquery.models.events.MajorTypeId;
 import com.bakdata.conquery.models.query.PrintSettings;
 import com.bakdata.conquery.sql.execution.ResultSetProcessor;
 import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import com.google.common.base.Preconditions;
 import lombok.AccessLevel;
 import lombok.EqualsAndHashCode;
@@ -29,28 +26,9 @@
 import lombok.extern.slf4j.Slf4j;
 import org.jetbrains.annotations.NotNull;
 
-@JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "type")
-@CPSBase
 @Slf4j
 public abstract class ResultType<T> {
 
-	public String printNullable(PrintSettings cfg, Object f) {
-		if (f == null) {
-			return "";
-		}
-		return print(cfg, f);
-	}
-
-	protected String print(PrintSettings cfg, @NonNull Object f) {
-		return f.toString();
-	}
-
-	public abstract String typeInfo();
-
-	public abstract T getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException;
-
-	protected abstract List<T> getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException;
-
 	public static ResultType<?> resolveResultType(MajorTypeId majorTypeId) {
 		return switch (majorTypeId) {
 			case STRING -> StringT.INSTANCE;
@@ -63,21 +41,27 @@ public static ResultType<?> resolveResultType(MajorTypeId majorTypeId) {
 		};
 	}
 
-	abstract static class PrimitiveResultType<T> extends ResultType<T> {
-		@Override
-		public String typeInfo() {
-			return this.getClass().getAnnotation(CPSType.class).id();
+	public String printNullable(PrintSettings cfg, Object f) {
+		if (f == null) {
+			return "";
 		}
+		return print(cfg, f);
+	}
 
-		@Override
-		public String toString() {
-			return typeInfo();
-		}
+	protected abstract String print(PrintSettings cfg, @NonNull Object f);
+
+	public String toString() {
+		return typeInfo();
 	}
 
-	@CPSType(id = "BOOLEAN", base = ResultType.class)
+	public abstract String typeInfo();
+
+	public abstract T getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException;
+
+	protected abstract List<T> getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException;
+
 	@NoArgsConstructor(access = AccessLevel.PRIVATE)
-	public static class BooleanT extends PrimitiveResultType<Boolean> {
+	public static class BooleanT extends ResultType<Boolean> {
 		@Getter(onMethod_ = @JsonCreator)
 		public static final BooleanT INSTANCE = new BooleanT();
 
@@ -93,6 +77,11 @@ public String print(PrintSettings cfg, Object f) {
 			return (Boolean) f ? "1" : "0";
 		}
 
+		@Override
+		public String typeInfo() {
+			return "BOOLEAN";
+		}
+
 		@Override
 		public Boolean getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
 			return resultSetProcessor.getBoolean(resultSet, columnIndex);
@@ -105,9 +94,8 @@ protected List<Boolean> getFromResultSetAsList(ResultSet resultSet, int columnIn
 	}
 
 
-	@CPSType(id = "INTEGER", base = ResultType.class)
 	@NoArgsConstructor(access = AccessLevel.PRIVATE)
-	public static class IntegerT extends PrimitiveResultType<Integer> {
+	public static class IntegerT extends ResultType<Integer> {
 		@Getter(onMethod_ = @JsonCreator)
 		public static final IntegerT INSTANCE = new IntegerT();
 
@@ -119,6 +107,11 @@ public String print(PrintSettings cfg, Object f) {
 			return f.toString();
 		}
 
+		@Override
+		public String typeInfo() {
+			return "INTEGER";
+		}
+
 		@Override
 		public Integer getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
 			return resultSetProcessor.getInteger(resultSet, columnIndex);
@@ -130,9 +123,8 @@ protected List<Integer> getFromResultSetAsList(ResultSet resultSet, int columnIn
 		}
 	}
 
-	@CPSType(id = "NUMERIC", base = ResultType.class)
 	@NoArgsConstructor(access = AccessLevel.PRIVATE)
-	public static class NumericT extends PrimitiveResultType<Double> {
+	public static class NumericT extends ResultType<Double> {
 		@Getter(onMethod_ = @JsonCreator)
 		public static final NumericT INSTANCE = new NumericT();
 
@@ -144,6 +136,11 @@ public String print(PrintSettings cfg, Object f) {
 			return f.toString();
 		}
 
+		@Override
+		public String typeInfo() {
+			return "NUMERIC";
+		}
+
 		@Override
 		public Double getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
 			return resultSetProcessor.getDouble(resultSet, columnIndex);
@@ -155,9 +152,8 @@ protected List<Double> getFromResultSetAsList(ResultSet resultSet, int columnInd
 		}
 	}
 
-	@CPSType(id = "DATE", base = ResultType.class)
 	@NoArgsConstructor(access = AccessLevel.PRIVATE)
-	public static class DateT extends PrimitiveResultType<Number> {
+	public static class DateT extends ResultType<Number> {
 		@Getter(onMethod_ = @JsonCreator)
 		public static final DateT INSTANCE = new DateT();
 
@@ -170,6 +166,15 @@ public String print(PrintSettings cfg, @NonNull Object f) {
 			return print(number, cfg.getDateFormatter());
 		}
 
+		public static String print(Number num, DateTimeFormatter formatter) {
+			return formatter.format(LocalDate.ofEpochDay(num.intValue()));
+		}
+
+		@Override
+		public String typeInfo() {
+			return "DATE";
+		}
+
 		@Override
 		public Number getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
 			return resultSetProcessor.getDate(resultSet, columnIndex);
@@ -179,49 +184,49 @@ public Number getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetPr
 		protected List<Number> getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
 			return resultSetProcessor.getDateList(resultSet, columnIndex);
 		}
-
-		public static String print(Number num, DateTimeFormatter formatter) {
-			return formatter.format(LocalDate.ofEpochDay(num.intValue()));
-		}
 	}
 
 	/**
 	 * A DateRange is provided by in a query result as two ints in a list, both standing for an epoch day (see {@link LocalDate#toEpochDay()}).
 	 * The first int describes the included lower bound of the range. The second int descibes the included upper bound.
 	 */
-	@CPSType(id = "DATE_RANGE", base = ResultType.class)
 	@NoArgsConstructor(access = AccessLevel.PRIVATE)
-	public static class DateRangeT extends PrimitiveResultType<List<Integer>> {
+	public static class DateRangeT extends ResultType<List<Integer>> {
 		@Getter(onMethod_ = @JsonCreator)
 		public static final DateRangeT INSTANCE = new DateRangeT();
 
 		@Override
 		public String print(PrintSettings cfg, @NonNull Object f) {
-			if (!(f instanceof List)) {
-				throw new IllegalStateException(String.format("Expected a List got %s (Type: %s, as string: %s)", f, f.getClass().getName(), f));
-			}
-			List<?> list = (List<?>) f;
-			if (list.size() != 2) {
-				throw new IllegalStateException("Expected a list with 2 elements, one min, one max. The list was: " + list);
-			}
+			Preconditions.checkArgument(f instanceof List<?>, "Expected a List got %s (Type: %s, as string: %s)", f, f.getClass().getName(), f);
+
+			final List<?> list = (List<?>) f;
+
+			Preconditions.checkArgument(list.size() == 2, "Expected a list with 2 elements: one min, one max. The list was: %s", list);
+
 			final DateTimeFormatter dateFormat = cfg.getDateFormatter();
 			final Integer min = (Integer) list.get(0);
 			final Integer max = (Integer) list.get(1);
+
 			if (min == null || max == null) {
 				log.warn("Encountered incomplete range, treating it as an open range. Either min or max was null: {}", list);
 			}
 			// Compute minString first because we need it either way
-			String minString = min == null || min == CDateRange.NEGATIVE_INFINITY ? "-∞" : ResultType.DateT.print(min, dateFormat);
+			final String minString = min == null || min == CDateRange.NEGATIVE_INFINITY ? "-∞" : ResultType.DateT.print(min, dateFormat);
 
 			if (cfg.isPrettyPrint() && min != null && min.equals(max)) {
 				// If the min and max are the same we print it like a singe date, not a range (only in pretty printing)
 				return minString;
 			}
-			String maxString = max == null || max == CDateRange.POSITIVE_INFINITY ? "+∞" : ResultType.DateT.print(max, dateFormat);
+			final String maxString = max == null || max == CDateRange.POSITIVE_INFINITY ? "+∞" : ResultType.DateT.print(max, dateFormat);
 
 			return minString + cfg.getDateRangeSeparator() + maxString;
 		}
 
+		@Override
+		public String typeInfo() {
+			return "DATE_RANGE";
+		}
+
 		@Override
 		public List<Integer> getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
 			return resultSetProcessor.getDateRange(resultSet, columnIndex);
@@ -233,9 +238,8 @@ public List<List<Integer>> getFromResultSetAsList(ResultSet resultSet, int colum
 		}
 	}
 
-	@CPSType(id = "STRING", base = ResultType.class)
 	@NoArgsConstructor(access = AccessLevel.PRIVATE)
-	public static class StringT extends PrimitiveResultType<String> {
+	public static class StringT extends ResultType<String> {
 		@Getter(onMethod_ = @JsonCreator)
 		public static final StringT INSTANCE = new StringT();
 
@@ -253,9 +257,14 @@ public StringT(BiFunction<Object, PrintSettings, String> valueMapper) {
 		@Override
 		protected String print(PrintSettings cfg, @NonNull Object f) {
 			if (valueMapper == null) {
-				return super.print(cfg, f);
+				return f.toString();
 			}
-			return super.print(cfg, valueMapper.apply(f, cfg));
+			return valueMapper.apply(f, cfg);
+		}
+
+		@Override
+		public String typeInfo() {
+			return "STRING";
 		}
 
 		@Override
@@ -269,9 +278,8 @@ protected List<String> getFromResultSetAsList(ResultSet resultSet, int columnInd
 		}
 	}
 
-	@CPSType(id = "MONEY", base = ResultType.class)
 	@NoArgsConstructor(access = AccessLevel.PRIVATE)
-	public static class MoneyT extends PrimitiveResultType<BigDecimal> {
+	public static class MoneyT extends ResultType<BigDecimal> {
 
 		@Getter(onMethod_ = @JsonCreator)
 		public static final MoneyT INSTANCE = new MoneyT();
@@ -284,24 +292,27 @@ public String print(PrintSettings cfg, Object f) {
 			return IntegerT.INSTANCE.print(cfg, f);
 		}
 
-		@Override
-		public BigDecimal getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
-			return resultSetProcessor.getMoney(resultSet, columnIndex);
-		}
-
-
 		@NotNull
 		public BigDecimal readIntermediateValue(PrintSettings cfg, Number f) {
 			return new BigDecimal(f.longValue()).movePointLeft(cfg.getCurrency().getDefaultFractionDigits());
 		}
 
+		@Override
+		public String typeInfo() {
+			return "MONEY";
+		}
+
+		@Override
+		public BigDecimal getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
+			return resultSetProcessor.getMoney(resultSet, columnIndex);
+		}
+
 		@Override
 		protected List<BigDecimal> getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
 			return resultSetProcessor.getMoneyList(resultSet, columnIndex);
 		}
 	}
 
-	@CPSType(id = "LIST", base = ResultType.class)
 	@Getter
 	@EqualsAndHashCode(callSuper = false)
 	public static class ListT<T> extends ResultType<List<T>> {
@@ -322,7 +333,7 @@ public String print(PrintSettings cfg, @NonNull Object f) {
 			}
 			// Not sure if this escaping is enough
 			final LocaleConfig.ListFormat listFormat = cfg.getListFormat();
-			StringJoiner joiner = listFormat.createListJoiner();
+			final StringJoiner joiner = listFormat.createListJoiner();
 			for (Object obj : (List<?>) f) {
 				joiner.add(listFormat.escapeListElement(elementType.print(cfg, obj)));
 			}
@@ -331,7 +342,7 @@ public String print(PrintSettings cfg, @NonNull Object f) {
 
 		@Override
 		public String typeInfo() {
-			return this.getClass().getAnnotation(CPSType.class).id() + "[" + elementType.typeInfo() + "]";
+			return "LIST[" + elementType.typeInfo() + "]";
 		}
 
 		@Override
@@ -343,10 +354,5 @@ public List<T> getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetP
 		protected List<List<T>> getFromResultSetAsList(final ResultSet resultSet, final int columnIndex, final ResultSetProcessor resultSetProcessor) {
 			throw new UnsupportedOperationException("Nested lists not supported in SQL mode");
 		}
-
-		@Override
-		public String toString() {
-			return typeInfo();
-		}
 	}
 }

From ffe1edc349cd910c350d7b3eee3b53b023a9abe8 Mon Sep 17 00:00:00 2001
From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com>
Date: Wed, 24 Jul 2024 18:17:29 +0200
Subject: [PATCH 17/41] remove dependence of DatasetRegistry on MetaStorage

---
 .../conquery/commands/ManagerNode.java        | 26 +++++++--------
 .../MetaIdReferenceDeserializer.java          | 10 ++----
 .../conquery/io/storage/MetaStorage.java      | 20 +++++------
 .../conquery/io/storage/NamespaceStorage.java |  3 +-
 .../io/storage/NamespacedStorage.java         |  4 ++-
 .../conquery/mode/DelegateManager.java        |  5 +--
 .../mode/InternalObjectMapperCreator.java     |  4 +--
 .../com/bakdata/conquery/mode/Manager.java    |  5 +--
 .../conquery/mode/ManagerProvider.java        |  3 --
 .../conquery/mode/NamespaceHandler.java       |  7 ++--
 .../mode/cluster/ClusterManagerProvider.java  |  6 ++--
 .../mode/local/LocalManagerProvider.java      | 12 ++++---
 .../conquery/models/config/StoreFactory.java  | 11 ++-----
 .../models/config/XodusStoreFactory.java      | 31 +++++++++--------
 .../models/identifiable/CentralRegistry.java  | 24 ++------------
 .../models/identifiable/ids/NamespacedId.java |  5 +--
 .../models/worker/DatasetRegistry.java        | 24 ++++----------
 .../models/worker/IdResolveContext.java       |  3 --
 .../conquery/models/worker/Namespace.java     |  6 ----
 .../worker/SingletonNamespaceCollection.java  | 10 ------
 .../conquery/models/worker/Worker.java        |  5 ---
 .../conquery/models/worker/Workers.java       |  5 ---
 .../resources/admin/AdminServlet.java         | 33 ++++---------------
 .../admin/rest/AdminDatasetProcessor.java     |  4 ++-
 .../integration/json/ConqueryTestSpec.java    | 33 +++++++------------
 .../io/AbstractSerializationTest.java         | 10 +++---
 .../io/jackson/serializer/IdRefrenceTest.java | 33 ++++++-------------
 .../serializer/SerializationTestUtil.java     |  4 +--
 .../concepts/tree/GroovyIndexedTest.java      | 22 +++++--------
 .../util/NonPersistentStoreFactory.java       | 19 +++--------
 .../util/support/StandaloneSupport.java       | 12 +++----
 .../conquery/util/support/TestConquery.java   | 10 +++---
 32 files changed, 148 insertions(+), 261 deletions(-)

diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java
index 004048186d..61b52502b7 100644
--- a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java
+++ b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java
@@ -113,7 +113,7 @@ public void run(Manager manager) throws InterruptedException {
 		// Create AdminServlet first to make it available to the realms
 		admin = new AdminServlet(this);
 
-		authController = new AuthorizationController(getStorage(), config, environment, admin);
+		authController = new AuthorizationController(getMetaStorage(), config, environment, admin);
 		environment.lifecycle().manage(authController);
 
 		// Register default components for the admin interface
@@ -145,14 +145,14 @@ public void run(Manager manager) throws InterruptedException {
 	private void registerTasks(Manager manager, Environment environment, ConqueryConfig config) {
 		environment.admin().addTask(formScanner);
 		environment.admin().addTask(
-				new QueryCleanupTask(getStorage(), Duration.of(
+				new QueryCleanupTask(getMetaStorage(), Duration.of(
 						config.getQueries().getOldQueriesTime().getQuantity(),
 						config.getQueries().getOldQueriesTime().getUnit().toChronoUnit()
 				)));
 
-		environment.admin().addTask(new PermissionCleanupTask(getStorage()));
+		environment.admin().addTask(new PermissionCleanupTask(getMetaStorage()));
 		manager.getAdminTasks().forEach(environment.admin()::addTask);
-		environment.admin().addTask(new ReloadMetaStorageTask(getStorage()));
+		environment.admin().addTask(new ReloadMetaStorageTask(getMetaStorage()));
 
 		final ShutdownTask shutdown = new ShutdownTask();
 		environment.admin().addTask(shutdown);
@@ -164,7 +164,7 @@ private void configureApiServlet(ConqueryConfig config, DropwizardResourceConfig
 		jerseyConfig.register(new AbstractBinder() {
 			@Override
 			protected void configure() {
-				bind(getStorage()).to(MetaStorage.class);
+				bind(getMetaStorage()).to(MetaStorage.class);
 				bind(getDatasetRegistry()).to(DatasetRegistry.class);
 			}
 		});
@@ -203,7 +203,7 @@ public void customizeApiObjectMapper(ObjectMapper objectMapper) {
 		injectableValues.add(Validator.class, getValidator());
 
 		getDatasetRegistry().injectInto(objectMapper);
-		getStorage().injectInto(objectMapper);
+		getMetaStorage().injectInto(objectMapper);
 		getConfig().injectInto(objectMapper);
 	}
 
@@ -219,10 +219,10 @@ public ObjectMapper createInternalObjectMapper(Class<? extends View> viewClass)
 
 	private void loadMetaStorage() {
 		log.info("Opening MetaStorage");
-		getStorage().openStores(getInternalObjectMapperCreator().createInternalObjectMapper(View.Persistence.Manager.class));
+		getMetaStorage().openStores(getInternalObjectMapperCreator().createInternalObjectMapper(View.Persistence.Manager.class));
 		log.info("Loading MetaStorage");
-		getStorage().loadData();
-		log.info("MetaStorage loaded {}", getStorage());
+		getMetaStorage().loadData();
+		log.info("MetaStorage loaded {}", getMetaStorage());
 	}
 
 	@SneakyThrows(InterruptedException.class)
@@ -236,7 +236,7 @@ public void loadNamespaces() {
 		final Collection<NamespaceStorage> namespaceStorages = getConfig().getStorage().discoverNamespaceStorages();
 		for (NamespaceStorage namespaceStorage : namespaceStorages) {
 			loaders.submit(() -> {
-				registry.createNamespace(namespaceStorage);
+				registry.createNamespace(namespaceStorage, getMetaStorage());
 			});
 		}
 
@@ -262,16 +262,16 @@ public void stop() throws Exception {
 				provider.close();
 			}
 			catch (Exception e) {
-				log.error(provider + " could not be closed", e);
+				log.error("{} could not be closed", provider, e);
 			}
 
 		}
 
 		try {
-			getStorage().close();
+			getMetaStorage().close();
 		}
 		catch (Exception e) {
-			log.error("{} could not be closed", getStorage(), e);
+			log.error("{} could not be closed", getMetaStorage(), e);
 		}
 
 	}
diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/MetaIdReferenceDeserializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/MetaIdReferenceDeserializer.java
index 0023d97757..e08b026eea 100644
--- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/MetaIdReferenceDeserializer.java
+++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/MetaIdReferenceDeserializer.java
@@ -4,18 +4,14 @@
 import java.util.InputMismatchException;
 import java.util.Optional;
 
+import com.bakdata.conquery.io.storage.MetaStorage;
 import com.bakdata.conquery.models.identifiable.CentralRegistry;
 import com.bakdata.conquery.models.identifiable.Identifiable;
 import com.bakdata.conquery.models.identifiable.ids.Id;
 import com.bakdata.conquery.models.identifiable.ids.IdUtil;
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonToken;
-import com.fasterxml.jackson.databind.BeanDescription;
-import com.fasterxml.jackson.databind.BeanProperty;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JavaType;
-import com.fasterxml.jackson.databind.JsonDeserializer;
-import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.*;
 import com.fasterxml.jackson.databind.deser.ContextualDeserializer;
 import com.fasterxml.jackson.databind.deser.SettableBeanProperty;
 import com.fasterxml.jackson.databind.jsontype.TypeDeserializer;
@@ -42,7 +38,7 @@ public T deserialize(JsonParser parser, DeserializationContext ctxt) throws IOEx
 		ID id = ctxt.readValue(parser, idClass);
 
 		try {
-			final CentralRegistry centralRegistry = CentralRegistry.get(ctxt);
+			final CentralRegistry centralRegistry = MetaStorage.get(ctxt).getCentralRegistry();
 
 			// Not all Components have registries, we leave it up to the validator to be angry.
 			if (centralRegistry == null) {
diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java
index d8c12a95b8..d0cf222cef 100644
--- a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java
+++ b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java
@@ -11,13 +11,9 @@
 import com.bakdata.conquery.models.execution.ManagedExecution;
 import com.bakdata.conquery.models.forms.configs.FormConfig;
 import com.bakdata.conquery.models.identifiable.CentralRegistry;
-import com.bakdata.conquery.models.identifiable.ids.specific.FormConfigId;
-import com.bakdata.conquery.models.identifiable.ids.specific.GroupId;
-import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId;
-import com.bakdata.conquery.models.identifiable.ids.specific.RoleId;
-import com.bakdata.conquery.models.identifiable.ids.specific.UserId;
-import com.bakdata.conquery.models.worker.DatasetRegistry;
-import com.bakdata.conquery.models.worker.Namespace;
+import com.bakdata.conquery.models.identifiable.ids.specific.*;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
@@ -34,8 +30,6 @@ public class MetaStorage extends ConqueryStorage implements Injectable {
 	protected final CentralRegistry centralRegistry = new CentralRegistry();
 	private final StoreFactory storageFactory;
 
-	@Getter
-	protected final DatasetRegistry<? extends Namespace> datasetRegistry;
 	private IdentifiableStore<ManagedExecution> executions;
 	private IdentifiableStore<FormConfig> formConfigs;
 	private IdentifiableStore<User> authUser;
@@ -47,8 +41,8 @@ public void openStores(ObjectMapper mapper) {
 		authRole = storageFactory.createRoleStore(centralRegistry, "meta", this, mapper);
 		authGroup = storageFactory.createGroupStore(centralRegistry, "meta", this, mapper);
 		// Executions depend on users
-		executions = storageFactory.createExecutionsStore(centralRegistry, datasetRegistry, "meta", mapper);
-		formConfigs = storageFactory.createFormConfigStore(centralRegistry, datasetRegistry, "meta", mapper);
+		executions = storageFactory.createExecutionsStore(centralRegistry, "meta", mapper);
+		formConfigs = storageFactory.createFormConfigStore(centralRegistry, "meta", mapper);
 
 	}
 
@@ -196,4 +190,8 @@ public void addFormConfig(FormConfig formConfig) {
 	public MutableInjectableValues inject(MutableInjectableValues values) {
 		return values.add(MetaStorage.class, this);
 	}
+
+	public static MetaStorage get(DeserializationContext ctxt) throws JsonMappingException {
+		return (MetaStorage) ctxt.findInjectableValue(MetaStorage.class.getName(), null, null);
+	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java
index f84b4dc544..7b29656e6d 100644
--- a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java
+++ b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java
@@ -17,7 +17,6 @@
 import com.bakdata.conquery.models.worker.WorkerToBucketsMap;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
-import jakarta.validation.Validator;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -32,7 +31,7 @@ public class NamespaceStorage extends NamespacedStorage {
 
 	protected CachedStore<String, Integer> entity2Bucket;
 
-	public NamespaceStorage(StoreFactory storageFactory, String pathName, Validator validator) {
+	public NamespaceStorage(StoreFactory storageFactory, String pathName) {
 		super(storageFactory, pathName);
 	}
 
diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java
index a4a4c07d54..a9a7378760 100644
--- a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java
+++ b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java
@@ -19,6 +19,7 @@
 import com.bakdata.conquery.models.identifiable.ids.specific.ImportId;
 import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId;
 import com.bakdata.conquery.models.identifiable.ids.specific.TableId;
+import com.bakdata.conquery.models.worker.SingletonNamespaceCollection;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
 import lombok.Getter;
@@ -56,7 +57,8 @@ public NamespacedStorage(StoreFactory storageFactory, String pathName) {
 	}
 
 	public void openStores(ObjectMapper objectMapper) {
-
+		// Before we start to parse the stores we need to replace the injected value for the IdResolveContext (from DatasetRegistry to this centralRegistry)
+		new SingletonNamespaceCollection(centralRegistry).injectInto(objectMapper);
 
 		dataset = storageFactory.createDatasetStore(pathName, objectMapper);
 		secondaryIds = storageFactory.createSecondaryIdDescriptionStore(centralRegistry, pathName, objectMapper);
diff --git a/backend/src/main/java/com/bakdata/conquery/mode/DelegateManager.java b/backend/src/main/java/com/bakdata/conquery/mode/DelegateManager.java
index f4a402adce..53ada11ccc 100644
--- a/backend/src/main/java/com/bakdata/conquery/mode/DelegateManager.java
+++ b/backend/src/main/java/com/bakdata/conquery/mode/DelegateManager.java
@@ -24,6 +24,7 @@ public class DelegateManager<N extends Namespace> implements Manager {
 	ConqueryConfig config;
 	Environment environment;
 	DatasetRegistry<N> datasetRegistry;
+	MetaStorage storage;
 	ImportHandler importHandler;
 	StorageListener storageListener;
 	Supplier<Collection<ShardNodeInformation>> nodeProvider;
@@ -42,7 +43,7 @@ public void stop() throws Exception {
 	}
 
 	@Override
-	public MetaStorage getStorage() {
-		return datasetRegistry.getMetaStorage();
+	public MetaStorage getMetaStorage() {
+		return storage;
 	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/mode/InternalObjectMapperCreator.java b/backend/src/main/java/com/bakdata/conquery/mode/InternalObjectMapperCreator.java
index 64d255dad8..53e1fcdf2f 100644
--- a/backend/src/main/java/com/bakdata/conquery/mode/InternalObjectMapperCreator.java
+++ b/backend/src/main/java/com/bakdata/conquery/mode/InternalObjectMapperCreator.java
@@ -27,9 +27,9 @@ public class InternalObjectMapperCreator {
 	private DatasetRegistry<? extends Namespace> datasetRegistry = null;
 	private MetaStorage storage = null;
 
-	public void init(DatasetRegistry<? extends Namespace> datasetRegistry) {
+	public void init(DatasetRegistry<? extends Namespace> datasetRegistry, MetaStorage storage) {
 		this.datasetRegistry = datasetRegistry;
-		this.storage = datasetRegistry.getMetaStorage();
+		this.storage = storage;
 	}
 
 	public ObjectMapper createInternalObjectMapper(@Nullable Class<? extends View> viewClass) {
diff --git a/backend/src/main/java/com/bakdata/conquery/mode/Manager.java b/backend/src/main/java/com/bakdata/conquery/mode/Manager.java
index bef1a3c444..f8de4d3035 100644
--- a/backend/src/main/java/com/bakdata/conquery/mode/Manager.java
+++ b/backend/src/main/java/com/bakdata/conquery/mode/Manager.java
@@ -10,9 +10,9 @@
 import com.bakdata.conquery.models.worker.DatasetRegistry;
 import com.bakdata.conquery.models.worker.Namespace;
 import com.bakdata.conquery.models.worker.ShardNodeInformation;
+import io.dropwizard.core.setup.Environment;
 import io.dropwizard.lifecycle.Managed;
 import io.dropwizard.servlets.tasks.Task;
-import io.dropwizard.core.setup.Environment;
 
 /**
  * A manager provides the implementations that differ by running mode.
@@ -27,5 +27,6 @@ public interface Manager extends Managed {
 	List<Task> getAdminTasks();
 	InternalObjectMapperCreator getInternalObjectMapperCreator();
 	JobManager getJobManager();
-	MetaStorage getStorage();
+
+	MetaStorage getMetaStorage();
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java
index c25f63a08e..706520ca08 100644
--- a/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java
+++ b/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java
@@ -2,7 +2,6 @@
 
 import jakarta.validation.Validator;
 
-import com.bakdata.conquery.io.storage.MetaStorage;
 import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.index.IndexService;
 import com.bakdata.conquery.models.jobs.JobManager;
@@ -40,8 +39,6 @@ static <N extends Namespace> DatasetRegistry<N> createDatasetRegistry(
 				namespaceHandler,
 				indexService
 		);
-		MetaStorage storage = new MetaStorage(config.getStorage(), datasetRegistry);
-		datasetRegistry.setMetaStorage(storage);
 		return datasetRegistry;
 	}
 
diff --git a/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java
index 8f3e4eb1e4..fef5334580 100644
--- a/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java
+++ b/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java
@@ -4,6 +4,7 @@
 import java.util.List;
 
 import com.bakdata.conquery.io.jackson.Injectable;
+import com.bakdata.conquery.io.jackson.Jackson;
 import com.bakdata.conquery.io.jackson.View;
 import com.bakdata.conquery.io.storage.MetaStorage;
 import com.bakdata.conquery.io.storage.NamespaceStorage;
@@ -32,6 +33,7 @@ public interface NamespaceHandler<N extends Namespace> {
 	static NamespaceSetupData createNamespaceSetup(NamespaceStorage storage, final ConqueryConfig config, final InternalObjectMapperCreator mapperCreator, IndexService indexService) {
 		List<Injectable> injectables = new ArrayList<>();
 		injectables.add(indexService);
+
 		ObjectMapper persistenceMapper = mapperCreator.createInternalObjectMapper(View.Persistence.Manager.class);
 		ObjectMapper communicationMapper = mapperCreator.createInternalObjectMapper(View.InternalCommunication.class);
 		ObjectMapper preprocessMapper = mapperCreator.createInternalObjectMapper(null);
@@ -40,8 +42,9 @@ static NamespaceSetupData createNamespaceSetup(NamespaceStorage storage, final C
 		injectables.forEach(i -> i.injectInto(communicationMapper));
 		injectables.forEach(i -> i.injectInto(preprocessMapper));
 
-		// Open and load the stores
-		storage.openStores(persistenceMapper);
+
+		// Each store needs its own mapper because each injects its own registry
+		storage.openStores(Jackson.copyMapperAndInjectables(persistenceMapper));
 		storage.loadData();
 
 		JobManager jobManager = new JobManager(storage.getDataset().getName(), config.isFailOnError());
diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java
index 1579acf868..0ea1ca24b3 100644
--- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java
+++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java
@@ -4,6 +4,7 @@
 import java.util.List;
 import java.util.function.Supplier;
 
+import com.bakdata.conquery.io.storage.MetaStorage;
 import com.bakdata.conquery.mode.*;
 import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.jobs.JobManager;
@@ -23,7 +24,8 @@ public ClusterManager provideManager(ConqueryConfig config, Environment environm
 		final ClusterState clusterState = new ClusterState();
 		final NamespaceHandler<DistributedNamespace> namespaceHandler = new ClusterNamespaceHandler(clusterState, config, creator);
 		final DatasetRegistry<DistributedNamespace> datasetRegistry = ManagerProvider.createDatasetRegistry(namespaceHandler, config, creator);
-		creator.init(datasetRegistry);
+		final MetaStorage storage = new MetaStorage(config.getStorage());
+		creator.init(datasetRegistry, storage);
 
 		final ClusterConnectionManager connectionManager =
 				new ClusterConnectionManager(datasetRegistry, jobManager, environment.getValidator(), config, creator, clusterState);
@@ -35,7 +37,7 @@ public ClusterManager provideManager(ConqueryConfig config, Environment environm
 
 		final DelegateManager<DistributedNamespace>
 				delegate =
-				new DelegateManager<>(config, environment, datasetRegistry, importHandler, extension, nodeProvider, adminTasks, creator, jobManager);
+				new DelegateManager<>(config, environment, datasetRegistry, storage, importHandler, extension, nodeProvider, adminTasks, creator, jobManager);
 
 		environment.healthChecks().register("cluster", new ClusterHealthCheck(clusterState));
 
diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java
index 762a754eb8..87bfb58b2c 100644
--- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java
+++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java
@@ -5,6 +5,7 @@
 import java.util.List;
 import java.util.function.Supplier;
 
+import com.bakdata.conquery.io.storage.MetaStorage;
 import com.bakdata.conquery.mode.DelegateManager;
 import com.bakdata.conquery.mode.InternalObjectMapperCreator;
 import com.bakdata.conquery.mode.ManagerProvider;
@@ -32,15 +33,18 @@ public LocalManagerProvider(SqlDialectFactory dialectFactory) {
 
 	public DelegateManager<LocalNamespace> provideManager(ConqueryConfig config, Environment environment) {
 
-		InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, environment.getValidator());
-		NamespaceHandler<LocalNamespace> namespaceHandler = new LocalNamespaceHandler(config, creator, dialectFactory);
-		DatasetRegistry<LocalNamespace> datasetRegistry = ManagerProvider.createDatasetRegistry(namespaceHandler, config, creator);
-		creator.init(datasetRegistry);
+		final InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, environment.getValidator());
+		final NamespaceHandler<LocalNamespace> namespaceHandler = new LocalNamespaceHandler(config, creator, dialectFactory);
+		final DatasetRegistry<LocalNamespace> datasetRegistry = ManagerProvider.createDatasetRegistry(namespaceHandler, config, creator);
+		final MetaStorage storage = new MetaStorage(config.getStorage());
+
+		creator.init(datasetRegistry, storage);
 
 		return new DelegateManager<>(
 				config,
 				environment,
 				datasetRegistry,
+				storage,
 				new FailingImportHandler(),
 				new LocalStorageListener(),
 				EMPTY_NODE_PROVIDER,
diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java
index f79907fa0a..78ec8aecea 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java
@@ -12,11 +12,7 @@
 import com.bakdata.conquery.models.auth.entities.Group;
 import com.bakdata.conquery.models.auth.entities.Role;
 import com.bakdata.conquery.models.auth.entities.User;
-import com.bakdata.conquery.models.datasets.Dataset;
-import com.bakdata.conquery.models.datasets.Import;
-import com.bakdata.conquery.models.datasets.PreviewConfig;
-import com.bakdata.conquery.models.datasets.SecondaryIdDescription;
-import com.bakdata.conquery.models.datasets.Table;
+import com.bakdata.conquery.models.datasets.*;
 import com.bakdata.conquery.models.datasets.concepts.Concept;
 import com.bakdata.conquery.models.datasets.concepts.StructureNode;
 import com.bakdata.conquery.models.events.Bucket;
@@ -27,7 +23,6 @@
 import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap;
 import com.bakdata.conquery.models.index.InternToExternMapper;
 import com.bakdata.conquery.models.index.search.SearchIndex;
-import com.bakdata.conquery.models.worker.DatasetRegistry;
 import com.bakdata.conquery.models.worker.WorkerInformation;
 import com.bakdata.conquery.models.worker.WorkerToBucketsMap;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
@@ -67,9 +62,9 @@ public interface StoreFactory {
 	SingletonStore<StructureNode[]> createStructureStore(String pathName, CentralRegistry centralRegistry, ObjectMapper objectMapper);
 
 	// MetaStorage
-	IdentifiableStore<ManagedExecution> createExecutionsStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName, ObjectMapper objectMapper);
+	IdentifiableStore<ManagedExecution> createExecutionsStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper);
 
-	IdentifiableStore<FormConfig> createFormConfigStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName, ObjectMapper objectMapper);
+	IdentifiableStore<FormConfig> createFormConfigStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper);
 
 	IdentifiableStore<User> createUserStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage, ObjectMapper objectMapper);
 
diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java
index ae74ba9f23..7529e92934 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java
@@ -52,7 +52,6 @@
 import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap;
 import com.bakdata.conquery.models.index.InternToExternMapper;
 import com.bakdata.conquery.models.index.search.SearchIndex;
-import com.bakdata.conquery.models.worker.DatasetRegistry;
 import com.bakdata.conquery.models.worker.WorkerInformation;
 import com.bakdata.conquery.models.worker.WorkerToBucketsMap;
 import com.bakdata.conquery.util.io.ConqueryMDC;
@@ -191,7 +190,7 @@ public ExecutorService getReaderExecutorService() {
 
 	@Override
 	public Collection<NamespaceStorage> discoverNamespaceStorages() {
-		return loadNamespacedStores("dataset_", (storePath) -> new NamespaceStorage(this, storePath, getValidator()), NAMESPACE_STORES);
+		return loadNamespacedStores("dataset_", (storePath) -> new NamespaceStorage(this, storePath), NAMESPACE_STORES);
 	}
 
 	@Override
@@ -258,22 +257,22 @@ public SingletonStore<Dataset> createDatasetStore(String pathName, ObjectMapper
 
 	@Override
 	public IdentifiableStore<SecondaryIdDescription> createSecondaryIdDescriptionStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) {
-		return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, SECONDARY_IDS, centralRegistry.injectIntoNew(objectMapper)), centralRegistry);
+		return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, SECONDARY_IDS, objectMapper), centralRegistry);
 	}
 
 	@Override
 	public IdentifiableStore<InternToExternMapper> createInternToExternMappingStore(String pathName, CentralRegistry centralRegistry, ObjectMapper objectMapper) {
-		return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, INTERN_TO_EXTERN, centralRegistry.injectIntoNew(objectMapper)), centralRegistry);
+		return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, INTERN_TO_EXTERN, objectMapper), centralRegistry);
 	}
 
 	@Override
 	public IdentifiableStore<SearchIndex> createSearchIndexStore(String pathName, CentralRegistry centralRegistry, ObjectMapper objectMapper) {
-		return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, SEARCH_INDEX, centralRegistry.injectIntoNew(objectMapper)), centralRegistry);
+		return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, SEARCH_INDEX, objectMapper), centralRegistry);
 	}
 
 	@Override
 	public SingletonStore<PreviewConfig> createPreviewStore(String pathName, CentralRegistry centralRegistry, ObjectMapper objectMapper) {
-		return StoreMappings.singleton(createStore(findEnvironment(pathName), validator, ENTITY_PREVIEW, centralRegistry.injectIntoNew(objectMapper)));
+		return StoreMappings.singleton(createStore(findEnvironment(pathName), validator, ENTITY_PREVIEW, objectMapper));
 	}
 
 	@Override
@@ -283,27 +282,27 @@ public CachedStore<String, Integer> createEntity2BucketStore(String pathName, Ob
 
 	@Override
 	public IdentifiableStore<Table> createTableStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) {
-		return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, TABLES, centralRegistry.injectIntoNew(objectMapper)), centralRegistry);
+		return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, TABLES, objectMapper), centralRegistry);
 	}
 
 	@Override
 	public IdentifiableStore<Concept<?>> createConceptStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) {
-		return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, CONCEPTS, centralRegistry.injectIntoNew(objectMapper)), centralRegistry);
+		return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, CONCEPTS, objectMapper), centralRegistry);
 	}
 
 	@Override
 	public IdentifiableStore<Import> createImportStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) {
-		return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, IMPORTS, centralRegistry.injectIntoNew(objectMapper)), centralRegistry);
+		return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, IMPORTS, objectMapper), centralRegistry);
 	}
 
 	@Override
 	public IdentifiableStore<CBlock> createCBlockStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) {
-		return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, C_BLOCKS, centralRegistry.injectIntoNew(objectMapper)), centralRegistry);
+		return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, C_BLOCKS, objectMapper), centralRegistry);
 	}
 
 	@Override
 	public IdentifiableStore<Bucket> createBucketStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) {
-		return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, BUCKETS, centralRegistry.injectIntoNew(objectMapper)), centralRegistry);
+		return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, BUCKETS, objectMapper), centralRegistry);
 	}
 
 	@Override
@@ -332,17 +331,17 @@ public SingletonStore<WorkerToBucketsMap> createWorkerToBucketsStore(String path
 
 	@Override
 	public SingletonStore<StructureNode[]> createStructureStore(String pathName, CentralRegistry centralRegistry, ObjectMapper objectMapper) {
-		return StoreMappings.singleton(createStore(findEnvironment(pathName), validator, STRUCTURE, centralRegistry.injectIntoNew(objectMapper)));
+		return StoreMappings.singleton(createStore(findEnvironment(pathName), validator, STRUCTURE, objectMapper));
 	}
 
 	@Override
-	public IdentifiableStore<ManagedExecution> createExecutionsStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName, ObjectMapper objectMapper) {
-		return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "executions")), validator, EXECUTIONS, datasetRegistry.injectInto(centralRegistry.injectIntoNew(objectMapper))), centralRegistry);
+	public IdentifiableStore<ManagedExecution> createExecutionsStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) {
+		return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "executions")), validator, EXECUTIONS, objectMapper), centralRegistry);
 	}
 
 	@Override
-	public IdentifiableStore<FormConfig> createFormConfigStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName, ObjectMapper objectMapper) {
-		return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "formConfigs")), validator, FORM_CONFIG, datasetRegistry.injectInto(centralRegistry.injectIntoNew(objectMapper))), centralRegistry);
+	public IdentifiableStore<FormConfig> createFormConfigStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) {
+		return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "formConfigs")), validator, FORM_CONFIG, objectMapper), centralRegistry);
 	}
 
 	@Override
diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/CentralRegistry.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/CentralRegistry.java
index f03b7a69b4..ae74f0de63 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/CentralRegistry.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/CentralRegistry.java
@@ -5,12 +5,8 @@
 import java.util.concurrent.ConcurrentMap;
 import java.util.function.Function;
 
-import com.bakdata.conquery.io.jackson.Injectable;
-import com.bakdata.conquery.io.jackson.MutableInjectableValues;
 import com.bakdata.conquery.models.error.ConqueryError.ExecutionCreationResolveError;
 import com.bakdata.conquery.models.identifiable.ids.Id;
-import com.bakdata.conquery.models.worker.IdResolveContext;
-import com.bakdata.conquery.models.worker.SingletonNamespaceCollection;
 import com.fasterxml.jackson.databind.DeserializationContext;
 import com.fasterxml.jackson.databind.JsonMappingException;
 import lombok.NoArgsConstructor;
@@ -22,7 +18,7 @@
 @SuppressWarnings({"rawtypes", "unchecked"})
 @NoArgsConstructor
 @ToString(of = "map")
-public class CentralRegistry implements Injectable {
+public class CentralRegistry {
 
 	private final IdMap map = new IdMap<>();
 	private final ConcurrentMap<Id<?>, Function<Id, Identifiable>> cacheables = new ConcurrentHashMap<>();
@@ -70,24 +66,8 @@ public synchronized void remove(Identifiable<?> ident) {
 		map.remove(id);
 	}
 
-	@Override
-	public MutableInjectableValues inject(MutableInjectableValues values) {
-		return values.add(CentralRegistry.class, this)
-					 // Possibly overriding mapping for DatasetRegistry
-					 .add(IdResolveContext.class, new SingletonNamespaceCollection(this));
-	}
-
 	public static CentralRegistry get(DeserializationContext ctxt) throws JsonMappingException {
-		CentralRegistry result = (CentralRegistry) ctxt.findInjectableValue(CentralRegistry.class.getName(), null, null);
-		if (result != null) {
-			return result;
-		}
-
-		IdResolveContext alternative = (IdResolveContext) ctxt.findInjectableValue(IdResolveContext.class.getName(), null, null);
-		if (alternative == null) {
-			return null;
-		}
-		return alternative.getMetaRegistry();
+		return (CentralRegistry) ctxt.findInjectableValue(CentralRegistry.class.getName(), null, null);
 	}
 
 	public void clear() {
diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/NamespacedId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/NamespacedId.java
index 936a4c313f..d19838ffc8 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/NamespacedId.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/NamespacedId.java
@@ -1,12 +1,13 @@
 package com.bakdata.conquery.models.identifiable.ids;
 
 import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId;
-import com.bakdata.conquery.models.worker.IdResolveContext;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import org.apache.commons.lang3.StringUtils;
 
 /**
- * Marker interface for {@link Id}s that are loaded via Namespaced CentralRegistry (see {@link com.bakdata.conquery.models.worker.IdResolveContext#findRegistry(DatasetId)}, as opposed to MetaRegistry (see {@link IdResolveContext#getMetaRegistry()}).
+ * Marker interface for {@link Id}s that are loaded via Namespaced CentralRegistry
+ * (see {@link com.bakdata.conquery.models.worker.IdResolveContext#findRegistry(DatasetId)},
+ * as opposed to Registry in the {@link com.bakdata.conquery.io.storage.MetaStorage}
  */
 public interface NamespacedId {
 
diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java b/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java
index afc490de52..8368f20a1a 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java
@@ -10,6 +10,7 @@
 import java.util.concurrent.ConcurrentMap;
 import java.util.stream.Collectors;
 
+import com.bakdata.conquery.io.jackson.Jackson;
 import com.bakdata.conquery.io.jackson.MutableInjectableValues;
 import com.bakdata.conquery.io.jackson.View;
 import com.bakdata.conquery.io.storage.MetaStorage;
@@ -30,7 +31,6 @@
 import jakarta.validation.Validator;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
-import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -47,30 +47,27 @@ public class DatasetRegistry<N extends Namespace> extends IdResolveContext imple
 
 	private final InternalObjectMapperCreator internalObjectMapperCreator;
 
-	@Getter
-	@Setter
-	private MetaStorage metaStorage;
-
 	private final NamespaceHandler<N> namespaceHandler;
 
 	private final IndexService indexService;
 
-	public N createNamespace(Dataset dataset, Validator validator) throws IOException {
+	public N createNamespace(Dataset dataset, MetaStorage metaStorage) throws IOException {
 		// Prepare empty storage
-		NamespaceStorage datasetStorage = new NamespaceStorage(config.getStorage(), "dataset_" + dataset.getName(), validator);
+		NamespaceStorage datasetStorage = new NamespaceStorage(config.getStorage(), "dataset_" + dataset.getName());
 		final ObjectMapper persistenceMapper = internalObjectMapperCreator.createInternalObjectMapper(View.Persistence.Manager.class);
 
-		datasetStorage.openStores(persistenceMapper);
+		// Each store injects its own IdResolveCtx so each needs its own mapper
+		datasetStorage.openStores(Jackson.copyMapperAndInjectables((persistenceMapper)));
 		datasetStorage.loadData();
 		datasetStorage.updateDataset(dataset);
 		datasetStorage.updateIdMapping(new EntityIdMap());
 		datasetStorage.setPreviewConfig(new PreviewConfig());
 		datasetStorage.close();
 
-		return createNamespace(datasetStorage);
+		return createNamespace(datasetStorage, metaStorage);
 	}
 
-	public N createNamespace(NamespaceStorage datasetStorage) {
+	public N createNamespace(NamespaceStorage datasetStorage, MetaStorage metaStorage) {
 		final N namespace = namespaceHandler.createNamespace(datasetStorage, metaStorage, indexService);
 		add(namespace);
 		return namespace;
@@ -88,7 +85,6 @@ public void removeNamespace(DatasetId id) {
 		N removed = datasets.remove(id);
 
 		if (removed != null) {
-			metaStorage.getCentralRegistry().remove(removed.getDataset());
 			namespaceHandler.removeNamespace(id, removed);
 			removed.remove();
 		}
@@ -103,12 +99,6 @@ public CentralRegistry findRegistry(DatasetId dataset) throws NoSuchElementExcep
 		return datasets.get(dataset).getStorage().getCentralRegistry();
 	}
 
-	@Override
-	public CentralRegistry getMetaRegistry() {
-		return metaStorage.getCentralRegistry();
-	}
-
-
 	public List<Dataset> getAllDatasets() {
 		return datasets.values().stream().map(Namespace::getStorage).map(NamespaceStorage::getDataset).collect(Collectors.toList());
 	}
diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/IdResolveContext.java b/backend/src/main/java/com/bakdata/conquery/models/worker/IdResolveContext.java
index 04fec43874..d65f7850c0 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/worker/IdResolveContext.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/worker/IdResolveContext.java
@@ -10,7 +10,6 @@
 import com.bakdata.conquery.models.identifiable.ids.Id;
 import com.bakdata.conquery.models.identifiable.ids.NamespacedId;
 import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId;
-import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.databind.DeserializationContext;
 import com.fasterxml.jackson.databind.JsonMappingException;
 import io.dropwizard.jackson.Jackson;
@@ -36,8 +35,6 @@ public MutableInjectableValues inject(MutableInjectableValues values) {
 	}
 
 	public abstract CentralRegistry findRegistry(DatasetId dataset) throws NoSuchElementException;
-	@JsonIgnore
-	public abstract CentralRegistry getMetaRegistry();
 
 	public <ID extends Id<T> & NamespacedId, T extends Identifiable<?>> T resolve(ID id) {
 		return findRegistry(id.getDataset()).resolve(id);
diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java
index 191265b8b2..1c952d0de7 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java
@@ -122,12 +122,6 @@ public CentralRegistry findRegistry(DatasetId dataset) throws NoSuchElementExcep
 		return storage.getCentralRegistry();
 	}
 
-	@Override
-	public CentralRegistry getMetaRegistry() {
-		throw new UnsupportedOperationException();
-	}
-
-
 	/**
 	 * Issues a job that initializes the search that is used by the frontend for recommendations in the filter interface of a concept.
 	 */
diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/SingletonNamespaceCollection.java b/backend/src/main/java/com/bakdata/conquery/models/worker/SingletonNamespaceCollection.java
index 78bc0c5b52..4f1d3799b3 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/worker/SingletonNamespaceCollection.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/worker/SingletonNamespaceCollection.java
@@ -8,21 +8,11 @@
 @RequiredArgsConstructor
 public class SingletonNamespaceCollection extends IdResolveContext {
 
-	public SingletonNamespaceCollection(CentralRegistry registry) {
-		this(registry, null);
-	}
-
 	@NonNull
 	private final CentralRegistry registry;
-	private final CentralRegistry metaRegistry;
 
 	@Override
 	public CentralRegistry findRegistry(DatasetId dataset) {
 		return registry;
 	}
-
-	@Override
-	public CentralRegistry getMetaRegistry() {
-		return metaRegistry;
-	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java
index 853830d3de..096554e5c5 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java
@@ -125,11 +125,6 @@ public MessageToManagerNode transform(NamespaceMessage message) {
 		return new ForwardToNamespace(getInfo().getDataset(), message);
 	}
 
-	public ObjectMapper inject(ObjectMapper binaryMapper) {
-		return new SingletonNamespaceCollection(storage.getCentralRegistry())
-				.injectIntoNew(binaryMapper);
-	}
-
 	@Override
 	public void close() {
 		// We do not close the executorService here because it does not belong to this class
diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java
index 99ddb7e633..395551aa64 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java
@@ -119,11 +119,6 @@ public CentralRegistry findRegistry(DatasetId dataset) {
 		return dataset2Worker.get(dataset).getStorage().getCentralRegistry();
 	}
 
-	@Override
-	public CentralRegistry getMetaRegistry() {
-		return null; // Workers simply have no MetaRegistry.
-	}
-
 	public void removeWorkerFor(DatasetId dataset) {
 		final Worker worker = dataset2Worker.get(dataset);
 
diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java
index 40a03b87c4..a41e6e8600 100644
--- a/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java
+++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java
@@ -4,6 +4,7 @@
 
 import java.nio.charset.StandardCharsets;
 import java.util.Collections;
+import jakarta.validation.Validator;
 
 import com.bakdata.conquery.commands.ManagerNode;
 import com.bakdata.conquery.io.freemarker.Freemarker;
@@ -18,35 +19,14 @@
 import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.jobs.JobManager;
 import com.bakdata.conquery.models.worker.DatasetRegistry;
-import com.bakdata.conquery.resources.admin.rest.AdminConceptsResource;
-import com.bakdata.conquery.resources.admin.rest.AdminDatasetProcessor;
-import com.bakdata.conquery.resources.admin.rest.AdminDatasetResource;
-import com.bakdata.conquery.resources.admin.rest.AdminDatasetsResource;
-import com.bakdata.conquery.resources.admin.rest.AdminProcessor;
-import com.bakdata.conquery.resources.admin.rest.AdminResource;
-import com.bakdata.conquery.resources.admin.rest.AdminTablesResource;
-import com.bakdata.conquery.resources.admin.rest.AuthOverviewResource;
-import com.bakdata.conquery.resources.admin.rest.GroupResource;
-import com.bakdata.conquery.resources.admin.rest.PermissionResource;
-import com.bakdata.conquery.resources.admin.rest.RoleResource;
-import com.bakdata.conquery.resources.admin.rest.UIProcessor;
-import com.bakdata.conquery.resources.admin.rest.UserResource;
-import com.bakdata.conquery.resources.admin.ui.AdminUIResource;
-import com.bakdata.conquery.resources.admin.ui.AuthOverviewUIResource;
-import com.bakdata.conquery.resources.admin.ui.ConceptsUIResource;
-import com.bakdata.conquery.resources.admin.ui.DatasetsUIResource;
-import com.bakdata.conquery.resources.admin.ui.GroupUIResource;
-import com.bakdata.conquery.resources.admin.ui.IndexServiceUIResource;
-import com.bakdata.conquery.resources.admin.ui.RoleUIResource;
-import com.bakdata.conquery.resources.admin.ui.TablesUIResource;
-import com.bakdata.conquery.resources.admin.ui.UserUIResource;
+import com.bakdata.conquery.resources.admin.rest.*;
+import com.bakdata.conquery.resources.admin.ui.*;
 import com.bakdata.conquery.resources.admin.ui.model.ConnectorUIResource;
 import io.dropwizard.core.setup.AdminEnvironment;
 import io.dropwizard.jersey.DropwizardResourceConfig;
 import io.dropwizard.jersey.jackson.JacksonMessageBodyProvider;
 import io.dropwizard.servlets.assets.AssetServlet;
 import io.dropwizard.views.common.ViewMessageBodyWriter;
-import jakarta.validation.Validator;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.glassfish.hk2.utilities.binding.AbstractBinder;
@@ -89,7 +69,7 @@ public AdminServlet(ManagerNode manager) {
 		adminProcessor = new AdminProcessor(
 				manager,
 				manager.getConfig(),
-				manager.getStorage(),
+				manager.getMetaStorage(),
 				manager.getDatasetRegistry(),
 				manager.getJobManager(),
 				manager.getMaintenanceService(),
@@ -101,6 +81,7 @@ public AdminServlet(ManagerNode manager) {
 				manager.getConfig(),
 				manager.getValidator(),
 				manager.getDatasetRegistry(),
+				manager.getMetaStorage(),
 				manager.getJobManager(),
 				manager.getImportHandler(),
 				manager.getStorageListener()
@@ -110,7 +91,7 @@ public AdminServlet(ManagerNode manager) {
 						@Override
 						protected void configure() {
 							bind(manager.getDatasetRegistry()).to(DatasetRegistry.class);
-							bind(manager.getStorage()).to(MetaStorage.class);
+							bind(manager.getMetaStorage()).to(MetaStorage.class);
 							bind(manager.getValidator()).to(Validator.class);
 							bind(manager.getJobManager()).to(JobManager.class);
 							bind(manager.getConfig()).to(ConqueryConfig.class);
@@ -134,7 +115,7 @@ protected void configure() {
 							  bind(adminProcessor).to(AdminProcessor.class);
 							  bindAsContract(UIProcessor.class);
 							  bind(manager.getDatasetRegistry()).to(DatasetRegistry.class);
-							  bind(manager.getStorage()).to(MetaStorage.class);
+							  bind(manager.getMetaStorage()).to(MetaStorage.class);
 							  bind(manager.getConfig()).to(ConqueryConfig.class);
 						  }
 					  })
diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java
index a8adad0a1e..8248842abc 100644
--- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java
+++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java
@@ -9,6 +9,7 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import com.bakdata.conquery.io.storage.MetaStorage;
 import com.bakdata.conquery.mode.ImportHandler;
 import com.bakdata.conquery.mode.StorageListener;
 import com.bakdata.conquery.models.config.ConqueryConfig;
@@ -59,6 +60,7 @@ public class AdminDatasetProcessor {
 	private final ConqueryConfig config;
 	private final Validator validator;
 	private final DatasetRegistry<? extends Namespace> datasetRegistry;
+	private final MetaStorage metaStorage;
 	private final JobManager jobManager;
 	private final ImportHandler importHandler;
 	private final StorageListener storageListener;
@@ -74,7 +76,7 @@ public synchronized Dataset addDataset(Dataset dataset) throws IOException {
 			throw new WebApplicationException("Dataset already exists", Response.Status.CONFLICT);
 		}
 
-		return datasetRegistry.createNamespace(dataset, getValidator()).getDataset();
+		return datasetRegistry.createNamespace(dataset, metaStorage).getDataset();
 	}
 
 	/**
diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java b/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java
index 07e8aea28a..e96c80c276 100644
--- a/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java
+++ b/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java
@@ -4,15 +4,12 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Consumer;
-
 import javax.annotation.Nullable;
 
 import com.bakdata.conquery.integration.IntegrationTest;
 import com.bakdata.conquery.io.cps.CPSBase;
 import com.bakdata.conquery.io.jackson.Jackson;
-import com.bakdata.conquery.io.jackson.MutableInjectableValues;
 import com.bakdata.conquery.io.jackson.View;
-import com.bakdata.conquery.io.storage.MetaStorage;
 import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.config.Dialect;
 import com.bakdata.conquery.models.config.IdColumnConfig;
@@ -21,7 +18,6 @@
 import com.bakdata.conquery.models.identifiable.Identifiable;
 import com.bakdata.conquery.models.identifiable.ids.Id;
 import com.bakdata.conquery.models.identifiable.ids.IdUtil;
-import com.bakdata.conquery.models.worker.SingletonNamespaceCollection;
 import com.bakdata.conquery.util.NonPersistentStoreFactory;
 import com.bakdata.conquery.util.support.StandaloneSupport;
 import com.bakdata.conquery.util.support.TestSupport;
@@ -94,16 +90,12 @@ public static <T> T parseSubTree(TestSupport support, JsonNode node, JavaType ex
 	}
 
 	public static <T> T parseSubTree(TestSupport support, JsonNode node, JavaType expectedType, Consumer<T> modifierBeforeValidation) throws IOException {
-		final ObjectMapper om = Jackson.MAPPER.copy();
-		ObjectMapper mapper = support.getDataset().injectIntoNew(
-				new SingletonNamespaceCollection(support.getNamespace().getStorage().getCentralRegistry(), support.getMetaStorage().getCentralRegistry())
-						.injectIntoNew(
-								om.addHandler(new DatasetPlaceHolderFiller(support))
-						)
-		);
-		final MutableInjectableValues injectableValues = (MutableInjectableValues) mapper.getInjectableValues();
-		injectableValues.add(ConqueryConfig.class, support.getConfig());
-		injectableValues.add(MetaStorage.class, support.getMetaStorage());
+		final ObjectMapper mapper = Jackson.copyMapperAndInjectables(Jackson.MAPPER);
+		support.getDataset().injectInto(mapper);
+		support.getNamespace().injectInto(mapper);
+		support.getMetaStorage().injectInto(mapper);
+		support.getConfig().injectInto(mapper);
+		mapper.addHandler(new DatasetPlaceHolderFiller(support));
 
 		T result = mapper.readerFor(expectedType).readValue(node);
 
@@ -117,13 +109,12 @@ public static <T> T parseSubTree(TestSupport support, JsonNode node, JavaType ex
 
 	public static <T> List<T> parseSubTreeList(TestSupport support, ArrayNode node, Class<?> expectedType, Consumer<T> modifierBeforeValidation)
 			throws IOException {
-		final ObjectMapper om = Jackson.MAPPER.copy();
-		ObjectMapper mapper = support.getDataset().injectInto(
-				new SingletonNamespaceCollection(support.getNamespace().getStorage().getCentralRegistry()).injectIntoNew(
-						om.addHandler(new DatasetPlaceHolderFiller(support))
-				)
-		);
-		support.getNamespace().getInjectables().forEach(i -> i.injectInto(mapper));
+		final ObjectMapper mapper = Jackson.copyMapperAndInjectables(Jackson.MAPPER);
+		support.getDataset().injectInto(mapper);
+		support.getNamespace().injectInto(mapper);
+		support.getMetaStorage().injectInto(mapper);
+		support.getConfig().injectInto(mapper);
+		mapper.addHandler(new DatasetPlaceHolderFiller(support));
 
 		mapper.setConfig(mapper.getDeserializationConfig().withView(View.Api.class));
 
diff --git a/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java b/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java
index 5c88386bde..5cb97c45f2 100644
--- a/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java
+++ b/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java
@@ -3,6 +3,8 @@
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.*;
 
+import jakarta.validation.Validator;
+
 import com.bakdata.conquery.commands.ManagerNode;
 import com.bakdata.conquery.commands.ShardNode;
 import com.bakdata.conquery.io.jackson.Jackson;
@@ -18,7 +20,6 @@
 import com.bakdata.conquery.util.NonPersistentStoreFactory;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import io.dropwizard.jersey.validation.Validators;
-import jakarta.validation.Validator;
 import lombok.Getter;
 import org.junit.jupiter.api.BeforeEach;
 
@@ -41,16 +42,15 @@ public void before() {
 		final IndexService indexService = new IndexService(config.getCsv().createCsvParserSettings(), "emptyDefaultLabel");
 		final ClusterNamespaceHandler clusterNamespaceHandler = new ClusterNamespaceHandler(new ClusterState(), config, creator);
 		datasetRegistry = new DatasetRegistry<>(0, config, null, clusterNamespaceHandler, indexService);
-		metaStorage = new MetaStorage(new NonPersistentStoreFactory(), datasetRegistry);
-		datasetRegistry.setMetaStorage(metaStorage);
-		creator.init(datasetRegistry);
+		metaStorage = new MetaStorage(new NonPersistentStoreFactory());
+		creator.init(datasetRegistry, metaStorage);
 
 		// Prepare manager node internal mapper
 		final ManagerNode managerNode = mock(ManagerNode.class);
 		when(managerNode.getConfig()).thenReturn(config);
 		when(managerNode.getValidator()).thenReturn(validator);
 		doReturn(datasetRegistry).when(managerNode).getDatasetRegistry();
-		when(managerNode.getStorage()).thenReturn(metaStorage);
+		when(managerNode.getMetaStorage()).thenReturn(metaStorage);
 		when(managerNode.getInternalObjectMapperCreator()).thenReturn(creator);
 
 		when(managerNode.createInternalObjectMapper(any())).thenCallRealMethod();
diff --git a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java
index 69f77b30f9..21282175d9 100644
--- a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java
+++ b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java
@@ -7,28 +7,21 @@
 import java.util.List;
 
 import com.bakdata.conquery.io.jackson.Jackson;
-import com.bakdata.conquery.io.jackson.MutableInjectableValues;
 import com.bakdata.conquery.io.storage.MetaStorage;
 import com.bakdata.conquery.models.auth.entities.User;
 import com.bakdata.conquery.models.datasets.Dataset;
 import com.bakdata.conquery.models.datasets.Table;
 import com.bakdata.conquery.models.identifiable.CentralRegistry;
-import com.bakdata.conquery.models.worker.DatasetRegistry;
-import com.bakdata.conquery.models.worker.DistributedNamespace;
 import com.bakdata.conquery.models.worker.SingletonNamespaceCollection;
 import com.bakdata.conquery.util.NonPersistentStoreFactory;
-import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import lombok.Getter;
-import lombok.RequiredArgsConstructor;
 import org.junit.jupiter.api.Test;
 
 public class IdRefrenceTest {
 
 	@Test
 	public void testListReferences() throws IOException {
-		final ObjectMapper mapper = Jackson.MAPPER.copy();
-		mapper.setInjectableValues(new MutableInjectableValues());
+		final ObjectMapper mapper = Jackson.copyMapperAndInjectables(Jackson.MAPPER);
 
 		CentralRegistry registry = new CentralRegistry();
 		Dataset dataset = new Dataset();
@@ -39,12 +32,9 @@ public void testListReferences() throws IOException {
 		registry.register(dataset);
 		registry.register(table);
 
-		final DatasetRegistry<DistributedNamespace> datasetRegistry = new DatasetRegistry<>(0, null, null, null, null);
-
-		final MetaStorage metaStorage = new MetaStorage(new NonPersistentStoreFactory(),datasetRegistry);
+		final MetaStorage metaStorage = new MetaStorage(new NonPersistentStoreFactory());
 
 		metaStorage.openStores(null);
-		datasetRegistry.setMetaStorage(metaStorage);
 
 
 		User user = new User("usermail", "userlabel", metaStorage);
@@ -61,20 +51,17 @@ public void testListReferences() throws IOException {
 				.contains("\"user.usermail\"")
 				.contains("\"dataset.table\"");
 
-		ListHolder holder = new SingletonNamespaceCollection(registry, metaStorage.getCentralRegistry())
-				.injectIntoNew(mapper.readerFor(ListHolder.class))
+		new SingletonNamespaceCollection(registry)
+				.injectInto(mapper);
+		metaStorage.injectInto(mapper);
+		ListHolder holder = mapper
+				.readerFor(ListHolder.class)
 				.readValue(json);
 
-		assertThat(holder.getUsers().get(0)).isSameAs(user);
-		assertThat(holder.getTables().get(0)).isSameAs(table);
+		assertThat(holder.users().get(0)).isSameAs(user);
+		assertThat(holder.tables().get(0)).isSameAs(table);
 	}
 
-	@Getter
-	@RequiredArgsConstructor(onConstructor_ = @JsonCreator)
-	public static class ListHolder {
-		@NsIdRefCollection
-		private final List<Table> tables;
-		@MetaIdRefCollection
-		private final List<User> users;
+	public record ListHolder(@NsIdRefCollection List<Table> tables, @MetaIdRefCollection List<User> users) {
 	}
 }
diff --git a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java
index 10c92e1004..a6a5de9174 100644
--- a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java
+++ b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java
@@ -8,6 +8,7 @@
 import java.lang.ref.WeakReference;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.UnaryOperator;
+import jakarta.validation.Validator;
 
 import com.bakdata.conquery.io.jackson.Injectable;
 import com.bakdata.conquery.io.jackson.Jackson;
@@ -23,7 +24,6 @@
 import com.fasterxml.jackson.databind.ObjectReader;
 import com.fasterxml.jackson.databind.ObjectWriter;
 import io.dropwizard.jersey.validation.Validators;
-import jakarta.validation.Validator;
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;
 import lombok.Setter;
@@ -116,7 +116,7 @@ public void test(T value) throws JSONException, IOException {
 	private void test(T value, T expected, ObjectMapper mapper) throws IOException {
 
 		if (registry != null) {
-			mapper = new SingletonNamespaceCollection(registry, registry).injectInto(mapper);
+			mapper = new SingletonNamespaceCollection(registry).injectInto(mapper);
 		}
 		for (Injectable injectable : injectables) {
 			mapper = injectable.injectInto(mapper);
diff --git a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java
index d5549e0723..80fcdd27d5 100644
--- a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java
+++ b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java
@@ -8,12 +8,10 @@
 import java.util.Random;
 import java.util.function.Supplier;
 import java.util.stream.Stream;
-
 import jakarta.validation.Validator;
 
 import com.bakdata.conquery.io.jackson.Injectable;
 import com.bakdata.conquery.io.jackson.Jackson;
-import com.bakdata.conquery.io.jackson.MutableInjectableValues;
 import com.bakdata.conquery.models.datasets.Column;
 import com.bakdata.conquery.models.datasets.Dataset;
 import com.bakdata.conquery.models.datasets.Table;
@@ -22,7 +20,9 @@
 import com.bakdata.conquery.models.exceptions.ConfigurationException;
 import com.bakdata.conquery.models.exceptions.JSONException;
 import com.bakdata.conquery.models.identifiable.CentralRegistry;
+import com.bakdata.conquery.models.worker.SingletonNamespaceCollection;
 import com.bakdata.conquery.util.CalculatedValue;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectReader;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.github.powerlibraries.io.In;
@@ -87,24 +87,20 @@ public static void init() throws IOException, JSONException, ConfigurationExcept
 
 
 		// Prepare Serdes injections
-		final Validator validator = Validators.newValidator();
-		final ObjectReader conceptReader = new Injectable(){
-			@Override
-			public MutableInjectableValues inject(MutableInjectableValues values) {
-				return values.add(Validator.class, validator);
-			}
-		}.injectInto(registry.injectIntoNew(dataset.injectIntoNew(Jackson.MAPPER))).readerFor(Concept.class);
-
-		// load tree twice to to avoid references
+		ObjectMapper mapper = Jackson.copyMapperAndInjectables(Jackson.MAPPER);
+		((Injectable) values -> values.add(Validator.class, Validators.newValidator())).injectInto(mapper);
+		new SingletonNamespaceCollection(registry).injectInto(mapper);
+		dataset.injectInto(mapper);
+		final ObjectReader conceptReader = mapper.readerFor(Concept.class);
+
+		// load tree twice to avoid references
 		indexedConcept = conceptReader.readValue(node);
 
 		indexedConcept.setDataset(dataset);
-		indexedConcept.initElements();
 
 		oldConcept = conceptReader.readValue(node);
 
 		oldConcept.setDataset(dataset);
-		oldConcept.initElements();
 	}
 
 
diff --git a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java
index c9622d73e1..ae13a560f6 100644
--- a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java
+++ b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java
@@ -6,22 +6,14 @@
 import java.util.concurrent.ConcurrentHashMap;
 
 import com.bakdata.conquery.io.cps.CPSType;
-import com.bakdata.conquery.io.storage.IdentifiableStore;
-import com.bakdata.conquery.io.storage.MetaStorage;
-import com.bakdata.conquery.io.storage.NamespaceStorage;
-import com.bakdata.conquery.io.storage.StoreMappings;
-import com.bakdata.conquery.io.storage.WorkerStorage;
+import com.bakdata.conquery.io.storage.*;
 import com.bakdata.conquery.io.storage.xodus.stores.CachedStore;
 import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore;
 import com.bakdata.conquery.models.auth.entities.Group;
 import com.bakdata.conquery.models.auth.entities.Role;
 import com.bakdata.conquery.models.auth.entities.User;
 import com.bakdata.conquery.models.config.StoreFactory;
-import com.bakdata.conquery.models.datasets.Dataset;
-import com.bakdata.conquery.models.datasets.Import;
-import com.bakdata.conquery.models.datasets.PreviewConfig;
-import com.bakdata.conquery.models.datasets.SecondaryIdDescription;
-import com.bakdata.conquery.models.datasets.Table;
+import com.bakdata.conquery.models.datasets.*;
 import com.bakdata.conquery.models.datasets.concepts.Concept;
 import com.bakdata.conquery.models.datasets.concepts.StructureNode;
 import com.bakdata.conquery.models.events.Bucket;
@@ -33,7 +25,6 @@
 import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap;
 import com.bakdata.conquery.models.index.InternToExternMapper;
 import com.bakdata.conquery.models.index.search.SearchIndex;
-import com.bakdata.conquery.models.worker.DatasetRegistry;
 import com.bakdata.conquery.models.worker.WorkerInformation;
 import com.bakdata.conquery.models.worker.WorkerToBucketsMap;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -153,12 +144,12 @@ public SingletonStore<StructureNode[]> createStructureStore(String pathName, Cen
 	}
 
 	@Override
-	public IdentifiableStore<ManagedExecution> createExecutionsStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName, ObjectMapper objectMapper) {
+	public IdentifiableStore<ManagedExecution> createExecutionsStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) {
 		return StoreMappings.identifiable(executionStore.computeIfAbsent(pathName, n -> new NonPersistentStore<>()), centralRegistry);
 	}
 
 	@Override
-	public IdentifiableStore<FormConfig> createFormConfigStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName, ObjectMapper objectMapper) {
+	public IdentifiableStore<FormConfig> createFormConfigStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) {
 		return StoreMappings.identifiable(formConfigStore.computeIfAbsent(pathName, n -> new NonPersistentStore<>()), centralRegistry);
 	}
 
@@ -181,7 +172,7 @@ public IdentifiableStore<Group> createGroupStore(CentralRegistry centralRegistry
 	 * @implNote intended for Unit-tests
 	 */
 	public MetaStorage createMetaStorage() {
-		final MetaStorage metaStorage = new MetaStorage(this, null);
+		final MetaStorage metaStorage = new MetaStorage(this);
 		metaStorage.openStores(null);
 		return metaStorage;
 	}
diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java b/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java
index 1abc9b0fc9..49042dada3 100644
--- a/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java
+++ b/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java
@@ -4,6 +4,11 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import jakarta.validation.Validator;
+import jakarta.ws.rs.client.Client;
+import jakarta.ws.rs.client.ClientRequestContext;
+import jakarta.ws.rs.client.ClientRequestFilter;
+import jakarta.ws.rs.core.UriBuilder;
 
 import com.bakdata.conquery.commands.PreprocessorCommand;
 import com.bakdata.conquery.commands.ShardNode;
@@ -23,11 +28,6 @@
 import com.bakdata.conquery.resources.admin.rest.AdminProcessor;
 import com.google.common.util.concurrent.MoreExecutors;
 import io.dropwizard.core.setup.Environment;
-import jakarta.validation.Validator;
-import jakarta.ws.rs.client.Client;
-import jakarta.ws.rs.client.ClientRequestContext;
-import jakarta.ws.rs.client.ClientRequestFilter;
-import jakarta.ws.rs.core.UriBuilder;
 import lombok.Data;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
@@ -94,7 +94,7 @@ public Validator getValidator() {
 	}
 
 	public MetaStorage getMetaStorage() {
-		return testConquery.getStandaloneCommand().getManagerNode().getStorage();
+		return testConquery.getStandaloneCommand().getManagerNode().getMetaStorage();
 	}
 
 	public NamespaceStorage getNamespaceStorage() {
diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java
index 4346306bc3..00f9822cd3 100644
--- a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java
+++ b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java
@@ -11,6 +11,8 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import jakarta.validation.Validator;
+import jakarta.ws.rs.client.Client;
 
 import com.bakdata.conquery.Conquery;
 import com.bakdata.conquery.commands.DistributedStandaloneCommand;
@@ -36,8 +38,6 @@
 import io.dropwizard.client.JerseyClientBuilder;
 import io.dropwizard.core.cli.Command;
 import io.dropwizard.testing.DropwizardTestSupport;
-import jakarta.validation.Validator;
-import jakarta.ws.rs.client.Client;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -149,7 +149,7 @@ public void afterEach() throws Exception {
 			}
 			openSupports.clear();
 		}
-		this.getStandaloneCommand().getManagerNode().getStorage().clear();
+		this.getStandaloneCommand().getManagerNode().getMetaStorage().clear();
 		waitUntilWorkDone();
 	}
 
@@ -190,7 +190,7 @@ public void waitUntilWorkDone() {
 	}
 
 	public void beforeEach() {
-		final MetaStorage storage = standaloneCommand.getManagerNode().getStorage();
+		final MetaStorage storage = standaloneCommand.getManagerNode().getMetaStorage();
 		testUser = standaloneCommand.getManagerNode().getConfig().getAuthorizationRealms().getInitialUsers().get(0).createOrOverwriteUser(storage);
 		storage.updateUser(testUser);
 	}
@@ -261,7 +261,7 @@ private boolean isBusy() {
 		boolean busy;
 		busy = standaloneCommand.getManagerNode().getJobManager().isSlowWorkerBusy();
 		busy |= standaloneCommand.getManagerNode()
-								 .getStorage()
+				.getMetaStorage()
 								 .getAllExecutions()
 								 .stream()
 								 .map(ManagedExecution::getState)

From 56e5ea59b56c7a06d47328260967860e0b6db522 Mon Sep 17 00:00:00 2001
From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com>
Date: Wed, 24 Jul 2024 18:28:55 +0200
Subject: [PATCH 18/41] simplify InternalObjectMapperCreator#init by pushing
 metaStorage to constructor

---
 .../conquery/mode/InternalObjectMapperCreator.java        | 5 ++---
 .../java/com/bakdata/conquery/mode/ManagerProvider.java   | 8 ++++----
 .../conquery/mode/cluster/ClusterManagerProvider.java     | 6 +++---
 .../bakdata/conquery/mode/local/LocalManagerProvider.java | 6 +++---
 .../bakdata/conquery/io/AbstractSerializationTest.java    | 8 ++++----
 5 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/backend/src/main/java/com/bakdata/conquery/mode/InternalObjectMapperCreator.java b/backend/src/main/java/com/bakdata/conquery/mode/InternalObjectMapperCreator.java
index 53e1fcdf2f..8fdcf63076 100644
--- a/backend/src/main/java/com/bakdata/conquery/mode/InternalObjectMapperCreator.java
+++ b/backend/src/main/java/com/bakdata/conquery/mode/InternalObjectMapperCreator.java
@@ -23,13 +23,12 @@
 @RequiredArgsConstructor
 public class InternalObjectMapperCreator {
 	private final ConqueryConfig config;
+	private final MetaStorage storage;
 	private final Validator validator;
 	private DatasetRegistry<? extends Namespace> datasetRegistry = null;
-	private MetaStorage storage = null;
 
-	public void init(DatasetRegistry<? extends Namespace> datasetRegistry, MetaStorage storage) {
+	public void init(DatasetRegistry<? extends Namespace> datasetRegistry) {
 		this.datasetRegistry = datasetRegistry;
-		this.storage = storage;
 	}
 
 	public ObjectMapper createInternalObjectMapper(@Nullable Class<? extends View> viewClass) {
diff --git a/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java
index 706520ca08..fe45f4ecbe 100644
--- a/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java
+++ b/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java
@@ -2,6 +2,7 @@
 
 import jakarta.validation.Validator;
 
+import com.bakdata.conquery.io.storage.MetaStorage;
 import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.index.IndexService;
 import com.bakdata.conquery.models.jobs.JobManager;
@@ -22,8 +23,8 @@ static JobManager newJobManager(ConqueryConfig config) {
 		return new JobManager(JOB_MANAGER_NAME, config.isFailOnError());
 	}
 
-	static InternalObjectMapperCreator newInternalObjectMapperCreator(ConqueryConfig config, Validator validator) {
-		return new InternalObjectMapperCreator(config, validator);
+	static InternalObjectMapperCreator newInternalObjectMapperCreator(ConqueryConfig config, MetaStorage metaStorage, Validator validator) {
+		return new InternalObjectMapperCreator(config, metaStorage, validator);
 	}
 
 	static <N extends Namespace> DatasetRegistry<N> createDatasetRegistry(
@@ -32,14 +33,13 @@ static <N extends Namespace> DatasetRegistry<N> createDatasetRegistry(
 			InternalObjectMapperCreator creator
 	) {
 		final IndexService indexService = new IndexService(config.getCsv().createCsvParserSettings(), config.getIndex().getEmptyLabel());
-		DatasetRegistry<N> datasetRegistry = new DatasetRegistry<>(
+		return new DatasetRegistry<>(
 				config.getCluster().getEntityBucketSize(),
 				config,
 				creator,
 				namespaceHandler,
 				indexService
 		);
-		return datasetRegistry;
 	}
 
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java
index 0ea1ca24b3..cafc855713 100644
--- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java
+++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java
@@ -20,12 +20,12 @@ public class ClusterManagerProvider implements ManagerProvider {
 
 	public ClusterManager provideManager(ConqueryConfig config, Environment environment) {
 		final JobManager jobManager = ManagerProvider.newJobManager(config);
-		final InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, environment.getValidator());
+		final MetaStorage storage = new MetaStorage(config.getStorage());
+		final InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, storage, environment.getValidator());
 		final ClusterState clusterState = new ClusterState();
 		final NamespaceHandler<DistributedNamespace> namespaceHandler = new ClusterNamespaceHandler(clusterState, config, creator);
 		final DatasetRegistry<DistributedNamespace> datasetRegistry = ManagerProvider.createDatasetRegistry(namespaceHandler, config, creator);
-		final MetaStorage storage = new MetaStorage(config.getStorage());
-		creator.init(datasetRegistry, storage);
+		creator.init(datasetRegistry);
 
 		final ClusterConnectionManager connectionManager =
 				new ClusterConnectionManager(datasetRegistry, jobManager, environment.getValidator(), config, creator, clusterState);
diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java
index 87bfb58b2c..41d3895cf3 100644
--- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java
+++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java
@@ -33,12 +33,12 @@ public LocalManagerProvider(SqlDialectFactory dialectFactory) {
 
 	public DelegateManager<LocalNamespace> provideManager(ConqueryConfig config, Environment environment) {
 
-		final InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, environment.getValidator());
+		final MetaStorage storage = new MetaStorage(config.getStorage());
+		final InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, storage, environment.getValidator());
 		final NamespaceHandler<LocalNamespace> namespaceHandler = new LocalNamespaceHandler(config, creator, dialectFactory);
 		final DatasetRegistry<LocalNamespace> datasetRegistry = ManagerProvider.createDatasetRegistry(namespaceHandler, config, creator);
-		final MetaStorage storage = new MetaStorage(config.getStorage());
 
-		creator.init(datasetRegistry, storage);
+		creator.init(datasetRegistry);
 
 		return new DelegateManager<>(
 				config,
diff --git a/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java b/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java
index 5cb97c45f2..8d25bc0197 100644
--- a/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java
+++ b/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java
@@ -38,12 +38,12 @@ public abstract class AbstractSerializationTest {
 
 	@BeforeEach
 	public void before() {
-		InternalObjectMapperCreator creator = new InternalObjectMapperCreator(config, validator);
+		metaStorage = new MetaStorage(new NonPersistentStoreFactory());
+		InternalObjectMapperCreator creator = new InternalObjectMapperCreator(config, metaStorage, validator);
 		final IndexService indexService = new IndexService(config.getCsv().createCsvParserSettings(), "emptyDefaultLabel");
 		final ClusterNamespaceHandler clusterNamespaceHandler = new ClusterNamespaceHandler(new ClusterState(), config, creator);
-		datasetRegistry = new DatasetRegistry<>(0, config, null, clusterNamespaceHandler, indexService);
-		metaStorage = new MetaStorage(new NonPersistentStoreFactory());
-		creator.init(datasetRegistry, metaStorage);
+		datasetRegistry = new DatasetRegistry<>(0, config, creator, clusterNamespaceHandler, indexService);
+		creator.init(datasetRegistry);
 
 		// Prepare manager node internal mapper
 		final ManagerNode managerNode = mock(ManagerNode.class);

From ca8b0eb1794b139489fd8b3581fbf866d9755ba3 Mon Sep 17 00:00:00 2001
From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com>
Date: Wed, 24 Jul 2024 18:49:34 +0200
Subject: [PATCH 19/41] revert record conversion is IdRefrenceTest because
 jackson fails

---
 .../io/jackson/serializer/IdRefrenceTest.java  | 18 +++++++++++++++---
 .../concepts/tree/GroovyIndexedTest.java       |  2 ++
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java
index 21282175d9..cc6cbebfde 100644
--- a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java
+++ b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java
@@ -14,7 +14,10 @@
 import com.bakdata.conquery.models.identifiable.CentralRegistry;
 import com.bakdata.conquery.models.worker.SingletonNamespaceCollection;
 import com.bakdata.conquery.util.NonPersistentStoreFactory;
+import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
 import org.junit.jupiter.api.Test;
 
 public class IdRefrenceTest {
@@ -58,10 +61,19 @@ public void testListReferences() throws IOException {
 				.readerFor(ListHolder.class)
 				.readValue(json);
 
-		assertThat(holder.users().get(0)).isSameAs(user);
-		assertThat(holder.tables().get(0)).isSameAs(table);
+		assertThat(holder.getUsers().get(0)).isSameAs(user);
+		assertThat(holder.getTables().get(0)).isSameAs(table);
 	}
 
-	public record ListHolder(@NsIdRefCollection List<Table> tables, @MetaIdRefCollection List<User> users) {
+	/**
+	 * @implNote this needs to be a class, because jackson ignores NsIdRefCollection on records
+	 */
+	@Getter
+	@RequiredArgsConstructor(onConstructor_ = @JsonCreator)
+	public static class ListHolder {
+		@NsIdRefCollection
+		private final List<Table> tables;
+		@MetaIdRefCollection
+		private final List<User> users;
 	}
 }
diff --git a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java
index 80fcdd27d5..3afe94bb22 100644
--- a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java
+++ b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java
@@ -97,10 +97,12 @@ public static void init() throws IOException, JSONException, ConfigurationExcept
 		indexedConcept = conceptReader.readValue(node);
 
 		indexedConcept.setDataset(dataset);
+		indexedConcept.initElements();
 
 		oldConcept = conceptReader.readValue(node);
 
 		oldConcept.setDataset(dataset);
+		oldConcept.initElements();
 	}
 
 

From 699f25a8802c9b736f2d85f20503ace7baf3bd78 Mon Sep 17 00:00:00 2001
From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com>
Date: Thu, 25 Jul 2024 08:25:33 +0200
Subject: [PATCH 20/41] small clean up to trigger CI

---
 .../main/java/com/bakdata/conquery/commands/ManagerNode.java    | 2 +-
 .../com/bakdata/conquery/models/worker/DatasetRegistry.java     | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java
index 61b52502b7..d1c5686040 100644
--- a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java
+++ b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java
@@ -46,7 +46,7 @@
 import org.glassfish.jersey.internal.inject.AbstractBinder;
 
 /**
- * Central node of Conquery. Hosts the frontend, api, meta data and takes care of query distribution to
+ * Central node of Conquery. Hosts the frontend, api, metadata and takes care of query distribution to
  * {@link ShardNode}s and respectively the {@link Worker}s hosted on them. The {@link ManagerNode} can also
  * forward queries or results to statistic backends. Finally, it collects the results of queries for access over the api.
  */
diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java b/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java
index 8368f20a1a..64606760ff 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java
@@ -28,7 +28,6 @@
 import com.fasterxml.jackson.annotation.JsonIgnoreType;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.cache.CacheStats;
-import jakarta.validation.Validator;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;

From fe2f01363a5f8f92b19c3645936fa130e58658dc Mon Sep 17 00:00:00 2001
From: awildturtok <1553491+awildturtok@users.noreply.github.com>
Date: Thu, 25 Jul 2024 10:21:03 +0200
Subject: [PATCH 21/41] use typeInfo instead of serializer magic

---
 .../apiv1/frontend/FrontendSelect.java        |  3 +--
 .../concepts/FrontEndConceptBuilder.java      | 19 ++++++++++++++++---
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendSelect.java b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendSelect.java
index b698e7b26a..95f9e1c94b 100644
--- a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendSelect.java
+++ b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendSelect.java
@@ -1,7 +1,6 @@
 package com.bakdata.conquery.apiv1.frontend;
 
 import com.bakdata.conquery.models.identifiable.ids.specific.SelectId;
-import com.bakdata.conquery.models.types.ResultType;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import lombok.Builder;
 import lombok.Data;
@@ -15,7 +14,7 @@ public class FrontendSelect {
 	private SelectId id;
 	private String label;
 	private String description;
-	private ResultType resultType;
+	private String resultType;
 	@JsonProperty("default")
 	private Boolean isDefault;
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java
index 07e3dd23b8..721fc2c920 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java
@@ -1,10 +1,23 @@
 package com.bakdata.conquery.models.datasets.concepts;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import com.bakdata.conquery.apiv1.frontend.*;
+import com.bakdata.conquery.apiv1.frontend.FrontendFilterConfiguration;
+import com.bakdata.conquery.apiv1.frontend.FrontendList;
+import com.bakdata.conquery.apiv1.frontend.FrontendNode;
+import com.bakdata.conquery.apiv1.frontend.FrontendRoot;
+import com.bakdata.conquery.apiv1.frontend.FrontendSecondaryId;
+import com.bakdata.conquery.apiv1.frontend.FrontendSelect;
+import com.bakdata.conquery.apiv1.frontend.FrontendTable;
+import com.bakdata.conquery.apiv1.frontend.FrontendValidityDate;
+import com.bakdata.conquery.apiv1.frontend.FrontendValue;
 import com.bakdata.conquery.io.storage.NamespaceStorage;
 import com.bakdata.conquery.models.auth.entities.Subject;
 import com.bakdata.conquery.models.auth.permissions.Ability;
@@ -159,7 +172,7 @@ public FrontendSelect createSelect(Select select) {
 							 .id(select.getId())
 							 .label(select.getLabel())
 							 .description(select.getDescription())
-							 .resultType(select.getResultType())
+							 .resultType(select.getResultType().typeInfo())
 							 .isDefault(select.isDefault())
 							 .build();
 	}

From 4fc4a89e5cf769666c22514b30110627c310e992 Mon Sep 17 00:00:00 2001
From: awildturtok <1553491+awildturtok@users.noreply.github.com>
Date: Thu, 25 Jul 2024 10:58:45 +0200
Subject: [PATCH 22/41] drop usage of `default` in ColumnStore

---
 .../bakdata/conquery/models/events/EmptyStore.java    |  4 ++++
 .../models/events/stores/primitive/BitSetStore.java   |  4 ++++
 .../events/stores/primitive/ByteArrayStore.java       |  5 +++++
 .../events/stores/primitive/DecimalArrayStore.java    |  5 +++++
 .../events/stores/primitive/DoubleArrayStore.java     |  5 +++++
 .../events/stores/primitive/FloatArrayStore.java      |  5 +++++
 .../models/events/stores/primitive/IntArrayStore.java |  5 +++++
 .../events/stores/primitive/IntegerDateStore.java     |  5 +++++
 .../events/stores/primitive/LongArrayStore.java       |  5 +++++
 .../events/stores/primitive/ShortArrayStore.java      |  5 +++++
 .../events/stores/primitive/StringStoreString.java    |  5 +++++
 .../models/events/stores/root/ColumnStore.java        | 11 +----------
 .../stores/specific/CompoundDateRangeStore.java       |  2 ++
 .../events/stores/specific/DirectDateRangeStore.java  |  5 +++++
 .../models/events/stores/specific/MoneyIntStore.java  |  5 +++++
 .../events/stores/specific/QuarterDateRangeStore.java |  5 +++++
 .../events/stores/specific/RebasingIntegerStore.java  |  5 +++++
 .../events/stores/specific/ScaledDecimalStore.java    |  5 +++++
 18 files changed, 81 insertions(+), 10 deletions(-)

diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/EmptyStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/EmptyStore.java
index 2d156a3226..7201ae0ac9 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/EmptyStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/EmptyStore.java
@@ -49,6 +49,10 @@ public void setDate(int event, int value) {
 
 	}
 
+	public void setParent(Bucket bucket) {
+		// not used
+	}
+
 	@Override
 	public Object createScriptValue(int event) {
 		return null;
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/BitSetStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/BitSetStore.java
index 6cfdf2e406..2e8c4731e8 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/BitSetStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/BitSetStore.java
@@ -6,6 +6,7 @@
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.io.jackson.serializer.BitSetDeserializer;
 import com.bakdata.conquery.io.jackson.serializer.BitSetSerializer;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.BooleanStore;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.fasterxml.jackson.annotation.JsonCreator;
@@ -81,6 +82,9 @@ public BitSetStore select(int[] starts, int[] lengths) {
 		return new BitSetStore(out, outNulls, length);
 	}
 
+	public void setParent(Bucket bucket) {
+		// not used
+	}
 
 	@Override
 	public boolean has(int event) {
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/ByteArrayStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/ByteArrayStore.java
index 7fdce17ce1..2c035314de 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/ByteArrayStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/ByteArrayStore.java
@@ -1,6 +1,7 @@
 package com.bakdata.conquery.models.events.stores.primitive;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.IntegerStore;
 import com.fasterxml.jackson.annotation.JsonCreator;
@@ -71,4 +72,8 @@ public long getInteger(int event) {
 		return values[event];
 	}
 
+	public void setParent(Bucket bucket) {
+		// not used
+	}
+
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/DecimalArrayStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/DecimalArrayStore.java
index f00ab0d574..60529dffe7 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/DecimalArrayStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/DecimalArrayStore.java
@@ -3,6 +3,7 @@
 import java.math.BigDecimal;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.DecimalStore;
 import com.fasterxml.jackson.annotation.JsonCreator;
@@ -66,4 +67,8 @@ public boolean has(int event) {
 	public BigDecimal getDecimal(int event) {
 		return values[event];
 	}
+
+	public void setParent(Bucket bucket) {
+		// not used
+	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/DoubleArrayStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/DoubleArrayStore.java
index 66427a2d56..b2dfc19931 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/DoubleArrayStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/DoubleArrayStore.java
@@ -1,6 +1,7 @@
 package com.bakdata.conquery.models.events.stores.primitive;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.RealStore;
 import com.fasterxml.jackson.annotation.JsonCreator;
@@ -66,4 +67,8 @@ public boolean has(int event) {
 	public double getReal(int event) {
 		return values[event];
 	}
+
+	public void setParent(Bucket bucket) {
+		// not used
+	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/FloatArrayStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/FloatArrayStore.java
index 21634d6572..ac59e91737 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/FloatArrayStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/FloatArrayStore.java
@@ -1,6 +1,7 @@
 package com.bakdata.conquery.models.events.stores.primitive;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.RealStore;
 import com.fasterxml.jackson.annotation.JsonCreator;
@@ -66,4 +67,8 @@ public boolean has(int event) {
 	public double getReal(int event) {
 		return values[event];
 	}
+
+	public void setParent(Bucket bucket) {
+		// not used
+	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/IntArrayStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/IntArrayStore.java
index 2890d913cd..64859ea28c 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/IntArrayStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/IntArrayStore.java
@@ -1,6 +1,7 @@
 package com.bakdata.conquery.models.events.stores.primitive;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.IntegerStore;
 import com.fasterxml.jackson.annotation.JsonCreator;
@@ -71,4 +72,8 @@ public boolean has(int event) {
 	public long getInteger(int event) {
 		return values[event];
 	}
+
+	public void setParent(Bucket bucket) {
+		// not used
+	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/IntegerDateStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/IntegerDateStore.java
index 83bed84cdd..7abdab5d38 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/IntegerDateStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/IntegerDateStore.java
@@ -2,6 +2,7 @@
 
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.models.common.CDate;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.DateStore;
 import com.bakdata.conquery.models.events.stores.root.IntegerStore;
@@ -69,4 +70,8 @@ public int getDate(int event) {
 		return (int) store.getInteger(event);
 	}
 
+	public void setParent(Bucket bucket) {
+		// not used
+	}
+
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/LongArrayStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/LongArrayStore.java
index 6a17aee351..3ab3b874bb 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/LongArrayStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/LongArrayStore.java
@@ -1,6 +1,7 @@
 package com.bakdata.conquery.models.events.stores.primitive;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.IntegerStore;
 import com.fasterxml.jackson.annotation.JsonCreator;
@@ -70,4 +71,8 @@ public long getInteger(int event) {
 		return values[event];
 	}
 
+	public void setParent(Bucket bucket) {
+		// not used
+	}
+
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/ShortArrayStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/ShortArrayStore.java
index fe2bbdce5b..a5180373e9 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/ShortArrayStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/ShortArrayStore.java
@@ -1,6 +1,7 @@
 package com.bakdata.conquery.models.events.stores.primitive;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.IntegerStore;
 import com.fasterxml.jackson.annotation.JsonCreator;
@@ -70,4 +71,8 @@ public long getInteger(int event) {
 		return values[event];
 	}
 
+	public void setParent(Bucket bucket) {
+		// not used
+	}
+
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/StringStoreString.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/StringStoreString.java
index 58524e069a..d78e25e11f 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/StringStoreString.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/StringStoreString.java
@@ -5,6 +5,7 @@
 import java.util.stream.Stream;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.StringStore;
 import com.fasterxml.jackson.annotation.JsonCreator;
@@ -96,4 +97,8 @@ public Stream<String> streamValues() {
 		return Arrays.stream(values).filter(Objects::nonNull);
 	}
 
+	public void setParent(Bucket bucket) {
+		// not used
+	}
+
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/ColumnStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/ColumnStore.java
index 93db593245..d0ca7b1cf6 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/ColumnStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/ColumnStore.java
@@ -31,9 +31,7 @@ public interface ColumnStore {
 	 * @implNote BackReference set here because Jackson does not support for fields in interfaces and abstract classes see also https://github.com/FasterXML/jackson-databind/issues/3304
 	 */
 	@JsonBackReference
-	default void setParent(Bucket bucket) {
-
-	}
+	void setParent(Bucket bucket);
 
 
 	/**
@@ -92,13 +90,6 @@ default long estimateMemoryConsumptionBytes() {
 	@JsonIgnore
 	int getLines();
 
-	/**
-	 * Bytes required to store auxilary data.
-	 */
-	default long estimateTypeSizeBytes() {
-		return 0;
-	}
-
 	/**
 	 * Create an empty store that's only a description of the transformation.
 	 */
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/CompoundDateRangeStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/CompoundDateRangeStore.java
index 9c72287b4e..c116f9142b 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/CompoundDateRangeStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/CompoundDateRangeStore.java
@@ -7,6 +7,7 @@
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.DateRangeStore;
 import com.bakdata.conquery.models.events.stores.root.DateStore;
+import com.fasterxml.jackson.annotation.JsonBackReference;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import jakarta.validation.constraints.NotEmpty;
 import jakarta.validation.constraints.NotNull;
@@ -89,6 +90,7 @@ public DateStore getEndStore() {
 	}
 
 	@Override
+	@JsonBackReference
 	public void setParent(@NonNull Bucket bucket) {
 		parent = bucket;
 	}
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/DirectDateRangeStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/DirectDateRangeStore.java
index fbaeadc9a8..dcef9bc254 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/DirectDateRangeStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/DirectDateRangeStore.java
@@ -2,6 +2,7 @@
 
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.models.common.daterange.CDateRange;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.primitive.IntegerDateStore;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.DateRangeStore;
@@ -29,6 +30,10 @@ public DirectDateRangeStore(DateStore minStore, DateStore maxStore) {
 		this.maxStore = maxStore;
 	}
 
+	public void setParent(Bucket bucket) {
+		// not used
+	}
+
 	@Override
 	public int getLines() {
 		// they can be unaligned, if one of them is empty.
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java
index c3d38474d8..c66faaae0e 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java
@@ -1,6 +1,7 @@
 package com.bakdata.conquery.models.events.stores.specific;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.IntegerStore;
 import com.bakdata.conquery.models.events.stores.root.MoneyStore;
@@ -61,4 +62,8 @@ public void setNull(int event) {
 	public final boolean has(int event) {
 		return numberType.has(event);
 	}
+
+	public void setParent(Bucket bucket) {
+		// not used
+	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/QuarterDateRangeStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/QuarterDateRangeStore.java
index 1173ac01bb..d441f771fd 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/QuarterDateRangeStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/QuarterDateRangeStore.java
@@ -6,6 +6,7 @@
 import com.bakdata.conquery.models.common.CDate;
 import com.bakdata.conquery.models.common.QuarterUtils;
 import com.bakdata.conquery.models.common.daterange.CDateRange;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.DateRangeStore;
 import com.bakdata.conquery.models.events.stores.root.IntegerStore;
@@ -75,4 +76,8 @@ public CDateRange getDateRange(int event) {
 
 		return CDateRange.of(begin, CDate.ofLocalDate(end));
 	}
+
+	public void setParent(Bucket bucket) {
+		// not used
+	}
 }
\ No newline at end of file
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/RebasingIntegerStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/RebasingIntegerStore.java
index 559457c0d8..ab26940bb8 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/RebasingIntegerStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/RebasingIntegerStore.java
@@ -1,6 +1,7 @@
 package com.bakdata.conquery.models.events.stores.specific;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.IntegerStore;
 import jakarta.validation.constraints.NotNull;
@@ -68,4 +69,8 @@ public boolean has(int event) {
 	public void setNull(int event) {
 		store.setNull(event);
 	}
+
+	public void setParent(Bucket bucket) {
+		// not used
+	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/ScaledDecimalStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/ScaledDecimalStore.java
index a7f368ba6f..802d1d6b92 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/ScaledDecimalStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/ScaledDecimalStore.java
@@ -4,6 +4,7 @@
 import java.math.BigInteger;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.DecimalStore;
 import com.bakdata.conquery.models.events.stores.root.IntegerStore;
@@ -75,4 +76,8 @@ public static BigDecimal scale(int scale, long value) {
 	public boolean has(int event) {
 		return subType.has(event);
 	}
+
+	public void setParent(Bucket bucket) {
+		// not used
+	}
 }
\ No newline at end of file

From 52772c1f52fe22bdec192657bfb9a08c0894d72d Mon Sep 17 00:00:00 2001
From: awildturtok <1553491+awildturtok@users.noreply.github.com>
Date: Thu, 25 Jul 2024 11:30:55 +0200
Subject: [PATCH 23/41] minor cleanup, ensure that setNull is not used as
 JsonProperty

---
 .../bakdata/conquery/models/events/stores/root/ColumnStore.java  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/ColumnStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/ColumnStore.java
index d0ca7b1cf6..1558fa67a2 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/ColumnStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/ColumnStore.java
@@ -106,6 +106,7 @@ static <T extends ColumnStore> T emptyCopy(T store) {
 	 */
 	<T extends ColumnStore> T select(int[] starts, int[] lengths);
 
+	@JsonIgnore
 	void setNull(int event);
 
 

From 6cbcb806bdd23e6c2516ee6c8ec5c6196a662616 Mon Sep 17 00:00:00 2001
From: Jonas Arnhold <jonas.arnhold@bakdata.com>
Date: Thu, 25 Jul 2024 11:48:23 +0200
Subject: [PATCH 24/41] Implement external ID lookup for SQL mode (#3496)

---
 .../conquery/apiv1/QueryProcessor.java        |   7 +-
 .../concept/specific/external/CQExternal.java | 252 +-----------------
 .../specific/external/EntityResolver.java     |  51 ++++
 .../specific/external/EntityResolverUtil.java | 179 +++++++++++++
 .../mode/cluster/ClusterEntityResolver.java   |  83 ++++++
 .../mode/cluster/ClusterNamespaceHandler.java |   1 +
 .../mode/local/LocalNamespaceHandler.java     |   2 +
 .../mode/local/SqlEntityResolver.java         | 206 ++++++++++++++
 .../conquery/models/config/ColumnConfig.java  |   7 +-
 .../identifiable/mapping/EntityIdMap.java     |   7 -
 .../identifiable/mapping/ExternalId.java      |  12 +
 .../models/worker/DistributedNamespace.java   |   5 +-
 .../models/worker/LocalNamespace.java         |   7 +-
 .../conquery/models/worker/Namespace.java     |   3 +
 .../conquery/sql/DslContextFactory.java       |   5 +-
 .../conversion/cqelement/CQYesConverter.java  |   6 +-
 .../cqelement/concept/PreprocessingCte.java   |   5 +-
 .../dialect/SqlFunctionProvider.java          |  30 +--
 .../sql/conversion/model/QueryStepJoiner.java |   9 +-
 .../query/TableExportQueryConverter.java      |   3 +-
 .../conquery/util/io/IdColumnUtil.java        |   6 +-
 .../integration/json/ConqueryTestSpec.java    |  18 +-
 .../integration/json/SqlTestDataImporter.java |  10 +
 .../identifiable/IdMapSerialisationTest.java  |   3 +-
 .../yes/persons.csv => shared/entities.csv}   |   7 +-
 .../test/resources/shared/entities.table.json |  14 +
 .../sql/yes/{CQYES.test.json => cqyes.json}   |  21 +-
 .../test/resources/tests/sql/yes/expected.csv |   9 +-
 .../test/resources/tests/sql/yes/no-op.csv    |   2 +
 29 files changed, 639 insertions(+), 331 deletions(-)
 create mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolver.java
 create mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolverUtil.java
 create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterEntityResolver.java
 create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java
 create mode 100644 backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/ExternalId.java
 rename backend/src/test/resources/{tests/sql/yes/persons.csv => shared/entities.csv} (86%)
 create mode 100644 backend/src/test/resources/shared/entities.table.json
 rename backend/src/test/resources/tests/sql/yes/{CQYES.test.json => cqyes.json} (54%)
 create mode 100644 backend/src/test/resources/tests/sql/yes/no-op.csv

diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java
index 1dbb9753d1..92d2b9b463 100644
--- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java
+++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java
@@ -39,6 +39,7 @@
 import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept;
 import com.bakdata.conquery.apiv1.query.concept.specific.CQOr;
 import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal;
+import com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolver;
 import com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider;
 import com.bakdata.conquery.io.storage.MetaStorage;
 import com.bakdata.conquery.metrics.ExecutionMetrics;
@@ -108,7 +109,6 @@ public class QueryProcessor {
 	private Validator validator;
 
 
-
 	public Stream<ExecutionStatus> getAllQueries(Dataset dataset, HttpServletRequest req, Subject subject, boolean allProviders) {
 		final Collection<ManagedExecution> allQueries = storage.getAllExecutions();
 
@@ -294,14 +294,13 @@ public FullExecutionStatus getQueryFullStatus(ManagedExecution query, Subject su
 	public ExternalUploadResult uploadEntities(Subject subject, Dataset dataset, ExternalUpload upload) {
 
 		final Namespace namespace = datasetRegistry.get(dataset.getId());
-		final CQExternal.ResolveStatistic statistic = CQExternal.resolveEntities(
+		final EntityResolver.ResolveStatistic statistic = namespace.getEntityResolver().resolveEntities(
 				upload.getValues(),
 				upload.getFormat(),
 				namespace.getStorage().getIdMapping(),
 				config.getIdColumns(),
 				config.getLocale().getDateReader(),
-				upload.isOneRowPerEntity(),
-				true
+				upload.isOneRowPerEntity()
 		);
 
 		// Resolving nothing is a problem thus we fail.
diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java
index 48466e2dd8..2ba0d02a0b 100644
--- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java
+++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java
@@ -3,7 +3,6 @@
 
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -11,7 +10,6 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
-import java.util.function.Function;
 import java.util.stream.Collectors;
 
 import com.bakdata.conquery.apiv1.query.CQElement;
@@ -21,7 +19,6 @@
 import com.bakdata.conquery.models.config.IdColumnConfig;
 import com.bakdata.conquery.models.error.ConqueryError;
 import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId;
-import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap;
 import com.bakdata.conquery.models.query.QueryExecutionContext;
 import com.bakdata.conquery.models.query.QueryPlanContext;
 import com.bakdata.conquery.models.query.QueryResolveContext;
@@ -33,16 +30,12 @@
 import com.bakdata.conquery.models.query.resultinfo.ResultInfo;
 import com.bakdata.conquery.models.query.resultinfo.SimpleResultInfo;
 import com.bakdata.conquery.models.types.ResultType;
-import com.bakdata.conquery.util.DateReader;
-import com.bakdata.conquery.util.io.IdColumnUtil;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonView;
 import com.google.common.collect.Streams;
 import io.dropwizard.validation.ValidationMethod;
 import jakarta.validation.constraints.NotEmpty;
-import jakarta.validation.constraints.NotNull;
 import lombok.AccessLevel;
-import lombok.Data;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -55,8 +48,6 @@
 @NoArgsConstructor
 public class CQExternal extends CQElement {
 
-	private static final String FORMAT_EXTRA = "EXTRA";
-
 	/**
 	 * Describes the format of {@code values}, how to extract data from each row:
 	 * <p>
@@ -118,7 +109,7 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) {
 		final String[] extraHeaders = Streams.zip(
 													 Arrays.stream(headers),
 													 format.stream(),
-													 (header, format) -> format.equals(FORMAT_EXTRA) ? header : null
+													 (header, format) -> format.equals(EntityResolverUtil.FORMAT_EXTRA) ? header : null
 											 )
 											 .filter(Objects::nonNull)
 											 .toArray(String[]::new);
@@ -180,78 +171,18 @@ private ExternalNode<List<String>> createExternalNodeForList(QueryPlanContext co
 		);
 	}
 
-	/**
-	 * For each row try and collect all dates.
-	 *
-	 * @return Row -> Dates
-	 */
-	private static CDateSet[] readDates(String[][] values, List<String> format, DateReader dateReader) {
-		final CDateSet[] out = new CDateSet[values.length];
-
-		final List<DateFormat> dateFormats = format.stream()
-												   .map(CQExternal::resolveDateFormat)
-												   // Don't use Stream#toList to preserve null-values
-												   .collect(Collectors.toList());
-
-
-		/*
-		 If no format is provided, put empty dates into output.
-		 This indicates that no date context was provided and
-		 the entries are not restricted by any date restriction,
-		 but can also don't contribute to any date aggregation.
-		 */
-		if (dateFormats.stream().allMatch(Objects::isNull)) {
-			// Initialize empty
-			for (int row = 0; row < values.length; row++) {
-				out[row] = CDateSet.createEmpty();
-			}
-			return out;
-		}
-
-		for (int row = 1; row < values.length; row++) {
-			try {
-				final CDateSet dates = CDateSet.createEmpty();
-
-				// Collect all specified dates into a single set.
-				for (int col = 0; col < dateFormats.size(); col++) {
-					final DateFormat dateFormat = dateFormats.get(col);
-
-					if (dateFormat == null) {
-						continue;
-					}
-					dateFormat.readDates(values[row][col], dateReader, dates);
-				}
-
-				if (dates.isEmpty()) {
-					continue;
-				}
-
-				if (out[row] == null) {
-					out[row] = CDateSet.createEmpty();
-				}
-
-				out[row].addAll(dates);
-			}
-			catch (Exception e) {
-				log.warn("Failed to parse Date from {}", row, e);
-			}
-		}
-
-		return out;
-	}
-
 	@Override
 	public void resolve(QueryResolveContext context) {
 		headers = values[0];
 
-		final ResolveStatistic resolved =
-				resolveEntities(values, format,
-								context.getNamespace().getStorage().getIdMapping(),
-								context.getConfig().getIdColumns(),
-								context.getConfig().getLocale().getDateReader(),
-								onlySingles,
-								context.getConfig().getSqlConnectorConfig().isEnabled()
-				);
+		final EntityResolver.ResolveStatistic resolved = context.getNamespace().getEntityResolver().resolveEntities(
+				values,
+				format,
+				context.getNamespace().getStorage().getIdMapping(),
+				context.getConfig().getIdColumns(),
+				context.getConfig().getLocale().getDateReader(),
+				onlySingles
+		);
 
 		if (resolved.getResolved().isEmpty()) {
 			throw new ConqueryError.ExternalResolveEmptyError();
@@ -277,158 +208,6 @@ public void resolve(QueryResolveContext context) {
 		extra = resolved.getExtra();
 	}
 
-	@Data
-	public static class ResolveStatistic {
-
-		@JsonIgnore
-		private final Map<String, CDateSet> resolved;
-
-		/**
-		 * Entity -> Column -> Values
-		 */
-		@JsonIgnore
-		private final Map<String, Map<String, List<String>>> extra;
-
-		private final List<String[]> unreadableDate;
-		private final List<String[]> unresolvedId;
-
-	}
-
-	/**
-	 * Helper method to try and resolve entities in values using the specified format.
-	 */
-	public static ResolveStatistic resolveEntities(@NotEmpty String[][] values, @NotEmpty List<String> format, EntityIdMap mapping, IdColumnConfig idColumnConfig, @NotNull DateReader dateReader, boolean onlySingles, boolean isInSqlMode) {
-		final Map<String, CDateSet> resolved = new HashMap<>();
-
-		final List<String[]> unresolvedDate = new ArrayList<>();
-		final List<String[]> unresolvedId = new ArrayList<>();
-
-		// extract dates from rows
-		final CDateSet[] rowDates = readDates(values, format, dateReader);
-
-		// Extract extra data from rows by Row, to be collected into by entities
-		// Row -> Column -> Value
-		final Map<String, String>[] extraDataByRow = readExtras(values, format);
-
-		final List<Function<String[], EntityIdMap.ExternalId>> readers = IdColumnUtil.getIdReaders(format, idColumnConfig.getIdMappers());
-
-		// We will not be able to resolve anything...
-		if (readers.isEmpty()) {
-			return new ResolveStatistic(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyList(), List.of(values));
-		}
-
-		// Entity -> Column -> Values
-		final Map<String, Map<String, List<String>>> extraDataByEntity = new HashMap<>();
-
-		// ignore the first row, because this is the header
-		for (int rowNum = 1; rowNum < values.length; rowNum++) {
-
-			final String[] row = values[rowNum];
-
-			if (rowDates[rowNum] == null) {
-				unresolvedDate.add(row);
-				continue;
-			}
-
-			// TODO proper implementation of EntityIdMap#resolve for SQL mode
-			String resolvedId = isInSqlMode
-								? String.valueOf(row[0])
-								: tryResolveId(row, readers, mapping);
-
-			if (resolvedId == null) {
-				unresolvedId.add(row);
-				continue;
-			}
-
-			//read the dates from the row
-			resolved.put(resolvedId, rowDates[rowNum]);
-
-			// Entity was resolved for row so we collect the data.
-			if (extraDataByRow[rowNum] != null) {
-
-				for (Map.Entry<String, String> entry : extraDataByRow[rowNum].entrySet()) {
-					extraDataByEntity.computeIfAbsent(resolvedId, (ignored) -> new HashMap<>())
-									 .computeIfAbsent(entry.getKey(), (ignored) -> new ArrayList<>())
-									 .add(entry.getValue());
-				}
-			}
-		}
-
-		if (onlySingles) {
-			// Check that there is at most one value per entity and per column
-			final boolean alright = extraDataByEntity.values().stream()
-													 .map(Map::values)
-													 .flatMap(Collection::stream)
-													 .allMatch(l -> l.size() <= 1);
-			if (!alright) {
-				throw new ConqueryError.ExternalResolveOnePerRowError();
-			}
-		}
-
-		return new ResolveStatistic(resolved, extraDataByEntity, unresolvedDate, unresolvedId);
-	}
-
-	/**
-	 * Try to extract a {@link com.bakdata.conquery.models.identifiable.mapping.EntityIdMap.ExternalId} from the row,
-	 * then try to map it to an internal {@link com.bakdata.conquery.models.query.entity.Entity}
-	 */
-	private static String tryResolveId(String[] row, List<Function<String[], EntityIdMap.ExternalId>> readers, EntityIdMap mapping) {
-		String resolvedId = null;
-
-		for (Function<String[], EntityIdMap.ExternalId> reader : readers) {
-			final EntityIdMap.ExternalId externalId = reader.apply(row);
-
-			if (externalId == null) {
-				continue;
-			}
-
-			String innerResolved = mapping.resolve(externalId);
-
-			if (innerResolved == null) {
-				continue;
-			}
-
-			// Only if all resolvable ids agree on the same entity, do we return the id.
-			if (resolvedId != null && !innerResolved.equals(resolvedId)) {
-				log.error("`{}` maps to different Entities", (Object) row);
-				continue;
-			}
-
-			resolvedId = innerResolved;
-		}
-		return resolvedId;
-	}
-
-	/**
-	 * Try and extract Extra data from input to be returned as extra-data in output.
-	 * <p>
-	 * Line -> ( Column -> Value )
-	 */
-	private static Map<String, String>[] readExtras(String[][] values, List<String> format) {
-		final String[] names = values[0];
-		final Map<String, String>[] extrasByRow = new Map[values.length];
-
-
-		for (int line = 1; line < values.length; line++) {
-			for (int col = 0; col < format.size(); col++) {
-				if (!format.get(col).equals(FORMAT_EXTRA)) {
-					continue;
-				}
-
-
-				if (extrasByRow[line] == null) {
-					extrasByRow[line] = new HashMap<>(names.length);
-				}
-
-				extrasByRow[line].put(names[col], values[line][col]);
-			}
-		}
-
-
-		return extrasByRow;
-	}
-
-
 	@Override
 	public RequiredEntities collectRequiredEntities(QueryExecutionContext context) {
 		return new RequiredEntities(valuesResolved.keySet());
@@ -441,7 +220,7 @@ public List<ResultInfo> getResultInfos() {
 		}
 		List<ResultInfo> resultInfos = new ArrayList<>();
 		for (int col = 0; col < format.size(); col++) {
-			if (!format.get(col).equals(FORMAT_EXTRA)) {
+			if (!format.get(col).equals(EntityResolverUtil.FORMAT_EXTRA)) {
 				continue;
 			}
 
@@ -484,15 +263,4 @@ public boolean isHeadersUnique() {
 		return false;
 	}
 
-	/**
-	 * Try to resolve a date format, return nothing if not possible.
-	 */
-	private static DateFormat resolveDateFormat(String name) {
-		try {
-			return DateFormat.valueOf(name);
-		}
-		catch (IllegalArgumentException e) {
-			return null; // Does not exist
-		}
-	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolver.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolver.java
new file mode 100644
index 0000000000..c93f1e3375
--- /dev/null
+++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolver.java
@@ -0,0 +1,51 @@
+package com.bakdata.conquery.apiv1.query.concept.specific.external;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import com.bakdata.conquery.models.common.CDateSet;
+import com.bakdata.conquery.models.config.IdColumnConfig;
+import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap;
+import com.bakdata.conquery.util.DateReader;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import jakarta.validation.constraints.NotEmpty;
+import jakarta.validation.constraints.NotNull;
+import lombok.Data;
+
+public interface EntityResolver {
+
+	/**
+	 * Helper method to try and resolve entities in values using the specified format.
+	 */
+	ResolveStatistic resolveEntities(
+			@NotEmpty String[][] values,
+			@NotEmpty List<String> format,
+			EntityIdMap mapping,
+			IdColumnConfig idColumnConfig,
+			@NotNull DateReader dateReader,
+			boolean onlySingles
+	);
+
+	@Data
+	class ResolveStatistic {
+
+		@JsonIgnore
+		private final Map<String, CDateSet> resolved;
+
+		/**
+		 * Entity -> Column -> Values
+		 */
+		@JsonIgnore
+		private final Map<String, Map<String, List<String>>> extra;
+
+		private final List<String[]> unreadableDate;
+		private final List<String[]> unresolvedId;
+
+		public static ResolveStatistic forEmptyReaders(String[][] values) {
+			return new ResolveStatistic(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyList(), List.of(values));
+		}
+
+	}
+
+}
diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolverUtil.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolverUtil.java
new file mode 100644
index 0000000000..3a9be46a25
--- /dev/null
+++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolverUtil.java
@@ -0,0 +1,179 @@
+package com.bakdata.conquery.apiv1.query.concept.specific.external;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import com.bakdata.conquery.models.common.CDateSet;
+import com.bakdata.conquery.models.error.ConqueryError;
+import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap;
+import com.bakdata.conquery.models.identifiable.mapping.ExternalId;
+import com.bakdata.conquery.util.DateReader;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class EntityResolverUtil {
+
+	public static final String FORMAT_EXTRA = "EXTRA";
+
+	/**
+	 * For each row try and collect all dates.
+	 *
+	 * @return Row -> Dates
+	 */
+	public static CDateSet[] readDates(String[][] values, List<String> format, DateReader dateReader) {
+		final CDateSet[] out = new CDateSet[values.length];
+
+		final List<DateFormat> dateFormats = format.stream()
+												   .map(EntityResolverUtil::resolveDateFormat)
+												   // Don't use Stream#toList to preserve null-values
+												   .collect(Collectors.toList());
+
+		/*
+		 If no format is provided, put empty dates into output.
+		 This indicates that no date context was provided and
+		 the entries are not restricted by any date restriction,
+		 but can also don't contribute to any date aggregation.
+		 */
+		if (dateFormats.stream().allMatch(Objects::isNull)) {
+			// Initialize empty
+			for (int row = 0; row < values.length; row++) {
+				out[row] = CDateSet.createEmpty();
+			}
+			return out;
+		}
+
+		for (int row = 1; row < values.length; row++) {
+			try {
+				final CDateSet dates = CDateSet.createEmpty();
+
+				// Collect all specified dates into a single set.
+				for (int col = 0; col < dateFormats.size(); col++) {
+					final DateFormat dateFormat = dateFormats.get(col);
+
+					if (dateFormat == null) {
+						continue;
+					}
+					dateFormat.readDates(values[row][col], dateReader, dates);
+				}
+
+				if (dates.isEmpty()) {
+					continue;
+				}
+
+				if (out[row] == null) {
+					out[row] = CDateSet.createEmpty();
+				}
+
+				out[row].addAll(dates);
+			}
+			catch (Exception e) {
+				log.warn("Failed to parse Date from {}", row, e);
+			}
+		}
+
+		return out;
+	}
+
+	public static void collectExtraData(Map<String, String>[] extraDataByRow, int rowNum, Map<String, Map<String, List<String>>> extraDataByEntity, String resolvedId) {
+		if (extraDataByRow[rowNum] != null) {
+			for (Map.Entry<String, String> entry : extraDataByRow[rowNum].entrySet()) {
+				extraDataByEntity.computeIfAbsent(resolvedId, (ignored) -> new HashMap<>())
+								 .computeIfAbsent(entry.getKey(), (ignored) -> new ArrayList<>())
+								 .add(entry.getValue());
+			}
+		}
+	}
+
+	public static void verifyOnlySingles(boolean onlySingles, Map<String, Map<String, List<String>>> extraDataByEntity) {
+		if (!onlySingles) {
+			return;
+		}
+		// Check that there is at most one value per entity and per column
+		final boolean alright = extraDataByEntity.values().stream()
+												 .map(Map::values)
+												 .flatMap(Collection::stream)
+												 .allMatch(l -> l.size() <= 1);
+		if (!alright) {
+			throw new ConqueryError.ExternalResolveOnePerRowError();
+		}
+	}
+
+	/**
+	 * Try to extract a {@link ExternalId} from the row,
+	 * then try to map it to an internal {@link com.bakdata.conquery.models.query.entity.Entity}
+	 */
+	public static String tryResolveId(String[] row, List<Function<String[], ExternalId>> readers, EntityIdMap mapping) {
+		String resolvedId = null;
+
+		for (Function<String[], ExternalId> reader : readers) {
+			final ExternalId externalId = reader.apply(row);
+
+			if (externalId == null) {
+				continue;
+			}
+
+			String innerResolved = mapping.resolve(externalId);
+
+			if (innerResolved == null) {
+				continue;
+			}
+
+			// Only if all resolvable ids agree on the same entity, do we return the id.
+			if (resolvedId != null && !innerResolved.equals(resolvedId)) {
+				log.error("`{}` maps to different Entities", (Object) row);
+				continue;
+			}
+
+			resolvedId = innerResolved;
+		}
+		return resolvedId;
+	}
+
+	/**
+	 * Try and extract Extra data from input to be returned as extra-data in output.
+	 * <p>
+	 * Line -> ( Column -> Value )
+	 */
+	public static Map<String, String>[] readExtras(String[][] values, List<String> format) {
+		final String[] names = values[0];
+		final Map<String, String>[] extrasByRow = new Map[values.length];
+
+
+		for (int line = 1; line < values.length; line++) {
+			for (int col = 0; col < format.size(); col++) {
+				if (!format.get(col).equals(FORMAT_EXTRA)) {
+					continue;
+				}
+
+
+				if (extrasByRow[line] == null) {
+					extrasByRow[line] = new HashMap<>(names.length);
+				}
+
+				extrasByRow[line].put(names[col], values[line][col]);
+			}
+		}
+
+
+		return extrasByRow;
+	}
+
+	/**
+	 * Try to resolve a date format, return nothing if not possible.
+	 */
+	private static DateFormat resolveDateFormat(String name) {
+		try {
+			return DateFormat.valueOf(name);
+		}
+		catch (IllegalArgumentException e) {
+			return null; // Does not exist
+		}
+	}
+
+}
diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterEntityResolver.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterEntityResolver.java
new file mode 100644
index 0000000000..8aa3d7c720
--- /dev/null
+++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterEntityResolver.java
@@ -0,0 +1,83 @@
+package com.bakdata.conquery.mode.cluster;
+
+import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.collectExtraData;
+import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.readDates;
+import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.tryResolveId;
+import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.verifyOnlySingles;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+import com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolver;
+import com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil;
+import com.bakdata.conquery.models.common.CDateSet;
+import com.bakdata.conquery.models.config.IdColumnConfig;
+import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap;
+import com.bakdata.conquery.models.identifiable.mapping.ExternalId;
+import com.bakdata.conquery.util.DateReader;
+import com.bakdata.conquery.util.io.IdColumnUtil;
+import jakarta.validation.constraints.NotEmpty;
+
+public class ClusterEntityResolver implements EntityResolver {
+
+	@Override
+	public ResolveStatistic resolveEntities(
+			@NotEmpty String[][] values,
+			List<String> format,
+			EntityIdMap mapping,
+			IdColumnConfig idColumnConfig,
+			DateReader dateReader,
+			boolean onlySingles
+	) {
+		final Map<String, CDateSet> resolved = new HashMap<>();
+		final List<String[]> unresolvedDate = new ArrayList<>();
+		final List<String[]> unresolvedId = new ArrayList<>();
+
+		// extract dates from rows
+		final CDateSet[] rowDates = readDates(values, format, dateReader);
+
+		// Extract extra data from rows by Row, to be collected into by entities
+		// Row -> Column -> Value
+		final Map<String, String>[] extraDataByRow = EntityResolverUtil.readExtras(values, format);
+
+		final List<Function<String[], ExternalId>> readers = IdColumnUtil.getIdReaders(format, idColumnConfig.getIdMappers());
+
+		// We will not be able to resolve anything...
+		if (readers.isEmpty()) {
+			return EntityResolver.ResolveStatistic.forEmptyReaders(values);
+		}
+
+		// Entity -> Column -> Values
+		final Map<String, Map<String, List<String>>> extraDataByEntity = new HashMap<>();
+
+		// ignore the first row, because this is the header
+		for (int rowNum = 1; rowNum < values.length; rowNum++) {
+
+			final String[] row = values[rowNum];
+
+			if (rowDates[rowNum] == null) {
+				unresolvedDate.add(row);
+				continue;
+			}
+
+			String resolvedId = tryResolveId(row, readers, mapping);
+
+			if (resolvedId == null) {
+				unresolvedId.add(row);
+				continue;
+			}
+
+			// read the dates from the row
+			resolved.put(resolvedId, rowDates[rowNum]);
+
+			// Entity was resolved for row, so we collect the data.
+			collectExtraData(extraDataByRow, rowNum, extraDataByEntity, resolvedId);
+		}
+
+		verifyOnlySingles(onlySingles, extraDataByEntity);
+		return new EntityResolver.ResolveStatistic(resolved, extraDataByEntity, unresolvedDate, unresolvedId);
+	}
+}
diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java
index acf04a20fe..617f166d72 100644
--- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java
+++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java
@@ -37,6 +37,7 @@ public DistributedNamespace createNamespace(NamespaceStorage storage, final Meta
 				namespaceData.getJobManager(),
 				namespaceData.getFilterSearch(),
 				namespaceData.getIndexService(),
+				new ClusterEntityResolver(),
 				namespaceData.getInjectables(),
 				workerHandler
 		);
diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java
index 52a389c5fe..a0f7f23c2d 100644
--- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java
+++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java
@@ -55,6 +55,7 @@ public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaSto
 		SqlConverter sqlConverter = new SqlConverter(nodeConversions);
 		ExecutionManager<SqlExecutionResult> executionManager = new SqlExecutionManager(sqlConverter, sqlExecutionService, metaStorage);
 		SqlStorageHandler sqlStorageHandler = new SqlStorageHandler(sqlExecutionService);
+		SqlEntityResolver sqlEntityResolver = new SqlEntityResolver(idColumns, dslContext, sqlDialect, sqlExecutionService);
 
 		return new LocalNamespace(
 				namespaceData.getPreprocessMapper(),
@@ -66,6 +67,7 @@ public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaSto
 				namespaceData.getJobManager(),
 				namespaceData.getFilterSearch(),
 				namespaceData.getIndexService(),
+				sqlEntityResolver,
 				namespaceData.getInjectables()
 		);
 	}
diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java b/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java
new file mode 100644
index 0000000000..681889ec86
--- /dev/null
+++ b/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java
@@ -0,0 +1,206 @@
+package com.bakdata.conquery.mode.local;
+
+import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.collectExtraData;
+import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.readDates;
+import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.verifyOnlySingles;
+import static org.jooq.impl.DSL.field;
+import static org.jooq.impl.DSL.name;
+import static org.jooq.impl.DSL.table;
+import static org.jooq.impl.DSL.val;
+import static org.jooq.impl.DSL.when;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolver;
+import com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil;
+import com.bakdata.conquery.models.common.CDateSet;
+import com.bakdata.conquery.models.config.IdColumnConfig;
+import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap;
+import com.bakdata.conquery.models.identifiable.mapping.ExternalId;
+import com.bakdata.conquery.sql.conversion.SharedAliases;
+import com.bakdata.conquery.sql.conversion.dialect.SqlDialect;
+import com.bakdata.conquery.sql.execution.SqlExecutionService;
+import com.bakdata.conquery.util.DateReader;
+import com.bakdata.conquery.util.io.IdColumnUtil;
+import jakarta.validation.constraints.NotEmpty;
+import lombok.RequiredArgsConstructor;
+import org.jooq.CommonTableExpression;
+import org.jooq.DSLContext;
+import org.jooq.Field;
+import org.jooq.Name;
+import org.jooq.Record;
+import org.jooq.Record2;
+import org.jooq.Record3;
+import org.jooq.Select;
+import org.jooq.SelectConditionStep;
+import org.jooq.Table;
+
+@RequiredArgsConstructor
+public class SqlEntityResolver implements EntityResolver {
+
+	private static final Name IS_RESOLVED_ALIAS = name("is_resolved");
+	private static final Name UNRESOLVED_CTE = name("ids_unresolved");
+	public static final String ROW_INDEX = "rowIndex";
+
+	private final IdColumnConfig idColumns;
+	private final DSLContext context;
+	private final SqlDialect dialect;
+	private final SqlExecutionService executionService;
+
+	@Override
+	public ResolveStatistic resolveEntities(
+			@NotEmpty String[][] values,
+			List<String> format,
+			EntityIdMap mapping,
+			IdColumnConfig idColumnConfig,
+			DateReader dateReader,
+			boolean onlySingles
+	) {
+		final Map<String, CDateSet> resolved = new HashMap<>();
+		final List<String[]> unresolvedDate = new ArrayList<>();
+		final List<String[]> unresolvedId = new ArrayList<>();
+
+		// extract dates from rows
+		final CDateSet[] rowDates = readDates(values, format, dateReader);
+
+		// Extract extra data from rows by Row, to be collected into by entities
+		// Row -> Column -> Value
+		final Map<String, String>[] extraDataByRow = EntityResolverUtil.readExtras(values, format);
+
+		final List<Function<String[], ExternalId>> readers = IdColumnUtil.getIdReaders(format, idColumnConfig.getIdMappers());
+
+		// We will not be able to resolve anything...
+		if (readers.isEmpty()) {
+			return EntityResolver.ResolveStatistic.forEmptyReaders(values);
+		}
+
+		// Entity -> Column -> Values
+		final Map<String, Map<String, List<String>>> extraDataByEntity = new HashMap<>();
+
+		// all IDs of this map had at least a matching reader
+		final Map<Integer, IdResolveInfo> resolvedIdsMap = resolveIds(values, readers);
+
+		// ignore the first row, because this is the header
+		for (int rowNum = 1; rowNum < values.length; rowNum++) {
+
+			final String[] row = values[rowNum];
+
+			final IdResolveInfo idResolveInfo = resolvedIdsMap.get(rowNum);
+			if (idResolveInfo == null) {
+				// row had no matching reader
+				unresolvedId.add(row);
+				continue;
+			}
+
+			// external ID could not be resolved internally
+			if (!idResolveInfo.isResolved()) {
+				unresolvedDate.add(row);
+				continue;
+			}
+
+			final String resolvedId = idResolveInfo.externalId();
+
+			if (rowDates[rowNum] == null) {
+				unresolvedDate.add(row);
+				continue;
+			}
+
+			// read the dates from the row
+			resolved.put(resolvedId, rowDates[rowNum]);
+
+			// Entity was resolved for row, so we collect the data.
+			collectExtraData(extraDataByRow, rowNum, extraDataByEntity, resolvedId);
+		}
+
+		verifyOnlySingles(onlySingles, extraDataByEntity);
+		return new EntityResolver.ResolveStatistic(resolved, extraDataByEntity, unresolvedDate, unresolvedId);
+
+	}
+
+	/**
+	 * Create a SQL query like this
+	 * <pre>
+	 *     {@code
+	 *      with "ids_unresolved" as (select 1   as "row",
+	 *                                      '1'  as "primary_id"
+	 *                                -- will select more ids here via union all)
+	 * 		select "row",
+	 * 		       "primary_id",
+	 * 		       case
+	 * 		           when "entities"."id" is not null then true
+	 * 		           else false
+	 * 		           end as "is_resolved"
+	 * 		from "entities"
+	 * 		         join "ids_unresolved"
+	 * 		              on "primary_id" = "entities"."id"
+	 * 		where "primary_id" = "pid"
+	 *     }
+	 * </pre>
+	 * <p>
+	 * For each ID, that had a matching reader, it will return an entry in the map with row number -> IdResolveInfo.
+	 */
+	private Map<Integer, IdResolveInfo> resolveIds(String[][] values, List<Function<String[], ExternalId>> readers) {
+
+		CommonTableExpression<?> unresolvedCte = createUnresolvedCte(values, readers);
+
+		Field<Integer> rowIndex = field(name(ROW_INDEX), Integer.class);
+		Field<String> externalPrimaryColumn = field(name(SharedAliases.PRIMARY_COLUMN.getAlias()), String.class);
+		Field<String> innerPrimaryColumn = field(name(idColumns.findPrimaryIdColumn().getField()), String.class);
+		Field<Boolean> isResolved = when(innerPrimaryColumn.isNotNull(), val(true))
+				.otherwise(false)
+				.as(IS_RESOLVED_ALIAS);
+
+		Table<Record> allIdsTable = table(name(idColumns.getTable()));
+		SelectConditionStep<Record3<Integer, String, Boolean>> resolveIdsQuery =
+				context.with(unresolvedCte)
+					   .select(rowIndex, externalPrimaryColumn, isResolved)
+					   .from(dialect.getFunctionProvider().innerJoin(allIdsTable, unresolvedCte, List.of(externalPrimaryColumn.eq(innerPrimaryColumn))))
+					   .where(externalPrimaryColumn.eq(innerPrimaryColumn));
+
+		return executionService.fetchStream(resolveIdsQuery)
+							   .collect(Collectors.toMap(
+									   record -> record.get(rowIndex),
+									   record -> new IdResolveInfo(record.get(externalPrimaryColumn), record.get(isResolved))
+							   ));
+	}
+
+	private CommonTableExpression<?> createUnresolvedCte(String[][] values, List<Function<String[], ExternalId>> readers) {
+
+		List<Select<Record2<Integer, String>>> selects = new ArrayList<>(values.length);
+		for (int i = 1; i < values.length; i++) {
+
+			final String[] row = values[i];
+
+			String resolvedId = null;
+			for (Function<String[], ExternalId> reader : readers) {
+				final ExternalId externalId = reader.apply(row);
+				resolvedId = externalId.getId();
+			}
+
+			// no matching reader found
+			if (resolvedId == null) {
+				continue;
+			}
+
+			Field<Integer> rowIndex = val(i).as(ROW_INDEX);
+			Field<String> externalPrimaryColumn = val(resolvedId).as(SharedAliases.PRIMARY_COLUMN.getAlias());
+			Select<Record2<Integer, String>> externalIdSelect = context.select(rowIndex, externalPrimaryColumn)
+																	   // some dialects can't just select static values without FROM clause
+																	   .from(dialect.getFunctionProvider().getNoOpTable());
+
+			selects.add(externalIdSelect);
+		}
+
+		return UNRESOLVED_CTE.as(selects.stream().reduce(Select::unionAll).orElseThrow(IllegalStateException::new));
+	}
+
+
+	private record IdResolveInfo(String externalId, boolean isResolved) {
+	}
+
+}
diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java
index 6c74ce0cc1..a708564a40 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java
@@ -6,6 +6,7 @@
 
 import com.bakdata.conquery.io.jackson.View;
 import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap;
+import com.bakdata.conquery.models.identifiable.mapping.ExternalId;
 import com.bakdata.conquery.resources.admin.rest.AdminDatasetProcessor;
 import com.fasterxml.jackson.annotation.JsonAlias;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
@@ -34,18 +35,18 @@
 @JsonIgnoreProperties(value = {"resolvable"}) // for backwards compatibility
 public class ColumnConfig {
 
-	public EntityIdMap.ExternalId read(String value) {
+	public ExternalId read(String value) {
 		if (Strings.isNullOrEmpty(value)) {
 			return null;
 		}
 
 		if (getLength() == -1 || getPad() == null) {
-			return new EntityIdMap.ExternalId(getName(), value);
+			return new ExternalId(getName(), value);
 		}
 
 		String padded = StringUtils.leftPad(value, getLength(), getPad());
 
-		return new EntityIdMap.ExternalId(getName(), padded);
+		return new ExternalId(getName(), padded);
 
 	}
 
diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java
index 83723c0b4b..52b56dd00c 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java
@@ -172,11 +172,4 @@ public void addInputMapping(String csvEntityId, ExternalId externalEntityId) {
 		}
 	}
 
-	@Data
-	@RequiredArgsConstructor(onConstructor_ = @JsonCreator)
-	public static class ExternalId {
-		private final String type;
-		private final String id;
-	}
-
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/ExternalId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/ExternalId.java
new file mode 100644
index 0000000000..b1f4683797
--- /dev/null
+++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/ExternalId.java
@@ -0,0 +1,12 @@
+package com.bakdata.conquery.models.identifiable.mapping;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
+
+@Data
+@RequiredArgsConstructor(onConstructor_ = @JsonCreator)
+public class ExternalId {
+	private final String type;
+	private final String id;
+}
diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java
index 19ecf66d1e..76a5d4664f 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java
@@ -8,6 +8,7 @@
 
 import com.bakdata.conquery.io.jackson.Injectable;
 import com.bakdata.conquery.io.storage.NamespaceStorage;
+import com.bakdata.conquery.mode.cluster.ClusterEntityResolver;
 import com.bakdata.conquery.models.datasets.Column;
 import com.bakdata.conquery.models.datasets.Dataset;
 import com.bakdata.conquery.models.datasets.concepts.Concept;
@@ -36,7 +37,6 @@ public class DistributedNamespace extends Namespace {
 	private final WorkerHandler workerHandler;
 	private final DistributedExecutionManager executionManager;
 
-
 	public DistributedNamespace(
 			ObjectMapper preprocessMapper,
 			ObjectMapper communicationMapper,
@@ -45,10 +45,11 @@ public DistributedNamespace(
 			JobManager jobManager,
 			FilterSearch filterSearch,
 			IndexService indexService,
+			ClusterEntityResolver clusterEntityResolver,
 			List<Injectable> injectables,
 			WorkerHandler workerHandler
 	) {
-		super(preprocessMapper, communicationMapper, storage, executionManager, jobManager, filterSearch, indexService, injectables);
+		super(preprocessMapper, communicationMapper, storage, executionManager, jobManager, filterSearch, indexService, clusterEntityResolver, injectables);
 		this.executionManager = executionManager;
 		this.workerHandler = workerHandler;
 	}
diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java
index b21aabf5bd..99fb6340a6 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java
@@ -8,6 +8,7 @@
 
 import com.bakdata.conquery.io.jackson.Injectable;
 import com.bakdata.conquery.io.storage.NamespaceStorage;
+import com.bakdata.conquery.mode.local.SqlEntityResolver;
 import com.bakdata.conquery.mode.local.SqlStorageHandler;
 import com.bakdata.conquery.models.datasets.Column;
 import com.bakdata.conquery.models.index.IndexService;
@@ -37,9 +38,10 @@ public LocalNamespace(
 			JobManager jobManager,
 			FilterSearch filterSearch,
 			IndexService indexService,
+			SqlEntityResolver sqlEntityResolver,
 			List<Injectable> injectables
 	) {
-		super(preprocessMapper, communicationMapper, storage, executionManager, jobManager, filterSearch, indexService, injectables);
+		super(preprocessMapper, communicationMapper, storage, executionManager, jobManager, filterSearch, indexService, sqlEntityResolver, injectables);
 		this.dslContextWrapper = dslContextWrapper;
 		this.storageHandler = storageHandler;
 	}
@@ -55,7 +57,8 @@ void registerColumnValuesInSearch(Set<Column> columns) {
 			try {
 				final Stream<String> stringStream = storageHandler.lookupColumnValues(getStorage(), column);
 				getFilterSearch().registerValues(column, stringStream.collect(Collectors.toSet()));
-			}catch (Exception e) {
+			}
+			catch (Exception e) {
 				log.error("Problem collecting column values for {}", column, e);
 			}
 		}
diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java
index 1c952d0de7..dcee848050 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java
@@ -6,6 +6,7 @@
 import java.util.NoSuchElementException;
 import java.util.Set;
 
+import com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolver;
 import com.bakdata.conquery.io.jackson.Injectable;
 import com.bakdata.conquery.io.storage.NamespaceStorage;
 import com.bakdata.conquery.models.datasets.Column;
@@ -49,6 +50,8 @@ public abstract class Namespace extends IdResolveContext {
 
 	private final IndexService indexService;
 
+	private final EntityResolver entityResolver;
+
 	// Jackson's injectables that are available when deserializing requests (see PathParamInjector) or items from the storage
 	private final List<Injectable> injectables;
 
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java b/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java
index b7ea1916d3..a1b2acf495 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java
@@ -5,6 +5,7 @@
 import com.zaxxer.hikari.HikariConfig;
 import com.zaxxer.hikari.HikariDataSource;
 import org.jooq.DSLContext;
+import org.jooq.conf.RenderOptionalKeyword;
 import org.jooq.conf.RenderQuotedNames;
 import org.jooq.conf.Settings;
 import org.jooq.impl.DSL;
@@ -24,7 +25,9 @@ public static DSLContextWrapper create(DatabaseConfig config, SqlConnectorConfig
 				.withRenderFormatted(connectorConfig.isWithPrettyPrinting())
 				// enforces all identifiers to be quoted if not explicitly unquoted via DSL.unquotedName()
 				// to prevent any lowercase/uppercase SQL dialect specific identifier naming issues
-				.withRenderQuotedNames(RenderQuotedNames.EXPLICIT_DEFAULT_QUOTED);
+				.withRenderQuotedNames(RenderQuotedNames.EXPLICIT_DEFAULT_QUOTED)
+				// always render "as" keyword for field aliases
+				.withRenderOptionalAsKeywordForFieldAliases(RenderOptionalKeyword.ON);
 
 		DSLContext dslContext = DSL.using(
 				hikariDataSource,
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java
index a1a3186946..eaa8dfaa86 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java
@@ -22,11 +22,7 @@ public Class<? extends CQYes> getConversionClass() {
 	@Override
 	public ConversionContext convert(CQYes cqYes, ConversionContext context) {
 
-		ColumnConfig primaryColumnConfig = context.getIdColumns().getIds().stream()
-												  .filter(ColumnConfig::isPrimaryId)
-												  .findFirst()
-												  .orElseThrow(() -> new IllegalStateException("SQL mode requires a primary key column in IdColumnConfig"));
-
+		ColumnConfig primaryColumnConfig = context.getIdColumns().findPrimaryIdColumn();
 		Field<Object> primaryColumn = DSL.field(DSL.name(primaryColumnConfig.getField()));
 		SqlIdColumns ids = new SqlIdColumns(primaryColumn);
 
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java
index 2a6edc2333..c660248531 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java
@@ -56,9 +56,10 @@ private static QueryStep.QueryStepBuilder joinWithStratificationTable(
 			List<Condition> conditions,
 			CQTableContext tableContext
 	) {
-		QueryStep stratificationTable = tableContext.getConversionContext().getStratificationTable();
+		QueryStep stratificationTableCte = tableContext.getConversionContext().getStratificationTable();
+		Table<Record> stratificationTable = DSL.table(DSL.name(stratificationTableCte.getCteName()));
 
-		Selects stratificationSelects = stratificationTable.getQualifiedSelects();
+		Selects stratificationSelects = stratificationTableCte.getQualifiedSelects();
 		SqlIdColumns stratificationIds = stratificationSelects.getIds();
 		SqlIdColumns rootTableIds = tableContext.getIds();
 		List<Condition> idConditions = stratificationIds.join(rootTableIds);
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java
index 68dfc0b099..46572215ea 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java
@@ -186,34 +186,16 @@ default Condition in(Field<String> column, String[] values) {
 		return column.in(values);
 	}
 
-	default TableOnConditionStep<Record> innerJoin(
-			Table<Record> leftPartQueryBase,
-			QueryStep rightPartQS,
-			List<Condition> joinConditions
-	) {
-		return leftPartQueryBase
-				.innerJoin(DSL.name(rightPartQS.getCteName()))
-				.on(joinConditions.toArray(Condition[]::new));
+	default TableOnConditionStep<Record> innerJoin(Table<?> leftPart, Table<?> rightPart, List<Condition> joinConditions) {
+		return leftPart.innerJoin(rightPart).on(joinConditions.toArray(Condition[]::new));
 	}
 
-	default TableOnConditionStep<Record> fullOuterJoin(
-			Table<Record> leftPartQueryBase,
-			QueryStep rightPartQS,
-			List<Condition> joinConditions
-	) {
-		return leftPartQueryBase
-				.fullOuterJoin(DSL.name(rightPartQS.getCteName()))
-				.on(joinConditions.toArray(Condition[]::new));
+	default TableOnConditionStep<Record> fullOuterJoin(Table<?> leftPart, Table<?> rightPart, List<Condition> joinConditions) {
+		return leftPart.fullOuterJoin(rightPart).on(joinConditions.toArray(Condition[]::new));
 	}
 
-	default TableOnConditionStep<Record> leftJoin(
-			Table<Record> leftPartQueryBase,
-			QueryStep rightPartQS,
-			List<Condition> joinConditions
-	) {
-		return leftPartQueryBase
-				.leftJoin(DSL.name(rightPartQS.getCteName()))
-				.on(joinConditions.toArray(Condition[]::new));
+	default TableOnConditionStep<Record> leftJoin(Table<?> leftPart, Table<?> rightPart, List<Condition> joinConditions) {
+		return leftPart.leftJoin(rightPart).on(joinConditions.toArray(Condition[]::new));
 	}
 
 	default Field<Date> toDateField(String dateExpression) {
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java
index 0a542da464..d58e373285 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java
@@ -108,7 +108,8 @@ public static TableLike<Record> constructJoinedTable(List<QueryStep> queriesToJo
 
 			List<Condition> joinConditions = Stream.concat(joinIdsCondition.stream(), Stream.of(joinDateCondition)).collect(Collectors.toList());
 
-			joinedQuery = joinType.join(joinedQuery, rightPartQS, joinConditions);
+			Table<Record> rightPartTable = DSL.table(DSL.name(rightPartQS.getCteName()));
+			joinedQuery = joinType.join(joinedQuery, rightPartTable, joinConditions);
 		}
 
 		return joinedQuery;
@@ -170,11 +171,7 @@ private static QueryStep buildStepAndAggregateDates(
 
 	@FunctionalInterface
 	private interface JoinType {
-		TableOnConditionStep<Record> join(
-				Table<Record> leftPartQueryBase,
-				QueryStep rightPartQS,
-				List<Condition> joinConditions
-		);
+		TableOnConditionStep<Record> join(Table<?> leftPart, Table<?> rightPart, List<Condition> joinConditions);
 	}
 
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java
index ce433e53b9..5b2b42d092 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java
@@ -199,8 +199,9 @@ private static Table<Record> joinConnectorTableWithPrerequisite(
 	) {
 		SqlFunctionProvider functionProvider = context.getSqlDialect().getFunctionProvider();
 		Table<Record> connectorTable = DSL.table(DSL.name(cqTable.getConnector().getTable().getName()));
+		Table<Record> convertedPrerequisiteTable = DSL.table(DSL.name(convertedPrerequisite.getCteName()));
 		List<Condition> joinOnIds = ids.join(convertedPrerequisite.getQualifiedSelects().getIds());
-		return functionProvider.innerJoin(connectorTable, convertedPrerequisite, joinOnIds);
+		return functionProvider.innerJoin(connectorTable, convertedPrerequisiteTable, joinOnIds);
 	}
 
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/util/io/IdColumnUtil.java b/backend/src/main/java/com/bakdata/conquery/util/io/IdColumnUtil.java
index 3db0a88d50..3f1716a3f6 100644
--- a/backend/src/main/java/com/bakdata/conquery/util/io/IdColumnUtil.java
+++ b/backend/src/main/java/com/bakdata/conquery/util/io/IdColumnUtil.java
@@ -11,7 +11,7 @@
 import com.bakdata.conquery.models.config.ColumnConfig;
 import com.bakdata.conquery.models.execution.ManagedExecution;
 import com.bakdata.conquery.models.identifiable.mapping.AutoIncrementingPseudomizer;
-import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap;
+import com.bakdata.conquery.models.identifiable.mapping.ExternalId;
 import com.bakdata.conquery.models.identifiable.mapping.FullIdPrinter;
 import com.bakdata.conquery.models.identifiable.mapping.IdPrinter;
 import com.bakdata.conquery.models.worker.Namespace;
@@ -23,8 +23,8 @@ public class IdColumnUtil {
 	/**
 	 * If a column contains an ID, create a reader for that ID.
 	 */
-	public static List<Function<String[], EntityIdMap.ExternalId>> getIdReaders(List<String> format, Map<String, ColumnConfig> idMappers) {
-		List<Function<String[], EntityIdMap.ExternalId>> out = new ArrayList<>(format.size());
+	public static List<Function<String[], ExternalId>> getIdReaders(List<String> format, Map<String, ColumnConfig> idMappers) {
+		List<Function<String[], ExternalId>> out = new ArrayList<>(format.size());
 
 		for (int index = 0; index < format.size(); index++) {
 			final ColumnConfig mapper = idMappers.get(format.get(index));
diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java b/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java
index e96c80c276..68c9b70818 100644
--- a/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java
+++ b/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java
@@ -3,6 +3,8 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
+import java.util.Map;
 import java.util.function.Consumer;
 import javax.annotation.Nullable;
 
@@ -10,6 +12,8 @@
 import com.bakdata.conquery.io.cps.CPSBase;
 import com.bakdata.conquery.io.jackson.Jackson;
 import com.bakdata.conquery.io.jackson.View;
+import com.bakdata.conquery.io.storage.MetaStorage;
+import com.bakdata.conquery.models.config.ColumnConfig;
 import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.config.Dialect;
 import com.bakdata.conquery.models.config.IdColumnConfig;
@@ -51,8 +55,18 @@ public abstract class ConqueryTestSpec {
 	@Nullable
 	SqlSpec sqlSpec;
 
-	@Nullable
-	private IdColumnConfig idColumns;
+	// default IdColumnConfig for SQL mode
+	private IdColumnConfig idColumns = new IdColumnConfig(
+			"entities",
+			List.of(
+					ColumnConfig.builder()
+								.name("ID")
+								.field("pid")
+								.primaryId(true)
+								.label(Map.of(Locale.ROOT, "result"))
+								.build()
+			)
+	);
 
 	public ConqueryConfig overrideConfig(ConqueryConfig config) {
 
diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java b/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java
index 21a37e7267..085a3ce853 100644
--- a/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java
+++ b/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java
@@ -17,17 +17,21 @@
 import com.bakdata.conquery.util.support.StandaloneSupport;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @RequiredArgsConstructor
 public class SqlTestDataImporter implements TestDataImporter {
 
+	private static final RequiredTable ALL_IDS_TABLE = importRequiredTable("/shared/entities.table.json");
+
 	private final CsvTableImporter csvTableImporter;
 
 	@Override
 	public void importQueryTestData(StandaloneSupport support, QueryTest test) throws Exception {
 		RequiredData content = test.getContent();
+		content.getTables().add(ALL_IDS_TABLE);
 		importSecondaryIds(support, content.getSecondaryIds());
 		importTables(support, content.getTables(), true);
 		importConcepts(support, test.getRawConcepts());
@@ -37,6 +41,7 @@ public void importQueryTestData(StandaloneSupport support, QueryTest test) throw
 	@Override
 	public void importFormTestData(StandaloneSupport support, FormTest test) throws Exception {
 		RequiredData content = test.getContent();
+		content.getTables().add(ALL_IDS_TABLE);
 		importSecondaryIds(support, content.getSecondaryIds());
 		importTables(support, content.getTables(), true);
 		importConcepts(support, test.getRawConcepts());
@@ -83,4 +88,9 @@ public void importTableContents(StandaloneSupport support, Collection<RequiredTa
 		}
 	}
 
+	@SneakyThrows
+	private static RequiredTable importRequiredTable(String fileResource) {
+		return RequiredTable.fromFile(fileResource);
+	}
+
 }
diff --git a/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java b/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java
index 42f895727c..ed9e8d76b9 100644
--- a/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java
+++ b/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java
@@ -2,13 +2,14 @@
 
 import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap;
 import com.bakdata.conquery.models.identifiable.mapping.EntityPrintId;
+import com.bakdata.conquery.models.identifiable.mapping.ExternalId;
 
 public class IdMapSerialisationTest {
 
 	public static EntityIdMap createTestPersistentMap() {
 		EntityIdMap entityIdMap = new EntityIdMap();
 
-		entityIdMap.addInputMapping("test1", new EntityIdMap.ExternalId("id", "a"));
+		entityIdMap.addInputMapping("test1", new ExternalId("id", "a"));
 
 		entityIdMap.addOutputMapping("test2", EntityPrintId.from("c"));
 
diff --git a/backend/src/test/resources/tests/sql/yes/persons.csv b/backend/src/test/resources/shared/entities.csv
similarity index 86%
rename from backend/src/test/resources/tests/sql/yes/persons.csv
rename to backend/src/test/resources/shared/entities.csv
index 5499484b3d..514b00398a 100644
--- a/backend/src/test/resources/tests/sql/yes/persons.csv
+++ b/backend/src/test/resources/shared/entities.csv
@@ -1,7 +1,8 @@
 pid,datum
-4,2013-11-10
-5,2013-11-10
-6,2013-11-10
 1,2012-01-01
 2,2013-11-10
 3,2013-11-10
+4,2013-11-10
+5,2013-11-10
+6,2013-11-10
+23,2014-01-01
diff --git a/backend/src/test/resources/shared/entities.table.json b/backend/src/test/resources/shared/entities.table.json
new file mode 100644
index 0000000000..c6a2561f58
--- /dev/null
+++ b/backend/src/test/resources/shared/entities.table.json
@@ -0,0 +1,14 @@
+{
+  "csv": "shared/entities.csv",
+  "name": "entities",
+  "primaryColumn": {
+    "name": "pid",
+    "type": "STRING"
+  },
+  "columns": [
+    {
+      "name": "datum",
+      "type": "DATE"
+    }
+  ]
+}
diff --git a/backend/src/test/resources/tests/sql/yes/CQYES.test.json b/backend/src/test/resources/tests/sql/yes/cqyes.json
similarity index 54%
rename from backend/src/test/resources/tests/sql/yes/CQYES.test.json
rename to backend/src/test/resources/tests/sql/yes/cqyes.json
index e5b9e5fe3b..e5d28e42d0 100644
--- a/backend/src/test/resources/tests/sql/yes/CQYES.test.json
+++ b/backend/src/test/resources/tests/sql/yes/cqyes.json
@@ -10,28 +10,11 @@
   },
   "concepts": [
   ],
-  "idColumns": {
-    "table": "persons",
-    "ids": [
-      {
-        "name": "PID",
-        "field": "pid",
-        "primaryId": true,
-        "label": {
-          "de": "PID"
-        },
-        "description": {
-          "en": "Pid of an Entity.",
-          "de": "Pid einer Entität."
-        }
-      }
-    ]
-  },
   "content": {
     "tables": [
       {
-        "csv": "tests/sql/yes/persons.csv",
-        "name": "persons",
+        "csv": "tests/sql/yes/no-op.csv",
+        "name": "no-op",
         "primaryColumn": {
           "name": "pid",
           "type": "STRING"
diff --git a/backend/src/test/resources/tests/sql/yes/expected.csv b/backend/src/test/resources/tests/sql/yes/expected.csv
index edf163f57b..0a872351ef 100644
--- a/backend/src/test/resources/tests/sql/yes/expected.csv
+++ b/backend/src/test/resources/tests/sql/yes/expected.csv
@@ -1,7 +1,8 @@
-PID,dates
-4,{}
-5,{}
-6,{}
+result,dates
 1,{}
 2,{}
+23,{}
 3,{}
+4,{}
+5,{}
+6,{}
diff --git a/backend/src/test/resources/tests/sql/yes/no-op.csv b/backend/src/test/resources/tests/sql/yes/no-op.csv
new file mode 100644
index 0000000000..d9a4329883
--- /dev/null
+++ b/backend/src/test/resources/tests/sql/yes/no-op.csv
@@ -0,0 +1,2 @@
+pid,datum
+irrelevant_id_for_cqyes,2013-11-10

From 58d3bcaa9f49219ab02a098a8aff6acde5645322 Mon Sep 17 00:00:00 2001
From: awildturtok <1553491+awildturtok@users.noreply.github.com>
Date: Wed, 24 Jul 2024 16:24:24 +0200
Subject: [PATCH 25/41] remove ResultType CPSType usage. Simply inline the
 type.

---
 .../conquery/models/types/ResultType.java     | 152 +++++++++---------
 1 file changed, 79 insertions(+), 73 deletions(-)

diff --git a/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java b/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java
index 02c82abfe1..52ccbfbea9 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java
@@ -11,15 +11,12 @@
 
 import c10n.C10N;
 import com.bakdata.conquery.internationalization.Results;
-import com.bakdata.conquery.io.cps.CPSBase;
-import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.models.common.daterange.CDateRange;
 import com.bakdata.conquery.models.config.LocaleConfig;
 import com.bakdata.conquery.models.events.MajorTypeId;
 import com.bakdata.conquery.models.query.PrintSettings;
 import com.bakdata.conquery.sql.execution.ResultSetProcessor;
 import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import com.google.common.base.Preconditions;
 import lombok.AccessLevel;
 import lombok.EqualsAndHashCode;
@@ -29,28 +26,9 @@
 import lombok.extern.slf4j.Slf4j;
 import org.jetbrains.annotations.NotNull;
 
-@JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "type")
-@CPSBase
 @Slf4j
 public abstract class ResultType<T> {
 
-	public String printNullable(PrintSettings cfg, Object f) {
-		if (f == null) {
-			return "";
-		}
-		return print(cfg, f);
-	}
-
-	protected String print(PrintSettings cfg, @NonNull Object f) {
-		return f.toString();
-	}
-
-	public abstract String typeInfo();
-
-	public abstract T getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException;
-
-	protected abstract List<T> getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException;
-
 	public static ResultType<?> resolveResultType(MajorTypeId majorTypeId) {
 		return switch (majorTypeId) {
 			case STRING -> StringT.INSTANCE;
@@ -63,21 +41,27 @@ public static ResultType<?> resolveResultType(MajorTypeId majorTypeId) {
 		};
 	}
 
-	abstract static class PrimitiveResultType<T> extends ResultType<T> {
-		@Override
-		public String typeInfo() {
-			return this.getClass().getAnnotation(CPSType.class).id();
+	public String printNullable(PrintSettings cfg, Object f) {
+		if (f == null) {
+			return "";
 		}
+		return print(cfg, f);
+	}
 
-		@Override
-		public String toString() {
-			return typeInfo();
-		}
+	protected abstract String print(PrintSettings cfg, @NonNull Object f);
+
+	public String toString() {
+		return typeInfo();
 	}
 
-	@CPSType(id = "BOOLEAN", base = ResultType.class)
+	public abstract String typeInfo();
+
+	public abstract T getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException;
+
+	protected abstract List<T> getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException;
+
 	@NoArgsConstructor(access = AccessLevel.PRIVATE)
-	public static class BooleanT extends PrimitiveResultType<Boolean> {
+	public static class BooleanT extends ResultType<Boolean> {
 		@Getter(onMethod_ = @JsonCreator)
 		public static final BooleanT INSTANCE = new BooleanT();
 
@@ -93,6 +77,11 @@ public String print(PrintSettings cfg, Object f) {
 			return (Boolean) f ? "1" : "0";
 		}
 
+		@Override
+		public String typeInfo() {
+			return "BOOLEAN";
+		}
+
 		@Override
 		public Boolean getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
 			return resultSetProcessor.getBoolean(resultSet, columnIndex);
@@ -105,9 +94,8 @@ protected List<Boolean> getFromResultSetAsList(ResultSet resultSet, int columnIn
 	}
 
 
-	@CPSType(id = "INTEGER", base = ResultType.class)
 	@NoArgsConstructor(access = AccessLevel.PRIVATE)
-	public static class IntegerT extends PrimitiveResultType<Integer> {
+	public static class IntegerT extends ResultType<Integer> {
 		@Getter(onMethod_ = @JsonCreator)
 		public static final IntegerT INSTANCE = new IntegerT();
 
@@ -119,6 +107,11 @@ public String print(PrintSettings cfg, Object f) {
 			return f.toString();
 		}
 
+		@Override
+		public String typeInfo() {
+			return "INTEGER";
+		}
+
 		@Override
 		public Integer getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
 			return resultSetProcessor.getInteger(resultSet, columnIndex);
@@ -130,9 +123,8 @@ protected List<Integer> getFromResultSetAsList(ResultSet resultSet, int columnIn
 		}
 	}
 
-	@CPSType(id = "NUMERIC", base = ResultType.class)
 	@NoArgsConstructor(access = AccessLevel.PRIVATE)
-	public static class NumericT extends PrimitiveResultType<Double> {
+	public static class NumericT extends ResultType<Double> {
 		@Getter(onMethod_ = @JsonCreator)
 		public static final NumericT INSTANCE = new NumericT();
 
@@ -144,6 +136,11 @@ public String print(PrintSettings cfg, Object f) {
 			return f.toString();
 		}
 
+		@Override
+		public String typeInfo() {
+			return "NUMERIC";
+		}
+
 		@Override
 		public Double getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
 			return resultSetProcessor.getDouble(resultSet, columnIndex);
@@ -155,9 +152,8 @@ protected List<Double> getFromResultSetAsList(ResultSet resultSet, int columnInd
 		}
 	}
 
-	@CPSType(id = "DATE", base = ResultType.class)
 	@NoArgsConstructor(access = AccessLevel.PRIVATE)
-	public static class DateT extends PrimitiveResultType<Number> {
+	public static class DateT extends ResultType<Number> {
 		@Getter(onMethod_ = @JsonCreator)
 		public static final DateT INSTANCE = new DateT();
 
@@ -170,6 +166,15 @@ public String print(PrintSettings cfg, @NonNull Object f) {
 			return print(number, cfg.getDateFormatter());
 		}
 
+		public static String print(Number num, DateTimeFormatter formatter) {
+			return formatter.format(LocalDate.ofEpochDay(num.intValue()));
+		}
+
+		@Override
+		public String typeInfo() {
+			return "DATE";
+		}
+
 		@Override
 		public Number getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
 			return resultSetProcessor.getDate(resultSet, columnIndex);
@@ -179,49 +184,49 @@ public Number getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetPr
 		protected List<Number> getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
 			return resultSetProcessor.getDateList(resultSet, columnIndex);
 		}
-
-		public static String print(Number num, DateTimeFormatter formatter) {
-			return formatter.format(LocalDate.ofEpochDay(num.intValue()));
-		}
 	}
 
 	/**
 	 * A DateRange is provided by in a query result as two ints in a list, both standing for an epoch day (see {@link LocalDate#toEpochDay()}).
 	 * The first int describes the included lower bound of the range. The second int descibes the included upper bound.
 	 */
-	@CPSType(id = "DATE_RANGE", base = ResultType.class)
 	@NoArgsConstructor(access = AccessLevel.PRIVATE)
-	public static class DateRangeT extends PrimitiveResultType<List<Integer>> {
+	public static class DateRangeT extends ResultType<List<Integer>> {
 		@Getter(onMethod_ = @JsonCreator)
 		public static final DateRangeT INSTANCE = new DateRangeT();
 
 		@Override
 		public String print(PrintSettings cfg, @NonNull Object f) {
-			if (!(f instanceof List)) {
-				throw new IllegalStateException(String.format("Expected a List got %s (Type: %s, as string: %s)", f, f.getClass().getName(), f));
-			}
-			List<?> list = (List<?>) f;
-			if (list.size() != 2) {
-				throw new IllegalStateException("Expected a list with 2 elements, one min, one max. The list was: " + list);
-			}
+			Preconditions.checkArgument(f instanceof List<?>, "Expected a List got %s (Type: %s, as string: %s)", f, f.getClass().getName(), f);
+
+			final List<?> list = (List<?>) f;
+
+			Preconditions.checkArgument(list.size() == 2, "Expected a list with 2 elements: one min, one max. The list was: %s", list);
+
 			final DateTimeFormatter dateFormat = cfg.getDateFormatter();
 			final Integer min = (Integer) list.get(0);
 			final Integer max = (Integer) list.get(1);
+
 			if (min == null || max == null) {
 				log.warn("Encountered incomplete range, treating it as an open range. Either min or max was null: {}", list);
 			}
 			// Compute minString first because we need it either way
-			String minString = min == null || min == CDateRange.NEGATIVE_INFINITY ? "-∞" : ResultType.DateT.print(min, dateFormat);
+			final String minString = min == null || min == CDateRange.NEGATIVE_INFINITY ? "-∞" : ResultType.DateT.print(min, dateFormat);
 
 			if (cfg.isPrettyPrint() && min != null && min.equals(max)) {
 				// If the min and max are the same we print it like a singe date, not a range (only in pretty printing)
 				return minString;
 			}
-			String maxString = max == null || max == CDateRange.POSITIVE_INFINITY ? "+∞" : ResultType.DateT.print(max, dateFormat);
+			final String maxString = max == null || max == CDateRange.POSITIVE_INFINITY ? "+∞" : ResultType.DateT.print(max, dateFormat);
 
 			return minString + cfg.getDateRangeSeparator() + maxString;
 		}
 
+		@Override
+		public String typeInfo() {
+			return "DATE_RANGE";
+		}
+
 		@Override
 		public List<Integer> getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
 			return resultSetProcessor.getDateRange(resultSet, columnIndex);
@@ -233,9 +238,8 @@ public List<List<Integer>> getFromResultSetAsList(ResultSet resultSet, int colum
 		}
 	}
 
-	@CPSType(id = "STRING", base = ResultType.class)
 	@NoArgsConstructor(access = AccessLevel.PRIVATE)
-	public static class StringT extends PrimitiveResultType<String> {
+	public static class StringT extends ResultType<String> {
 		@Getter(onMethod_ = @JsonCreator)
 		public static final StringT INSTANCE = new StringT();
 
@@ -253,9 +257,14 @@ public StringT(BiFunction<Object, PrintSettings, String> valueMapper) {
 		@Override
 		protected String print(PrintSettings cfg, @NonNull Object f) {
 			if (valueMapper == null) {
-				return super.print(cfg, f);
+				return f.toString();
 			}
-			return super.print(cfg, valueMapper.apply(f, cfg));
+			return valueMapper.apply(f, cfg);
+		}
+
+		@Override
+		public String typeInfo() {
+			return "STRING";
 		}
 
 		@Override
@@ -269,9 +278,8 @@ protected List<String> getFromResultSetAsList(ResultSet resultSet, int columnInd
 		}
 	}
 
-	@CPSType(id = "MONEY", base = ResultType.class)
 	@NoArgsConstructor(access = AccessLevel.PRIVATE)
-	public static class MoneyT extends PrimitiveResultType<BigDecimal> {
+	public static class MoneyT extends ResultType<BigDecimal> {
 
 		@Getter(onMethod_ = @JsonCreator)
 		public static final MoneyT INSTANCE = new MoneyT();
@@ -284,24 +292,27 @@ public String print(PrintSettings cfg, Object f) {
 			return IntegerT.INSTANCE.print(cfg, f);
 		}
 
-		@Override
-		public BigDecimal getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
-			return resultSetProcessor.getMoney(resultSet, columnIndex);
-		}
-
-
 		@NotNull
 		public BigDecimal readIntermediateValue(PrintSettings cfg, Number f) {
 			return new BigDecimal(f.longValue()).movePointLeft(cfg.getCurrency().getDefaultFractionDigits());
 		}
 
+		@Override
+		public String typeInfo() {
+			return "MONEY";
+		}
+
+		@Override
+		public BigDecimal getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
+			return resultSetProcessor.getMoney(resultSet, columnIndex);
+		}
+
 		@Override
 		protected List<BigDecimal> getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
 			return resultSetProcessor.getMoneyList(resultSet, columnIndex);
 		}
 	}
 
-	@CPSType(id = "LIST", base = ResultType.class)
 	@Getter
 	@EqualsAndHashCode(callSuper = false)
 	public static class ListT<T> extends ResultType<List<T>> {
@@ -322,7 +333,7 @@ public String print(PrintSettings cfg, @NonNull Object f) {
 			}
 			// Not sure if this escaping is enough
 			final LocaleConfig.ListFormat listFormat = cfg.getListFormat();
-			StringJoiner joiner = listFormat.createListJoiner();
+			final StringJoiner joiner = listFormat.createListJoiner();
 			for (Object obj : (List<?>) f) {
 				joiner.add(listFormat.escapeListElement(elementType.print(cfg, obj)));
 			}
@@ -331,7 +342,7 @@ public String print(PrintSettings cfg, @NonNull Object f) {
 
 		@Override
 		public String typeInfo() {
-			return this.getClass().getAnnotation(CPSType.class).id() + "[" + elementType.typeInfo() + "]";
+			return "LIST[" + elementType.typeInfo() + "]";
 		}
 
 		@Override
@@ -343,10 +354,5 @@ public List<T> getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetP
 		protected List<List<T>> getFromResultSetAsList(final ResultSet resultSet, final int columnIndex, final ResultSetProcessor resultSetProcessor) {
 			throw new UnsupportedOperationException("Nested lists not supported in SQL mode");
 		}
-
-		@Override
-		public String toString() {
-			return typeInfo();
-		}
 	}
 }

From a2aad8a29ceb5de9fa62fbfd26961780f6c56092 Mon Sep 17 00:00:00 2001
From: awildturtok <1553491+awildturtok@users.noreply.github.com>
Date: Thu, 25 Jul 2024 10:21:03 +0200
Subject: [PATCH 26/41] use typeInfo instead of serializer magic

---
 .../apiv1/frontend/FrontendSelect.java        |  3 +--
 .../concepts/FrontEndConceptBuilder.java      | 19 ++++++++++++++++---
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendSelect.java b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendSelect.java
index b698e7b26a..95f9e1c94b 100644
--- a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendSelect.java
+++ b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendSelect.java
@@ -1,7 +1,6 @@
 package com.bakdata.conquery.apiv1.frontend;
 
 import com.bakdata.conquery.models.identifiable.ids.specific.SelectId;
-import com.bakdata.conquery.models.types.ResultType;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import lombok.Builder;
 import lombok.Data;
@@ -15,7 +14,7 @@ public class FrontendSelect {
 	private SelectId id;
 	private String label;
 	private String description;
-	private ResultType resultType;
+	private String resultType;
 	@JsonProperty("default")
 	private Boolean isDefault;
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java
index 07e3dd23b8..721fc2c920 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java
@@ -1,10 +1,23 @@
 package com.bakdata.conquery.models.datasets.concepts;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import com.bakdata.conquery.apiv1.frontend.*;
+import com.bakdata.conquery.apiv1.frontend.FrontendFilterConfiguration;
+import com.bakdata.conquery.apiv1.frontend.FrontendList;
+import com.bakdata.conquery.apiv1.frontend.FrontendNode;
+import com.bakdata.conquery.apiv1.frontend.FrontendRoot;
+import com.bakdata.conquery.apiv1.frontend.FrontendSecondaryId;
+import com.bakdata.conquery.apiv1.frontend.FrontendSelect;
+import com.bakdata.conquery.apiv1.frontend.FrontendTable;
+import com.bakdata.conquery.apiv1.frontend.FrontendValidityDate;
+import com.bakdata.conquery.apiv1.frontend.FrontendValue;
 import com.bakdata.conquery.io.storage.NamespaceStorage;
 import com.bakdata.conquery.models.auth.entities.Subject;
 import com.bakdata.conquery.models.auth.permissions.Ability;
@@ -159,7 +172,7 @@ public FrontendSelect createSelect(Select select) {
 							 .id(select.getId())
 							 .label(select.getLabel())
 							 .description(select.getDescription())
-							 .resultType(select.getResultType())
+							 .resultType(select.getResultType().typeInfo())
 							 .isDefault(select.isDefault())
 							 .build();
 	}

From 12b0497ef90ad4f97b071fdd0c3377bcf9ff6934 Mon Sep 17 00:00:00 2001
From: Jonas Arnhold <jonas.arnhold@bakdata.com>
Date: Thu, 25 Jul 2024 11:48:23 +0200
Subject: [PATCH 27/41] Implement external ID lookup for SQL mode (#3496)

---
 .../conquery/apiv1/QueryProcessor.java        |   7 +-
 .../concept/specific/external/CQExternal.java | 252 +-----------------
 .../specific/external/EntityResolver.java     |  51 ++++
 .../specific/external/EntityResolverUtil.java | 179 +++++++++++++
 .../mode/cluster/ClusterEntityResolver.java   |  83 ++++++
 .../mode/cluster/ClusterNamespaceHandler.java |   1 +
 .../mode/local/LocalNamespaceHandler.java     |   2 +
 .../mode/local/SqlEntityResolver.java         | 206 ++++++++++++++
 .../conquery/models/config/ColumnConfig.java  |   7 +-
 .../identifiable/mapping/EntityIdMap.java     |   7 -
 .../identifiable/mapping/ExternalId.java      |  12 +
 .../models/worker/DistributedNamespace.java   |   5 +-
 .../models/worker/LocalNamespace.java         |   7 +-
 .../conquery/models/worker/Namespace.java     |   3 +
 .../conquery/sql/DslContextFactory.java       |   5 +-
 .../conversion/cqelement/CQYesConverter.java  |   6 +-
 .../cqelement/concept/PreprocessingCte.java   |   5 +-
 .../dialect/SqlFunctionProvider.java          |  30 +--
 .../sql/conversion/model/QueryStepJoiner.java |   9 +-
 .../query/TableExportQueryConverter.java      |   3 +-
 .../conquery/util/io/IdColumnUtil.java        |   6 +-
 .../integration/json/ConqueryTestSpec.java    |  18 +-
 .../integration/json/SqlTestDataImporter.java |  10 +
 .../identifiable/IdMapSerialisationTest.java  |   3 +-
 .../yes/persons.csv => shared/entities.csv}   |   7 +-
 .../test/resources/shared/entities.table.json |  14 +
 .../sql/yes/{CQYES.test.json => cqyes.json}   |  21 +-
 .../test/resources/tests/sql/yes/expected.csv |   9 +-
 .../test/resources/tests/sql/yes/no-op.csv    |   2 +
 29 files changed, 639 insertions(+), 331 deletions(-)
 create mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolver.java
 create mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolverUtil.java
 create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterEntityResolver.java
 create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java
 create mode 100644 backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/ExternalId.java
 rename backend/src/test/resources/{tests/sql/yes/persons.csv => shared/entities.csv} (86%)
 create mode 100644 backend/src/test/resources/shared/entities.table.json
 rename backend/src/test/resources/tests/sql/yes/{CQYES.test.json => cqyes.json} (54%)
 create mode 100644 backend/src/test/resources/tests/sql/yes/no-op.csv

diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java
index 1dbb9753d1..92d2b9b463 100644
--- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java
+++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java
@@ -39,6 +39,7 @@
 import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept;
 import com.bakdata.conquery.apiv1.query.concept.specific.CQOr;
 import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal;
+import com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolver;
 import com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider;
 import com.bakdata.conquery.io.storage.MetaStorage;
 import com.bakdata.conquery.metrics.ExecutionMetrics;
@@ -108,7 +109,6 @@ public class QueryProcessor {
 	private Validator validator;
 
 
-
 	public Stream<ExecutionStatus> getAllQueries(Dataset dataset, HttpServletRequest req, Subject subject, boolean allProviders) {
 		final Collection<ManagedExecution> allQueries = storage.getAllExecutions();
 
@@ -294,14 +294,13 @@ public FullExecutionStatus getQueryFullStatus(ManagedExecution query, Subject su
 	public ExternalUploadResult uploadEntities(Subject subject, Dataset dataset, ExternalUpload upload) {
 
 		final Namespace namespace = datasetRegistry.get(dataset.getId());
-		final CQExternal.ResolveStatistic statistic = CQExternal.resolveEntities(
+		final EntityResolver.ResolveStatistic statistic = namespace.getEntityResolver().resolveEntities(
 				upload.getValues(),
 				upload.getFormat(),
 				namespace.getStorage().getIdMapping(),
 				config.getIdColumns(),
 				config.getLocale().getDateReader(),
-				upload.isOneRowPerEntity(),
-				true
+				upload.isOneRowPerEntity()
 		);
 
 		// Resolving nothing is a problem thus we fail.
diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java
index 48466e2dd8..2ba0d02a0b 100644
--- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java
+++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java
@@ -3,7 +3,6 @@
 
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -11,7 +10,6 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
-import java.util.function.Function;
 import java.util.stream.Collectors;
 
 import com.bakdata.conquery.apiv1.query.CQElement;
@@ -21,7 +19,6 @@
 import com.bakdata.conquery.models.config.IdColumnConfig;
 import com.bakdata.conquery.models.error.ConqueryError;
 import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId;
-import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap;
 import com.bakdata.conquery.models.query.QueryExecutionContext;
 import com.bakdata.conquery.models.query.QueryPlanContext;
 import com.bakdata.conquery.models.query.QueryResolveContext;
@@ -33,16 +30,12 @@
 import com.bakdata.conquery.models.query.resultinfo.ResultInfo;
 import com.bakdata.conquery.models.query.resultinfo.SimpleResultInfo;
 import com.bakdata.conquery.models.types.ResultType;
-import com.bakdata.conquery.util.DateReader;
-import com.bakdata.conquery.util.io.IdColumnUtil;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonView;
 import com.google.common.collect.Streams;
 import io.dropwizard.validation.ValidationMethod;
 import jakarta.validation.constraints.NotEmpty;
-import jakarta.validation.constraints.NotNull;
 import lombok.AccessLevel;
-import lombok.Data;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -55,8 +48,6 @@
 @NoArgsConstructor
 public class CQExternal extends CQElement {
 
-	private static final String FORMAT_EXTRA = "EXTRA";
-
 	/**
 	 * Describes the format of {@code values}, how to extract data from each row:
 	 * <p>
@@ -118,7 +109,7 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) {
 		final String[] extraHeaders = Streams.zip(
 													 Arrays.stream(headers),
 													 format.stream(),
-													 (header, format) -> format.equals(FORMAT_EXTRA) ? header : null
+													 (header, format) -> format.equals(EntityResolverUtil.FORMAT_EXTRA) ? header : null
 											 )
 											 .filter(Objects::nonNull)
 											 .toArray(String[]::new);
@@ -180,78 +171,18 @@ private ExternalNode<List<String>> createExternalNodeForList(QueryPlanContext co
 		);
 	}
 
-	/**
-	 * For each row try and collect all dates.
-	 *
-	 * @return Row -> Dates
-	 */
-	private static CDateSet[] readDates(String[][] values, List<String> format, DateReader dateReader) {
-		final CDateSet[] out = new CDateSet[values.length];
-
-		final List<DateFormat> dateFormats = format.stream()
-												   .map(CQExternal::resolveDateFormat)
-												   // Don't use Stream#toList to preserve null-values
-												   .collect(Collectors.toList());
-
-
-		/*
-		 If no format is provided, put empty dates into output.
-		 This indicates that no date context was provided and
-		 the entries are not restricted by any date restriction,
-		 but can also don't contribute to any date aggregation.
-		 */
-		if (dateFormats.stream().allMatch(Objects::isNull)) {
-			// Initialize empty
-			for (int row = 0; row < values.length; row++) {
-				out[row] = CDateSet.createEmpty();
-			}
-			return out;
-		}
-
-		for (int row = 1; row < values.length; row++) {
-			try {
-				final CDateSet dates = CDateSet.createEmpty();
-
-				// Collect all specified dates into a single set.
-				for (int col = 0; col < dateFormats.size(); col++) {
-					final DateFormat dateFormat = dateFormats.get(col);
-
-					if (dateFormat == null) {
-						continue;
-					}
-					dateFormat.readDates(values[row][col], dateReader, dates);
-				}
-
-				if (dates.isEmpty()) {
-					continue;
-				}
-
-				if (out[row] == null) {
-					out[row] = CDateSet.createEmpty();
-				}
-
-				out[row].addAll(dates);
-			}
-			catch (Exception e) {
-				log.warn("Failed to parse Date from {}", row, e);
-			}
-		}
-
-		return out;
-	}
-
 	@Override
 	public void resolve(QueryResolveContext context) {
 		headers = values[0];
 
-		final ResolveStatistic resolved =
-				resolveEntities(values, format,
-								context.getNamespace().getStorage().getIdMapping(),
-								context.getConfig().getIdColumns(),
-								context.getConfig().getLocale().getDateReader(),
-								onlySingles,
-								context.getConfig().getSqlConnectorConfig().isEnabled()
-				);
+		final EntityResolver.ResolveStatistic resolved = context.getNamespace().getEntityResolver().resolveEntities(
+				values,
+				format,
+				context.getNamespace().getStorage().getIdMapping(),
+				context.getConfig().getIdColumns(),
+				context.getConfig().getLocale().getDateReader(),
+				onlySingles
+		);
 
 		if (resolved.getResolved().isEmpty()) {
 			throw new ConqueryError.ExternalResolveEmptyError();
@@ -277,158 +208,6 @@ public void resolve(QueryResolveContext context) {
 		extra = resolved.getExtra();
 	}
 
-	@Data
-	public static class ResolveStatistic {
-
-		@JsonIgnore
-		private final Map<String, CDateSet> resolved;
-
-		/**
-		 * Entity -> Column -> Values
-		 */
-		@JsonIgnore
-		private final Map<String, Map<String, List<String>>> extra;
-
-		private final List<String[]> unreadableDate;
-		private final List<String[]> unresolvedId;
-
-	}
-
-	/**
-	 * Helper method to try and resolve entities in values using the specified format.
-	 */
-	public static ResolveStatistic resolveEntities(@NotEmpty String[][] values, @NotEmpty List<String> format, EntityIdMap mapping, IdColumnConfig idColumnConfig, @NotNull DateReader dateReader, boolean onlySingles, boolean isInSqlMode) {
-		final Map<String, CDateSet> resolved = new HashMap<>();
-
-		final List<String[]> unresolvedDate = new ArrayList<>();
-		final List<String[]> unresolvedId = new ArrayList<>();
-
-		// extract dates from rows
-		final CDateSet[] rowDates = readDates(values, format, dateReader);
-
-		// Extract extra data from rows by Row, to be collected into by entities
-		// Row -> Column -> Value
-		final Map<String, String>[] extraDataByRow = readExtras(values, format);
-
-		final List<Function<String[], EntityIdMap.ExternalId>> readers = IdColumnUtil.getIdReaders(format, idColumnConfig.getIdMappers());
-
-		// We will not be able to resolve anything...
-		if (readers.isEmpty()) {
-			return new ResolveStatistic(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyList(), List.of(values));
-		}
-
-		// Entity -> Column -> Values
-		final Map<String, Map<String, List<String>>> extraDataByEntity = new HashMap<>();
-
-		// ignore the first row, because this is the header
-		for (int rowNum = 1; rowNum < values.length; rowNum++) {
-
-			final String[] row = values[rowNum];
-
-			if (rowDates[rowNum] == null) {
-				unresolvedDate.add(row);
-				continue;
-			}
-
-			// TODO proper implementation of EntityIdMap#resolve for SQL mode
-			String resolvedId = isInSqlMode
-								? String.valueOf(row[0])
-								: tryResolveId(row, readers, mapping);
-
-			if (resolvedId == null) {
-				unresolvedId.add(row);
-				continue;
-			}
-
-			//read the dates from the row
-			resolved.put(resolvedId, rowDates[rowNum]);
-
-			// Entity was resolved for row so we collect the data.
-			if (extraDataByRow[rowNum] != null) {
-
-				for (Map.Entry<String, String> entry : extraDataByRow[rowNum].entrySet()) {
-					extraDataByEntity.computeIfAbsent(resolvedId, (ignored) -> new HashMap<>())
-									 .computeIfAbsent(entry.getKey(), (ignored) -> new ArrayList<>())
-									 .add(entry.getValue());
-				}
-			}
-		}
-
-		if (onlySingles) {
-			// Check that there is at most one value per entity and per column
-			final boolean alright = extraDataByEntity.values().stream()
-													 .map(Map::values)
-													 .flatMap(Collection::stream)
-													 .allMatch(l -> l.size() <= 1);
-			if (!alright) {
-				throw new ConqueryError.ExternalResolveOnePerRowError();
-			}
-		}
-
-		return new ResolveStatistic(resolved, extraDataByEntity, unresolvedDate, unresolvedId);
-	}
-
-	/**
-	 * Try to extract a {@link com.bakdata.conquery.models.identifiable.mapping.EntityIdMap.ExternalId} from the row,
-	 * then try to map it to an internal {@link com.bakdata.conquery.models.query.entity.Entity}
-	 */
-	private static String tryResolveId(String[] row, List<Function<String[], EntityIdMap.ExternalId>> readers, EntityIdMap mapping) {
-		String resolvedId = null;
-
-		for (Function<String[], EntityIdMap.ExternalId> reader : readers) {
-			final EntityIdMap.ExternalId externalId = reader.apply(row);
-
-			if (externalId == null) {
-				continue;
-			}
-
-			String innerResolved = mapping.resolve(externalId);
-
-			if (innerResolved == null) {
-				continue;
-			}
-
-			// Only if all resolvable ids agree on the same entity, do we return the id.
-			if (resolvedId != null && !innerResolved.equals(resolvedId)) {
-				log.error("`{}` maps to different Entities", (Object) row);
-				continue;
-			}
-
-			resolvedId = innerResolved;
-		}
-		return resolvedId;
-	}
-
-	/**
-	 * Try and extract Extra data from input to be returned as extra-data in output.
-	 * <p>
-	 * Line -> ( Column -> Value )
-	 */
-	private static Map<String, String>[] readExtras(String[][] values, List<String> format) {
-		final String[] names = values[0];
-		final Map<String, String>[] extrasByRow = new Map[values.length];
-
-
-		for (int line = 1; line < values.length; line++) {
-			for (int col = 0; col < format.size(); col++) {
-				if (!format.get(col).equals(FORMAT_EXTRA)) {
-					continue;
-				}
-
-
-				if (extrasByRow[line] == null) {
-					extrasByRow[line] = new HashMap<>(names.length);
-				}
-
-				extrasByRow[line].put(names[col], values[line][col]);
-			}
-		}
-
-
-		return extrasByRow;
-	}
-
-
 	@Override
 	public RequiredEntities collectRequiredEntities(QueryExecutionContext context) {
 		return new RequiredEntities(valuesResolved.keySet());
@@ -441,7 +220,7 @@ public List<ResultInfo> getResultInfos() {
 		}
 		List<ResultInfo> resultInfos = new ArrayList<>();
 		for (int col = 0; col < format.size(); col++) {
-			if (!format.get(col).equals(FORMAT_EXTRA)) {
+			if (!format.get(col).equals(EntityResolverUtil.FORMAT_EXTRA)) {
 				continue;
 			}
 
@@ -484,15 +263,4 @@ public boolean isHeadersUnique() {
 		return false;
 	}
 
-	/**
-	 * Try to resolve a date format, return nothing if not possible.
-	 */
-	private static DateFormat resolveDateFormat(String name) {
-		try {
-			return DateFormat.valueOf(name);
-		}
-		catch (IllegalArgumentException e) {
-			return null; // Does not exist
-		}
-	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolver.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolver.java
new file mode 100644
index 0000000000..c93f1e3375
--- /dev/null
+++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolver.java
@@ -0,0 +1,51 @@
+package com.bakdata.conquery.apiv1.query.concept.specific.external;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import com.bakdata.conquery.models.common.CDateSet;
+import com.bakdata.conquery.models.config.IdColumnConfig;
+import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap;
+import com.bakdata.conquery.util.DateReader;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import jakarta.validation.constraints.NotEmpty;
+import jakarta.validation.constraints.NotNull;
+import lombok.Data;
+
+public interface EntityResolver {
+
+	/**
+	 * Helper method to try and resolve entities in values using the specified format.
+	 */
+	ResolveStatistic resolveEntities(
+			@NotEmpty String[][] values,
+			@NotEmpty List<String> format,
+			EntityIdMap mapping,
+			IdColumnConfig idColumnConfig,
+			@NotNull DateReader dateReader,
+			boolean onlySingles
+	);
+
+	@Data
+	class ResolveStatistic {
+
+		@JsonIgnore
+		private final Map<String, CDateSet> resolved;
+
+		/**
+		 * Entity -> Column -> Values
+		 */
+		@JsonIgnore
+		private final Map<String, Map<String, List<String>>> extra;
+
+		private final List<String[]> unreadableDate;
+		private final List<String[]> unresolvedId;
+
+		public static ResolveStatistic forEmptyReaders(String[][] values) {
+			return new ResolveStatistic(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyList(), List.of(values));
+		}
+
+	}
+
+}
diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolverUtil.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolverUtil.java
new file mode 100644
index 0000000000..3a9be46a25
--- /dev/null
+++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolverUtil.java
@@ -0,0 +1,179 @@
+package com.bakdata.conquery.apiv1.query.concept.specific.external;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import com.bakdata.conquery.models.common.CDateSet;
+import com.bakdata.conquery.models.error.ConqueryError;
+import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap;
+import com.bakdata.conquery.models.identifiable.mapping.ExternalId;
+import com.bakdata.conquery.util.DateReader;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class EntityResolverUtil {
+
+	public static final String FORMAT_EXTRA = "EXTRA";
+
+	/**
+	 * For each row try and collect all dates.
+	 *
+	 * @return Row -> Dates
+	 */
+	public static CDateSet[] readDates(String[][] values, List<String> format, DateReader dateReader) {
+		final CDateSet[] out = new CDateSet[values.length];
+
+		final List<DateFormat> dateFormats = format.stream()
+												   .map(EntityResolverUtil::resolveDateFormat)
+												   // Don't use Stream#toList to preserve null-values
+												   .collect(Collectors.toList());
+
+		/*
+		 If no format is provided, put empty dates into output.
+		 This indicates that no date context was provided and
+		 the entries are not restricted by any date restriction,
+		 but can also don't contribute to any date aggregation.
+		 */
+		if (dateFormats.stream().allMatch(Objects::isNull)) {
+			// Initialize empty
+			for (int row = 0; row < values.length; row++) {
+				out[row] = CDateSet.createEmpty();
+			}
+			return out;
+		}
+
+		for (int row = 1; row < values.length; row++) {
+			try {
+				final CDateSet dates = CDateSet.createEmpty();
+
+				// Collect all specified dates into a single set.
+				for (int col = 0; col < dateFormats.size(); col++) {
+					final DateFormat dateFormat = dateFormats.get(col);
+
+					if (dateFormat == null) {
+						continue;
+					}
+					dateFormat.readDates(values[row][col], dateReader, dates);
+				}
+
+				if (dates.isEmpty()) {
+					continue;
+				}
+
+				if (out[row] == null) {
+					out[row] = CDateSet.createEmpty();
+				}
+
+				out[row].addAll(dates);
+			}
+			catch (Exception e) {
+				log.warn("Failed to parse Date from {}", row, e);
+			}
+		}
+
+		return out;
+	}
+
+	public static void collectExtraData(Map<String, String>[] extraDataByRow, int rowNum, Map<String, Map<String, List<String>>> extraDataByEntity, String resolvedId) {
+		if (extraDataByRow[rowNum] != null) {
+			for (Map.Entry<String, String> entry : extraDataByRow[rowNum].entrySet()) {
+				extraDataByEntity.computeIfAbsent(resolvedId, (ignored) -> new HashMap<>())
+								 .computeIfAbsent(entry.getKey(), (ignored) -> new ArrayList<>())
+								 .add(entry.getValue());
+			}
+		}
+	}
+
+	public static void verifyOnlySingles(boolean onlySingles, Map<String, Map<String, List<String>>> extraDataByEntity) {
+		if (!onlySingles) {
+			return;
+		}
+		// Check that there is at most one value per entity and per column
+		final boolean alright = extraDataByEntity.values().stream()
+												 .map(Map::values)
+												 .flatMap(Collection::stream)
+												 .allMatch(l -> l.size() <= 1);
+		if (!alright) {
+			throw new ConqueryError.ExternalResolveOnePerRowError();
+		}
+	}
+
+	/**
+	 * Try to extract a {@link ExternalId} from the row,
+	 * then try to map it to an internal {@link com.bakdata.conquery.models.query.entity.Entity}
+	 */
+	public static String tryResolveId(String[] row, List<Function<String[], ExternalId>> readers, EntityIdMap mapping) {
+		String resolvedId = null;
+
+		for (Function<String[], ExternalId> reader : readers) {
+			final ExternalId externalId = reader.apply(row);
+
+			if (externalId == null) {
+				continue;
+			}
+
+			String innerResolved = mapping.resolve(externalId);
+
+			if (innerResolved == null) {
+				continue;
+			}
+
+			// Only if all resolvable ids agree on the same entity, do we return the id.
+			if (resolvedId != null && !innerResolved.equals(resolvedId)) {
+				log.error("`{}` maps to different Entities", (Object) row);
+				continue;
+			}
+
+			resolvedId = innerResolved;
+		}
+		return resolvedId;
+	}
+
+	/**
+	 * Try and extract Extra data from input to be returned as extra-data in output.
+	 * <p>
+	 * Line -> ( Column -> Value )
+	 */
+	public static Map<String, String>[] readExtras(String[][] values, List<String> format) {
+		final String[] names = values[0];
+		final Map<String, String>[] extrasByRow = new Map[values.length];
+
+
+		for (int line = 1; line < values.length; line++) {
+			for (int col = 0; col < format.size(); col++) {
+				if (!format.get(col).equals(FORMAT_EXTRA)) {
+					continue;
+				}
+
+
+				if (extrasByRow[line] == null) {
+					extrasByRow[line] = new HashMap<>(names.length);
+				}
+
+				extrasByRow[line].put(names[col], values[line][col]);
+			}
+		}
+
+
+		return extrasByRow;
+	}
+
+	/**
+	 * Try to resolve a date format, return nothing if not possible.
+	 */
+	private static DateFormat resolveDateFormat(String name) {
+		try {
+			return DateFormat.valueOf(name);
+		}
+		catch (IllegalArgumentException e) {
+			return null; // Does not exist
+		}
+	}
+
+}
diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterEntityResolver.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterEntityResolver.java
new file mode 100644
index 0000000000..8aa3d7c720
--- /dev/null
+++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterEntityResolver.java
@@ -0,0 +1,83 @@
+package com.bakdata.conquery.mode.cluster;
+
+import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.collectExtraData;
+import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.readDates;
+import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.tryResolveId;
+import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.verifyOnlySingles;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+import com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolver;
+import com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil;
+import com.bakdata.conquery.models.common.CDateSet;
+import com.bakdata.conquery.models.config.IdColumnConfig;
+import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap;
+import com.bakdata.conquery.models.identifiable.mapping.ExternalId;
+import com.bakdata.conquery.util.DateReader;
+import com.bakdata.conquery.util.io.IdColumnUtil;
+import jakarta.validation.constraints.NotEmpty;
+
+public class ClusterEntityResolver implements EntityResolver {
+
+	@Override
+	public ResolveStatistic resolveEntities(
+			@NotEmpty String[][] values,
+			List<String> format,
+			EntityIdMap mapping,
+			IdColumnConfig idColumnConfig,
+			DateReader dateReader,
+			boolean onlySingles
+	) {
+		final Map<String, CDateSet> resolved = new HashMap<>();
+		final List<String[]> unresolvedDate = new ArrayList<>();
+		final List<String[]> unresolvedId = new ArrayList<>();
+
+		// extract dates from rows
+		final CDateSet[] rowDates = readDates(values, format, dateReader);
+
+		// Extract extra data from rows by Row, to be collected into by entities
+		// Row -> Column -> Value
+		final Map<String, String>[] extraDataByRow = EntityResolverUtil.readExtras(values, format);
+
+		final List<Function<String[], ExternalId>> readers = IdColumnUtil.getIdReaders(format, idColumnConfig.getIdMappers());
+
+		// We will not be able to resolve anything...
+		if (readers.isEmpty()) {
+			return EntityResolver.ResolveStatistic.forEmptyReaders(values);
+		}
+
+		// Entity -> Column -> Values
+		final Map<String, Map<String, List<String>>> extraDataByEntity = new HashMap<>();
+
+		// ignore the first row, because this is the header
+		for (int rowNum = 1; rowNum < values.length; rowNum++) {
+
+			final String[] row = values[rowNum];
+
+			if (rowDates[rowNum] == null) {
+				unresolvedDate.add(row);
+				continue;
+			}
+
+			String resolvedId = tryResolveId(row, readers, mapping);
+
+			if (resolvedId == null) {
+				unresolvedId.add(row);
+				continue;
+			}
+
+			// read the dates from the row
+			resolved.put(resolvedId, rowDates[rowNum]);
+
+			// Entity was resolved for row, so we collect the data.
+			collectExtraData(extraDataByRow, rowNum, extraDataByEntity, resolvedId);
+		}
+
+		verifyOnlySingles(onlySingles, extraDataByEntity);
+		return new EntityResolver.ResolveStatistic(resolved, extraDataByEntity, unresolvedDate, unresolvedId);
+	}
+}
diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java
index acf04a20fe..617f166d72 100644
--- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java
+++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java
@@ -37,6 +37,7 @@ public DistributedNamespace createNamespace(NamespaceStorage storage, final Meta
 				namespaceData.getJobManager(),
 				namespaceData.getFilterSearch(),
 				namespaceData.getIndexService(),
+				new ClusterEntityResolver(),
 				namespaceData.getInjectables(),
 				workerHandler
 		);
diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java
index 52a389c5fe..a0f7f23c2d 100644
--- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java
+++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java
@@ -55,6 +55,7 @@ public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaSto
 		SqlConverter sqlConverter = new SqlConverter(nodeConversions);
 		ExecutionManager<SqlExecutionResult> executionManager = new SqlExecutionManager(sqlConverter, sqlExecutionService, metaStorage);
 		SqlStorageHandler sqlStorageHandler = new SqlStorageHandler(sqlExecutionService);
+		SqlEntityResolver sqlEntityResolver = new SqlEntityResolver(idColumns, dslContext, sqlDialect, sqlExecutionService);
 
 		return new LocalNamespace(
 				namespaceData.getPreprocessMapper(),
@@ -66,6 +67,7 @@ public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaSto
 				namespaceData.getJobManager(),
 				namespaceData.getFilterSearch(),
 				namespaceData.getIndexService(),
+				sqlEntityResolver,
 				namespaceData.getInjectables()
 		);
 	}
diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java b/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java
new file mode 100644
index 0000000000..681889ec86
--- /dev/null
+++ b/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java
@@ -0,0 +1,206 @@
+package com.bakdata.conquery.mode.local;
+
+import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.collectExtraData;
+import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.readDates;
+import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.verifyOnlySingles;
+import static org.jooq.impl.DSL.field;
+import static org.jooq.impl.DSL.name;
+import static org.jooq.impl.DSL.table;
+import static org.jooq.impl.DSL.val;
+import static org.jooq.impl.DSL.when;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolver;
+import com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil;
+import com.bakdata.conquery.models.common.CDateSet;
+import com.bakdata.conquery.models.config.IdColumnConfig;
+import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap;
+import com.bakdata.conquery.models.identifiable.mapping.ExternalId;
+import com.bakdata.conquery.sql.conversion.SharedAliases;
+import com.bakdata.conquery.sql.conversion.dialect.SqlDialect;
+import com.bakdata.conquery.sql.execution.SqlExecutionService;
+import com.bakdata.conquery.util.DateReader;
+import com.bakdata.conquery.util.io.IdColumnUtil;
+import jakarta.validation.constraints.NotEmpty;
+import lombok.RequiredArgsConstructor;
+import org.jooq.CommonTableExpression;
+import org.jooq.DSLContext;
+import org.jooq.Field;
+import org.jooq.Name;
+import org.jooq.Record;
+import org.jooq.Record2;
+import org.jooq.Record3;
+import org.jooq.Select;
+import org.jooq.SelectConditionStep;
+import org.jooq.Table;
+
+@RequiredArgsConstructor
+public class SqlEntityResolver implements EntityResolver {
+
+	private static final Name IS_RESOLVED_ALIAS = name("is_resolved");
+	private static final Name UNRESOLVED_CTE = name("ids_unresolved");
+	public static final String ROW_INDEX = "rowIndex";
+
+	private final IdColumnConfig idColumns;
+	private final DSLContext context;
+	private final SqlDialect dialect;
+	private final SqlExecutionService executionService;
+
+	@Override
+	public ResolveStatistic resolveEntities(
+			@NotEmpty String[][] values,
+			List<String> format,
+			EntityIdMap mapping,
+			IdColumnConfig idColumnConfig,
+			DateReader dateReader,
+			boolean onlySingles
+	) {
+		final Map<String, CDateSet> resolved = new HashMap<>();
+		final List<String[]> unresolvedDate = new ArrayList<>();
+		final List<String[]> unresolvedId = new ArrayList<>();
+
+		// extract dates from rows
+		final CDateSet[] rowDates = readDates(values, format, dateReader);
+
+		// Extract extra data from rows by Row, to be collected into by entities
+		// Row -> Column -> Value
+		final Map<String, String>[] extraDataByRow = EntityResolverUtil.readExtras(values, format);
+
+		final List<Function<String[], ExternalId>> readers = IdColumnUtil.getIdReaders(format, idColumnConfig.getIdMappers());
+
+		// We will not be able to resolve anything...
+		if (readers.isEmpty()) {
+			return EntityResolver.ResolveStatistic.forEmptyReaders(values);
+		}
+
+		// Entity -> Column -> Values
+		final Map<String, Map<String, List<String>>> extraDataByEntity = new HashMap<>();
+
+		// all IDs of this map had at least a matching reader
+		final Map<Integer, IdResolveInfo> resolvedIdsMap = resolveIds(values, readers);
+
+		// ignore the first row, because this is the header
+		for (int rowNum = 1; rowNum < values.length; rowNum++) {
+
+			final String[] row = values[rowNum];
+
+			final IdResolveInfo idResolveInfo = resolvedIdsMap.get(rowNum);
+			if (idResolveInfo == null) {
+				// row had no matching reader
+				unresolvedId.add(row);
+				continue;
+			}
+
+			// external ID could not be resolved internally
+			if (!idResolveInfo.isResolved()) {
+				unresolvedDate.add(row);
+				continue;
+			}
+
+			final String resolvedId = idResolveInfo.externalId();
+
+			if (rowDates[rowNum] == null) {
+				unresolvedDate.add(row);
+				continue;
+			}
+
+			// read the dates from the row
+			resolved.put(resolvedId, rowDates[rowNum]);
+
+			// Entity was resolved for row, so we collect the data.
+			collectExtraData(extraDataByRow, rowNum, extraDataByEntity, resolvedId);
+		}
+
+		verifyOnlySingles(onlySingles, extraDataByEntity);
+		return new EntityResolver.ResolveStatistic(resolved, extraDataByEntity, unresolvedDate, unresolvedId);
+
+	}
+
+	/**
+	 * Create a SQL query like this
+	 * <pre>
+	 *     {@code
+	 *      with "ids_unresolved" as (select 1   as "row",
+	 *                                      '1'  as "primary_id"
+	 *                                -- will select more ids here via union all)
+	 * 		select "row",
+	 * 		       "primary_id",
+	 * 		       case
+	 * 		           when "entities"."id" is not null then true
+	 * 		           else false
+	 * 		           end as "is_resolved"
+	 * 		from "entities"
+	 * 		         join "ids_unresolved"
+	 * 		              on "primary_id" = "entities"."id"
+	 * 		where "primary_id" = "pid"
+	 *     }
+	 * </pre>
+	 * <p>
+	 * For each ID, that had a matching reader, it will return an entry in the map with row number -> IdResolveInfo.
+	 */
+	private Map<Integer, IdResolveInfo> resolveIds(String[][] values, List<Function<String[], ExternalId>> readers) {
+
+		CommonTableExpression<?> unresolvedCte = createUnresolvedCte(values, readers);
+
+		Field<Integer> rowIndex = field(name(ROW_INDEX), Integer.class);
+		Field<String> externalPrimaryColumn = field(name(SharedAliases.PRIMARY_COLUMN.getAlias()), String.class);
+		Field<String> innerPrimaryColumn = field(name(idColumns.findPrimaryIdColumn().getField()), String.class);
+		Field<Boolean> isResolved = when(innerPrimaryColumn.isNotNull(), val(true))
+				.otherwise(false)
+				.as(IS_RESOLVED_ALIAS);
+
+		Table<Record> allIdsTable = table(name(idColumns.getTable()));
+		SelectConditionStep<Record3<Integer, String, Boolean>> resolveIdsQuery =
+				context.with(unresolvedCte)
+					   .select(rowIndex, externalPrimaryColumn, isResolved)
+					   .from(dialect.getFunctionProvider().innerJoin(allIdsTable, unresolvedCte, List.of(externalPrimaryColumn.eq(innerPrimaryColumn))))
+					   .where(externalPrimaryColumn.eq(innerPrimaryColumn));
+
+		return executionService.fetchStream(resolveIdsQuery)
+							   .collect(Collectors.toMap(
+									   record -> record.get(rowIndex),
+									   record -> new IdResolveInfo(record.get(externalPrimaryColumn), record.get(isResolved))
+							   ));
+	}
+
+	private CommonTableExpression<?> createUnresolvedCte(String[][] values, List<Function<String[], ExternalId>> readers) {
+
+		List<Select<Record2<Integer, String>>> selects = new ArrayList<>(values.length);
+		for (int i = 1; i < values.length; i++) {
+
+			final String[] row = values[i];
+
+			String resolvedId = null;
+			for (Function<String[], ExternalId> reader : readers) {
+				final ExternalId externalId = reader.apply(row);
+				resolvedId = externalId.getId();
+			}
+
+			// no matching reader found
+			if (resolvedId == null) {
+				continue;
+			}
+
+			Field<Integer> rowIndex = val(i).as(ROW_INDEX);
+			Field<String> externalPrimaryColumn = val(resolvedId).as(SharedAliases.PRIMARY_COLUMN.getAlias());
+			Select<Record2<Integer, String>> externalIdSelect = context.select(rowIndex, externalPrimaryColumn)
+																	   // some dialects can't just select static values without FROM clause
+																	   .from(dialect.getFunctionProvider().getNoOpTable());
+
+			selects.add(externalIdSelect);
+		}
+
+		return UNRESOLVED_CTE.as(selects.stream().reduce(Select::unionAll).orElseThrow(IllegalStateException::new));
+	}
+
+
+	private record IdResolveInfo(String externalId, boolean isResolved) {
+	}
+
+}
diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java
index 6c74ce0cc1..a708564a40 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java
@@ -6,6 +6,7 @@
 
 import com.bakdata.conquery.io.jackson.View;
 import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap;
+import com.bakdata.conquery.models.identifiable.mapping.ExternalId;
 import com.bakdata.conquery.resources.admin.rest.AdminDatasetProcessor;
 import com.fasterxml.jackson.annotation.JsonAlias;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
@@ -34,18 +35,18 @@
 @JsonIgnoreProperties(value = {"resolvable"}) // for backwards compatibility
 public class ColumnConfig {
 
-	public EntityIdMap.ExternalId read(String value) {
+	public ExternalId read(String value) {
 		if (Strings.isNullOrEmpty(value)) {
 			return null;
 		}
 
 		if (getLength() == -1 || getPad() == null) {
-			return new EntityIdMap.ExternalId(getName(), value);
+			return new ExternalId(getName(), value);
 		}
 
 		String padded = StringUtils.leftPad(value, getLength(), getPad());
 
-		return new EntityIdMap.ExternalId(getName(), padded);
+		return new ExternalId(getName(), padded);
 
 	}
 
diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java
index 83723c0b4b..52b56dd00c 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java
@@ -172,11 +172,4 @@ public void addInputMapping(String csvEntityId, ExternalId externalEntityId) {
 		}
 	}
 
-	@Data
-	@RequiredArgsConstructor(onConstructor_ = @JsonCreator)
-	public static class ExternalId {
-		private final String type;
-		private final String id;
-	}
-
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/ExternalId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/ExternalId.java
new file mode 100644
index 0000000000..b1f4683797
--- /dev/null
+++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/ExternalId.java
@@ -0,0 +1,12 @@
+package com.bakdata.conquery.models.identifiable.mapping;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
+
+@Data
+@RequiredArgsConstructor(onConstructor_ = @JsonCreator)
+public class ExternalId {
+	private final String type;
+	private final String id;
+}
diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java
index 19ecf66d1e..76a5d4664f 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java
@@ -8,6 +8,7 @@
 
 import com.bakdata.conquery.io.jackson.Injectable;
 import com.bakdata.conquery.io.storage.NamespaceStorage;
+import com.bakdata.conquery.mode.cluster.ClusterEntityResolver;
 import com.bakdata.conquery.models.datasets.Column;
 import com.bakdata.conquery.models.datasets.Dataset;
 import com.bakdata.conquery.models.datasets.concepts.Concept;
@@ -36,7 +37,6 @@ public class DistributedNamespace extends Namespace {
 	private final WorkerHandler workerHandler;
 	private final DistributedExecutionManager executionManager;
 
-
 	public DistributedNamespace(
 			ObjectMapper preprocessMapper,
 			ObjectMapper communicationMapper,
@@ -45,10 +45,11 @@ public DistributedNamespace(
 			JobManager jobManager,
 			FilterSearch filterSearch,
 			IndexService indexService,
+			ClusterEntityResolver clusterEntityResolver,
 			List<Injectable> injectables,
 			WorkerHandler workerHandler
 	) {
-		super(preprocessMapper, communicationMapper, storage, executionManager, jobManager, filterSearch, indexService, injectables);
+		super(preprocessMapper, communicationMapper, storage, executionManager, jobManager, filterSearch, indexService, clusterEntityResolver, injectables);
 		this.executionManager = executionManager;
 		this.workerHandler = workerHandler;
 	}
diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java
index b21aabf5bd..99fb6340a6 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java
@@ -8,6 +8,7 @@
 
 import com.bakdata.conquery.io.jackson.Injectable;
 import com.bakdata.conquery.io.storage.NamespaceStorage;
+import com.bakdata.conquery.mode.local.SqlEntityResolver;
 import com.bakdata.conquery.mode.local.SqlStorageHandler;
 import com.bakdata.conquery.models.datasets.Column;
 import com.bakdata.conquery.models.index.IndexService;
@@ -37,9 +38,10 @@ public LocalNamespace(
 			JobManager jobManager,
 			FilterSearch filterSearch,
 			IndexService indexService,
+			SqlEntityResolver sqlEntityResolver,
 			List<Injectable> injectables
 	) {
-		super(preprocessMapper, communicationMapper, storage, executionManager, jobManager, filterSearch, indexService, injectables);
+		super(preprocessMapper, communicationMapper, storage, executionManager, jobManager, filterSearch, indexService, sqlEntityResolver, injectables);
 		this.dslContextWrapper = dslContextWrapper;
 		this.storageHandler = storageHandler;
 	}
@@ -55,7 +57,8 @@ void registerColumnValuesInSearch(Set<Column> columns) {
 			try {
 				final Stream<String> stringStream = storageHandler.lookupColumnValues(getStorage(), column);
 				getFilterSearch().registerValues(column, stringStream.collect(Collectors.toSet()));
-			}catch (Exception e) {
+			}
+			catch (Exception e) {
 				log.error("Problem collecting column values for {}", column, e);
 			}
 		}
diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java
index 1c952d0de7..dcee848050 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java
@@ -6,6 +6,7 @@
 import java.util.NoSuchElementException;
 import java.util.Set;
 
+import com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolver;
 import com.bakdata.conquery.io.jackson.Injectable;
 import com.bakdata.conquery.io.storage.NamespaceStorage;
 import com.bakdata.conquery.models.datasets.Column;
@@ -49,6 +50,8 @@ public abstract class Namespace extends IdResolveContext {
 
 	private final IndexService indexService;
 
+	private final EntityResolver entityResolver;
+
 	// Jackson's injectables that are available when deserializing requests (see PathParamInjector) or items from the storage
 	private final List<Injectable> injectables;
 
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java b/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java
index b7ea1916d3..a1b2acf495 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java
@@ -5,6 +5,7 @@
 import com.zaxxer.hikari.HikariConfig;
 import com.zaxxer.hikari.HikariDataSource;
 import org.jooq.DSLContext;
+import org.jooq.conf.RenderOptionalKeyword;
 import org.jooq.conf.RenderQuotedNames;
 import org.jooq.conf.Settings;
 import org.jooq.impl.DSL;
@@ -24,7 +25,9 @@ public static DSLContextWrapper create(DatabaseConfig config, SqlConnectorConfig
 				.withRenderFormatted(connectorConfig.isWithPrettyPrinting())
 				// enforces all identifiers to be quoted if not explicitly unquoted via DSL.unquotedName()
 				// to prevent any lowercase/uppercase SQL dialect specific identifier naming issues
-				.withRenderQuotedNames(RenderQuotedNames.EXPLICIT_DEFAULT_QUOTED);
+				.withRenderQuotedNames(RenderQuotedNames.EXPLICIT_DEFAULT_QUOTED)
+				// always render "as" keyword for field aliases
+				.withRenderOptionalAsKeywordForFieldAliases(RenderOptionalKeyword.ON);
 
 		DSLContext dslContext = DSL.using(
 				hikariDataSource,
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java
index a1a3186946..eaa8dfaa86 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java
@@ -22,11 +22,7 @@ public Class<? extends CQYes> getConversionClass() {
 	@Override
 	public ConversionContext convert(CQYes cqYes, ConversionContext context) {
 
-		ColumnConfig primaryColumnConfig = context.getIdColumns().getIds().stream()
-												  .filter(ColumnConfig::isPrimaryId)
-												  .findFirst()
-												  .orElseThrow(() -> new IllegalStateException("SQL mode requires a primary key column in IdColumnConfig"));
-
+		ColumnConfig primaryColumnConfig = context.getIdColumns().findPrimaryIdColumn();
 		Field<Object> primaryColumn = DSL.field(DSL.name(primaryColumnConfig.getField()));
 		SqlIdColumns ids = new SqlIdColumns(primaryColumn);
 
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java
index 2a6edc2333..c660248531 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java
@@ -56,9 +56,10 @@ private static QueryStep.QueryStepBuilder joinWithStratificationTable(
 			List<Condition> conditions,
 			CQTableContext tableContext
 	) {
-		QueryStep stratificationTable = tableContext.getConversionContext().getStratificationTable();
+		QueryStep stratificationTableCte = tableContext.getConversionContext().getStratificationTable();
+		Table<Record> stratificationTable = DSL.table(DSL.name(stratificationTableCte.getCteName()));
 
-		Selects stratificationSelects = stratificationTable.getQualifiedSelects();
+		Selects stratificationSelects = stratificationTableCte.getQualifiedSelects();
 		SqlIdColumns stratificationIds = stratificationSelects.getIds();
 		SqlIdColumns rootTableIds = tableContext.getIds();
 		List<Condition> idConditions = stratificationIds.join(rootTableIds);
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java
index 68dfc0b099..46572215ea 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java
@@ -186,34 +186,16 @@ default Condition in(Field<String> column, String[] values) {
 		return column.in(values);
 	}
 
-	default TableOnConditionStep<Record> innerJoin(
-			Table<Record> leftPartQueryBase,
-			QueryStep rightPartQS,
-			List<Condition> joinConditions
-	) {
-		return leftPartQueryBase
-				.innerJoin(DSL.name(rightPartQS.getCteName()))
-				.on(joinConditions.toArray(Condition[]::new));
+	default TableOnConditionStep<Record> innerJoin(Table<?> leftPart, Table<?> rightPart, List<Condition> joinConditions) {
+		return leftPart.innerJoin(rightPart).on(joinConditions.toArray(Condition[]::new));
 	}
 
-	default TableOnConditionStep<Record> fullOuterJoin(
-			Table<Record> leftPartQueryBase,
-			QueryStep rightPartQS,
-			List<Condition> joinConditions
-	) {
-		return leftPartQueryBase
-				.fullOuterJoin(DSL.name(rightPartQS.getCteName()))
-				.on(joinConditions.toArray(Condition[]::new));
+	default TableOnConditionStep<Record> fullOuterJoin(Table<?> leftPart, Table<?> rightPart, List<Condition> joinConditions) {
+		return leftPart.fullOuterJoin(rightPart).on(joinConditions.toArray(Condition[]::new));
 	}
 
-	default TableOnConditionStep<Record> leftJoin(
-			Table<Record> leftPartQueryBase,
-			QueryStep rightPartQS,
-			List<Condition> joinConditions
-	) {
-		return leftPartQueryBase
-				.leftJoin(DSL.name(rightPartQS.getCteName()))
-				.on(joinConditions.toArray(Condition[]::new));
+	default TableOnConditionStep<Record> leftJoin(Table<?> leftPart, Table<?> rightPart, List<Condition> joinConditions) {
+		return leftPart.leftJoin(rightPart).on(joinConditions.toArray(Condition[]::new));
 	}
 
 	default Field<Date> toDateField(String dateExpression) {
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java
index 0a542da464..d58e373285 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java
@@ -108,7 +108,8 @@ public static TableLike<Record> constructJoinedTable(List<QueryStep> queriesToJo
 
 			List<Condition> joinConditions = Stream.concat(joinIdsCondition.stream(), Stream.of(joinDateCondition)).collect(Collectors.toList());
 
-			joinedQuery = joinType.join(joinedQuery, rightPartQS, joinConditions);
+			Table<Record> rightPartTable = DSL.table(DSL.name(rightPartQS.getCteName()));
+			joinedQuery = joinType.join(joinedQuery, rightPartTable, joinConditions);
 		}
 
 		return joinedQuery;
@@ -170,11 +171,7 @@ private static QueryStep buildStepAndAggregateDates(
 
 	@FunctionalInterface
 	private interface JoinType {
-		TableOnConditionStep<Record> join(
-				Table<Record> leftPartQueryBase,
-				QueryStep rightPartQS,
-				List<Condition> joinConditions
-		);
+		TableOnConditionStep<Record> join(Table<?> leftPart, Table<?> rightPart, List<Condition> joinConditions);
 	}
 
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java
index ce433e53b9..5b2b42d092 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java
@@ -199,8 +199,9 @@ private static Table<Record> joinConnectorTableWithPrerequisite(
 	) {
 		SqlFunctionProvider functionProvider = context.getSqlDialect().getFunctionProvider();
 		Table<Record> connectorTable = DSL.table(DSL.name(cqTable.getConnector().getTable().getName()));
+		Table<Record> convertedPrerequisiteTable = DSL.table(DSL.name(convertedPrerequisite.getCteName()));
 		List<Condition> joinOnIds = ids.join(convertedPrerequisite.getQualifiedSelects().getIds());
-		return functionProvider.innerJoin(connectorTable, convertedPrerequisite, joinOnIds);
+		return functionProvider.innerJoin(connectorTable, convertedPrerequisiteTable, joinOnIds);
 	}
 
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/util/io/IdColumnUtil.java b/backend/src/main/java/com/bakdata/conquery/util/io/IdColumnUtil.java
index 3db0a88d50..3f1716a3f6 100644
--- a/backend/src/main/java/com/bakdata/conquery/util/io/IdColumnUtil.java
+++ b/backend/src/main/java/com/bakdata/conquery/util/io/IdColumnUtil.java
@@ -11,7 +11,7 @@
 import com.bakdata.conquery.models.config.ColumnConfig;
 import com.bakdata.conquery.models.execution.ManagedExecution;
 import com.bakdata.conquery.models.identifiable.mapping.AutoIncrementingPseudomizer;
-import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap;
+import com.bakdata.conquery.models.identifiable.mapping.ExternalId;
 import com.bakdata.conquery.models.identifiable.mapping.FullIdPrinter;
 import com.bakdata.conquery.models.identifiable.mapping.IdPrinter;
 import com.bakdata.conquery.models.worker.Namespace;
@@ -23,8 +23,8 @@ public class IdColumnUtil {
 	/**
 	 * If a column contains an ID, create a reader for that ID.
 	 */
-	public static List<Function<String[], EntityIdMap.ExternalId>> getIdReaders(List<String> format, Map<String, ColumnConfig> idMappers) {
-		List<Function<String[], EntityIdMap.ExternalId>> out = new ArrayList<>(format.size());
+	public static List<Function<String[], ExternalId>> getIdReaders(List<String> format, Map<String, ColumnConfig> idMappers) {
+		List<Function<String[], ExternalId>> out = new ArrayList<>(format.size());
 
 		for (int index = 0; index < format.size(); index++) {
 			final ColumnConfig mapper = idMappers.get(format.get(index));
diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java b/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java
index e96c80c276..68c9b70818 100644
--- a/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java
+++ b/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java
@@ -3,6 +3,8 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
+import java.util.Map;
 import java.util.function.Consumer;
 import javax.annotation.Nullable;
 
@@ -10,6 +12,8 @@
 import com.bakdata.conquery.io.cps.CPSBase;
 import com.bakdata.conquery.io.jackson.Jackson;
 import com.bakdata.conquery.io.jackson.View;
+import com.bakdata.conquery.io.storage.MetaStorage;
+import com.bakdata.conquery.models.config.ColumnConfig;
 import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.config.Dialect;
 import com.bakdata.conquery.models.config.IdColumnConfig;
@@ -51,8 +55,18 @@ public abstract class ConqueryTestSpec {
 	@Nullable
 	SqlSpec sqlSpec;
 
-	@Nullable
-	private IdColumnConfig idColumns;
+	// default IdColumnConfig for SQL mode
+	private IdColumnConfig idColumns = new IdColumnConfig(
+			"entities",
+			List.of(
+					ColumnConfig.builder()
+								.name("ID")
+								.field("pid")
+								.primaryId(true)
+								.label(Map.of(Locale.ROOT, "result"))
+								.build()
+			)
+	);
 
 	public ConqueryConfig overrideConfig(ConqueryConfig config) {
 
diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java b/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java
index b356786602..90211c9b65 100644
--- a/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java
+++ b/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java
@@ -8,17 +8,21 @@
 import com.bakdata.conquery.integration.sql.CsvTableImporter;
 import com.bakdata.conquery.util.support.StandaloneSupport;
 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @RequiredArgsConstructor
 public class SqlTestDataImporter implements TestDataImporter {
 
+	private static final RequiredTable ALL_IDS_TABLE = importRequiredTable("/shared/entities.table.json");
+
 	private final CsvTableImporter csvTableImporter;
 
 	@Override
 	public void importQueryTestData(StandaloneSupport support, QueryTest test) throws Exception {
 		RequiredData content = test.getContent();
+		content.getTables().add(ALL_IDS_TABLE);
 		importSecondaryIds(support, content.getSecondaryIds());
 		importTables(support, content.getTables(), true);
 		importConcepts(support, test.getRawConcepts());
@@ -31,6 +35,7 @@ public void importQueryTestData(StandaloneSupport support, QueryTest test) throw
 	@Override
 	public void importFormTestData(StandaloneSupport support, FormTest test) throws Exception {
 		RequiredData content = test.getContent();
+		content.getTables().add(ALL_IDS_TABLE);
 		importSecondaryIds(support, content.getSecondaryIds());
 		importTables(support, content.getTables(), true);
 		importConcepts(support, test.getRawConcepts());
@@ -51,4 +56,9 @@ public void importTableContents(StandaloneSupport support, Collection<RequiredTa
 		}
 	}
 
+	@SneakyThrows
+	private static RequiredTable importRequiredTable(String fileResource) {
+		return RequiredTable.fromFile(fileResource);
+	}
+
 }
diff --git a/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java b/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java
index 42f895727c..ed9e8d76b9 100644
--- a/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java
+++ b/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java
@@ -2,13 +2,14 @@
 
 import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap;
 import com.bakdata.conquery.models.identifiable.mapping.EntityPrintId;
+import com.bakdata.conquery.models.identifiable.mapping.ExternalId;
 
 public class IdMapSerialisationTest {
 
 	public static EntityIdMap createTestPersistentMap() {
 		EntityIdMap entityIdMap = new EntityIdMap();
 
-		entityIdMap.addInputMapping("test1", new EntityIdMap.ExternalId("id", "a"));
+		entityIdMap.addInputMapping("test1", new ExternalId("id", "a"));
 
 		entityIdMap.addOutputMapping("test2", EntityPrintId.from("c"));
 
diff --git a/backend/src/test/resources/tests/sql/yes/persons.csv b/backend/src/test/resources/shared/entities.csv
similarity index 86%
rename from backend/src/test/resources/tests/sql/yes/persons.csv
rename to backend/src/test/resources/shared/entities.csv
index 5499484b3d..514b00398a 100644
--- a/backend/src/test/resources/tests/sql/yes/persons.csv
+++ b/backend/src/test/resources/shared/entities.csv
@@ -1,7 +1,8 @@
 pid,datum
-4,2013-11-10
-5,2013-11-10
-6,2013-11-10
 1,2012-01-01
 2,2013-11-10
 3,2013-11-10
+4,2013-11-10
+5,2013-11-10
+6,2013-11-10
+23,2014-01-01
diff --git a/backend/src/test/resources/shared/entities.table.json b/backend/src/test/resources/shared/entities.table.json
new file mode 100644
index 0000000000..c6a2561f58
--- /dev/null
+++ b/backend/src/test/resources/shared/entities.table.json
@@ -0,0 +1,14 @@
+{
+  "csv": "shared/entities.csv",
+  "name": "entities",
+  "primaryColumn": {
+    "name": "pid",
+    "type": "STRING"
+  },
+  "columns": [
+    {
+      "name": "datum",
+      "type": "DATE"
+    }
+  ]
+}
diff --git a/backend/src/test/resources/tests/sql/yes/CQYES.test.json b/backend/src/test/resources/tests/sql/yes/cqyes.json
similarity index 54%
rename from backend/src/test/resources/tests/sql/yes/CQYES.test.json
rename to backend/src/test/resources/tests/sql/yes/cqyes.json
index e5b9e5fe3b..e5d28e42d0 100644
--- a/backend/src/test/resources/tests/sql/yes/CQYES.test.json
+++ b/backend/src/test/resources/tests/sql/yes/cqyes.json
@@ -10,28 +10,11 @@
   },
   "concepts": [
   ],
-  "idColumns": {
-    "table": "persons",
-    "ids": [
-      {
-        "name": "PID",
-        "field": "pid",
-        "primaryId": true,
-        "label": {
-          "de": "PID"
-        },
-        "description": {
-          "en": "Pid of an Entity.",
-          "de": "Pid einer Entität."
-        }
-      }
-    ]
-  },
   "content": {
     "tables": [
       {
-        "csv": "tests/sql/yes/persons.csv",
-        "name": "persons",
+        "csv": "tests/sql/yes/no-op.csv",
+        "name": "no-op",
         "primaryColumn": {
           "name": "pid",
           "type": "STRING"
diff --git a/backend/src/test/resources/tests/sql/yes/expected.csv b/backend/src/test/resources/tests/sql/yes/expected.csv
index edf163f57b..0a872351ef 100644
--- a/backend/src/test/resources/tests/sql/yes/expected.csv
+++ b/backend/src/test/resources/tests/sql/yes/expected.csv
@@ -1,7 +1,8 @@
-PID,dates
-4,{}
-5,{}
-6,{}
+result,dates
 1,{}
 2,{}
+23,{}
 3,{}
+4,{}
+5,{}
+6,{}
diff --git a/backend/src/test/resources/tests/sql/yes/no-op.csv b/backend/src/test/resources/tests/sql/yes/no-op.csv
new file mode 100644
index 0000000000..d9a4329883
--- /dev/null
+++ b/backend/src/test/resources/tests/sql/yes/no-op.csv
@@ -0,0 +1,2 @@
+pid,datum
+irrelevant_id_for_cqyes,2013-11-10

From 92149a0f80c8fa8ad4eb716aa273e0586a093284 Mon Sep 17 00:00:00 2001
From: awildturtok <1553491+awildturtok@users.noreply.github.com>
Date: Thu, 25 Jul 2024 10:58:45 +0200
Subject: [PATCH 28/41] drop usage of `default` in ColumnStore

---
 .../bakdata/conquery/models/events/EmptyStore.java    |  4 ++++
 .../models/events/stores/primitive/BitSetStore.java   |  4 ++++
 .../events/stores/primitive/ByteArrayStore.java       |  5 +++++
 .../events/stores/primitive/DecimalArrayStore.java    |  5 +++++
 .../events/stores/primitive/DoubleArrayStore.java     |  5 +++++
 .../events/stores/primitive/FloatArrayStore.java      |  5 +++++
 .../models/events/stores/primitive/IntArrayStore.java |  5 +++++
 .../events/stores/primitive/IntegerDateStore.java     |  5 +++++
 .../events/stores/primitive/LongArrayStore.java       |  5 +++++
 .../events/stores/primitive/ShortArrayStore.java      |  5 +++++
 .../events/stores/primitive/StringStoreString.java    |  5 +++++
 .../models/events/stores/root/ColumnStore.java        | 11 +----------
 .../stores/specific/CompoundDateRangeStore.java       |  2 ++
 .../events/stores/specific/DirectDateRangeStore.java  |  5 +++++
 .../models/events/stores/specific/MoneyIntStore.java  |  5 +++++
 .../events/stores/specific/QuarterDateRangeStore.java |  5 +++++
 .../events/stores/specific/RebasingIntegerStore.java  |  5 +++++
 .../events/stores/specific/ScaledDecimalStore.java    |  5 +++++
 18 files changed, 81 insertions(+), 10 deletions(-)

diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/EmptyStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/EmptyStore.java
index 2d156a3226..7201ae0ac9 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/EmptyStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/EmptyStore.java
@@ -49,6 +49,10 @@ public void setDate(int event, int value) {
 
 	}
 
+	public void setParent(Bucket bucket) {
+		// not used
+	}
+
 	@Override
 	public Object createScriptValue(int event) {
 		return null;
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/BitSetStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/BitSetStore.java
index 6cfdf2e406..2e8c4731e8 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/BitSetStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/BitSetStore.java
@@ -6,6 +6,7 @@
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.io.jackson.serializer.BitSetDeserializer;
 import com.bakdata.conquery.io.jackson.serializer.BitSetSerializer;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.BooleanStore;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.fasterxml.jackson.annotation.JsonCreator;
@@ -81,6 +82,9 @@ public BitSetStore select(int[] starts, int[] lengths) {
 		return new BitSetStore(out, outNulls, length);
 	}
 
+	public void setParent(Bucket bucket) {
+		// not used
+	}
 
 	@Override
 	public boolean has(int event) {
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/ByteArrayStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/ByteArrayStore.java
index 7fdce17ce1..2c035314de 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/ByteArrayStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/ByteArrayStore.java
@@ -1,6 +1,7 @@
 package com.bakdata.conquery.models.events.stores.primitive;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.IntegerStore;
 import com.fasterxml.jackson.annotation.JsonCreator;
@@ -71,4 +72,8 @@ public long getInteger(int event) {
 		return values[event];
 	}
 
+	public void setParent(Bucket bucket) {
+		// not used
+	}
+
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/DecimalArrayStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/DecimalArrayStore.java
index f00ab0d574..60529dffe7 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/DecimalArrayStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/DecimalArrayStore.java
@@ -3,6 +3,7 @@
 import java.math.BigDecimal;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.DecimalStore;
 import com.fasterxml.jackson.annotation.JsonCreator;
@@ -66,4 +67,8 @@ public boolean has(int event) {
 	public BigDecimal getDecimal(int event) {
 		return values[event];
 	}
+
+	public void setParent(Bucket bucket) {
+		// not used
+	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/DoubleArrayStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/DoubleArrayStore.java
index 66427a2d56..b2dfc19931 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/DoubleArrayStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/DoubleArrayStore.java
@@ -1,6 +1,7 @@
 package com.bakdata.conquery.models.events.stores.primitive;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.RealStore;
 import com.fasterxml.jackson.annotation.JsonCreator;
@@ -66,4 +67,8 @@ public boolean has(int event) {
 	public double getReal(int event) {
 		return values[event];
 	}
+
+	public void setParent(Bucket bucket) {
+		// not used
+	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/FloatArrayStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/FloatArrayStore.java
index 21634d6572..ac59e91737 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/FloatArrayStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/FloatArrayStore.java
@@ -1,6 +1,7 @@
 package com.bakdata.conquery.models.events.stores.primitive;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.RealStore;
 import com.fasterxml.jackson.annotation.JsonCreator;
@@ -66,4 +67,8 @@ public boolean has(int event) {
 	public double getReal(int event) {
 		return values[event];
 	}
+
+	public void setParent(Bucket bucket) {
+		// not used
+	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/IntArrayStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/IntArrayStore.java
index 2890d913cd..64859ea28c 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/IntArrayStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/IntArrayStore.java
@@ -1,6 +1,7 @@
 package com.bakdata.conquery.models.events.stores.primitive;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.IntegerStore;
 import com.fasterxml.jackson.annotation.JsonCreator;
@@ -71,4 +72,8 @@ public boolean has(int event) {
 	public long getInteger(int event) {
 		return values[event];
 	}
+
+	public void setParent(Bucket bucket) {
+		// not used
+	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/IntegerDateStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/IntegerDateStore.java
index 83bed84cdd..7abdab5d38 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/IntegerDateStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/IntegerDateStore.java
@@ -2,6 +2,7 @@
 
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.models.common.CDate;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.DateStore;
 import com.bakdata.conquery.models.events.stores.root.IntegerStore;
@@ -69,4 +70,8 @@ public int getDate(int event) {
 		return (int) store.getInteger(event);
 	}
 
+	public void setParent(Bucket bucket) {
+		// not used
+	}
+
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/LongArrayStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/LongArrayStore.java
index 6a17aee351..3ab3b874bb 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/LongArrayStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/LongArrayStore.java
@@ -1,6 +1,7 @@
 package com.bakdata.conquery.models.events.stores.primitive;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.IntegerStore;
 import com.fasterxml.jackson.annotation.JsonCreator;
@@ -70,4 +71,8 @@ public long getInteger(int event) {
 		return values[event];
 	}
 
+	public void setParent(Bucket bucket) {
+		// not used
+	}
+
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/ShortArrayStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/ShortArrayStore.java
index fe2bbdce5b..a5180373e9 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/ShortArrayStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/ShortArrayStore.java
@@ -1,6 +1,7 @@
 package com.bakdata.conquery.models.events.stores.primitive;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.IntegerStore;
 import com.fasterxml.jackson.annotation.JsonCreator;
@@ -70,4 +71,8 @@ public long getInteger(int event) {
 		return values[event];
 	}
 
+	public void setParent(Bucket bucket) {
+		// not used
+	}
+
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/StringStoreString.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/StringStoreString.java
index 58524e069a..d78e25e11f 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/StringStoreString.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/StringStoreString.java
@@ -5,6 +5,7 @@
 import java.util.stream.Stream;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.StringStore;
 import com.fasterxml.jackson.annotation.JsonCreator;
@@ -96,4 +97,8 @@ public Stream<String> streamValues() {
 		return Arrays.stream(values).filter(Objects::nonNull);
 	}
 
+	public void setParent(Bucket bucket) {
+		// not used
+	}
+
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/ColumnStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/ColumnStore.java
index 93db593245..d0ca7b1cf6 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/ColumnStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/ColumnStore.java
@@ -31,9 +31,7 @@ public interface ColumnStore {
 	 * @implNote BackReference set here because Jackson does not support for fields in interfaces and abstract classes see also https://github.com/FasterXML/jackson-databind/issues/3304
 	 */
 	@JsonBackReference
-	default void setParent(Bucket bucket) {
-
-	}
+	void setParent(Bucket bucket);
 
 
 	/**
@@ -92,13 +90,6 @@ default long estimateMemoryConsumptionBytes() {
 	@JsonIgnore
 	int getLines();
 
-	/**
-	 * Bytes required to store auxilary data.
-	 */
-	default long estimateTypeSizeBytes() {
-		return 0;
-	}
-
 	/**
 	 * Create an empty store that's only a description of the transformation.
 	 */
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/CompoundDateRangeStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/CompoundDateRangeStore.java
index 9c72287b4e..c116f9142b 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/CompoundDateRangeStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/CompoundDateRangeStore.java
@@ -7,6 +7,7 @@
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.DateRangeStore;
 import com.bakdata.conquery.models.events.stores.root.DateStore;
+import com.fasterxml.jackson.annotation.JsonBackReference;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import jakarta.validation.constraints.NotEmpty;
 import jakarta.validation.constraints.NotNull;
@@ -89,6 +90,7 @@ public DateStore getEndStore() {
 	}
 
 	@Override
+	@JsonBackReference
 	public void setParent(@NonNull Bucket bucket) {
 		parent = bucket;
 	}
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/DirectDateRangeStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/DirectDateRangeStore.java
index fbaeadc9a8..dcef9bc254 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/DirectDateRangeStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/DirectDateRangeStore.java
@@ -2,6 +2,7 @@
 
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.models.common.daterange.CDateRange;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.primitive.IntegerDateStore;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.DateRangeStore;
@@ -29,6 +30,10 @@ public DirectDateRangeStore(DateStore minStore, DateStore maxStore) {
 		this.maxStore = maxStore;
 	}
 
+	public void setParent(Bucket bucket) {
+		// not used
+	}
+
 	@Override
 	public int getLines() {
 		// they can be unaligned, if one of them is empty.
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java
index c3d38474d8..c66faaae0e 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java
@@ -1,6 +1,7 @@
 package com.bakdata.conquery.models.events.stores.specific;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.IntegerStore;
 import com.bakdata.conquery.models.events.stores.root.MoneyStore;
@@ -61,4 +62,8 @@ public void setNull(int event) {
 	public final boolean has(int event) {
 		return numberType.has(event);
 	}
+
+	public void setParent(Bucket bucket) {
+		// not used
+	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/QuarterDateRangeStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/QuarterDateRangeStore.java
index 1173ac01bb..d441f771fd 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/QuarterDateRangeStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/QuarterDateRangeStore.java
@@ -6,6 +6,7 @@
 import com.bakdata.conquery.models.common.CDate;
 import com.bakdata.conquery.models.common.QuarterUtils;
 import com.bakdata.conquery.models.common.daterange.CDateRange;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.DateRangeStore;
 import com.bakdata.conquery.models.events.stores.root.IntegerStore;
@@ -75,4 +76,8 @@ public CDateRange getDateRange(int event) {
 
 		return CDateRange.of(begin, CDate.ofLocalDate(end));
 	}
+
+	public void setParent(Bucket bucket) {
+		// not used
+	}
 }
\ No newline at end of file
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/RebasingIntegerStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/RebasingIntegerStore.java
index 559457c0d8..ab26940bb8 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/RebasingIntegerStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/RebasingIntegerStore.java
@@ -1,6 +1,7 @@
 package com.bakdata.conquery.models.events.stores.specific;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.IntegerStore;
 import jakarta.validation.constraints.NotNull;
@@ -68,4 +69,8 @@ public boolean has(int event) {
 	public void setNull(int event) {
 		store.setNull(event);
 	}
+
+	public void setParent(Bucket bucket) {
+		// not used
+	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/ScaledDecimalStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/ScaledDecimalStore.java
index a7f368ba6f..802d1d6b92 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/ScaledDecimalStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/ScaledDecimalStore.java
@@ -4,6 +4,7 @@
 import java.math.BigInteger;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.bakdata.conquery.models.events.stores.root.DecimalStore;
 import com.bakdata.conquery.models.events.stores.root.IntegerStore;
@@ -75,4 +76,8 @@ public static BigDecimal scale(int scale, long value) {
 	public boolean has(int event) {
 		return subType.has(event);
 	}
+
+	public void setParent(Bucket bucket) {
+		// not used
+	}
 }
\ No newline at end of file

From 5e1e058b15c243441572a8da19c1aa9720ee1c31 Mon Sep 17 00:00:00 2001
From: awildturtok <1553491+awildturtok@users.noreply.github.com>
Date: Thu, 25 Jul 2024 11:30:55 +0200
Subject: [PATCH 29/41] minor cleanup, ensure that setNull is not used as
 JsonProperty

---
 .../bakdata/conquery/models/events/stores/root/ColumnStore.java  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/ColumnStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/ColumnStore.java
index d0ca7b1cf6..1558fa67a2 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/ColumnStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/ColumnStore.java
@@ -106,6 +106,7 @@ static <T extends ColumnStore> T emptyCopy(T store) {
 	 */
 	<T extends ColumnStore> T select(int[] starts, int[] lengths);
 
+	@JsonIgnore
 	void setNull(int event);
 
 

From 69778bd950cabf4b419ed12c40f971d954499d84 Mon Sep 17 00:00:00 2001
From: awildturtok <1553491+awildturtok@users.noreply.github.com>
Date: Thu, 25 Jul 2024 15:44:06 +0200
Subject: [PATCH 30/41] dont use config override in Conquery

---
 .../conquery/models/config/IdColumnConfig.java  |  4 ++--
 .../integration/json/ConqueryTestSpec.java      | 17 ++---------------
 2 files changed, 4 insertions(+), 17 deletions(-)

diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java
index d49d6fcbb7..9c8d1202f6 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java
@@ -44,7 +44,7 @@ public class IdColumnConfig {
 	/**
 	 * Relevant in SQL-Mode, used as AllIdsTable for CQExternal and CQYes.
 	 */
-	private String table;
+	private String table = "entities";
 
 	/**
 	 * List of resolvable and printable ids.
@@ -56,7 +56,7 @@ public class IdColumnConfig {
 	private List<ColumnConfig> ids = List.of(
 			ColumnConfig.builder()
 						.name("ID")
-						.field("result")
+						.field("pid")
 						.label(Map.of(Locale.ROOT, "result"))
 						.primaryId(true)
 						.print(true)
diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java b/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java
index 68c9b70818..cefa4caf5c 100644
--- a/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java
+++ b/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java
@@ -3,17 +3,14 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Locale;
-import java.util.Map;
 import java.util.function.Consumer;
+
 import javax.annotation.Nullable;
 
 import com.bakdata.conquery.integration.IntegrationTest;
 import com.bakdata.conquery.io.cps.CPSBase;
 import com.bakdata.conquery.io.jackson.Jackson;
 import com.bakdata.conquery.io.jackson.View;
-import com.bakdata.conquery.io.storage.MetaStorage;
-import com.bakdata.conquery.models.config.ColumnConfig;
 import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.config.Dialect;
 import com.bakdata.conquery.models.config.IdColumnConfig;
@@ -56,17 +53,7 @@ public abstract class ConqueryTestSpec {
 	SqlSpec sqlSpec;
 
 	// default IdColumnConfig for SQL mode
-	private IdColumnConfig idColumns = new IdColumnConfig(
-			"entities",
-			List.of(
-					ColumnConfig.builder()
-								.name("ID")
-								.field("pid")
-								.primaryId(true)
-								.label(Map.of(Locale.ROOT, "result"))
-								.build()
-			)
-	);
+	private IdColumnConfig idColumns = null;
 
 	public ConqueryConfig overrideConfig(ConqueryConfig config) {
 

From 4895e068356be49ade69550b13e0dd92d7d44a90 Mon Sep 17 00:00:00 2001
From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com>
Date: Fri, 26 Jul 2024 09:46:32 +0200
Subject: [PATCH 31/41] catches serdes error for specific mapper and prints its
 view

---
 .../jackson/serializer/SerializationTestUtil.java | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java
index a6a5de9174..bec921c38e 100644
--- a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java
+++ b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java
@@ -101,11 +101,16 @@ public void test(T value, T expected) throws JSONException, IOException {
 		}
 
 		for (ObjectMapper objectMapper : objectMappers) {
-			test(
-					value,
-					expected,
-					objectMapper
-			);
+			try {
+				test(
+						value,
+						expected,
+						objectMapper
+				);
+			} catch (Exception e) {
+				Class<?> activeView = objectMapper.getSerializationConfig().getActiveView();
+				throw new IllegalStateException("Serdes failed with object mapper using view '" + activeView + "'", e);
+			}
 		}
 	}
 

From 038f93ad4034d51005cd24172c37824c3e83dd0d Mon Sep 17 00:00:00 2001
From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com>
Date: Mon, 29 Jul 2024 13:59:11 +0200
Subject: [PATCH 32/41] remove TestConqueryConfig class as it was not used as
 an JUnit Extention anymore

---
 .../integration/IntegrationTests.java         | 32 ++-----------------
 .../conquery/util/support/ConfigOverride.java | 19 +++--------
 2 files changed, 8 insertions(+), 43 deletions(-)

diff --git a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java
index 29ace90e72..4506672856 100644
--- a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java
+++ b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java
@@ -7,14 +7,7 @@
 import java.io.InputStream;
 import java.net.URI;
 import java.nio.file.Files;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
+import java.util.*;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -39,17 +32,12 @@
 import com.fasterxml.jackson.databind.ObjectWriter;
 import com.google.common.base.Strings;
 import io.github.classgraph.Resource;
-import lombok.EqualsAndHashCode;
 import lombok.Getter;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.junit.jupiter.api.DynamicContainer;
 import org.junit.jupiter.api.DynamicNode;
 import org.junit.jupiter.api.DynamicTest;
-import org.junit.jupiter.api.extension.BeforeAllCallback;
-import org.junit.jupiter.api.extension.Extension;
-import org.junit.jupiter.api.extension.ExtensionContext;
-import org.junit.jupiter.api.extension.RegisterExtension;
 
 @Slf4j
 public class IntegrationTests {
@@ -79,15 +67,14 @@ public class IntegrationTests {
 	@Getter
 	private final File workDir;
 	@Getter
-	@RegisterExtension
-	public TestConqueryConfig config;
+	public ConqueryConfig config;
 
 	@SneakyThrows(IOException.class)
 	public IntegrationTests(String defaultTestRoot, String defaultTestRootPackage) {
 		this.defaultTestRoot = defaultTestRoot;
 		this.defaultTestRootPackage = defaultTestRootPackage;
 		this.workDir = Files.createTempDirectory("conqueryIntegrationTest").toFile();
-		this.config = new TestConqueryConfig();
+		this.config = new ConqueryConfig();
 		ConfigOverride.configurePathsAndLogging(this.config, this.workDir);
 	}
 
@@ -262,19 +249,6 @@ public synchronized TestConquery getCachedConqueryInstance(File workDir, Conquer
 		return conquery;
 	}
 
-	@EqualsAndHashCode(callSuper = true)
-	public static class TestConqueryConfig extends ConqueryConfig implements Extension, BeforeAllCallback {
-
-		@Override
-		public void beforeAll(ExtensionContext context) throws Exception {
-
-			context.getTestInstance()
-				   .filter(ConfigOverride.class::isInstance)
-				   .map(ConfigOverride.class::cast)
-				   .ifPresent(co -> co.override(this));
-		}
-	}
-
 	private static ResourceTree scanForResources(String testRoot, String pattern) {
 		ResourceTree tree = new ResourceTree(null, null);
 		tree.addAll(CPSTypeIdResolver.SCAN_RESULT.getResourcesMatchingPattern(Pattern.compile("^" + testRoot + pattern)));
diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/ConfigOverride.java b/backend/src/test/java/com/bakdata/conquery/util/support/ConfigOverride.java
index ec595bb877..3d5dfa73a1 100644
--- a/backend/src/test/java/com/bakdata/conquery/util/support/ConfigOverride.java
+++ b/backend/src/test/java/com/bakdata/conquery/util/support/ConfigOverride.java
@@ -10,18 +10,17 @@
 import io.dropwizard.jetty.ConnectorFactory;
 import io.dropwizard.jetty.HttpConnectorFactory;
 import lombok.SneakyThrows;
+import lombok.experimental.UtilityClass;
 import org.apache.commons.collections4.CollectionUtils;
-import org.junit.jupiter.api.TestInstance;
-import org.junit.jupiter.api.TestInstance.Lifecycle;
 
 /**
  * This interface allows to override the configuration used in tests.
  */
-@TestInstance(Lifecycle.PER_CLASS)
-public interface ConfigOverride {
+@UtilityClass
+public final class ConfigOverride {
 
 	@SneakyThrows
-	static void configurePathsAndLogging(ConqueryConfig config, File tmpDir) {
+	public static void configurePathsAndLogging(ConqueryConfig config, File tmpDir) {
 
 		config.setFailOnError(true);
 
@@ -37,7 +36,7 @@ static void configurePathsAndLogging(ConqueryConfig config, File tmpDir) {
 	}
 
 	@SneakyThrows
-	static void configureRandomPorts(ConqueryConfig config) {
+	public static void configureRandomPorts(ConqueryConfig config) {
 
 		// set random open ports
 		final Collection<ConnectorFactory> connectorFactories = CollectionUtils.union(
@@ -54,12 +53,4 @@ static void configureRandomPorts(ConqueryConfig config) {
 			config.getCluster().setPort(s.getLocalPort());
 		}
 	}
-
-	/**
-	 * Is called upon initialization of the test instance of Conquery.
-	 *
-	 * @param config The configuration that is initialized with the defaults.
-	 */
-	void override(ConqueryConfig config);
-
 }

From 3667d795ade6a79b0998a777b62e0e59c35bdc67 Mon Sep 17 00:00:00 2001
From: Jonas Arnhold <jonas.arnhold@bakdata.com>
Date: Mon, 29 Jul 2024 16:04:56 +0200
Subject: [PATCH 33/41] Fix ConceptColumnSelect conversion if there is only 1
 connector (#3495)

---
 .../cqelement/concept/CQConceptConverter.java |   8 +-
 .../cqelement/concept/ConceptSqlTables.java   |   7 +-
 .../cqelement/concept/ConnectorSqlTables.java |   8 ++
 .../cqelement/concept/TablePath.java          |   3 +-
 .../CountQuartersSqlAggregator.java           |   3 +-
 .../model/aggregator/CountSqlAggregator.java  |   3 +-
 .../aggregator/DateDistanceSqlAggregator.java |   3 +-
 .../model/aggregator/FlagSqlAggregator.java   |   3 +-
 .../model/aggregator/SumSqlAggregator.java    |   3 +-
 .../select/ConceptColumnSelectConverter.java  |  61 +++++-----
 .../select/DateUnionSelectConverter.java      |   3 +-
 .../model/select/DaterangeSelectUtil.java     |   7 +-
 .../model/select/DistinctSelectConverter.java |   7 +-
 .../select/DurationSumSelectConverter.java    |   3 +-
 .../select/EventDateUnionSelectConverter.java |   8 +-
 .../EventDurationSumSelectConverter.java      |   8 +-
 .../model/select/ExistsSelectConverter.java   |   8 +-
 .../select/FirstValueSelectConverter.java     |   3 +-
 .../select/LastValueSelectConverter.java      |   3 +-
 .../select/RandomValueSelectConverter.java    |   3 +-
 .../model/select/SelectContext.java           |  17 +--
 .../model/select/SelectConverter.java         |   6 +-
 .../model/select/ValueSelectUtil.java         |   3 +-
 .../sql/selects/concept_values/content.csv    |   8 ++
 .../sql/selects/concept_values/content2.csv   |   6 +
 .../sql/selects/concept_values/expected.csv   |   5 +
 .../concept_values/single_connector.json      | 112 ++++++++++++++++++
 ...oncept_values.json => two_connectors.json} |   8 +-
 28 files changed, 210 insertions(+), 110 deletions(-)
 create mode 100644 backend/src/test/resources/tests/sql/selects/concept_values/content.csv
 create mode 100644 backend/src/test/resources/tests/sql/selects/concept_values/content2.csv
 create mode 100644 backend/src/test/resources/tests/sql/selects/concept_values/expected.csv
 create mode 100644 backend/src/test/resources/tests/sql/selects/concept_values/single_connector.json
 rename backend/src/test/resources/tests/sql/selects/concept_values/{concept_values.json => two_connectors.json} (89%)

diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java
index 6cac5ba991..a8fa813a3f 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java
@@ -11,11 +11,9 @@
 import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept;
 import com.bakdata.conquery.models.datasets.Column;
 import com.bakdata.conquery.models.datasets.concepts.ConceptElement;
-import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.datasets.concepts.select.concept.ConceptColumnSelect;
 import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeChild;
 import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeNode;
-import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept;
 import com.bakdata.conquery.models.query.queryplan.DateAggregationAction;
 import com.bakdata.conquery.sql.conversion.NodeConverter;
 import com.bakdata.conquery.sql.conversion.SharedAliases;
@@ -99,7 +97,7 @@ private static QueryStep finishConceptConversion(QueryStep predecessor, CQConcep
 		Optional<ColumnDateRange> validityDate = predecessorSelects.getValidityDate();
 		SqlIdColumns ids = predecessorSelects.getIds();
 
-		SelectContext<TreeConcept, ConceptSqlTables> selectContext = SelectContext.create(cqConcept, ids, validityDate, universalTables, context);
+		SelectContext<ConceptSqlTables> selectContext = SelectContext.create(cqConcept, ids, validityDate, universalTables, context);
 		List<ConceptSqlSelects> converted = cqConcept.getSelects().stream()
 													 .map(select -> select.createConverter().conceptSelect(select, selectContext))
 													 .toList();
@@ -153,7 +151,7 @@ private CQTableContext createTableContext(TablePath tablePath, CQConcept cqConce
 		getDateRestriction(conversionContext, tablesValidityDate).ifPresent(allSqlFiltersForTable::add);
 
 		// convert selects
-		SelectContext<Connector, ConnectorSqlTables> selectContext = SelectContext.create(cqTable, ids, tablesValidityDate, connectorTables, conversionContext);
+		SelectContext<ConnectorSqlTables> selectContext = SelectContext.create(cqTable, ids, tablesValidityDate, connectorTables, conversionContext);
 		List<ConnectorSqlSelects> allSelectsForTable = new ArrayList<>();
 		ConnectorSqlSelects conceptColumnSelect = createConceptColumnConnectorSqlSelects(cqConcept, selectContext);
 		allSelectsForTable.add(conceptColumnSelect);
@@ -277,7 +275,7 @@ private static Optional<SqlFilters> getDateRestriction(ConversionContext context
 		));
 	}
 
-	private static ConnectorSqlSelects createConceptColumnConnectorSqlSelects(CQConcept cqConcept, SelectContext<Connector, ConnectorSqlTables> selectContext) {
+	private static ConnectorSqlSelects createConceptColumnConnectorSqlSelects(CQConcept cqConcept, SelectContext<ConnectorSqlTables> selectContext) {
 		return cqConcept.getSelects().stream()
 						.filter(select -> select instanceof ConceptColumnSelect)
 						.findFirst()
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java
index ce9c58994f..2fda1a3295 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java
@@ -4,22 +4,21 @@
 import java.util.Map;
 
 import com.bakdata.conquery.sql.conversion.model.CteStep;
+import com.bakdata.conquery.sql.conversion.model.SqlTables;
 import lombok.Getter;
 
 @Getter
-public class ConceptSqlTables extends ConnectorSqlTables {
+public class ConceptSqlTables extends SqlTables {
 
 	private final List<ConnectorSqlTables> connectorTables;
 
 	public ConceptSqlTables(
-			String conceptConnectorLabel,
 			String rootTable,
 			Map<CteStep, String> cteNameMap,
 			Map<CteStep, CteStep> predecessorMap,
-			boolean containsIntervalPacking,
 			List<ConnectorSqlTables> connectorTables
 	) {
-		super(conceptConnectorLabel, rootTable, cteNameMap, predecessorMap, containsIntervalPacking);
+		super(rootTable, cteNameMap, predecessorMap);
 		this.connectorTables = connectorTables;
 	}
 
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java
index 6f226a4119..fe9443edc1 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java
@@ -2,6 +2,7 @@
 
 import java.util.Map;
 
+import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep;
 import com.bakdata.conquery.sql.conversion.model.CteStep;
 import com.bakdata.conquery.sql.conversion.model.SqlTables;
@@ -20,7 +21,13 @@ public class ConnectorSqlTables extends SqlTables {
 	 */
 	private final boolean withIntervalPacking;
 
+	/**
+	 * Corresponding {@link Connector} of these {@link SqlTables}.
+	 */
+	private final Connector connector;
+
 	public ConnectorSqlTables(
+			Connector connector,
 			String conceptConnectorLabel,
 			String rootTable,
 			Map<CteStep, String> cteNameMap,
@@ -28,6 +35,7 @@ public ConnectorSqlTables(
 			boolean containsIntervalPacking
 	) {
 		super(rootTable, cteNameMap, predecessorMap);
+		this.connector = connector;
 		this.label = conceptConnectorLabel;
 		this.withIntervalPacking = containsIntervalPacking;
 	}
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java
index 452a0b147f..5b48c2d3be 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java
@@ -54,6 +54,7 @@ private static ConnectorSqlTables createConnectorTables(CQConcept cqConcept, CQT
 		Map<CteStep, String> cteNameMap = CteStep.createCteNameMap(tableInfo.getMappings().keySet(), conceptConnectorLabel, context.getNameGenerator());
 
 		return new ConnectorSqlTables(
+				cqTable.getConnector(),
 				conceptConnectorLabel,
 				tableInfo.getRootTable(),
 				cteNameMap,
@@ -70,11 +71,9 @@ public ConceptSqlTables createConceptTables(QueryStep predecessor) {
 		List<ConnectorSqlTables> connectorSqlTables = this.connectorTableMap.values().stream().toList();
 
 		return new ConceptSqlTables(
-				conceptName,
 				tableInfo.getRootTable(),
 				cteNameMap,
 				tableInfo.getMappings(),
-				tableInfo.isContainsIntervalPacking(),
 				connectorSqlTables
 		);
 	}
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java
index 7dd3494050..6c8e6fd6a7 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java
@@ -6,7 +6,6 @@
 
 import com.bakdata.conquery.models.common.Range;
 import com.bakdata.conquery.models.datasets.Column;
-import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.datasets.concepts.filters.specific.CountQuartersFilter;
 import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.CountQuartersSelect;
 import com.bakdata.conquery.models.events.MajorTypeId;
@@ -34,7 +33,7 @@
 public class CountQuartersSqlAggregator implements SelectConverter<CountQuartersSelect>, FilterConverter<CountQuartersFilter, Range.LongRange>, SqlAggregator {
 
 	@Override
-	public ConnectorSqlSelects connectorSelect(CountQuartersSelect countQuartersSelect, SelectContext<Connector, ConnectorSqlTables> selectContext) {
+	public ConnectorSqlSelects connectorSelect(CountQuartersSelect countQuartersSelect, SelectContext<ConnectorSqlTables> selectContext) {
 
 		String alias = selectContext.getNameGenerator().selectName(countQuartersSelect);
 		ConnectorSqlTables tables = selectContext.getTables();
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java
index 8d41ce66a7..501340d79f 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java
@@ -2,7 +2,6 @@
 
 import com.bakdata.conquery.models.common.Range;
 import com.bakdata.conquery.models.datasets.Column;
-import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.datasets.concepts.filters.specific.CountFilter;
 import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.CountSelect;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
@@ -36,7 +35,7 @@ public static CountType fromBoolean(boolean value) {
 	}
 
 	@Override
-	public ConnectorSqlSelects connectorSelect(CountSelect countSelect, SelectContext<Connector, ConnectorSqlTables> selectContext) {
+	public ConnectorSqlSelects connectorSelect(CountSelect countSelect, SelectContext<ConnectorSqlTables> selectContext) {
 
 		ConnectorSqlTables tables = selectContext.getTables();
 		CountType countType = CountType.fromBoolean(countSelect.isDistinct());
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java
index 9545d3b39b..ff7aba4043 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java
@@ -7,7 +7,6 @@
 import com.bakdata.conquery.models.common.Range;
 import com.bakdata.conquery.models.common.daterange.CDateRange;
 import com.bakdata.conquery.models.datasets.Column;
-import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.datasets.concepts.filters.specific.DateDistanceFilter;
 import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.DateDistanceSelect;
 import com.bakdata.conquery.models.events.MajorTypeId;
@@ -36,7 +35,7 @@
 public class DateDistanceSqlAggregator implements SelectConverter<DateDistanceSelect>, FilterConverter<DateDistanceFilter, Range.LongRange> {
 
 	@Override
-	public ConnectorSqlSelects connectorSelect(DateDistanceSelect select, SelectContext<Connector, ConnectorSqlTables> selectContext) {
+	public ConnectorSqlSelects connectorSelect(DateDistanceSelect select, SelectContext<ConnectorSqlTables> selectContext) {
 
 		Column column = select.getColumn();
 		String alias = selectContext.getNameGenerator().selectName(select);
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java
index 5c379e53f5..a1505b3447 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java
@@ -7,7 +7,6 @@
 import java.util.stream.Collectors;
 
 import com.bakdata.conquery.models.datasets.Column;
-import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.datasets.concepts.filters.specific.FlagFilter;
 import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.FlagSelect;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
@@ -73,7 +72,7 @@ public class FlagSqlAggregator implements SelectConverter<FlagSelect>, FilterCon
 	private static final Param<Integer> NUMERIC_TRUE_VAL = DSL.val(1);
 
 	@Override
-	public ConnectorSqlSelects connectorSelect(FlagSelect flagSelect, SelectContext<Connector, ConnectorSqlTables> selectContext) {
+	public ConnectorSqlSelects connectorSelect(FlagSelect flagSelect, SelectContext<ConnectorSqlTables> selectContext) {
 
 		SqlFunctionProvider functionProvider = selectContext.getConversionContext().getSqlDialect().getFunctionProvider();
 		SqlTables connectorTables = selectContext.getTables();
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java
index 5ea0d55527..95c2d39122 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java
@@ -8,7 +8,6 @@
 
 import com.bakdata.conquery.models.common.IRange;
 import com.bakdata.conquery.models.datasets.Column;
-import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.datasets.concepts.filters.specific.SumFilter;
 import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.SumSelect;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
@@ -96,7 +95,7 @@ private enum SumDistinctCteStep implements CteStep {
 	private static final String SUM_DISTINCT_SUFFIX = "sum_distinct";
 
 	@Override
-	public ConnectorSqlSelects connectorSelect(SumSelect sumSelect, SelectContext<Connector, ConnectorSqlTables> selectContext) {
+	public ConnectorSqlSelects connectorSelect(SumSelect sumSelect, SelectContext<ConnectorSqlTables> selectContext) {
 
 		Column sumColumn = sumSelect.getColumn();
 		Column subtractColumn = sumSelect.getSubtractColumn();
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java
index b91da43125..02e770e89d 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java
@@ -7,7 +7,6 @@
 
 import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.datasets.concepts.select.concept.ConceptColumnSelect;
-import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept;
 import com.bakdata.conquery.sql.conversion.SharedAliases;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables;
@@ -42,8 +41,8 @@ private enum CONCEPT_COLUMN_STEPS implements CteStep {
 	}
 
 	@Override
-	public ConnectorSqlSelects connectorSelect(ConceptColumnSelect select, SelectContext<Connector, ConnectorSqlTables> selectContext) {
-		Connector connector = selectContext.getSelectHolder();
+	public ConnectorSqlSelects connectorSelect(ConceptColumnSelect select, SelectContext<ConnectorSqlTables> selectContext) {
+		Connector connector = selectContext.getTables().getConnector();
 		if (connector.getColumn() == null) {
 			return ConnectorSqlSelects.none();
 		}
@@ -56,17 +55,17 @@ public ConnectorSqlSelects connectorSelect(ConceptColumnSelect select, SelectCon
 	}
 
 	@Override
-	public ConceptSqlSelects conceptSelect(ConceptColumnSelect select, SelectContext<TreeConcept, ConceptSqlTables> selectContext) {
+	public ConceptSqlSelects conceptSelect(ConceptColumnSelect select, SelectContext<ConceptSqlTables> selectContext) {
 
 		// we will do a union distinct on all Connector tables
 		List<? extends Connector> connectors;
-		if (isSingleConnectorConcept(selectContext.getSelectHolder())) {
+		if (isSingleConnector(selectContext.getTables())) {
 			// we union the Connector table with itself if there is only 1 Connector
-			Connector connector = selectContext.getSelectHolder().getConcept().getConnectors().get(0);
+			Connector connector = selectContext.getTables().getConnectorTables().get(0).getConnector();
 			connectors = List.of(connector, connector);
 		}
 		else {
-			connectors = selectContext.getSelectHolder().getConnectors();
+			connectors = selectContext.getTables().getConnectorTables().stream().map(ConnectorSqlTables::getConnector).toList();
 		}
 
 		NameGenerator nameGenerator = selectContext.getNameGenerator();
@@ -97,44 +96,42 @@ public ConceptSqlSelects conceptSelect(ConceptColumnSelect select, SelectContext
 								.build();
 	}
 
-	private static boolean isSingleConnectorConcept(TreeConcept treeConcept) {
-		return treeConcept.getConcept().getConnectors().size() == 1;
+	private static boolean isSingleConnector(ConceptSqlTables tables) {
+		return tables.getConnectorTables().size() == 1;
 	}
 
 	private static QueryStep createUnionConnectorConnectorsStep(
 			List<? extends Connector> connectors,
 			String alias,
-			SelectContext<TreeConcept, ConceptSqlTables> selectContext
+			SelectContext<ConceptSqlTables> selectContext
 	) {
-		List<QueryStep> unionSteps = selectContext.getTables()
-												  .getConnectorTables()
-												  .stream()
-												  .map(tables -> createConnectorColumnSelectQuery(tables, connectors, alias, selectContext))
-												  .toList();
-
+		List<QueryStep> unionSteps = connectors.stream().map(connector -> createConnectorColumnSelectQuery(connector, alias, selectContext)).toList();
 		String unionedColumnsCteName = selectContext.getNameGenerator().cteStepName(CONCEPT_COLUMN_STEPS.UNIONED_COLUMNS, alias);
 		return QueryStep.createUnionStep(unionSteps, unionedColumnsCteName, Collections.emptyList());
 	}
 
 	private static QueryStep createConnectorColumnSelectQuery(
-			ConnectorSqlTables tables,
-			List<? extends Connector> connectors,
+			Connector connector,
 			String alias,
-			SelectContext<TreeConcept, ConceptSqlTables> selectContext
+			SelectContext<ConceptSqlTables> selectContext
 	) {
-		Connector matchingConnector =
-				connectors.stream()
-						  .filter(connector -> isMatchingConnector(tables, connector))
-						  .findFirst()
-						  .orElseThrow(() -> new IllegalStateException("Could not find matching connector for ConnectorSqlTables %s".formatted(tables)));
-
-		Table<Record> connectorTable = DSL.table(DSL.name(tables.cteName(ConceptCteStep.EVENT_FILTER)));
-
-		Field<Object> primaryColumn = TablePrimaryColumnUtil.findPrimaryColumn(matchingConnector.getTable(), selectContext.getConversionContext().getConfig());
+		// a  ConceptColumn select uses all connectors a Concept has, even if they are not part of the CQConcept
+		// but if they are, we need to make sure we use the event-filtered table instead of the root table
+		String tableName = selectContext.getTables()
+										.getConnectorTables()
+										.stream()
+										.filter(tables -> Objects.equals(tables.getRootTable(), connector.getTable().getName()))
+										.findFirst()
+										.map(tables -> tables.cteName(ConceptCteStep.EVENT_FILTER))
+										.orElse(connector.getTable().getName());
+
+		Table<Record> connectorTable = DSL.table(DSL.name(tableName));
+
+		Field<Object> primaryColumn = TablePrimaryColumnUtil.findPrimaryColumn(connector.getTable(), selectContext.getConversionContext().getConfig());
 		Field<Object> qualifiedPrimaryColumn = QualifyingUtil.qualify(primaryColumn, connectorTable.getName()).as(SharedAliases.PRIMARY_COLUMN.getAlias());
 		SqlIdColumns ids = new SqlIdColumns(qualifiedPrimaryColumn);
 
-		Field<Object> connectorColumn = DSL.field(DSL.name(connectorTable.getName(), matchingConnector.getColumn().getName()));
+		Field<Object> connectorColumn = DSL.field(DSL.name(connectorTable.getName(), connector.getColumn().getName()));
 		Field<String> casted = selectContext.getFunctionProvider().cast(connectorColumn, SQLDataType.VARCHAR).as(alias);
 		FieldWrapper<String> connectorSelect = new FieldWrapper<>(casted);
 
@@ -149,11 +146,7 @@ private static QueryStep createConnectorColumnSelectQuery(
 						.build();
 	}
 
-	private static boolean isMatchingConnector(final ConnectorSqlTables tables, final Connector connector) {
-		return connector.getColumn() != null && (Objects.equals(tables.getRootTable(), connector.getTable().getName()));
-	}
-
-	private static FieldWrapper<String> createConnectorColumnStringAgg(SelectContext<TreeConcept, ConceptSqlTables> selectContext, QueryStep unionStep, String alias) {
+	private static FieldWrapper<String> createConnectorColumnStringAgg(SelectContext<ConceptSqlTables> selectContext, QueryStep unionStep, String alias) {
 		SqlFunctionProvider functionProvider = selectContext.getFunctionProvider();
 		Field<String> unionedColumn = DSL.field(DSL.name(unionStep.getCteName(), alias), String.class);
 		return new FieldWrapper<>(
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java
index d2c04e9c34..b479b6859b 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java
@@ -1,13 +1,12 @@
 package com.bakdata.conquery.sql.conversion.model.select;
 
-import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.DateUnionSelect;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables;
 
 public class DateUnionSelectConverter implements SelectConverter<DateUnionSelect> {
 
 	@Override
-	public ConnectorSqlSelects connectorSelect(DateUnionSelect select, SelectContext<Connector, ConnectorSqlTables> selectContext) {
+	public ConnectorSqlSelects connectorSelect(DateUnionSelect select, SelectContext<ConnectorSqlTables> selectContext) {
 		return DaterangeSelectUtil.createConnectorSqlSelects(
 				select,
 				(daterange, alias, functionProvider) -> new FieldWrapper<>(functionProvider.daterangeStringAggregation(daterange).as(alias)),
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java
index e44f03b12b..04bbc0b0cf 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java
@@ -14,7 +14,6 @@
 import java.util.Optional;
 import java.util.stream.Collectors;
 
-import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.datasets.concepts.DaterangeSelectOrFilter;
 import com.bakdata.conquery.models.identifiable.Named;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
@@ -45,7 +44,7 @@ public interface AggregationFunction {
 	public static <S extends DaterangeSelectOrFilter & Named<?>> ConnectorSqlSelects createConnectorSqlSelects(
 			S select,
 			AggregationFunction aggregationFunction,
-			SelectContext<Connector, ConnectorSqlTables> context
+			SelectContext<ConnectorSqlTables> context
 	) {
 		String alias = context.getNameGenerator().selectName(select);
 		ConnectorSqlTables tables = context.getTables();
@@ -95,7 +94,7 @@ private static Condition containsInfinityDate(ColumnDateRange validityDate, SqlF
 		return validityDate.getStart().eq(negativeInfinity).or(validityDate.getEnd().eq(positiveInfinity));
 	}
 
-	private static SqlTables createTables(String alias, SelectContext<Connector, ConnectorSqlTables> context) {
+	private static SqlTables createTables(String alias, SelectContext<ConnectorSqlTables> context) {
 		Map<CteStep, CteStep> predecessorMapping = new HashMap<>();
 		String eventFilterCteName = context.getTables().cteName(EVENT_FILTER);
 		predecessorMapping.putAll(IntervalPackingCteStep.getMappings(context.getSqlDialect()));
@@ -110,7 +109,7 @@ private static SqlTables createTables(String alias, SelectContext<Connector, Con
 		return new SqlTables(eventFilterCteName, cteNameMap, predecessorMapping);
 	}
 
-	private static QueryStep applyIntervalPacking(ColumnDateRange daterange, SqlTables dateUnionTables, SelectContext<?, ?> context) {
+	private static QueryStep applyIntervalPacking(ColumnDateRange daterange, SqlTables dateUnionTables, SelectContext<?> context) {
 
 		String eventFilterCteName = context.getTables().cteName(EVENT_FILTER);
 		IntervalPackingContext intervalPackingContext = IntervalPackingContext.builder()
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java
index ec2bf25a5f..5cde2abe5b 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java
@@ -6,7 +6,6 @@
 import java.util.List;
 import java.util.Optional;
 
-import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.datasets.concepts.select.connector.DistinctSelect;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables;
@@ -64,7 +63,7 @@ private enum DistinctSelectCteStep implements CteStep {
 	}
 
 	@Override
-	public ConnectorSqlSelects connectorSelect(DistinctSelect distinctSelect, SelectContext<Connector, ConnectorSqlTables> selectContext) {
+	public ConnectorSqlSelects connectorSelect(DistinctSelect distinctSelect, SelectContext<ConnectorSqlTables> selectContext) {
 
 		String alias = selectContext.getNameGenerator().selectName(distinctSelect);
 
@@ -84,7 +83,7 @@ public ConnectorSqlSelects connectorSelect(DistinctSelect distinctSelect, Select
 	}
 
 	private static QueryStep createAggregationCte(
-			SelectContext<Connector, ConnectorSqlTables> selectContext,
+			SelectContext<ConnectorSqlTables> selectContext,
 			FieldWrapper<Object> preprocessingSelect,
 			QueryStep distinctSelectCte,
 			String alias
@@ -113,7 +112,7 @@ private static QueryStep createAggregationCte(
 	private static QueryStep createDistinctSelectCte(
 			FieldWrapper<Object> preprocessingSelect,
 			String alias,
-			SelectContext<Connector, ConnectorSqlTables> selectContext
+			SelectContext<ConnectorSqlTables> selectContext
 	) {
 		// values to aggregate must be event-filtered first
 		String eventFilterTable = selectContext.getTables().cteName(ConceptCteStep.EVENT_FILTER);
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DurationSumSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DurationSumSelectConverter.java
index 01800ea36a..de8ec3496f 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DurationSumSelectConverter.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DurationSumSelectConverter.java
@@ -1,6 +1,5 @@
 package com.bakdata.conquery.sql.conversion.model.select;
 
-import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.DurationSumSelect;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables;
 import com.bakdata.conquery.sql.conversion.model.ColumnDateRange;
@@ -8,7 +7,7 @@
 public class DurationSumSelectConverter implements SelectConverter<DurationSumSelect> {
 
 	@Override
-	public ConnectorSqlSelects connectorSelect(DurationSumSelect select, SelectContext<Connector, ConnectorSqlTables> selectContext) {
+	public ConnectorSqlSelects connectorSelect(DurationSumSelect select, SelectContext<ConnectorSqlTables> selectContext) {
 		return DaterangeSelectUtil.createConnectorSqlSelects(
 				select,
 				(daterange, alias, functionProvider) -> {
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java
index 0558140279..852960ead1 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java
@@ -1,8 +1,6 @@
 package com.bakdata.conquery.sql.conversion.model.select;
 
-import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDateUnionSelect;
-import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables;
@@ -13,7 +11,7 @@
 public class EventDateUnionSelectConverter implements SelectConverter<EventDateUnionSelect> {
 
 	@Override
-	public ConnectorSqlSelects connectorSelect(EventDateUnionSelect select, SelectContext<Connector, ConnectorSqlTables> selectContext) {
+	public ConnectorSqlSelects connectorSelect(EventDateUnionSelect select, SelectContext<ConnectorSqlTables> selectContext) {
 
 		FieldWrapper<String> stringAggregation = createEventDateUnionAggregation(select, selectContext);
 		ExtractingSqlSelect<?> finalSelect = stringAggregation.qualify(selectContext.getTables().getPredecessor(ConceptCteStep.AGGREGATION_FILTER));
@@ -25,7 +23,7 @@ public ConnectorSqlSelects connectorSelect(EventDateUnionSelect select, SelectCo
 	}
 
 	@Override
-	public ConceptSqlSelects conceptSelect(EventDateUnionSelect select, SelectContext<TreeConcept, ConceptSqlTables> selectContext) {
+	public ConceptSqlSelects conceptSelect(EventDateUnionSelect select, SelectContext<ConceptSqlTables> selectContext) {
 
 		FieldWrapper<String> stringAggregation = createEventDateUnionAggregation(select, selectContext);
 		ExtractingSqlSelect<?> finalSelect = stringAggregation.qualify(selectContext.getTables().getPredecessor(ConceptCteStep.UNIVERSAL_SELECTS));
@@ -36,7 +34,7 @@ public ConceptSqlSelects conceptSelect(EventDateUnionSelect select, SelectContex
 								.build();
 	}
 
-	private static FieldWrapper<String> createEventDateUnionAggregation(EventDateUnionSelect select, SelectContext<?, ?> selectContext) {
+	private static FieldWrapper<String> createEventDateUnionAggregation(EventDateUnionSelect select, SelectContext<?> selectContext) {
 
 		Preconditions.checkArgument(selectContext.getValidityDate().isPresent(), "Can't convert an EventDateUnionSelect without a validity date being present");
 		ColumnDateRange validityDate = selectContext.getValidityDate().get();
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java
index 87ab0900b8..369f4f8489 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java
@@ -4,9 +4,7 @@
 import java.sql.Date;
 import java.time.temporal.ChronoUnit;
 
-import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDurationSumSelect;
-import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables;
@@ -20,7 +18,7 @@
 public class EventDurationSumSelectConverter implements SelectConverter<EventDurationSumSelect> {
 
 	@Override
-	public ConnectorSqlSelects connectorSelect(EventDurationSumSelect select, SelectContext<Connector, ConnectorSqlTables> selectContext) {
+	public ConnectorSqlSelects connectorSelect(EventDurationSumSelect select, SelectContext<ConnectorSqlTables> selectContext) {
 
 		FieldWrapper<BigDecimal> stringAggregation = createEventDurationSumAggregation(select, selectContext);
 		ExtractingSqlSelect<?> finalSelect = stringAggregation.qualify(selectContext.getTables().getPredecessor(ConceptCteStep.AGGREGATION_FILTER));
@@ -32,7 +30,7 @@ public ConnectorSqlSelects connectorSelect(EventDurationSumSelect select, Select
 	}
 
 	@Override
-	public ConceptSqlSelects conceptSelect(EventDurationSumSelect select, SelectContext<TreeConcept, ConceptSqlTables> selectContext) {
+	public ConceptSqlSelects conceptSelect(EventDurationSumSelect select, SelectContext<ConceptSqlTables> selectContext) {
 
 		FieldWrapper<BigDecimal> stringAggregation = createEventDurationSumAggregation(select, selectContext);
 		ExtractingSqlSelect<?> finalSelect = stringAggregation.qualify(selectContext.getTables().getPredecessor(ConceptCteStep.UNIVERSAL_SELECTS));
@@ -43,7 +41,7 @@ public ConceptSqlSelects conceptSelect(EventDurationSumSelect select, SelectCont
 								.build();
 	}
 
-	private FieldWrapper<BigDecimal> createEventDurationSumAggregation(EventDurationSumSelect select, SelectContext<?, ?> selectContext) {
+	private FieldWrapper<BigDecimal> createEventDurationSumAggregation(EventDurationSumSelect select, SelectContext<?> selectContext) {
 
 		Preconditions.checkArgument(selectContext.getValidityDate().isPresent(), "Can't convert an EventDateUnionSelect without a validity date being present");
 		String predecessorCteName = selectContext.getTables().getPredecessor(ConceptCteStep.INTERVAL_PACKING_SELECTS);
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java
index b0c8561aa9..b63d1d0422 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java
@@ -1,15 +1,13 @@
 package com.bakdata.conquery.sql.conversion.model.select;
 
-import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.ExistsSelect;
-import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables;
 
 public class ExistsSelectConverter implements SelectConverter<ExistsSelect> {
 
 	@Override
-	public ConnectorSqlSelects connectorSelect(ExistsSelect select, SelectContext<Connector, ConnectorSqlTables> selectContext) {
+	public ConnectorSqlSelects connectorSelect(ExistsSelect select, SelectContext<ConnectorSqlTables> selectContext) {
 		ExistsSqlSelect existsSqlSelect = createExistsSelect(select, selectContext);
 		return ConnectorSqlSelects.builder()
 								  .finalSelect(existsSqlSelect)
@@ -17,14 +15,14 @@ public ConnectorSqlSelects connectorSelect(ExistsSelect select, SelectContext<Co
 	}
 
 	@Override
-	public ConceptSqlSelects conceptSelect(ExistsSelect select, SelectContext<TreeConcept, ConceptSqlTables> selectContext) {
+	public ConceptSqlSelects conceptSelect(ExistsSelect select, SelectContext<ConceptSqlTables> selectContext) {
 		ExistsSqlSelect existsSqlSelect = createExistsSelect(select, selectContext);
 		return ConceptSqlSelects.builder()
 								.finalSelect(existsSqlSelect)
 								.build();
 	}
 
-	private static ExistsSqlSelect createExistsSelect(ExistsSelect select, SelectContext<?, ?> selectContext) {
+	private static ExistsSqlSelect createExistsSelect(ExistsSelect select, SelectContext<?> selectContext) {
 		String alias = selectContext.getNameGenerator().selectName(select);
 		return new ExistsSqlSelect(alias);
 	}
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FirstValueSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FirstValueSelectConverter.java
index 5a81632ba2..64b19538bb 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FirstValueSelectConverter.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FirstValueSelectConverter.java
@@ -1,13 +1,12 @@
 package com.bakdata.conquery.sql.conversion.model.select;
 
-import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.datasets.concepts.select.connector.FirstValueSelect;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables;
 
 public class FirstValueSelectConverter implements SelectConverter<FirstValueSelect> {
 
 	@Override
-	public ConnectorSqlSelects connectorSelect(FirstValueSelect select, SelectContext<Connector, ConnectorSqlTables> selectContext) {
+	public ConnectorSqlSelects connectorSelect(FirstValueSelect select, SelectContext<ConnectorSqlTables> selectContext) {
 		return ValueSelectUtil.createValueSelect(
 				select.getColumn(),
 				selectContext.getNameGenerator().selectName(select),
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/LastValueSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/LastValueSelectConverter.java
index 2ad575be52..06a27c7484 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/LastValueSelectConverter.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/LastValueSelectConverter.java
@@ -1,13 +1,12 @@
 package com.bakdata.conquery.sql.conversion.model.select;
 
-import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.datasets.concepts.select.connector.LastValueSelect;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables;
 
 public class LastValueSelectConverter implements SelectConverter<LastValueSelect> {
 
 	@Override
-	public ConnectorSqlSelects connectorSelect(LastValueSelect select, SelectContext<Connector, ConnectorSqlTables> selectContext) {
+	public ConnectorSqlSelects connectorSelect(LastValueSelect select, SelectContext<ConnectorSqlTables> selectContext) {
 		return ValueSelectUtil.createValueSelect(
 				select.getColumn(),
 				selectContext.getNameGenerator().selectName(select),
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java
index 8e1fbf5eef..208860a993 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java
@@ -1,6 +1,5 @@
 package com.bakdata.conquery.sql.conversion.model.select;
 
-import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.datasets.concepts.select.connector.RandomValueSelect;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables;
@@ -9,7 +8,7 @@
 public class RandomValueSelectConverter implements SelectConverter<RandomValueSelect> {
 
 	@Override
-	public ConnectorSqlSelects connectorSelect(RandomValueSelect select, SelectContext<Connector, ConnectorSqlTables> selectContext) {
+	public ConnectorSqlSelects connectorSelect(RandomValueSelect select, SelectContext<ConnectorSqlTables> selectContext) {
 
 		ConnectorSqlTables tables = selectContext.getTables();
 
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java
index 1f4ff5aee0..d37e743641 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java
@@ -4,9 +4,6 @@
 
 import com.bakdata.conquery.apiv1.query.concept.filter.CQTable;
 import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept;
-import com.bakdata.conquery.models.datasets.concepts.Connector;
-import com.bakdata.conquery.models.datasets.concepts.SelectHolder;
-import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept;
 import com.bakdata.conquery.sql.conversion.Context;
 import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables;
@@ -21,35 +18,31 @@
 
 @Value
 @AllArgsConstructor(access = AccessLevel.PRIVATE)
-public class SelectContext<S extends SelectHolder<?>, T extends SqlTables> implements Context {
+public class SelectContext<T extends SqlTables> implements Context {
 
-	S selectHolder;
 	SqlIdColumns ids;
 	Optional<ColumnDateRange> validityDate;
 	T tables;
 	ConversionContext conversionContext;
 
-	public static SelectContext<Connector, ConnectorSqlTables> create(
+	public static SelectContext<ConnectorSqlTables> create(
 			CQTable cqTable,
 			SqlIdColumns ids,
 			Optional<ColumnDateRange> validityDate,
 			ConnectorSqlTables tables,
 			ConversionContext conversionContext
 	) {
-		return new SelectContext<>(cqTable.getConnector(), ids, validityDate, tables, conversionContext);
+		return new SelectContext<>(ids, validityDate, tables, conversionContext);
 	}
 
-	public static SelectContext<TreeConcept, ConceptSqlTables> create(
+	public static SelectContext<ConceptSqlTables> create(
 			CQConcept cqConcept,
 			SqlIdColumns ids,
 			Optional<ColumnDateRange> validityDate,
 			ConceptSqlTables tables,
 			ConversionContext conversionContext
 	) {
-		if (!(cqConcept.getConcept() instanceof TreeConcept treeConcept)) {
-			throw new IllegalArgumentException("Cannot create a select context for a concept that is not a TreeConcept");
-		}
-		return new SelectContext<>(treeConcept, ids, validityDate, tables, conversionContext);
+		return new SelectContext<>(ids, validityDate, tables, conversionContext);
 	}
 
 	public SqlFunctionProvider getFunctionProvider() {
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectConverter.java
index fc043a84b6..3e8a8bfb29 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectConverter.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectConverter.java
@@ -1,18 +1,16 @@
 package com.bakdata.conquery.sql.conversion.model.select;
 
-import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.datasets.concepts.select.Select;
-import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables;
 
 public interface SelectConverter<S extends Select> {
 
-	default ConnectorSqlSelects connectorSelect(S select, SelectContext<Connector, ConnectorSqlTables> selectContext) {
+	default ConnectorSqlSelects connectorSelect(S select, SelectContext<ConnectorSqlTables> selectContext) {
 		throw new UnsupportedOperationException("Conversion of Select %s not implemented on Connector-level".formatted(select.getClass()));
 	}
 
-	default ConceptSqlSelects conceptSelect(S select, SelectContext<TreeConcept, ConceptSqlTables> selectContext) {
+	default ConceptSqlSelects conceptSelect(S select, SelectContext<ConceptSqlTables> selectContext) {
 		throw new UnsupportedOperationException("Conversion of Select %s not implemented or not possible on Concept-level".formatted(select.getClass()));
 	}
 
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java
index 633a520df1..f6da877a02 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java
@@ -5,7 +5,6 @@
 import java.util.function.BiFunction;
 
 import com.bakdata.conquery.models.datasets.Column;
-import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables;
 import com.bakdata.conquery.sql.conversion.model.ColumnDateRange;
@@ -17,7 +16,7 @@ public static ConnectorSqlSelects createValueSelect(
 			Column column,
 			String alias,
 			BiFunction<Field<?>, List<Field<?>>, Field<?>> aggregationFunction,
-			SelectContext<Connector, ConnectorSqlTables> selectContext
+			SelectContext<ConnectorSqlTables> selectContext
 	) {
 		ConnectorSqlTables tables = selectContext.getTables();
 
diff --git a/backend/src/test/resources/tests/sql/selects/concept_values/content.csv b/backend/src/test/resources/tests/sql/selects/concept_values/content.csv
new file mode 100644
index 0000000000..88a503b22b
--- /dev/null
+++ b/backend/src/test/resources/tests/sql/selects/concept_values/content.csv
@@ -0,0 +1,8 @@
+pid,datum,test_column
+1,2012-01-01,"A1"
+1,2012-01-01,"A1"
+1,2012-01-02,"B2"
+1,2012-01-02,"B2"
+2,2010-07-15,"B2"
+3,2013-11-10,"A1"
+4,2012-11-11,"B2"
diff --git a/backend/src/test/resources/tests/sql/selects/concept_values/content2.csv b/backend/src/test/resources/tests/sql/selects/concept_values/content2.csv
new file mode 100644
index 0000000000..44d8d7121e
--- /dev/null
+++ b/backend/src/test/resources/tests/sql/selects/concept_values/content2.csv
@@ -0,0 +1,6 @@
+pid,datum,test_column
+1,2012-01-01,"B2"
+1,2012-01-02,"A1"
+2,2010-07-15,"B2"
+3,2013-11-10,"A1"
+4,2012-11-11,"B2"
diff --git a/backend/src/test/resources/tests/sql/selects/concept_values/expected.csv b/backend/src/test/resources/tests/sql/selects/concept_values/expected.csv
new file mode 100644
index 0000000000..320a84c64c
--- /dev/null
+++ b/backend/src/test/resources/tests/sql/selects/concept_values/expected.csv
@@ -0,0 +1,5 @@
+result,dates,tree select
+1,{2012-01-01/2012-01-02},"{A1,B2}"
+2,{2010-07-15/2010-07-15},{B2}
+3,{2013-11-10/2013-11-10},{A1}
+4,{2012-11-11/2012-11-11},{B2}
\ No newline at end of file
diff --git a/backend/src/test/resources/tests/sql/selects/concept_values/single_connector.json b/backend/src/test/resources/tests/sql/selects/concept_values/single_connector.json
new file mode 100644
index 0000000000..dd9193c671
--- /dev/null
+++ b/backend/src/test/resources/tests/sql/selects/concept_values/single_connector.json
@@ -0,0 +1,112 @@
+{
+  "type": "QUERY_TEST",
+  "label": "CONCEPT_VALUES Test with a single connector",
+  "expectedCsv": "tests/sql/selects/concept_values/expected.csv",
+  "query": {
+    "type": "CONCEPT_QUERY",
+    "root": {
+      "type": "CONCEPT",
+      "selects": [
+        "tree.select"
+      ],
+      "ids": [
+        "tree"
+      ],
+      "tables": [
+        {
+          "id": "tree.test_column"
+        }
+      ]
+    }
+  },
+  "concepts": [
+    {
+      "label": "tree",
+      "type": "TREE",
+      "selects": [
+        {
+          "type": "CONCEPT_VALUES",
+          "name": "select",
+          "asIds": false
+        }
+      ],
+      "connectors": [
+        {
+          "name": "test_column",
+          "column": "table.test_column",
+          "validityDates": {
+            "label": "datum",
+            "column": "table.datum"
+          }
+        },
+        {
+          "label": "tree_label2",
+          "name": "test_column2",
+          "column": "table2.test_column",
+          "validityDates": {
+            "label": "datum",
+            "column": "table2.datum"
+          }
+        }
+      ],
+      "children": [
+        {
+          "label": "test_child1",
+          "condition": {
+            "type": "PREFIX_LIST",
+            "prefixes": "A"
+          },
+          "children": []
+        },
+        {
+          "label": "test_child2",
+          "condition": {
+            "type": "PREFIX_LIST",
+            "prefixes": "B"
+          },
+          "children": []
+        }
+      ]
+    }
+  ],
+  "content": {
+    "tables": [
+      {
+        "csv": "tests/sql/selects/concept_values/content.csv",
+        "name": "table",
+        "primaryColumn": {
+          "name": "pid",
+          "type": "STRING"
+        },
+        "columns": [
+          {
+            "name": "datum",
+            "type": "DATE"
+          },
+          {
+            "name": "test_column",
+            "type": "STRING"
+          }
+        ]
+      },
+      {
+        "csv": "tests/sql/selects/concept_values/content2.csv",
+        "name": "table2",
+        "primaryColumn": {
+          "name": "pid",
+          "type": "STRING"
+        },
+        "columns": [
+          {
+            "name": "datum",
+            "type": "DATE"
+          },
+          {
+            "name": "test_column",
+            "type": "STRING"
+          }
+        ]
+      }
+    ]
+  }
+}
diff --git a/backend/src/test/resources/tests/sql/selects/concept_values/concept_values.json b/backend/src/test/resources/tests/sql/selects/concept_values/two_connectors.json
similarity index 89%
rename from backend/src/test/resources/tests/sql/selects/concept_values/concept_values.json
rename to backend/src/test/resources/tests/sql/selects/concept_values/two_connectors.json
index c98cc4559f..746ca1563f 100644
--- a/backend/src/test/resources/tests/sql/selects/concept_values/concept_values.json
+++ b/backend/src/test/resources/tests/sql/selects/concept_values/two_connectors.json
@@ -1,7 +1,7 @@
 {
   "type": "QUERY_TEST",
-  "label": "CONCEPT_VALUES Test",
-  "expectedCsv": "tests/aggregator/CONCEPT_COLUMN_SELECTS/expected_raw.csv",
+  "label": "CONCEPT_VALUES Test with 2 connectors",
+  "expectedCsv": "tests/sql/selects/concept_values/expected.csv",
   "query": {
     "type": "CONCEPT_QUERY",
     "root": {
@@ -75,7 +75,7 @@
   "content": {
     "tables": [
       {
-        "csv": "tests/aggregator/CONCEPT_COLUMN_SELECTS/content.csv",
+        "csv": "tests/sql/selects/concept_values/content.csv",
         "name": "table",
         "primaryColumn": {
           "name": "pid",
@@ -93,7 +93,7 @@
         ]
       },
       {
-        "csv": "tests/aggregator/CONCEPT_COLUMN_SELECTS/content2.csv",
+        "csv": "tests/sql/selects/concept_values/content2.csv",
         "name": "table2",
         "primaryColumn": {
           "name": "pid",

From 7c98229ef4b599a0eae2a06c20f28595713434e7 Mon Sep 17 00:00:00 2001
From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com>
Date: Mon, 29 Jul 2024 16:10:13 +0200
Subject: [PATCH 34/41] makes ShardNode a dropwizard bundle and the shardnode
 command a server command

---
 .../java/com/bakdata/conquery/Conquery.java   | 24 ++++++----
 .../commands/CollectEntitiesCommand.java      |  0
 .../DistributedStandaloneCommand.java         |  2 +-
 .../conquery/commands/ShardCommand.java       | 39 ++++++++++++++++
 .../bakdata/conquery/commands/ShardNode.java  | 18 ++++----
 .../conquery/models/config/PluginConfig.java  |  3 +-
 .../specific/ReportConsistency.java           |  9 ++--
 .../conquery/models/worker/Workers.java       |  5 ++-
 .../conquery/util/commands/NoOpConquery.java  | 20 +++++++++
 .../conquery/command/ShardCommandTest.java    | 44 +++++++++++++++++++
 10 files changed, 135 insertions(+), 29 deletions(-)
 delete mode 100644 backend/src/main/java/com/bakdata/conquery/commands/CollectEntitiesCommand.java
 create mode 100644 backend/src/main/java/com/bakdata/conquery/commands/ShardCommand.java
 create mode 100644 backend/src/main/java/com/bakdata/conquery/util/commands/NoOpConquery.java
 create mode 100644 backend/src/test/java/com/bakdata/conquery/command/ShardCommandTest.java

diff --git a/backend/src/main/java/com/bakdata/conquery/Conquery.java b/backend/src/main/java/com/bakdata/conquery/Conquery.java
index 313eab5f96..b6acb72947 100644
--- a/backend/src/main/java/com/bakdata/conquery/Conquery.java
+++ b/backend/src/main/java/com/bakdata/conquery/Conquery.java
@@ -1,12 +1,9 @@
 package com.bakdata.conquery;
 
+import jakarta.validation.Validator;
+
 import ch.qos.logback.classic.Level;
-import com.bakdata.conquery.commands.DistributedStandaloneCommand;
-import com.bakdata.conquery.commands.ManagerNode;
-import com.bakdata.conquery.commands.MigrateCommand;
-import com.bakdata.conquery.commands.PreprocessorCommand;
-import com.bakdata.conquery.commands.RecodeStoreCommand;
-import com.bakdata.conquery.commands.ShardNode;
+import com.bakdata.conquery.commands.*;
 import com.bakdata.conquery.io.jackson.Jackson;
 import com.bakdata.conquery.io.jackson.MutableInjectableValues;
 import com.bakdata.conquery.metrics.prometheus.PrometheusBundle;
@@ -22,7 +19,6 @@
 import io.dropwizard.core.ConfiguredBundle;
 import io.dropwizard.core.setup.Bootstrap;
 import io.dropwizard.core.setup.Environment;
-import jakarta.validation.Validator;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.Setter;
@@ -51,7 +47,7 @@ public void initialize(Bootstrap<ConqueryConfig> bootstrap) {
 		// main config file is json
 		bootstrap.setConfigurationFactoryFactory(JsonConfigurationFactory::new);
 
-		bootstrap.addCommand(new ShardNode());
+		bootstrap.addCommand(new ShardCommand());
 		bootstrap.addCommand(new PreprocessorCommand());
 		bootstrap.addCommand(new DistributedStandaloneCommand(this));
 		bootstrap.addCommand(new RecodeStoreCommand());
@@ -84,6 +80,18 @@ public void initialize(Bootstrap<?> bootstrap) {
 			}
 		});
 
+		bootstrap.addBundle(new ConfiguredBundle<ConqueryConfig>() {
+			@Override
+			public void run(ConqueryConfig configuration, Environment environment) throws Exception {
+				ConfiguredBundle.super.run(configuration, environment);
+			}
+
+			@Override
+			public void initialize(Bootstrap<?> bootstrap) {
+				ConfiguredBundle.super.initialize(bootstrap);
+			}
+		});
+
 		bootstrap.addBundle(new PrometheusBundle());
 	}
 
diff --git a/backend/src/main/java/com/bakdata/conquery/commands/CollectEntitiesCommand.java b/backend/src/main/java/com/bakdata/conquery/commands/CollectEntitiesCommand.java
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java b/backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java
index 1a9a0ab165..aa314cdfaa 100644
--- a/backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java
+++ b/backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java
@@ -104,7 +104,7 @@ public void startStandalone(Environment environment, Namespace namespace, Conque
 					clone = config.withStorage(((XodusStoreFactory) config.getStorage()).withDirectory(managerDir));
 				}
 
-				sc.run(environment, namespace, clone);
+				sc.run(clone, environment);
 				return sc;
 			}));
 		}
diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ShardCommand.java b/backend/src/main/java/com/bakdata/conquery/commands/ShardCommand.java
new file mode 100644
index 0000000000..bf52085427
--- /dev/null
+++ b/backend/src/main/java/com/bakdata/conquery/commands/ShardCommand.java
@@ -0,0 +1,39 @@
+package com.bakdata.conquery.commands;
+
+import java.util.Collections;
+
+import com.bakdata.conquery.models.config.ConqueryConfig;
+import com.bakdata.conquery.util.commands.NoOpConquery;
+import io.dropwizard.core.cli.ServerCommand;
+import io.dropwizard.core.server.DefaultServerFactory;
+import io.dropwizard.core.setup.Bootstrap;
+import io.dropwizard.core.setup.Environment;
+import net.sourceforge.argparse4j.inf.Namespace;
+
+/**
+ * Command to run conquery as a shard node.
+ */
+public class ShardCommand extends ServerCommand<ConqueryConfig> {
+
+	public ShardCommand() {
+		super(new NoOpConquery(), "shard", "Connects this instance as a ShardNode to a running ManagerNode.");
+	}
+
+	@Override
+	protected void run(Bootstrap<ConqueryConfig> bootstrap, Namespace namespace, ConqueryConfig configuration) throws Exception {
+		bootstrap.addBundle(new ShardNode());
+
+		super.run(bootstrap, namespace, configuration);
+	}
+
+	@Override
+	protected void run(Environment environment, Namespace namespace, ConqueryConfig configuration) throws Exception {
+		/*
+		 Clear application connectors for a shard, before building the server,
+		 as we only expose the metrics through the admin connector.
+		 */
+		((DefaultServerFactory)configuration.getServerFactory()).setApplicationConnectors(Collections.emptyList());
+
+		super.run(environment, namespace, configuration);
+	}
+}
diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java
index 2293959fec..211936ad8f 100644
--- a/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java
+++ b/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java
@@ -7,15 +7,12 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import jakarta.validation.Validator;
 
 import com.bakdata.conquery.io.jackson.Jackson;
 import com.bakdata.conquery.io.jackson.MutableInjectableValues;
 import com.bakdata.conquery.io.jackson.View;
-import com.bakdata.conquery.io.mina.BinaryJacksonCoder;
-import com.bakdata.conquery.io.mina.CQProtocolCodecFilter;
-import com.bakdata.conquery.io.mina.ChunkReader;
-import com.bakdata.conquery.io.mina.ChunkWriter;
-import com.bakdata.conquery.io.mina.NetworkSession;
+import com.bakdata.conquery.io.mina.*;
 import com.bakdata.conquery.io.storage.WorkerStorage;
 import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.jobs.JobManager;
@@ -37,14 +34,13 @@
 import com.fasterxml.jackson.databind.DeserializationConfig;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationConfig;
+import io.dropwizard.core.ConfiguredBundle;
 import io.dropwizard.core.setup.Environment;
 import io.dropwizard.lifecycle.Managed;
 import io.dropwizard.util.Duration;
-import jakarta.validation.Validator;
 import lombok.Getter;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
-import net.sourceforge.argparse4j.inf.Namespace;
 import org.apache.mina.core.RuntimeIoException;
 import org.apache.mina.core.future.ConnectFuture;
 import org.apache.mina.core.service.IoHandler;
@@ -61,10 +57,12 @@
  */
 @Slf4j
 @Getter
-public class ShardNode extends ConqueryCommand implements IoHandler, Managed {
+public class ShardNode implements ConfiguredBundle<ConqueryConfig>, IoHandler, Managed {
 
 	public static final String DEFAULT_NAME = "shard-node";
 
+	private final String name;
+
 	private NioSocketConnector connector;
 	private ConnectFuture future;
 	private JobManager jobManager;
@@ -82,12 +80,12 @@ public ShardNode() {
 	}
 
 	public ShardNode(String name) {
-		super(name, "Connects this instance as a ShardNode to a running ManagerNode.");
+		this.name = name;
 	}
 
 
 	@Override
-	protected void run(Environment environment, Namespace namespace, ConqueryConfig config) throws Exception {
+	public void run(ConqueryConfig config, Environment environment) throws Exception {
 		this.environment = environment;
 		this.config = config;
 
diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/PluginConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/PluginConfig.java
index 8629074ccd..824e00fa5f 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/config/PluginConfig.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/config/PluginConfig.java
@@ -3,11 +3,10 @@
 import com.bakdata.conquery.commands.ManagerNode;
 import com.bakdata.conquery.io.cps.CPSBase;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
-import freemarker.core.Environment;
 
 @JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "type")
 @CPSBase
 public interface PluginConfig {
 
-	public default void initialize(ManagerNode managerNode){};
+	default void initialize(ManagerNode managerNode){};
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ReportConsistency.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ReportConsistency.java
index 0239857c86..692fa2f986 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ReportConsistency.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ReportConsistency.java
@@ -3,6 +3,7 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import com.bakdata.conquery.commands.ManagerNode;
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.models.datasets.Import;
 import com.bakdata.conquery.models.identifiable.ids.Id;
@@ -13,16 +14,12 @@
 import com.bakdata.conquery.models.messages.namespaces.NamespacedMessage;
 import com.bakdata.conquery.models.worker.DistributedNamespace;
 import com.google.common.collect.Sets;
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.NonNull;
-import lombok.Setter;
+import lombok.*;
 import lombok.extern.slf4j.Slf4j;
 
 /**
  * Compares the the ids of imports and buckets of a {@link com.bakdata.conquery.models.worker.Worker} with the those
- * the {@link com.bakdata.conquery.commands.ManagerNode} assumed the Worker to have and reports an error if there are
+ * the {@link ManagerNode} assumed the Worker to have and reports an error if there are
  * inconsistencies.
  */
 @CPSType(id="REPORT_CONSISTENCY", base= NamespacedMessage.class)
diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java
index 395551aa64..cb7f603021 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java
@@ -8,7 +8,9 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
+import jakarta.validation.Validator;
 
+import com.bakdata.conquery.commands.ShardNode;
 import com.bakdata.conquery.io.storage.WorkerStorage;
 import com.bakdata.conquery.models.config.StoreFactory;
 import com.bakdata.conquery.models.config.ThreadPoolDefinition;
@@ -18,14 +20,13 @@
 import com.bakdata.conquery.models.identifiable.ids.specific.WorkerId;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import jakarta.validation.Validator;
 import lombok.Getter;
 import lombok.NonNull;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
 
 /**
- * {@link com.bakdata.conquery.commands.ShardNode} container of {@link Worker}.
+ * {@link ShardNode} container of {@link Worker}.
  *
  * Each Shard contains one {@link Worker} per {@link Dataset}.
  */
diff --git a/backend/src/main/java/com/bakdata/conquery/util/commands/NoOpConquery.java b/backend/src/main/java/com/bakdata/conquery/util/commands/NoOpConquery.java
new file mode 100644
index 0000000000..102c35c452
--- /dev/null
+++ b/backend/src/main/java/com/bakdata/conquery/util/commands/NoOpConquery.java
@@ -0,0 +1,20 @@
+package com.bakdata.conquery.util.commands;
+
+import com.bakdata.conquery.Conquery;
+import com.bakdata.conquery.commands.ManagerNode;
+import com.bakdata.conquery.models.config.ConqueryConfig;
+import io.dropwizard.core.setup.Environment;
+
+/**
+ * {@link io.dropwizard.core.Application} "placeholder" for {@link io.dropwizard.core.cli.ServerCommand}/{@link io.dropwizard.core.cli.EnvironmentCommand}s that should not
+ * run {@link ManagerNode} related code.
+ * <p/>
+ * The {@link io.dropwizard.core.cli.EnvironmentCommand} calls normally {@link Conquery#run(ConqueryConfig, Environment)} which brings ub the manager.
+ */
+public class NoOpConquery extends Conquery {
+
+	@Override
+	public void run(ConqueryConfig configuration, Environment environment) throws Exception {
+		// Do nothing
+	}
+}
diff --git a/backend/src/test/java/com/bakdata/conquery/command/ShardCommandTest.java b/backend/src/test/java/com/bakdata/conquery/command/ShardCommandTest.java
new file mode 100644
index 0000000000..8ea837d1d9
--- /dev/null
+++ b/backend/src/test/java/com/bakdata/conquery/command/ShardCommandTest.java
@@ -0,0 +1,44 @@
+package com.bakdata.conquery.command;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import jakarta.ws.rs.client.Client;
+import jakarta.ws.rs.core.Response;
+
+import com.bakdata.conquery.Conquery;
+import com.bakdata.conquery.commands.ShardCommand;
+import com.bakdata.conquery.models.config.ConqueryConfig;
+import com.bakdata.conquery.util.NonPersistentStoreFactory;
+import com.bakdata.conquery.util.support.ConfigOverride;
+import io.dropwizard.testing.junit5.DropwizardAppExtension;
+import io.dropwizard.testing.junit5.DropwizardExtensionsSupport;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith(DropwizardExtensionsSupport.class)
+public class ShardCommandTest {
+
+	private static final ConqueryConfig CONQUERY_CONFIG = new ConqueryConfig() {{
+		ConfigOverride.configureRandomPorts(this);
+		this.setStorage(new NonPersistentStoreFactory());
+	}};
+
+	private static final DropwizardAppExtension<ConqueryConfig> EXT = new DropwizardAppExtension<>(
+			Conquery.class,
+			CONQUERY_CONFIG,
+			application -> new ShardCommand()
+	);
+
+	@Test
+	void checkHttpUp() {
+		Client client = EXT.client();
+
+		Response response = client.target(
+						String.format("http://localhost:%d/metrics", EXT.getAdminPort()))
+				.request()
+				.get();
+
+		assertThat(response.getStatus()).isEqualTo(200);
+	}
+
+}

From ffcb8fa51894bc8629800f32510732378338e616 Mon Sep 17 00:00:00 2001
From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com>
Date: Mon, 29 Jul 2024 16:12:26 +0200
Subject: [PATCH 35/41] clean up

---
 .../src/main/java/com/bakdata/conquery/Conquery.java | 12 ------------
 .../commands/DistributedStandaloneCommand.java       |  2 +-
 .../bakdata/conquery/models/config/PluginConfig.java |  2 +-
 3 files changed, 2 insertions(+), 14 deletions(-)

diff --git a/backend/src/main/java/com/bakdata/conquery/Conquery.java b/backend/src/main/java/com/bakdata/conquery/Conquery.java
index b6acb72947..6051d23a9c 100644
--- a/backend/src/main/java/com/bakdata/conquery/Conquery.java
+++ b/backend/src/main/java/com/bakdata/conquery/Conquery.java
@@ -80,18 +80,6 @@ public void initialize(Bootstrap<?> bootstrap) {
 			}
 		});
 
-		bootstrap.addBundle(new ConfiguredBundle<ConqueryConfig>() {
-			@Override
-			public void run(ConqueryConfig configuration, Environment environment) throws Exception {
-				ConfiguredBundle.super.run(configuration, environment);
-			}
-
-			@Override
-			public void initialize(Bootstrap<?> bootstrap) {
-				ConfiguredBundle.super.initialize(bootstrap);
-			}
-		});
-
 		bootstrap.addBundle(new PrometheusBundle());
 	}
 
diff --git a/backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java b/backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java
index aa314cdfaa..8e368fc189 100644
--- a/backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java
+++ b/backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java
@@ -26,7 +26,7 @@ public class DistributedStandaloneCommand extends ServerCommand<ConqueryConfig>
 
 	private final Conquery conquery;
 	private ClusterManager manager;
-	private ManagerNode managerNode = new ManagerNode();
+	private final ManagerNode managerNode = new ManagerNode();
 	private final List<ShardNode> shardNodes = new Vector<>();
 
 	// TODO clean up the command structure, so we can use the Environment from EnvironmentCommand
diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/PluginConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/PluginConfig.java
index 824e00fa5f..07f13b1082 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/config/PluginConfig.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/config/PluginConfig.java
@@ -8,5 +8,5 @@
 @CPSBase
 public interface PluginConfig {
 
-	default void initialize(ManagerNode managerNode){};
+	default void initialize(ManagerNode managerNode){}
 }

From cc98c700088ed093bf26a96d02637ada8b77228c Mon Sep 17 00:00:00 2001
From: MT <12283268+thoniTUB@users.noreply.github.com>
Date: Tue, 30 Jul 2024 09:49:10 +0200
Subject: [PATCH 36/41] Update
 backend/src/main/java/com/bakdata/conquery/util/commands/NoOpConquery.java

fix typo

Co-authored-by: awildturtok <1553491+awildturtok@users.noreply.github.com>
---
 .../java/com/bakdata/conquery/util/commands/NoOpConquery.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/src/main/java/com/bakdata/conquery/util/commands/NoOpConquery.java b/backend/src/main/java/com/bakdata/conquery/util/commands/NoOpConquery.java
index 102c35c452..9bae79415c 100644
--- a/backend/src/main/java/com/bakdata/conquery/util/commands/NoOpConquery.java
+++ b/backend/src/main/java/com/bakdata/conquery/util/commands/NoOpConquery.java
@@ -9,7 +9,7 @@
  * {@link io.dropwizard.core.Application} "placeholder" for {@link io.dropwizard.core.cli.ServerCommand}/{@link io.dropwizard.core.cli.EnvironmentCommand}s that should not
  * run {@link ManagerNode} related code.
  * <p/>
- * The {@link io.dropwizard.core.cli.EnvironmentCommand} calls normally {@link Conquery#run(ConqueryConfig, Environment)} which brings ub the manager.
+ * The {@link io.dropwizard.core.cli.EnvironmentCommand} calls normally {@link Conquery#run(ConqueryConfig, Environment)} which brings up the manager.
  */
 public class NoOpConquery extends Conquery {
 

From 9ff36aa0091267768bfe26135bd89e347d04f63a Mon Sep 17 00:00:00 2001
From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com>
Date: Wed, 31 Jul 2024 10:17:49 +0200
Subject: [PATCH 37/41] deploys complete cluster in test

---
 .../bakdata/conquery/commands/ShardNode.java  |   8 +-
 .../conquery/mode/cluster/ClusterState.java   |   1 -
 .../conquery/models/config/ClusterConfig.java |   7 +-
 .../models/worker/ClusterHealthCheck.java     |  13 +-
 .../command/DistributedCommandsTest.java      | 116 ++++++++++++++++++
 .../conquery/command/ShardCommandTest.java    |  44 -------
 6 files changed, 134 insertions(+), 55 deletions(-)
 create mode 100644 backend/src/test/java/com/bakdata/conquery/command/DistributedCommandsTest.java
 delete mode 100644 backend/src/test/java/com/bakdata/conquery/command/ShardCommandTest.java

diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java
index 211936ad8f..26e41c8945 100644
--- a/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java
+++ b/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java
@@ -12,7 +12,11 @@
 import com.bakdata.conquery.io.jackson.Jackson;
 import com.bakdata.conquery.io.jackson.MutableInjectableValues;
 import com.bakdata.conquery.io.jackson.View;
-import com.bakdata.conquery.io.mina.*;
+import com.bakdata.conquery.io.mina.BinaryJacksonCoder;
+import com.bakdata.conquery.io.mina.CQProtocolCodecFilter;
+import com.bakdata.conquery.io.mina.ChunkReader;
+import com.bakdata.conquery.io.mina.ChunkWriter;
+import com.bakdata.conquery.io.mina.NetworkSession;
 import com.bakdata.conquery.io.storage.WorkerStorage;
 import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.jobs.JobManager;
@@ -341,7 +345,7 @@ private void connectToCluster() {
 
 				future.cancel();
 				// Sleep thirty seconds then retry.
-				TimeUnit.SECONDS.sleep(30);
+				TimeUnit.SECONDS.sleep(config.getCluster().getConnectRetryTimeout().toSeconds());
 
 			}
 			catch (RuntimeIoException e) {
diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterState.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterState.java
index 36356fa9fc..18e725f1d0 100644
--- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterState.java
+++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterState.java
@@ -31,5 +31,4 @@ public WorkerInformation getWorker(final WorkerId workerId, final DatasetId id)
 					   .flatMap(ns -> ns.getWorkers().getOptional(workerId))
 					   .orElseThrow(() -> new NoSuchElementException("Unknown worker worker '%s' for dataset '%s'".formatted(workerId, id)));
 	}
-
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ClusterConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ClusterConfig.java
index c98582ea00..c71c7fde1c 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/config/ClusterConfig.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/config/ClusterConfig.java
@@ -1,13 +1,13 @@
 package com.bakdata.conquery.models.config;
 
 import java.net.InetAddress;
+import jakarta.validation.Valid;
+import jakarta.validation.constraints.Min;
+import jakarta.validation.constraints.NotNull;
 
 import io.dropwizard.core.Configuration;
 import io.dropwizard.util.Duration;
 import io.dropwizard.validation.PortRange;
-import jakarta.validation.Valid;
-import jakarta.validation.constraints.Min;
-import jakarta.validation.constraints.NotNull;
 import lombok.Getter;
 import lombok.Setter;
 
@@ -26,6 +26,7 @@ public class ClusterConfig extends Configuration {
 	private int entityBucketSize = 1000;
 
 	private Duration idleTimeOut =  Duration.minutes(5);
+	private Duration connectRetryTimeout = Duration.seconds(30);
 
 	/**
 	 * Amount of backpressure before jobs can volunteer to block to send messages to their shards.
diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/ClusterHealthCheck.java b/backend/src/main/java/com/bakdata/conquery/models/worker/ClusterHealthCheck.java
index ffcf3e5e04..a52e35667d 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/worker/ClusterHealthCheck.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/worker/ClusterHealthCheck.java
@@ -1,5 +1,6 @@
 package com.bakdata.conquery.models.worker;
 
+import java.util.Collection;
 import java.util.List;
 import java.util.function.Predicate;
 
@@ -10,19 +11,21 @@
 @Data
 public class ClusterHealthCheck extends HealthCheck {
 
+	public static final String HEALTHY_MESSAGE_FMT = "All %d known shards are connected.";
 	private final ClusterState clusterState;
 
 	@Override
 	protected Result check() throws Exception {
 
+		Collection<ShardNodeInformation> knownShards = clusterState.getShardNodes().values();
 		final List<String> disconnectedWorkers =
-				clusterState.getShardNodes().values().stream()
-							.filter(Predicate.not(ShardNodeInformation::isConnected))
-							.map(ShardNodeInformation::toString)
-							.toList();
+				knownShards.stream()
+						   .filter(Predicate.not(ShardNodeInformation::isConnected))
+						   .map(ShardNodeInformation::toString)
+						   .toList();
 
 		if (disconnectedWorkers.isEmpty()){
-			return Result.healthy("All known shards are connected.");
+			return Result.healthy(HEALTHY_MESSAGE_FMT, knownShards.size());
 		}
 
 		return Result.unhealthy("The shard(s) %s are no longer connected.".formatted(String.join(",", disconnectedWorkers)));
diff --git a/backend/src/test/java/com/bakdata/conquery/command/DistributedCommandsTest.java b/backend/src/test/java/com/bakdata/conquery/command/DistributedCommandsTest.java
new file mode 100644
index 0000000000..5fbd90e3c2
--- /dev/null
+++ b/backend/src/test/java/com/bakdata/conquery/command/DistributedCommandsTest.java
@@ -0,0 +1,116 @@
+package com.bakdata.conquery.command;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.Map;
+import jakarta.ws.rs.client.Client;
+import jakarta.ws.rs.core.GenericType;
+import jakarta.ws.rs.core.Response;
+
+import com.bakdata.conquery.Conquery;
+import com.bakdata.conquery.commands.ShardCommand;
+import com.bakdata.conquery.models.config.ConqueryConfig;
+import com.bakdata.conquery.models.worker.ClusterHealthCheck;
+import com.bakdata.conquery.util.NonPersistentStoreFactory;
+import com.bakdata.conquery.util.support.ConfigOverride;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import io.dropwizard.core.cli.ServerCommand;
+import io.dropwizard.testing.junit5.DropwizardAppExtension;
+import io.dropwizard.testing.junit5.DropwizardExtensionsSupport;
+import io.dropwizard.util.Duration;
+import lombok.Data;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith(DropwizardExtensionsSupport.class)
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+public class DistributedCommandsTest {
+
+	public static final Duration CONNECT_RETRY_TIMEOUT = Duration.seconds(1);
+
+	private static final ConqueryConfig CONQUERY_CONFIG_MANAGER = new ConqueryConfig() {{
+		ConfigOverride.configureRandomPorts(this);
+		this.setStorage(new NonPersistentStoreFactory());
+	}};
+
+	private static final ConqueryConfig CONQUERY_CONFIG_SHARD = new ConqueryConfig() {{
+		ConfigOverride.configureRandomPorts(this);
+		this.getCluster().setPort(CONQUERY_CONFIG_MANAGER.getCluster().getPort());
+		this.getCluster().setConnectRetryTimeout(CONNECT_RETRY_TIMEOUT);
+		this.setStorage(new NonPersistentStoreFactory());
+	}};
+
+	private static final DropwizardAppExtension<ConqueryConfig> MANAGER = new DropwizardAppExtension<>(
+			Conquery.class,
+			CONQUERY_CONFIG_MANAGER,
+			ServerCommand::new
+	);
+
+	private static final DropwizardAppExtension<ConqueryConfig> SHARD = new DropwizardAppExtension<>(
+			Conquery.class,
+			CONQUERY_CONFIG_SHARD,
+			application -> new ShardCommand()
+	);
+
+	@Test
+	@Order(0)
+	void checkHttpUpShard() {
+		Client client = SHARD.client();
+
+		Response response = client.target(
+										  String.format("http://localhost:%d/ping", SHARD.getAdminPort()))
+				.request()
+				.get();
+
+		assertThat(response.getStatus()).isEqualTo(200);
+	}
+
+	@Test
+	@Order(0)
+	void checkHttpUpManager() {
+		Client client = MANAGER.client();
+
+		Response response = client.target(
+										  String.format("http://localhost:%d/ping", MANAGER.getAdminPort()))
+								  .request()
+								  .get();
+
+		assertThat(response.getStatus()).isEqualTo(200);
+	}
+
+	@Test
+	@Order(1)
+	void clusterEstablished() throws InterruptedException {
+		Client client = MANAGER.client();
+
+		// Wait for Shard to be connected
+		// May use https://github.com/awaitility/awaitility here in the future
+		Thread.sleep(2 * CONNECT_RETRY_TIMEOUT.toMilliseconds());
+
+		Response response = client.target(
+										  String.format("http://localhost:%d/healthcheck", MANAGER.getAdminPort()))
+								  .request()
+								  .get();
+
+
+		assertThat(response.getStatus()).isEqualTo(200);
+
+		Map<String, GenericHealthCheckResult> healthCheck = response.readEntity(new GenericType<>() {
+		});
+
+		assertThat(healthCheck).containsKey("cluster");
+		assertThat(healthCheck.get("cluster").healthy).isTrue();
+		assertThat(healthCheck.get("cluster").getMessage()).isEqualTo(String.format(ClusterHealthCheck.HEALTHY_MESSAGE_FMT, 1));
+	}
+
+	@Data
+	@JsonIgnoreProperties(ignoreUnknown = true)
+	private static class GenericHealthCheckResult {
+		private boolean healthy;
+		private String message;
+	}
+
+}
diff --git a/backend/src/test/java/com/bakdata/conquery/command/ShardCommandTest.java b/backend/src/test/java/com/bakdata/conquery/command/ShardCommandTest.java
deleted file mode 100644
index 8ea837d1d9..0000000000
--- a/backend/src/test/java/com/bakdata/conquery/command/ShardCommandTest.java
+++ /dev/null
@@ -1,44 +0,0 @@
-package com.bakdata.conquery.command;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-import jakarta.ws.rs.client.Client;
-import jakarta.ws.rs.core.Response;
-
-import com.bakdata.conquery.Conquery;
-import com.bakdata.conquery.commands.ShardCommand;
-import com.bakdata.conquery.models.config.ConqueryConfig;
-import com.bakdata.conquery.util.NonPersistentStoreFactory;
-import com.bakdata.conquery.util.support.ConfigOverride;
-import io.dropwizard.testing.junit5.DropwizardAppExtension;
-import io.dropwizard.testing.junit5.DropwizardExtensionsSupport;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-
-@ExtendWith(DropwizardExtensionsSupport.class)
-public class ShardCommandTest {
-
-	private static final ConqueryConfig CONQUERY_CONFIG = new ConqueryConfig() {{
-		ConfigOverride.configureRandomPorts(this);
-		this.setStorage(new NonPersistentStoreFactory());
-	}};
-
-	private static final DropwizardAppExtension<ConqueryConfig> EXT = new DropwizardAppExtension<>(
-			Conquery.class,
-			CONQUERY_CONFIG,
-			application -> new ShardCommand()
-	);
-
-	@Test
-	void checkHttpUp() {
-		Client client = EXT.client();
-
-		Response response = client.target(
-						String.format("http://localhost:%d/metrics", EXT.getAdminPort()))
-				.request()
-				.get();
-
-		assertThat(response.getStatus()).isEqualTo(200);
-	}
-
-}

From 7c9c9fa691026a6b9bcbe45a2acc132c3f515b38 Mon Sep 17 00:00:00 2001
From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com>
Date: Wed, 31 Jul 2024 10:45:43 +0200
Subject: [PATCH 38/41] adapts to changes in conquery

---
 .../com/bakdata/conquery/integration/IntegrationTests.java     | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java
index 4506672856..e37943068e 100644
--- a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java
+++ b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java
@@ -67,14 +67,13 @@ public class IntegrationTests {
 	@Getter
 	private final File workDir;
 	@Getter
-	public ConqueryConfig config;
+	public final ConqueryConfig config  = new ConqueryConfig();
 
 	@SneakyThrows(IOException.class)
 	public IntegrationTests(String defaultTestRoot, String defaultTestRootPackage) {
 		this.defaultTestRoot = defaultTestRoot;
 		this.defaultTestRootPackage = defaultTestRootPackage;
 		this.workDir = Files.createTempDirectory("conqueryIntegrationTest").toFile();
-		this.config = new ConqueryConfig();
 		ConfigOverride.configurePathsAndLogging(this.config, this.workDir);
 	}
 

From b1ad47a1fb722226a2ee80ce1c65a15ee8cf766c Mon Sep 17 00:00:00 2001
From: Jonas Arnhold <jonas.arnhold@bakdata.com>
Date: Thu, 1 Aug 2024 17:25:34 +0200
Subject: [PATCH 39/41] Fix TableExportQuery conversion (#3509)

---
 .../sql/conversion/model/NumberMapUtil.java   |  5 +
 .../model/aggregator/SumSqlAggregator.java    |  9 +-
 .../model/filter/NumberFilterConverter.java   |  2 +-
 .../query/TableExportQueryConverter.java      | 91 +++++++++----------
 .../resources/shared/geschlecht.concept.json  | 25 +++--
 .../src/test/resources/shared/vers_stamm.csv  | 46 +++++-----
 .../resources/shared/vers_stamm.table.json    |  4 +
 .../FULL_EXPORT_FORM_SECONDARY_ID.json        | 15 ++-
 .../sql/form/FULL_EXPORT_FORM/expected.csv    |  9 +-
 9 files changed, 117 insertions(+), 89 deletions(-)

diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NumberMapUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NumberMapUtil.java
index 1523e36515..f3a4f2362e 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NumberMapUtil.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NumberMapUtil.java
@@ -3,6 +3,7 @@
 import java.math.BigDecimal;
 import java.util.Map;
 
+import com.bakdata.conquery.models.datasets.Column;
 import com.bakdata.conquery.models.events.MajorTypeId;
 
 public class NumberMapUtil {
@@ -14,4 +15,8 @@ public class NumberMapUtil {
 			MajorTypeId.INTEGER, Integer.class
 	);
 
+	public static Class<? extends Number> getType(Column column) {
+		return NUMBER_MAP.get(column.getType());
+	}
+
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java
index 95c2d39122..8dbceba3be 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java
@@ -171,7 +171,8 @@ public Condition convertForTableExport(SumFilter<RANGE> filter, FilterContext<RA
 		Column column = filter.getColumn();
 		String tableName = column.getTable().getName();
 		String columnName = column.getName();
-		Field<Number> field = DSL.field(DSL.name(tableName, columnName), Number.class);
+		Class<? extends Number> numberClass = NumberMapUtil.getType(column);
+		Field<? extends Number> field = DSL.field(DSL.name(tableName, columnName), numberClass);
 
 		Column subtractColumn = filter.getSubtractColumn();
 		if (subtractColumn == null) {
@@ -180,13 +181,13 @@ public Condition convertForTableExport(SumFilter<RANGE> filter, FilterContext<RA
 
 		String subtractColumnName = subtractColumn.getName();
 		String subtractTableName = subtractColumn.getTable().getName();
-		Field<Number> subtractField = DSL.field(DSL.name(subtractTableName, subtractColumnName), Number.class);
+		Field<? extends Number> subtractField = DSL.field(DSL.name(subtractTableName, subtractColumnName), numberClass);
 		return new SumCondition(field.minus(subtractField), filterContext.getValue()).condition();
 	}
 
 	private CommonAggregationSelect<BigDecimal> createSumAggregationSelect(Column sumColumn, Column subtractColumn, String alias, ConnectorSqlTables tables) {
 
-		Class<? extends Number> numberClass = NumberMapUtil.NUMBER_MAP.get(sumColumn.getType());
+		Class<? extends Number> numberClass = NumberMapUtil.getType(sumColumn);
 		List<ExtractingSqlSelect<?>> preprocessingSelects = new ArrayList<>();
 
 		ExtractingSqlSelect<? extends Number> rootSelect = new ExtractingSqlSelect<>(tables.getRootTable(), sumColumn.getName(), numberClass);
@@ -228,7 +229,7 @@ private CommonAggregationSelect<BigDecimal> createDistinctSumAggregationSelect(
 	) {
 		List<ExtractingSqlSelect<?>> preprocessingSelects = new ArrayList<>();
 
-		Class<? extends Number> numberClass = NumberMapUtil.NUMBER_MAP.get(sumColumn.getType());
+		Class<? extends Number> numberClass = NumberMapUtil.getType(sumColumn);
 		ExtractingSqlSelect<? extends Number> rootSelect = new ExtractingSqlSelect<>(tables.getRootTable(), sumColumn.getName(), numberClass);
 		preprocessingSelects.add(rootSelect);
 
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java
index bbfb1dce5f..1eb1daa5fe 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java
@@ -26,7 +26,7 @@ public SqlFilters convertToSqlFilter(NumberFilter<RANGE> filter, FilterContext<R
 		Column column = filter.getColumn();
 		ConnectorSqlTables tables = filterContext.getTables();
 
-		Class<? extends Number> numberClass = NumberMapUtil.NUMBER_MAP.get(column.getType());
+		Class<? extends Number> numberClass = NumberMapUtil.getType(column);
 		ExtractingSqlSelect<? extends Number> rootSelect = new ExtractingSqlSelect<>(tables.getRootTable(), column.getName(), numberClass);
 
 		Field<? extends Number> eventFilterCtePredecessor = rootSelect.qualify(tables.getPredecessor(ConceptCteStep.EVENT_FILTER)).select();
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java
index 5b2b42d092..8b7df87fd0 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java
@@ -1,12 +1,11 @@
 package com.bakdata.conquery.sql.conversion.query;
 
-import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import com.bakdata.conquery.apiv1.query.Query;
 import com.bakdata.conquery.apiv1.query.TableExportQuery;
@@ -28,7 +27,6 @@
 import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper;
 import com.bakdata.conquery.util.TablePrimaryColumnUtil;
 import com.google.common.base.Preconditions;
-import com.google.common.base.Predicate;
 import lombok.RequiredArgsConstructor;
 import org.jooq.Condition;
 import org.jooq.Field;
@@ -113,24 +111,17 @@ private static QueryStep convertTable(
 		Field<Object> primaryColumn = TablePrimaryColumnUtil.findPrimaryColumn(cqTable.getConnector().getTable(), context.getConfig());
 		SqlIdColumns ids = new SqlIdColumns(primaryColumn);
 		String conceptConnectorName = context.getNameGenerator().conceptConnectorName(concept, cqTable.getConnector());
-		Optional<ColumnDateRange> validityDate = convertTablesValidityDate(cqTable, conceptConnectorName, dateRestriction, context);
-
-		List<Field<?>> exportColumns = new ArrayList<>();
-		exportColumns.add(createSourceInfoSelect(cqTable));
-
-		positions.entrySet().stream()
-				 .sorted(Comparator.comparingInt(Map.Entry::getValue))
-				 .map(entry -> createColumnSelect(cqTable, entry))
-				 .forEach(exportColumns::add);
+		Optional<ColumnDateRange> validityDate = convertTablesValidityDate(cqTable, conceptConnectorName, context);
 
+		List<FieldWrapper<?>> exportColumns = initializeFields(cqTable, positions);
 		Selects selects = Selects.builder()
 								 .ids(ids)
 								 .validityDate(validityDate)
-								 .sqlSelects(exportColumns.stream().map(FieldWrapper::new).collect(Collectors.toList()))
+								 .sqlSelects(exportColumns)
 								 .build();
 
 		List<Condition> filters = cqTable.getFilters().stream().map(filterValue -> filterValue.convertForTableExport(ids, context)).toList();
-		Table<Record> joinedTable = joinConnectorTableWithPrerequisite(cqTable, ids, convertedPrerequisite, context);
+		Table<Record> joinedTable = joinConnectorTableWithPrerequisite(cqTable, ids, convertedPrerequisite, dateRestriction, context);
 
 		return QueryStep.builder()
 						.cteName(conceptConnectorName)
@@ -140,68 +131,76 @@ private static QueryStep convertTable(
 						.build();
 	}
 
-	private static Optional<ColumnDateRange> convertTablesValidityDate(CQTable table, String alias, CDateRange dateRestriction, ConversionContext context) {
+	private static Optional<ColumnDateRange> convertTablesValidityDate(CQTable table, String alias, ConversionContext context) {
 		if (table.findValidityDate() == null) {
 			return Optional.of(ColumnDateRange.empty());
 		}
 		SqlFunctionProvider functionProvider = context.getSqlDialect().getFunctionProvider();
-		ColumnDateRange validityDate = functionProvider.forValidityDate(table.findValidityDate(), dateRestriction);
+		ColumnDateRange validityDate = functionProvider.forValidityDate(table.findValidityDate());
 		// when exporting tables, we want the validity date as a single-column daterange string expression straightaway
 		Field<String> asStringExpression = functionProvider.encloseInCurlyBraces(functionProvider.daterangeStringExpression(validityDate));
 		return Optional.of(ColumnDateRange.of(asStringExpression).asValidityDateRange(alias));
 	}
 
-	private static Field<String> createSourceInfoSelect(CQTable cqTable) {
-		String tableName = cqTable.getConnector().getTable().getName();
-		return DSL.val(tableName).as(SharedAliases.SOURCE.getAlias());
-	}
+	private static List<FieldWrapper<?>> initializeFields(CQTable cqTable, Map<Column, Integer> positions) {
 
-	private static Field<?> createColumnSelect(CQTable table, Map.Entry<Column, Integer> entry) {
-
-		Column column = entry.getKey();
-		Integer columnPosition = entry.getValue();
-		String columnName = "%s-%s".formatted(column.getName(), columnPosition);
-
-		if (!isColumnOfTable(column, table)) {
-			return DSL.inline(null, Object.class).as(columnName);
+		Field<?>[] exportColumns = createPlaceholders(positions, cqTable);
+		for (Column column : cqTable.getConnector().getTable().getColumns()) {
+			// e.g. date column(s) are handled separately and not part of positions
+			if (!positions.containsKey(column)) {
+				continue;
+			}
+			int position = positions.get(column) - 1;
+			exportColumns[position] = createColumnSelect(column, position);
 		}
-		return DSL.field(DSL.name(column.getTable().getName(), column.getName()))
-				  .as(columnName);
-	}
 
-	private static boolean isColumnOfTable(Column column, CQTable table) {
-		return columnIsConnectorColumn(column, table)
-			   || columnIsSecondaryIdOfConnectorTable(column, table)
-			   || columnIsConnectorTableColumn(column, table);
+		return Arrays.stream(exportColumns).map(FieldWrapper::new).collect(Collectors.toList());
 	}
 
-	private static boolean columnIsConnectorTableColumn(Column column, CQTable table) {
-		return matchesTableColumnOn(table, tableColumn -> tableColumn == column);
-	}
+	private static Field<?>[] createPlaceholders(Map<Column, Integer> positions, CQTable cqTable) {
+
+		Field<?>[] exportColumns = new Field[positions.size() + 1];
+		exportColumns[0] = createSourceInfoSelect(cqTable);
+
+		// if columns have the same computed position, they can share a common name because they will be unioned over multiple tables anyway
+		positions.forEach((column, position) -> {
+			int shifted = position - 1;
+			Field<?> columnSelect = DSL.inline(null, Object.class).as("%s-%d".formatted(column.getName(), shifted));
+			exportColumns[shifted] = columnSelect;
+		});
 
-	private static boolean columnIsSecondaryIdOfConnectorTable(Column column, CQTable table) {
-		return column.getSecondaryId() != null && matchesTableColumnOn(table, tableColumn -> tableColumn.getSecondaryId() == column.getSecondaryId());
+		return exportColumns;
 	}
 
-	private static boolean matchesTableColumnOn(CQTable table, Predicate<Column> condition) {
-		return Arrays.stream(table.getConnector().getTable().getColumns()).anyMatch(condition);
+	private static Field<String> createSourceInfoSelect(CQTable cqTable) {
+		String tableName = cqTable.getConnector().getTable().getName();
+		return DSL.val(tableName).as(SharedAliases.SOURCE.getAlias());
 	}
 
-	private static boolean columnIsConnectorColumn(Column column, CQTable table) {
-		return table.getConnector().getColumn() != null && table.getConnector().getColumn() == column;
+	private static Field<?> createColumnSelect(Column column, int position) {
+		String columnName = "%s-%s".formatted(column.getName(), position);
+		return DSL.field(DSL.name(column.getTable().getName(), column.getName()))
+				  .as(columnName);
 	}
 
 	private static Table<Record> joinConnectorTableWithPrerequisite(
 			CQTable cqTable,
 			SqlIdColumns ids,
 			QueryStep convertedPrerequisite,
+			CDateRange dateRestriction,
 			ConversionContext context
 	) {
 		SqlFunctionProvider functionProvider = context.getSqlDialect().getFunctionProvider();
 		Table<Record> connectorTable = DSL.table(DSL.name(cqTable.getConnector().getTable().getName()));
 		Table<Record> convertedPrerequisiteTable = DSL.table(DSL.name(convertedPrerequisite.getCteName()));
-		List<Condition> joinOnIds = ids.join(convertedPrerequisite.getQualifiedSelects().getIds());
-		return functionProvider.innerJoin(connectorTable, convertedPrerequisiteTable, joinOnIds);
+
+		ColumnDateRange validityDate = functionProvider.forValidityDate(cqTable.findValidityDate());
+		List<Condition> joinConditions = Stream.concat(
+				ids.join(convertedPrerequisite.getQualifiedSelects().getIds()).stream(),
+				Stream.of(functionProvider.dateRestriction(functionProvider.forCDateRange(dateRestriction), validityDate))
+		).toList();
+
+		return functionProvider.innerJoin(connectorTable, convertedPrerequisiteTable, joinConditions);
 	}
 
 }
diff --git a/backend/src/test/resources/shared/geschlecht.concept.json b/backend/src/test/resources/shared/geschlecht.concept.json
index bc519e8c17..e613e3a1c3 100644
--- a/backend/src/test/resources/shared/geschlecht.concept.json
+++ b/backend/src/test/resources/shared/geschlecht.concept.json
@@ -25,16 +25,23 @@
       }
     ],
     "table": "vers_stamm",
-    "filters": {
-      "type": "SELECT",
-      "name": "geschlecht",
-      "label": "Geschlecht",
-      "column": "vers_stamm.geschlecht",
-      "labels": {
-        "1": "Weiblich",
-        "2": "Männlich"
+    "filters": [
+      {
+        "type": "SELECT",
+        "name": "geschlecht",
+        "label": "Geschlecht",
+        "column": "vers_stamm.geschlecht",
+        "labels": {
+          "1": "Weiblich",
+          "2": "Männlich"
+        }
+      },
+      {
+        "type": "SUM",
+        "name": "sum",
+        "column": "vers_stamm.value"
       }
-    },
+    ],
     "selects": [
       {
         "type": "FIRST",
diff --git a/backend/src/test/resources/shared/vers_stamm.csv b/backend/src/test/resources/shared/vers_stamm.csv
index 65f2be3481..4093e69f07 100644
--- a/backend/src/test/resources/shared/vers_stamm.csv
+++ b/backend/src/test/resources/shared/vers_stamm.csv
@@ -1,23 +1,23 @@
-pid,date_start,date_end,geburtsdatum,geschlecht
-1,2012-01-01,2012-12-31,1957-01-01,1
-2,2012-01-01,2012-12-31,1988-04-01,2
-3,2012-03-01,2012-12-31,1989-01-01,2
-4,2013-01-01,2013-12-31,1989-01-01,2
-5,2012-07-01,2012-12-31,1972-07-01,2
-6,2012-01-01,2012-12-31,1960-10-01,2
-7,2012-01-01,2012-12-31,1983-01-01,1
-8,2012-01-01,2012-12-31,1956-10-01,2
-9,2012-01-01,2012-12-31,1951-07-01,2
-10,2012-01-01,2012-12-31,1964-04-01,2
-11,2012-01-01,2012-12-31,1966-04-01,1
-12,2013-01-01,2013-12-31,1966-04-01,1
-13,2012-01-01,2012-12-31,1995-01-01,1
-14,2012-01-01,2012-12-31,1952-10-01,2
-15,2012-01-01,2012-12-31,1983-01-01,1
-16,2012-01-01,2012-12-31,1981-01-01,1
-17,2012-01-01,2012-12-31,1979-01-01,2
-18,2012-01-01,2012-12-31,1959-04-01,1
-19,2012-01-01,2012-12-31,1991-10-01,1
-20,2012-01-01,2012-12-31,1987-01-01,2
-21,2012-01-01,2012-12-31,1952-01-01,1
-22,2012-01-01,2012-10-01,1955-01-01,2
+pid,date_start,date_end,geburtsdatum,geschlecht,value
+1,2012-01-01,2012-12-31,1957-01-01,1,12
+2,2012-01-01,2012-12-31,1988-04-01,2,34
+3,2012-03-01,2012-12-31,1989-01-01,2,24
+4,2013-01-01,2013-12-31,1989-01-01,2,66
+5,2012-07-01,2012-12-31,1972-07-01,2,31
+6,2012-01-01,2012-12-31,1960-10-01,2,41
+7,2012-01-01,2012-12-31,1983-01-01,1,55
+8,2012-01-01,2012-12-31,1956-10-01,2,7
+9,2012-01-01,2012-12-31,1951-07-01,2,4
+10,2012-01-01,2012-12-31,1964-04-01,2,1
+11,2012-01-01,2012-12-31,1966-04-01,1,22
+12,2013-01-01,2013-12-31,1966-04-01,1,4
+13,2012-01-01,2012-12-31,1995-01-01,1,6
+14,2012-01-01,2012-12-31,1952-10-01,2,788
+15,2012-01-01,2012-12-31,1983-01-01,1,55
+16,2012-01-01,2012-12-31,1981-01-01,1,2
+17,2012-01-01,2012-12-31,1979-01-01,2,24
+18,2012-01-01,2012-12-31,1959-04-01,1,68
+19,2012-01-01,2012-12-31,1991-10-01,1,90
+20,2012-01-01,2012-12-31,1987-01-01,2,765
+21,2012-01-01,2012-12-31,1952-01-01,1,78
+22,2012-01-01,2012-10-01,1955-01-01,2,5
diff --git a/backend/src/test/resources/shared/vers_stamm.table.json b/backend/src/test/resources/shared/vers_stamm.table.json
index c33e5bf70b..4d66bf89d8 100644
--- a/backend/src/test/resources/shared/vers_stamm.table.json
+++ b/backend/src/test/resources/shared/vers_stamm.table.json
@@ -21,6 +21,10 @@
     {
       "name": "geschlecht",
       "type": "STRING"
+    },
+    {
+      "name": "value",
+      "type": "INTEGER"
     }
   ]
 }
diff --git a/backend/src/test/resources/tests/sql/form/FULL_EXPORT_FORM/FULL_EXPORT_FORM_SECONDARY_ID.json b/backend/src/test/resources/tests/sql/form/FULL_EXPORT_FORM/FULL_EXPORT_FORM_SECONDARY_ID.json
index 022f6e70ba..0df4f0311b 100644
--- a/backend/src/test/resources/tests/sql/form/FULL_EXPORT_FORM/FULL_EXPORT_FORM_SECONDARY_ID.json
+++ b/backend/src/test/resources/tests/sql/form/FULL_EXPORT_FORM/FULL_EXPORT_FORM_SECONDARY_ID.json
@@ -6,6 +6,10 @@
   },
   "form": {
     "type": "FULL_EXPORT_FORM",
+    "dateRange": {
+      "min": "2012-01-01",
+      "max": "2012-03-03"
+    },
     "tables": [
       {
         "type": "CONCEPT",
@@ -30,7 +34,16 @@
           "id": "geschlecht.geschlecht",
           "dateColumn": {
             "value": "geschlecht.geschlecht.erster_tag"
-          }
+          },
+          "filters": [
+            {
+              "filter": "geschlecht.geschlecht.sum",
+              "type": "REAL_RANGE",
+              "value": {
+                "min": 10
+              }
+            }
+          ]
         }
       }
     ],
diff --git a/backend/src/test/resources/tests/sql/form/FULL_EXPORT_FORM/expected.csv b/backend/src/test/resources/tests/sql/form/FULL_EXPORT_FORM/expected.csv
index fa644087e7..f87a4f72a1 100644
--- a/backend/src/test/resources/tests/sql/form/FULL_EXPORT_FORM/expected.csv
+++ b/backend/src/test/resources/tests/sql/form/FULL_EXPORT_FORM/expected.csv
@@ -1,5 +1,4 @@
-result,dates,source,secondary,table column,vers_stamm date_end,vers_stamm geburtsdatum,vers_stamm geschlecht
-1,{2012-03-01/2012-03-01},table,1,A,,,
-1,{2012-03-02/2012-03-02},table,1,A,,,
-1,{2012-03-04/2012-03-04},table,2,A,,,
-1,{2012-01-01/2012-01-01},vers_stamm,,,2012-12-31,1957-01-01,1
+result,dates,source,secondary,table column,vers_stamm date_end,vers_stamm geburtsdatum,vers_stamm geschlecht,vers_stamm value
+1,{2012-03-01/2012-03-01},table,1,A,,,,
+1,{2012-03-02/2012-03-02},table,1,A,,,,
+1,{2012-01-01/2012-01-01},vers_stamm,,,2012-12-31,1957-01-01,1,12

From c766d18895ab11bff4a6b5523c65fc8cf00814e7 Mon Sep 17 00:00:00 2001
From: Jonas Arnhold <jonas.arnhold@bakdata.com>
Date: Thu, 1 Aug 2024 18:04:24 +0200
Subject: [PATCH 40/41] Fix full outer joining when converting forms (#3510)

---
 .../sql/conversion/model/ColumnDateRange.java | 13 +++
 .../sql/conversion/model/QueryStepJoiner.java | 13 ++-
 .../MULTIPLE_FEATURES_AND_CONNECTOR.test.json | 63 ---------------
 .../MULTIPLE_FEATURES_AND_CONNECTOR.json      | 63 +++++++++------
 .../MULTIPLE_FEATURES_OR_CONNECTOR.json       | 79 +++++++++++++++++++
 .../expected_and_connector.csv                | 14 ++--
 .../expected_or_connector.csv                 |  7 ++
 7 files changed, 157 insertions(+), 95 deletions(-)
 delete mode 100644 backend/src/test/resources/tests/form/MULTIPLE_FEATURES_AND_CONNECTOR.test.json
 create mode 100644 backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/MULTIPLE_FEATURES_OR_CONNECTOR.json
 create mode 100644 backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/expected_or_connector.csv

diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java
index dea4df7760..5cc417d988 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java
@@ -107,6 +107,19 @@ public ColumnDateRange as(String alias) {
 		);
 	}
 
+	public ColumnDateRange coalesce(ColumnDateRange right) {
+		if (this.isSingleColumnRange() != right.isSingleColumnRange()) {
+			throw new UnsupportedOperationException("Can only join ColumnDateRanges of same type");
+		}
+		if (isSingleColumnRange()) {
+			return ColumnDateRange.of(DSL.coalesce(this.range, right.getRange())).as(this.alias);
+		}
+		return ColumnDateRange.of(
+				DSL.coalesce(this.start, right.getStart()),
+				DSL.coalesce(this.end, right.getEnd())
+		).as(this.alias);
+	}
+
 	public Condition join(ColumnDateRange right) {
 		if (this.isSingleColumnRange() != right.isSingleColumnRange()) {
 			throw new UnsupportedOperationException("Can only join ColumnDateRanges of same type");
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java
index d58e373285..4c792dcf57 100644
--- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java
+++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java
@@ -63,8 +63,8 @@ public static QueryStep joinSteps(
 
 		DateAggregationDates dateAggregationDates = DateAggregationDates.forSteps(queriesToJoin);
 		if (dateAggregationAction == DateAggregationAction.BLOCK || dateAggregationDates.dateAggregationImpossible()) {
-			// for forms, date aggregation is allways blocked // TODO check if this is really correct
-			Optional<ColumnDateRange> stratificationDate = queriesToJoin.get(0).getQualifiedSelects().getStratificationDate();
+			// for forms, date aggregation is allways blocked, but dates need to be coalesced in case we do a fulll outer join
+			Optional<ColumnDateRange> stratificationDate = coalesceStratificationDates(queriesToJoin);
 			joinedStep = buildJoinedStep(ids, mergedSelects, Optional.empty(), stratificationDate, joinedStepBuilder);
 		}
 		// if there is only 1 child node containing a validity date, we just keep it as overall validity date for the joined node
@@ -169,6 +169,15 @@ private static QueryStep buildStepAndAggregateDates(
 		);
 	}
 
+	private static Optional<ColumnDateRange> coalesceStratificationDates(List<QueryStep> queriesToJoin) {
+		return queriesToJoin.stream()
+							.map(QueryStep::getQualifiedSelects)
+							.map(Selects::getStratificationDate)
+							.filter(Optional::isPresent)
+							.map(Optional::get)
+							.reduce(ColumnDateRange::coalesce);
+	}
+
 	@FunctionalInterface
 	private interface JoinType {
 		TableOnConditionStep<Record> join(Table<?> leftPart, Table<?> rightPart, List<Condition> joinConditions);
diff --git a/backend/src/test/resources/tests/form/MULTIPLE_FEATURES_AND_CONNECTOR.test.json b/backend/src/test/resources/tests/form/MULTIPLE_FEATURES_AND_CONNECTOR.test.json
deleted file mode 100644
index d949c3e938..0000000000
--- a/backend/src/test/resources/tests/form/MULTIPLE_FEATURES_AND_CONNECTOR.test.json
+++ /dev/null
@@ -1,63 +0,0 @@
-{
-  "type": "FORM_TEST",
-  "label": "ABS-EXPORT-FORM with multiple features and AND connector",
-  "expectedCsv": {
-    "results": "tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/expected_and_connector.csv"
-  },
-  "form": {
-    "type": "EXPORT_FORM",
-    "queryGroup": "00000000-0000-0000-0000-000000000001",
-    "resolution": "QUARTERS",
-    "alsoCreateCoarserSubdivisions": true,
-    "features": [
-      {
-        "type": "AND",
-        "children": [
-          {
-            "ids": [
-              "alter"
-            ],
-            "type": "CONCEPT",
-            "tables": [
-              {
-                "id": "alter.alter",
-                "filters": []
-              }
-            ]
-          },
-          {
-            "ids": [
-              "geschlecht"
-            ],
-            "type": "CONCEPT",
-            "tables": [
-              {
-                "id": "geschlecht.geschlecht",
-                "filters": []
-              }
-            ]
-          }
-        ]
-      }
-    ],
-    "timeMode": {
-      "value": "ABSOLUTE",
-      "dateRange": {
-        "min": "2012-01-16",
-        "max": "2012-12-17"
-      }
-    }
-  },
-  "concepts": [
-    "/tests/form/shared/alter.concept.json",
-    "/tests/form/shared/geschlecht.concept.json"
-  ],
-  "content": {
-    "tables": [
-      "/tests/form/shared/vers_stamm.table.json"
-    ],
-    "previousQueryResults": [
-      "tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/query_results_1.csv"
-    ]
-  }
-}
diff --git a/backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/MULTIPLE_FEATURES_AND_CONNECTOR.json b/backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/MULTIPLE_FEATURES_AND_CONNECTOR.json
index ca5f139e9f..3d84f213ce 100644
--- a/backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/MULTIPLE_FEATURES_AND_CONNECTOR.json
+++ b/backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/MULTIPLE_FEATURES_AND_CONNECTOR.json
@@ -14,28 +14,28 @@
         "type": "AND",
         "children": [
           {
-            "ids": [
-              "alter"
-            ],
             "type": "CONCEPT",
-            "tables": [
-              {
-                "id": "alter.alter",
-                "filters": []
-              }
-            ]
+            "ids": "abc-concept",
+            "tables": {
+              "id": "abc-concept.connector2",
+              "filters": [
+                {
+                  "filter": "abc-concept.connector2.value",
+                  "type": "MULTI_SELECT",
+                  "value": [
+                    "A",
+                    "B"
+                  ]
+                }
+              ]
+            }
           },
           {
-            "ids": [
-              "geschlecht"
-            ],
             "type": "CONCEPT",
-            "tables": [
-              {
-                "id": "geschlecht.geschlecht",
-                "filters": []
-              }
-            ]
+            "ids": "geschlecht",
+            "tables": {
+              "id": "geschlecht.geschlecht"
+            }
           }
         ]
       }
@@ -43,21 +43,38 @@
     "timeMode": {
       "value": "ABSOLUTE",
       "dateRange": {
-        "min": "2012-01-16",
+        "min": "2012-01-01",
         "max": "2012-12-17"
       }
     }
   },
   "concepts": [
-    "/shared/alter.concept.json",
+    "/tests/form/shared/abc.concept.json",
     "/shared/geschlecht.concept.json"
   ],
   "content": {
     "tables": [
-      "/shared/vers_stamm.table.json"
+      "/shared/vers_stamm.table.json",
+      "/tests/form/shared/abc.table.json"
     ],
-    "previousQueryResults": [
-      "tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/query_results_1.csv"
+    "secondaryIds": [
+      {
+        "name": "secondary"
+      }
+    ],
+    "previousQueries": [
+      {
+        "type": "CONCEPT_QUERY",
+        "root": {
+          "ids": [
+            "abc-concept"
+          ],
+          "tables": {
+            "id": "abc-concept.connector"
+          },
+          "type": "CONCEPT"
+        }
+      }
     ]
   }
 }
diff --git a/backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/MULTIPLE_FEATURES_OR_CONNECTOR.json b/backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/MULTIPLE_FEATURES_OR_CONNECTOR.json
new file mode 100644
index 0000000000..4b04c0306e
--- /dev/null
+++ b/backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/MULTIPLE_FEATURES_OR_CONNECTOR.json
@@ -0,0 +1,79 @@
+{
+  "type": "FORM_TEST",
+  "label": "ABS-EXPORT-FORM with multiple features and OR connector",
+  "expectedCsv": {
+    "results": "tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/expected_or_connector.csv"
+  },
+  "form": {
+    "type": "EXPORT_FORM",
+    "queryGroup": "00000000-0000-0000-0000-000000000001",
+    "resolution": "QUARTERS",
+    "alsoCreateCoarserSubdivisions": true,
+    "features": [
+      {
+        "type": "OR",
+        "children": [
+          {
+            "type": "CONCEPT",
+            "ids": "abc-concept",
+            "tables": {
+              "id": "abc-concept.connector2",
+              "filters": [
+                {
+                  "filter": "abc-concept.connector2.value",
+                  "type": "MULTI_SELECT",
+                  "value": [
+                    "A"
+                  ]
+                }
+              ]
+            }
+          },
+          {
+            "type": "CONCEPT",
+            "ids": "geschlecht",
+            "tables": {
+              "id": "geschlecht.geschlecht"
+            }
+          }
+        ]
+      }
+    ],
+    "timeMode": {
+      "value": "ABSOLUTE",
+      "dateRange": {
+        "min": "2012-01-01",
+        "max": "2012-12-17"
+      }
+    }
+  },
+  "concepts": [
+    "/tests/form/shared/abc.concept.json",
+    "/shared/geschlecht.concept.json"
+  ],
+  "content": {
+    "tables": [
+      "/shared/vers_stamm.table.json",
+      "/tests/form/shared/abc.table.json"
+    ],
+    "secondaryIds": [
+      {
+        "name": "secondary"
+      }
+    ],
+    "previousQueries": [
+      {
+        "type": "CONCEPT_QUERY",
+        "root": {
+          "ids": [
+            "abc-concept"
+          ],
+          "tables": {
+            "id": "abc-concept.connector"
+          },
+          "type": "CONCEPT"
+        }
+      }
+    ]
+  }
+}
diff --git a/backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/expected_and_connector.csv b/backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/expected_and_connector.csv
index b6502df1ef..77e2811e9d 100644
--- a/backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/expected_and_connector.csv
+++ b/backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/expected_and_connector.csv
@@ -1,7 +1,7 @@
-result,resolution,index,date_range,Alter and Geschlecht
-1,complete,,2012-01-16/2012-12-17,1
-1,year,1,2012-01-16/2012-12-17,1
-1,quarter,1,2012-01-16/2012-03-31,1
-1,quarter,2,2012-04-01/2012-06-30,1
-1,quarter,3,2012-07-01/2012-09-30,1
-1,quarter,4,2012-10-01/2012-12-17,1
+result,resolution,index,date_range,abc-concept and Geschlecht
+1,complete,,2012-01-01/2012-12-17,1
+1,year,1,2012-01-01/2012-12-17,1
+1,quarter,1,2012-01-01/2012-03-31,1
+1,quarter,2,2012-04-01/2012-06-30,
+1,quarter,3,2012-07-01/2012-09-30,
+1,quarter,4,2012-10-01/2012-12-17,
diff --git a/backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/expected_or_connector.csv b/backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/expected_or_connector.csv
new file mode 100644
index 0000000000..7818c4018d
--- /dev/null
+++ b/backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/expected_or_connector.csv
@@ -0,0 +1,7 @@
+result,resolution,index,date_range,abc-concept or Geschlecht
+1,complete,,2012-01-01/2012-12-17,1
+1,quarter,1,2012-01-01/2012-03-31,1
+1,quarter,2,2012-04-01/2012-06-30,1
+1,quarter,3,2012-07-01/2012-09-30,1
+1,quarter,4,2012-10-01/2012-12-17,1
+1,year,1,2012-01-01/2012-12-17,1

From bcad24d1b97dee82fd53c1cf24933a2e5c8ef69b Mon Sep 17 00:00:00 2001
From: awildturtok <1553491+awildturtok@users.noreply.github.com>
Date: Mon, 5 Aug 2024 16:00:51 +0200
Subject: [PATCH 41/41] Split imports on preprocess (#3389)

Implementation of buckets split at preprocess:

Prior implementation split data on import, which meant loading large amounts of data only to chunk them. This merge offloads that responsibility onto the preprocess using stable hashing of the entities ID. The hashing is dependent on the number of buckets, defined at preprocess.

This Merge serves to reduce the memory requirements of the manager node, and improve upload time. It also lays the foundation to make preprocessing more memory efficient by compressing buckets separetly.
---
 .../commands/PreprocessorCommand.java         | 118 +++---
 .../conquery/io/storage/NamespaceStorage.java |  18 +-
 .../conquery/io/storage/WorkerStorage.java    |   8 +-
 .../io/storage/xodus/stores/CachedStore.java  |   3 +-
 .../mode/cluster/ClusterImportHandler.java    | 160 ++++++--
 .../mode/cluster/ClusterManagerProvider.java  |   2 +-
 .../mode/cluster/RegisterImportEntities.java  |  42 ++
 .../concepts/tree/ConceptTreeCache.java       |   7 +-
 .../datasets/concepts/tree/TreeConcept.java   |   4 +-
 .../conquery/models/events/Bucket.java        |  25 +-
 .../conquery/models/events/BucketManager.java |  13 +-
 .../conquery/models/events/CBlock.java        |   3 +-
 .../conquery/models/events/EmptyBucket.java   |   2 +-
 .../InjectingCentralRegistry.java             |  26 --
 .../identifiable/mapping/EntityIdMap.java     |   2 +-
 .../models/jobs/CalculateCBlocksJob.java      | 103 +++--
 .../conquery/models/jobs/ImportJob.java       | 372 ------------------
 .../namespaces/specific/AddImport.java        |   2 +-
 .../namespaces/specific/ImportBucket.java     |   6 +-
 .../conquery/models/preproc/Preprocessed.java |  79 +++-
 .../models/preproc/PreprocessedData.java      |   3 +-
 .../models/preproc/PreprocessedHeader.java    |  72 ++++
 .../models/preproc/PreprocessedReader.java    |  44 +--
 .../conquery/models/preproc/Preprocessor.java |   4 +-
 .../models/worker/DistributedNamespace.java   |  14 +-
 .../models/worker/ShardNodeInformation.java   |   7 +-
 .../conquery/models/worker/WorkerHandler.java |  28 +-
 .../models/worker/WorkerInformation.java      |  21 +-
 .../models/worker/WorkerToBucketsMap.java     |  27 +-
 .../resources/api/ConceptsProcessor.java      |   4 +-
 .../ProgressReporterImpl.java                 |  30 +-
 .../integration/common/LoadingUtil.java       |  65 ++-
 .../integration/tests/ImportUpdateTest.java   |  12 +-
 .../integration/tests/RestartTest.java        |   2 +
 .../ConceptUpdateAndDeletionTest.java         |   4 +
 .../tests/deletion/ImportDeletionTest.java    |   8 +-
 .../conquery/models/SerializationTests.java   |   4 +-
 .../concepts/tree/GroovyIndexedTest.java      |   9 +-
 .../util/support/StandaloneSupport.java       |   6 +-
 39 files changed, 660 insertions(+), 699 deletions(-)
 create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/cluster/RegisterImportEntities.java
 delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/identifiable/InjectingCentralRegistry.java
 delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/jobs/ImportJob.java

diff --git a/backend/src/main/java/com/bakdata/conquery/commands/PreprocessorCommand.java b/backend/src/main/java/com/bakdata/conquery/commands/PreprocessorCommand.java
index ea5f91cd78..686e8200eb 100644
--- a/backend/src/main/java/com/bakdata/conquery/commands/PreprocessorCommand.java
+++ b/backend/src/main/java/com/bakdata/conquery/commands/PreprocessorCommand.java
@@ -44,6 +44,7 @@
 import net.sourceforge.argparse4j.inf.ArgumentGroup;
 import net.sourceforge.argparse4j.inf.Namespace;
 import net.sourceforge.argparse4j.inf.Subparser;
+import org.jetbrains.annotations.NotNull;
 
 @Slf4j
 @FieldNameConstants
@@ -52,7 +53,7 @@ public class PreprocessorCommand extends ConqueryCommand {
 	private final List<String> failed = Collections.synchronizedList(new ArrayList<>());
 	private final List<String> success = Collections.synchronizedList(new ArrayList<>());
 	private ExecutorService pool;
-	private boolean isFailFast = false;
+	private boolean isFailFast;
 	private boolean isStrict = true;
 
 	public PreprocessorCommand() {
@@ -71,14 +72,14 @@ public static boolean requiresProcessing(PreprocessingJob preprocessingJob) {
 
 			log.info("EXISTS ALREADY");
 
-			int currentHash = preprocessingJob.getDescriptor()
-											  .calculateValidityHash(preprocessingJob.getCsvDirectory(), preprocessingJob.getTag());
+			final int currentHash = preprocessingJob.getDescriptor()
+													.calculateValidityHash(preprocessingJob.getCsvDirectory(), preprocessingJob.getTag());
 
 
 			final ObjectMapper om = Jackson.BINARY_MAPPER.copy();
 			try (final PreprocessedReader parser = new PreprocessedReader(new GZIPInputStream(new FileInputStream(preprocessingJob.getPreprocessedFile())), om)) {
 
-				PreprocessedHeader header = parser.readHeader();
+				final PreprocessedHeader header = parser.readHeader();
 
 				if (header.getValidityHash() == currentHash) {
 					log.info("\tHASH STILL VALID");
@@ -133,13 +134,18 @@ public void configure(Subparser subparser) {
 		group.addArgument("--fast-fail")
 			 .action(Arguments.storeTrue())
 			 .setDefault(false)
-			 .help("Stop preprocessing and exit with failure if an error occures that prevents the generation of a cqpp.");
+			 .help("Stop preprocessing and exit with failure if an error occurs that prevents the generation of a cqpp.");
 
 		group.addArgument("--strict")
 			 .type(new BooleanArgumentType())
 			 .setDefault(true)
 			 .help("Escalate missing files to errors.");
 
+		group.addArgument("--buckets")
+			 .type(Integer.class)
+			 .setDefault(100)
+			 .help("Number of buckets to use for id-hashing. This value is required to be a constant per-dataset.");
+
 	}
 
 	@Override
@@ -150,41 +156,49 @@ protected void run(Environment environment, Namespace namespace, ConqueryConfig
 
 		// Tag if present is appended to input-file csvs, output-file cqpp and used as id of cqpps
 
+		// Seems to be a bug with dropwizard and boolean default-values
 		isFailFast = Optional.ofNullable(namespace.getBoolean("fast-fail")).orElse(false);
-		isStrict = Optional.ofNullable(namespace.getBoolean("strict")).orElse(true);
+		isStrict = Optional.ofNullable(namespace.getBoolean("strict")).orElse(false);
 
-		final List<String> tags = namespace.<String>getList("tag");
+		final List<String> tags = namespace.getList("tag");
 
 		final File inDir = namespace.get("in");
 		final File outDir = namespace.get("out");
-		final List<File> descriptionFiles = namespace.<File>getList("desc");
+		final List<File> descriptionFilesRoot = namespace.getList("desc");
+		final int buckets = namespace.getInt("buckets");
 
 
 		log.info("Preprocessing from command line config.");
 
-		final Collection<PreprocessingJob> jobs = new ArrayList<>();
+		final Collection<PreprocessingJob> jobs = collectJobs(descriptionFilesRoot, tags, inDir, outDir, environment);
 
-		if (tags == null || tags.isEmpty()) {
-			for (File desc : descriptionFiles) {
-				final List<PreprocessingJob> descriptions =
-						findPreprocessingDescriptions(desc, inDir, outDir, Optional.empty(), environment.getValidator());
-				jobs.addAll(descriptions);
-			}
+		final List<PreprocessingJob> broken = validateJobs(jobs, environment);
+
+		jobs.removeIf(Predicate.not(PreprocessorCommand::requiresProcessing));
+
+		preprocessJobs(jobs, buckets, config);
+
+
+		log.info("Successfully Preprocess {} Jobs:", success.size());
+		success.forEach(desc -> log.info("\tSucceeded Preprocessing for {}", desc));
+
+		if (!broken.isEmpty()) {
+			log.warn("Did not find {} Files", broken.size());
+			broken.forEach(desc -> log.warn("\tDid not find file for {}", desc));
 		}
-		else {
-			for (String tag : tags) {
-				for (File desc : descriptionFiles) {
-					final List<PreprocessingJob> jobDescriptions =
-							findPreprocessingDescriptions(desc, inDir, outDir, Optional.of(tag), environment.getValidator());
 
-					jobs.addAll(jobDescriptions);
-				}
-			}
+		if (isFailed()) {
+			log.error("Failed {} Preprocessing Jobs:", failed.size());
+			failed.forEach(desc -> log.error("\tFailed Preprocessing for {}", desc));
+			doFail();
 		}
+	}
 
-		List<PreprocessingJob> broken = new ArrayList<>();
+	@NotNull
+	private List<PreprocessingJob> validateJobs(Collection<PreprocessingJob> jobs, Environment environment) {
+		final List<PreprocessingJob> broken = new ArrayList<>();
 
-		for (Iterator<PreprocessingJob> iterator = jobs.iterator(); iterator.hasNext(); ) {
+		for (final Iterator<PreprocessingJob> iterator = jobs.iterator(); iterator.hasNext(); ) {
 			final PreprocessingJob job = iterator.next();
 
 			try {
@@ -213,22 +227,48 @@ protected void run(Environment environment, Namespace namespace, ConqueryConfig
 			log.error("FAILED Preprocessing, files are missing or invalid.");
 			doFail();
 		}
+		return broken;
+	}
 
-		jobs.removeIf(Predicate.not(PreprocessorCommand::requiresProcessing));
+	@NotNull
+	private Collection<PreprocessingJob> collectJobs(List<File> descriptionFiles, List<String> tags, File inDir, File outDir, Environment environment)
+			throws IOException {
+		final Collection<PreprocessingJob> jobs = new ArrayList<>();
 
+		if (tags == null || tags.isEmpty()) {
+			for (File desc : descriptionFiles) {
+				final List<PreprocessingJob> descriptions =
+						findPreprocessingDescriptions(desc, inDir, outDir, Optional.empty(), environment.getValidator());
+				jobs.addAll(descriptions);
+			}
+		}
+		else {
+			for (String tag : tags) {
+				for (File desc : descriptionFiles) {
+					final List<PreprocessingJob> jobDescriptions =
+							findPreprocessingDescriptions(desc, inDir, outDir, Optional.of(tag), environment.getValidator());
+
+					jobs.addAll(jobDescriptions);
+				}
+			}
+		}
+		return jobs;
+	}
+
+	private void preprocessJobs(Collection<PreprocessingJob> jobs, int buckets, ConqueryConfig config) throws InterruptedException {
 		final long totalSize = jobs.stream()
 								   .mapToLong(PreprocessingJob::estimateTotalCsvSizeBytes)
 								   .sum();
 
 		log.info("Required to preprocess {} in total", BinaryByteUnit.format(totalSize));
 
-		ProgressBar totalProgress = new ProgressBar(totalSize, System.out);
+		final ProgressBar totalProgress = new ProgressBar(totalSize, System.out);
 
 		for (PreprocessingJob job : jobs) {
 			pool.submit(() -> {
 				ConqueryMDC.setLocation(job.toString());
 				try {
-					Preprocessor.preprocess(job, totalProgress, config);
+					Preprocessor.preprocess(job, totalProgress, config, buckets);
 					success.add(job.toString());
 				}
 				catch (FileNotFoundException e) {
@@ -246,23 +286,6 @@ protected void run(Environment environment, Namespace namespace, ConqueryConfig
 		pool.awaitTermination(24, TimeUnit.HOURS);
 
 		ConqueryMDC.clearLocation();
-
-
-		if (!success.isEmpty()) {
-			log.info("Successfully Preprocess {} Jobs:", success.size());
-			success.forEach(desc -> log.info("\tSucceeded Preprocessing for {}", desc));
-		}
-
-		if (!broken.isEmpty()) {
-			log.warn("Did not find {} Files", broken.size());
-			broken.forEach(desc -> log.warn("\tDid not find file for {}", desc));
-		}
-
-		if (isFailed()) {
-			log.error("Failed {} Preprocessing Jobs:", failed.size());
-			failed.forEach(desc -> log.error("\tFailed Preprocessing for {}", desc));
-			doFail();
-		}
 	}
 
 	private void addMissing(PreprocessingJob job) {
@@ -281,7 +304,7 @@ private void addFailed(PreprocessingJob job) {
 
 	public List<PreprocessingJob> findPreprocessingDescriptions(File descriptionFiles, File inDir, File outputDir, Optional<String> tag, Validator validator)
 			throws IOException {
-		List<PreprocessingJob> out = new ArrayList<>();
+		final List<PreprocessingJob> out = new ArrayList<>();
 
 		final File[] files = descriptionFiles.isFile()
 							 ? new File[]{descriptionFiles}
@@ -302,8 +325,7 @@ private boolean isFailed() {
 		return !failed.isEmpty();
 	}
 
-	private Optional<PreprocessingJob> tryExtractDescriptor(Validator validator, Optional<String> tag, File descriptionFile, File outputDir, File csvDir)
-			throws IOException {
+	private Optional<PreprocessingJob> tryExtractDescriptor(Validator validator, Optional<String> tag, File descriptionFile, File outputDir, File csvDir) {
 		try {
 			final TableImportDescriptor
 					descriptor =
diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java
index 7b29656e6d..7de472e7b3 100644
--- a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java
+++ b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java
@@ -2,7 +2,6 @@
 
 import java.util.Collection;
 import java.util.Objects;
-import java.util.OptionalInt;
 
 import com.bakdata.conquery.io.storage.xodus.stores.CachedStore;
 import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore;
@@ -109,22 +108,13 @@ public int getNumberOfEntities() {
 		return entity2Bucket.count();
 	}
 
-	public OptionalInt getEntityBucket(String entity) {
-		final Integer bucket = entity2Bucket.get(entity);
 
-		if(bucket == null){
-			return OptionalInt.empty();
-		}
-
-		return OptionalInt.of(bucket);
+	public boolean containsEntity(String entity) {
+		return entity2Bucket.get(entity) != null;
 	}
 
-	public int assignEntityBucket(String entity, int bucketSize) {
-		final int bucket = (int) Math.ceil((1d + getNumberOfEntities()) / (double) bucketSize);
-
-		entity2Bucket.add(entity, bucket);
-
-		return bucket;
+	public void registerEntity(String entity, int bucket) {
+		entity2Bucket.update(entity, bucket);
 	}
 
 
diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/WorkerStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/WorkerStorage.java
index adfe9b2841..84ff4e90d6 100644
--- a/backend/src/main/java/com/bakdata/conquery/io/storage/WorkerStorage.java
+++ b/backend/src/main/java/com/bakdata/conquery/io/storage/WorkerStorage.java
@@ -72,7 +72,7 @@ private void decorateCBlockStore(IdentifiableStore<CBlock> baseStoreCreator) {
 
 
 	public void addCBlock(CBlock cBlock) {
-		log.debug("Adding CBlock[{}]", cBlock.getId());
+		log.trace("Adding CBlock[{}]", cBlock.getId());
 		cBlocks.add(cBlock);
 	}
 
@@ -81,7 +81,7 @@ public CBlock getCBlock(CBlockId id) {
 	}
 
 	public void removeCBlock(CBlockId id) {
-		log.debug("Removing CBlock[{}]", id);
+		log.trace("Removing CBlock[{}]", id);
 		cBlocks.remove(id);
 	}
 
@@ -90,7 +90,7 @@ public Collection<CBlock> getAllCBlocks() {
 	}
 
 	public void addBucket(Bucket bucket) {
-		log.debug("Adding Bucket[{}]", bucket.getId());
+		log.trace("Adding Bucket[{}]", bucket.getId());
 		buckets.add(bucket);
 	}
 
@@ -99,7 +99,7 @@ public Bucket getBucket(BucketId id) {
 	}
 
 	public void removeBucket(BucketId id) {
-		log.debug("Removing Bucket[{}]", id);
+		log.trace("Removing Bucket[{}]", id);
 		buckets.remove(id);
 	}
 
diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/CachedStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/CachedStore.java
index 0840ca471a..bf6588683f 100644
--- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/CachedStore.java
+++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/CachedStore.java
@@ -33,8 +33,7 @@ public void add(KEY key, VALUE value) {
 
 	@Override
 	public VALUE get(KEY key) {
-		// TODO: 08.01.2020 fk: This assumes that all values have been read at some point!
-		return cache.get(key);
+		return cache.computeIfAbsent(key, store::get);
 	}
 
 	@Override
diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterImportHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterImportHandler.java
index 991d5efd64..f5a1b5179b 100644
--- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterImportHandler.java
+++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterImportHandler.java
@@ -1,49 +1,155 @@
 package com.bakdata.conquery.mode.cluster;
 
+import java.io.IOException;
 import java.io.InputStream;
 import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 import com.bakdata.conquery.mode.ImportHandler;
-import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.datasets.Import;
 import com.bakdata.conquery.models.datasets.Table;
 import com.bakdata.conquery.models.datasets.concepts.Concept;
 import com.bakdata.conquery.models.datasets.concepts.Connector;
+import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId;
-import com.bakdata.conquery.models.jobs.ImportJob;
+import com.bakdata.conquery.models.identifiable.ids.specific.ImportId;
+import com.bakdata.conquery.models.identifiable.ids.specific.TableId;
+import com.bakdata.conquery.models.identifiable.ids.specific.WorkerId;
+import com.bakdata.conquery.models.messages.namespaces.specific.AddImport;
+import com.bakdata.conquery.models.messages.namespaces.specific.ImportBucket;
 import com.bakdata.conquery.models.messages.namespaces.specific.RemoveImportJob;
+import com.bakdata.conquery.models.preproc.PreprocessedData;
+import com.bakdata.conquery.models.preproc.PreprocessedHeader;
+import com.bakdata.conquery.models.preproc.PreprocessedReader;
 import com.bakdata.conquery.models.worker.DatasetRegistry;
 import com.bakdata.conquery.models.worker.DistributedNamespace;
 import com.bakdata.conquery.models.worker.Namespace;
+import com.bakdata.conquery.models.worker.WorkerInformation;
+import jakarta.ws.rs.BadRequestException;
+import jakarta.ws.rs.NotFoundException;
+import jakarta.ws.rs.WebApplicationException;
+import jakarta.ws.rs.core.Response;
 import lombok.AllArgsConstructor;
 import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
 
 /**
  * Handler of {@link Import} requests that realizes them both on the manager and the cluster's shards.
  */
 @AllArgsConstructor
-public
-class ClusterImportHandler implements ImportHandler {
+@Slf4j
+public class ClusterImportHandler implements ImportHandler {
 
-	private final ConqueryConfig config;
 	private final DatasetRegistry<DistributedNamespace> datasetRegistry;
 
 	@SneakyThrows
 	@Override
 	public void updateImport(Namespace namespace, InputStream inputStream) {
-		ImportJob job = ImportJob.createOrUpdate(
-				datasetRegistry.get(namespace.getDataset().getId()),
-				inputStream,
-				config.getCluster().getEntityBucketSize(),
-				true
-		);
+		handleImport(namespace, inputStream, true);
+	}
+
+	private static void handleImport(Namespace namespace, InputStream inputStream, boolean update) throws IOException {
+		try (PreprocessedReader parser = new PreprocessedReader(inputStream, namespace.getPreprocessMapper())) {
+			// We parse semi-manually as the incoming file consist of multiple documents we read progressively:
+			// 1) the header to check metadata
+			// 2...) The chunked Buckets
+
+			final PreprocessedHeader header = parser.readHeader();
+
+			final Table table = validateImportable(((DistributedNamespace) namespace), header, update);
+
+			readAndDistributeImport(((DistributedNamespace) namespace), table, header, parser);
+
+			clearDependentConcepts(namespace.getStorage().getAllConcepts(), table);
+		}
+	}
+
+	/**
+	 * Handle validity and update logic.
+	 */
+	private static Table validateImportable(DistributedNamespace namespace, PreprocessedHeader header, boolean update) {
+		final TableId tableId = new TableId(namespace.getDataset().getId(), header.getTable());
+		final ImportId importId = new ImportId(tableId, header.getName());
+
+		final Table table = namespace.getStorage().getTable(tableId);
+
+		if (table == null) {
+			throw new BadRequestException("Table[%s] does not exist.".formatted(tableId));
+		}
+
+		// Ensure that Import and Table have the same schema
+		final List<String> errors = header.assertMatch(table);
+
+		if (!errors.isEmpty()) {
+			final String errorsMessage = String.join("\n - ", errors);
+
+			log.error("Problems concerning Import `{}`:\n{}", importId, errorsMessage);
+			throw new BadRequestException("Headers[%s] do not match Table[%s]:\n%s".formatted(importId, table.getId(), errorsMessage));
+		}
+
+		final Import processedImport = namespace.getStorage().getImport(importId);
+
+		if (update) {
+			if (processedImport == null) {
+				throw new NotFoundException("Import[%s] is not present.".formatted(importId));
+			}
+
+			// before updating the import, make sure that all workers removed the prior import
+			namespace.getWorkerHandler().sendToAll(new RemoveImportJob(processedImport));
+			namespace.getStorage().removeImport(importId);
+		}
+		else if (processedImport != null) {
+			throw new WebApplicationException("Import[%s] is already present.".formatted(importId), Response.Status.CONFLICT);
+		}
+
+		return table;
+	}
 
-		namespace.getJobManager().addSlowJob(job);
+	private static void readAndDistributeImport(DistributedNamespace namespace, Table table, PreprocessedHeader header, PreprocessedReader reader) {
+		final TableId tableId = new TableId(namespace.getDataset().getId(), header.getTable());
+		final ImportId importId = new ImportId(tableId, header.getName());
+
+		log.info("BEGIN importing {} into {}", header.getName(), table);
+
+		Import imp = null;
+
+		final Map<Integer, Collection<String>> collectedEntities = new HashMap<>();
+
+		for (PreprocessedData container : (Iterable<? extends PreprocessedData>) () -> reader) {
+
+			if (imp == null) {
+				// We need a container to create a description.
+				imp = header.createImportDescription(table, container.getStores());
+
+				namespace.getWorkerHandler().sendToAll(new AddImport(imp));
+				namespace.getStorage().updateImport(imp);
+			}
+
+
+			final Bucket bucket = Bucket.fromPreprocessed(table, container, imp);
+
+			log.trace("DONE reading bucket `{}`, contains {} entities.", bucket.getId(), bucket.entities().size());
+
+			final WorkerInformation responsibleWorker = namespace.getWorkerHandler().assignResponsibleWorker(bucket.getId());
+
+			sendBucket(bucket, responsibleWorker);
+
+			// NOTE: I want the bucket to be GC'd as early as possible, so I just store the part(s) I need later.
+
+			collectedEntities.put(bucket.getBucket(), bucket.entities());
+		}
+
+		namespace.getJobManager().addSlowJob(new RegisterImportEntities(collectedEntities, namespace, importId));
+
+		log.debug("Successfully read {} Buckets, containing {} entities for `{}`", header.getNumberOfBuckets(), header.getNumberOfEntities(), importId);
+
+		namespace.getWorkerHandler().sendUpdatedWorkerInformation();
 
-		clearDependentConcepts(namespace.getStorage().getAllConcepts(), job.getTable());
 	}
 
-	private void clearDependentConcepts(Collection<Concept<?>> allConcepts, Table table) {
+	private static void clearDependentConcepts(Collection<Concept<?>> allConcepts, Table table) {
 		for (Concept<?> c : allConcepts) {
 			for (Connector con : c.getConnectors()) {
 				if (!con.getTable().equals(table)) {
@@ -55,24 +161,29 @@ private void clearDependentConcepts(Collection<Concept<?>> allConcepts, Table ta
 		}
 	}
 
+	/**
+	 * select, then send buckets.
+	 */
+	public static WorkerId sendBucket(Bucket bucket, WorkerInformation responsibleWorker) {
+
+		responsibleWorker.awaitFreeJobQueue();
+
+		log.trace("Sending Bucket[{}] to {}", bucket.getId(), responsibleWorker.getId());
+		responsibleWorker.send(new ImportBucket(bucket.getId().toString(), bucket));
+
+		return responsibleWorker.getId();
+	}
+
 	@SneakyThrows
 	@Override
 	public void addImport(Namespace namespace, InputStream inputStream) {
-		ImportJob job = ImportJob.createOrUpdate(
-				datasetRegistry.get(namespace.getDataset().getId()),
-				inputStream,
-				config.getCluster().getEntityBucketSize(),
-				false
-		);
-		namespace.getJobManager().addSlowJob(job);
-
-		clearDependentConcepts(namespace.getStorage().getAllConcepts(), job.getTable());
+		handleImport(namespace, inputStream, false);
 	}
 
 	@Override
 	public void deleteImport(Import imp) {
 
-		DatasetId id = imp.getTable().getDataset().getId();
+		final DatasetId id = imp.getTable().getDataset().getId();
 		final DistributedNamespace namespace = datasetRegistry.get(id);
 
 		clearDependentConcepts(namespace.getStorage().getAllConcepts(), imp.getTable());
@@ -83,4 +194,5 @@ public void deleteImport(Import imp) {
 		// Remove bucket assignments for consistency report
 		namespace.getWorkerHandler().removeBucketAssignmentsForImportFormWorkers(imp);
 	}
+
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java
index cafc855713..ed682e7112 100644
--- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java
+++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java
@@ -30,7 +30,7 @@ public ClusterManager provideManager(ConqueryConfig config, Environment environm
 		final ClusterConnectionManager connectionManager =
 				new ClusterConnectionManager(datasetRegistry, jobManager, environment.getValidator(), config, creator, clusterState);
 
-		final ImportHandler importHandler = new ClusterImportHandler(config, datasetRegistry);
+		final ImportHandler importHandler = new ClusterImportHandler(datasetRegistry);
 		final StorageListener extension = new ClusterStorageListener(jobManager, datasetRegistry);
 		final Supplier<Collection<ShardNodeInformation>> nodeProvider = () -> clusterState.getShardNodes().values();
 		final List<Task> adminTasks = List.of(new ReportConsistencyTask(clusterState));
diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/RegisterImportEntities.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/RegisterImportEntities.java
new file mode 100644
index 0000000000..e6c89725c1
--- /dev/null
+++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/RegisterImportEntities.java
@@ -0,0 +1,42 @@
+package com.bakdata.conquery.mode.cluster;
+
+import java.util.Collection;
+import java.util.Map;
+
+import com.bakdata.conquery.models.identifiable.ids.specific.ImportId;
+import com.bakdata.conquery.models.jobs.Job;
+import com.bakdata.conquery.models.worker.DistributedNamespace;
+import lombok.Data;
+
+/**
+ * This class handles registration of entities. Relevant for counting and resolving entities from external sources.
+ */
+@Data
+class RegisterImportEntities extends Job {
+
+	private final Map<Integer, Collection<String>> collectedEntities;
+
+
+	private final DistributedNamespace namespace;
+	private final ImportId importId;
+
+	@Override
+	public void execute() {
+		// This task is quite slow, so be delay it as far as possible.
+		for (Map.Entry<Integer, Collection<String>> bucket2Entities : collectedEntities.entrySet()) {
+			for (String entity : bucket2Entities.getValue()) {
+
+				if (namespace.getStorage().containsEntity(entity)) {
+					continue;
+				}
+
+				namespace.getStorage().registerEntity(entity, bucket2Entities.getKey());
+			}
+		}
+	}
+
+	@Override
+	public String getLabel() {
+		return "Handle Bucket %s assignments.".formatted(importId);
+	}
+}
diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeCache.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeCache.java
index 5875e89264..128dcf61bd 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeCache.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeCache.java
@@ -4,6 +4,7 @@
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 
+import com.bakdata.conquery.models.datasets.concepts.ConceptElement;
 import com.bakdata.conquery.models.exceptions.ConceptConfigurationException;
 import com.bakdata.conquery.util.CalculatedValue;
 import com.fasterxml.jackson.annotation.JsonIgnore;
@@ -35,7 +36,7 @@ public class ConceptTreeCache {
 	 * @implNote ConcurrentHashMap does not allow null values, but we want to have null values in the map. So we wrap the values in Optional.
 	 */
 	@JsonIgnore
-	private final Map<String, Optional<ConceptTreeChild>> cached = new ConcurrentHashMap<>();;
+	private final Map<String, Optional<ConceptElement<?>>> cached = new ConcurrentHashMap<>();;
 
 
 	/**
@@ -43,7 +44,7 @@ public class ConceptTreeCache {
 	 *
 	 * @param value
 	 */
-	public ConceptTreeChild findMostSpecificChild(String value, CalculatedValue<Map<String, Object>> rowMap) throws ConceptConfigurationException {
+	public ConceptElement<?> findMostSpecificChild(String value, CalculatedValue<Map<String, Object>> rowMap) throws ConceptConfigurationException {
 
 		if(cached.containsKey(value)) {
 			hits++;
@@ -52,7 +53,7 @@ public ConceptTreeChild findMostSpecificChild(String value, CalculatedValue<Map<
 
 		misses++;
 
-		final ConceptTreeChild child = treeConcept.findMostSpecificChild(value, rowMap);
+		final ConceptElement<?> child = treeConcept.findMostSpecificChild(value, rowMap);
 
 		if(!rowMap.isCalculated()) {
 			cached.put(value, Optional.ofNullable(child));
diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/TreeConcept.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/TreeConcept.java
index ab302384e9..0c2c90d8fd 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/TreeConcept.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/TreeConcept.java
@@ -126,11 +126,11 @@ public void initElements() throws ConfigurationException, JSONException {
 		}
 	}
 
-	public ConceptTreeChild findMostSpecificChild(String stringValue, CalculatedValue<Map<String, Object>> rowMap) throws ConceptConfigurationException {
+	public ConceptElement findMostSpecificChild(String stringValue, CalculatedValue<Map<String, Object>> rowMap) throws ConceptConfigurationException {
 		return findMostSpecificChild(stringValue, rowMap, null, getChildren());
 	}
 
-	private ConceptTreeChild findMostSpecificChild(String stringValue, CalculatedValue<Map<String, Object>> rowMap, ConceptTreeChild best, List<ConceptTreeChild> currentList)
+	private ConceptElement findMostSpecificChild(String stringValue, CalculatedValue<Map<String, Object>> rowMap, ConceptElement best, List<ConceptTreeChild> currentList)
 			throws ConceptConfigurationException {
 
 		while (currentList != null && !currentList.isEmpty()) {
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/Bucket.java b/backend/src/main/java/com/bakdata/conquery/models/events/Bucket.java
index 2257ab8b2c..530aa94187 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/Bucket.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/Bucket.java
@@ -5,6 +5,7 @@
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Objects;
 
 import com.bakdata.conquery.io.jackson.serializer.NsIdRef;
 import com.bakdata.conquery.models.common.CDateSet;
@@ -26,12 +27,14 @@
 import com.bakdata.conquery.models.identifiable.IdentifiableImpl;
 import com.bakdata.conquery.models.identifiable.ids.NamespacedIdentifiable;
 import com.bakdata.conquery.models.identifiable.ids.specific.BucketId;
+import com.bakdata.conquery.models.preproc.PreprocessedData;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonManagedReference;
 import com.google.common.collect.ImmutableSet;
 import io.dropwizard.validation.ValidationMethod;
 import it.unimi.dsi.fastutil.objects.Object2IntMap;
+import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
 import jakarta.validation.constraints.Min;
 import jakarta.validation.constraints.NotNull;
 import lombok.AccessLevel;
@@ -50,16 +53,16 @@
 @FieldNameConstants
 @Getter
 @Setter
-@ToString(of = {"numberOfEvents", "stores"}, callSuper = true)
+@ToString(onlyExplicitlyIncluded = true, callSuper = true)
 @AllArgsConstructor
 @RequiredArgsConstructor(onConstructor_ = {@JsonCreator}, access = AccessLevel.PROTECTED)
+
 public class Bucket extends IdentifiableImpl<BucketId> implements NamespacedIdentifiable<BucketId> {
 
 	@Min(0)
 	private final int bucket;
 
-	@Min(0)
-	private final int numberOfEvents;
+	@ToString.Include
 	@JsonManagedReference
 	@Setter(AccessLevel.PROTECTED)
 	private ColumnStore[] stores;
@@ -74,9 +77,25 @@ public class Bucket extends IdentifiableImpl<BucketId> implements NamespacedIden
 	 */
 	private final Object2IntMap<String> ends;
 
+	private final int numberOfEvents;
+
 	@NsIdRef
 	private final Import imp;
 
+	private static ColumnStore[] sortColumns(Table table, Map<String, ColumnStore> stores) {
+		return Arrays.stream(table.getColumns())
+					 .map(Column::getName)
+					 .map(stores::get)
+					 .map(Objects::requireNonNull)
+					 .toArray(ColumnStore[]::new);
+	}
+
+	public static Bucket fromPreprocessed(Table table, PreprocessedData container, Import imp) {
+		final ColumnStore[] storesSorted = sortColumns(table, container.getStores());
+		final int numberOfEvents = container.getEnds().values().stream().mapToInt(i -> i).max().orElse(0);
+
+		return new Bucket(container.getBucketId(), storesSorted, new Object2IntOpenHashMap<>(container.getStarts()), new Object2IntOpenHashMap<>(container.getEnds()),numberOfEvents, imp);
+	}
 
 	@JsonIgnore
 	@ValidationMethod(message = "Number of events does not match the length of some stores.")
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/BucketManager.java b/backend/src/main/java/com/bakdata/conquery/models/events/BucketManager.java
index fe91ec2ba8..d7f2a99f4a 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/BucketManager.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/BucketManager.java
@@ -124,7 +124,11 @@ private static void registerCBlock(CBlock cBlock, Map<Connector, Int2ObjectMap<M
 	public void fullUpdate() {
 		final CalculateCBlocksJob job = new CalculateCBlocksJob(storage, this, worker.getJobsExecutorService());
 
-		for (Concept<?> c : storage.getAllConcepts()) {
+		final Collection<Concept<?>> allConcepts = storage.getAllConcepts();
+
+		log.info("BEGIN full update for {} concepts.", allConcepts.size());
+
+		for (Concept<?> c : allConcepts) {
 			if (!(c instanceof TreeConcept)) {
 				continue;
 			}
@@ -142,7 +146,7 @@ public void fullUpdate() {
 						continue;
 					}
 
-					log.warn("CBlock[{}] missing in Storage. Queuing recalculation", cBlockId);
+					log.trace("CBlock[{}] missing in Storage. Queuing recalculation", cBlockId);
 					job.addCBlock(bucket, con);
 				}
 			}
@@ -365,6 +369,11 @@ public void addConcept(Concept<?> concept) {
 				job.addCBlock(bucket, connector);
 			}
 		}
+
+		if(job.isEmpty()){
+			return;
+		}
+
 		jobManager.addSlowJob(job);
 	}
 
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/CBlock.java b/backend/src/main/java/com/bakdata/conquery/models/events/CBlock.java
index 424fe60dc1..cfc5e0182b 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/CBlock.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/CBlock.java
@@ -12,6 +12,7 @@
 import com.bakdata.conquery.models.datasets.Dataset;
 import com.bakdata.conquery.models.datasets.Table;
 import com.bakdata.conquery.models.datasets.concepts.Concept;
+import com.bakdata.conquery.models.datasets.concepts.ConceptElement;
 import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.datasets.concepts.conditions.CTCondition;
 import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeCache;
@@ -164,7 +165,7 @@ else if (treeConcept.countElements() == 1) {
 					continue;
 				}
 
-				final ConceptTreeChild child = cache == null
+				final ConceptElement<?> child = cache == null
 											   ? treeConcept.findMostSpecificChild(stringValue, rowMap)
 											   : cache.findMostSpecificChild(stringValue, rowMap);
 
diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/EmptyBucket.java b/backend/src/main/java/com/bakdata/conquery/models/events/EmptyBucket.java
index f0b6f64951..5d1551cb8d 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/events/EmptyBucket.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/events/EmptyBucket.java
@@ -20,7 +20,7 @@ public class EmptyBucket extends Bucket {
 	private static final EmptyBucket Instance = new EmptyBucket();
 
 	public EmptyBucket() {
-		super(0, 0, Object2IntMaps.emptyMap(), Object2IntMaps.emptyMap(), null);
+		super(0, Object2IntMaps.emptyMap(), Object2IntMaps.emptyMap(), 0, null);
 		this.setStores(new ColumnStore[0]);
 	}
 
diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/InjectingCentralRegistry.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/InjectingCentralRegistry.java
deleted file mode 100644
index 05d4395dde..0000000000
--- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/InjectingCentralRegistry.java
+++ /dev/null
@@ -1,26 +0,0 @@
-package com.bakdata.conquery.models.identifiable;
-
-import java.util.Map;
-
-import com.bakdata.conquery.models.identifiable.ids.Id;
-import lombok.Data;
-import lombok.NonNull;
-import lombok.RequiredArgsConstructor;
-
-/**
- * Central Registry used to wire up incoming ids with already established ids.
- */
-@RequiredArgsConstructor
-@Data
-public class InjectingCentralRegistry extends CentralRegistry{
-	/**
-	 * This map is intentionally NOT an IdMap as it allows wiring up mismatched ids.
-	 */
-	@NonNull
-	private final Map<Id<?>, Identifiable<?>> injections;
-
-	@Override
-	protected <T extends Identifiable<?>> T get(Id<T> name) {
-		return (T) injections.get(name);
-	}
-}
diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java
index 52b56dd00c..6bd986ed3d 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java
@@ -147,7 +147,7 @@ public String resolve(ExternalId key) {
 		}
 
 		// Maybe we can find them directly in the dictionary?
-		if (storage.getEntityBucket(key.getId()).isPresent()) {
+		if (storage.containsEntity(key.getId())) {
 			return key.getId();
 		}
 
diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/CalculateCBlocksJob.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/CalculateCBlocksJob.java
index afad884d47..91287384b6 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/jobs/CalculateCBlocksJob.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/CalculateCBlocksJob.java
@@ -3,6 +3,9 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.stream.Collectors;
 
 import com.bakdata.conquery.io.storage.WorkerStorage;
@@ -15,9 +18,9 @@
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.ListeningExecutorService;
 import com.google.common.util.concurrent.MoreExecutors;
-import lombok.Getter;
+import lombok.Data;
 import lombok.RequiredArgsConstructor;
-import lombok.Setter;
+import lombok.ToString;
 import lombok.extern.slf4j.Slf4j;
 
 /**
@@ -27,44 +30,65 @@
  */
 @RequiredArgsConstructor
 @Slf4j
-
+@Data
+@ToString(onlyExplicitlyIncluded = true)
 public class CalculateCBlocksJob extends Job {
 
-	private final List<CalculationInformation> infos = new ArrayList<>();
+	private final List<CalculationInformationProcessor> tasks = new ArrayList<>();
 	private final WorkerStorage storage;
 	private final BucketManager bucketManager;
 	private final ExecutorService executorService;
 
+	@ToString.Include
 	@Override
 	public String getLabel() {
-		return "Calculate CBlocks[" + infos.size() + "]";
+		return "Calculate CBlocks[" + tasks.size() + "]";
 	}
 
 	public void addCBlock(Bucket bucket, ConceptTreeConnector connector) {
-		infos.add(new CalculationInformation(connector, bucket));
+		tasks.add(createInformationProcessor(connector, bucket));
+	}
+
+	private CalculationInformationProcessor createInformationProcessor(ConceptTreeConnector connector, Bucket bucket) {
+		return new CalculationInformationProcessor(connector, bucket, bucketManager, storage);
 	}
 
 	@Override
 	public void execute() throws Exception {
-		if (infos.isEmpty()) {
+		if (tasks.isEmpty()) {
 			return;
 		}
 
-		getProgressReporter().setMax(infos.size());
+		log.info("BEGIN calculate CBlocks for {} entries.", tasks.size());
 
-		final ListeningExecutorService executorService = MoreExecutors.listeningDecorator(this.executorService);
+		getProgressReporter().setMax(tasks.size());
 
-		final List<? extends ListenableFuture<?>> futures = infos.stream()
-																 .map(this::createInformationProcessor)
-																 .map(executorService::submit)
-																 .peek(f -> f.addListener(this::incrementProgressReporter, MoreExecutors.directExecutor()))
-																 .collect(Collectors.toList());
+		final ListeningExecutorService executorService = MoreExecutors.listeningDecorator(getExecutorService());
 
-		Futures.allAsList(futures).get();
-	}
+		final List<? extends ListenableFuture<?>> futures =
+				tasks.stream()
+					 .map(executorService::submit)
+					 .peek(fut -> fut.addListener(this::incrementProgressReporter, MoreExecutors.directExecutor()))
+					 .collect(Collectors.toList());
+
+
+		final ListenableFuture<?> all = Futures.allAsList(futures);
+
+		while (!all.isDone()) {
+			try {
+				all.get(1, TimeUnit.MINUTES);
+			}
+			catch (TimeoutException exception) {
+				log.debug("submitted={}, pool={}", tasks.size(), getExecutorService());
+
+				if (log.isTraceEnabled() && getExecutorService() instanceof ThreadPoolExecutor) {
+					log.trace("Waiting for {}", ((ThreadPoolExecutor) getExecutorService()).getQueue());
+				}
+			}
+		}
+
+		log.debug("DONE CalculateCBlocks for {} entries.", tasks.size());
 
-	private CalculationInformationProcessor createInformationProcessor(CalculationInformation info) {
-		return new CalculationInformationProcessor(info, bucketManager, storage);
 	}
 
 	private void incrementProgressReporter() {
@@ -72,52 +96,45 @@ private void incrementProgressReporter() {
 	}
 
 	public boolean isEmpty() {
-		return infos.isEmpty();
+		return tasks.isEmpty();
 	}
 
-	@RequiredArgsConstructor
-	@Getter
-	@Setter
-	private static class CalculationInformation {
+
+	@Data
+	@ToString(onlyExplicitlyIncluded = true)
+	private static class CalculationInformationProcessor implements Runnable {
 		private final ConceptTreeConnector connector;
 		private final Bucket bucket;
 
-		public CBlockId getCBlockId() {
-			return new CBlockId(getBucket().getId(), getConnector().getId());
-		}
-	}
-
-
-	@RequiredArgsConstructor
-	private static class CalculationInformationProcessor implements Runnable {
-		private final CalculationInformation info;
 		private final BucketManager bucketManager;
 		private final WorkerStorage storage;
 
 		@Override
 		public void run() {
 			try {
-				if (bucketManager.hasCBlock(info.getCBlockId())) {
-					log.trace("Skipping calculation of CBlock[{}] because its already present in the BucketManager.", info.getCBlockId());
+				if (bucketManager.hasCBlock(getCBlockId())) {
+					log.trace("Skipping calculation of CBlock[{}] because its already present in the BucketManager.", getCBlockId());
 					return;
 				}
 
-				CBlock cBlock = CBlock.createCBlock(info.getConnector(), info.getBucket(), bucketManager.getEntityBucketSize());
+				log.trace("BEGIN calculating CBlock for {}", getCBlockId());
+
+				final CBlock cBlock = CBlock.createCBlock(getConnector(), getBucket(), bucketManager.getEntityBucketSize());
+
+				log.trace("DONE calculating CBlock for {}", getCBlockId());
 
 				bucketManager.addCalculatedCBlock(cBlock);
 				storage.addCBlock(cBlock);
 			}
 			catch (Exception e) {
-				throw new RuntimeException(
-						String.format(
-								"Exception in CalculateCBlocksJob (CBlock=%s, connector=%s)",
-								info.getCBlockId(),
-								info.getConnector()
-						),
-						e
-				);
+				throw new RuntimeException("Exception in CalculateCBlocksJob %s".formatted(getCBlockId()), e);
 			}
 		}
 
+		@ToString.Include
+		public CBlockId getCBlockId() {
+			return new CBlockId(getBucket().getId(), getConnector().getId());
+		}
+
 	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/ImportJob.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/ImportJob.java
deleted file mode 100644
index 92fbd6ee47..0000000000
--- a/backend/src/main/java/com/bakdata/conquery/models/jobs/ImportJob.java
+++ /dev/null
@@ -1,372 +0,0 @@
-package com.bakdata.conquery.models.jobs;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import com.bakdata.conquery.models.datasets.Column;
-import com.bakdata.conquery.models.datasets.Dataset;
-import com.bakdata.conquery.models.datasets.Import;
-import com.bakdata.conquery.models.datasets.ImportColumn;
-import com.bakdata.conquery.models.datasets.Table;
-import com.bakdata.conquery.models.events.Bucket;
-import com.bakdata.conquery.models.events.MajorTypeId;
-import com.bakdata.conquery.models.events.stores.root.ColumnStore;
-import com.bakdata.conquery.models.exceptions.JSONException;
-import com.bakdata.conquery.models.identifiable.ids.specific.BucketId;
-import com.bakdata.conquery.models.identifiable.ids.specific.ImportId;
-import com.bakdata.conquery.models.identifiable.ids.specific.TableId;
-import com.bakdata.conquery.models.identifiable.ids.specific.WorkerId;
-import com.bakdata.conquery.models.messages.namespaces.specific.AddImport;
-import com.bakdata.conquery.models.messages.namespaces.specific.ImportBucket;
-import com.bakdata.conquery.models.messages.namespaces.specific.RemoveImportJob;
-import com.bakdata.conquery.models.preproc.PPColumn;
-import com.bakdata.conquery.models.preproc.PreprocessedData;
-import com.bakdata.conquery.models.preproc.PreprocessedHeader;
-import com.bakdata.conquery.models.preproc.PreprocessedReader;
-import com.bakdata.conquery.models.worker.DistributedNamespace;
-import com.bakdata.conquery.models.worker.WorkerHandler;
-import com.bakdata.conquery.models.worker.WorkerInformation;
-import com.bakdata.conquery.util.progressreporter.ProgressReporter;
-import com.google.common.base.Functions;
-import it.unimi.dsi.fastutil.ints.IntArrayList;
-import it.unimi.dsi.fastutil.ints.IntList;
-import it.unimi.dsi.fastutil.objects.Object2IntMap;
-import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
-import jakarta.ws.rs.BadRequestException;
-import jakarta.ws.rs.WebApplicationException;
-import jakarta.ws.rs.core.Response;
-import lombok.Getter;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-/**
- * This is the main routine to load data into Conquery.
- */
-@RequiredArgsConstructor
-@Slf4j
-public class ImportJob extends Job {
-
-	private static final int NUMBER_OF_STEPS = /* directly in execute = */4;
-	private final DistributedNamespace namespace;
-	@Getter
-	private final Table table;
-	private final int bucketSize;
-	private final PreprocessedHeader header;
-	private final PreprocessedData container;
-
-	public static ImportJob createOrUpdate(DistributedNamespace namespace, InputStream inputStream, int entityBucketSize, boolean update)
-			throws IOException {
-
-		try (PreprocessedReader parser = new PreprocessedReader(inputStream, namespace.getPreprocessMapper())) {
-
-			final Dataset ds = namespace.getDataset();
-
-			// We parse semi-manually as the incoming file consist of multiple documents we only read progressively:
-			// 1) the header to check metadata
-			// 2) The Dictionaries to be imported and transformed
-			// 3) The ColumnStores themselves which contain references to the previously imported dictionaries.
-
-
-			final PreprocessedHeader header = parser.readHeader();
-
-			final TableId tableId = new TableId(ds.getId(), header.getTable());
-			final Table table = namespace.getStorage().getTable(tableId);
-
-			if (table == null) {
-				throw new BadRequestException(String.format("Table[%s] does not exist.", tableId));
-			}
-
-			// Ensure that Import and Table have the same schema
-			final List<String> validationErrors = ensureHeadersMatch(table, header);
-
-			if(!validationErrors.isEmpty()){
-				final String errorMessage = String.join("\n -", validationErrors);
-
-				log.error("Problems concerning Import `{}`:{}", header.getName(), errorMessage);
-				throw new BadRequestException(String.format("Import[%s.%s] does not match Table[%s]:%s", header.getTable(), header.getName(), table.getId(), errorMessage));
-			}
-
-			final ImportId importId = new ImportId(table.getId(), header.getName());
-			final Import processedImport = namespace.getStorage().getImport(importId);
-
-			if (update) {
-				if (processedImport == null) {
-					throw new WebApplicationException(String.format("Import[%s] is not present.", importId), Response.Status.NOT_FOUND);
-				}
-				// before updating the import, make sure that all workers removed the last import
-				namespace.getWorkerHandler().sendToAll(new RemoveImportJob(processedImport));
-				namespace.getStorage().removeImport(importId);
-			}
-			else if (processedImport != null) {
-				throw new WebApplicationException(String.format("Import[%s] is already present.", importId), Response.Status.CONFLICT);
-			}
-
-
-			log.trace("Begin reading data.");
-
-			final PreprocessedData container = parser.readData();
-
-			log.debug("Done reading data. Contains {} Entities.", container.size());
-
-			log.info("Importing {} into {}", header.getName(), tableId);
-
-			return new ImportJob(
-					namespace,
-					table,
-					entityBucketSize,
-					header,
-					container
-			);
-		}
-	}
-
-	/**
-	 * Verify that the supplied table matches the preprocessed data in shape.
-	 */
-	public static List<String> ensureHeadersMatch(Table table, PreprocessedHeader importHeaders) {
-//		final StringJoiner errors = new StringJoiner("\n - ", "\n - ", "");
-
-		final List<String> errors = new ArrayList<>();
-
-		if (table.getColumns().length != importHeaders.getColumns().length) {
-			errors.add(String.format("Import column count=%d does not match table column count=%d", importHeaders.getColumns().length, table.getColumns().length));
-		}
-
-		final Map<String, MajorTypeId> typesByName = Arrays.stream(importHeaders.getColumns()).collect(Collectors.toMap(PPColumn::getName, PPColumn::getType));
-
-		for (PPColumn column : importHeaders.getColumns()) {
-			if (!typesByName.containsKey(column.getName())) {
-				errors.add("Column[%s] is missing."
-								   .formatted(column.getName()));
-			}
-			else if (!typesByName.get(column.getName()).equals(column.getType())) {
-				errors.add("Column[%s] Types do not match %s != %s"
-								   .formatted(column.getName(), typesByName.get(column.getName()), column.getType()));
-			}
-		}
-
-		return errors;
-	}
-
-
-	@Override
-	public void execute() throws JSONException, InterruptedException, IOException {
-
-		getProgressReporter().setMax(NUMBER_OF_STEPS);
-
-		log.trace("Updating primary dictionary");
-
-		getProgressReporter().report(1);
-
-		// Distribute the new IDs among workers
-		distributeWorkerResponsibilities(container.entities());
-
-		getProgressReporter().report(1);
-
-		final Import imp = createImport(header, container.getStores(), table.getColumns(), container.size());
-
-		namespace.getStorage().updateImport(imp);
-
-		final Map<Integer, List<String>> buckets2LocalEntities = groupEntitiesByBucket(container.entities(), bucketSize);
-
-
-		final ColumnStore[] storesSorted = Arrays.stream(table.getColumns())
-												 .map(Column::getName)
-												 .map(container.getStores()::get)
-												 .map(Objects::requireNonNull)
-												 .toArray(ColumnStore[]::new);
-
-
-		log.info("Start sending {} Buckets", buckets2LocalEntities.size());
-
-		// we use this to track assignment to workers.
-		final Map<WorkerId, Set<BucketId>> workerAssignments =
-				sendBuckets(container.getStarts(), container.getLengths(), imp, buckets2LocalEntities, storesSorted);
-
-		final WorkerHandler handler = namespace.getWorkerHandler();
-		workerAssignments.forEach(handler::addBucketsToWorker);
-
-	}
-
-	private void distributeWorkerResponsibilities(Set<String> entities) {
-		log.debug("Updating bucket assignments.");
-
-		synchronized (namespace) {
-			for (String entity : entities) {
-				final int bucket = namespace.getBucket(entity, bucketSize);
-
-				if (namespace.getWorkerHandler().getResponsibleWorkerForBucket(bucket) != null) {
-					continue;
-				}
-
-				namespace.getWorkerHandler().addResponsibility(bucket);
-			}
-		}
-	}
-
-	private Import createImport(PreprocessedHeader header, Map<String, ColumnStore> stores, Column[] columns, int size) {
-		final Import imp = new Import(table);
-
-		imp.setName(header.getName());
-		imp.setNumberOfEntries(header.getRows());
-		imp.setNumberOfEntities(size);
-
-		final ImportColumn[] importColumns = new ImportColumn[columns.length];
-
-		for (int i = 0; i < columns.length; i++) {
-			final ColumnStore store = stores.get(columns[i].getName());
-
-			final ImportColumn col = new ImportColumn(imp, store.createDescription(), store.getLines(), store.estimateMemoryConsumptionBytes());
-
-			col.setName(columns[i].getName());
-
-			importColumns[i] = col;
-		}
-
-		imp.setColumns(importColumns);
-
-		namespace.getWorkerHandler().sendToAll(new AddImport(imp));
-		return imp;
-	}
-
-	/**
-	 * Group entities by their global bucket id.
-	 */
-	private Map<Integer, List<String>> groupEntitiesByBucket(Set<String> entities, int bucketSize) {
-		return entities.stream()
-					   .collect(Collectors.groupingBy(entity -> namespace.getBucket(entity, bucketSize)));
-
-	}
-
-	/**
-	 * select, then send buckets.
-	 */
-	private Map<WorkerId, Set<BucketId>> sendBuckets(Map<String, Integer> starts, Map<String, Integer> lengths, Import imp, Map<Integer, List<String>> buckets2LocalEntities, ColumnStore[] storesSorted) {
-
-		final Map<WorkerId, Set<BucketId>> newWorkerAssignments = new HashMap<>();
-
-		final ProgressReporter subJob = getProgressReporter().subJob(buckets2LocalEntities.size());
-
-		for (Map.Entry<Integer, List<String>> bucket2entities : buckets2LocalEntities.entrySet()) {
-
-
-			final int bucketId = bucket2entities.getKey();
-			final List<String> entities = bucket2entities.getValue();
-
-			final WorkerInformation responsibleWorker = Objects.requireNonNull(
-					namespace
-							.getWorkerHandler()
-							.getResponsibleWorkerForBucket(bucketId),
-					() -> "No responsible worker for Bucket#" + bucketId
-			);
-
-			awaitFreeJobQueue(responsibleWorker);
-
-			final Map<String, Integer> bucketStarts = entities.stream()
-															  .filter(starts::containsKey)
-															  .collect(Collectors.toMap(Functions.identity(), starts::get));
-
-			final Map<String, Integer> bucketLengths = entities.stream()
-															   .filter(lengths::containsKey)
-															   .collect(Collectors.toMap(Functions.identity(), lengths::get));
-
-
-			assert !Collections.disjoint(bucketStarts.keySet(), bucketLengths.keySet());
-
-
-			final Bucket bucket =
-					selectBucket(bucketStarts, bucketLengths, storesSorted, imp, bucketId);
-
-			newWorkerAssignments.computeIfAbsent(responsibleWorker.getId(), (ignored) -> new HashSet<>())
-								.add(bucket.getId());
-
-			log.trace("Sending Bucket[{}] to {}", bucket.getId(), responsibleWorker.getId());
-			responsibleWorker.send(ImportBucket.forBucket(bucket));
-
-			subJob.report(1);
-		}
-
-		return newWorkerAssignments;
-	}
-
-	private void awaitFreeJobQueue(WorkerInformation responsibleWorker) {
-		try {
-			responsibleWorker.getConnectedShardNode().waitForFreeJobQueue();
-		}
-		catch (InterruptedException e) {
-			log.error("Interrupted while waiting for worker[{}] to have free space in queue", responsibleWorker, e);
-		}
-	}
-
-	/**
-	 * - remap Entity-Ids to global
-	 * - calculate per-Entity regions of Bucklet (start/end)
-	 * - split stores
-	 */
-	private Bucket selectBucket(Map<String, Integer> localStarts, Map<String, Integer> localLengths, ColumnStore[] stores, Import imp, int bucketId) {
-
-
-		final IntList selectionStart = new IntArrayList();
-		final IntList selectionLength = new IntArrayList();
-
-
-		// First entity of Bucket starts at 0, the following are appended.
-		final Object2IntMap<String> entityStarts = new Object2IntOpenHashMap<>();
-		final Object2IntMap<String> entityEnds = new Object2IntOpenHashMap<>();
-
-
-		int currentStart = 0;
-
-		for (Map.Entry<String, Integer> entity2Start : localStarts.entrySet()) {
-			final String entity = entity2Start.getKey();
-			final int start = entity2Start.getValue();
-
-			final int length = localLengths.get(entity);
-
-			selectionStart.add(start);
-
-			selectionLength.add(length);
-
-			entityStarts.put(entity, currentStart);
-			entityEnds.put(entity, currentStart + length);
-
-			currentStart += length;
-		}
-
-		// copy only the parts of the bucket we need
-		final ColumnStore[] bucketStores =
-				Arrays.stream(stores)
-					  .map(store -> store.select(selectionStart.toIntArray(), selectionLength.toIntArray()))
-					  .toArray(ColumnStore[]::new);
-
-		return new Bucket(
-				bucketId,
-				selectionLength.intStream().sum(),
-				bucketStores,
-				entityStarts,
-				entityEnds,
-				imp
-		);
-	}
-
-	private Dataset getDataset() {
-		return namespace.getDataset();
-	}
-
-
-	@Override
-	public String getLabel() {
-		return "Importing into " + table + " from " + header.getName();
-	}
-
-}
diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/AddImport.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/AddImport.java
index 5470d205c9..659f2a2493 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/AddImport.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/AddImport.java
@@ -20,7 +20,7 @@ public class AddImport extends WorkerMessage {
 
 	@Override
 	public void react(Worker context) throws Exception {
-		log.info("Received Import[{}], containing {} entries.", imp.getId(), imp.getNumberOfEntries());
+		log.trace("Received Import[{}], containing {} entries.", imp.getId(), imp.getNumberOfEntries());
 		context.addImport(imp);
 	}
 
diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ImportBucket.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ImportBucket.java
index b2b02666d8..def021735d 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ImportBucket.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ImportBucket.java
@@ -22,13 +22,9 @@ public class ImportBucket extends WorkerMessage {
 
 	private final Bucket bucket;
 
-	public static ImportBucket forBucket(Bucket bucket)  {
-		return new ImportBucket(bucket.getId().toString(),bucket);
-	}
-
 	@Override
 	public void react(Worker context) throws Exception {
-		log.trace("Received {}", bucket.getId());
+		log.debug("Received {}, containing {} entities", bucket.getId(), bucket.entities().size());
 
 		context.addBucket(bucket);
 	}
diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/Preprocessed.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/Preprocessed.java
index 3bb69761f2..e32f51b359 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/preproc/Preprocessed.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/Preprocessed.java
@@ -6,7 +6,9 @@
 import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.IntSummaryStatistics;
 import java.util.List;
 import java.util.Map;
@@ -22,11 +24,14 @@
 import com.bakdata.conquery.models.preproc.parser.Parser;
 import com.bakdata.conquery.models.preproc.parser.specific.StringParser;
 import com.fasterxml.jackson.core.JsonGenerator;
+import com.google.common.collect.Maps;
+import com.google.common.hash.Hashing;
 import it.unimi.dsi.fastutil.ints.IntArrayList;
 import it.unimi.dsi.fastutil.ints.IntList;
 import it.unimi.dsi.fastutil.ints.IntLists;
 import it.unimi.dsi.fastutil.objects.Object2IntAVLTreeMap;
 import it.unimi.dsi.fastutil.objects.Object2IntMap;
+import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 
@@ -79,7 +84,7 @@ public Preprocessed(ConqueryConfig config, PreprocessingJob preprocessingJob) th
 	}
 
 
-	public void write(File file) throws IOException {
+	public void write(File file, int buckets) throws IOException {
 
 		final Object2IntMap<String> entityStart = new Object2IntAVLTreeMap<>();
 		final Object2IntMap<String> entityLength = new Object2IntAVLTreeMap<>();
@@ -95,15 +100,23 @@ public void write(File file) throws IOException {
 
 		log.debug("Writing Headers");
 
-		final int hash = descriptor.calculateValidityHash(job.getCsvDirectory(), job.getTag());
+		//TODO this could actually be done at read-time, avoiding large allocations entirely. But in a different smaller PR.
+		final Map<Integer, Collection<String>> bucket2Entity = entityStart.keySet().stream()
+																		  .collect(Collectors.groupingBy(id -> getEntityBucket(buckets, id)))
+																		  .entrySet().stream()
+																		  .collect(Collectors.toMap(Map.Entry::getKey, entry -> new HashSet<>(entry.getValue())));
 
-		final PreprocessedHeader header = new PreprocessedHeader(descriptor.getName(), descriptor.getTable(), rows, columns, hash);
 
+		final int hash = descriptor.calculateValidityHash(job.getCsvDirectory(), job.getTag());
 
-		final PreprocessedData data = new PreprocessedData(entityStart, entityLength, columnStores);
+		final PreprocessedHeader header =
+				new PreprocessedHeader(descriptor.getName(), descriptor.getTable(), rows, entityStart.size(), bucket2Entity.size(), columns, hash);
 
+		writePreprocessed(file, header, entityStart, entityLength, columnStores, bucket2Entity);
+	}
 
-		writePreprocessed(file, header, data);
+	public static int getEntityBucket(int buckets, String id) {
+		return Hashing.consistentHash(id.hashCode(), buckets);
 	}
 
 	/**
@@ -172,7 +185,7 @@ private Map<String, ColumnStore> combineStores(Object2IntMap<String> entityStart
 		return columnStores;
 	}
 
-	private static void writePreprocessed(File file, PreprocessedHeader header, PreprocessedData data) throws IOException {
+	private static void writePreprocessed(File file, PreprocessedHeader header, Map<String, Integer> globalStarts, Map<String, Integer> globalLengths, Map<String, ColumnStore> data, Map<Integer, Collection<String>> bucket2Entities) throws IOException {
 		final OutputStream out = new GZIPOutputStream(new FileOutputStream(file));
 		try (JsonGenerator generator = Jackson.BINARY_MAPPER.copy().enable(JsonGenerator.Feature.AUTO_CLOSE_TARGET).getFactory().createGenerator(out)) {
 
@@ -180,11 +193,61 @@ private static void writePreprocessed(File file, PreprocessedHeader header, Prep
 
 			generator.writeObject(header);
 
-
 			log.debug("Writing data");
 
-			generator.writeObject(data);
+			for (Map.Entry<Integer, Collection<String>> bucketIds : bucket2Entities.entrySet()) {
+				final Collection<String> entities = bucketIds.getValue();
+
+				final Map<String, Integer> starts = Maps.filterKeys(globalStarts, entities::contains);
+				final Map<String, Integer> lengths = Maps.filterKeys(globalLengths, entities::contains);
+
+				final PreprocessedData preprocessedData = selectBucket(bucketIds.getKey(), starts, lengths, data);
+
+				generator.writeObject(preprocessedData);
+			}
+		}
+	}
+
+	private static PreprocessedData selectBucket(int bucket, Map<String, Integer> localStarts, Map<String, Integer> localLengths, Map<String, ColumnStore> stores) {
+
+
+		final IntList selectionStart = new IntArrayList();
+		final IntList selectionLength = new IntArrayList();
+
+
+		// First entity of Bucket starts at 0, the following are appended.
+		final Object2IntMap<String> entityStarts = new Object2IntOpenHashMap<>();
+		final Object2IntMap<String> entityEnds = new Object2IntOpenHashMap<>();
+
+
+		int currentStart = 0;
+
+		for (Map.Entry<String, Integer> entity2Start : localStarts.entrySet()) {
+			final String entity = entity2Start.getKey();
+			final int start = entity2Start.getValue();
+
+			final int length = localLengths.get(entity);
+
+			selectionStart.add(start);
+
+			selectionLength.add(length);
+
+			entityStarts.put(entity, currentStart);
+			entityEnds.put(entity, currentStart + length);
+
+			currentStart += length;
 		}
+
+		final Map<String, ColumnStore> selected = new HashMap<>();
+
+		for (Map.Entry<String, ColumnStore> entry : stores.entrySet()) {
+			final String name = entry.getKey();
+			final ColumnStore store = entry.getValue();
+
+			selected.put(name, store.select(selectionStart.toIntArray(), selectionLength.toIntArray()));
+		}
+
+		return new PreprocessedData(bucket, entityStarts, entityEnds, selected);
 	}
 
 	public synchronized String addPrimary(String primary) {
diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/PreprocessedData.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/PreprocessedData.java
index 313d44ee09..0df0e7ccc7 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/preproc/PreprocessedData.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/PreprocessedData.java
@@ -12,9 +12,10 @@
 @Data
 @AllArgsConstructor(onConstructor_ = @JsonCreator)
 public class PreprocessedData {
+	private final int bucketId;
 
 	private final Map<String, Integer> starts;
-	private final Map<String, Integer> lengths;
+	private final Map<String, Integer> ends;
 
 	private final Map<String, ColumnStore> stores;
 
diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/PreprocessedHeader.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/PreprocessedHeader.java
index b032dbb8a8..6b33c7abd8 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/preproc/PreprocessedHeader.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/PreprocessedHeader.java
@@ -1,5 +1,17 @@
 package com.bakdata.conquery.models.preproc;
 
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import com.bakdata.conquery.models.datasets.Column;
+import com.bakdata.conquery.models.datasets.Import;
+import com.bakdata.conquery.models.datasets.ImportColumn;
+import com.bakdata.conquery.models.datasets.Table;
+import com.bakdata.conquery.models.events.MajorTypeId;
+import com.bakdata.conquery.models.events.stores.root.ColumnStore;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import lombok.AllArgsConstructor;
 import lombok.Data;
@@ -33,6 +45,10 @@ public class PreprocessedHeader {
 	 * Number of rows in the Preprocessed file.
 	 */
 	private long rows;
+	private long numberOfEntities;
+
+	//TODO use Set<Integer> to track actually included buckets,to split phase bucket assignment.
+	private int numberOfBuckets;
 
 	/**
 	 * The specific columns and their associated MajorType for validation.
@@ -44,4 +60,60 @@ public class PreprocessedHeader {
 	 */
 	private int validityHash;
 
+	public Import createImportDescription(Table table, Map<String, ColumnStore> stores) {
+		final Import imp = new Import(table);
+
+		imp.setName(getName());
+		imp.setNumberOfEntries(getRows());
+		imp.setNumberOfEntities(getNumberOfEntities());
+
+		final ImportColumn[] importColumns = new ImportColumn[columns.length];
+
+		for (int i = 0; i < columns.length; i++) {
+			final ColumnStore store = stores.get(columns[i].getName());
+
+			final ImportColumn col = new ImportColumn(imp, store.createDescription(), store.getLines(), numberOfBuckets * store.estimateMemoryConsumptionBytes());
+
+			col.setName(columns[i].getName());
+
+			importColumns[i] = col;
+		}
+
+		imp.setColumns(importColumns);
+
+		return imp;
+	}
+
+
+	/**
+	 * Verify that the supplied table matches the preprocessed' data in shape.
+	 *
+	 * @return
+	 */
+	public List<String> assertMatch(Table table) {
+		final List<String> errors = new ArrayList<>();
+
+		if (table.getColumns().length != getColumns().length) {
+			errors.add("Import column count=`%d` does not match table column count=`%d`".formatted(getColumns().length, table.getColumns().length));
+		}
+
+		final Map<String, MajorTypeId> typesByName = Arrays.stream(getColumns()).collect(Collectors.toMap(PPColumn::getName, PPColumn::getType));
+
+		for (int i = 0; i < Math.min(table.getColumns().length, getColumns().length); i++) {
+			final Column column = table.getColumns()[i];
+
+			if (!typesByName.containsKey(column.getName())) {
+				errors.add("Column[%s] is missing.".formatted(column.getName()));
+				continue;
+			}
+
+			if (!typesByName.get(column.getName()).equals(column.getType())) {
+				errors.add("Column[%s] Types do not match %s != %s".formatted(column.getName(), typesByName.get(column.getName()), column.getType()));
+			}
+		}
+
+		return errors;
+	}
+
+
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/PreprocessedReader.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/PreprocessedReader.java
index abdd8574d1..340433ae6c 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/preproc/PreprocessedReader.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/PreprocessedReader.java
@@ -2,13 +2,8 @@
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.Iterator;
 
-import com.bakdata.conquery.models.identifiable.Identifiable;
-import com.bakdata.conquery.models.identifiable.InjectingCentralRegistry;
-import com.bakdata.conquery.models.identifiable.ids.Id;
-import com.bakdata.conquery.models.worker.SingletonNamespaceCollection;
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -16,6 +11,7 @@
 import lombok.AccessLevel;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.experimental.Accessors;
 
 /**
@@ -23,7 +19,7 @@
  * Header then Dictionaries then Data. Only this order is possible.
  */
 @RequiredArgsConstructor(access = AccessLevel.PACKAGE)
-public class PreprocessedReader implements AutoCloseable {
+public class PreprocessedReader implements AutoCloseable, Iterator<PreprocessedData> {
 	@Override
 	public void close() throws IOException {
 		parser.close();
@@ -40,45 +36,39 @@ public enum LastRead {
 
 	@Getter
 	private LastRead lastRead = LastRead.BEGIN;
+	private int bucketsRemaining;
 	private final JsonParser parser;
-	private final Map<Id<?>, Identifiable<?>> replacements = new HashMap<>();
 
 	public PreprocessedReader(InputStream inputStream, ObjectMapper objectMapper) throws IOException {
-		final InjectingCentralRegistry injectingCentralRegistry = new InjectingCentralRegistry(replacements);
-		final SingletonNamespaceCollection namespaceCollection = new SingletonNamespaceCollection(injectingCentralRegistry);
 
-		parser = namespaceCollection.injectIntoNew(objectMapper)
-				.enable(JsonGenerator.Feature.AUTO_CLOSE_TARGET)
-				.getFactory()
-				.createParser(inputStream);
+		parser = objectMapper.copy().enable(JsonGenerator.Feature.AUTO_CLOSE_TARGET)
+							 .getFactory()
+							 .createParser(inputStream);
 	}
 
-	public void addReplacement(Id<?> id, Identifiable<?> replacement) {
-		this.replacements.put(id, replacement);
-	}
-
-	public <K extends Id<?>, V extends Identifiable<?>> void addAllReplacements(Map<K, V> replacements) {
-		this.replacements.putAll(replacements);
-	}
 
 	public PreprocessedHeader readHeader() throws IOException {
 		Preconditions.checkState(lastRead.equals(LastRead.BEGIN));
 
 		final PreprocessedHeader header = parser.readValueAs(PreprocessedHeader.class);
+		bucketsRemaining = header.getNumberOfBuckets();
 
 		lastRead = lastRead.next();
 		return header;
 	}
 
 
+	@Override
+	public boolean hasNext() {
+		return bucketsRemaining > 0;
+	}
 
-	public PreprocessedData readData() throws IOException {
-		Preconditions.checkState(lastRead.equals(LastRead.HEADER));
-
-		final PreprocessedData dictionaries = parser.readValueAs(PreprocessedData.class);
+	@SneakyThrows
+	@Override
+	public PreprocessedData next() {
+		bucketsRemaining--;
 
-		lastRead = lastRead.next();
-		return dictionaries;
+		return parser.readValueAs(PreprocessedData.class);
 	}
 
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/Preprocessor.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/Preprocessor.java
index a7908aa324..f3e31e6078 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/preproc/Preprocessor.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/Preprocessor.java
@@ -54,7 +54,7 @@ public static File getTaggedVersion(File file, String tag, String extension) {
 	 * <p>
 	 * Reads CSV file, per row extracts the primary key, then applies other transformations on each row, then compresses the data with {@link ColumnStore}.
 	 */
-	public static void preprocess(PreprocessingJob preprocessingJob, ProgressBar totalProgress, ConqueryConfig config) throws IOException {
+	public static void preprocess(PreprocessingJob preprocessingJob, ProgressBar totalProgress, ConqueryConfig config, int buckets) throws IOException {
 
 		final File preprocessedFile = preprocessingJob.getPreprocessedFile();
 		TableImportDescriptor descriptor = preprocessingJob.getDescriptor();
@@ -209,7 +209,7 @@ else if (errors == config.getPreprocessor().getMaximumPrintedErrors()) {
 			exceptions.forEach((clazz, count) -> log.warn("Got {} `{}`", count, clazz.getSimpleName()));
 		}
 
-		result.write(tmp);
+		result.write(tmp, buckets);
 
 		if (errors > 0) {
 			log.warn("Had {}% faulty lines ({} of ~{} lines)", String.format("%.2f", 100d * (double) errors / (double) lineId), errors, lineId);
diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java
index 76a5d4664f..dda525ee97 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java
@@ -54,18 +54,12 @@ public DistributedNamespace(
 		this.workerHandler = workerHandler;
 	}
 
-	public int getBucket(String entity, int bucketSize) {
-		final NamespaceStorage storage = getStorage();
-		return storage.getEntityBucket(entity)
-					  .orElseGet(() -> storage.assignEntityBucket(entity, bucketSize));
-	}
-
 	@Override
 	void updateMatchingStats() {
-		final Collection<Concept<?>> concepts = this.getStorage().getAllConcepts()
-													.stream()
-													.filter(concept -> concept.getMatchingStats() == null)
-													.collect(Collectors.toSet());
+		final Collection<Concept<?>> concepts = getStorage().getAllConcepts()
+															.stream()
+															.filter(concept -> concept.getMatchingStats() == null)
+															.collect(Collectors.toSet());
 		getWorkerHandler().sendToAll(new UpdateMatchingStatsMessage(concepts));
 	}
 
diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/ShardNodeInformation.java b/backend/src/main/java/com/bakdata/conquery/models/worker/ShardNodeInformation.java
index fe15ed6524..cc6c780ddf 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/worker/ShardNodeInformation.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/worker/ShardNodeInformation.java
@@ -12,6 +12,7 @@
 import com.bakdata.conquery.models.messages.network.MessageToShardNode;
 import com.codahale.metrics.SharedMetricRegistries;
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.google.common.base.Stopwatch;
 import lombok.Getter;
 import lombok.ToString;
 import lombok.extern.slf4j.Slf4j;
@@ -102,8 +103,12 @@ public void waitForFreeJobQueue() throws InterruptedException {
 		}
 
 		synchronized (jobManagerSync) {
-			log.trace("Have to wait for free JobQueue");
+			final Stopwatch waiting = Stopwatch.createStarted();
+			log.trace("Shard {}, have to wait for free JobQueue (backpressure={})", session, backpressure);
+
 			jobManagerSync.wait();
+
+			log.debug("Shard {}, Waited {} for free JobQueue", session, waiting.stop());
 		}
 	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/WorkerHandler.java b/backend/src/main/java/com/bakdata/conquery/models/worker/WorkerHandler.java
index 070e546974..c6c65dae41 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/worker/WorkerHandler.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/worker/WorkerHandler.java
@@ -103,24 +103,24 @@ public synchronized void removeBucketAssignmentsForImportFormWorkers(@NonNull Im
 		sendUpdatedWorkerInformation();
 	}
 
-	private synchronized void sendUpdatedWorkerInformation() {
+	public synchronized void sendUpdatedWorkerInformation() {
 		for (WorkerInformation w : workers.values()) {
 			w.send(new UpdateWorkerBucket(w));
 		}
 	}
 
-	public synchronized void addBucketsToWorker(@NonNull WorkerId id, @NonNull Set<BucketId> bucketIds) {
+	public synchronized void registerBucketForWorker(@NonNull WorkerId id, @NonNull BucketId bucketId) {
 		// Ensure that add and remove are not executed at the same time.
 		// We don't make assumptions about the underlying implementation regarding thread safety
 		WorkerToBucketsMap workerBuckets = storage.getWorkerBuckets();
+
 		if (workerBuckets == null) {
 			workerBuckets = createWorkerBucketsMap();
 		}
-		workerBuckets.addBucketForWorker(id, bucketIds);
 
-		storage.setWorkerToBucketsMap(workerBuckets);
+		workerBuckets.addBucketForWorker(id, bucketId);
 
-		sendUpdatedWorkerInformation();
+		storage.setWorkerToBucketsMap(workerBuckets);
 	}
 
 	private synchronized WorkerToBucketsMap createWorkerBucketsMap() {
@@ -138,11 +138,12 @@ public synchronized WorkerInformation getResponsibleWorkerForBucket(int bucket)
 	}
 
 	/**
+	 * @return
 	 * @implNote Currently the least occupied Worker receives a new Bucket, this can change in later implementations. (For example for
 	 * dedicated Workers, or entity weightings)
 	 */
 
-	public synchronized void addResponsibility(int bucket) {
+	public synchronized WorkerInformation addResponsibility(int bucket) {
 		final WorkerInformation smallest = workers
 				.stream()
 				.min(Comparator.comparing(si -> si.getIncludedBuckets().size()))
@@ -153,6 +154,8 @@ public synchronized void addResponsibility(int bucket) {
 		bucket2WorkerMap.put(bucket, smallest);
 
 		smallest.getIncludedBuckets().add(bucket);
+
+		return smallest;
 	}
 
 	public void register(ShardNodeInformation node, WorkerInformation info) {
@@ -192,6 +195,19 @@ public Set<BucketId> getBucketsForWorker(WorkerId workerId) {
 		return workerBuckets.getBucketsForWorker(workerId);
 	}
 
+	public synchronized WorkerInformation assignResponsibleWorker(BucketId bucket) {
+
+		WorkerInformation responsibleWorkerForBucket = getResponsibleWorkerForBucket(bucket.getBucket());
+
+		if (responsibleWorkerForBucket == null) {
+			responsibleWorkerForBucket = addResponsibility(bucket.getBucket());
+		}
+
+		registerBucketForWorker(responsibleWorkerForBucket.getId(), bucket);
+
+		return responsibleWorkerForBucket;
+	}
+
 	private record PendingReaction(UUID callerId, Set<WorkerId> pendingWorkers, ActionReactionMessage parent) {
 
 		/**
diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/WorkerInformation.java b/backend/src/main/java/com/bakdata/conquery/models/worker/WorkerInformation.java
index def028d002..8eb023e427 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/worker/WorkerInformation.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/worker/WorkerInformation.java
@@ -12,13 +12,13 @@
 import it.unimi.dsi.fastutil.ints.IntArraySet;
 import jakarta.validation.constraints.Min;
 import jakarta.validation.constraints.NotNull;
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.Setter;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 
-@Getter
-@Setter
-@EqualsAndHashCode(callSuper = true)
+@Data
+@NoArgsConstructor
+@Slf4j
 public class WorkerInformation extends NamedImpl<WorkerId> implements MessageSender.Transforming<WorkerMessage, MessageToShardNode> {
 	@NotNull
 	private DatasetId dataset;
@@ -32,6 +32,15 @@ public class WorkerInformation extends NamedImpl<WorkerId> implements MessageSen
 	@Min(0)
 	private int entityBucketSize;
 
+	public void awaitFreeJobQueue() {
+		try {
+			getConnectedShardNode().waitForFreeJobQueue();
+		}
+		catch (InterruptedException e) {
+			log.error("Interrupted while waiting for worker[{}] to have free space in queue", this, e);
+		}
+	}
+
 
 	@Override
 	public WorkerId createId() {
diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/WorkerToBucketsMap.java b/backend/src/main/java/com/bakdata/conquery/models/worker/WorkerToBucketsMap.java
index 4f7b03072f..70b98cd832 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/worker/WorkerToBucketsMap.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/worker/WorkerToBucketsMap.java
@@ -9,7 +9,6 @@
 import com.bakdata.conquery.models.identifiable.ids.specific.BucketId;
 import com.bakdata.conquery.models.identifiable.ids.specific.ImportId;
 import com.bakdata.conquery.models.identifiable.ids.specific.WorkerId;
-import com.google.common.collect.ImmutableSet;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
@@ -21,23 +20,23 @@
 @Getter
 @Setter
 public class WorkerToBucketsMap {
-    private Map<WorkerId, Set<BucketId>> map = new ConcurrentHashMap<>();
+	private Map<WorkerId, Set<BucketId>> map = new ConcurrentHashMap<>();
 
-    public Set<BucketId> getBucketsForWorker(WorkerId workerId) {
-        // Don't modify the underlying map here
-        Set<BucketId> buckets = map.get(workerId);
-        if (buckets != null) {
+	public Set<BucketId> getBucketsForWorker(WorkerId workerId) {
+		// Don't modify the underlying map here
+		Set<BucketId> buckets = map.get(workerId);
+		if (buckets != null) {
 			// Don't allow modification
 			return Collections.unmodifiableSet(buckets);
 		}
-        return Collections.emptySet();
-    }
+		return Collections.emptySet();
+	}
 
-    public void addBucketForWorker(@NonNull WorkerId id, @NonNull Set<BucketId> bucketIds) {
-        map.computeIfAbsent(id, k -> new HashSet<>()).addAll(bucketIds);
-    }
+	public void addBucketForWorker(@NonNull WorkerId id, @NonNull BucketId bucketId) {
+		map.computeIfAbsent(id, k -> new HashSet<>()).add(bucketId);
+	}
 
-    public void removeBucketsOfImport(@NonNull ImportId importId) {
-        map.values().forEach(set -> set.removeIf(bucketId -> bucketId.getImp().equals(importId)));
-    }
+	public void removeBucketsOfImport(@NonNull ImportId importId) {
+		map.values().forEach(set -> set.removeIf(bucketId -> bucketId.getImp().equals(importId)));
+	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java
index 9f643cd18b..908aa69235 100644
--- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java
+++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java
@@ -26,9 +26,9 @@
 import com.bakdata.conquery.models.datasets.Dataset;
 import com.bakdata.conquery.models.datasets.PreviewConfig;
 import com.bakdata.conquery.models.datasets.concepts.Concept;
+import com.bakdata.conquery.models.datasets.concepts.ConceptElement;
 import com.bakdata.conquery.models.datasets.concepts.FrontEndConceptBuilder;
 import com.bakdata.conquery.models.datasets.concepts.filters.specific.SelectFilter;
-import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeChild;
 import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept;
 import com.bakdata.conquery.models.exceptions.ConceptConfigurationException;
 import com.bakdata.conquery.models.exceptions.ValidatorHelper;
@@ -294,7 +294,7 @@ public ResolvedConceptsResult resolveConceptElements(TreeConcept concept, List<S
 
 		for (String conceptCode : conceptCodes) {
 			try {
-				final ConceptTreeChild child = concept.findMostSpecificChild(conceptCode, new CalculatedValue<>(Collections::emptyMap));
+				final ConceptElement<?> child = concept.findMostSpecificChild(conceptCode, new CalculatedValue<>(Collections::emptyMap));
 
 				if (child != null) {
 					resolvedCodes.add(child.getId());
diff --git a/backend/src/main/java/com/bakdata/conquery/util/progressreporter/ProgressReporterImpl.java b/backend/src/main/java/com/bakdata/conquery/util/progressreporter/ProgressReporterImpl.java
index c6c12cf9e0..ce429d4d82 100644
--- a/backend/src/main/java/com/bakdata/conquery/util/progressreporter/ProgressReporterImpl.java
+++ b/backend/src/main/java/com/bakdata/conquery/util/progressreporter/ProgressReporterImpl.java
@@ -2,6 +2,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
 
 import com.fasterxml.jackson.annotation.JsonValue;
 import lombok.Getter;
@@ -10,11 +11,13 @@
 @Slf4j
 public class ProgressReporterImpl implements ProgressReporter {
 
+	public long getMax() {
+		return max.get();
+	}
 
-	@Getter(onMethod_ = @Override)
-	private long max = 1;
-	private long innerProgress = 0;
-	private long reservedForChildren = 0;
+	private final AtomicLong max = new AtomicLong(1);
+	private final AtomicLong innerProgress = new AtomicLong(0);
+	private final AtomicLong reservedForChildren = new AtomicLong(0);
 	private final List<ProgressReporterImpl> children = new ArrayList<>();
 
 	@Getter
@@ -53,7 +56,7 @@ public double getProgress() {
 	}
 	
 	public long getAbsoluteProgress() {
-		long absoluteProgress = innerProgress;
+		long absoluteProgress = innerProgress.get();
 
 		for (ProgressReporterImpl child : children) {
 			absoluteProgress += child.getAbsoluteProgress();
@@ -63,7 +66,7 @@ public long getAbsoluteProgress() {
 	}
 	
 	public long getAbsoluteMax() {
-		long absoluteMax = max;
+		long absoluteMax = max.get();
 
 		for (ProgressReporterImpl child : children) {
 			absoluteMax += child.getAbsoluteMax();
@@ -78,7 +81,7 @@ public ProgressReporter subJob(long steps) {
 			throw new IllegalStateException("You need to start the Progress Reporter before you can add subjobs");
 		}
 
-		reservedForChildren += steps;
+		reservedForChildren.addAndGet(steps);
 
 		ProgressReporterImpl childPr = new ProgressReporterImpl();
 		childPr.start();
@@ -98,12 +101,12 @@ public void report(int steps) {
 			log.warn("Progress reporter was not started");
 			return;
 		}
-		if (innerProgress + steps > max) {
+		if (innerProgress.get() + steps > max.get()) {
 			log.warn("Progress({}) + ChildProgressReserve({}) + Steps({}) is bigger than the maximum Progress({}). There might be to many reports in the code.", innerProgress, reservedForChildren, steps, max);
 			return;
 		}
 
-		innerProgress += steps;
+		innerProgress.addAndGet(steps);
 	}
 
 	@Override
@@ -114,12 +117,7 @@ public void setMax(long max) {
 			return;
 		}
 
-		if (this.max > max) {
-			log.warn("Max cannot be decreased.");
-			return;
-		}
-
-		this.max = max;
+		this.max.set(max);
 	}
 
 	@Override
@@ -138,7 +136,7 @@ public void done() {
 			log.trace("Done was called before all steps were been reported. There might be missing reporting steps in the code.");
 		}
 
-		innerProgress = max - reservedForChildren;
+		innerProgress.set(max.get() - reservedForChildren.get());
 	}
 	
 
diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java b/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java
index 3b64377522..ef12549871 100644
--- a/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java
+++ b/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java
@@ -17,11 +17,6 @@
 import java.util.Map;
 import java.util.UUID;
 
-import jakarta.ws.rs.client.Entity;
-import jakarta.ws.rs.client.Invocation;
-import jakarta.ws.rs.core.MediaType;
-import jakarta.ws.rs.core.Response;
-
 import com.bakdata.conquery.ConqueryConstants;
 import com.bakdata.conquery.apiv1.query.ConceptQuery;
 import com.bakdata.conquery.apiv1.query.Query;
@@ -53,6 +48,10 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.univocity.parsers.csv.CsvParser;
+import jakarta.ws.rs.client.Entity;
+import jakarta.ws.rs.client.Invocation;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
 import lombok.NonNull;
 import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;
@@ -132,10 +131,6 @@ private static void uploadTable(StandaloneSupport support, Table table) {
 		}
 	}
 
-	public static void importTableContents(StandaloneSupport support, RequiredTable[] tables) throws Exception {
-		importTableContents(support, Arrays.asList(tables));
-	}
-
 	public static List<File> generateCqpp(StandaloneSupport support, Collection<RequiredTable> tables) throws Exception {
 		List<File> preprocessedFiles = new ArrayList<>();
 		List<File> descriptions = new ArrayList<>();
@@ -177,57 +172,49 @@ public static List<File> generateCqpp(StandaloneSupport support, Collection<Requ
 		return preprocessedFiles;
 	}
 
-	public static void importCqppFile(StandaloneSupport support, File cqpp) {
-		assertThat(cqpp).exists();
+	public static void uploadCqpp(StandaloneSupport support, File cqpp, boolean update, Response.Status.Family expectedResponseFamily) {
+		if(update) {
+			assertThat(cqpp).exists();
+		}
+
+		final String methodName = update ? "updateImport" : "addImport";
+
+		final URI addImport =
+				HierarchyHelper.hierarchicalPath(support.defaultAdminURIBuilder(), AdminDatasetResource.class, methodName)
+							   .queryParam("file", cqpp)
+							   .buildFromMap(Map.of(ResourceConstants.DATASET, support.getDataset().getId()));
 
-		final URI addImport = HierarchyHelper.hierarchicalPath(support.defaultAdminURIBuilder(), AdminDatasetResource.class, "addImport")
-											 .queryParam("file", cqpp)
-											 .buildFromMap(Map.of(ResourceConstants.DATASET, support.getDataset().getId().toString()));
+		final Entity<Entity<String>> entity = Entity.entity(Entity.json(""), MediaType.APPLICATION_JSON_TYPE);
 
 		final Invocation.Builder request = support.getClient()
 												  .target(addImport)
 												  .request(MediaType.APPLICATION_JSON);
-		try (final Response response = request
-				.post(Entity.entity(null, MediaType.APPLICATION_JSON_TYPE))) {
-
-			assertThat(response.getStatusInfo().getFamily())
-					.describedAs(new LazyTextDescription(() -> response.readEntity(String.class)))
-					.isEqualTo(Response.Status.Family.SUCCESSFUL);
-		}
-	}
 
-	public static void updateCqppFile(StandaloneSupport support, File cqpp, Response.Status.Family expectedResponseFamily, String expectedReason) {
-		assertThat(cqpp).exists();
+		final Invocation invocation = update ? request.buildPut(entity) : request.buildPost(entity);
 
-		final URI addImport = HierarchyHelper.hierarchicalPath(support.defaultAdminURIBuilder(), AdminDatasetResource.class, "updateImport")
-											 .queryParam("file", cqpp)
-											 .buildFromMap(Map.of(
-													 ResourceConstants.DATASET, support.getDataset().getId()
-											 ));
+		log.info("sending CQPP with {}", invocation);
 
-		final Invocation.Builder request = support.getClient()
-												  .target(addImport)
-												  .request(MediaType.APPLICATION_JSON);
-		try (final Response response = request
-				.put(Entity.entity(Entity.json(""), MediaType.APPLICATION_JSON_TYPE))) {
+		try (final Response response = invocation.invoke()) {
 
 			assertThat(response.getStatusInfo().getFamily())
 					.describedAs(new LazyTextDescription(() -> response.readEntity(String.class)))
 					.isEqualTo(expectedResponseFamily);
-			assertThat(response.getStatusInfo().getReasonPhrase())
-					.describedAs(new LazyTextDescription(() -> response.readEntity(String.class)))
-					.isEqualTo(expectedReason);
 		}
 	}
 
 	public static void importCqppFiles(StandaloneSupport support, List<File> cqppFiles) {
 		for (File cqpp : cqppFiles) {
-			importCqppFile(support, cqpp);
+			uploadCqpp(support, cqpp, false, Response.Status.Family.SUCCESSFUL);
 		}
+
+		support.waitUntilWorkDone();
+
+
 	}
 
 	public static void importTableContents(StandaloneSupport support, Collection<RequiredTable> tables) throws Exception {
 		List<File> cqpps = generateCqpp(support, tables);
+
 		importCqppFiles(support, cqpps);
 	}
 
@@ -275,6 +262,8 @@ public static void updateConcepts(StandaloneSupport support, ArrayNode rawConcep
 		for (Concept<?> concept : concepts) {
 			updateConcept(support, concept, expectedResponseFamily);
 		}
+
+
 	}
 
 	private static void updateConcept(@NonNull StandaloneSupport support, @NonNull Concept<?> concept, @NonNull Response.Status.Family expectedResponseFamily) {
diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ImportUpdateTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ImportUpdateTest.java
index 7e35786d72..3291d3ac4f 100644
--- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ImportUpdateTest.java
+++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ImportUpdateTest.java
@@ -6,8 +6,6 @@
 import java.io.File;
 import java.util.List;
 
-import jakarta.ws.rs.core.Response;
-
 import com.bakdata.conquery.ConqueryConstants;
 import com.bakdata.conquery.apiv1.query.Query;
 import com.bakdata.conquery.commands.ShardNode;
@@ -32,6 +30,7 @@
 import com.bakdata.conquery.util.support.StandaloneSupport;
 import com.bakdata.conquery.util.support.TestConquery;
 import com.github.powerlibraries.io.In;
+import jakarta.ws.rs.core.Response;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.FileUtils;
 
@@ -78,7 +77,9 @@ public void execute(String name, TestConquery testConquery) throws Exception {
 			assertThat(cqpps.size()).isEqualTo(tables.size());
 
 			LoadingUtil.importCqppFiles(conquery, List.of(cqpps.get(0)));
+
 			conquery.waitUntilWorkDone();
+
 		}
 		final Query query = IntegrationUtils.parseQuery(conquery, test.getRawQuery());
 
@@ -125,7 +126,7 @@ public void execute(String name, TestConquery testConquery) throws Exception {
 		}
 
 		//Try to update an import that does not exist should throw a Not-Found Webapplication Exception
-		LoadingUtil.updateCqppFile(conquery, cqpps.get(1), Response.Status.Family.CLIENT_ERROR, "Not Found");
+		LoadingUtil.uploadCqpp(conquery, cqpps.get(1), true, Response.Status.Family.CLIENT_ERROR);
 		conquery.waitUntilWorkDone();
 
 		//Load manually new data for import and update the concerned import
@@ -170,7 +171,10 @@ public void execute(String name, TestConquery testConquery) throws Exception {
 
 			log.info("updating import");
 			//correct update of the import
-			LoadingUtil.updateCqppFile(conquery, newPreprocessedFile, Response.Status.Family.SUCCESSFUL, "No Content");
+			LoadingUtil.uploadCqpp(conquery, newPreprocessedFile, true, Response.Status.Family.SUCCESSFUL);
+			conquery.waitUntilWorkDone();
+
+
 			conquery.waitUntilWorkDone();
 		}
 
diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java
index 5b3e8280fc..d1f0360479 100644
--- a/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java
+++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java
@@ -148,6 +148,8 @@ public void execute(String name, TestConquery testConquery) throws Exception {
 
 		test.executeTest(support);
 
+		storage = support.getMetaStorage();
+
 		{// Auth actual tests
 			User userStored = storage.getUser(user.getId());
 			assertThat(userStored).isEqualTo(user);
diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ConceptUpdateAndDeletionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ConceptUpdateAndDeletionTest.java
index 8795093de6..0b1e1ee887 100644
--- a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ConceptUpdateAndDeletionTest.java
+++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ConceptUpdateAndDeletionTest.java
@@ -114,9 +114,13 @@ public void execute(String name, TestConquery testConquery) throws Exception {
 		// To perform the update, the old concept will be deleted first and the new concept will be added. That means the deletion of concept is also covered here
 		{
 			log.info("Executing  update");
+
 			LoadingUtil.updateConcepts(conquery, test2.getRawConcepts(), Response.Status.Family.SUCCESSFUL);
 			conquery.waitUntilWorkDone();
+
 			log.info("Update executed");
+
+
 		}
 
 
diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java
index e123998d0c..5536f967cf 100644
--- a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java
+++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java
@@ -10,9 +10,6 @@
 import java.util.Map;
 import java.util.zip.GZIPInputStream;
 
-import jakarta.ws.rs.core.MediaType;
-import jakarta.ws.rs.core.Response;
-
 import com.bakdata.conquery.ConqueryConstants;
 import com.bakdata.conquery.apiv1.query.Query;
 import com.bakdata.conquery.commands.ShardNode;
@@ -42,6 +39,8 @@
 import com.bakdata.conquery.util.support.StandaloneSupport;
 import com.bakdata.conquery.util.support.TestConquery;
 import com.github.powerlibraries.io.In;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.FileUtils;
 
@@ -242,6 +241,9 @@ public void execute(String name, TestConquery testConquery) throws Exception {
 			//import preprocessedFiles
 			conquery.getDatasetsProcessor().addImport(conquery.getNamespace(), new GZIPInputStream(new FileInputStream(preprocessedFile)));
 			conquery.waitUntilWorkDone();
+
+
+			conquery.waitUntilWorkDone();
 		}
 
 		// State after reimport.
diff --git a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java
index 2ae22e13bf..99e0d08580 100644
--- a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java
+++ b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java
@@ -220,7 +220,7 @@ public void bucketCompoundDateRange() throws JSONException, IOException {
 		ColumnStore startStore = new IntegerDateStore(new ShortArrayStore(new short[]{1, 2, 3, 4}, Short.MIN_VALUE));
 		ColumnStore endStore = new IntegerDateStore(new ShortArrayStore(new short[]{5, 6, 7, 8}, Short.MIN_VALUE));
 
-		Bucket bucket = new Bucket(0, 4, new ColumnStore[]{startStore, endStore, compoundStore}, Object2IntMaps.emptyMap(), Object2IntMaps.emptyMap(), imp);
+		Bucket bucket = new Bucket(0, new ColumnStore[]{startStore, endStore, compoundStore}, Object2IntMaps.singleton("0", 0), Object2IntMaps.singleton("0", 4),4, imp);
 
 		compoundStore.setParent(bucket);
 
@@ -583,7 +583,7 @@ public void serialize() throws IOException, JSONException {
 		final Import imp = new Import(table);
 		imp.setName("import");
 
-		final Bucket bucket = new Bucket(0, 0, new ColumnStore[0], Object2IntMaps.emptyMap(), Object2IntMaps.emptyMap(), imp);
+		final Bucket bucket = new Bucket(0, new ColumnStore[0], Object2IntMaps.emptyMap(), Object2IntMaps.emptyMap(),0, imp);
 
 
 		final CBlock cBlock = CBlock.createCBlock(connector, bucket, 10);
diff --git a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java
index 3afe94bb22..97cef7a5df 100644
--- a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java
+++ b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java
@@ -8,7 +8,6 @@
 import java.util.Random;
 import java.util.function.Supplier;
 import java.util.stream.Stream;
-import jakarta.validation.Validator;
 
 import com.bakdata.conquery.io.jackson.Injectable;
 import com.bakdata.conquery.io.jackson.Jackson;
@@ -16,6 +15,7 @@
 import com.bakdata.conquery.models.datasets.Dataset;
 import com.bakdata.conquery.models.datasets.Table;
 import com.bakdata.conquery.models.datasets.concepts.Concept;
+import com.bakdata.conquery.models.datasets.concepts.ConceptElement;
 import com.bakdata.conquery.models.events.MajorTypeId;
 import com.bakdata.conquery.models.exceptions.ConfigurationException;
 import com.bakdata.conquery.models.exceptions.JSONException;
@@ -27,6 +27,7 @@
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.github.powerlibraries.io.In;
 import io.dropwizard.jersey.validation.Validators;
+import jakarta.validation.Validator;
 import lombok.extern.slf4j.Slf4j;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.parallel.Execution;
@@ -111,11 +112,11 @@ public static void init() throws IOException, JSONException, ConfigurationExcept
 	public void basic(String key, CalculatedValue<Map<String, Object>> rowMap) throws JSONException {
 		log.trace("Searching for {}", key);
 
-		ConceptTreeChild idxResult = indexedConcept.findMostSpecificChild(key, rowMap);
-		ConceptTreeChild oldResult = oldConcept.findMostSpecificChild(key, rowMap);
+		ConceptElement idxResult = indexedConcept.findMostSpecificChild(key, rowMap);
+		ConceptElement oldResult = oldConcept.findMostSpecificChild(key, rowMap);
 
 		assertThat(oldResult.getId()).describedAs("%s hierarchical name", key).isEqualTo(idxResult.getId());
 	}
 
 
-}
\ No newline at end of file
+}
diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java b/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java
index 49042dada3..8f3410c085 100644
--- a/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java
+++ b/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java
@@ -73,8 +73,10 @@ public void preprocessTmp(File tmpDir, List<File> descriptions) throws Exception
 				Map.of(
 						"in", tmpDir,
 						"out", tmpDir,
-						"desc", descriptions
-
+						"desc", descriptions,
+						"buckets", 10,
+						"strict", true,
+						"fast-fail", true
 				)
 		);