Merge branch 'develop' into master

ingef · Aug 13, 2024 · ed703ea · ed703ea
2 parents f94dbc6 + c7fed2a
commit ed703ea
Show file tree

Hide file tree

Showing 36 changed files with 519 additions and 353 deletions.
diff --git a/backend/pom.xml b/backend/pom.xml
@@ -54,20 +54,6 @@
                             </excludes>
                         </configuration>
                     </execution>
-                    <execution>
-                        <id>default-jar</id>
-                        <goals>
-                            <goal>jar</goal>
-                        </goals>
-                        <configuration>
-                            <archive>
-                                <manifestEntries>
-                                    <Add-Opens>java.base/java.lang ALL-UNNAMED java.base/java.util ALL-UNNAMED java.base/java.nio ALL-UNNAMED java.base/sun.nio.ch ALL-UNNAMED
-</Add-Opens>
-                                </manifestEntries>
-                            </archive>
-                        </configuration>
-                    </execution>
                 </executions>
             </plugin>
             <plugin>

diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java
@@ -1,7 +1,6 @@
 package com.bakdata.conquery.apiv1;
 
 import java.net.URI;
-
 import jakarta.validation.constraints.NotEmpty;
 import jakarta.validation.constraints.NotNull;
 
@@ -15,6 +14,7 @@
 import com.bakdata.conquery.models.identifiable.ids.specific.SearchIndexId;
 import com.bakdata.conquery.models.index.FrontendValueIndex;
 import com.bakdata.conquery.models.index.FrontendValueIndexKey;
+import com.bakdata.conquery.models.index.IndexCreationException;
 import com.bakdata.conquery.models.index.IndexService;
 import com.bakdata.conquery.models.index.search.SearchIndex;
 import com.bakdata.conquery.util.io.FileUtil;
@@ -41,8 +41,6 @@
 @CPSType(id = "CSV_TEMPLATE", base = SearchIndex.class)
 public class FilterTemplate extends IdentifiableImpl<SearchIndexId> implements Searchable, SearchIndex {
 
-	private static final long serialVersionUID = 1L;
-
 	@NotNull
 	@NsIdRef
 	private Dataset dataset;
@@ -89,7 +87,7 @@ public boolean isSearchDisabled() {
 		return false;
 	}
 
-	public TrieSearch<FrontendValue> createTrieSearch(IndexConfig config) {
+	public TrieSearch<FrontendValue> createTrieSearch(IndexConfig config) throws IndexCreationException {
 
 		final URI resolvedURI = FileUtil.getResolvedUri(config.getBaseUrl(), getFilePath());
 		log.trace("Resolved filter template reference url for search '{}': {}", this.getId(), resolvedURI);

diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java
@@ -85,7 +85,7 @@ public void run(Manager manager) throws InterruptedException {
 		this.manager = manager;
 
 		final ObjectMapper objectMapper = environment.getObjectMapper();
-		customizeApiObjectMapper(objectMapper);
+		customizeApiObjectMapper(objectMapper, getDatasetRegistry(), getMetaStorage(), config, validator);
 
 
 		// FormScanner needs to be instantiated before plugins are initialized
@@ -182,7 +182,12 @@ protected void configure() {
 	 *
 	 * @param objectMapper to be configured (should be a JSON mapper)
 	 */
-	public void customizeApiObjectMapper(ObjectMapper objectMapper) {
+	public static void customizeApiObjectMapper(
+			ObjectMapper objectMapper,
+			DatasetRegistry<?> datasetRegistry,
+			MetaStorage metaStorage,
+			ConqueryConfig config,
+			Validator validator) {
 
 		// Set serialization config
 		SerializationConfig serializationConfig = objectMapper.getSerializationConfig();
@@ -200,26 +205,26 @@ public void customizeApiObjectMapper(ObjectMapper objectMapper) {
 
 		final MutableInjectableValues injectableValues = new MutableInjectableValues();
 		objectMapper.setInjectableValues(injectableValues);
-		injectableValues.add(Validator.class, getValidator());
+		injectableValues.add(Validator.class, validator);
 
-		getDatasetRegistry().injectInto(objectMapper);
-		getMetaStorage().injectInto(objectMapper);
-		getConfig().injectInto(objectMapper);
+		datasetRegistry.injectInto(objectMapper);
+		metaStorage.injectInto(objectMapper);
+		config.injectInto(objectMapper);
 	}
 
 	/**
 	 * Create a new internal object mapper for binary (de-)serialization that is equipped with {@link ManagerNode} related injectables.
 	 *
 	 * @return a preconfigured binary object mapper
-	 * @see ManagerNode#customizeApiObjectMapper(ObjectMapper)
+	 * @see ManagerNode#customizeApiObjectMapper(ObjectMapper, DatasetRegistry, MetaStorage, ConqueryConfig, Validator)
 	 */
 	public ObjectMapper createInternalObjectMapper(Class<? extends View> viewClass) {
 		return getInternalObjectMapperCreator().createInternalObjectMapper(viewClass);
 	}
 
 	private void loadMetaStorage() {
 		log.info("Opening MetaStorage");
-		getMetaStorage().openStores(getInternalObjectMapperCreator().createInternalObjectMapper(View.Persistence.Manager.class));
+		getMetaStorage().openStores(createInternalObjectMapper(View.Persistence.Manager.class));
 		log.info("Loading MetaStorage");
 		getMetaStorage().loadData();
 		log.info("MetaStorage loaded {}", getMetaStorage());

diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/Initializing.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/Initializing.java
@@ -0,0 +1,29 @@
+package com.bakdata.conquery.io.jackson;
+
+import com.fasterxml.jackson.databind.util.StdConverter;
+
+/**
+ * Interface for class instances that need initialization after deserialization and value injection.
+ * Let the class implement this interface and annotate the class with:
+ * <pre>
+ *  {@code
+ *  @JsonDeserialize(converter = Initializing.Converter.class )
+ *  }
+ * </pre>
+ * @param <T>
+ * @implNote Every class that inherits from an initializing class needs to define its own Converter.
+ * Otherwise, the class is parsed as the super class, and its overrides are not called.
+ */
+public interface Initializing {
+
+	void init();
+
+	class Converter<T extends Initializing> extends StdConverter<T, T> {
+
+		@Override
+		public T convert(T value) {
+			value.init();
+			return value;
+		}
+	}
+}
diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java
@@ -3,6 +3,8 @@
 import java.util.Collection;
 import java.util.Objects;
 
+import com.bakdata.conquery.io.jackson.Injectable;
+import com.bakdata.conquery.io.jackson.MutableInjectableValues;
 import com.bakdata.conquery.io.storage.xodus.stores.CachedStore;
 import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore;
 import com.bakdata.conquery.models.config.StoreFactory;
@@ -16,10 +18,12 @@
 import com.bakdata.conquery.models.worker.WorkerToBucketsMap;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
+import lombok.ToString;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
-public class NamespaceStorage extends NamespacedStorage {
+@ToString
+public class NamespaceStorage extends NamespacedStorage implements Injectable {
 
 	protected IdentifiableStore<InternToExternMapper> internToExternMappers;
 	protected IdentifiableStore<SearchIndex> searchIndexes;
@@ -40,11 +44,6 @@ private void decorateIdMapping(SingletonStore<EntityIdMap> idMapping) {
 				.onAdd(mapping -> mapping.setStorage(this));
 	}
 
-	private void decorateInternToExternMappingStore(IdentifiableStore<InternToExternMapper> store) {
-		// We don't call internToExternMapper::init this is done by the first select that needs the mapping
-	}
-
-
 	@Override
 	public void openStores(ObjectMapper objectMapper) {
 		super.openStores(objectMapper);
@@ -57,7 +56,6 @@ public void openStores(ObjectMapper objectMapper) {
 		preview = getStorageFactory().createPreviewStore(super.getPathName(), getCentralRegistry(), objectMapper);
 		entity2Bucket = getStorageFactory().createEntity2BucketStore(super.getPathName(), objectMapper);
 
-		decorateInternToExternMappingStore(internToExternMappers);
 		decorateIdMapping(idMapping);
 	}
 
@@ -169,4 +167,10 @@ public PreviewConfig getPreviewConfig() {
 	public void removePreviewConfig() {
 		preview.remove();
 	}
+
+
+	@Override
+	public MutableInjectableValues inject(MutableInjectableValues values) {
+		return super.inject(values).add(NamespaceStorage.class, this);
+	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java
@@ -4,6 +4,8 @@
 import java.util.Collection;
 import java.util.List;
 
+import com.bakdata.conquery.io.jackson.Injectable;
+import com.bakdata.conquery.io.jackson.MutableInjectableValues;
 import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore;
 import com.bakdata.conquery.models.config.StoreFactory;
 import com.bakdata.conquery.models.datasets.Column;
@@ -35,7 +37,7 @@
  */
 @Slf4j
 @ToString(onlyExplicitlyIncluded = true)
-public abstract class NamespacedStorage extends ConqueryStorage {
+public abstract class NamespacedStorage extends ConqueryStorage implements Injectable {
 
 	@Getter
 	protected final CentralRegistry centralRegistry = new CentralRegistry();
@@ -59,6 +61,7 @@ public NamespacedStorage(StoreFactory storageFactory, String pathName) {
 	public void openStores(ObjectMapper objectMapper) {
 		// Before we start to parse the stores we need to replace the injected value for the IdResolveContext (from DatasetRegistry to this centralRegistry)
 		new SingletonNamespaceCollection(centralRegistry).injectInto(objectMapper);
+		this.injectInto(objectMapper);
 
 		dataset = storageFactory.createDatasetStore(pathName, objectMapper);
 		secondaryIds = storageFactory.createSecondaryIdDescriptionStore(centralRegistry, pathName, objectMapper);
@@ -67,9 +70,7 @@ public void openStores(ObjectMapper objectMapper) {
 		concepts = storageFactory.createConceptStore(centralRegistry, pathName, objectMapper);
 
 		decorateDatasetStore(dataset);
-		decorateSecondaryIdDescriptionStore(secondaryIds);
 		decorateTableStore(tables);
-		decorateImportStore(imports);
 		decorateConceptStore(concepts);
 	}
 
@@ -88,10 +89,6 @@ private void decorateDatasetStore(SingletonStore<Dataset> store) {
 		store.onAdd(centralRegistry::register).onRemove(centralRegistry::remove);
 	}
 
-	private void decorateSecondaryIdDescriptionStore(IdentifiableStore<SecondaryIdDescription> store) {
-		// Nothing to decorate
-	}
-
 	private void decorateTableStore(IdentifiableStore<Table> store) {
 		store.onAdd(table -> {
 				 for (Column column : table.getColumns()) {
@@ -109,13 +106,13 @@ private void decorateTableStore(IdentifiableStore<Table> store) {
 	private void decorateConceptStore(IdentifiableStore<Concept<?>> store) {
 		store.onAdd(concept -> {
 
-			if (concept.getDataset() != null && !concept.getDataset().equals(dataset.get())) {
-				throw new IllegalStateException("Concept is not for this dataset.");
+			if (concept.getDataset() == null) {
+				throw new IllegalStateException("Concept had no dataset set");
 			}
 
-			concept.setDataset(dataset.get());
-
-			concept.initElements();
+			if (!concept.getDataset().equals(dataset.get())) {
+				throw new IllegalStateException("Concept is not for this dataset.");
+			}
 
 			concept.getSelects().forEach(centralRegistry::register);
 			for (Connector connector : concept.getConnectors()) {
@@ -145,11 +142,6 @@ private void decorateConceptStore(IdentifiableStore<Concept<?>> store) {
 		});
 	}
 
-	private void decorateImportStore(IdentifiableStore<Import> store) {
-		// Intentionally left blank
-	}
-
-
 	public void addImport(Import imp) {
 		imports.add(imp);
 	}
@@ -231,4 +223,9 @@ public Collection<Concept<?>> getAllConcepts() {
 		return concepts.getAll();
 	}
 
+
+	@Override
+	public MutableInjectableValues inject(MutableInjectableValues values) {
+		return values.add(NamespacedStorage.class, this);
+	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/WorkerStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/WorkerStorage.java
@@ -2,6 +2,8 @@
 
 import java.util.Collection;
 
+import com.bakdata.conquery.io.jackson.Injectable;
+import com.bakdata.conquery.io.jackson.MutableInjectableValues;
 import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore;
 import com.bakdata.conquery.models.config.StoreFactory;
 import com.bakdata.conquery.models.datasets.concepts.Concept;
@@ -13,19 +15,18 @@
 import com.bakdata.conquery.models.worker.WorkerInformation;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
-import jakarta.validation.Validator;
 import lombok.ToString;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @ToString(of = "worker")
-public class WorkerStorage extends NamespacedStorage {
+public class WorkerStorage extends NamespacedStorage implements Injectable {
 
 	private SingletonStore<WorkerInformation> worker;
 	private IdentifiableStore<Bucket> buckets;
 	private IdentifiableStore<CBlock> cBlocks;
 
-	public WorkerStorage(StoreFactory storageFactory, Validator validator, String pathName) {
+	public WorkerStorage(StoreFactory storageFactory, String pathName) {
 		super(storageFactory, pathName);
 	}
 
@@ -36,10 +37,6 @@ public void openStores(ObjectMapper objectMapper) {
 		worker = getStorageFactory().createWorkerInformationStore(getPathName(), objectMapper);
 		buckets = getStorageFactory().createBucketStore(centralRegistry, getPathName(), objectMapper);
 		cBlocks = getStorageFactory().createCBlockStore(centralRegistry, getPathName(), objectMapper);
-
-		decorateWorkerStore(worker);
-		decorateBucketStore(buckets);
-		decorateCBlockStore(cBlocks);
 	}
 
 	@Override
@@ -58,19 +55,6 @@ public ImmutableList<ManagedStore> getStores() {
 	}
 
 
-	private void decorateWorkerStore(SingletonStore<WorkerInformation> store) {
-		// Nothing to decorate
-	}
-
-	private void decorateBucketStore(IdentifiableStore<Bucket> store) {
-		// Nothing to decorate
-	}
-
-	private void decorateCBlockStore(IdentifiableStore<CBlock> baseStoreCreator) {
-		// Nothing to decorate
-	}
-
-
 	public void addCBlock(CBlock cBlock) {
 		log.trace("Adding CBlock[{}]", cBlock.getId());
 		cBlocks.add(cBlock);
@@ -129,4 +113,9 @@ public void removeConcept(ConceptId id) {
 		log.debug("Removing Concept[{}]", id);
 		concepts.remove(id);
 	}
+
+	@Override
+	public MutableInjectableValues inject(MutableInjectableValues values) {
+		return super.inject(values).add(WorkerStorage.class, this);
+	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java
@@ -33,14 +33,17 @@ public interface NamespaceHandler<N extends Namespace> {
 	static NamespaceSetupData createNamespaceSetup(NamespaceStorage storage, final ConqueryConfig config, final InternalObjectMapperCreator mapperCreator, IndexService indexService) {
 		List<Injectable> injectables = new ArrayList<>();
 		injectables.add(indexService);
+		injectables.add(storage);
 
 		ObjectMapper persistenceMapper = mapperCreator.createInternalObjectMapper(View.Persistence.Manager.class);
 		ObjectMapper communicationMapper = mapperCreator.createInternalObjectMapper(View.InternalCommunication.class);
 		ObjectMapper preprocessMapper = mapperCreator.createInternalObjectMapper(null);
 
-		injectables.forEach(i -> i.injectInto(persistenceMapper));
-		injectables.forEach(i -> i.injectInto(communicationMapper));
-		injectables.forEach(i -> i.injectInto(preprocessMapper));
+		injectables.forEach(i -> {
+			i.injectInto(persistenceMapper);
+			i.injectInto(communicationMapper);
+			i.injectInto(preprocessMapper);
+		});
 
 
 		// Each store needs its own mapper because each injects its own registry