Merge Release

.github/workflows/create_backend_branch_image.yml

@@ -12,7 +12,7 @@ on:
       - master
       - release
       - staging/**
-      
+
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}-backend
@@ -48,6 +48,7 @@ jobs:
         uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
         with:
           context: .
+          file: backend.Dockerfile
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

.github/workflows/create_backend_image.yml

@@ -47,6 +47,7 @@ jobs:
         uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
         with:
           context: .
+          file: backend.Dockerfile
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

.github/workflows/create_frontend_branch_image.yml

@@ -12,7 +12,7 @@ on:
       - master
       - release
       - staging/*
-      
+
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}-frontend
@@ -47,7 +47,8 @@ jobs:
       - name: Build and push Docker image
         uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
         with:
-          context: ./frontend
+          context: .
+          file: frontend.Dockerfile
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

.github/workflows/create_frontend_image.yml

@@ -44,7 +44,8 @@ jobs:
       - name: Build and push Docker image
         uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
         with:
-          context: ./frontend
+          context: .
+          file: frontend.Dockerfile
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

.github/workflows/test_build_backend_image.yml

@@ -12,13 +12,13 @@ on:
       - develop
       - master
     paths:
-      - 'backend/**'
-      - 'executable/**'
-      - 'Dockerfile'
-      - 'pom.xml'
-      - 'lombok.config'
-      - 'scripts/**'
-      - '.github/workflows/test_build_backend_image.yml'
+      - "backend/**"
+      - "executable/**"
+      - "Dockerfile"
+      - "pom.xml"
+      - "lombok.config"
+      - "scripts/**"
+      - ".github/workflows/test_build_backend_image.yml"
 
 env:
   REGISTRY: ghcr.io
@@ -42,4 +42,5 @@ jobs:
         uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
         with:
           context: .
+          file: backend.Dockerfile
           push: false

.github/workflows/test_build_frontend_image.yml

@@ -12,9 +12,9 @@ on:
       - develop
       - master
     paths:
-      - 'frontend/*'
-      - 'scripts/*'
-      - '.github/workflows/test_build_frontend_image.yml'
+      - "frontend/*"
+      - "scripts/*"
+      - ".github/workflows/test_build_frontend_image.yml"
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}-frontend
@@ -36,5 +36,6 @@ jobs:
       - name: Build docker image
         uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
         with:
-          context: ./frontend
+          context: .
+          file: frontend.Dockerfile
           push: false

backend/src/main/java/com/bakdata/conquery/Conquery.java

@@ -3,7 +3,6 @@
 import javax.validation.Validator;
 
 import ch.qos.logback.classic.Level;
-import com.bakdata.conquery.commands.CollectEntitiesCommand;
 import com.bakdata.conquery.commands.DistributedStandaloneCommand;
 import com.bakdata.conquery.commands.ManagerNode;
 import com.bakdata.conquery.commands.MigrateCommand;
@@ -54,7 +53,6 @@ public void initialize(Bootstrap<ConqueryConfig> bootstrap) {
 
 		bootstrap.addCommand(new ShardNode());
 		bootstrap.addCommand(new PreprocessorCommand());
-		bootstrap.addCommand(new CollectEntitiesCommand());
 		bootstrap.addCommand(new DistributedStandaloneCommand(this));
 		bootstrap.addCommand(new RecodeStoreCommand());
 		bootstrap.addCommand(new MigrateCommand());

backend/src/main/java/com/bakdata/conquery/ConqueryConstants.java

@@ -45,7 +45,6 @@ public class ConqueryConstants {
 			OBSERVATION_SCOPE_INFO =
 			new LocalizedDefaultResultInfo((l) -> C10N.get(ResultHeadersC10n.class, l).observationScope(), new ResultType.StringT(FeatureGroup::localizeValue), Set.of());
 
-	public static final String PRIMARY_DICTIONARY = "PRIMARY_DICTIONARY";
 	/**
 	 * Drawn from random.org
 	 */

backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java

@@ -8,7 +8,6 @@
 import com.bakdata.conquery.apiv1.frontend.FrontendValue;
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.io.jackson.serializer.NsIdRef;
-import com.bakdata.conquery.io.storage.NamespaceStorage;
 import com.bakdata.conquery.models.config.IndexConfig;
 import com.bakdata.conquery.models.datasets.Dataset;
 import com.bakdata.conquery.models.datasets.concepts.Searchable;
@@ -40,7 +39,7 @@
 @ToString
 @Slf4j
 @CPSType(id = "CSV_TEMPLATE", base = SearchIndex.class)
-public class FilterTemplate extends IdentifiableImpl<SearchIndexId> implements Searchable<SearchIndexId>, SearchIndex {
+public class FilterTemplate extends IdentifiableImpl<SearchIndexId> implements Searchable, SearchIndex {
 
 	private static final long serialVersionUID = 1L;
 
@@ -90,7 +89,7 @@ public boolean isSearchDisabled() {
 		return false;
 	}
 
-	public TrieSearch<FrontendValue> createTrieSearch(IndexConfig config, NamespaceStorage storage) {
+	public TrieSearch<FrontendValue> createTrieSearch(IndexConfig config) {
 
 		final URI resolvedURI = FileUtil.getResolvedUri(config.getBaseUrl(), getFilePath());
 		log.trace("Resolved filter template reference url for search '{}': {}", this.getId(), resolvedURI);

backend/src/main/java/com/bakdata/conquery/apiv1/LabelMap.java

@@ -0,0 +1,71 @@
+package com.bakdata.conquery.apiv1;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+import com.bakdata.conquery.apiv1.frontend.FrontendValue;
+import com.bakdata.conquery.models.config.IndexConfig;
+import com.bakdata.conquery.models.datasets.concepts.Searchable;
+import com.bakdata.conquery.models.identifiable.ids.specific.FilterId;
+import com.bakdata.conquery.models.query.FilterSearch;
+import com.bakdata.conquery.util.search.TrieSearch;
+import com.google.common.collect.BiMap;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.Delegate;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.time.StopWatch;
+
+@Getter
+@RequiredArgsConstructor
+@Slf4j
+@EqualsAndHashCode
+public class LabelMap implements Searchable {
+
+	private final FilterId id;
+	@Delegate
+	private final BiMap<String, String> delegate;
+	private final int minSuffixLength;
+	private final boolean generateSearchSuffixes;
+
+	@Override
+	public TrieSearch<FrontendValue> createTrieSearch(IndexConfig config) {
+
+		final TrieSearch<FrontendValue> search = config.createTrieSearch(true);
+
+		final List<FrontendValue> collected = delegate.entrySet().stream()
+													  .map(entry -> new FrontendValue(entry.getKey(), entry.getValue()))
+													  .collect(Collectors.toList());
+
+		if (log.isTraceEnabled()) {
+			log.trace("Labels for {}: `{}`", getId(), collected.stream().map(FrontendValue::toString).collect(Collectors.toList()));
+		}
+
+		StopWatch timer = StopWatch.createStarted();
+		log.trace("START-SELECT ADDING_ITEMS for {}", getId());
+
+		collected.forEach(feValue -> search.addItem(feValue, FilterSearch.extractKeywords(feValue)));
+
+		log.trace("DONE-SELECT ADDING_ITEMS for {} in {}", getId(), timer);
+
+		timer.reset();
+		log.trace("START-SELECT SHRINKING for {}", getId());
+
+		search.shrinkToFit();
+
+		log.trace("DONE-SELECT SHRINKING for {} in {}", getId(), timer);
+
+		return search;
+	}
+
+	@Override
+	public boolean isGenerateSuffixes() {
+		return generateSearchSuffixes;
+	}
+
+	@Override
+	public boolean isSearchDisabled() {
+		return false;
+	}
+}

backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java

@@ -555,15 +555,14 @@ public Stream<Map<String, String>> resolveEntities(Subject subject, List<FilterV
 					 .filter(Predicate.not(Map::isEmpty));
 	}
 
-	public ResultStatistics getResultStatistics(ManagedQuery managedQuery) {
-		final Query query = managedQuery.getQuery();
-		final List<ResultInfo> resultInfos = query.getResultInfos();
+	public ResultStatistics getResultStatistics(SingleTableResult managedQuery) {
+		final List<ResultInfo> resultInfos = managedQuery.getResultInfos();
 
 		final Optional<ResultInfo>
 				dateInfo =
-				query.getResultInfos().stream().filter(info -> info.getSemantics().contains(new SemanticType.EventDateT())).findFirst();
+				resultInfos.stream().filter(info -> info.getSemantics().contains(new SemanticType.EventDateT())).findFirst();
 
-		final int dateIndex = dateInfo.map(resultInfos::indexOf).orElse(0 /*Discarded if dateInfo is not present*/);
+		final Optional<Integer> dateIndex = dateInfo.map(resultInfos::indexOf);
 
 		final Locale locale = I18n.LOCALE.get();
 		final NumberFormat decimalFormat = NumberFormat.getNumberInstance(locale);
@@ -576,7 +575,6 @@ public ResultStatistics getResultStatistics(ManagedQuery managedQuery) {
 				new PrintSettings(true, locale, managedQuery.getNamespace(), config, null, null, decimalFormat, integerFormat);
 		final UniqueNamer uniqueNamer = new UniqueNamer(printSettings);
 
-
 		return ResultStatistics.collectResultStatistics(managedQuery, resultInfos, dateInfo, dateIndex, printSettings, uniqueNamer, config);
 	}
 

backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoRole.java

@@ -0,0 +1,60 @@
+package com.bakdata.conquery.apiv1.auth;
+
+import java.util.Collections;
+import java.util.Objects;
+import java.util.Set;
+
+import javax.validation.constraints.NotEmpty;
+import javax.validation.constraints.NotNull;
+
+import com.bakdata.conquery.io.storage.MetaStorage;
+import com.bakdata.conquery.models.auth.entities.Role;
+import com.bakdata.conquery.models.auth.permissions.WildcardPermission;
+import com.bakdata.conquery.models.identifiable.ids.specific.RoleId;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NonNull;
+
+/**
+ * Factory class to create configured initial roles.
+
+ */
+@Getter
+@Builder
+public class ProtoRole {
+
+	private String label;
+
+	@NotEmpty
+	private final String name;
+
+	/**
+	 * String permissions in the form of
+	 * {@link org.apache.shiro.authz.permission.WildcardPermission}, that the user
+	 * should hold after initialization.
+	 */
+	@Builder.Default
+	@NotNull
+	private Set<String> permissions = Collections.emptySet();
+
+	public Role createOrOverwriteRole(@NonNull MetaStorage storage) {
+		label = Objects.requireNonNullElse(label, name);
+
+
+		Role role = new Role(name, label, storage);
+
+		storage.updateRole(role);
+
+		for (String permission : permissions) {
+			role.addPermission(new WildcardPermission(permission));
+		}
+
+		return role;
+	}
+
+	@JsonIgnore
+	public RoleId createId() {
+		return new RoleId(name);
+	}
+}

backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java

@@ -10,26 +10,29 @@
 import com.bakdata.conquery.io.storage.MetaStorage;
 import com.bakdata.conquery.models.auth.UserManageable;
 import com.bakdata.conquery.models.auth.basic.LocalAuthenticationRealm;
+import com.bakdata.conquery.models.auth.entities.Role;
 import com.bakdata.conquery.models.auth.entities.User;
 import com.bakdata.conquery.models.auth.permissions.WildcardPermission;
+import com.bakdata.conquery.models.identifiable.ids.specific.RoleId;
 import com.bakdata.conquery.models.identifiable.ids.specific.UserId;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import lombok.Builder;
 import lombok.Getter;
 import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
 
 /**
  * Factory class to create configured initial users.
-
  */
 @Getter
 @Builder
+@Slf4j
 public class ProtoUser {
 
-	private String label;
 	@NotEmpty
 	private final String name;
-
+	private final Set<String> roles;
+	private String label;
 	/**
 	 * String permissions in the form of
 	 * {@link org.apache.shiro.authz.permission.WildcardPermission}, that the user
@@ -51,11 +54,29 @@ public User createOrOverwriteUser(@NonNull MetaStorage storage) {
 		if (label == null) {
 			label = name;
 		}
-		User user = new User(name, label, storage);
+
+		final User user = new User(name, label, storage);
 		storage.updateUser(user);
+
+		if (roles != null){
+			for (String roleId : roles) {
+				final Role role = storage.getRole(new RoleId(roleId));
+
+				if(role == null){
+					log.warn("Unknown Role[{}] for {}", roleId, this);
+					continue;
+				}
+
+				user.addRole(role);
+			}
+		}
+
+
 		for (String sPermission : permissions) {
 			user.addPermission(new WildcardPermission(sPermission));
 		}
+
+
 		return user;
 	}
 

backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/FullExportForm.java

@@ -127,6 +127,6 @@ public String getLocalizedTypeLabel() {
 
 	@Override
 	public ManagedInternalForm<FullExportForm> toManagedExecution(User user, Dataset submittedDataset, MetaStorage storage) {
-		return new ManagedInternalForm<FullExportForm>(this, user, submittedDataset, storage);
+		return new ManagedInternalForm<>(this, user, submittedDataset, storage);
 	}
 }