Skip to content

Commit

Permalink
Merge pull request #3366 from ingef/release
Browse files Browse the repository at this point in the history
Merge Release
  • Loading branch information
awildturtok authored Apr 8, 2024
2 parents 191c117 + 32d05c4 commit ca4576b
Show file tree
Hide file tree
Showing 363 changed files with 5,508 additions and 7,521 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/create_backend_branch_image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ on:
- master
- release
- staging/**

env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}-backend
Expand Down Expand Up @@ -48,6 +48,7 @@ jobs:
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
with:
context: .
file: backend.Dockerfile
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
1 change: 1 addition & 0 deletions .github/workflows/create_backend_image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ jobs:
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
with:
context: .
file: backend.Dockerfile
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
5 changes: 3 additions & 2 deletions .github/workflows/create_frontend_branch_image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ on:
- master
- release
- staging/*

env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}-frontend
Expand Down Expand Up @@ -47,7 +47,8 @@ jobs:
- name: Build and push Docker image
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
with:
context: ./frontend
context: .
file: frontend.Dockerfile
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
3 changes: 2 additions & 1 deletion .github/workflows/create_frontend_image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,8 @@ jobs:
- name: Build and push Docker image
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
with:
context: ./frontend
context: .
file: frontend.Dockerfile
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
15 changes: 8 additions & 7 deletions .github/workflows/test_build_backend_image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,13 @@ on:
- develop
- master
paths:
- 'backend/**'
- 'executable/**'
- 'Dockerfile'
- 'pom.xml'
- 'lombok.config'
- 'scripts/**'
- '.github/workflows/test_build_backend_image.yml'
- "backend/**"
- "executable/**"
- "Dockerfile"
- "pom.xml"
- "lombok.config"
- "scripts/**"
- ".github/workflows/test_build_backend_image.yml"

env:
REGISTRY: ghcr.io
Expand All @@ -42,4 +42,5 @@ jobs:
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
with:
context: .
file: backend.Dockerfile
push: false
9 changes: 5 additions & 4 deletions .github/workflows/test_build_frontend_image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@ on:
- develop
- master
paths:
- 'frontend/*'
- 'scripts/*'
- '.github/workflows/test_build_frontend_image.yml'
- "frontend/*"
- "scripts/*"
- ".github/workflows/test_build_frontend_image.yml"
env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}-frontend
Expand All @@ -36,5 +36,6 @@ jobs:
- name: Build docker image
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
with:
context: ./frontend
context: .
file: frontend.Dockerfile
push: false
File renamed without changes.
2 changes: 0 additions & 2 deletions backend/src/main/java/com/bakdata/conquery/Conquery.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import javax.validation.Validator;

import ch.qos.logback.classic.Level;
import com.bakdata.conquery.commands.CollectEntitiesCommand;
import com.bakdata.conquery.commands.DistributedStandaloneCommand;
import com.bakdata.conquery.commands.ManagerNode;
import com.bakdata.conquery.commands.MigrateCommand;
Expand Down Expand Up @@ -54,7 +53,6 @@ public void initialize(Bootstrap<ConqueryConfig> bootstrap) {

bootstrap.addCommand(new ShardNode());
bootstrap.addCommand(new PreprocessorCommand());
bootstrap.addCommand(new CollectEntitiesCommand());
bootstrap.addCommand(new DistributedStandaloneCommand(this));
bootstrap.addCommand(new RecodeStoreCommand());
bootstrap.addCommand(new MigrateCommand());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@ public class ConqueryConstants {
OBSERVATION_SCOPE_INFO =
new LocalizedDefaultResultInfo((l) -> C10N.get(ResultHeadersC10n.class, l).observationScope(), new ResultType.StringT(FeatureGroup::localizeValue), Set.of());

public static final String PRIMARY_DICTIONARY = "PRIMARY_DICTIONARY";
/**
* Drawn from random.org
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
import com.bakdata.conquery.apiv1.frontend.FrontendValue;
import com.bakdata.conquery.io.cps.CPSType;
import com.bakdata.conquery.io.jackson.serializer.NsIdRef;
import com.bakdata.conquery.io.storage.NamespaceStorage;
import com.bakdata.conquery.models.config.IndexConfig;
import com.bakdata.conquery.models.datasets.Dataset;
import com.bakdata.conquery.models.datasets.concepts.Searchable;
Expand Down Expand Up @@ -40,7 +39,7 @@
@ToString
@Slf4j
@CPSType(id = "CSV_TEMPLATE", base = SearchIndex.class)
public class FilterTemplate extends IdentifiableImpl<SearchIndexId> implements Searchable<SearchIndexId>, SearchIndex {
public class FilterTemplate extends IdentifiableImpl<SearchIndexId> implements Searchable, SearchIndex {

private static final long serialVersionUID = 1L;

Expand Down Expand Up @@ -90,7 +89,7 @@ public boolean isSearchDisabled() {
return false;
}

public TrieSearch<FrontendValue> createTrieSearch(IndexConfig config, NamespaceStorage storage) {
public TrieSearch<FrontendValue> createTrieSearch(IndexConfig config) {

final URI resolvedURI = FileUtil.getResolvedUri(config.getBaseUrl(), getFilePath());
log.trace("Resolved filter template reference url for search '{}': {}", this.getId(), resolvedURI);
Expand Down
71 changes: 71 additions & 0 deletions backend/src/main/java/com/bakdata/conquery/apiv1/LabelMap.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
package com.bakdata.conquery.apiv1;

import java.util.List;
import java.util.stream.Collectors;

import com.bakdata.conquery.apiv1.frontend.FrontendValue;
import com.bakdata.conquery.models.config.IndexConfig;
import com.bakdata.conquery.models.datasets.concepts.Searchable;
import com.bakdata.conquery.models.identifiable.ids.specific.FilterId;
import com.bakdata.conquery.models.query.FilterSearch;
import com.bakdata.conquery.util.search.TrieSearch;
import com.google.common.collect.BiMap;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.experimental.Delegate;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.time.StopWatch;

@Getter
@RequiredArgsConstructor
@Slf4j
@EqualsAndHashCode
public class LabelMap implements Searchable {

private final FilterId id;
@Delegate
private final BiMap<String, String> delegate;
private final int minSuffixLength;
private final boolean generateSearchSuffixes;

@Override
public TrieSearch<FrontendValue> createTrieSearch(IndexConfig config) {

final TrieSearch<FrontendValue> search = config.createTrieSearch(true);

final List<FrontendValue> collected = delegate.entrySet().stream()
.map(entry -> new FrontendValue(entry.getKey(), entry.getValue()))
.collect(Collectors.toList());

if (log.isTraceEnabled()) {
log.trace("Labels for {}: `{}`", getId(), collected.stream().map(FrontendValue::toString).collect(Collectors.toList()));
}

StopWatch timer = StopWatch.createStarted();
log.trace("START-SELECT ADDING_ITEMS for {}", getId());

collected.forEach(feValue -> search.addItem(feValue, FilterSearch.extractKeywords(feValue)));

log.trace("DONE-SELECT ADDING_ITEMS for {} in {}", getId(), timer);

timer.reset();
log.trace("START-SELECT SHRINKING for {}", getId());

search.shrinkToFit();

log.trace("DONE-SELECT SHRINKING for {} in {}", getId(), timer);

return search;
}

@Override
public boolean isGenerateSuffixes() {
return generateSearchSuffixes;
}

@Override
public boolean isSearchDisabled() {
return false;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -555,15 +555,14 @@ public Stream<Map<String, String>> resolveEntities(Subject subject, List<FilterV
.filter(Predicate.not(Map::isEmpty));
}

public ResultStatistics getResultStatistics(ManagedQuery managedQuery) {
final Query query = managedQuery.getQuery();
final List<ResultInfo> resultInfos = query.getResultInfos();
public ResultStatistics getResultStatistics(SingleTableResult managedQuery) {
final List<ResultInfo> resultInfos = managedQuery.getResultInfos();

final Optional<ResultInfo>
dateInfo =
query.getResultInfos().stream().filter(info -> info.getSemantics().contains(new SemanticType.EventDateT())).findFirst();
resultInfos.stream().filter(info -> info.getSemantics().contains(new SemanticType.EventDateT())).findFirst();

final int dateIndex = dateInfo.map(resultInfos::indexOf).orElse(0 /*Discarded if dateInfo is not present*/);
final Optional<Integer> dateIndex = dateInfo.map(resultInfos::indexOf);

final Locale locale = I18n.LOCALE.get();
final NumberFormat decimalFormat = NumberFormat.getNumberInstance(locale);
Expand All @@ -576,7 +575,6 @@ public ResultStatistics getResultStatistics(ManagedQuery managedQuery) {
new PrintSettings(true, locale, managedQuery.getNamespace(), config, null, null, decimalFormat, integerFormat);
final UniqueNamer uniqueNamer = new UniqueNamer(printSettings);


return ResultStatistics.collectResultStatistics(managedQuery, resultInfos, dateInfo, dateIndex, printSettings, uniqueNamer, config);
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package com.bakdata.conquery.apiv1.auth;

import java.util.Collections;
import java.util.Objects;
import java.util.Set;

import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.NotNull;

import com.bakdata.conquery.io.storage.MetaStorage;
import com.bakdata.conquery.models.auth.entities.Role;
import com.bakdata.conquery.models.auth.permissions.WildcardPermission;
import com.bakdata.conquery.models.identifiable.ids.specific.RoleId;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Builder;
import lombok.Getter;
import lombok.NonNull;

/**
* Factory class to create configured initial roles.
*/
@Getter
@Builder
public class ProtoRole {

private String label;

@NotEmpty
private final String name;

/**
* String permissions in the form of
* {@link org.apache.shiro.authz.permission.WildcardPermission}, that the user
* should hold after initialization.
*/
@Builder.Default
@NotNull
private Set<String> permissions = Collections.emptySet();

public Role createOrOverwriteRole(@NonNull MetaStorage storage) {
label = Objects.requireNonNullElse(label, name);


Role role = new Role(name, label, storage);

storage.updateRole(role);

for (String permission : permissions) {
role.addPermission(new WildcardPermission(permission));
}

return role;
}

@JsonIgnore
public RoleId createId() {
return new RoleId(name);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,26 +10,29 @@
import com.bakdata.conquery.io.storage.MetaStorage;
import com.bakdata.conquery.models.auth.UserManageable;
import com.bakdata.conquery.models.auth.basic.LocalAuthenticationRealm;
import com.bakdata.conquery.models.auth.entities.Role;
import com.bakdata.conquery.models.auth.entities.User;
import com.bakdata.conquery.models.auth.permissions.WildcardPermission;
import com.bakdata.conquery.models.identifiable.ids.specific.RoleId;
import com.bakdata.conquery.models.identifiable.ids.specific.UserId;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Builder;
import lombok.Getter;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;

/**
* Factory class to create configured initial users.
*/
@Getter
@Builder
@Slf4j
public class ProtoUser {

private String label;
@NotEmpty
private final String name;

private final Set<String> roles;
private String label;
/**
* String permissions in the form of
* {@link org.apache.shiro.authz.permission.WildcardPermission}, that the user
Expand All @@ -51,11 +54,29 @@ public User createOrOverwriteUser(@NonNull MetaStorage storage) {
if (label == null) {
label = name;
}
User user = new User(name, label, storage);

final User user = new User(name, label, storage);
storage.updateUser(user);

if (roles != null){
for (String roleId : roles) {
final Role role = storage.getRole(new RoleId(roleId));

if(role == null){
log.warn("Unknown Role[{}] for {}", roleId, this);
continue;
}

user.addRole(role);
}
}


for (String sPermission : permissions) {
user.addPermission(new WildcardPermission(sPermission));
}


return user;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,6 @@ public String getLocalizedTypeLabel() {

@Override
public ManagedInternalForm<FullExportForm> toManagedExecution(User user, Dataset submittedDataset, MetaStorage storage) {
return new ManagedInternalForm<FullExportForm>(this, user, submittedDataset, storage);
return new ManagedInternalForm<>(this, user, submittedDataset, storage);
}
}
Loading

0 comments on commit ca4576b

Please sign in to comment.