Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Handle system index set scoping #21174

Open
wants to merge 9 commits into
base: master
Choose a base branch
from
Open

Handle system index set scoping #21174

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
2f8f3a5
handle system index set scoping
patrickmann Dec 12, 2024
52110c0
CL
patrickmann Dec 12, 2024
70e7ff0
unit tests
patrickmann Dec 12, 2024
2f5f8a6
check isRegular to determine scope
patrickmann Dec 12, 2024
e08d499
remove useless test
patrickmann Dec 12, 2024
64fba31
force rebuild
patrickmann Dec 12, 2024
2b6985c
force rebuild
patrickmann Dec 12, 2024
43c88e9
force rebuild
patrickmann Dec 13, 2024
5541d48
Merge branch 'master' into indexsetScoping
patrickmann Dec 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions changelog/unreleased/pr-21174.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
type = "c"
message = "Handle scoping for index sets: System index sets cannot be deleted. "

pulls = ["21174"]
issues = ["Graylog2/graylog-plugin-enterprise#5904"]
2 changes: 2 additions & 0 deletions graylog2-server/src/main/java/org/graylog2/database/entities/ScopedEntitiesModule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,15 @@
*/
package org.graylog2.database.entities;

import org.graylog2.indexer.indexset.SystemIndexSetScope;
import org.graylog2.plugin.PluginModule;

public class ScopedEntitiesModule extends PluginModule {
@Override
protected void configure() {

addEntityScope(DefaultEntityScope.class);
addEntityScope(SystemIndexSetScope.class);

addSystemRestResource(EntityScopeResource.class);
}
Expand Down
6 changes: 5 additions & 1 deletion graylog2-server/src/main/java/org/graylog2/indexer/indexset/IndexSetConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,10 @@ public static IndexSetConfig create(@Id @ObjectId @JsonProperty("_id") @Nullable
fieldTypeRefreshIntervalValue = writableValue ? DEFAULT_FIELD_TYPE_REFRESH_INTERVAL : Duration.ZERO;
}

if (scope == null) {
scope = Boolean.FALSE.equals(isRegular) ? SystemIndexSetScope.NAME : DefaultEntityScope.NAME;
}

return AutoValue_IndexSetConfig.builder()
.id(id)
.title(title)
Expand All @@ -131,7 +135,7 @@ public static IndexSetConfig create(@Id @ObjectId @JsonProperty("_id") @Nullable
.customFieldMappings(customFieldMappings == null ? new CustomFieldMappings() : customFieldMappings)
.fieldTypeProfile(fieldTypeProfile)
.dataTieringConfig(dataTiering)
.scope(scope == null ? DefaultEntityScope.NAME : scope)
.scope(scope)
.build();
}

Expand Down
17 changes: 15 additions & 2 deletions graylog2-server/src/main/java/org/graylog2/indexer/indexset/MongoIndexSetService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,9 @@
import jakarta.inject.Inject;
import org.bson.types.ObjectId;
import org.graylog2.database.MongoCollections;
import org.graylog2.database.entities.EntityScopeService;
import org.graylog2.database.utils.MongoUtils;
import org.graylog2.database.utils.ScopedEntityMongoUtils;
import org.graylog2.events.ClusterEventBus;
import org.graylog2.indexer.indexset.events.IndexSetCreatedEvent;
import org.graylog2.indexer.indexset.events.IndexSetDeletedEvent;
Expand All @@ -56,6 +58,7 @@ public class MongoIndexSetService implements IndexSetService {
public static final String FIELD_TITLE = "title";

private final MongoCollection<IndexSetConfig> collection;
private final ScopedEntityMongoUtils<IndexSetConfig> scopedEntityMongoUtils;
private final MongoUtils<IndexSetConfig> mongoUtils;
private final ClusterConfigService clusterConfigService;
private final ClusterEventBus clusterEventBus;
Expand All @@ -65,9 +68,12 @@ public class MongoIndexSetService implements IndexSetService {
public MongoIndexSetService(MongoCollections mongoCollections,
StreamService streamService,
ClusterConfigService clusterConfigService,
ClusterEventBus clusterEventBus) {
ClusterEventBus clusterEventBus,
EntityScopeService entityScopeService) {
this.collection = mongoCollections.collection(COLLECTION_NAME, IndexSetConfig.class);
this.mongoUtils = mongoCollections.utils(this.collection);
this.scopedEntityMongoUtils = mongoCollections.scopedEntityUtils(collection, entityScopeService);

this.streamService = streamService;
this.clusterConfigService = clusterConfigService;
this.clusterEventBus = requireNonNull(clusterEventBus);
Expand Down Expand Up @@ -156,8 +162,10 @@ public List<IndexSetConfig> searchByTitle(String searchString) {
*/
@Override
public IndexSetConfig save(IndexSetConfig indexSetConfig) {
scopedEntityMongoUtils.ensureValidScope(indexSetConfig);
String id = indexSetConfig.id();
if (id != null) {
scopedEntityMongoUtils.ensureMutability(indexSetConfig);
collection.replaceOne(idEq(id), indexSetConfig, new ReplaceOptions().upsert(true));
} else {
final InsertOneResult insertOneResult = collection.insertOne(indexSetConfig);
Expand Down Expand Up @@ -193,7 +201,12 @@ public int delete(ObjectId id) {
if (!isDeletable(id)) {
return 0;
}
int removedEntries = mongoUtils.deleteById(id) ? 1 : 0;
int removedEntries;
try {
removedEntries = scopedEntityMongoUtils.deleteById(id) ? 1 : 0;
} catch (IllegalArgumentException e) {
return 0;
}
if (removedEntries > 0) {
final IndexSetDeletedEvent deletedEvent = IndexSetDeletedEvent.create(id.toHexString());
clusterEventBus.post(deletedEvent);
Expand Down
1 change: 1 addition & 0 deletions graylog2-server/src/main/java/org/graylog2/migrations/MigrationsModule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,5 +71,6 @@ protected void configure() {
addMigration(V20240312140000_RemoveFieldTypeMappingsManagerRole.class);
addMigration(V202404170856_UpdateIndexSetTemplates.class);
addMigration(V20240927120300_DataNodeMigrationIndexSet.class);
addMigration(V20241212102900_IndexSetScopeMigration.class);
}
}
55 changes: 55 additions & 0 deletions ...-server/src/main/java/org/graylog2/migrations/V20241212102900_IndexSetScopeMigration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/*
* Copyright (C) 2020 Graylog, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
package org.graylog2.migrations;

import jakarta.inject.Inject;
import org.graylog2.indexer.indexset.IndexSetService;
import org.graylog2.indexer.indexset.SystemIndexSetScope;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.time.ZonedDateTime;

/**
* Make system index sets (regular==false) non-deletable by assigning SystemIndexSetScope
*/
public class V20241212102900_IndexSetScopeMigration extends Migration {
private static final Logger LOG = LoggerFactory.getLogger(V20241212102900_IndexSetScopeMigration.class);

private final IndexSetService indexSetService;

@Inject
public V20241212102900_IndexSetScopeMigration(final IndexSetService indexSetService) {
this.indexSetService = indexSetService;
}

@Override
public ZonedDateTime createdAt() {
return ZonedDateTime.parse("2024-12-12T10:29:00Z");
}

@Override
public void upgrade() {
indexSetService.findAll().forEach(indexSetConfig -> {
if (!indexSetConfig.isRegularIndex() && !indexSetConfig.scope().equalsIgnoreCase(SystemIndexSetScope.NAME)) {
indexSetService.save(indexSetConfig.toBuilder().scope(SystemIndexSetScope.NAME).build());
LOG.info("Successfully updated scope for index set: {}", indexSetConfig.title());
}
});
}

}
80 changes: 61 additions & 19 deletions graylog2-server/src/test/java/org/graylog2/indexer/indexset/MongoIndexSetServiceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
import org.graylog2.buffers.processors.fakestreams.FakeStream;
import org.graylog2.cluster.ClusterConfigServiceImpl;
import org.graylog2.database.MongoCollections;
import org.graylog2.database.entities.DefaultEntityScope;
import org.graylog2.database.entities.EntityScopeService;
import org.graylog2.events.ClusterEventBus;
import org.graylog2.indexer.indexset.events.IndexSetCreatedEvent;
import org.graylog2.indexer.indexset.events.IndexSetDeletedEvent;
Expand Down Expand Up @@ -53,10 +55,12 @@
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;

import static org.assertj.core.api.Assertions.assertThat;
import static org.graylog2.indexer.EventIndexTemplateProvider.EVENT_TEMPLATE_TYPE;
import static org.graylog2.indexer.indexset.SimpleIndexSetConfig.DEFAULT_FIELD_TYPE_REFRESH_INTERVAL;
import static org.mockito.Mockito.when;

public class MongoIndexSetServiceTest {
Expand All @@ -78,19 +82,20 @@ public class MongoIndexSetServiceTest {
private final NodeId nodeId = new SimpleNodeId("5ca1ab1e-0000-4000-a000-000000000000");

@Before
public void setUp() throws Exception {
public void setUp() {
clusterEventBus = new ClusterEventBus();
clusterConfigService = new ClusterConfigServiceImpl(objectMapperProvider, mongodb.mongoConnection(),
nodeId, new RestrictedChainingClassLoader(
new ChainingClassLoader(getClass().getClassLoader()), SafeClasses.allGraylogInternal()),
clusterEventBus);
MongoCollections mongoCollections = new MongoCollections(objectMapperProvider, mongodb.mongoConnection());
indexSetService = new MongoIndexSetService(mongoCollections, streamService, clusterConfigService, clusterEventBus);
final EntityScopeService entityScopeService = new EntityScopeService(Set.of(new DefaultEntityScope(), new SystemIndexSetScope()));
indexSetService = new MongoIndexSetService(mongoCollections, streamService, clusterConfigService, clusterEventBus, entityScopeService);
}

@Test
@MongoDBFixtures("MongoIndexSetServiceTest.json")
public void getWithStringId() throws Exception {
public void getWithStringId() {
final Optional<IndexSetConfig> indexSetConfig = indexSetService.get("57f3d721a43c2d59cb750001");
assertThat(indexSetConfig)
.isPresent()
Expand Down Expand Up @@ -119,7 +124,7 @@ public void getWithStringId() throws Exception {

@Test
@MongoDBFixtures("MongoIndexSetServiceTest.json")
public void getReturnsExistingIndexSetConfig() throws Exception {
public void getReturnsExistingIndexSetConfig() {
final Optional<IndexSetConfig> indexSetConfig = indexSetService.get(new ObjectId("57f3d721a43c2d59cb750001"));
assertThat(indexSetConfig)
.isPresent()
Expand Down Expand Up @@ -147,14 +152,14 @@ public void getReturnsExistingIndexSetConfig() throws Exception {
}

@Test
public void getReturnsAbsentOptionalIfIndexSetConfigDoesNotExist() throws Exception {
public void getReturnsAbsentOptionalIfIndexSetConfigDoesNotExist() {
final Optional<IndexSetConfig> indexSetConfig = indexSetService.get(new ObjectId("57f3d3f0a43c2d595eb0a348"));
assertThat(indexSetConfig).isEmpty();
}

@Test
@MongoDBFixtures("MongoIndexSetServiceTest.json")
public void getDefault() throws Exception {
public void getDefault() {
clusterConfigService.write(DefaultIndexSetConfig.create("57f3d721a43c2d59cb750002"));

final IndexSetConfig indexSetConfig = indexSetService.getDefault();
Expand All @@ -165,13 +170,13 @@ public void getDefault() throws Exception {

@Test(expected = IllegalStateException.class)
@MongoDBFixtures("MongoIndexSetServiceTest.json")
public void getDefaultWithoutDefault() throws Exception {
public void getDefaultWithoutDefault() {
indexSetService.getDefault();
}

@Test
@MongoDBFixtures("MongoIndexSetServiceTest.json")
public void findOne() throws Exception {
public void findOne() {
final Optional<IndexSetConfig> config3 = indexSetService.findOne(DBQuery.is("title", "Test 2"));
assertThat(config3).isPresent();
assertThat(config3.get().id()).isEqualTo("57f3d721a43c2d59cb750002");
Expand All @@ -182,12 +187,12 @@ public void findOne() throws Exception {

@Test
@MongoDBFixtures("MongoIndexSetServiceTest.json")
public void findAll() throws Exception {
public void findAll() {
final List<IndexSetConfig> configs = indexSetService.findAll();

assertThat(configs)
.isNotEmpty()
.hasSize(3)
.hasSize(4)
.containsExactly(
IndexSetConfig.create(
"57f3d721a43c2d59cb750001",
Expand Down Expand Up @@ -245,12 +250,35 @@ public void findAll() throws Exception {
EVENT_TEMPLATE_TYPE,
1,
false
),
IndexSetConfig.create(
"57f3d721a43c2d59cb750004",
SystemIndexSetScope.NAME,
"Test 4",
"Index with system scope - not deletable",
true, null,
"test_4",
null, null,
1,
0,
MessageCountRotationStrategy.class.getCanonicalName(),
MessageCountRotationStrategyConfig.create(2500),
NoopRetentionStrategy.class.getCanonicalName(),
NoopRetentionStrategyConfig.create(25),
ZonedDateTime.of(2016, 10, 4, 18, 0, 0, 0, ZoneOffset.UTC),
"standard",
"test_4",
EVENT_TEMPLATE_TYPE,
1,
false,
DEFAULT_FIELD_TYPE_REFRESH_INTERVAL,
null, null, null
)
);
}

@Test
public void save() throws Exception {
public void save() {
final IndexSetCreatedSubscriber subscriber = new IndexSetCreatedSubscriber();
clusterEventBus.registerClusterEventSubscriber(subscriber);
final IndexSetConfig indexSetConfig = IndexSetConfig.create(
Expand Down Expand Up @@ -285,7 +313,7 @@ public void save() throws Exception {

@Test
@MongoDBFixtures("MongoIndexSetServiceTest.json")
public void deleteWithStringId() throws Exception {
public void deleteWithStringId() {
final IndexSetDeletedSubscriber subscriber = new IndexSetDeletedSubscriber();
clusterEventBus.registerClusterEventSubscriber(subscriber);

Expand All @@ -300,7 +328,7 @@ public void deleteWithStringId() throws Exception {

@Test
@MongoDBFixtures("MongoIndexSetServiceTest.json")
public void deleteRemovesExistingIndexSetConfig() throws Exception {
public void deleteRemovesExistingIndexSetConfig() {
final IndexSetDeletedSubscriber subscriber = new IndexSetDeletedSubscriber();
clusterEventBus.registerClusterEventSubscriber(subscriber);

Expand All @@ -315,22 +343,36 @@ public void deleteRemovesExistingIndexSetConfig() throws Exception {

@Test
@MongoDBFixtures("MongoIndexSetServiceTest.json")
public void deleteDoesNothingIfIndexSetConfigDoesNotExist() throws Exception {
public void deleteDoesNothingIfIndexSetConfigDoesNotExist() {
final IndexSetDeletedSubscriber subscriber = new IndexSetDeletedSubscriber();
clusterEventBus.registerClusterEventSubscriber(subscriber);

final int deletedEntries = indexSetService.delete("57f3d721a43c2d59cb750009");
assertThat(deletedEntries).isEqualTo(0);
assertThat(deletedEntries).isZero();
assertThat(indexSetService.get("57f3d721a43c2d59cb750001")).isPresent();
assertThat(indexSetService.get("57f3d721a43c2d59cb750009")).isEmpty();
assertThat(indexSetService.findAll()).hasSize(3);
assertThat(indexSetService.findAll()).hasSize(4);

assertThat(subscriber.getEvents()).isEmpty();
}


@Test
@MongoDBFixtures("MongoIndexSetServiceTest.json")
public void deleteThrowsIfInvalidScope() {
final IndexSetDeletedSubscriber subscriber = new IndexSetDeletedSubscriber();
clusterEventBus.registerClusterEventSubscriber(subscriber);

int deletedEntries = indexSetService.delete("57f3d721a43c2d59cb750004");

assertThat(deletedEntries).isZero();
assertThat(indexSetService.findAll()).hasSize(4);
assertThat(subscriber.getEvents()).isEmpty();
}

@Test
@MongoDBFixtures("MongoIndexSetServiceTest.json")
public void deleteWithAssignedStreams() throws Exception {
public void deleteWithAssignedStreams() {
final IndexSetDeletedSubscriber subscriber = new IndexSetDeletedSubscriber();
clusterEventBus.registerClusterEventSubscriber(subscriber);

Expand All @@ -342,9 +384,9 @@ public void deleteWithAssignedStreams() throws Exception {
when(streamService.loadAllWithIndexSet(streamId)).thenReturn(Collections.singletonList(stream1));

final int deletedEntries = indexSetService.delete(streamId);
assertThat(deletedEntries).isEqualTo(0);
assertThat(deletedEntries).isZero();
assertThat(indexSetService.get(streamId)).isPresent();
assertThat(indexSetService.findAll()).hasSize(3);
assertThat(indexSetService.findAll()).hasSize(4);

assertThat(subscriber.getEvents()).isEmpty();
}
Expand Down
Loading
Loading