From a4c16342e06ba14d9635ffb2f48345f77ed9d1c8 Mon Sep 17 00:00:00 2001 From: Tomas Dvorak Date: Thu, 12 Dec 2024 10:32:11 +0100 Subject: [PATCH 1/7] Datanode/opensearch configuration beans (#21082) * Preflight check for usable space and cache size * add warning when search cache size occupies more than 80% of free disk space * Added changelog * unified searchable snapshots configuration for datanode * removed node cache from OpensearchConfiguration (is managed by searchable snaphots config) * code cleanup * better node search cache documentation * updated changelog * Opensearch configuration beans refactoring * code cleanup, renaming * further configuration simplifications * fixed configuration, seed hosts file moved under configuration * fixed ssl configuration checks * Added support for opensearch configuration files * opensearch config dir handling overhaul, files cleanup, temp dirs * Safer input stream creation for configuration files * Fix opensearch configuration files copy from jar * renamed and standardized configuration beans * license * Code cleanup * move configuration beans out of opensearch package --------- Co-authored-by: Matthias Oesterheld <33032967+moesterheld@users.noreply.github.com> --- .../bindings/OpensearchProcessBindings.java | 28 +- .../bindings/PreflightChecksBindings.java | 2 - .../bootstrap/preflight/FullDirSync.java | 116 --------- .../preflight/OpensearchConfigSync.java | 101 -------- .../configuration/ConfigDirRemovalThread.java | 47 ++++ .../configuration/DatanodeDirectories.java | 59 ++--- .../configuration/DatanodeKeystore.java | 13 + .../DatanodeTrustManagerProvider.java | 17 +- .../OpensearchConfigurationDir.java | 35 +++ .../OpensearchConfigurationException.java | 4 + .../OpensearchConfigurationService.java | 136 ++-------- .../OpensearchKeystoreProvider.java | 58 ++--- .../configuration/TruststoreCreator.java | 36 ++- ...eystoreOpensearchCertificatesProvider.java | 52 ++++ ...ocalConfigurationCertificatesProvider.java | 136 ++++++++++ .../LocalKeystoreSecureConfiguration.java | 87 ------- ... => NoOpensearchCertificatesProvider.java} | 11 +- .../variants/OpensearchCertificates.java | 45 ++++ ...va => OpensearchCertificatesProvider.java} | 8 +- .../OpensearchSecurityConfiguration.java | 222 ---------------- .../variants/SecureConfiguration.java | 72 ------ .../UploadedCertFilesSecureConfiguration.java | 162 ------------ .../datanode/initializers/JerseyService.java | 56 ++-- .../opensearch/OpensearchProcessImpl.java | 81 ++---- .../opensearch/OpensearchProcessService.java | 2 +- .../opensearch/cli/OpensearchCli.java | 4 +- .../cli/OpensearchCommandLineProcess.java | 41 ++- .../OpensearchConfiguration.java | 195 +++++++------- .../OpensearchConfigurationParams.java | 27 ++ .../beans/OpensearchConfigurationPart.java | 59 ----- .../OpensearchClusterConfigurationBean.java | 97 +++++++ .../OpensearchCommonConfigurationBean.java | 78 ++++++ .../OpensearchDefaultConfigFilesBean.java | 123 +++++++++ .../OpensearchSecurityConfigurationBean.java | 239 ++++++++++++++++++ .../SearchableSnapshotsConfigurationBean.java | 13 +- .../statemachine/OpensearchStateMachine.java | 2 +- .../beans/ConfigurationBuildParams.java} | 5 +- .../beans/DatanodeConfigurationBean.java | 22 ++ .../beans/DatanodeConfigurationPart.java | 132 ++++++++++ .../files/DatanodeConfigFile.java | 34 +++ .../files/InputStreamConfigFile.java | 29 +++ .../files/KeystoreConfigFile.java | 36 +++ .../OpensearchSecurityConfigurationFile.java | 63 +++++ .../configuration/files/TextConfigFile.java | 33 +++ .../configuration/files/YamlConfigFile.java | 34 +++ .../opensearch.yml.example | 228 ----------------- .../bootstrap/preflight/FullDirSyncTest.java | 122 --------- .../DatanodeDirectoriesTest.java | 7 +- .../opensearch/OpensearchProcessImplTest.java | 6 +- ...ava => DatanodeConfigurationPartTest.java} | 12 +- ...rchableSnapshotsConfigurationBeanTest.java | 16 +- 51 files changed, 1600 insertions(+), 1643 deletions(-) delete mode 100644 data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/FullDirSync.java delete mode 100644 data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/OpensearchConfigSync.java create mode 100644 data-node/src/main/java/org/graylog/datanode/configuration/ConfigDirRemovalThread.java create mode 100644 data-node/src/main/java/org/graylog/datanode/configuration/OpensearchConfigurationDir.java create mode 100644 data-node/src/main/java/org/graylog/datanode/configuration/variants/DatanodeKeystoreOpensearchCertificatesProvider.java create mode 100644 data-node/src/main/java/org/graylog/datanode/configuration/variants/LocalConfigurationCertificatesProvider.java delete mode 100644 data-node/src/main/java/org/graylog/datanode/configuration/variants/LocalKeystoreSecureConfiguration.java rename data-node/src/main/java/org/graylog/datanode/configuration/variants/{InSecureConfiguration.java => NoOpensearchCertificatesProvider.java} (65%) create mode 100644 data-node/src/main/java/org/graylog/datanode/configuration/variants/OpensearchCertificates.java rename data-node/src/main/java/org/graylog/datanode/configuration/variants/{SecurityConfigurationVariant.java => OpensearchCertificatesProvider.java} (74%) delete mode 100644 data-node/src/main/java/org/graylog/datanode/configuration/variants/OpensearchSecurityConfiguration.java delete mode 100644 data-node/src/main/java/org/graylog/datanode/configuration/variants/SecureConfiguration.java delete mode 100644 data-node/src/main/java/org/graylog/datanode/configuration/variants/UploadedCertFilesSecureConfiguration.java create mode 100644 data-node/src/main/java/org/graylog/datanode/opensearch/configuration/OpensearchConfigurationParams.java delete mode 100644 data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/OpensearchConfigurationPart.java create mode 100644 data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/OpensearchClusterConfigurationBean.java create mode 100644 data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/OpensearchCommonConfigurationBean.java create mode 100644 data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/OpensearchDefaultConfigFilesBean.java create mode 100644 data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/OpensearchSecurityConfigurationBean.java rename data-node/src/main/java/org/graylog/datanode/{opensearch/configuration/beans/OpensearchConfigurationBean.java => process/configuration/beans/ConfigurationBuildParams.java} (79%) create mode 100644 data-node/src/main/java/org/graylog/datanode/process/configuration/beans/DatanodeConfigurationBean.java create mode 100644 data-node/src/main/java/org/graylog/datanode/process/configuration/beans/DatanodeConfigurationPart.java create mode 100644 data-node/src/main/java/org/graylog/datanode/process/configuration/files/DatanodeConfigFile.java create mode 100644 data-node/src/main/java/org/graylog/datanode/process/configuration/files/InputStreamConfigFile.java create mode 100644 data-node/src/main/java/org/graylog/datanode/process/configuration/files/KeystoreConfigFile.java create mode 100644 data-node/src/main/java/org/graylog/datanode/process/configuration/files/OpensearchSecurityConfigurationFile.java create mode 100644 data-node/src/main/java/org/graylog/datanode/process/configuration/files/TextConfigFile.java create mode 100644 data-node/src/main/java/org/graylog/datanode/process/configuration/files/YamlConfigFile.java delete mode 100644 data-node/src/main/resources/opensearch/config/opensearch-security/opensearch.yml.example delete mode 100644 data-node/src/test/java/org/graylog/datanode/bootstrap/preflight/FullDirSyncTest.java rename data-node/src/test/java/org/graylog/datanode/opensearch/configuration/beans/{OpensearchConfigurationPartTest.java => DatanodeConfigurationPartTest.java} (75%) diff --git a/data-node/src/main/java/org/graylog/datanode/bindings/OpensearchProcessBindings.java b/data-node/src/main/java/org/graylog/datanode/bindings/OpensearchProcessBindings.java index 438501a78b58..f09ae986c598 100644 --- a/data-node/src/main/java/org/graylog/datanode/bindings/OpensearchProcessBindings.java +++ b/data-node/src/main/java/org/graylog/datanode/bindings/OpensearchProcessBindings.java @@ -18,16 +18,25 @@ import com.google.common.util.concurrent.Service; import com.google.inject.AbstractModule; +import com.google.inject.TypeLiteral; import com.google.inject.multibindings.Multibinder; import org.graylog.datanode.configuration.DatanodeTrustManagerProvider; import org.graylog.datanode.configuration.OpensearchConfigurationService; +import org.graylog.datanode.configuration.variants.DatanodeKeystoreOpensearchCertificatesProvider; +import org.graylog.datanode.configuration.variants.LocalConfigurationCertificatesProvider; +import org.graylog.datanode.configuration.variants.NoOpensearchCertificatesProvider; +import org.graylog.datanode.configuration.variants.OpensearchCertificatesProvider; import org.graylog.datanode.metrics.ConfigureMetricsIndexSettings; import org.graylog.datanode.opensearch.OpensearchProcess; import org.graylog.datanode.opensearch.OpensearchProcessImpl; import org.graylog.datanode.opensearch.OpensearchProcessService; +import org.graylog.datanode.opensearch.configuration.OpensearchConfigurationParams; import org.graylog.datanode.opensearch.configuration.OpensearchUsableSpace; import org.graylog.datanode.opensearch.configuration.OpensearchUsableSpaceProvider; -import org.graylog.datanode.opensearch.configuration.beans.OpensearchConfigurationBean; +import org.graylog.datanode.opensearch.configuration.beans.impl.OpensearchClusterConfigurationBean; +import org.graylog.datanode.opensearch.configuration.beans.impl.OpensearchCommonConfigurationBean; +import org.graylog.datanode.opensearch.configuration.beans.impl.OpensearchDefaultConfigFilesBean; +import org.graylog.datanode.opensearch.configuration.beans.impl.OpensearchSecurityConfigurationBean; import org.graylog.datanode.opensearch.configuration.beans.impl.SearchableSnapshotsConfigurationBean; import org.graylog.datanode.opensearch.statemachine.OpensearchStateMachine; import org.graylog.datanode.opensearch.statemachine.OpensearchStateMachineProvider; @@ -35,6 +44,7 @@ import org.graylog.datanode.opensearch.statemachine.tracer.OpensearchWatchdog; import org.graylog.datanode.opensearch.statemachine.tracer.StateMachineTracer; import org.graylog.datanode.opensearch.statemachine.tracer.StateMachineTransitionLogger; +import org.graylog.datanode.process.configuration.beans.DatanodeConfigurationBean; public class OpensearchProcessBindings extends AbstractModule { @@ -49,9 +59,21 @@ protected void configure() { bind(OpensearchUsableSpace.class).toProvider(OpensearchUsableSpaceProvider.class).asEagerSingleton(); - //opensearch configuration beans - Multibinder opensearchConfigurationBeanMultibinder = Multibinder.newSetBinder(binder(), OpensearchConfigurationBean.class); + //opensearch certificate providers + Multibinder opensearchCertificatesProviders = Multibinder.newSetBinder(binder(), OpensearchCertificatesProvider.class); + opensearchCertificatesProviders.addBinding().to(LocalConfigurationCertificatesProvider.class).asEagerSingleton(); + opensearchCertificatesProviders.addBinding().to(DatanodeKeystoreOpensearchCertificatesProvider.class).asEagerSingleton(); + opensearchCertificatesProviders.addBinding().to(NoOpensearchCertificatesProvider.class).asEagerSingleton(); + + + //opensearch configuration beans. The order of the beans is important here! + + Multibinder> opensearchConfigurationBeanMultibinder = Multibinder.newSetBinder(binder(), new TypeLiteral>() {}); + opensearchConfigurationBeanMultibinder.addBinding().to(OpensearchDefaultConfigFilesBean.class).asEagerSingleton(); + opensearchConfigurationBeanMultibinder.addBinding().to(OpensearchCommonConfigurationBean.class).asEagerSingleton(); + opensearchConfigurationBeanMultibinder.addBinding().to(OpensearchClusterConfigurationBean.class).asEagerSingleton(); opensearchConfigurationBeanMultibinder.addBinding().to(SearchableSnapshotsConfigurationBean.class).asEagerSingleton(); + opensearchConfigurationBeanMultibinder.addBinding().to(OpensearchSecurityConfigurationBean.class).asEagerSingleton(); // this service both starts and provides the opensearch process serviceBinder.addBinding().to(OpensearchConfigurationService.class).asEagerSingleton(); diff --git a/data-node/src/main/java/org/graylog/datanode/bindings/PreflightChecksBindings.java b/data-node/src/main/java/org/graylog/datanode/bindings/PreflightChecksBindings.java index f32c4b52d6e5..857deedeccf9 100644 --- a/data-node/src/main/java/org/graylog/datanode/bindings/PreflightChecksBindings.java +++ b/data-node/src/main/java/org/graylog/datanode/bindings/PreflightChecksBindings.java @@ -23,7 +23,6 @@ import org.graylog.datanode.bootstrap.preflight.DatanodeKeystoreCheck; import org.graylog.datanode.bootstrap.preflight.OpenSearchPreconditionsCheck; import org.graylog.datanode.bootstrap.preflight.OpensearchBinPreflightCheck; -import org.graylog.datanode.bootstrap.preflight.OpensearchConfigSync; import org.graylog.datanode.bootstrap.preflight.OpensearchDataDirCompatibilityCheck; import org.graylog.datanode.opensearch.CsrRequester; import org.graylog.datanode.opensearch.CsrRequesterImpl; @@ -41,7 +40,6 @@ protected void configure() { bind(CsrRequester.class).to(CsrRequesterImpl.class).asEagerSingleton(); bind(CertificateExchange.class).to(CertificateExchangeImpl.class); - addPreflightCheck(OpensearchConfigSync.class); addPreflightCheck(DatanodeDnsPreflightCheck.class); addPreflightCheck(OpensearchBinPreflightCheck.class); addPreflightCheck(DatanodeDirectoriesLockfileCheck.class); diff --git a/data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/FullDirSync.java b/data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/FullDirSync.java deleted file mode 100644 index 9826f73c3315..000000000000 --- a/data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/FullDirSync.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright (C) 2020 Graylog, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * . - */ -package org.graylog.datanode.bootstrap.preflight; - -import org.apache.commons.io.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.StandardCopyOption; -import java.nio.file.attribute.BasicFileAttributes; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; - -public class FullDirSync { - private static final Logger LOG = LoggerFactory.getLogger(FullDirSync.class); - - /** - * The execute bit for directories means that owner can traverse these and access their content. - */ - protected static final Set DIRECTORY_PERMISSIONS = Set.of(PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_EXECUTE); - protected static final Set FILE_PERMISSIONS = Set.of(PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_READ); - - public static void run(Path source, Path target) throws IOException { - - final List existingPaths = collectExistingPaths(target); - existingPaths.remove(target); // exclude the target, we don't want to remove it in following step - deletePaths(existingPaths); - copyFiles(source, target); - } - - private static void deletePaths(List existingPaths) throws IOException { - for (Path path : existingPaths) { - if (Files.isDirectory(path)) { - LOG.info("Deleting obsolete directory " + path); - FileUtils.deleteDirectory(path.toFile()); - } else { - try { - LOG.info("Deleting obsolete file " + path); - Files.deleteIfExists(path); - } catch (IOException e) { - LOG.info("Failed to delete obsolete file " + path); - } - } - - } - } - - private static List copyFiles(Path source, Path target) throws IOException { - - List copiedPaths = new LinkedList<>(); - - Files.walkFileTree(source, new SimpleFileVisitor<>() { - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - Path currentTarget = target.resolve(source.relativize(dir).toString()); - Files.createDirectories(currentTarget, PosixFilePermissions.asFileAttribute((DIRECTORY_PERMISSIONS))); - copiedPaths.add(currentTarget); - LOG.info("Synchronizing directory {}", currentTarget); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - final Path currentTarget = target.resolve(source.relativize(file).toString()); - Files.copy(file, currentTarget, StandardCopyOption.REPLACE_EXISTING); - Files.setPosixFilePermissions(currentTarget, FILE_PERMISSIONS); - copiedPaths.add(currentTarget); - LOG.info("Synchronizing file {}", currentTarget); - return FileVisitResult.CONTINUE; - } - }); - - return copiedPaths; - } - - private static List collectExistingPaths(Path target) throws IOException { - final List pathsToDelete = new LinkedList<>(); - - Files.walkFileTree(target, new SimpleFileVisitor<>() { - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { - pathsToDelete.add(dir); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { - pathsToDelete.add(file); - return FileVisitResult.CONTINUE; - } - }); - return pathsToDelete; - } -} diff --git a/data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/OpensearchConfigSync.java b/data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/OpensearchConfigSync.java deleted file mode 100644 index 92193960debd..000000000000 --- a/data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/OpensearchConfigSync.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (C) 2020 Graylog, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * . - */ -package org.graylog.datanode.bootstrap.preflight; - -import org.graylog.datanode.configuration.DatanodeConfiguration; -import org.graylog2.bootstrap.preflight.PreflightCheck; -import org.graylog2.bootstrap.preflight.PreflightCheckException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import jakarta.inject.Inject; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.FileSystem; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; -import java.util.Collections; -import java.util.Set; - -public class OpensearchConfigSync implements PreflightCheck { - - private static final Logger LOG = LoggerFactory.getLogger(OpensearchConfigSync.class); - - private final DatanodeConfiguration configuration; - - @Inject - public OpensearchConfigSync(DatanodeConfiguration datanodeConfiguration) { - this.configuration = datanodeConfiguration; - } - - @Override - public void runCheck() throws PreflightCheckException { - try { - - final Path opensearchProcessConfigurationDir = configuration.datanodeDirectories().createOpensearchProcessConfigurationDir(); - LOG.info("Directory used for Opensearch process configuration is {}", opensearchProcessConfigurationDir.toAbsolutePath()); - - // this is a directory in main/resources that holds all the initial configuration files needed by the opensearch - // we manage this directory in git. Generally we assume that this is a read-only location and we need to copy - // its content to a read-write location for the managed opensearch process. - // This copy happens during each opensearch process start and will override any files that already exist - // from previous runs. - final Path sourceOfInitialConfiguration = Path.of("opensearch", "config"); - synchronizeConfig(sourceOfInitialConfiguration, opensearchProcessConfigurationDir); - } catch (IOException | URISyntaxException e) { - throw new RuntimeException("Failed to prepare opensearch config directory", e); - } - } - - public void synchronizeConfig(Path configRelativePath, final Path target) throws URISyntaxException, IOException { - final URI uriToConfig = OpensearchConfigSync.class.getResource("/" + configRelativePath.toString()).toURI(); - if ("jar".equals(uriToConfig.getScheme())) { - copyFromJar(configRelativePath, target, uriToConfig); - } else { - copyFromLocalFs(configRelativePath, target); - } - } - - private static void copyFromJar(Path configRelativePath, Path target, URI uri) throws IOException { - try ( - final FileSystem fs = FileSystems.newFileSystem(uri, Collections.emptyMap()); - ) { - // Get hold of the path to the top level directory of the JAR file - final Path resourcesRoot = fs.getPath("/"); - final Path source = resourcesRoot.resolve(configRelativePath.toString()); // caution, the toString is needed here to resolve properly! - copyRecursively(source, target); - } - } - - private static void copyFromLocalFs(Path configRelativePath, Path target) throws URISyntaxException, IOException { - final Path resourcesRoot = Paths.get(OpensearchConfigSync.class.getResource("/").toURI()); - final Path source = resourcesRoot.resolve(configRelativePath); - copyRecursively(source, target); - } - - private static void copyRecursively(Path source, Path target) throws IOException { - LOG.info("Synchronizing Opensearch configuration"); - FullDirSync.run(source, target); - } -} diff --git a/data-node/src/main/java/org/graylog/datanode/configuration/ConfigDirRemovalThread.java b/data-node/src/main/java/org/graylog/datanode/configuration/ConfigDirRemovalThread.java new file mode 100644 index 000000000000..67f2f4b3210b --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/configuration/ConfigDirRemovalThread.java @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.configuration; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; + +public class ConfigDirRemovalThread extends Thread { + private final Path opensearchConfigDir; + + public ConfigDirRemovalThread(Path opensearchConfigDir) { + this.opensearchConfigDir = opensearchConfigDir; + } + + @Override + public void run() { + deleteDirectory(opensearchConfigDir); + } + private void deleteDirectory(Path toBeDeleted) { + try { + if (Files.isDirectory(toBeDeleted)) { + try (final Stream list = Files.list(toBeDeleted)) { + list.forEach(this::deleteDirectory); + } + } + Files.delete(toBeDeleted); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/configuration/DatanodeDirectories.java b/data-node/src/main/java/org/graylog/datanode/configuration/DatanodeDirectories.java index 0c45a0233ea1..d652f904920f 100644 --- a/data-node/src/main/java/org/graylog/datanode/configuration/DatanodeDirectories.java +++ b/data-node/src/main/java/org/graylog/datanode/configuration/DatanodeDirectories.java @@ -17,7 +17,6 @@ package org.graylog.datanode.configuration; import jakarta.annotation.Nonnull; -import org.apache.commons.io.FileUtils; import org.graylog.datanode.Configuration; import org.graylog2.plugin.system.NodeId; import org.slf4j.Logger; @@ -32,6 +31,7 @@ import java.nio.file.attribute.PosixFilePermissions; import java.util.Optional; import java.util.Set; +import java.util.stream.Stream; /** * This is a collection of pointers to directories used to store data, logs and configuration of the managed opensearch. @@ -40,6 +40,11 @@ public class DatanodeDirectories { private static final Logger LOG = LoggerFactory.getLogger(DatanodeDirectories.class); + /** + * The execute bit for directories means that owner can traverse these and access their content. + */ + protected static final FileAttribute> DIRECTORY_PERMISSIONS = PosixFilePermissions.asFileAttribute(Set.of(PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_EXECUTE)); + private final Path dataTargetDir; private final Path logsTargetDir; private final Path configurationSourceDir; @@ -91,10 +96,6 @@ public Path getDataTargetDir() { return dataTargetDir.toAbsolutePath(); } - public String getDataTargetDirSpace() { - return FileUtils.byteCountToDisplaySize(dataTargetDir.toFile().getFreeSpace()); - } - /** * This directory is used by the managed opensearch to store its logs in it. * Read-write permissions required. @@ -103,11 +104,6 @@ public Path getLogsTargetDir() { return logsTargetDir.toAbsolutePath(); } - public String getLogsTargetDirSpace() { - return FileUtils.byteCountToDisplaySize(logsTargetDir.toFile().getFreeSpace()); - } - - /** * This directory is provided by system admin to the datanode. We read our configuration from this location, * we read certificates from here. We'll never write anything to it. @@ -129,52 +125,42 @@ public Optional resolveConfigurationSourceFile(String filename) { /** * This directory is used by us to store all runtime-generated configuration of datanode. This * could be truststores, private keys, certificates and other generated config files. - * We also synchronize and generate opensearch configuration into a subdir of this dir, see {@link #getOpensearchProcessConfigurationDir()} + * We also synchronize and generate opensearch configuration into a subdir of this dir, see {@link #createUniqueOpensearchProcessConfigurationDir()} * Read-write permissions required. */ public Path getConfigurationTargetDir() { return configurationTargetDir.toAbsolutePath(); } - public String getConfigurationTargetDirSpace() { - return FileUtils.byteCountToDisplaySize(configurationTargetDir.toFile().getFreeSpace()); - } - - public Path createConfigurationFile(Path relativePath) throws IOException { - final Path resolvedPath = getConfigurationTargetDir().resolve(relativePath); - return createRestrictedAccessFile(resolvedPath); - } - @Nonnull - private static Path createRestrictedAccessFile(Path resolvedPath) throws IOException { + static Path createRestrictedAccessFile(Path resolvedPath) throws IOException { Files.deleteIfExists(resolvedPath); final Set permissions = Set.of(PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_READ); final FileAttribute> fileAttributes = PosixFilePermissions.asFileAttribute(permissions); return Files.createFile(resolvedPath, fileAttributes); } + /** * This is a subdirectory of {@link #getConfigurationTargetDir()}. It's used by us to synchronize and generate opensearch * configuration. Opensearch is then instructed to accept this dir as its base configuration dir (OPENSEARCH_PATH_CONF env property). - * @see org.graylog.datanode.bootstrap.preflight.OpensearchConfigSync + * Opensearch configuration is always regenerated during runtime, so the target dir may be temp and deleted when + * the JVM terminates. This prevents concurrency collisions, outdated files, need to remove existing but not needed. */ - public Path getOpensearchProcessConfigurationDir() { - return getConfigurationTargetDir().resolve("opensearch"); + public OpensearchConfigurationDir createUniqueOpensearchProcessConfigurationDir() { + final Path configRootDir = getConfigurationTargetDir(); + try { + final Path opensearchConfigDir = Files.createTempDirectory(configRootDir, "opensearch", DIRECTORY_PERMISSIONS); + // the process configuration dir can be safely removed when this JVM terminates. It will be generated + // again next time we'll start a process. + Runtime.getRuntime().addShutdownHook(new ConfigDirRemovalThread(opensearchConfigDir)); + return new OpensearchConfigurationDir(opensearchConfigDir); + } catch (IOException e) { + throw new OpensearchConfigurationException("Failed to create opensearch configuration directory", e); + } } - public Path createOpensearchProcessConfigurationDir() throws IOException { - final Path dir = getOpensearchProcessConfigurationDir(); - // TODO: should we always delete existing process configuration dir and recreate it here? IMHO yes - final Set permissions = Set.of(PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_READ); - final FileAttribute> fileAttributes = PosixFilePermissions.asFileAttribute(permissions); - Files.createDirectories(dir, fileAttributes); - return dir; - } - public Path createOpensearchProcessConfigurationFile(Path relativePath) throws IOException { - final Path resolvedPath = getOpensearchProcessConfigurationDir().resolve(relativePath); - return createRestrictedAccessFile(resolvedPath); - } @Override public String toString() { @@ -183,7 +169,6 @@ public String toString() { ", logsTargetDir='" + getLogsTargetDir() + '\'' + ", configurationSourceDir='" + getConfigurationSourceDir() + '\'' + ", configurationTargetDir='" + getConfigurationTargetDir() + '\'' + - ", opensearchProcessConfigurationDir='" + getOpensearchProcessConfigurationDir() + '\'' + '}'; } } diff --git a/data-node/src/main/java/org/graylog/datanode/configuration/DatanodeKeystore.java b/data-node/src/main/java/org/graylog/datanode/configuration/DatanodeKeystore.java index b3d9b3177d81..a24be6b8fd10 100644 --- a/data-node/src/main/java/org/graylog/datanode/configuration/DatanodeKeystore.java +++ b/data-node/src/main/java/org/graylog/datanode/configuration/DatanodeKeystore.java @@ -21,6 +21,7 @@ import jakarta.annotation.Nullable; import jakarta.inject.Inject; import jakarta.inject.Named; +import org.apache.commons.lang3.RandomStringUtils; import org.bouncycastle.pkcs.PKCS10CertificationRequest; import org.graylog.security.certutil.CertConstants; import org.graylog.security.certutil.KeyPair; @@ -28,6 +29,7 @@ import org.graylog.security.certutil.csr.CsrGenerator; import org.graylog.security.certutil.csr.InMemoryKeystoreInformation; import org.graylog.security.certutil.csr.exceptions.CSRGenerationException; +import org.graylog.security.certutil.keystore.storage.KeystoreUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,6 +38,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.security.GeneralSecurityException; import java.security.InvalidKeyException; import java.security.Key; import java.security.KeyStore; @@ -167,6 +170,16 @@ public synchronized KeyStore loadKeystore() throws DatanodeKeystoreException { } } + public synchronized InMemoryKeystoreInformation getSafeCopy() throws DatanodeKeystoreException { + final char[] randomKeystorePassword = RandomStringUtils.randomAlphabetic(256).toCharArray(); + try { + final KeyStore reencrypted = KeystoreUtils.newStoreCopyContent(loadKeystore(), passwordSecret.toCharArray(), randomKeystorePassword); + return new InMemoryKeystoreInformation(reencrypted, randomKeystorePassword); + } catch (GeneralSecurityException | IOException e) { + throw new DatanodeKeystoreException(e); + } + } + public synchronized PKCS10CertificationRequest createCertificateSigningRequest(String hostname, List altNames) throws DatanodeKeystoreException, CSRGenerationException { final InMemoryKeystoreInformation keystore = new InMemoryKeystoreInformation(loadKeystore(), passwordSecret.toCharArray()); return CsrGenerator.generateCSR(keystore, DATANODE_KEY_ALIAS, hostname, altNames); diff --git a/data-node/src/main/java/org/graylog/datanode/configuration/DatanodeTrustManagerProvider.java b/data-node/src/main/java/org/graylog/datanode/configuration/DatanodeTrustManagerProvider.java index 299150d8e094..56e6d6f1726b 100644 --- a/data-node/src/main/java/org/graylog/datanode/configuration/DatanodeTrustManagerProvider.java +++ b/data-node/src/main/java/org/graylog/datanode/configuration/DatanodeTrustManagerProvider.java @@ -21,19 +21,13 @@ import jakarta.inject.Inject; import jakarta.inject.Provider; import jakarta.inject.Singleton; -import org.graylog.datanode.configuration.variants.OpensearchSecurityConfiguration; import org.graylog.datanode.opensearch.OpensearchConfigurationChangeEvent; import org.graylog2.security.CustomCAX509TrustManager; import org.graylog2.security.TrustManagerAggregator; import javax.net.ssl.X509TrustManager; -import java.io.IOException; import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.CertificateException; import java.util.List; -import java.util.Optional; @Singleton public class DatanodeTrustManagerProvider implements Provider { @@ -49,16 +43,7 @@ public DatanodeTrustManagerProvider(CustomCAX509TrustManager CustomCAX509TrustMa @Subscribe public void onOpensearchConfigurationChange(OpensearchConfigurationChangeEvent e) { - Optional.ofNullable(e.config().opensearchSecurityConfiguration()) - .flatMap(OpensearchSecurityConfiguration::getTruststore) - .map(t -> { - try { - return t.loadKeystore(); - } catch (KeyStoreException | IOException | CertificateException | NoSuchAlgorithmException ex) { - throw new RuntimeException(ex); - } - }) - .ifPresent(this::setTruststore); + setTruststore(e.config().trustStore()); } private void setTruststore(KeyStore keyStore) { diff --git a/data-node/src/main/java/org/graylog/datanode/configuration/OpensearchConfigurationDir.java b/data-node/src/main/java/org/graylog/datanode/configuration/OpensearchConfigurationDir.java new file mode 100644 index 000000000000..14754c37d252 --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/configuration/OpensearchConfigurationDir.java @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.configuration; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +public record OpensearchConfigurationDir(java.nio.file.Path configurationRoot) { + public Path createOpensearchProcessConfigurationFile(Path relativePath) throws IOException { + if (relativePath.isAbsolute()) { + throw new IllegalArgumentException("Only relative paths supported here!" + relativePath); + } + + final Path resolvedPath = configurationRoot.resolve(relativePath); + + // recursively create all parent directories + Files.createDirectories(resolvedPath.getParent(), DatanodeDirectories.DIRECTORY_PERMISSIONS); + return DatanodeDirectories.createRestrictedAccessFile(resolvedPath); + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/configuration/OpensearchConfigurationException.java b/data-node/src/main/java/org/graylog/datanode/configuration/OpensearchConfigurationException.java index 3556bec37e01..9fb14dbfceec 100644 --- a/data-node/src/main/java/org/graylog/datanode/configuration/OpensearchConfigurationException.java +++ b/data-node/src/main/java/org/graylog/datanode/configuration/OpensearchConfigurationException.java @@ -24,4 +24,8 @@ public OpensearchConfigurationException(String message) { public OpensearchConfigurationException(Exception cause) { super(cause); } + + public OpensearchConfigurationException(String message, Exception cause) { + super(message, cause); + } } diff --git a/data-node/src/main/java/org/graylog/datanode/configuration/OpensearchConfigurationService.java b/data-node/src/main/java/org/graylog/datanode/configuration/OpensearchConfigurationService.java index bdf996b076bc..529f6df09667 100644 --- a/data-node/src/main/java/org/graylog/datanode/configuration/OpensearchConfigurationService.java +++ b/data-node/src/main/java/org/graylog/datanode/configuration/OpensearchConfigurationService.java @@ -16,35 +16,22 @@ */ package org.graylog.datanode.configuration; -import com.google.common.collect.ImmutableMap; import com.google.common.eventbus.EventBus; import com.google.common.eventbus.Subscribe; import com.google.common.util.concurrent.AbstractIdleService; import jakarta.inject.Inject; import jakarta.inject.Singleton; import org.graylog.datanode.Configuration; -import org.graylog.datanode.configuration.variants.InSecureConfiguration; -import org.graylog.datanode.configuration.variants.LocalKeystoreSecureConfiguration; -import org.graylog.datanode.configuration.variants.OpensearchSecurityConfiguration; -import org.graylog.datanode.configuration.variants.SecurityConfigurationVariant; -import org.graylog.datanode.configuration.variants.UploadedCertFilesSecureConfiguration; import org.graylog.datanode.opensearch.OpensearchConfigurationChangeEvent; +import org.graylog.datanode.opensearch.configuration.OpensearchConfigurationParams; import org.graylog.datanode.opensearch.configuration.OpensearchConfiguration; -import org.graylog.datanode.opensearch.configuration.beans.OpensearchConfigurationBean; -import org.graylog.datanode.opensearch.configuration.beans.OpensearchConfigurationPart; -import org.graylog.security.certutil.ca.exceptions.KeyStoreStorageException; -import org.graylog2.cluster.Node; -import org.graylog2.cluster.nodes.DataNodeDto; -import org.graylog2.cluster.nodes.NodeService; -import org.graylog2.security.JwtSecret; - -import java.io.IOException; -import java.security.GeneralSecurityException; +import org.graylog.datanode.process.configuration.beans.DatanodeConfigurationBean; +import org.graylog.datanode.process.configuration.beans.DatanodeConfigurationPart; + import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -52,18 +39,13 @@ @Singleton public class OpensearchConfigurationService extends AbstractIdleService { private final Configuration localConfiguration; - private final UploadedCertFilesSecureConfiguration uploadedCertFilesSecureConfiguration; - private final LocalKeystoreSecureConfiguration localKeystoreSecureConfiguration; - private final InSecureConfiguration inSecureConfiguration; private final DatanodeConfiguration datanodeConfiguration; - private final JwtSecret signingKey; - private final NodeService nodeService; - private final Set opensearchConfigurationBeans; + private final Set> opensearchConfigurationBeans; /** * This configuration won't survive datanode restart. But it can be repeatedly provided to the managed opensearch */ - private final Map transientConfiguration = new ConcurrentHashMap<>(); + private final Map transientConfiguration = new ConcurrentHashMap<>(); private final List trustedCertificates = new ArrayList<>(); private final EventBus eventBus; @@ -71,20 +53,10 @@ public class OpensearchConfigurationService extends AbstractIdleService { @Inject public OpensearchConfigurationService(final Configuration localConfiguration, final DatanodeConfiguration datanodeConfiguration, - final UploadedCertFilesSecureConfiguration uploadedCertFilesSecureConfiguration, - final LocalKeystoreSecureConfiguration localKeystoreSecureConfiguration, - final InSecureConfiguration inSecureConfiguration, - final NodeService nodeService, - JwtSecret jwtSecret, - final Set opensearchConfigurationBeans, + final Set> opensearchConfigurationBeans, final EventBus eventBus) { this.localConfiguration = localConfiguration; this.datanodeConfiguration = datanodeConfiguration; - this.uploadedCertFilesSecureConfiguration = uploadedCertFilesSecureConfiguration; - this.localKeystoreSecureConfiguration = localKeystoreSecureConfiguration; - this.inSecureConfiguration = inSecureConfiguration; - this.signingKey = jwtSecret; - this.nodeService = nodeService; this.opensearchConfigurationBeans = opensearchConfigurationBeans; this.eventBus = eventBus; eventBus.register(this); @@ -110,14 +82,14 @@ public void onKeystoreChange(DatanodeKeystoreChangedEvent event) { public void setAllowlist(List allowlist, List trustedCertificates) { this.trustedCertificates.addAll(trustedCertificates); - setTransientConfiguration("reindex.remote.allowlist", allowlist); + setTransientConfiguration("reindex.remote.allowlist", String.join(", ", allowlist)); } public void removeAllowlist() { removeTransientConfiguration("reindex.remote.allowlist"); } - public void setTransientConfiguration(String key, Object value) { + public void setTransientConfiguration(String key, String value) { this.transientConfiguration.put(key, value); triggerConfigurationChangedEvent(); } @@ -130,88 +102,18 @@ public void removeTransientConfiguration(String key) { } private OpensearchConfiguration get() { - //TODO: at some point bind the whole list, for now there is too much experiments with order and prerequisites - List securityConfigurationTypes = List.of( - inSecureConfiguration, - uploadedCertFilesSecureConfiguration, - localKeystoreSecureConfiguration - ); - - Optional chosenSecurityConfigurationVariant = securityConfigurationTypes.stream() - .filter(s -> s.isConfigured(localConfiguration)) - .findFirst(); - - try { - ImmutableMap.Builder opensearchProperties = ImmutableMap.builder(); - - if (localConfiguration.getInitialClusterManagerNodes() != null && !localConfiguration.getInitialClusterManagerNodes().isBlank()) { - opensearchProperties.put("cluster.initial_cluster_manager_nodes", localConfiguration.getInitialClusterManagerNodes()); - } else { - final var nodeList = String.join(",", nodeService.allActive().values().stream().map(Node::getHostname).collect(Collectors.toSet())); - opensearchProperties.put("cluster.initial_cluster_manager_nodes", nodeList); - } - opensearchProperties.putAll(commonOpensearchConfig(localConfiguration)); - - OpensearchSecurityConfiguration securityConfiguration = null; - if (chosenSecurityConfigurationVariant.isPresent()) { - securityConfiguration = chosenSecurityConfigurationVariant.get() - .build() - .configure(datanodeConfiguration, trustedCertificates, signingKey); - opensearchProperties.putAll(securityConfiguration.getProperties()); - } - - final Set configurationParts = opensearchConfigurationBeans.stream() - .map(OpensearchConfigurationBean::buildConfigurationPart) - .collect(Collectors.toSet()); - - return new OpensearchConfiguration( - datanodeConfiguration.opensearchDistributionProvider().get(), - datanodeConfiguration.datanodeDirectories(), - localConfiguration.getBindAddress(), - localConfiguration.getHostname(), - localConfiguration.getOpensearchHttpPort(), - localConfiguration.getOpensearchTransportPort(), - localConfiguration.getClustername(), - localConfiguration.getDatanodeNodeName(), - localConfiguration.getNodeRoles(), - localConfiguration.getOpensearchDiscoverySeedHosts(), - securityConfiguration, - configurationParts, - opensearchProperties.build() - ); - } catch (GeneralSecurityException | KeyStoreStorageException | IOException e) { - throw new OpensearchConfigurationException(e); - } - } - - private ImmutableMap commonOpensearchConfig(final Configuration localConfiguration) { - final ImmutableMap.Builder config = ImmutableMap.builder(); - localConfiguration.getOpensearchNetworkHost().ifPresent( - networkHost -> config.put("network.host", networkHost)); - config.put("path.data", datanodeConfiguration.datanodeDirectories().getDataTargetDir().toString()); - config.put("path.logs", datanodeConfiguration.datanodeDirectories().getLogsTargetDir().toString()); - config.put("network.bind_host", localConfiguration.getBindAddress()); + final List configurationParts = opensearchConfigurationBeans.stream() + .map(bean -> bean.buildConfigurationPart(new OpensearchConfigurationParams(trustedCertificates, transientConfiguration))) + .collect(Collectors.toList()); - config.put("network.publish_host", localConfiguration.getHostname()); - - if (localConfiguration.getOpensearchDebug() != null && !localConfiguration.getOpensearchDebug().isBlank()) { - config.put("logger.org.opensearch", localConfiguration.getOpensearchDebug()); - } - - if (localConfiguration.getOpensearchAuditLog() != null && !localConfiguration.getOpensearchAuditLog().isBlank()) { - config.put("plugins.security.audit.type", localConfiguration.getOpensearchAuditLog()); - } - - // common OpenSearch config parameters from our docs - config.put("indices.query.bool.max_clause_count", localConfiguration.getIndicesQueryBoolMaxClauseCount().toString()); - - // enable admin access via the REST API - config.put("plugins.security.restapi.admin.enabled", "true"); - - config.putAll(transientConfiguration); - - return config.build(); + return new OpensearchConfiguration( + datanodeConfiguration.opensearchDistributionProvider().get(), + datanodeConfiguration.datanodeDirectories(), + localConfiguration.getHostname(), + localConfiguration.getOpensearchHttpPort(), + configurationParts + ); } private void triggerConfigurationChangedEvent() { diff --git a/data-node/src/main/java/org/graylog/datanode/configuration/OpensearchKeystoreProvider.java b/data-node/src/main/java/org/graylog/datanode/configuration/OpensearchKeystoreProvider.java index a079f4a9d48b..400ff4d86c1b 100644 --- a/data-node/src/main/java/org/graylog/datanode/configuration/OpensearchKeystoreProvider.java +++ b/data-node/src/main/java/org/graylog/datanode/configuration/OpensearchKeystoreProvider.java @@ -17,21 +17,21 @@ package org.graylog.datanode.configuration; -import com.google.common.base.Suppliers; import com.google.common.eventbus.EventBus; import com.google.common.eventbus.Subscribe; +import io.jsonwebtoken.lang.Collections; +import jakarta.annotation.Nonnull; import jakarta.inject.Inject; import jakarta.inject.Provider; import jakarta.inject.Singleton; -import org.graylog.datanode.configuration.variants.OpensearchSecurityConfiguration; import org.graylog.datanode.opensearch.OpensearchConfigurationChangeEvent; import org.graylog.security.certutil.KeyStoreDto; +import org.graylog.security.certutil.csr.KeystoreInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.HashMap; import java.util.Map; -import java.util.function.Supplier; +import java.util.concurrent.ConcurrentHashMap; @Singleton public class OpensearchKeystoreProvider implements Provider> { @@ -40,7 +40,7 @@ public enum Store {CONFIGURED, TRUSTSTORE, HTTP, TRANSPORT} private static final Logger log = LoggerFactory.getLogger(OpensearchKeystoreProvider.class); - private Supplier opensearchSecurityConfiguration; + private final Map keystores = new ConcurrentHashMap<>(); @Inject public OpensearchKeystoreProvider(EventBus eventBus) { @@ -50,44 +50,34 @@ public OpensearchKeystoreProvider(EventBus eventBus) { @Subscribe @SuppressWarnings("unused") public void onConfigurationChangeEvent(OpensearchConfigurationChangeEvent event) { - this.opensearchSecurityConfiguration = Suppliers.memoize(() -> event.config().opensearchSecurityConfiguration()); - } - - @Override - public Map get() { - if (opensearchSecurityConfiguration == null) { - return Map.of(); - } - OpensearchSecurityConfiguration config = opensearchSecurityConfiguration.get(); + try { + keystores.put(Store.TRUSTSTORE, KeyStoreDto.fromKeyStore(event.config().trustStore())); - Map certificates = new HashMap<>(); + event.config().httpCertificate() + .map(OpensearchKeystoreProvider::toDto) + .ifPresentOrElse(dto -> keystores.put(Store.HTTP, dto), () -> keystores.remove(Store.HTTP)); - certificates.put(Store.TRUSTSTORE, config.getTruststore().map(t -> { - try { - return KeyStoreDto.fromKeyStore(t.loadKeystore()); - } catch (Exception e) { - log.error("Error reading truststore", e); - return KeyStoreDto.empty(); - } - }).orElse(KeyStoreDto.empty())); + event.config().transportCertificate() + .map(OpensearchKeystoreProvider::toDto) + .ifPresentOrElse(dto -> keystores.put(Store.TRANSPORT, dto), () -> keystores.remove(Store.TRANSPORT)); - KeyStoreDto http = KeyStoreDto.empty(); - try { - http = KeyStoreDto.fromKeyStore(config.getHttpCertificate().loadKeystore()); } catch (Exception e) { - log.error("Error reading http certificate", e); - + log.error("Error reading truststore", e); } - certificates.put(Store.HTTP, http); + } - KeyStoreDto transport = KeyStoreDto.empty(); + @Nonnull + private static KeyStoreDto toDto(KeystoreInformation cert) { try { - transport = KeyStoreDto.fromKeyStore(config.getTransportCertificate().loadKeystore()); + return KeyStoreDto.fromKeyStore(cert.loadKeystore()); } catch (Exception e) { - log.error("Error reading transport certificate", e); + throw new RuntimeException(e); } - certificates.put(Store.TRANSPORT, transport); - return certificates; + } + + @Override + public Map get() { + return Collections.immutable(keystores); } } diff --git a/data-node/src/main/java/org/graylog/datanode/configuration/TruststoreCreator.java b/data-node/src/main/java/org/graylog/datanode/configuration/TruststoreCreator.java index cea4e2415dd1..57950d5dea11 100644 --- a/data-node/src/main/java/org/graylog/datanode/configuration/TruststoreCreator.java +++ b/data-node/src/main/java/org/graylog/datanode/configuration/TruststoreCreator.java @@ -19,10 +19,9 @@ import jakarta.annotation.Nonnull; import org.graylog.security.certutil.CertConstants; import org.graylog.security.certutil.csr.FilesystemKeystoreInformation; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.graylog.security.certutil.csr.InMemoryKeystoreInformation; +import org.graylog.security.certutil.csr.KeystoreInformation; -import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Path; @@ -60,9 +59,14 @@ public static TruststoreCreator newEmpty() { } } - public TruststoreCreator addRootCert(final String name, FilesystemKeystoreInformation keystoreInformation, - final String alias) throws IOException, GeneralSecurityException { - final X509Certificate rootCert = findRootCert(keystoreInformation.location(), keystoreInformation.password(), alias); + public TruststoreCreator addRootCert(final String name, KeystoreInformation keystoreInformation, + final String alias) throws GeneralSecurityException { + final X509Certificate rootCert; + try { + rootCert = findRootCert(keystoreInformation, alias); + } catch (Exception e) { + throw new RuntimeException(e); + } this.truststore.setCertificateEntry(name, rootCert); return this; } @@ -91,11 +95,14 @@ public KeyStore getTruststore() { return this.truststore; } + public KeystoreInformation toKeystoreInformation(final char[] truststorePassword) { + return new InMemoryKeystoreInformation(this.truststore, truststorePassword); + } + - private static X509Certificate findRootCert(Path keystorePath, - char[] password, - final String alias) throws IOException, GeneralSecurityException { - final KeyStore keystore = loadKeystore(keystorePath, password); + private static X509Certificate findRootCert(KeystoreInformation keystoreInformation, + final String alias) throws Exception { + final KeyStore keystore = keystoreInformation.loadKeystore(); final Certificate[] certs = keystore.getCertificateChain(alias); return Arrays.stream(certs) @@ -109,13 +116,4 @@ private static X509Certificate findRootCert(Path keystorePath, private static boolean isRootCaCertificate(X509Certificate cert) { return cert.getSubjectX500Principal().equals(cert.getIssuerX500Principal()); } - - private static KeyStore loadKeystore(final Path keystorePath, - final char[] password) throws IOException, GeneralSecurityException { - KeyStore nodeKeystore = KeyStore.getInstance(CertConstants.PKCS12); - try (final FileInputStream is = new FileInputStream(keystorePath.toFile())) { - nodeKeystore.load(is, password); - } - return nodeKeystore; - } } diff --git a/data-node/src/main/java/org/graylog/datanode/configuration/variants/DatanodeKeystoreOpensearchCertificatesProvider.java b/data-node/src/main/java/org/graylog/datanode/configuration/variants/DatanodeKeystoreOpensearchCertificatesProvider.java new file mode 100644 index 000000000000..2736bbfb80d3 --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/configuration/variants/DatanodeKeystoreOpensearchCertificatesProvider.java @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.configuration.variants; + +import jakarta.inject.Inject; +import org.graylog.datanode.Configuration; +import org.graylog.datanode.configuration.DatanodeKeystore; +import org.graylog.datanode.configuration.DatanodeKeystoreException; +import org.graylog.datanode.configuration.OpensearchConfigurationException; +import org.graylog.security.certutil.csr.InMemoryKeystoreInformation; + +public final class DatanodeKeystoreOpensearchCertificatesProvider implements OpensearchCertificatesProvider { + private final DatanodeKeystore datanodeKeystore; + + @Inject + public DatanodeKeystoreOpensearchCertificatesProvider(final DatanodeKeystore datanodeKeystore) { + this.datanodeKeystore = datanodeKeystore; + } + + @Override + public boolean isConfigured(Configuration localConfiguration) { + try { + return datanodeKeystore.exists() && datanodeKeystore.hasSignedCertificate(); + } catch (DatanodeKeystoreException e) { + throw new OpensearchConfigurationException(e); + } + } + + @Override + public OpensearchCertificates build() { + try { + final InMemoryKeystoreInformation safeCopy = this.datanodeKeystore.getSafeCopy(); + return new OpensearchCertificates(safeCopy, safeCopy); + } catch (DatanodeKeystoreException e) { + throw new OpensearchConfigurationException(e); + } + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/configuration/variants/LocalConfigurationCertificatesProvider.java b/data-node/src/main/java/org/graylog/datanode/configuration/variants/LocalConfigurationCertificatesProvider.java new file mode 100644 index 000000000000..adc8198dcd43 --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/configuration/variants/LocalConfigurationCertificatesProvider.java @@ -0,0 +1,136 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.configuration.variants; + +import jakarta.annotation.Nonnull; +import jakarta.inject.Inject; +import org.apache.commons.lang3.RandomStringUtils; +import org.graylog.datanode.Configuration; +import org.graylog.datanode.configuration.DatanodeConfiguration; +import org.graylog.datanode.configuration.OpensearchConfigurationException; +import org.graylog.security.certutil.csr.FilesystemKeystoreInformation; +import org.graylog.security.certutil.csr.InMemoryKeystoreInformation; +import org.graylog.security.certutil.csr.KeystoreInformation; +import org.graylog.security.certutil.keystore.storage.KeystoreUtils; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyStore; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; + +import static org.graylog.datanode.Configuration.HTTP_CERTIFICATE_PASSWORD_PROPERTY; +import static org.graylog.datanode.Configuration.TRANSPORT_CERTIFICATE_PASSWORD_PROPERTY; + +public final class LocalConfigurationCertificatesProvider implements OpensearchCertificatesProvider { + + private final String tranportCertificateFile; + private final String httpCertificateFile; + private final String transportCertificatePassword; + private final String httpCertificatePassword; + private final DatanodeConfiguration datanodeConfiguration; + + @Inject + public LocalConfigurationCertificatesProvider(final Configuration localConfiguration, + final DatanodeConfiguration datanodeConfiguration) { + this.datanodeConfiguration = datanodeConfiguration; + + this.tranportCertificateFile = localConfiguration.getDatanodeTransportCertificate(); + this.transportCertificatePassword = localConfiguration.getDatanodeTransportCertificatePassword(); + + this.httpCertificateFile = localConfiguration.getDatanodeHttpCertificate(); + this.httpCertificatePassword = localConfiguration.getDatanodeHttpCertificatePassword(); + } + + @Override + public boolean isConfigured(Configuration localConfiguration) throws OpensearchConfigurationException { + + if (noneOfRequiredConfigOptionsProvided()) { + return false; // none of the uploaded cert options is provided => not usable for this security config, skip this config + } + + List errors = new LinkedList<>(); + + if (isBlank(transportCertificatePassword)) { + errors.add(TRANSPORT_CERTIFICATE_PASSWORD_PROPERTY + " required. Please configure password to your transport certificates keystore."); + } + + if (!fileExists(tranportCertificateFile)) { + errors.add("transport_certificate required. Please provide a path to a certificate file in your configuration."); + } + + if (isBlank(httpCertificatePassword)) { + errors.add(HTTP_CERTIFICATE_PASSWORD_PROPERTY + " required. Please configure password to your http certificates keystore."); + } + + if (!fileExists(httpCertificateFile)) { + errors.add("http_certificate required. Please provide a path to a certificate file in your configuration."); + } + + if (!errors.isEmpty()) { + throw new OpensearchConfigurationException("Configuration incomplete, check the following settings: " + String.join(", ", errors)); + } + + return true; + } + + private boolean isBlank(String value) { + return value == null || value.isBlank(); + } + + private boolean fileExists(String filename) { + return Optional.ofNullable(filename) + .flatMap(fileName -> datanodeConfiguration.datanodeDirectories().resolveConfigurationSourceFile(filename)) + .map(Files::exists) + .orElse(false); + } + + /** + * We require either full set of http and transport certificates and their keys or nothing. Anything in-between will + * lead to an exception, it's a mismatched configuration and would cause problems in the future. + */ + private boolean noneOfRequiredConfigOptionsProvided() { + return isBlank(transportCertificatePassword) && + isBlank(httpCertificatePassword) && + isBlank(httpCertificateFile) && + isBlank(tranportCertificateFile); + } + + @Override + public OpensearchCertificates build() { + + final Path transportCertPath = datanodeConfiguration.datanodeDirectories().resolveConfigurationSourceFile(tranportCertificateFile).orElseThrow(() -> new RuntimeException("This should not happen, certificate expected")); + final InMemoryKeystoreInformation transportKeystore = reencrypt(new FilesystemKeystoreInformation(transportCertPath, transportCertificatePassword.toCharArray())); + + final Path httpCertPath = datanodeConfiguration.datanodeDirectories().resolveConfigurationSourceFile(httpCertificateFile).orElseThrow(() -> new RuntimeException("This should not happen, certificate expected")); + final InMemoryKeystoreInformation httpKeystore = reencrypt(new FilesystemKeystoreInformation(httpCertPath, httpCertificatePassword.toCharArray())); + + return new OpensearchCertificates(transportKeystore, httpKeystore); + } + + @Nonnull + private static InMemoryKeystoreInformation reencrypt(KeystoreInformation keystoreInformation) { + try { + final char[] oneTimePassword = RandomStringUtils.randomAlphabetic(256).toCharArray(); + final KeyStore reencrypted = KeystoreUtils.newStoreCopyContent(keystoreInformation.loadKeystore(), keystoreInformation.password(), oneTimePassword); + return new InMemoryKeystoreInformation(reencrypted, oneTimePassword); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/configuration/variants/LocalKeystoreSecureConfiguration.java b/data-node/src/main/java/org/graylog/datanode/configuration/variants/LocalKeystoreSecureConfiguration.java deleted file mode 100644 index 20f5f261136a..000000000000 --- a/data-node/src/main/java/org/graylog/datanode/configuration/variants/LocalKeystoreSecureConfiguration.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (C) 2020 Graylog, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * . - */ -package org.graylog.datanode.configuration.variants; - -import jakarta.inject.Inject; -import jakarta.inject.Named; -import org.graylog.datanode.Configuration; -import org.graylog.datanode.configuration.DatanodeConfiguration; -import org.graylog.datanode.configuration.DatanodeKeystore; -import org.graylog.datanode.configuration.DatanodeKeystoreException; -import org.graylog.security.certutil.ca.exceptions.KeyStoreStorageException; -import org.graylog.security.certutil.csr.FilesystemKeystoreInformation; - -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.file.Path; -import java.security.GeneralSecurityException; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.CertificateException; - -public final class LocalKeystoreSecureConfiguration extends SecureConfiguration { - private final DatanodeKeystore datanodeKeystore; - private final char[] secret; - - @Inject - public LocalKeystoreSecureConfiguration(final DatanodeKeystore datanodeKeystore, - final DatanodeConfiguration datanodeConfiguration, - final @Named("password_secret") String passwordSecret - ) { - super(datanodeConfiguration); - this.datanodeKeystore = datanodeKeystore; - this.secret = passwordSecret.toCharArray(); - } - - @Override - public boolean isConfigured(Configuration localConfiguration) { - try { - return datanodeKeystore.exists() && datanodeKeystore.hasSignedCertificate(); - } catch (DatanodeKeystoreException e) { - throw new RuntimeException(e); - } - } - - @Override - public OpensearchSecurityConfiguration build() throws KeyStoreStorageException, IOException, GeneralSecurityException { - - final Path targetTransportKeystoreLocation = getTransportKeystoreLocation(); - final Path targetHttpKeystoreLocation = getHttpKeystoreLocation(); - - try { - final KeyStore datanodeKeystore = this.datanodeKeystore.loadKeystore(); - copy(datanodeKeystore, targetTransportKeystoreLocation, secret); - copy(datanodeKeystore, targetHttpKeystoreLocation, secret); - - return new OpensearchSecurityConfiguration( - new FilesystemKeystoreInformation(targetTransportKeystoreLocation.toAbsolutePath(), secret), - new FilesystemKeystoreInformation(targetHttpKeystoreLocation.toAbsolutePath(), secret) - ); - } catch (DatanodeKeystoreException e) { - throw new RuntimeException(e); - } - } - - private void copy(final KeyStore originalKeystore, final Path targetLocation, final char[] password) { - try (FileOutputStream fos = new FileOutputStream(targetLocation.toFile())) { - originalKeystore.store(fos, password); - } catch (IOException | CertificateException | KeyStoreException | NoSuchAlgorithmException e) { - throw new RuntimeException(e); - } - } -} diff --git a/data-node/src/main/java/org/graylog/datanode/configuration/variants/InSecureConfiguration.java b/data-node/src/main/java/org/graylog/datanode/configuration/variants/NoOpensearchCertificatesProvider.java similarity index 65% rename from data-node/src/main/java/org/graylog/datanode/configuration/variants/InSecureConfiguration.java rename to data-node/src/main/java/org/graylog/datanode/configuration/variants/NoOpensearchCertificatesProvider.java index a6951471e41b..2aacfbde2cfc 100644 --- a/data-node/src/main/java/org/graylog/datanode/configuration/variants/InSecureConfiguration.java +++ b/data-node/src/main/java/org/graylog/datanode/configuration/variants/NoOpensearchCertificatesProvider.java @@ -17,14 +17,13 @@ package org.graylog.datanode.configuration.variants; import org.graylog.datanode.Configuration; -import org.graylog.security.certutil.ca.exceptions.KeyStoreStorageException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @Deprecated -public class InSecureConfiguration implements SecurityConfigurationVariant { +public class NoOpensearchCertificatesProvider implements OpensearchCertificatesProvider { - private static final Logger LOG = LoggerFactory.getLogger(InSecureConfiguration.class); + private static final Logger LOG = LoggerFactory.getLogger(NoOpensearchCertificatesProvider.class); @Override public boolean isConfigured(final Configuration localConfiguration) { @@ -32,8 +31,8 @@ public boolean isConfigured(final Configuration localConfiguration) { } @Override - public OpensearchSecurityConfiguration build() throws KeyStoreStorageException { - LOG.warn("Insecure configuration is deprecated. Please use selfsigned_setup to create fully encrypted setups."); - return OpensearchSecurityConfiguration.disabled(); + public OpensearchCertificates build() { + LOG.warn("Insecure configuration is deprecated. Please use selfsigned_startup to create fully encrypted setups."); + return OpensearchCertificates.none(); } } diff --git a/data-node/src/main/java/org/graylog/datanode/configuration/variants/OpensearchCertificates.java b/data-node/src/main/java/org/graylog/datanode/configuration/variants/OpensearchCertificates.java new file mode 100644 index 000000000000..05b5986eea38 --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/configuration/variants/OpensearchCertificates.java @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.configuration.variants; + +import jakarta.annotation.Nullable; +import org.graylog.security.certutil.csr.KeystoreInformation; + +public class OpensearchCertificates { + + private final KeystoreInformation transportCertificate; + private final KeystoreInformation httpCertificate; + + public OpensearchCertificates(KeystoreInformation transportCertificate, KeystoreInformation httpCertificate) { + this.transportCertificate = transportCertificate; + this.httpCertificate = httpCertificate; + } + + public static OpensearchCertificates none() { + return new OpensearchCertificates(null, null); + } + + @Nullable + public KeystoreInformation getTransportCertificate() { + return transportCertificate; + } + + @Nullable + public KeystoreInformation getHttpCertificate() { + return httpCertificate; + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/configuration/variants/SecurityConfigurationVariant.java b/data-node/src/main/java/org/graylog/datanode/configuration/variants/OpensearchCertificatesProvider.java similarity index 74% rename from data-node/src/main/java/org/graylog/datanode/configuration/variants/SecurityConfigurationVariant.java rename to data-node/src/main/java/org/graylog/datanode/configuration/variants/OpensearchCertificatesProvider.java index 391b5887ea73..61169d69d4a0 100644 --- a/data-node/src/main/java/org/graylog/datanode/configuration/variants/SecurityConfigurationVariant.java +++ b/data-node/src/main/java/org/graylog/datanode/configuration/variants/OpensearchCertificatesProvider.java @@ -18,14 +18,10 @@ import org.graylog.datanode.Configuration; import org.graylog.datanode.configuration.OpensearchConfigurationException; -import org.graylog.security.certutil.ca.exceptions.KeyStoreStorageException; -import java.io.IOException; -import java.security.GeneralSecurityException; - -public interface SecurityConfigurationVariant { +public interface OpensearchCertificatesProvider { boolean isConfigured(final Configuration localConfiguration) throws OpensearchConfigurationException; - OpensearchSecurityConfiguration build() throws KeyStoreStorageException, IOException, GeneralSecurityException; + OpensearchCertificates build(); } diff --git a/data-node/src/main/java/org/graylog/datanode/configuration/variants/OpensearchSecurityConfiguration.java b/data-node/src/main/java/org/graylog/datanode/configuration/variants/OpensearchSecurityConfiguration.java deleted file mode 100644 index e319d457d2a6..000000000000 --- a/data-node/src/main/java/org/graylog/datanode/configuration/variants/OpensearchSecurityConfiguration.java +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Copyright (C) 2020 Graylog, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * . - */ -package org.graylog.datanode.configuration.variants; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; -import com.google.common.collect.ImmutableMap; -import org.apache.commons.lang3.RandomStringUtils; -import org.graylog.datanode.configuration.DatanodeConfiguration; -import org.graylog.datanode.configuration.TruststoreCreator; -import org.graylog.security.certutil.CertConstants; -import org.graylog.security.certutil.csr.FilesystemKeystoreInformation; -import org.graylog2.security.JwtSecret; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.nio.file.Path; -import java.security.GeneralSecurityException; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.Certificate; -import java.security.cert.CertificateException; -import java.security.cert.X509Certificate; -import java.util.Base64; -import java.util.Enumeration; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.stream.Collectors; - -public class OpensearchSecurityConfiguration { - - private static final Logger LOG = LoggerFactory.getLogger(OpensearchSecurityConfiguration.class); - - private static final String KEYSTORE_FORMAT = "PKCS12"; - private static final String TRUSTSTORE_FORMAT = "PKCS12"; - private static final Path TRUSTSTORE_FILE = Path.of("datanode-truststore.p12"); - - private final FilesystemKeystoreInformation transportCertificate; - private final FilesystemKeystoreInformation httpCertificate; - private FilesystemKeystoreInformation truststore; - private String opensearchHeap; - - public OpensearchSecurityConfiguration(FilesystemKeystoreInformation transportCertificate, FilesystemKeystoreInformation httpCertificate) { - this.transportCertificate = transportCertificate; - this.httpCertificate = httpCertificate; - } - - public static OpensearchSecurityConfiguration disabled() { - return new OpensearchSecurityConfiguration(null, null); - } - - /** - * Caution: side effects! - * - * This method will take the current security setup and apply it to the managed opensearch. It will change the - * initial set of opensearch users, it will create and persist a truststore - */ - public OpensearchSecurityConfiguration configure(DatanodeConfiguration datanodeConfiguration, List trustedCertificates, JwtSecret signingKey) throws GeneralSecurityException, IOException { - opensearchHeap = datanodeConfiguration.opensearchHeap(); - if (securityEnabled()) { - - logCertificateInformation("transport certificate", transportCertificate); - logCertificateInformation("HTTP certificate", httpCertificate); - - final Path opensearchConfigDir = datanodeConfiguration.datanodeDirectories().getOpensearchProcessConfigurationDir(); - - final Path trustStorePath = datanodeConfiguration.datanodeDirectories().createOpensearchProcessConfigurationFile(TRUSTSTORE_FILE); - final String truststorePassword = RandomStringUtils.randomAlphabetic(256); - - this.truststore = TruststoreCreator.newDefaultJvm() - .addRootCert("datanode-transport-chain-CA-root", transportCertificate, CertConstants.DATANODE_KEY_ALIAS) - .addRootCert("datanode-http-chain-CA-root", httpCertificate, CertConstants.DATANODE_KEY_ALIAS) - .addCertificates(trustedCertificates) - .persist(trustStorePath, truststorePassword.toCharArray()); - - enableJwtAuthenticationInConfig(opensearchConfigDir, signingKey); - } - return this; - } - - public Map getProperties() throws GeneralSecurityException, IOException { - final ImmutableMap.Builder config = ImmutableMap.builder(); - if (securityEnabled()) { - config.putAll(commonSecureConfig()); - - config.put("plugins.security.ssl.transport.keystore_type", KEYSTORE_FORMAT); - config.put("plugins.security.ssl.transport.keystore_filepath", transportCertificate.location().getFileName().toString()); // todo: this should be computed as a relative path - config.put("plugins.security.ssl.transport.keystore_alias", CertConstants.DATANODE_KEY_ALIAS); - - config.put("plugins.security.ssl.transport.truststore_type", TRUSTSTORE_FORMAT); - config.put("plugins.security.ssl.transport.truststore_filepath", TRUSTSTORE_FILE.toString()); - - config.put("plugins.security.ssl.http.enabled", "true"); - - config.put("plugins.security.ssl.http.keystore_type", KEYSTORE_FORMAT); - config.put("plugins.security.ssl.http.keystore_filepath", httpCertificate.location().getFileName().toString()); // todo: this should be computed as a relative path - config.put("plugins.security.ssl.http.keystore_alias", CertConstants.DATANODE_KEY_ALIAS); - - config.put("plugins.security.ssl.http.truststore_type", TRUSTSTORE_FORMAT); - config.put("plugins.security.ssl.http.truststore_filepath", TRUSTSTORE_FILE.toString()); - - // enable client cert auth - config.put("plugins.security.ssl.http.clientauth_mode", "OPTIONAL"); - } else { - config.put("plugins.security.disabled", "true"); - config.put("plugins.security.ssl.http.enabled", "false"); - } - return config.build(); - } - - public Map getKeystoreItems() { - final ImmutableMap.Builder config = ImmutableMap.builder(); - if (securityEnabled()) { - config.put("plugins.security.ssl.transport.keystore_password_secure", new String(transportCertificate.password())); - config.put("plugins.security.ssl.transport.truststore_password_secure", new String(truststore.password())); - config.put("plugins.security.ssl.http.keystore_password_secure", new String(httpCertificate.password())); - config.put("plugins.security.ssl.http.truststore_password_secure", new String(truststore.password())); - } - return config.build(); - } - - private Map filterConfigurationMap(final Map map, final String... keys) { - Map result = map; - for (final String key : List.of(keys)) { - result = (Map) result.get(key); - } - return result; - } - - private void enableJwtAuthenticationInConfig(final Path opensearchConfigDir, final JwtSecret signingKey) throws IOException { - final ObjectMapper objectMapper = new YAMLMapper(); - final File file = opensearchConfigDir.resolve(Path.of("opensearch-security", "config.yml")).toFile(); - Map contents = objectMapper.readValue(file, new TypeReference<>() {}); - - Map config = filterConfigurationMap(contents, "config", "dynamic", "authc", "jwt_auth_domain", "http_authenticator", "config"); - config.put("signing_key", signingKey.getBase64Encoded()); - - objectMapper.writeValue(file, contents); - } - - public boolean securityEnabled() { - return !Objects.isNull(httpCertificate) && !Objects.isNull(transportCertificate); - } - - public FilesystemKeystoreInformation getTransportCertificate() { - return transportCertificate; - } - - public FilesystemKeystoreInformation getHttpCertificate() { - return httpCertificate; - } - - public Optional getTruststore() { - return Optional.ofNullable(truststore); - } - - protected ImmutableMap commonSecureConfig() { - final ImmutableMap.Builder config = ImmutableMap.builder(); - - config.put("plugins.security.disabled", "false"); - //config.put(SSL_PREFIX + "http.enabled", "true"); - - config.put("plugins.security.nodes_dn", "CN=*"); - config.put("plugins.security.allow_default_init_securityindex", "true"); - //config.put("plugins.security.authcz.admin_dn", "CN=kirk,OU=client,O=client,L=test,C=de"); - - config.put("plugins.security.enable_snapshot_restore_privilege", "true"); - config.put("plugins.security.check_snapshot_restore_write_privileges", "true"); - config.put("plugins.security.restapi.roles_enabled", "all_access,security_rest_api_access,readall"); - config.put("plugins.security.system_indices.enabled", "true"); - config.put("plugins.security.system_indices.indices", ".plugins-ml-model,.plugins-ml-task,.opendistro-alerting-config,.opendistro-alerting-alert*,.opendistro-anomaly-results*,.opendistro-anomaly-detector*,.opendistro-anomaly-checkpoints,.opendistro-anomaly-detection-state,.opendistro-reports-*,.opensearch-notifications-*,.opensearch-notebooks,.opensearch-observability,.opendistro-asynchronous-search-response*,.replication-metadata-store"); - config.put("node.max_local_storage_nodes", "3"); - - return config.build(); - } - - private void logCertificateInformation(String certificateType, FilesystemKeystoreInformation keystore) throws KeyStoreException, IOException, CertificateException, NoSuchAlgorithmException { - final KeyStore instance = KeyStore.getInstance(KEYSTORE_FORMAT); - try (final FileInputStream is = new FileInputStream(keystore.location().toFile())) { - instance.load(is, keystore.password()); - final Enumeration aliases = instance.aliases(); - while (aliases.hasMoreElements()) { - final Certificate cert = instance.getCertificate(aliases.nextElement()); - if (cert instanceof X509Certificate x509Certificate) { - final String alternativeNames = x509Certificate.getSubjectAlternativeNames() - .stream() - .map(san -> san.get(1)) - .map(Object::toString) - .collect(Collectors.joining(", ")); - LOG.info("Opensearch {} has following alternative names: {}", certificateType, alternativeNames); - LOG.info("Opensearch {} has following serial number: {}", certificateType, ((X509Certificate) cert).getSerialNumber()); - LOG.info("Opensearch {} has following validity: {} - {}", certificateType, ((X509Certificate) cert).getNotBefore(), ((X509Certificate) cert).getNotAfter()); - } - } - } - } - - public String getOpensearchHeap() { - return opensearchHeap; - } -} diff --git a/data-node/src/main/java/org/graylog/datanode/configuration/variants/SecureConfiguration.java b/data-node/src/main/java/org/graylog/datanode/configuration/variants/SecureConfiguration.java deleted file mode 100644 index a6c4eea3ce23..000000000000 --- a/data-node/src/main/java/org/graylog/datanode/configuration/variants/SecureConfiguration.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (C) 2020 Graylog, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * . - */ -package org.graylog.datanode.configuration.variants; - -import com.google.common.base.Suppliers; -import org.graylog.datanode.configuration.DatanodeConfiguration; - -import java.io.IOException; -import java.nio.file.Path; -import java.util.function.Supplier; - -public abstract class SecureConfiguration implements SecurityConfigurationVariant { - - /** - * This filename is used only internally - we copy user-provided certificates to this location and - * we configure opensearch to read this file. It doesn't have to match naming provided by user. - * The target configuration is regenerated during each startup, so it could also be a random filename - * as long as we use the same name as a copy-target and opensearch config property. - */ - private static final Path TARGET_DATANODE_HTTP_KEYSTORE_FILENAME = Path.of("http-keystore.p12"); - /** - * This filename is used only internally - we copy user-provided certificates to this location and - * we configure opensearch to read this file. It doesn't have to match naming provided by user. - * The target configuration is regenerated during each startup, so it could also be a random filename - * as long as we use the same name as a copy-target and opensearch config property. - */ - private static final Path TARGET_DATANODE_TRANSPORT_KEYSTORE_FILENAME = Path.of("transport-keystore.p12"); - - private final Supplier httpKeystoreLocation; - private final Supplier transportKeystoreLocation; - - public SecureConfiguration(final DatanodeConfiguration datanodeConfiguration) { - this.httpKeystoreLocation = Suppliers.memoize(() -> { - try { - return datanodeConfiguration.datanodeDirectories().createOpensearchProcessConfigurationFile(TARGET_DATANODE_HTTP_KEYSTORE_FILENAME); - } catch (IOException e) { - throw new RuntimeException("Failed to create http keystore file", e); - } - }); - - this.transportKeystoreLocation = Suppliers.memoize(() -> { - try { - return datanodeConfiguration.datanodeDirectories().createOpensearchProcessConfigurationFile(TARGET_DATANODE_TRANSPORT_KEYSTORE_FILENAME); - } catch (IOException e) { - throw new RuntimeException("Failed to create transport keystore file", e); - } - }); - } - - Path getHttpKeystoreLocation() { - return httpKeystoreLocation.get(); - } - - - Path getTransportKeystoreLocation() { - return transportKeystoreLocation.get(); - } -} diff --git a/data-node/src/main/java/org/graylog/datanode/configuration/variants/UploadedCertFilesSecureConfiguration.java b/data-node/src/main/java/org/graylog/datanode/configuration/variants/UploadedCertFilesSecureConfiguration.java deleted file mode 100644 index b7eb39ff8ba1..000000000000 --- a/data-node/src/main/java/org/graylog/datanode/configuration/variants/UploadedCertFilesSecureConfiguration.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright (C) 2020 Graylog, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * . - */ -package org.graylog.datanode.configuration.variants; - -import jakarta.inject.Inject; -import org.apache.commons.lang3.RandomStringUtils; -import org.graylog.datanode.Configuration; -import org.graylog.datanode.configuration.DatanodeConfiguration; -import org.graylog.datanode.configuration.OpensearchConfigurationException; -import org.graylog.security.certutil.ca.exceptions.KeyStoreStorageException; -import org.graylog.security.certutil.csr.FilesystemKeystoreInformation; -import org.graylog.security.certutil.keystore.storage.KeystoreFileStorage; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.security.GeneralSecurityException; -import java.security.KeyStore; -import java.util.LinkedList; -import java.util.List; -import java.util.Optional; - -import static org.graylog.datanode.Configuration.HTTP_CERTIFICATE_PASSWORD_PROPERTY; -import static org.graylog.datanode.Configuration.TRANSPORT_CERTIFICATE_PASSWORD_PROPERTY; - -public final class UploadedCertFilesSecureConfiguration extends SecureConfiguration { - - private final String uploadedTransportKeystoreFileName; - private final String uploadedHttpKeystoreFileName; - private final String datanodeTransportCertificatePassword; - private final String datanodeHttpCertificatePassword; - private final KeystoreFileStorage keystoreFileStorage; - private final DatanodeConfiguration datanodeConfiguration; - - @Inject - public UploadedCertFilesSecureConfiguration(final Configuration localConfiguration, - final DatanodeConfiguration datanodeConfiguration, - KeystoreFileStorage keystoreFileStorage) { - super(datanodeConfiguration); - this.datanodeConfiguration = datanodeConfiguration; - this.keystoreFileStorage = keystoreFileStorage; - - this.uploadedTransportKeystoreFileName = localConfiguration.getDatanodeTransportCertificate(); - this.uploadedHttpKeystoreFileName = localConfiguration.getDatanodeHttpCertificate(); - - this.datanodeTransportCertificatePassword = localConfiguration.getDatanodeTransportCertificatePassword(); - this.datanodeHttpCertificatePassword = localConfiguration.getDatanodeHttpCertificatePassword(); - } - - @Override - public boolean isConfigured(Configuration localConfiguration) throws OpensearchConfigurationException { - - if (noneOfRequiredConfigOptionsProvided()) { - return false; // none of the uploaded cert options is provided => not usable for this security config, skip this config - } - - List errors = new LinkedList<>(); - - if (isBlank(datanodeTransportCertificatePassword)) { - errors.add(TRANSPORT_CERTIFICATE_PASSWORD_PROPERTY + " required. Please configure password to your transport certificates keystore."); - } - - if (!fileExists(uploadedTransportKeystoreFileName)) { - errors.add("transport_certificate required. Please provide a path to a certificate file in your configuration."); - } - - if (isBlank(datanodeHttpCertificatePassword)) { - errors.add(HTTP_CERTIFICATE_PASSWORD_PROPERTY + " required. Please configure password to your http certificates keystore."); - } - - if (!fileExists(uploadedHttpKeystoreFileName)) { - errors.add("http_certificate required. Please provide a path to a certificate file in your configuration."); - } - - if (!errors.isEmpty()) { - throw new OpensearchConfigurationException("Configuration incomplete, check the following settings: " + String.join(", ", errors)); - } - - return true; - } - - private boolean isBlank(String value) { - return value == null || value.isBlank(); - } - - private boolean fileExists(String filename) { - return Optional.ofNullable(filename) - .flatMap(fileName -> datanodeConfiguration.datanodeDirectories().resolveConfigurationSourceFile(filename)) - .map(Files::exists) - .orElse(false); - } - - /** - * We require either full set of http and transport certificates and their keys or nothing. Anything in-between will - * lead to an exception, it's a mismatched configuration and would cause problems in the future. - */ - private boolean noneOfRequiredConfigOptionsProvided() { - return isBlank(datanodeTransportCertificatePassword) && - isBlank(datanodeHttpCertificatePassword) && - isBlank(uploadedHttpKeystoreFileName) && - isBlank(uploadedTransportKeystoreFileName); - } - - @Override - public OpensearchSecurityConfiguration build() throws KeyStoreStorageException, IOException, GeneralSecurityException { - - final Path targetTransportKeystoreLocation = getTransportKeystoreLocation(); - final Path targetHttpKeystoreLocation = getHttpKeystoreLocation(); - - final char[] transportOTP = reEncyptWithOtp(datanodeConfiguration.datanodeDirectories().resolveConfigurationSourceFile(uploadedTransportKeystoreFileName).orElseThrow(() -> new RuntimeException("This should not happen, certificate expected")), - datanodeTransportCertificatePassword.toCharArray(), - targetTransportKeystoreLocation); - - final char[] httpOTP = reEncyptWithOtp(datanodeConfiguration.datanodeDirectories().resolveConfigurationSourceFile(uploadedHttpKeystoreFileName).orElseThrow(() -> new RuntimeException("This should not happen, certificate expected")), - datanodeHttpCertificatePassword.toCharArray(), - targetHttpKeystoreLocation); - - return new OpensearchSecurityConfiguration( - new FilesystemKeystoreInformation(targetTransportKeystoreLocation.toAbsolutePath(), transportOTP), - new FilesystemKeystoreInformation(targetHttpKeystoreLocation.toAbsolutePath(), httpOTP) - ); - } - - public char[] reEncyptWithOtp(final Path originalLocation, - final char[] originalPassword, - final Path targetLocation - ) throws KeyStoreStorageException { - // caution! This password changes during each configuration request!!! - final char[] oneTimePassword = RandomStringUtils.randomAlphabetic(256).toCharArray(); - reEncypt(originalLocation, originalPassword, targetLocation, oneTimePassword); - return oneTimePassword; - } - - private void reEncypt(final Path originalLocation, - final char[] originalPassword, - final Path targetLocation, - final char[] newPassword - ) throws KeyStoreStorageException { - - final Optional keyStore = keystoreFileStorage.readKeyStore(originalLocation, originalPassword); - if (keyStore.isPresent()) { - final KeyStore originalKeystore = keyStore.get(); - keystoreFileStorage.writeKeyStore(targetLocation, originalKeystore, originalPassword, newPassword); - } else { - throw new KeyStoreStorageException("No keystore present in : " + originalLocation); - } - } -} diff --git a/data-node/src/main/java/org/graylog/datanode/initializers/JerseyService.java b/data-node/src/main/java/org/graylog/datanode/initializers/JerseyService.java index c481d6bb29a8..fe25ebf6b5fa 100644 --- a/data-node/src/main/java/org/graylog/datanode/initializers/JerseyService.java +++ b/data-node/src/main/java/org/graylog/datanode/initializers/JerseyService.java @@ -44,12 +44,10 @@ import org.glassfish.jersey.server.ServerProperties; import org.glassfish.jersey.server.model.Resource; import org.graylog.datanode.Configuration; -import org.graylog.datanode.configuration.variants.OpensearchSecurityConfiguration; import org.graylog.datanode.opensearch.OpensearchConfigurationChangeEvent; import org.graylog.datanode.opensearch.configuration.OpensearchConfiguration; import org.graylog.datanode.rest.config.SecuredNodeAnnotationFilter; -import org.graylog.security.certutil.CertConstants; -import org.graylog.security.certutil.csr.FilesystemKeystoreInformation; +import org.graylog.security.certutil.csr.KeystoreInformation; import org.graylog2.configuration.TLSProtocolsConfiguration; import org.graylog2.plugin.inject.Graylog2Module; import org.graylog2.rest.MoreMediaTypes; @@ -60,10 +58,7 @@ import org.slf4j.LoggerFactory; import javax.net.ssl.SSLContext; -import java.io.IOException; import java.net.URI; -import java.nio.file.Files; -import java.security.GeneralSecurityException; import java.security.KeyStore; import java.util.Map; import java.util.Set; @@ -130,14 +125,10 @@ public synchronized void handleOpensearchConfigurationChange(OpensearchConfigura doStartup(extractSslConfiguration(event.config())); } - private SSLEngineConfigurator extractSslConfiguration(OpensearchConfiguration config) throws GeneralSecurityException, IOException { - final OpensearchSecurityConfiguration securityConfiguration = config.opensearchSecurityConfiguration(); - if (securityConfiguration != null && securityConfiguration.securityEnabled()) { - return buildSslEngineConfigurator(securityConfiguration.getHttpCertificate()); - } else { - return null; - } - + private SSLEngineConfigurator extractSslConfiguration(OpensearchConfiguration config) { + return config.httpCertificate() + .map(this::buildSslEngineConfigurator) + .orElse(null); } @Override @@ -263,35 +254,26 @@ private HttpServer setUp(URI listenUri, return httpServer; } - private SSLEngineConfigurator buildSslEngineConfigurator(FilesystemKeystoreInformation keystoreInformation) - throws GeneralSecurityException, IOException { - if (keystoreInformation == null || !Files.isRegularFile(keystoreInformation.location()) || !Files.isReadable(keystoreInformation.location())) { + private SSLEngineConfigurator buildSslEngineConfigurator(KeystoreInformation keystoreInformation) { + + if (keystoreInformation == null) { throw new IllegalArgumentException("Unreadable to read private key"); } - final SSLContextConfigurator sslContextConfigurator = new SSLContextConfigurator(); final char[] password = firstNonNull(keystoreInformation.password(), new char[]{}); - final KeyStore keyStore = readKeystore(keystoreInformation); - - sslContextConfigurator.setKeyStorePass(password); - sslContextConfigurator.setKeyStoreBytes(KeyStoreUtils.getBytes(keyStore, password)); - - final SSLContext sslContext = sslContextConfigurator.createSSLContext(true); - final SSLEngineConfigurator sslEngineConfigurator = new SSLEngineConfigurator(sslContext, false, false, false); - sslEngineConfigurator.setEnabledProtocols(tlsConfiguration.getEnabledTlsProtocols().toArray(new String[0])); - return sslEngineConfigurator; - } - - private static KeyStore readKeystore(FilesystemKeystoreInformation keystoreInformation) { - LOG.info("Jersey is using keystore located in {}", keystoreInformation.location().toAbsolutePath()); - try (var in = Files.newInputStream(keystoreInformation.location())) { - KeyStore caKeystore = KeyStore.getInstance(CertConstants.PKCS12); - caKeystore.load(in, keystoreInformation.password()); - return caKeystore; - } catch (IOException | GeneralSecurityException ex) { - throw new RuntimeException("Could not read keystore: " + ex.getMessage(), ex); + try { + final KeyStore keyStore = keystoreInformation.loadKeystore(); + sslContextConfigurator.setKeyStorePass(password); + sslContextConfigurator.setKeyStoreBytes(KeyStoreUtils.getBytes(keyStore, password)); + + final SSLContext sslContext = sslContextConfigurator.createSSLContext(true); + final SSLEngineConfigurator sslEngineConfigurator = new SSLEngineConfigurator(sslContext, false, false, false); + sslEngineConfigurator.setEnabledProtocols(tlsConfiguration.getEnabledTlsProtocols().toArray(new String[0])); + return sslEngineConfigurator; + } catch (Exception e) { + throw new RuntimeException("Could not read keystore: " + e.getMessage(), e); } } diff --git a/data-node/src/main/java/org/graylog/datanode/opensearch/OpensearchProcessImpl.java b/data-node/src/main/java/org/graylog/datanode/opensearch/OpensearchProcessImpl.java index 9704815533c0..348c1cf069bc 100644 --- a/data-node/src/main/java/org/graylog/datanode/opensearch/OpensearchProcessImpl.java +++ b/data-node/src/main/java/org/graylog/datanode/opensearch/OpensearchProcessImpl.java @@ -25,7 +25,6 @@ import org.apache.http.client.utils.URIBuilder; import org.graylog.datanode.Configuration; import org.graylog.datanode.configuration.DatanodeConfiguration; -import org.graylog.datanode.configuration.variants.OpensearchSecurityConfiguration; import org.graylog.datanode.opensearch.cli.OpensearchCommandLineProcess; import org.graylog.datanode.opensearch.configuration.OpensearchConfiguration; import org.graylog.datanode.opensearch.rest.OpensearchRestClient; @@ -35,7 +34,6 @@ import org.graylog.datanode.periodicals.ClusterStateResponse; import org.graylog.datanode.process.ProcessInformation; import org.graylog.datanode.process.ProcessListener; -import org.graylog.security.certutil.csr.FilesystemKeystoreInformation; import org.graylog.shaded.opensearch2.org.opensearch.OpenSearchStatusException; import org.graylog.shaded.opensearch2.org.opensearch.action.admin.cluster.health.ClusterHealthRequest; import org.graylog.shaded.opensearch2.org.opensearch.action.admin.cluster.health.ClusterHealthResponse; @@ -50,8 +48,6 @@ import org.graylog.shaded.opensearch2.org.opensearch.client.RestHighLevelClient; import org.graylog.shaded.opensearch2.org.opensearch.common.settings.Settings; import org.graylog.storage.opensearch2.OpenSearchClient; -import org.graylog2.cluster.nodes.DataNodeDto; -import org.graylog2.cluster.nodes.NodeService; import org.graylog2.datanode.DataNodeLifecycleEvent; import org.graylog2.datanode.DataNodeLifecycleTrigger; import org.graylog2.plugin.system.NodeId; @@ -65,28 +61,20 @@ import javax.net.ssl.X509TrustManager; import java.io.IOException; import java.net.URI; -import java.nio.charset.Charset; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.CertificateException; +import java.security.KeyStore; import java.util.List; import java.util.Locale; import java.util.Objects; import java.util.Optional; import java.util.Queue; -import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; public class OpensearchProcessImpl implements OpensearchProcess, ProcessListener { private static final Logger LOG = LoggerFactory.getLogger(OpensearchProcessImpl.class); - public static final Path UNICAST_HOSTS_FILE = Path.of("unicast_hosts.txt"); + @SuppressWarnings("OptionalUsedAsFieldOrParameterType") private Optional opensearchConfiguration = Optional.empty(); @@ -103,7 +91,6 @@ public class OpensearchProcessImpl implements OpensearchProcess, ProcessListener private final Queue stdout; private final Queue stderr; private final CustomCAX509TrustManager trustManager; - private final NodeService nodeService; private final Configuration configuration; private final ObjectMapper objectMapper; private final String nodeName; @@ -117,14 +104,13 @@ public class OpensearchProcessImpl implements OpensearchProcess, ProcessListener @Inject OpensearchProcessImpl(DatanodeConfiguration datanodeConfiguration, final CustomCAX509TrustManager trustManager, - final Configuration configuration, final NodeService nodeService, + final Configuration configuration, ObjectMapper objectMapper, OpensearchStateMachine processState, NodeId nodeId, EventBus eventBus) { this.datanodeConfiguration = datanodeConfiguration; this.processState = processState; this.stdout = new CircularFifoQueue<>(datanodeConfiguration.processLogsBufferSize()); this.stderr = new CircularFifoQueue<>(datanodeConfiguration.processLogsBufferSize()); this.trustManager = trustManager; - this.nodeService = nodeService; this.configuration = configuration; this.objectMapper = objectMapper; this.nodeName = configuration.getDatanodeNodeName(); @@ -134,9 +120,7 @@ public class OpensearchProcessImpl implements OpensearchProcess, ProcessListener private RestHighLevelClient createRestClient(OpensearchConfiguration configuration) { - final TrustManager trustManager = configuration.opensearchSecurityConfiguration().getTruststore() - .map(this::createAggregatedTrustManager) - .orElse(this.trustManager); + final TrustManager trustManager = createAggregatedTrustManager(configuration.trustStore()); return OpensearchRestClient.build(configuration, datanodeConfiguration, trustManager); } @@ -144,16 +128,13 @@ private RestHighLevelClient createRestClient(OpensearchConfiguration configurati /** * We have to combine the system-wide trust manager with a manager that trusts certificates used to secure * the datanode's opensearch process. + * * @param truststore truststore containing certificates used to secure datanode's opensearch * @return combined trust manager */ @Nonnull - private X509TrustManager createAggregatedTrustManager(FilesystemKeystoreInformation truststore) { - try { - return new TrustManagerAggregator(List.of(this.trustManager, TrustManagerAggregator.trustManagerFromKeystore(truststore.loadKeystore()))); - } catch (KeyStoreException | IOException | CertificateException | NoSuchAlgorithmException e) { - throw new RuntimeException(e); - } + private X509TrustManager createAggregatedTrustManager(KeyStore truststore) { + return new TrustManagerAggregator(List.of(this.trustManager, TrustManagerAggregator.trustManagerFromKeystore(truststore))); } @Override @@ -175,7 +156,7 @@ public Optional openSearchClient() { } public OpensearchInfo processInfo() { - return new OpensearchInfo(configuration.getDatanodeNodeName(), processState.getState(), getOpensearchBaseUrl().toString(), commandLineProcess != null ? commandLineProcess.processInfo() : ProcessInformation.empty()); + return new OpensearchInfo(configuration.getDatanodeNodeName(), processState.getState(), getOpensearchBaseUrl().toString(), commandLineProcess != null ? commandLineProcess.processInfo() : ProcessInformation.empty()); } @Override @@ -196,9 +177,7 @@ public String getOpensearchClusterUrl() { @Override public String getDatanodeRestApiUrl() { - final boolean secured = opensearchConfiguration.map(OpensearchConfiguration::opensearchSecurityConfiguration) - .map(OpensearchSecurityConfiguration::securityEnabled) - .orElse(false); + final boolean secured = opensearchConfiguration.flatMap(OpensearchConfiguration::httpCertificate).isPresent(); String protocol = secured ? "https" : "http"; String host = configuration.getHostname(); final int port = configuration.getDatanodeHttpPort(); @@ -225,40 +204,28 @@ private void configure() { (config -> { // refresh TM if the SSL certs changed trustManager.refresh(); - // refresh the seed hosts - writeSeedHostsList(); }), () -> {throw new IllegalArgumentException("Opensearch configuration required but not supplied!");} ); } - private void writeSeedHostsList() { - try { - final Path hostsfile = datanodeConfiguration.datanodeDirectories().createOpensearchProcessConfigurationFile(UNICAST_HOSTS_FILE); - final Set current = nodeService.allActive().values().stream().map(DataNodeDto::getClusterAddress).filter(Objects::nonNull).collect(Collectors.toSet()); - Files.write(hostsfile, current, Charset.defaultCharset(), StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE); - } catch (IOException iox) { - LOG.error("Could not write to file: {} - {}", UNICAST_HOSTS_FILE, iox.getMessage()); - } - - } @Override public synchronized void start() { - opensearchConfiguration.ifPresentOrElse( - (config -> { - boolean startedPreviously = Objects.nonNull(commandLineProcess) && commandLineProcess.processInfo().alive(); - if (startedPreviously) { - stop(); - } - - commandLineProcess = new OpensearchCommandLineProcess(config, this); - commandLineProcess.start(); - - restClient = Optional.of(createRestClient(config)); - openSearchClient = restClient.map(c -> new OpenSearchClient(c, objectMapper)); - }), - () -> {throw new IllegalArgumentException("Opensearch configuration required but not supplied!");} - ); + opensearchConfiguration.ifPresentOrElse( + (config -> { + boolean startedPreviously = Objects.nonNull(commandLineProcess) && commandLineProcess.processInfo().alive(); + if (startedPreviously) { + stop(); + } + + commandLineProcess = new OpensearchCommandLineProcess(config, this); + commandLineProcess.start(); + + restClient = Optional.of(createRestClient(config)); + openSearchClient = restClient.map(c -> new OpenSearchClient(c, objectMapper)); + }), + () -> {throw new IllegalArgumentException("Opensearch configuration required but not supplied!");} + ); } /** diff --git a/data-node/src/main/java/org/graylog/datanode/opensearch/OpensearchProcessService.java b/data-node/src/main/java/org/graylog/datanode/opensearch/OpensearchProcessService.java index 7ccfbbdce333..fb01fe100ea6 100644 --- a/data-node/src/main/java/org/graylog/datanode/opensearch/OpensearchProcessService.java +++ b/data-node/src/main/java/org/graylog/datanode/opensearch/OpensearchProcessService.java @@ -132,7 +132,7 @@ private void onConfigurationChange(OpensearchConfiguration config) { LOG.info("OpenSearch starting up"); checkWritePreflightFinishedOnInsecureStartup(); try { - lockfileCheck.checkDatanodeLock(config.datanodeDirectories().getDataTargetDir()); + lockfileCheck.checkDatanodeLock(config.getDatanodeDirectories().getDataTargetDir()); if (stateMachine.isInState(OpensearchState.WAITING_FOR_CONFIGURATION) && !this.processAutostart) { stateMachine.fire(OpensearchEvent.PROCESS_PREPARED); this.processAutostart = true; // reset to default diff --git a/data-node/src/main/java/org/graylog/datanode/opensearch/cli/OpensearchCli.java b/data-node/src/main/java/org/graylog/datanode/opensearch/cli/OpensearchCli.java index c26bfdeea120..fbdd8cc1d693 100644 --- a/data-node/src/main/java/org/graylog/datanode/opensearch/cli/OpensearchCli.java +++ b/data-node/src/main/java/org/graylog/datanode/opensearch/cli/OpensearchCli.java @@ -30,8 +30,8 @@ public class OpensearchCli { public OpensearchCli(OpensearchConfiguration config) { this( - config.datanodeDirectories().getOpensearchProcessConfigurationDir(), - config.opensearchDistribution().getOpensearchBinDirPath() + config.getOpensearchConfigurationDir().configurationRoot(), + config.getOpensearchDistribution().getOpensearchBinDirPath() ); } diff --git a/data-node/src/main/java/org/graylog/datanode/opensearch/cli/OpensearchCommandLineProcess.java b/data-node/src/main/java/org/graylog/datanode/opensearch/cli/OpensearchCommandLineProcess.java index eae2377ac07c..a020e3bad86c 100644 --- a/data-node/src/main/java/org/graylog/datanode/opensearch/cli/OpensearchCommandLineProcess.java +++ b/data-node/src/main/java/org/graylog/datanode/opensearch/cli/OpensearchCommandLineProcess.java @@ -16,8 +16,6 @@ */ package org.graylog.datanode.opensearch.cli; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.github.rholder.retry.Attempt; import com.github.rholder.retry.RetryException; import com.github.rholder.retry.RetryListener; @@ -26,22 +24,25 @@ import com.github.rholder.retry.WaitStrategies; import jakarta.validation.constraints.NotNull; import org.apache.commons.exec.OS; +import org.graylog.datanode.configuration.OpensearchConfigurationDir; +import org.graylog.datanode.configuration.OpensearchConfigurationException; import org.graylog.datanode.opensearch.configuration.OpensearchConfiguration; import org.graylog.datanode.process.CommandLineProcess; import org.graylog.datanode.process.CommandLineProcessListener; import org.graylog.datanode.process.ProcessInformation; import org.graylog.datanode.process.ProcessListener; +import org.graylog.datanode.process.configuration.files.DatanodeConfigFile; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.BufferedReader; import java.io.Closeable; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; @@ -52,8 +53,7 @@ public class OpensearchCommandLineProcess implements Closeable { private final CommandLineProcess commandLineProcess; private final CommandLineProcessListener resultHandler; - private final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); - private static final Path CONFIG = Path.of("opensearch.yml"); + /** * as long as OpenSearch is not supported on macOS, we have to fix the jdk path if we want to @@ -63,7 +63,7 @@ public class OpensearchCommandLineProcess implements Closeable { */ private void fixJdkOnMac(final OpensearchConfiguration config) { final var isMacOS = OS.isFamilyMac(); - final var jdk = config.opensearchDistribution().directory().resolve("jdk.app"); + final var jdk = config.getOpensearchDistribution().directory().resolve("jdk.app"); final var jdkNotLinked = !Files.exists(jdk); if (isMacOS && jdkNotLinked) { // Link System jdk into startup folder, get path: @@ -90,18 +90,25 @@ private void fixJdkOnMac(final OpensearchConfiguration config) { } private void writeOpenSearchConfig(final OpensearchConfiguration config) { + final OpensearchConfigurationDir confDir = config.getOpensearchConfigurationDir(); + config.configFiles().forEach(cf -> persistConfigFile(confDir, cf)); + } + + private static void persistConfigFile(OpensearchConfigurationDir confDir, DatanodeConfigFile cf) { try { - final Path configFile = config.datanodeDirectories().createOpensearchProcessConfigurationFile(CONFIG); - mapper.writeValue(configFile.toFile(), getOpensearchConfigurationArguments(config)); + final Path targetFile = confDir.createOpensearchProcessConfigurationFile(cf.relativePath()); + try (final FileOutputStream file = new FileOutputStream(targetFile.toFile())) { + cf.write(file); + } } catch (IOException e) { - throw new RuntimeException("Could not generate OpenSearch config: " + e.getMessage(), e); + throw new OpensearchConfigurationException("Failed to create opensearch config file " + cf.relativePath(), e); } } public OpensearchCommandLineProcess(OpensearchConfiguration config, ProcessListener listener) { fixJdkOnMac(config); configureOpensearchKeystoreSecrets(config); - final Path executable = config.opensearchDistribution().getOpensearchExecutable(); + final Path executable = config.getOpensearchDistribution().getOpensearchExecutable(); writeOpenSearchConfig(config); resultHandler = new CommandLineProcessListener(listener); commandLineProcess = new CommandLineProcess(executable, List.of(), resultHandler, config.getEnv()); @@ -117,20 +124,6 @@ private void configureOpensearchKeystoreSecrets(OpensearchConfiguration config) LOG.info("Added {} keystore items", keystoreItems.size()); } - - private static Map getOpensearchConfigurationArguments(OpensearchConfiguration config) { - Map allArguments = new LinkedHashMap<>(config.asMap()); - - // now copy all the environment values to the configuration arguments. Opensearch won't do it for us, - // because we are using tar distriburion and opensearch does this only for docker dist. See opensearch-env script - // additionally, the env variables have to be prefixed with opensearch. (e.g. "opensearch.cluster.routing.allocation.disk.threshold_enabled") - config.getEnv().getEnv().entrySet().stream() - .filter(entry -> entry.getKey().matches("^opensearch\\.[a-z0-9_]+(?:\\.[a-z0-9_]+)+")) - .peek(entry -> LOG.info("Detected pass-through opensearch property {}:{}", entry.getKey().substring("opensearch.".length()), entry.getValue())) - .forEach(entry -> allArguments.put(entry.getKey().substring("opensearch.".length()), entry.getValue())); - return allArguments; - } - public void start() { commandLineProcess.start(); } diff --git a/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/OpensearchConfiguration.java b/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/OpensearchConfiguration.java index b6dcc4a0143e..9dab37bc4b77 100644 --- a/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/OpensearchConfiguration.java +++ b/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/OpensearchConfiguration.java @@ -16,140 +16,159 @@ */ package org.graylog.datanode.opensearch.configuration; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import jakarta.annotation.Nonnull; -import org.apache.commons.exec.OS; import org.graylog.datanode.OpensearchDistribution; import org.graylog.datanode.configuration.DatanodeDirectories; -import org.graylog.datanode.configuration.S3RepositoryConfiguration; -import org.graylog.datanode.configuration.variants.OpensearchSecurityConfiguration; -import org.graylog.datanode.opensearch.configuration.beans.OpensearchConfigurationPart; +import org.graylog.datanode.configuration.OpensearchConfigurationDir; +import org.graylog.datanode.process.configuration.beans.DatanodeConfigurationPart; +import org.graylog.datanode.process.configuration.files.DatanodeConfigFile; +import org.graylog.datanode.process.configuration.files.YamlConfigFile; import org.graylog.datanode.process.Environment; +import org.graylog.security.certutil.csr.KeystoreInformation; import org.graylog.shaded.opensearch2.org.apache.http.HttpHost; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.nio.file.Path; +import java.security.KeyStore; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Set; +import java.util.Objects; +import java.util.Optional; import java.util.stream.Collectors; -import java.util.stream.Stream; - -public record OpensearchConfiguration( - OpensearchDistribution opensearchDistribution, - DatanodeDirectories datanodeDirectories, - String bindAddress, - String hostname, - int httpPort, - int transportPort, - String clusterName, - String nodeName, - List nodeRoles, - List discoverySeedHosts, - OpensearchSecurityConfiguration opensearchSecurityConfiguration, - Set configurationParts, - - Map additionalConfiguration -) { - public Map asMap() { - Map config = new LinkedHashMap<>(); - - config.put("action.auto_create_index", "false"); - - // currently, startup fails on macOS without disabling this filter. - // for a description of the filter (although it's for ES), see https://www.elastic.co/guide/en/elasticsearch/reference/7.17/_system_call_filter_check.html - if (OS.isFamilyMac()) { - config.put("bootstrap.system_call_filter", "false"); - } - - if (bindAddress != null && !bindAddress.isBlank()) { - config.put("network.host", bindAddress); - } - config.put("http.port", String.valueOf(httpPort)); - config.put("transport.port", String.valueOf(transportPort)); - if (clusterName != null && !clusterName.isBlank()) { - config.put("cluster.name", clusterName); - } - - config.put("node.name", nodeName); - config.put("node.roles", buildRolesList()); +public class OpensearchConfiguration { - if (discoverySeedHosts != null && !discoverySeedHosts.isEmpty()) { - config.put("discovery.seed_hosts", toValuesList(discoverySeedHosts)); - } + private static final Logger LOG = LoggerFactory.getLogger(OpensearchConfiguration.class); - config.put("discovery.seed_providers", "file"); + private final OpensearchDistribution opensearchDistribution; + private final String hostname; + private final int httpPort; + private final List configurationParts; + private final OpensearchConfigurationDir opensearchConfigurationDir; + private final DatanodeDirectories datanodeDirectories; - configurationParts.stream() - .map(OpensearchConfigurationPart::properties) - .forEach(config::putAll); - - config.putAll(additionalConfiguration); - return config; + public OpensearchConfiguration(OpensearchDistribution opensearchDistribution, DatanodeDirectories datanodeDirectories, String hostname, int httpPort, List configurationParts) { + this.opensearchDistribution = opensearchDistribution; + this.hostname = hostname; + this.httpPort = httpPort; + this.configurationParts = configurationParts; + this.datanodeDirectories = datanodeDirectories; + this.opensearchConfigurationDir = datanodeDirectories.createUniqueOpensearchProcessConfigurationDir(); } @Nonnull private String buildRolesList() { - final ImmutableList.Builder roles = ImmutableList.builder(); - if (nodeRoles != null) { - roles.addAll(nodeRoles); - } - configurationParts.stream() - .map(OpensearchConfigurationPart::nodeRoles) - .forEach(roles::addAll); - - return toValuesList(roles.build()); - } - - private String toValuesList(List values) { - return String.join(",", values); + return configurationParts.stream() + .flatMap(cfg -> cfg.nodeRoles().stream()) + .collect(Collectors.joining(",")); } public Environment getEnv() { final Environment env = new Environment(System.getenv()); List javaOpts = new LinkedList<>(); - javaOpts.add("-Xms%s".formatted(opensearchSecurityConfiguration.getOpensearchHeap())); - javaOpts.add("-Xmx%s".formatted(opensearchSecurityConfiguration.getOpensearchHeap())); - javaOpts.add("-Dopensearch.transport.cname_in_publish_address=true"); - opensearchSecurityConfiguration.getTruststore().ifPresent(truststore -> { - javaOpts.add("-Djavax.net.ssl.trustStore=" + truststore.location().toAbsolutePath()); - javaOpts.add("-Djavax.net.ssl.trustStorePassword=" + new String(truststore.password())); - javaOpts.add("-Djavax.net.ssl.trustStoreType=pkcs12"); - }); + configurationParts.stream().map(DatanodeConfigurationPart::javaOpts) + .forEach(javaOpts::addAll); env.put("OPENSEARCH_JAVA_OPTS", String.join(" ", javaOpts)); - env.put("OPENSEARCH_PATH_CONF", datanodeDirectories.getOpensearchProcessConfigurationDir().toString()); + env.put("OPENSEARCH_PATH_CONF", opensearchConfigurationDir.configurationRoot().toString()); return env; } public HttpHost getRestBaseUrl() { - final boolean sslEnabled = Boolean.parseBoolean(asMap().getOrDefault("plugins.security.ssl.http.enabled", "false").toString()); - return new HttpHost(hostname(), httpPort(), sslEnabled ? "https" : "http"); + return new HttpHost(hostname, httpPort, isHttpsEnabled() ? "https" : "http"); } - public HttpHost getClusterBaseUrl() { - final boolean sslEnabled = Boolean.parseBoolean(asMap().getOrDefault("plugins.security.ssl.http.enabled", "false").toString()); - return new HttpHost(hostname(), transportPort(), sslEnabled ? "https" : "http"); + public boolean isHttpsEnabled() { + return httpCertificate().isPresent(); } + /** + * Are there any {@link org.graylog.datanode.configuration.variants.OpensearchCertificatesProvider} configured? + */ public boolean securityConfigured() { - return opensearchSecurityConfiguration() != null; + return configurationParts.stream().anyMatch(DatanodeConfigurationPart::securityConfigured); } public Map getKeystoreItems() { - final ImmutableMap.Builder builder = ImmutableMap.builder(); - builder.putAll(opensearchSecurityConfiguration.getKeystoreItems()); - configurationParts.stream() - .map(OpensearchConfigurationPart::keystoreItems) + .map(DatanodeConfigurationPart::keystoreItems) .forEach(builder::putAll); return builder.build(); } + + public KeyStore trustStore() { + return configurationParts.stream() + .map(DatanodeConfigurationPart::trustStore) + .filter(Objects::nonNull) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("This should not happen, truststore should always be present")); + } + + public Optional httpCertificate() { + return configurationParts.stream() + .map(DatanodeConfigurationPart::httpCertificate) + .filter(Objects::nonNull) + .findFirst(); + } + + public Optional transportCertificate() { + return configurationParts.stream() + .map(DatanodeConfigurationPart::transportCertificate) + .filter(Objects::nonNull) + .findFirst(); + } + + public List configFiles() { + + final List configFiles = new LinkedList<>(); + + configurationParts.stream() + .flatMap(cp -> cp.configFiles().stream()) + .forEach(configFiles::add); + + configFiles.add(new YamlConfigFile(Path.of("opensearch.yml"), opensearchYmlConfig())); + + return configFiles; + } + + private Map opensearchYmlConfig() { + Map config = new LinkedHashMap<>(); + + // this needs special treatment as it's as an aggregation of other configuration parts + config.put("node.roles", buildRolesList()); + + configurationParts.stream() + .map(DatanodeConfigurationPart::properties) + .forEach(config::putAll); + + // now copy all the environment values to the configuration arguments. Opensearch won't do it for us, + // because we are using tar distriburion and opensearch does this only for docker dist. See opensearch-env script + // additionally, the env variables have to be prefixed with opensearch. (e.g. "opensearch.cluster.routing.allocation.disk.threshold_enabled") + getEnv().getEnv().entrySet().stream() + .filter(entry -> entry.getKey().matches("^opensearch\\.[a-z0-9_]+(?:\\.[a-z0-9_]+)+")) + .peek(entry -> LOG.info("Detected pass-through opensearch property {}:{}", entry.getKey().substring("opensearch.".length()), entry.getValue())) + .forEach(entry -> config.put(entry.getKey().substring("opensearch.".length()), entry.getValue())); + return config; + } + + public OpensearchDistribution getOpensearchDistribution() { + return opensearchDistribution; + } + + public OpensearchConfigurationDir getOpensearchConfigurationDir() { + return opensearchConfigurationDir; + } + + public DatanodeDirectories getDatanodeDirectories() { + return datanodeDirectories; + } } diff --git a/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/OpensearchConfigurationParams.java b/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/OpensearchConfigurationParams.java new file mode 100644 index 000000000000..675020249399 --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/OpensearchConfigurationParams.java @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.opensearch.configuration; + + +import org.graylog.datanode.process.configuration.beans.ConfigurationBuildParams; + +import java.security.cert.X509Certificate; +import java.util.List; + +public record OpensearchConfigurationParams(List trustedCertificates, + java.util.Map transientConfiguration) implements ConfigurationBuildParams { +} diff --git a/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/OpensearchConfigurationPart.java b/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/OpensearchConfigurationPart.java deleted file mode 100644 index bfe2986a6083..000000000000 --- a/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/OpensearchConfigurationPart.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (C) 2020 Graylog, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * . - */ -package org.graylog.datanode.opensearch.configuration.beans; - -import com.google.auto.value.AutoValue; -import com.google.common.collect.ImmutableList; - -import java.util.Collections; -import java.util.List; -import java.util.Map; - -@AutoValue -public abstract class OpensearchConfigurationPart { - public abstract List nodeRoles(); - - public abstract Map keystoreItems(); - - public abstract Map properties(); - - public static Builder builder() { - return new AutoValue_OpensearchConfigurationPart.Builder() - .nodeRoles(Collections.emptyList()) - .keystoreItems(Collections.emptyMap()) - .properties(Collections.emptyMap()); - } - - @AutoValue.Builder - public abstract static class Builder { - - public abstract Builder nodeRoles(List nodeRoles); - - abstract ImmutableList.Builder nodeRolesBuilder(); - - public final Builder addNodeRole(String role) { - nodeRolesBuilder().add(role); - return this; - } - - public abstract Builder keystoreItems(Map keystoreItems); - - public abstract Builder properties(Map properties); - - public abstract OpensearchConfigurationPart build(); - } -} diff --git a/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/OpensearchClusterConfigurationBean.java b/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/OpensearchClusterConfigurationBean.java new file mode 100644 index 000000000000..1e8de037a67a --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/OpensearchClusterConfigurationBean.java @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.opensearch.configuration.beans.impl; + +import com.google.common.collect.ImmutableMap; +import jakarta.inject.Inject; +import org.graylog.datanode.Configuration; +import org.graylog.datanode.opensearch.configuration.OpensearchConfigurationParams; +import org.graylog.datanode.process.configuration.beans.DatanodeConfigurationBean; +import org.graylog.datanode.process.configuration.beans.DatanodeConfigurationPart; +import org.graylog.datanode.process.configuration.files.TextConfigFile; +import org.graylog2.cluster.Node; +import org.graylog2.cluster.nodes.DataNodeDto; +import org.graylog2.cluster.nodes.NodeService; + +import java.nio.file.Path; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +public class OpensearchClusterConfigurationBean implements DatanodeConfigurationBean { + + public static final Path UNICAST_HOSTS_FILE = Path.of("unicast_hosts.txt"); + + private final Configuration localConfiguration; + private final NodeService nodeService; + + @Inject + public OpensearchClusterConfigurationBean(Configuration localConfiguration, NodeService nodeService) { + this.localConfiguration = localConfiguration; + this.nodeService = nodeService; + } + + @Override + public DatanodeConfigurationPart buildConfigurationPart(OpensearchConfigurationParams trustedCertificates) { + ImmutableMap.Builder properties = ImmutableMap.builder(); + + properties.put("network.bind_host", localConfiguration.getBindAddress()); + properties.put("network.publish_host", localConfiguration.getHostname()); + + if (localConfiguration.getClustername() != null && !localConfiguration.getClustername().isBlank()) { + properties.put("cluster.name", localConfiguration.getClustername()); + } + + if (localConfiguration.getBindAddress() != null && !localConfiguration.getBindAddress().isBlank()) { + properties.put("network.host", localConfiguration.getBindAddress()); + } + properties.put("http.port", String.valueOf(localConfiguration.getOpensearchHttpPort())); + properties.put("transport.port", String.valueOf(localConfiguration.getOpensearchTransportPort())); + + properties.put("node.name", localConfiguration.getDatanodeNodeName()); + + if (localConfiguration.getInitialClusterManagerNodes() != null && !localConfiguration.getInitialClusterManagerNodes().isBlank()) { + properties.put("cluster.initial_cluster_manager_nodes", localConfiguration.getInitialClusterManagerNodes()); + } else { + final var nodeList = String.join(",", nodeService.allActive().values().stream().map(Node::getHostname).collect(Collectors.toSet())); + properties.put("cluster.initial_cluster_manager_nodes", nodeList); + } + + final List discoverySeedHosts = localConfiguration.getOpensearchDiscoverySeedHosts(); + if (discoverySeedHosts != null && !discoverySeedHosts.isEmpty()) { + properties.put("discovery.seed_hosts", String.join(",", discoverySeedHosts)); + } + + properties.put("discovery.seed_providers", "file"); + + // TODO: why do we have this configured? + properties.put("node.max_local_storage_nodes", "3"); + + return DatanodeConfigurationPart.builder() + .properties(properties.build()) + .withConfigFile(seedHostFile()) + .build(); + } + + private TextConfigFile seedHostFile() { + final String data = nodeService.allActive().values().stream() + .map(DataNodeDto::getClusterAddress) + .filter(Objects::nonNull) + .collect(Collectors.joining("\n")); + return new TextConfigFile(UNICAST_HOSTS_FILE, data); + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/OpensearchCommonConfigurationBean.java b/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/OpensearchCommonConfigurationBean.java new file mode 100644 index 000000000000..6705e69244fa --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/OpensearchCommonConfigurationBean.java @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.opensearch.configuration.beans.impl; + +import com.google.common.collect.ImmutableMap; +import jakarta.inject.Inject; +import org.apache.commons.exec.OS; +import org.graylog.datanode.Configuration; +import org.graylog.datanode.configuration.DatanodeConfiguration; +import org.graylog.datanode.opensearch.configuration.OpensearchConfigurationParams; +import org.graylog.datanode.process.configuration.beans.DatanodeConfigurationBean; +import org.graylog.datanode.process.configuration.beans.DatanodeConfigurationPart; + +import java.util.Map; + +public class OpensearchCommonConfigurationBean implements DatanodeConfigurationBean { + + private final Configuration localConfiguration; + private final DatanodeConfiguration datanodeConfiguration; + + @Inject + public OpensearchCommonConfigurationBean(Configuration localConfiguration, DatanodeConfiguration datanodeConfiguration) { + this.localConfiguration = localConfiguration; + this.datanodeConfiguration = datanodeConfiguration; + } + + @Override + public DatanodeConfigurationPart buildConfigurationPart(OpensearchConfigurationParams buildParams) { + return DatanodeConfigurationPart.builder() + .properties(commonOpensearchConfig(buildParams)) + .nodeRoles(localConfiguration.getNodeRoles()) + .javaOpt("-Xms%s".formatted(datanodeConfiguration.opensearchHeap())) + .javaOpt("-Xmx%s".formatted(datanodeConfiguration.opensearchHeap())) + .javaOpt("-Dopensearch.transport.cname_in_publish_address=true") + .build(); + } + + private Map commonOpensearchConfig(OpensearchConfigurationParams buildParams) { + final ImmutableMap.Builder config = ImmutableMap.builder(); + localConfiguration.getOpensearchNetworkHost().ifPresent( + networkHost -> config.put("network.host", networkHost)); + config.put("path.data", datanodeConfiguration.datanodeDirectories().getDataTargetDir().toString()); + config.put("path.logs", datanodeConfiguration.datanodeDirectories().getLogsTargetDir().toString()); + + if (localConfiguration.getOpensearchDebug() != null && !localConfiguration.getOpensearchDebug().isBlank()) { + config.put("logger.org.opensearch", localConfiguration.getOpensearchDebug()); + } + + // common OpenSearch config parameters from our docs + config.put("indices.query.bool.max_clause_count", localConfiguration.getIndicesQueryBoolMaxClauseCount().toString()); + + config.put("action.auto_create_index", "false"); + + // currently, startup fails on macOS without disabling this filter. + // for a description of the filter (although it's for ES), see https://www.elastic.co/guide/en/elasticsearch/reference/7.17/_system_call_filter_check.html + if (OS.isFamilyMac()) { + config.put("bootstrap.system_call_filter", "false"); + } + + config.putAll(buildParams.transientConfiguration()); + + return config.build(); + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/OpensearchDefaultConfigFilesBean.java b/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/OpensearchDefaultConfigFilesBean.java new file mode 100644 index 000000000000..ea35d8643149 --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/OpensearchDefaultConfigFilesBean.java @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.opensearch.configuration.beans.impl; + +import jakarta.annotation.Nonnull; +import org.graylog.datanode.opensearch.configuration.OpensearchConfigurationParams; +import org.graylog.datanode.process.configuration.beans.DatanodeConfigurationBean; +import org.graylog.datanode.process.configuration.beans.DatanodeConfigurationPart; +import org.graylog.datanode.process.configuration.files.DatanodeConfigFile; +import org.graylog.datanode.process.configuration.files.InputStreamConfigFile; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; + +public class OpensearchDefaultConfigFilesBean implements DatanodeConfigurationBean { + + @Override + public DatanodeConfigurationPart buildConfigurationPart(OpensearchConfigurationParams trustedCertificates) { + return DatanodeConfigurationPart.builder() + .configFiles(collectConfigFiles()) + .build(); + } + + private List collectConfigFiles() { + // this is a directory in main/resources that holds all the initial configuration files needed by the opensearch + // we manage this directory in git. Generally we assume that this is a read-only location and we need to copy + // its content to a read-write location for the managed opensearch process. + // This copy happens during each opensearch process start and will override any files that already exist + // from previous runs. + final Path sourceOfInitialConfiguration = Path.of("opensearch", "config"); + try { + return synchronizeConfig(sourceOfInitialConfiguration); + } catch (URISyntaxException | IOException e) { + throw new RuntimeException(e); + } + } + + public List synchronizeConfig(Path configRelativePath) throws URISyntaxException, IOException { + final URI uriToConfig = OpensearchDefaultConfigFilesBean.class.getResource("/" + configRelativePath.toString()).toURI(); + if ("jar".equals(uriToConfig.getScheme())) { + return copyFromJar(configRelativePath, uriToConfig); + } else { + return copyFromLocalFs(configRelativePath); + } + } + + private static List copyFromJar(Path configRelativePath, URI uri) throws IOException { + try ( + final FileSystem fs = FileSystems.newFileSystem(uri, Collections.emptyMap()); + ) { + // Get hold of the path to the top level directory of the JAR file + final Path resourcesRoot = fs.getPath("/"); + final Path source = resourcesRoot.resolve(configRelativePath.toString()); // caution, the toString is needed here to resolve properly! + return collectRecursively(source); + } + } + + private static List copyFromLocalFs(Path configRelativePath) throws URISyntaxException, IOException { + final Path resourcesRoot = Paths.get(OpensearchDefaultConfigFilesBean.class.getResource("/").toURI()); + final Path source = resourcesRoot.resolve(configRelativePath); + return collectRecursively(source); + } + + private static List collectRecursively(Path source) throws IOException { + List configFiles = new LinkedList<>(); + Files.walkFileTree(source, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path sourceFile, BasicFileAttributes attrs) { + final Path relativePath = source.relativize(sourceFile); + // the relative path may come from a different provider than we'll use later, triggering ProviderMismatchException + // We need to disconnect it from the existing provider and use the default one. We can't use relative paths from zip/jar + // to create configuration files on a local file system. + final Path relativePathWithoutProvider = Path.of(relativePath.toString()); + try { + final ByteArrayInputStream stream = copyToMemory(sourceFile); + configFiles.add(new InputStreamConfigFile(relativePathWithoutProvider, stream)); + } catch (IOException e) { + throw new RuntimeException(e); + } + return FileVisitResult.CONTINUE; + } + }); + return configFiles; + } + + @Nonnull + private static ByteArrayInputStream copyToMemory(Path sourceFile) throws IOException { + try (final InputStream inputStream = Files.newInputStream(sourceFile)) { + ByteArrayOutputStream memory = new ByteArrayOutputStream(); + inputStream.transferTo(memory); + return new ByteArrayInputStream(memory.toByteArray()); + } + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/OpensearchSecurityConfigurationBean.java b/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/OpensearchSecurityConfigurationBean.java new file mode 100644 index 000000000000..f026d2bda49f --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/OpensearchSecurityConfigurationBean.java @@ -0,0 +1,239 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.opensearch.configuration.beans.impl; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import jakarta.inject.Inject; +import org.apache.commons.lang3.RandomStringUtils; +import org.graylog.datanode.Configuration; +import org.graylog.datanode.configuration.OpensearchConfigurationException; +import org.graylog.datanode.configuration.TruststoreCreator; +import org.graylog.datanode.configuration.variants.OpensearchCertificates; +import org.graylog.datanode.configuration.variants.OpensearchCertificatesProvider; +import org.graylog.datanode.opensearch.configuration.OpensearchConfigurationParams; +import org.graylog.datanode.process.configuration.beans.DatanodeConfigurationBean; +import org.graylog.datanode.process.configuration.beans.DatanodeConfigurationPart; +import org.graylog.datanode.process.configuration.files.KeystoreConfigFile; +import org.graylog.datanode.process.configuration.files.OpensearchSecurityConfigurationFile; +import org.graylog.security.certutil.CertConstants; +import org.graylog.security.certutil.csr.KeystoreInformation; +import org.graylog2.security.JwtSecret; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Path; +import java.security.GeneralSecurityException; +import java.security.KeyStore; +import java.security.cert.Certificate; +import java.security.cert.X509Certificate; +import java.util.Enumeration; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +public class OpensearchSecurityConfigurationBean implements DatanodeConfigurationBean { + + private static final Logger LOG = LoggerFactory.getLogger(OpensearchSecurityConfigurationBean.class); + + private static final String KEYSTORE_FORMAT = "PKCS12"; + private static final String TRUSTSTORE_FORMAT = "PKCS12"; + + /** + * This filename is used only internally - we copy user-provided certificates to this location and + * we configure opensearch to read this file. It doesn't have to match naming provided by user. + * The target configuration is regenerated during each startup, so it could also be a random filename + * as long as we use the same name as a copy-target and opensearch config property. + */ + private static final String TARGET_DATANODE_HTTP_KEYSTORE_FILENAME = "http-keystore.p12"; + /** + * This filename is used only internally - we copy user-provided certificates to this location and + * we configure opensearch to read this file. It doesn't have to match naming provided by user. + * The target configuration is regenerated during each startup, so it could also be a random filename + * as long as we use the same name as a copy-target and opensearch config property. + */ + private static final String TARGET_DATANODE_TRANSPORT_KEYSTORE_FILENAME = "transport-keystore.p12"; + + private static final Path TRUSTSTORE_FILE = Path.of("datanode-truststore.p12"); + + private final Set opensearchCertificatesProviders; + private final Configuration localConfiguration; + private final JwtSecret jwtSecret; + + @Inject + public OpensearchSecurityConfigurationBean(Set opensearchCertificatesProviders, + final Configuration localConfiguration, + final JwtSecret jwtSecret) { + this.opensearchCertificatesProviders = opensearchCertificatesProviders; + this.localConfiguration = localConfiguration; + this.jwtSecret = jwtSecret; + } + + @Override + public DatanodeConfigurationPart buildConfigurationPart(OpensearchConfigurationParams opensearchConfigurationParams) { + + final DatanodeConfigurationPart.Builder configurationBuilder = DatanodeConfigurationPart.builder(); + + Optional opensearchCertificates = opensearchCertificatesProviders.stream() + .filter(s -> s.isConfigured(localConfiguration)) + .findFirst() + .map(OpensearchCertificatesProvider::build); + + configurationBuilder.securityConfigured(opensearchCertificates.isPresent()); // Caution, this may include insecure_startup config with no certs! + + final String truststorePassword = RandomStringUtils.randomAlphabetic(256); + + final TruststoreCreator truststoreCreator = TruststoreCreator.newDefaultJvm() + .addCertificates(opensearchConfigurationParams.trustedCertificates()); + + final Optional httpCert = opensearchCertificates + .map(OpensearchCertificates::getHttpCertificate); + + final Optional transportCert = opensearchCertificates + .map(OpensearchCertificates::getTransportCertificate); + + httpCert.ifPresent(cert -> { + try { + configurationBuilder.httpCertificate(cert); + configurationBuilder.withConfigFile(new KeystoreConfigFile(Path.of(TARGET_DATANODE_HTTP_KEYSTORE_FILENAME), cert)); + truststoreCreator.addRootCert("http-cert", cert, CertConstants.DATANODE_KEY_ALIAS); + logCertificateInformation("HTTP certificate", cert); + } catch (GeneralSecurityException e) { + throw new RuntimeException(e); + } + }); + + transportCert.ifPresent(cert -> { + try { + configurationBuilder.transportCertificate(cert); + configurationBuilder.withConfigFile(new KeystoreConfigFile(Path.of(TARGET_DATANODE_TRANSPORT_KEYSTORE_FILENAME), cert)); + truststoreCreator.addRootCert("transport-cert", cert, CertConstants.DATANODE_KEY_ALIAS); + logCertificateInformation("Transport certificate", cert); + } catch (GeneralSecurityException e) { + throw new RuntimeException(e); + } + }); + + return configurationBuilder + .properties(properties(httpCert, transportCert)) + .keystoreItems(keystoreItems(truststorePassword, httpCert, transportCert)) + .javaOpts(javaOptions(truststorePassword)) + .trustStore(truststoreCreator.getTruststore()) + .withConfigFile(new KeystoreConfigFile(TRUSTSTORE_FILE, truststoreCreator.toKeystoreInformation(truststorePassword.toCharArray()))) + .withConfigFile(new OpensearchSecurityConfigurationFile(jwtSecret)) + .build(); + } + + + private Map properties(Optional httpCert, Optional transportCert) { + final ImmutableMap.Builder config = ImmutableMap.builder(); + + if (localConfiguration.getOpensearchAuditLog() != null && !localConfiguration.getOpensearchAuditLog().isBlank()) { + config.put("plugins.security.audit.type", localConfiguration.getOpensearchAuditLog()); + } + + // enable admin access via the REST API + config.put("plugins.security.restapi.admin.enabled", "true"); + + + if (httpCert.isPresent() && transportCert.isPresent()) { + config.putAll(commonSecurityConfig()); + + config.put("plugins.security.ssl.transport.keystore_type", KEYSTORE_FORMAT); + config.put("plugins.security.ssl.transport.keystore_filepath", TARGET_DATANODE_TRANSPORT_KEYSTORE_FILENAME); + config.put("plugins.security.ssl.transport.keystore_alias", CertConstants.DATANODE_KEY_ALIAS); + + config.put("plugins.security.ssl.transport.truststore_type", TRUSTSTORE_FORMAT); + config.put("plugins.security.ssl.transport.truststore_filepath", TRUSTSTORE_FILE.toString()); + + config.put("plugins.security.ssl.http.enabled", "true"); + + config.put("plugins.security.ssl.http.keystore_type", KEYSTORE_FORMAT); + config.put("plugins.security.ssl.http.keystore_filepath", TARGET_DATANODE_HTTP_KEYSTORE_FILENAME); + config.put("plugins.security.ssl.http.keystore_alias", CertConstants.DATANODE_KEY_ALIAS); + + config.put("plugins.security.ssl.http.truststore_type", TRUSTSTORE_FORMAT); + config.put("plugins.security.ssl.http.truststore_filepath", TRUSTSTORE_FILE.toString()); + + // enable client cert auth + config.put("plugins.security.ssl.http.clientauth_mode", "OPTIONAL"); + } else { + config.put("plugins.security.disabled", "true"); + config.put("plugins.security.ssl.http.enabled", "false"); + } + return config.build(); + } + + private List javaOptions(String truststorePassword) { + final ImmutableList.Builder builder = ImmutableList.builder(); + builder.add("-Djavax.net.ssl.trustStore=" + TRUSTSTORE_FILE); + builder.add("-Djavax.net.ssl.trustStorePassword=" + truststorePassword); + builder.add("-Djavax.net.ssl.trustStoreType=pkcs12"); + return builder.build(); + } + + private Map commonSecurityConfig() { + final ImmutableMap.Builder config = ImmutableMap.builder(); + config.put("plugins.security.disabled", "false"); + + config.put("plugins.security.nodes_dn", "CN=*"); + config.put("plugins.security.allow_default_init_securityindex", "true"); + //config.put("plugins.security.authcz.admin_dn", "CN=kirk,OU=client,O=client,L=test,C=de"); + + config.put("plugins.security.enable_snapshot_restore_privilege", "true"); + config.put("plugins.security.check_snapshot_restore_write_privileges", "true"); + config.put("plugins.security.restapi.roles_enabled", "all_access,security_rest_api_access,readall"); + config.put("plugins.security.system_indices.enabled", "true"); + config.put("plugins.security.system_indices.indices", ".plugins-ml-model,.plugins-ml-task,.opendistro-alerting-config,.opendistro-alerting-alert*,.opendistro-anomaly-results*,.opendistro-anomaly-detector*,.opendistro-anomaly-checkpoints,.opendistro-anomaly-detection-state,.opendistro-reports-*,.opensearch-notifications-*,.opensearch-notebooks,.opensearch-observability,.opendistro-asynchronous-search-response*,.replication-metadata-store"); + + return config.build(); + } + + private Map keystoreItems(String truststorePassword, Optional httpCert, Optional transportCert) { + final ImmutableMap.Builder config = ImmutableMap.builder(); + config.put("plugins.security.ssl.transport.truststore_password_secure", new String(truststorePassword)); + config.put("plugins.security.ssl.http.truststore_password_secure", new String(truststorePassword)); + httpCert.ifPresent(c -> config.put("plugins.security.ssl.http.keystore_password_secure", new String(c.password()))); + transportCert.ifPresent(c -> config.put("plugins.security.ssl.transport.keystore_password_secure", new String(c.password()))); + return config.build(); + } + + + private void logCertificateInformation(String certificateType, KeystoreInformation keystore) { + try { + final KeyStore instance = keystore.loadKeystore(); + final Enumeration aliases = instance.aliases(); + while (aliases.hasMoreElements()) { + final Certificate cert = instance.getCertificate(aliases.nextElement()); + if (cert instanceof X509Certificate x509Certificate) { + final String alternativeNames = x509Certificate.getSubjectAlternativeNames() + .stream() + .map(san -> san.get(1)) + .map(Object::toString) + .collect(Collectors.joining(", ")); + LOG.info("Opensearch {} has following alternative names: {}", certificateType, alternativeNames); + LOG.info("Opensearch {} has following serial number: {}", certificateType, ((X509Certificate) cert).getSerialNumber()); + LOG.info("Opensearch {} has following validity: {} - {}", certificateType, ((X509Certificate) cert).getNotBefore(), ((X509Certificate) cert).getNotAfter()); + } + } + } catch (Exception e) { + throw new OpensearchConfigurationException("Failed to load kestore", e); + } + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/SearchableSnapshotsConfigurationBean.java b/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/SearchableSnapshotsConfigurationBean.java index 9684806cc981..62fa38f325ac 100644 --- a/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/SearchableSnapshotsConfigurationBean.java +++ b/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/impl/SearchableSnapshotsConfigurationBean.java @@ -24,9 +24,10 @@ import org.graylog.datanode.Configuration; import org.graylog.datanode.configuration.OpensearchConfigurationException; import org.graylog.datanode.configuration.S3RepositoryConfiguration; +import org.graylog.datanode.opensearch.configuration.OpensearchConfigurationParams; import org.graylog.datanode.opensearch.configuration.OpensearchUsableSpace; -import org.graylog.datanode.opensearch.configuration.beans.OpensearchConfigurationBean; -import org.graylog.datanode.opensearch.configuration.beans.OpensearchConfigurationPart; +import org.graylog.datanode.process.configuration.beans.DatanodeConfigurationBean; +import org.graylog.datanode.process.configuration.beans.DatanodeConfigurationPart; import org.graylog2.bootstrap.preflight.PreflightCheckException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,7 +45,7 @@ * If there is neither S3 nor local filesystem snapshot configuration, both search role and cache are disabled, * preventing unnecessary disk space consumption on the node. */ -public class SearchableSnapshotsConfigurationBean implements OpensearchConfigurationBean { +public class SearchableSnapshotsConfigurationBean implements DatanodeConfigurationBean { private static final Logger LOG = LoggerFactory.getLogger(SearchableSnapshotsConfigurationBean.class); @@ -61,17 +62,17 @@ public SearchableSnapshotsConfigurationBean(Configuration localConfiguration, S3 } @Override - public OpensearchConfigurationPart buildConfigurationPart() { + public DatanodeConfigurationPart buildConfigurationPart(OpensearchConfigurationParams trustedCertificates) { if (snapshotsAreEnabled()) { validateUsableSpace(); - return OpensearchConfigurationPart.builder() + return DatanodeConfigurationPart.builder() .properties(properties()) .keystoreItems(keystoreItems()) .addNodeRole(SEARCH_NODE_ROLE) .build(); } else { LOG.info("Opensearch snapshots not configured, skipping search role and cache configuration."); - return OpensearchConfigurationPart.builder().build(); + return DatanodeConfigurationPart.builder().build(); } } diff --git a/data-node/src/main/java/org/graylog/datanode/opensearch/statemachine/OpensearchStateMachine.java b/data-node/src/main/java/org/graylog/datanode/opensearch/statemachine/OpensearchStateMachine.java index e332839d0808..842928349de4 100644 --- a/data-node/src/main/java/org/graylog/datanode/opensearch/statemachine/OpensearchStateMachine.java +++ b/data-node/src/main/java/org/graylog/datanode/opensearch/statemachine/OpensearchStateMachine.java @@ -146,7 +146,7 @@ private void fire(OpensearchEvent trigger, OpensearchEvent errorEvent) { try { super.fire(trigger); } catch (Exception e) { - LOG.error(e.getMessage()); + LOG.error("Failed to fire event " + trigger, e); super.fire(errorEvent); } } diff --git a/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/OpensearchConfigurationBean.java b/data-node/src/main/java/org/graylog/datanode/process/configuration/beans/ConfigurationBuildParams.java similarity index 79% rename from data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/OpensearchConfigurationBean.java rename to data-node/src/main/java/org/graylog/datanode/process/configuration/beans/ConfigurationBuildParams.java index cf7bdea6da7b..70cfb938cf96 100644 --- a/data-node/src/main/java/org/graylog/datanode/opensearch/configuration/beans/OpensearchConfigurationBean.java +++ b/data-node/src/main/java/org/graylog/datanode/process/configuration/beans/ConfigurationBuildParams.java @@ -14,8 +14,7 @@ * along with this program. If not, see * . */ -package org.graylog.datanode.opensearch.configuration.beans; +package org.graylog.datanode.process.configuration.beans; -public interface OpensearchConfigurationBean { - OpensearchConfigurationPart buildConfigurationPart(); +public interface ConfigurationBuildParams { } diff --git a/data-node/src/main/java/org/graylog/datanode/process/configuration/beans/DatanodeConfigurationBean.java b/data-node/src/main/java/org/graylog/datanode/process/configuration/beans/DatanodeConfigurationBean.java new file mode 100644 index 000000000000..c1a0a26ffe6e --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/process/configuration/beans/DatanodeConfigurationBean.java @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.process.configuration.beans; + + +public interface DatanodeConfigurationBean { + DatanodeConfigurationPart buildConfigurationPart(T trustedCertificates); +} diff --git a/data-node/src/main/java/org/graylog/datanode/process/configuration/beans/DatanodeConfigurationPart.java b/data-node/src/main/java/org/graylog/datanode/process/configuration/beans/DatanodeConfigurationPart.java new file mode 100644 index 000000000000..66b8b9c8f200 --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/process/configuration/beans/DatanodeConfigurationPart.java @@ -0,0 +1,132 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.process.configuration.beans; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import jakarta.annotation.Nullable; +import org.graylog.datanode.process.configuration.files.DatanodeConfigFile; +import org.graylog.security.certutil.csr.KeystoreInformation; + +import java.security.KeyStore; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +@AutoValue +public abstract class DatanodeConfigurationPart { + public abstract List nodeRoles(); + + public abstract Map keystoreItems(); + + public abstract Map properties(); + + public abstract List javaOpts(); + + public abstract Map systemProperties(); + + @Nullable + public abstract KeystoreInformation httpCertificate(); + + @Nullable + public abstract KeystoreInformation transportCertificate(); + + public abstract boolean securityConfigured(); + + @Nullable + public abstract KeyStore trustStore(); + + public abstract List configFiles(); + + public static Builder builder() { + return new AutoValue_DatanodeConfigurationPart.Builder() + .nodeRoles(Collections.emptyList()) + .keystoreItems(Collections.emptyMap()) + .properties(Collections.emptyMap()) + .javaOpts(Collections.emptyList()) + .configFiles(Collections.emptyList()) + .securityConfigured(false) + .trustStore(null) + .systemProperties(Collections.emptyMap()); + } + + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder nodeRoles(List nodeRoles); + + abstract ImmutableList.Builder nodeRolesBuilder(); + + public final Builder addNodeRole(String role) { + nodeRolesBuilder().add(role); + return this; + } + + public abstract Builder javaOpts(List javaOpts); + + abstract ImmutableList.Builder javaOptsBuilder(); + + public final Builder javaOpt(String opt) { + javaOptsBuilder().add(opt); + return this; + } + + public abstract Builder configFiles(List configFiles); + + abstract ImmutableList.Builder configFilesBuilder(); + + public Builder withConfigFile(DatanodeConfigFile configFile) { + configFilesBuilder().add(configFile); + return this; + } + + public abstract Builder keystoreItems(Map keystoreItems); + + public abstract Builder properties(Map properties); + + public abstract Builder httpCertificate(KeystoreInformation httpCertificate); + + public abstract Builder transportCertificate(KeystoreInformation httpCertificate); + + @Deprecated + public abstract Builder securityConfigured(boolean securityConfigured); + + public abstract Builder trustStore(@Nullable KeyStore truststore); + + + private final ImmutableMap.Builder systemPropertiesBuilder = ImmutableMap.builder(); + + ImmutableMap.Builder systemPropertiesBuilder() { + return systemPropertiesBuilder; + } + + abstract Builder systemProperties(Map systemProperties); // not public + + abstract DatanodeConfigurationPart autoBuild(); // not public + + public DatanodeConfigurationPart build() { + systemProperties(systemPropertiesBuilder.buildKeepingLast()); + return autoBuild(); + } + + public Builder systemProperty(String key, String value) { + systemPropertiesBuilder().put(key, value); + return this; + } + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/process/configuration/files/DatanodeConfigFile.java b/data-node/src/main/java/org/graylog/datanode/process/configuration/files/DatanodeConfigFile.java new file mode 100644 index 000000000000..8137daee5755 --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/process/configuration/files/DatanodeConfigFile.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.process.configuration.files; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Path; + +public interface DatanodeConfigFile { + + /** + * Target relative path of the configuration file. May include parent directories. + */ + Path relativePath(); + + /** + * Given a file stream, write the configuration file content in it. Everything will be automatically flushed and closed. + */ + void write(OutputStream stream) throws IOException; +} diff --git a/data-node/src/main/java/org/graylog/datanode/process/configuration/files/InputStreamConfigFile.java b/data-node/src/main/java/org/graylog/datanode/process/configuration/files/InputStreamConfigFile.java new file mode 100644 index 000000000000..f22d25283ba5 --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/process/configuration/files/InputStreamConfigFile.java @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.process.configuration.files; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Path; + +public record InputStreamConfigFile(Path relativePath, ByteArrayInputStream inputStream) implements DatanodeConfigFile { + @Override + public void write(OutputStream output) throws IOException { + inputStream.transferTo(output); + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/process/configuration/files/KeystoreConfigFile.java b/data-node/src/main/java/org/graylog/datanode/process/configuration/files/KeystoreConfigFile.java new file mode 100644 index 000000000000..976abc04ace7 --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/process/configuration/files/KeystoreConfigFile.java @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.process.configuration.files; + +import org.graylog.datanode.configuration.OpensearchConfigurationException; +import org.graylog.security.certutil.csr.KeystoreInformation; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Path; + +public record KeystoreConfigFile(Path relativePath, KeystoreInformation keystoreInformation) implements DatanodeConfigFile { + + @Override + public void write(OutputStream stream) throws IOException { + try { + keystoreInformation().loadKeystore().store(stream, keystoreInformation.password()); + } catch (Exception e) { + throw new OpensearchConfigurationException("Failed to persist opensearch keystore file " + relativePath, e); + } + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/process/configuration/files/OpensearchSecurityConfigurationFile.java b/data-node/src/main/java/org/graylog/datanode/process/configuration/files/OpensearchSecurityConfigurationFile.java new file mode 100644 index 000000000000..d49786fdaaa3 --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/process/configuration/files/OpensearchSecurityConfigurationFile.java @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.process.configuration.files; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; +import org.graylog2.security.JwtSecret; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +public class OpensearchSecurityConfigurationFile implements DatanodeConfigFile { + + private static final ObjectMapper OBJECT_MAPPER = new YAMLMapper(); + private static final Path TARGET_PATH = Path.of("opensearch-security", "config.yml"); + private final JwtSecret signingKey; + + public OpensearchSecurityConfigurationFile(final JwtSecret signingKey) { + this.signingKey = signingKey; + } + + @Override + public Path relativePath() { + return TARGET_PATH; + } + + @Override + public void write(OutputStream stream) throws IOException { + final InputStream configSource = getClass().getResourceAsStream("/opensearch/config/opensearch-security/config.yml"); + Map contents = OBJECT_MAPPER.readValue(configSource, new TypeReference<>() {}); + Map config = filterConfigurationMap(contents, "config", "dynamic", "authc", "jwt_auth_domain", "http_authenticator", "config"); + config.put("signing_key", signingKey.getBase64Encoded()); + OBJECT_MAPPER.writeValue(stream, contents); + } + + + private Map filterConfigurationMap(final Map map, final String... keys) { + Map result = map; + for (final String key : List.of(keys)) { + result = (Map) result.get(key); + } + return result; + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/process/configuration/files/TextConfigFile.java b/data-node/src/main/java/org/graylog/datanode/process/configuration/files/TextConfigFile.java new file mode 100644 index 000000000000..21c356a5151f --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/process/configuration/files/TextConfigFile.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.process.configuration.files; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; + +public record TextConfigFile(Path relativePath, String text) implements DatanodeConfigFile { + + @Override + public void write(OutputStream output) throws IOException { + try (final ByteArrayInputStream input = new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8))) { + input.transferTo(output); + } + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/process/configuration/files/YamlConfigFile.java b/data-node/src/main/java/org/graylog/datanode/process/configuration/files/YamlConfigFile.java new file mode 100644 index 000000000000..65a94dc21bfc --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/process/configuration/files/YamlConfigFile.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.process.configuration.files; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Path; +import java.util.Map; + +public record YamlConfigFile(Path relativePath, Map config) implements DatanodeConfigFile { + private static final ObjectMapper MAPPER = new ObjectMapper(new YAMLFactory()); + + @Override + public void write(OutputStream output) throws IOException { + MAPPER.writeValue(output, config); + } +} diff --git a/data-node/src/main/resources/opensearch/config/opensearch-security/opensearch.yml.example b/data-node/src/main/resources/opensearch/config/opensearch-security/opensearch.yml.example deleted file mode 100644 index 3b4df645dea5..000000000000 --- a/data-node/src/main/resources/opensearch/config/opensearch-security/opensearch.yml.example +++ /dev/null @@ -1,228 +0,0 @@ -############## OpenSearch Security configuration ############### - -########################################################### -# Add the following settings to your standard opensearch.yml -# alongside with the OpenSearch Security TLS settings. -# Settings must always be the same on all nodes in the cluster. - -############## Common configuration settings ############## - -# Specify a list of DNs which denote the other nodes in the cluster. -# This settings support wildcards and regular expressions -# The list of DNs are also read from security index **in addition** to the yml configuration if -# plugins.security.nodes_dn_dynamic_config_enabled is true. -# NOTE: This setting only has effect if 'plugins.security.cert.intercluster_request_evaluator_class' is not set. -plugins.security.nodes_dn: - - "CN=*.example.com, OU=SSL, O=Test, L=Test, C=DE" - - "CN=node.other.com, OU=SSL, O=Test, L=Test, C=DE" - -# The nodes_dn_dynamic_config_enabled settings is geared towards cross_cluster usecases where there is a need to -# manage the whitelisted nodes_dn without having to restart the nodes everytime a new cross_cluster remote is configured -# Setting nodes_dn_dynamic_config_enabled to true enables **super-admin callable** /_opendistro/_security/api/nodesdn APIs -# which provide means to update/retrieve nodesdn dynamically. -# -# NOTE: The overall whitelisted nodes_dn evaluated comes from both the plugins.security.nodes_dn and the ones stored -# in security index. -# (default: false) -# NOTE2: This setting only has effect if 'plugins.security.cert.intercluster_request_evaluator_class' is not set. -plugins.security.nodes_dn_dynamic_config_enabled: false - -# Defines the DNs (distinguished names) of certificates -# to which admin privileges should be assigned (mandatory) -plugins.security.authcz.admin_dn: - - "CN=kirk,OU=client,O=client,l=tEst, C=De" - -# Define how backend roles should be mapped to Security roles -# MAPPING_ONLY - mappings must be configured explicitely in roles_mapping.yml (default) -# BACKENDROLES_ONLY - backend roles are mapped to Security roles directly. Settings in roles_mapping.yml have no effect. -# BOTH - backend roles are mapped to Security roles mapped directly and via roles_mapping.yml in addition -plugins.security.roles_mapping_resolution: MAPPING_ONLY - -############## REST Management API configuration settings ############## -# Enable or disable role based access to the REST management API -# Default is that no role is allowed to access the REST management API. -#plugins.security.restapi.roles_enabled: ["all_access","xyz_role"] - -# Disable particular endpoints and their HTTP methods for roles. -# By default all endpoints/methods are allowed. -#plugins.security.restapi.endpoints_disabled..: -# Example: -#plugins.security.restapi.endpoints_disabled.all_access.ACTIONGROUPS: ["PUT","POST","DELETE"] -#plugins.security.restapi.endpoints_disabled.xyz_role.LICENSE: ["DELETE"] - -# The following endpoints exist: -# ACTIONGROUPS -# CACHE -# CONFIG -# ROLES -# ROLESMAPPING -# INTERNALUSERS -# SYSTEMINFO -# PERMISSIONSINFO - -############## Auditlog configuration settings ############## -# General settings - -# Enable/disable rest request logging (default: true) -#plugins.security.audit.enable_rest: true -# Enable/disable transport request logging (default: false) -#plugins.security.audit.enable_transport: false -# Enable/disable bulk request logging (default: false) -# If enabled all subrequests in bulk requests will be logged too -#plugins.security.audit.resolve_bulk_requests: false -# Disable some categories -#plugins.security.audit.config.disabled_categories: ["AUTHENTICATED","GRANTED_PRIVILEGES"] -# Disable some requests (wildcard or regex of actions or rest request paths) -#plugins.security.audit.ignore_requests: ["indices:data/read/*","*_bulk"] -# Tune threadpool size, default is 10 -#plugins.security.audit.threadpool.size: 10 -# Tune threadpool max size queue length, default is 100000 -#plugins.security.audit.threadpool.max_queue_len: 100000 - -# Ignore users, e.g. do not log audit requests from that users (default: no ignored users) -#plugins.security.audit.ignore_users: ['kibanaserver','some*user','/also.*regex possible/']" - -# Destination of the auditlog events -plugins.security.audit.type: internal_opensearch -#plugins.security.audit.type: external_opensearch -#plugins.security.audit.type: debug -#plugins.security.audit.type: webhook - -# external_opensearch settings -#plugins.security.audit.config.http_endpoints: ['localhost:9200','localhost:9201','localhost:9202']" -# Auditlog index can be a static one or one with a date pattern (default is 'auditlog6') -#plugins.security.audit.config.index: auditlog6 # make sure you secure this index properly -#plugins.security.audit.config.index: "'auditlog6-'YYYY.MM.dd" #rotates index daily - make sure you secure this index properly -#plugins.security.audit.config.type: auditlog -#plugins.security.audit.config.username: auditloguser -#plugins.security.audit.config.password: auditlogpassword -#plugins.security.audit.config.enable_ssl: false -#plugins.security.audit.config.verify_hostnames: false -#plugins.security.audit.config.enable_ssl_client_auth: false -#plugins.security.audit.config.cert_alias: mycert -#plugins.security.audit.config.pemkey_filepath: key.pem -#plugins.security.audit.config.pemkey_content: <...pem base 64 content> -#plugins.security.audit.config.pemkey_password: secret -#plugins.security.audit.config.pemcert_filepath: cert.pem -#plugins.security.audit.config.pemcert_content: <...pem base 64 content> -#plugins.security.audit.config.pemtrustedcas_filepath: ca.pem -#plugins.security.audit.config.pemtrustedcas_content: <...pem base 64 content> - -# webhook settings -#plugins.security.audit.config.webhook.url: "http://mywebhook/endpoint" -# One of URL_PARAMETER_GET,URL_PARAMETER_POST,TEXT,JSON,SLACK -#plugins.security.audit.config.webhook.format: JSON -#plugins.security.audit.config.webhook.ssl.verify: false -#plugins.security.audit.config.webhook.ssl.pemtrustedcas_filepath: ca.pem -#plugins.security.audit.config.webhook.ssl.pemtrustedcas_content: <...pem base 64 content> - -# log4j settings -#plugins.security.audit.config.log4j.logger_name: auditlogger -#plugins.security.audit.config.log4j.level: INFO - -############## Kerberos configuration settings ############## -# If Kerberos authentication should be used you have to configure: - -# The Path to the krb5.conf file -# Can be absolute or relative to the OpenSearch config directory -#plugins.security.kerberos.krb5_filepath: '/etc/krb5.conf' - -# The Path to the keytab where the acceptor_principal credentials are stored. -# Must be relative to the OpenSearch config directory -#plugins.security.kerberos.acceptor_keytab_filepath: 'eskeytab.tab' - -# Acceptor (Server) Principal name, must be present in acceptor_keytab_path file -#plugins.security.kerberos.acceptor_principal: 'HTTP/localhost' - -############## Advanced configuration settings ############## -# Enable transport layer impersonation -# Allow DNs (distinguished names) to impersonate as other users -#plugins.security.authcz.impersonation_dn: -# "CN=spock,OU=client,O=client,L=Test,C=DE": -# - worf -# "cn=webuser,ou=IT,ou=IT,dc=company,dc=com": -# - user2 -# - user1 - -# Enable rest layer impersonation -# Allow users to impersonate as other users -#plugins.security.authcz.rest_impersonation_user: -# "picard": -# - worf -# "john": -# - steve -# - martin - -# If this is set to true OpenSearch Security will automatically initialize the configuration index -# with the files in the config directory if the index does not exist. -# WARNING: This will use well-known default passwords. -# Use only in a private network/environment. -#plugins.security.allow_default_init_securityindex: false - -# If this is set to true then allow to startup with demo certificates. -# These are certificates issued by floragunn GmbH for demo purposes. -# WARNING: This certificates are well known and therefore unsafe -# Use only in a private network/environment. -#plugins.security.allow_unsafe_democertificates: false - - - -# Password strength rules for password complexity. -# If you want to set up password strength rules for internal users, you can use the below settings for it. -# Password validation rules can be configured through regex. In the below regex example, a user must need -# a password with minimum 8 characters length and must include minimum one uppercase, one lower case, one digit, and one special character.  -# And a custom error message can be configured, in case if a password is not created according to the password strength rule.    -# plugins.security.restapi.password_validation_regex: '(?=.*[A-Z])(?=.*[^a-zA-Z\d])(?=.*[0-9])(?=.*[a-z]).{8,}' -# plugins.security.restapi.password_validation_error_message: "A password must be at least 8 characters long and contain at least one uppercase letter, one lowercase letter, one digit, and one special character." - - -############## Expert settings ############## -# WARNING: Expert settings, do only use if you know what you are doing -# If you set wrong values here this this could be a security risk -# or make OpenSearch Security stop working - -# Name of the index where .opendistro_security stores its configuration. - -#plugins.security.config_index_name: .opendistro_security - -# This defines the OID of server node certificates -#plugins.security.cert.oid: '1.2.3.4.5.5' - -# This specifies the implementation of org.opensearch.security.transport.InterClusterRequestEvaluator -# that is used to determine inter-cluster request. -# Instances of org.opensearch.security.transport.InterClusterRequestEvaluator must implement a single argument -# constructor that takes an org.opensearch.common.settings.Settings -#plugins.security.cert.intercluster_request_evaluator_class: org.opensearch.security.transport.DefaultInterClusterRequestEvaluator - -# By default, normal users can restore snapshots if they have the priviliges 'cluster:admin/snapshot/restore', -# 'indices:admin/create', and 'indices:data/write/index' for the indices to be restored. -# To disable snapshot restore for normal users set 'plugins.security.enable_snapshot_restore_privilege: false'. -# This makes it so that only snapshot restore requests signed by an admin TLS certificate are accepted. -# A snapshot can only be restored when it does not contain global state and does not restore the '.opendistro_security' index -# If 'plugins.security.check_snapshot_restore_write_privileges: false' is set then the additional indices checks are omitted. -#plugins.security.enable_snapshot_restore_privilege: true -#plugins.security.check_snapshot_restore_write_privileges: true - -# Authentication cache timeout in minutes (A value of 0 disables caching, default is 60) -#plugins.security.cache.ttl_minutes: 60 - -# Disable OpenSearch Security -# WARNING: This can expose your configuration (including passwords) to the public. -#plugins.security.disabled: false - - -# Protected indices are even more secure than normal indices. These indices require a role to access like any other index, but they require an additional role -# to be visible, listed in the plugins.security.protected_indices.roles setting. -# Enable protected indices -# plugins.security.protected_indices.enabled: true -# Specify a list of roles a user must be member of to touch any protected index. -# plugins.security.protected_indices.roles: ['all_access'] -# Specify a list of indices to mark as protected. These indices will only be visible / mutable by members of the above setting, in addition to needing permission to the index via a normal role. -# plugins.security.protected_indices.indices: [] - -# System indices are similar to security index, except the contents are not encrypted. -# Indices configured as system indices can be accessed by only super-admin and no role will provide access to these indices. -# Enable system indices -# plugins.security.system_indices.enabled: true -# Specify a list of indices to mark as system. These indices will only be visible / mutable by members of the above setting, in addition to needing permission to the index via a normal role. -# plugins.security.system_indices.indices: ['.opendistro-alerting-config', '.opendistro-ism-*', '.opendistro-reports-*', '.opensearch-notifications-*', '.opensearch-notebooks', '.opensearch-observability', '.opendistro-asynchronous-search-response*', '.replication-metadata-store'] diff --git a/data-node/src/test/java/org/graylog/datanode/bootstrap/preflight/FullDirSyncTest.java b/data-node/src/test/java/org/graylog/datanode/bootstrap/preflight/FullDirSyncTest.java deleted file mode 100644 index 19f4516a5e35..000000000000 --- a/data-node/src/test/java/org/graylog/datanode/bootstrap/preflight/FullDirSyncTest.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright (C) 2020 Graylog, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * . - */ -package org.graylog.datanode.bootstrap.preflight; - -import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.nio.file.attribute.PosixFilePermission; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; - -class FullDirSyncTest { - - @TempDir - private Path source; - - @TempDir - private Path target; - - @BeforeEach - void setUp() throws IOException { - - /* - source: - -a.txt - -b.txt - -subdir - -c.txt - - target: - -a.txt - -x.txt - -empty-dir - -subdir - -y.txt - -unused-dir - -z.txt - */ - - Files.createFile(source.resolve("a.txt")); - Files.createFile(source.resolve("b.txt")); - Files.createDirectories(source.resolve("subdir")); - Files.createFile(source.resolve("subdir").resolve("c.txt")); - - Files.createFile(target.resolve("a.txt")); - Files.createFile(target.resolve("x.txt")); - Files.createDirectories(target.resolve("subdir")); - Files.createDirectories(target.resolve("empty-dir")); - Files.createFile(target.resolve("subdir").resolve("y.txt")); - Files.createDirectories(target.resolve("unused-dir")); - Files.createFile(target.resolve("unused-dir").resolve("z.txt")); - } - - @Test - void run() throws IOException { - FullDirSync.run(source, target); - - List afterSyncState = new ArrayList<>(); - - Files.walkFileTree(target, new SimpleFileVisitor<>() { - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - if(!target.equals(dir)) { - afterSyncState.add(dir); - } - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - afterSyncState.add(file); - return super.visitFile(file, attrs); - } - }); - - Assertions.assertThat(afterSyncState) - .extracting(target::relativize) - .extracting(Path::toString) - .hasSize(4) - .contains("a.txt", "b.txt", "subdir", "subdir/c.txt"); - - Assertions.assertThat(afterSyncState) - .filteredOn(Files::isDirectory) - .allSatisfy(p -> { - final Set permission = Files.getPosixFilePermissions(p); - Assertions.assertThat(permission).containsExactlyInAnyOrderElementsOf(FullDirSync.DIRECTORY_PERMISSIONS); - }); - - Assertions.assertThat(afterSyncState) - .filteredOn(Files::isRegularFile) - .allSatisfy(p -> { - final Set permission = Files.getPosixFilePermissions(p); - Assertions.assertThat(permission).containsExactlyInAnyOrderElementsOf(FullDirSync.FILE_PERMISSIONS); - }); - - - - } -} diff --git a/data-node/src/test/java/org/graylog/datanode/configuration/DatanodeDirectoriesTest.java b/data-node/src/test/java/org/graylog/datanode/configuration/DatanodeDirectoriesTest.java index 5ca2c4176fae..d39f0a9794ac 100644 --- a/data-node/src/test/java/org/graylog/datanode/configuration/DatanodeDirectoriesTest.java +++ b/data-node/src/test/java/org/graylog/datanode/configuration/DatanodeDirectoriesTest.java @@ -31,15 +31,16 @@ class DatanodeDirectoriesTest { @Test void testConfigDirPermissions(@TempDir Path dataDir, @TempDir Path logsDir, @TempDir Path configSourceDir, @TempDir Path configTargetDir) throws IOException { final DatanodeDirectories datanodeDirectories = new DatanodeDirectories(dataDir, logsDir, configSourceDir, configTargetDir); - final Path dir = datanodeDirectories.createOpensearchProcessConfigurationDir(); - Assertions.assertThat(Files.getPosixFilePermissions(dir)). + final OpensearchConfigurationDir dir = datanodeDirectories.createUniqueOpensearchProcessConfigurationDir(); + + Assertions.assertThat(Files.getPosixFilePermissions(dir.configurationRoot())). contains( PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_READ ); - final Path keyFile = datanodeDirectories.createOpensearchProcessConfigurationFile(Path.of("my-secret-file.key")); + final Path keyFile = dir.createOpensearchProcessConfigurationFile(Path.of("my-secret-file.key")); Assertions.assertThat(Files.getPosixFilePermissions(keyFile)). contains( PosixFilePermission.OWNER_WRITE, diff --git a/data-node/src/test/java/org/graylog/datanode/opensearch/OpensearchProcessImplTest.java b/data-node/src/test/java/org/graylog/datanode/opensearch/OpensearchProcessImplTest.java index 5c81e1eda10d..77914ef48a61 100644 --- a/data-node/src/test/java/org/graylog/datanode/opensearch/OpensearchProcessImplTest.java +++ b/data-node/src/test/java/org/graylog/datanode/opensearch/OpensearchProcessImplTest.java @@ -29,8 +29,6 @@ import org.graylog.shaded.opensearch2.org.opensearch.client.RequestOptions; import org.graylog.shaded.opensearch2.org.opensearch.client.RestHighLevelClient; import org.graylog.shaded.opensearch2.org.opensearch.common.settings.Settings; -import org.graylog2.cluster.nodes.DataNodeDto; -import org.graylog2.cluster.nodes.NodeService; import org.graylog2.plugin.system.NodeId; import org.graylog2.plugin.system.SimpleNodeId; import org.graylog2.security.CustomCAX509TrustManager; @@ -66,8 +64,6 @@ public class OpensearchProcessImplTest { @Mock private Configuration configuration; @Mock - private NodeService nodeService; - @Mock private ObjectMapper objectMapper; @Mock private OpensearchStateMachine processState; @@ -86,7 +82,7 @@ public void setup() throws IOException { when(datanodeConfiguration.processLogsBufferSize()).thenReturn(100); when(configuration.getDatanodeNodeName()).thenReturn(nodeName); this.opensearchProcess = spy(new OpensearchProcessImpl(datanodeConfiguration, trustmManager, configuration, - nodeService, objectMapper, processState, nodeId, eventBus)); + objectMapper, processState, nodeId, eventBus)); when(opensearchProcess.restClient()).thenReturn(Optional.of(restClient)); when(restClient.cluster()).thenReturn(clusterClient); } diff --git a/data-node/src/test/java/org/graylog/datanode/opensearch/configuration/beans/OpensearchConfigurationPartTest.java b/data-node/src/test/java/org/graylog/datanode/opensearch/configuration/beans/DatanodeConfigurationPartTest.java similarity index 75% rename from data-node/src/test/java/org/graylog/datanode/opensearch/configuration/beans/OpensearchConfigurationPartTest.java rename to data-node/src/test/java/org/graylog/datanode/opensearch/configuration/beans/DatanodeConfigurationPartTest.java index 0cfeff8e15a0..3b43c8114644 100644 --- a/data-node/src/test/java/org/graylog/datanode/opensearch/configuration/beans/OpensearchConfigurationPartTest.java +++ b/data-node/src/test/java/org/graylog/datanode/opensearch/configuration/beans/DatanodeConfigurationPartTest.java @@ -17,20 +17,23 @@ package org.graylog.datanode.opensearch.configuration.beans; import org.assertj.core.api.Assertions; +import org.graylog.datanode.process.configuration.beans.DatanodeConfigurationPart; import org.junit.jupiter.api.Test; import java.util.Collections; -class OpensearchConfigurationPartTest { +class DatanodeConfigurationPartTest { @Test void testConfigBuild() { - final OpensearchConfigurationPart configurationPart = OpensearchConfigurationPart.builder() + final DatanodeConfigurationPart configurationPart = DatanodeConfigurationPart.builder() .addNodeRole("cluster_manager") .addNodeRole("data") .addNodeRole("search") .keystoreItems(Collections.singletonMap("foo", "bar")) .properties(Collections.singletonMap("reindex.remote.allowlist", "localhost:9201")) + .systemProperty("file.encoding", "utf-8") + .systemProperty("java.home", "/jdk") .build(); Assertions.assertThat(configurationPart.nodeRoles()) @@ -44,5 +47,10 @@ void testConfigBuild() { Assertions.assertThat(configurationPart.properties()) .hasSize(1) .containsEntry("reindex.remote.allowlist", "localhost:9201"); + + Assertions.assertThat(configurationPart.systemProperties()) + .hasSize(2) + .containsEntry("file.encoding", "utf-8") + .containsEntry("java.home", "/jdk"); } } diff --git a/data-node/src/test/java/org/graylog/datanode/opensearch/configuration/beans/impl/SearchableSnapshotsConfigurationBeanTest.java b/data-node/src/test/java/org/graylog/datanode/opensearch/configuration/beans/impl/SearchableSnapshotsConfigurationBeanTest.java index 069fa5ff9547..760024357bf5 100644 --- a/data-node/src/test/java/org/graylog/datanode/opensearch/configuration/beans/impl/SearchableSnapshotsConfigurationBeanTest.java +++ b/data-node/src/test/java/org/graylog/datanode/opensearch/configuration/beans/impl/SearchableSnapshotsConfigurationBeanTest.java @@ -24,12 +24,14 @@ import org.graylog.datanode.Configuration; import org.graylog.datanode.configuration.OpensearchConfigurationException; import org.graylog.datanode.configuration.S3RepositoryConfiguration; +import org.graylog.datanode.opensearch.configuration.OpensearchConfigurationParams; import org.graylog.datanode.opensearch.configuration.OpensearchUsableSpace; -import org.graylog.datanode.opensearch.configuration.beans.OpensearchConfigurationPart; +import org.graylog.datanode.process.configuration.beans.DatanodeConfigurationPart; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import java.nio.file.Path; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -51,7 +53,7 @@ void testS3Repo(@TempDir Path tempDir) throws ValidationException, RepositoryExc config, () -> new OpensearchUsableSpace(tempDir, 20L * 1024 * 1024 * 1024)); - final OpensearchConfigurationPart configurationPart = bean.buildConfigurationPart(); + final DatanodeConfigurationPart configurationPart = bean.buildConfigurationPart(emptyBuildParams()); Assertions.assertThat(configurationPart.nodeRoles()) .contains(SearchableSnapshotsConfigurationBean.SEARCH_NODE_ROLE); @@ -63,6 +65,10 @@ void testS3Repo(@TempDir Path tempDir) throws ValidationException, RepositoryExc .containsKeys("s3.client.default.endpoint", "node.search.cache.size"); } + private OpensearchConfigurationParams emptyBuildParams() { + return new OpensearchConfigurationParams(Collections.emptyList(), Collections.emptyMap()); + } + @Test void testLocalFilesystemRepo(@TempDir Path tempDir) throws ValidationException, RepositoryException { // no s3 repo configuration properties given by the user @@ -77,7 +83,7 @@ void testLocalFilesystemRepo(@TempDir Path tempDir) throws ValidationException, config, () -> new OpensearchUsableSpace(tempDir, 20L * 1024 * 1024 * 1024)); - final OpensearchConfigurationPart configurationPart = bean.buildConfigurationPart(); + final DatanodeConfigurationPart configurationPart = bean.buildConfigurationPart(emptyBuildParams()); Assertions.assertThat(configurationPart.nodeRoles()) .contains(SearchableSnapshotsConfigurationBean.SEARCH_NODE_ROLE); @@ -103,7 +109,7 @@ void testNoSnapshotConfiguration(@TempDir Path tempDir) throws ValidationExcepti config, () -> new OpensearchUsableSpace(tempDir, 20L * 1024 * 1024 * 1024)); - final OpensearchConfigurationPart configurationPart = bean.buildConfigurationPart(); + final DatanodeConfigurationPart configurationPart = bean.buildConfigurationPart(emptyBuildParams()); Assertions.assertThat(configurationPart.nodeRoles()) .isEmpty(); // no search role should be provided @@ -132,7 +138,7 @@ void testCacheSizeValidation(@TempDir Path tempDir) throws ValidationException, () -> new OpensearchUsableSpace(tempDir, 8L * 1024 * 1024 * 1024)); // 10GB cache requested on 8GB of free space, needs to throw an exception! - Assertions.assertThatThrownBy(bean::buildConfigurationPart) + Assertions.assertThatThrownBy(() -> bean.buildConfigurationPart(emptyBuildParams())) .isInstanceOf(OpensearchConfigurationException.class) .hasMessageContaining("There is not enough usable space for the node search cache. Your system has only 8gb available"); From fad12ccbcfb8bb4d81d6850dec3391c51f53829d Mon Sep 17 00:00:00 2001 From: Dennis Oelkers Date: Thu, 12 Dec 2024 12:14:29 +0100 Subject: [PATCH 2/7] Improving Events Services. (#21172) * Widen parameter type for easier use with different collections. * Using methods for bulk retrieval to reduce db roundtrips. * Extracting helper function for parsing timestamp filters. * Improving typing. * Removing leading underscore from variables. * Adding license header. --- .../processor/DBEventDefinitionService.java | 3 +- .../events/search/EventsSearchService.java | 20 ++----- .../parseTimerangeFilter.ts | 52 +++++++++++++++++++ .../events/events/ColumnRenderers.tsx | 14 ++--- .../src/components/events/fetchEvents.ts | 39 ++------------ 5 files changed, 69 insertions(+), 59 deletions(-) create mode 100644 graylog2-web-interface/src/components/common/PaginatedEntityTable/parseTimerangeFilter.ts diff --git a/graylog2-server/src/main/java/org/graylog/events/processor/DBEventDefinitionService.java b/graylog2-server/src/main/java/org/graylog/events/processor/DBEventDefinitionService.java index eb7cf236b0e8..278229163b75 100644 --- a/graylog2-server/src/main/java/org/graylog/events/processor/DBEventDefinitionService.java +++ b/graylog2-server/src/main/java/org/graylog/events/processor/DBEventDefinitionService.java @@ -38,6 +38,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collection; import java.util.List; import java.util.Locale; import java.util.Optional; @@ -212,7 +213,7 @@ public Stream streamAll() { return stream(collection.find()); } - public List getByIds(List ids) { + public List getByIds(Collection ids) { return MongoUtils.stream(collection.find(MongoUtils.stringIdsIn(ids))) .map(this::getEventDefinitionWithRefetchedFilters) .toList(); diff --git a/graylog2-server/src/main/java/org/graylog/events/search/EventsSearchService.java b/graylog2-server/src/main/java/org/graylog/events/search/EventsSearchService.java index 6a20dbb6afea..37cf6dd47829 100644 --- a/graylog2-server/src/main/java/org/graylog/events/search/EventsSearchService.java +++ b/graylog2-server/src/main/java/org/graylog/events/search/EventsSearchService.java @@ -24,7 +24,6 @@ import org.graylog.events.event.EventDto; import org.graylog.events.processor.DBEventDefinitionService; import org.graylog.events.processor.EventDefinitionDto; -import org.graylog2.database.NotFoundException; import org.graylog2.indexer.IndexMapping; import org.graylog2.plugin.database.Persisted; import org.graylog2.plugin.indexer.searches.timeranges.TimeRange; @@ -37,8 +36,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.stream.Collector; import java.util.stream.Collectors; @@ -231,23 +228,14 @@ private Set forbiddenSourceStreams(Subject subject) { } private Map lookupStreams(Set streams) { - return streams.stream() - .map(streamId -> { - try { - return streamService.load(streamId); - } catch (NotFoundException e) { - return null; - } - }) - .filter(Objects::nonNull) + return streamService.loadByIds(streams) + .stream() .collect(Collectors.toMap(Persisted::getId, s -> EventsSearchResult.ContextEntity.create(s.getId(), s.getTitle(), s.getDescription()))); } private Map lookupEventDefinitions(Set eventDefinitions) { - return eventDefinitions.stream() - .map(eventDefinitionService::get) - .filter(Optional::isPresent) - .map(Optional::get) + return eventDefinitionService.getByIds(eventDefinitions) + .stream() .collect(Collectors.toMap(EventDefinitionDto::id, d -> EventsSearchResult.ContextEntity.create(d.id(), d.title(), d.description(), d.remediationSteps()))); } diff --git a/graylog2-web-interface/src/components/common/PaginatedEntityTable/parseTimerangeFilter.ts b/graylog2-web-interface/src/components/common/PaginatedEntityTable/parseTimerangeFilter.ts new file mode 100644 index 000000000000..685d96fea6e2 --- /dev/null +++ b/graylog2-web-interface/src/components/common/PaginatedEntityTable/parseTimerangeFilter.ts @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +import moment from 'moment'; +import trim from 'lodash/trim'; + +import { extractRangeFromString } from 'components/common/EntityFilters/helpers/timeRange'; +import { adjustFormat } from 'util/DateTime'; +import type { TimeRange, RelativeTimeRange } from 'views/logic/queries/Query'; + +const allTimesRange: RelativeTimeRange = { type: 'relative', range: 0 }; + +const isNullOrBlank = (s: string | undefined) => { + if (!s) { + return true; + } + + return trim(s) === ''; +}; + +const parseTimerangeFilter = (timestamp: string | undefined): TimeRange => { + if (!timestamp) { + return allTimesRange; + } + + const [from, to] = extractRangeFromString(timestamp); + + if (!from && !to) { + return allTimesRange; + } + + return { + type: 'absolute', + from: isNullOrBlank(from) ? adjustFormat(moment(0).utc(), 'internal') : from, + to: isNullOrBlank(to) ? adjustFormat(moment().utc(), 'internal') : to, + }; +}; + +export default parseTimerangeFilter; diff --git a/graylog2-web-interface/src/components/events/events/ColumnRenderers.tsx b/graylog2-web-interface/src/components/events/events/ColumnRenderers.tsx index 90fb31fb9c0a..44161af1d95c 100644 --- a/graylog2-web-interface/src/components/events/events/ColumnRenderers.tsx +++ b/graylog2-web-interface/src/components/events/events/ColumnRenderers.tsx @@ -110,25 +110,25 @@ export const getGeneralEventAttributeRenderers = , + renderCell: (message: string, event) => , }, key: { - renderCell: (_key: string) => {_key || No Key set for this Event.}, + renderCell: (key: string) => {key || No Key set for this Event.}, staticWidth: 200, }, id: { staticWidth: 300, }, alert: { - renderCell: (_alert: boolean) => , + renderCell: (alert: boolean) => , staticWidth: 100, }, priority: { - renderCell: (_priority: number) => , + renderCell: (priority: number) => , staticWidth: 100, }, event_definition_type: { - renderCell: (_type: string) => , + renderCell: (type: string) => , staticWidth: 200, }, group_by_fields: { @@ -142,10 +142,10 @@ const customColumnRenderers = (): ColumnRenderers => ({ event_definition_id: { minWidth: 300, width: 0.3, - renderCell: (_eventDefinitionId: string, _, __, meta: EventsAdditionalData) => , + renderCell: (eventDefinitionId: string, _, __, meta: EventsAdditionalData) => , }, fields: { - renderCell: (_fields: Record) => , + renderCell: (fields: Record) => , staticWidth: 400, }, remediation_steps: { diff --git a/graylog2-web-interface/src/components/events/fetchEvents.ts b/graylog2-web-interface/src/components/events/fetchEvents.ts index 5c23bfbd198f..6134a3f1551f 100644 --- a/graylog2-web-interface/src/components/events/fetchEvents.ts +++ b/graylog2-web-interface/src/components/events/fetchEvents.ts @@ -16,7 +16,6 @@ */ import moment from 'moment'; -import trim from 'lodash/trim'; import * as URLUtils from 'util/URLUtils'; import { adjustFormat } from 'util/DateTime'; @@ -27,6 +26,8 @@ import type { Event, EventsAdditionalData } from 'components/events/events/types import { additionalAttributes } from 'components/events/Constants'; import { extractRangeFromString } from 'components/common/EntityFilters/helpers/timeRange'; import type { UrlQueryFilters } from 'components/common/EntityFilters/types'; +import parseTimerangeFilter from 'components/common/PaginatedEntityTable/parseTimerangeFilter'; +import type { TimeRange } from 'views/logic/queries/Query'; const url = URLUtils.qualifyUrl('/events/search'); @@ -38,39 +39,7 @@ type FiltersResult = { aggregation_timerange?: { from?: string, to?: string, type: string, range?: number }, key?: Array, }, - timerange?: { from?: string, to?: string, type: string, range?: number }, -}; - -const allTimesRange = { type: 'relative', range: 0 }; - -const isNullOrBlank = (s: string | undefined) => { - if (!s) { - return true; - } - - if (trim(s) === '') { - return true; - } - - return false; -}; - -const parseTimestampFilter = (timestamp: string | undefined) => { - if (!timestamp) { - return allTimesRange; - } - - const [from, to] = extractRangeFromString(timestamp); - - if (!from && !to) { - return allTimesRange; - } - - return { - type: 'absolute', - from: isNullOrBlank(from) ? adjustFormat(moment(0).utc(), 'internal') : from, - to: isNullOrBlank(to) ? adjustFormat(moment().utc(), 'internal') : to, - }; + timerange?: TimeRange, }; export const parseTypeFilter = (alert: string) => { @@ -87,7 +56,7 @@ export const parseTypeFilter = (alert: string) => { const parseFilters = (filters: UrlQueryFilters) => { const result: FiltersResult = { filter: {} }; - result.timerange = parseTimestampFilter(filters.get('timestamp')?.[0]); + result.timerange = parseTimerangeFilter(filters.get('timestamp')?.[0]); if (filters.get('timerange_start')?.[0]) { const [from, to] = extractRangeFromString(filters.get('timerange_start')[0]); From dad084d46b516e8b64fb17f5153dfb68f237e950 Mon Sep 17 00:00:00 2001 From: Dennis Oelkers Date: Thu, 12 Dec 2024 13:43:39 +0100 Subject: [PATCH 3/7] Introducing and reusing `DefaultLayout` type. (#21173) * Introducing and reusing `DefaultLayout` type. * Fixing linter hints. --- .../common/EntityDataTable/hooks/useTableLayout.ts | 9 ++------- .../src/components/common/EntityDataTable/types.ts | 7 +++++++ .../PaginatedEntityTable/PaginatedEntityTable.tsx | 10 +++++----- .../src/components/events/ExpandedSection.tsx | 4 ++-- .../events/events/hooks/useTableComponents.tsx | 4 ++-- 5 files changed, 18 insertions(+), 16 deletions(-) diff --git a/graylog2-web-interface/src/components/common/EntityDataTable/hooks/useTableLayout.ts b/graylog2-web-interface/src/components/common/EntityDataTable/hooks/useTableLayout.ts index b5ee4905f9da..1b5b0ea5773f 100644 --- a/graylog2-web-interface/src/components/common/EntityDataTable/hooks/useTableLayout.ts +++ b/graylog2-web-interface/src/components/common/EntityDataTable/hooks/useTableLayout.ts @@ -16,16 +16,11 @@ */ import { useMemo } from 'react'; -import type { Sort } from 'stores/PaginationTypes'; +import type { DefaultLayout } from 'components/common/EntityDataTable/types'; import useUserLayoutPreferences from './useUserLayoutPreferences'; -const useTableLayout = ({ entityTableId, defaultSort, defaultPageSize, defaultDisplayedAttributes }: { - entityTableId: string, - defaultSort: Sort, - defaultDisplayedAttributes: Array - defaultPageSize: number, -}) => { +const useTableLayout = ({ entityTableId, defaultSort, defaultPageSize, defaultDisplayedAttributes }: DefaultLayout) => { const { data: userLayoutPreferences = {}, isInitialLoading } = useUserLayoutPreferences(entityTableId); return useMemo(() => ({ diff --git a/graylog2-web-interface/src/components/common/EntityDataTable/types.ts b/graylog2-web-interface/src/components/common/EntityDataTable/types.ts index 342b839dde2e..f1c20ba00cfb 100644 --- a/graylog2-web-interface/src/components/common/EntityDataTable/types.ts +++ b/graylog2-web-interface/src/components/common/EntityDataTable/types.ts @@ -72,3 +72,10 @@ export type ExpandedSectionRenderer = { actions?: (entity: Entity) => React.ReactNode, disableHeader?: boolean, } + +export type DefaultLayout = { + entityTableId: string, + defaultSort: Sort, + defaultDisplayedAttributes: Array + defaultPageSize: number, +} diff --git a/graylog2-web-interface/src/components/common/PaginatedEntityTable/PaginatedEntityTable.tsx b/graylog2-web-interface/src/components/common/PaginatedEntityTable/PaginatedEntityTable.tsx index 2c451bf0fcaf..4b02fcc43a42 100644 --- a/graylog2-web-interface/src/components/common/PaginatedEntityTable/PaginatedEntityTable.tsx +++ b/graylog2-web-interface/src/components/common/PaginatedEntityTable/PaginatedEntityTable.tsx @@ -25,7 +25,7 @@ import useUpdateUserLayoutPreferences from 'components/common/EntityDataTable/ho import { useTableEventHandlers } from 'components/common/EntityDataTable'; import { Spinner, PaginatedList, SearchForm, NoSearchResult, EntityDataTable } from 'components/common'; import type { Attribute, SearchParams } from 'stores/PaginationTypes'; -import type { EntityBase } from 'components/common/EntityDataTable/types'; +import type { EntityBase, DefaultLayout } from 'components/common/EntityDataTable/types'; import EntityFilters from 'components/common/EntityFilters'; import useUrlQueryFilters from 'components/common/EntityFilters/hooks/useUrlQueryFilters'; import type { UrlQueryFilters } from 'components/common/EntityFilters/types'; @@ -59,7 +59,7 @@ type Props = { keyFn: (options: SearchParams) => Array, queryHelpComponent?: React.ReactNode, searchPlaceholder?: string, - tableLayout: Parameters[0], + tableLayout: DefaultLayout, topRightCol?: React.ReactNode, } @@ -78,9 +78,9 @@ const INITIAL_DATA = { */ const PaginatedEntityTable = ({ actionsCellWidth = 160, columnsOrder, entityActions, tableLayout, fetchEntities, keyFn, - humanName, columnRenderers, queryHelpComponent, filterValueRenderers, - expandedSectionsRenderer, bulkSelection, additionalAttributes = [], - entityAttributesAreCamelCase, topRightCol, searchPlaceholder, fetchOptions: reactQueryOptions, + humanName, columnRenderers, queryHelpComponent = undefined, filterValueRenderers = undefined, + expandedSectionsRenderer = undefined, bulkSelection = undefined, additionalAttributes = [], + entityAttributesAreCamelCase, topRightCol = undefined, searchPlaceholder = undefined, fetchOptions: reactQueryOptions = undefined, }: Props) => { const [urlQueryFilters, setUrlQueryFilters] = useUrlQueryFilters(); const [query, setQuery] = useQueryParam('query', StringParam); diff --git a/graylog2-web-interface/src/components/events/ExpandedSection.tsx b/graylog2-web-interface/src/components/events/ExpandedSection.tsx index 74ab165a4340..6c2e617cd0c1 100644 --- a/graylog2-web-interface/src/components/events/ExpandedSection.tsx +++ b/graylog2-web-interface/src/components/events/ExpandedSection.tsx @@ -16,14 +16,14 @@ */ import React from 'react'; -import type useTableLayout from 'components/common/EntityDataTable/hooks/useTableLayout'; import type { Event, EventsAdditionalData } from 'components/events/events/types'; import useMetaDataContext from 'components/common/EntityDataTable/hooks/useMetaDataContext'; import GeneralEventDetailsTable from 'components/events/events/GeneralEventDetailsTable'; import useNonDisplayedAttributes from 'components/events/events/hooks/useNonDisplayedAttributes'; +import type { DefaultLayout } from 'components/common/EntityDataTable/types'; type Props = { - defaultLayout: Parameters[0], + defaultLayout: DefaultLayout, event: Event, } diff --git a/graylog2-web-interface/src/components/events/events/hooks/useTableComponents.tsx b/graylog2-web-interface/src/components/events/events/hooks/useTableComponents.tsx index 18cbcdd4115b..03f9d98dec5d 100644 --- a/graylog2-web-interface/src/components/events/events/hooks/useTableComponents.tsx +++ b/graylog2-web-interface/src/components/events/events/hooks/useTableComponents.tsx @@ -21,11 +21,11 @@ import keyBy from 'lodash/keyBy'; import EventActions from 'components/events/events/EventActions'; import type { Event } from 'components/events/events/types'; import ExpandedSection from 'components/events/ExpandedSection'; -import type useTableLayout from 'components/common/EntityDataTable/hooks/useTableLayout'; import BulkActions from 'components/events/events/BulkActions'; +import type { DefaultLayout } from 'components/common/EntityDataTable/types'; const useTableElements = ({ defaultLayout }: { - defaultLayout: Parameters[0], + defaultLayout: DefaultLayout, }) => { const entityActions = useCallback((event: Event) => ( From 4b8e798dc282f640cf7d412b2e65edbeedcacfb4 Mon Sep 17 00:00:00 2001 From: Laura Date: Thu, 12 Dec 2024 14:02:13 +0100 Subject: [PATCH 4/7] ISW - Add Start Input Step (#21163) --- .../src/components/inputs/InputListItem.tsx | 14 +- .../InputSetupWizard.SetupRouting.test.tsx | 52 +-- .../InputSetupWizard.StartInput.test.tsx | 354 +++++++++++++++++ .../InputSetupWizard.test.tsx | 56 +-- .../InputSetupWizard/InputSetupWizard.tsx | 103 +---- .../inputs/InputSetupWizard/Wizard.tsx | 137 +++++++ .../contexts/InputSetupWizardContext.tsx | 13 +- .../InputSetupWizardProvider.test.tsx | 55 --- .../contexts/InputSetupWizardProvider.tsx | 49 +-- .../InputSetupWizardStepsContext.tsx} | 21 +- .../InputSetupWizardStepsProvider.tsx | 43 +++ .../helpers/stepHelper.test.ts | 28 +- .../InputSetupWizard/helpers/stepHelper.ts | 41 +- .../hooks/useInputSetupWizardSteps.ts | 31 ++ .../hooks/useSetupInputMutations.ts | 104 +++++ .../inputs/InputSetupWizard/index.ts | 4 +- .../steps/SetupRoutingStep.tsx | 106 +++-- .../InputSetupWizard/steps/StartInputStep.tsx | 362 +++++++++++++++++- .../steps/components/CreateStreamForm.tsx | 6 +- .../steps/components/ProgressMessage.tsx | 97 +++++ .../inputs/InputSetupWizard/types.ts | 17 +- .../components/inputs/InputStateControl.tsx | 10 +- .../src/components/inputs/InputsList.tsx | 93 ++--- .../src/routing/ApiRoutes.ts | 1 + .../src/stores/streams/StreamsStore.ts | 12 +- 25 files changed, 1401 insertions(+), 408 deletions(-) create mode 100644 graylog2-web-interface/src/components/inputs/InputSetupWizard/InputSetupWizard.StartInput.test.tsx create mode 100644 graylog2-web-interface/src/components/inputs/InputSetupWizard/Wizard.tsx delete mode 100644 graylog2-web-interface/src/components/inputs/InputSetupWizard/contexts/InputSetupWizardProvider.test.tsx rename graylog2-web-interface/src/components/inputs/InputSetupWizard/{steps/components/CreateStream.tsx => contexts/InputSetupWizardStepsContext.tsx} (59%) create mode 100644 graylog2-web-interface/src/components/inputs/InputSetupWizard/contexts/InputSetupWizardStepsProvider.tsx create mode 100644 graylog2-web-interface/src/components/inputs/InputSetupWizard/hooks/useInputSetupWizardSteps.ts create mode 100644 graylog2-web-interface/src/components/inputs/InputSetupWizard/hooks/useSetupInputMutations.ts create mode 100644 graylog2-web-interface/src/components/inputs/InputSetupWizard/steps/components/ProgressMessage.tsx diff --git a/graylog2-web-interface/src/components/inputs/InputListItem.tsx b/graylog2-web-interface/src/components/inputs/InputListItem.tsx index 8f3b630744f9..725a87f88100 100644 --- a/graylog2-web-interface/src/components/inputs/InputListItem.tsx +++ b/graylog2-web-interface/src/components/inputs/InputListItem.tsx @@ -45,7 +45,7 @@ import { TELEMETRY_EVENT_TYPE } from 'logic/telemetry/Constants'; import useSendTelemetry from 'logic/telemetry/useSendTelemetry'; import useLocation from 'routing/useLocation'; import useFeature from 'hooks/useFeature'; -import { INPUT_SETUP_MODE_FEATURE_FLAG } from 'components/inputs/InputSetupWizard'; +import { INPUT_SETUP_MODE_FEATURE_FLAG, InputSetupWizard } from 'components/inputs/InputSetupWizard'; type Props = { input: Input, @@ -68,12 +68,21 @@ const InputListItem = ({ input, currentNode, permissions }: Props) => { const [showConfirmDeleteDialog, setShowConfirmDeleteDialog] = useState(false); const [showStaticFieldForm, setShowStaticFieldForm] = useState(false); const [showConfigurationForm, setShowConfigurationForm] = useState(false); + const [showWizard, setShowWizard] = useState(false); const sendTelemetry = useSendTelemetry(); const { pathname } = useLocation(); const { inputTypes, inputDescriptions } = useStore(InputTypesStore); const { inputStates } = useStore(InputStatesStore) as { inputStates: InputStates }; const inputSetupFeatureFlagIsEnabled = useFeature(INPUT_SETUP_MODE_FEATURE_FLAG); + const openWizard = () => { + setShowWizard(true); + }; + + const closeWizard = () => { + setShowWizard(false); + }; + const deleteInput = () => { setShowConfirmDeleteDialog(true); }; @@ -187,7 +196,7 @@ const InputListItem = ({ input, currentNode, permissions }: Props) => { ); } - actions.push(); + actions.push(); } actions.push( @@ -261,6 +270,7 @@ const InputListItem = ({ input, currentNode, permissions }: Props) => { const additionalContent = (
+ {inputSetupFeatureFlagIsEnabled && showWizard && ()} { - const { openWizard, setActiveStep } = useInputSetupWizard(); - - const open = () => { - setActiveStep(INPUT_WIZARD_STEPS.SETUP_ROUTING); - openWizard(wizardData); - }; - - return (); +import InputSetupWizardProvider from './contexts/InputSetupWizardProvider'; +import InputSetupWizard from './Wizard'; + +const input = { + id: 'inputId', + title: 'inputTitle', + type: 'type', + global: false, + name: 'inputName', + created_at: '', + creator_user_id: 'creatorId', + static_fields: { }, + attributes: { }, }; -const renderWizard = (wizardData: WizardData = {}) => ( +const onClose = jest.fn(); + +const renderWizard = () => ( render( - - + , ) ); @@ -170,12 +170,6 @@ const getStreamCreateFormFields = async () => { }; }; -const openWizard = async () => { - const openButton = await screen.findByRole('button', { name: /Open Wizard!/ }); - - fireEvent.click(openButton); -}; - beforeEach(() => { asMock(useStreams).mockReturnValue(useStreamsResult()); asMock(usePipelinesConnectedStream).mockReturnValue(pipelinesConnectedMock()); @@ -185,11 +179,10 @@ beforeEach(() => { describe('InputSetupWizard Setup Routing', () => { it('should render the Setup Routing step', async () => { renderWizard(); - openWizard(); - const wizard = await screen.findByText('Setup Routing'); + const routingStepText = await screen.findByText(/Choose a Destination Stream to route Messages from this Input to./i); - expect(wizard).toBeInTheDocument(); + expect(routingStepText).toBeInTheDocument(); }); it('should only show editable existing streams', async () => { @@ -201,7 +194,6 @@ describe('InputSetupWizard Setup Routing', () => { )); renderWizard(); - openWizard(); const streamSelect = await screen.findByLabelText(/All messages \(Default\)/i); @@ -223,7 +215,6 @@ describe('InputSetupWizard Setup Routing', () => { )); renderWizard(); - openWizard(); const streamSelect = await screen.findByLabelText(/All messages \(Default\)/i); @@ -245,7 +236,6 @@ describe('InputSetupWizard Setup Routing', () => { )); renderWizard(); - openWizard(); const streamSelect = await screen.findByLabelText(/All messages \(Default\)/i); @@ -268,7 +258,6 @@ describe('InputSetupWizard Setup Routing', () => { ])); renderWizard(); - openWizard(); const streamSelect = await screen.findByLabelText(/All messages \(Default\)/i); @@ -290,7 +279,6 @@ describe('InputSetupWizard Setup Routing', () => { asMock(useIndexSetsList).mockReturnValue(useIndexSetsListResult); renderWizard(); - openWizard(); const createStreamButton = await screen.findByRole('button', { name: /Create Stream/i, @@ -328,7 +316,6 @@ describe('InputSetupWizard Setup Routing', () => { asMock(useIndexSetsList).mockReturnValue(useIndexSetsListResult); renderWizard(); - openWizard(); const createStreamButton = await screen.findByRole('button', { name: /Create Stream/i, @@ -369,7 +356,6 @@ describe('InputSetupWizard Setup Routing', () => { asMock(useIndexSetsList).mockReturnValue(useIndexSetsListResult); renderWizard(); - openWizard(); const createStreamButton = await screen.findByRole('button', { name: /Create Stream/i, diff --git a/graylog2-web-interface/src/components/inputs/InputSetupWizard/InputSetupWizard.StartInput.test.tsx b/graylog2-web-interface/src/components/inputs/InputSetupWizard/InputSetupWizard.StartInput.test.tsx new file mode 100644 index 000000000000..be6dbf2d7c92 --- /dev/null +++ b/graylog2-web-interface/src/components/inputs/InputSetupWizard/InputSetupWizard.StartInput.test.tsx @@ -0,0 +1,354 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +import * as React from 'react'; +import { render, screen, fireEvent, waitFor } from 'wrappedTestingLibrary'; +import selectEvent from 'react-select-event'; + +import fetch from 'logic/rest/FetchProvider'; +import { asMock, StoreMock as MockStore } from 'helpers/mocking'; +import usePipelinesConnectedStream from 'hooks/usePipelinesConnectedStream'; +import useStreams from 'components/streams/hooks/useStreams'; +import useIndexSetsList from 'components/indices/hooks/useIndexSetsList'; +import { streams } from 'fixtures/streams'; +import { InputStatesStore } from 'stores/inputs/InputStatesStore'; + +import InputSetupWizardProvider from './contexts/InputSetupWizardProvider'; +import InputSetupWizard from './Wizard'; + +jest.mock('components/streams/hooks/useStreams', () => jest.fn()); +jest.mock('hooks/usePipelinesConnectedStream'); +jest.mock('components/indices/hooks/useIndexSetsList'); +jest.mock('logic/rest/FetchProvider', () => jest.fn(() => Promise.resolve())); +jest.mock('views/stores/StreamsStore', () => ({ StreamsStore: MockStore() })); +jest.mock('stores/system/SystemStore', () => ({ SystemStore: MockStore() })); + +jest.mock('stores/inputs/InputStatesStore', () => ({ + InputStatesStore: MockStore( + ['start', jest.fn(() => Promise.resolve())], + ['stop', jest.fn(() => Promise.resolve())], + ), +})); + +jest.mock('stores/nodes/NodesStore', () => ({ + NodesStore: MockStore(), +})); + +const input = { + id: 'input-test-id', + title: 'inputTitle', + type: 'type', + global: false, + name: 'inputName', + created_at: '', + creator_user_id: 'creatorId', + static_fields: { }, + attributes: { }, +}; + +const onClose = jest.fn(); + +const renderWizard = () => ( + render( + + + , + ) +); + +const useStreamsResult = { + data: { + list: streams, + pagination: { total: 1 }, + attributes: [], + }, + isInitialLoading: false, + isFetching: false, + error: undefined, + refetch: () => {}, +}; + +const pipelinesConnectedMock = (response = []) => ({ + data: response, + refetch: jest.fn(), + isInitialLoading: false, + error: undefined, + isError: false, +}); + +const useIndexSetsListResult = { + data: { + indexSets: + [ + { + id: 'default_id', + title: 'Default', + description: 'default index set', + index_prefix: 'default', + shards: 1, + replicas: 1, + rotation_strategy_class: 'org.graylog2.indexer.rotation.strategies.MessageCountRotationStrategy', + rotation_strategy: { + type: 'org.graylog2.indexer.rotation.strategies.MessageCountRotationStrategyConfig', + max_docs_per_index: 20000000, + }, + retention_strategy_class: 'org.graylog2.indexer.retention.strategies.NoopRetentionStrategy', + retention_strategy: { + type: 'org.graylog2.indexer.retention.strategies.NoopRetentionStrategyConfig', + max_number_of_indices: 2147483647, + }, + index_analyzer: '', + index_optimization_max_num_segments: 0, + index_optimization_disabled: false, + field_type_refresh_interval: 1, + writable: true, + default: true, + can_be_default: true, + }, + { + id: 'nox_id', + title: 'Nox', + description: 'nox index set', + index_prefix: 'nox', + shards: 1, + replicas: 1, + rotation_strategy_class: 'org.graylog2.indexer.rotation.strategies.MessageCountRotationStrategy', + rotation_strategy: { + type: 'org.graylog2.indexer.rotation.strategies.MessageCountRotationStrategyConfig', + max_docs_per_index: 20000000, + }, + retention_strategy_class: 'org.graylog2.indexer.retention.strategies.NoopRetentionStrategy', + retention_strategy: { + type: 'org.graylog2.indexer.retention.strategies.NoopRetentionStrategyConfig', + max_number_of_indices: 2147483647, + }, + index_analyzer: '', + index_optimization_max_num_segments: 0, + index_optimization_disabled: false, + field_type_refresh_interval: 1, + writable: true, + default: false, + can_be_default: true, + }, + ], + indexSetsCount: 2, + indexSetStats: null, + }, + isInitialLoading: false, + isSuccess: true, + error: undefined, + refetch: () => {}, +}; + +const updateRoutingUrlRegEx = /.+(system\/pipelines\/pipeline\/routing)/i; +const createStreamUrl = '/streams'; +const startStreamUrl = (streamId) => `/streams/${streamId}/resume`; +const createPipelineUrlRegEx = /.+(system\/pipelines\/pipeline)/i; + +const newStreamConfig = { + description: 'Wingardium new stream', + index_set_id: 'default_id', + remove_matches_from_default_stream: undefined, + title: 'Wingardium', +}; + +const newPipelineConfig = { + description: 'Pipeline for Stream: Wingardium created by the Input Setup Wizard.', + source: 'pipeline "Wingardium"\nstage 0 match either\nend', + title: 'Wingardium', +}; + +const goToStartInputStep = async () => { + const nextButton = await screen.findByRole('button', { name: /Finish & Start Input/i, hidden: true }); + + fireEvent.click(nextButton); +}; + +const startInput = async () => { + const startInputButton = await screen.findByTestId('start-input-button'); + + fireEvent.click(startInputButton); +}; + +const createStream = async (newPipeline = false) => { + const createStreamButton = await screen.findByRole('button', { + name: /Create Stream/i, + hidden: true, + }); + + fireEvent.click(createStreamButton); + + await screen.findByRole('heading', { name: /Create new stream/i, hidden: true }); + + const titleInput = await screen.findByRole('textbox', { + name: /Title/i, + hidden: true, + }); + + const descriptionInput = await screen.findByRole('textbox', { + name: /Description/i, + hidden: true, + }); + + const newPipelineCheckbox = await screen.findByRole('checkbox', { + name: /Create a new pipeline for this stream/i, + hidden: true, + }); + + const submitButton = await screen.findByRole('button', { + name: 'Create', + hidden: true, + }); + + fireEvent.change(titleInput, { target: { value: 'Wingardium' } }); + fireEvent.change(descriptionInput, { target: { value: 'Wingardium new stream' } }); + + if (newPipeline) { + fireEvent.click(newPipelineCheckbox); + } + + await waitFor(() => expect(submitButton).toBeEnabled()); + fireEvent.click(submitButton); +}; + +beforeEach(() => { + asMock(useStreams).mockReturnValue(useStreamsResult); + asMock(usePipelinesConnectedStream).mockReturnValue(pipelinesConnectedMock()); + asMock(useIndexSetsList).mockReturnValue(useIndexSetsListResult); + asMock(fetch).mockImplementation(() => Promise.resolve({})); +}); + +describe('InputSetupWizard Start Input', () => { + it('should render the Start Input step', async () => { + renderWizard(); + goToStartInputStep(); + + expect(await screen.findByText(/Set up and start the Input according to the configuration made./i)).toBeInTheDocument(); + }); + + it('should start when default stream is selected', async () => { + renderWizard(); + goToStartInputStep(); + startInput(); + + await waitFor(() => expect(InputStatesStore.start).toHaveBeenCalledWith(input)); + + expect(await screen.findByRole('heading', { name: /Setting up Input.../i, hidden: true })).toBeInTheDocument(); + expect(await screen.findByText(/Input started sucessfully!/i)).toBeInTheDocument(); + }); + + it('should start input when an existing stream is selected', async () => { + renderWizard(); + + const streamSelect = await screen.findByLabelText(/All messages \(Default\)/i); + + await selectEvent.openMenu(streamSelect); + + await selectEvent.select(streamSelect, 'One Stream'); + + goToStartInputStep(); + startInput(); + + await waitFor(() => expect(InputStatesStore.start).toHaveBeenCalledWith(input)); + + await waitFor(() => expect(fetch).toHaveBeenCalledWith( + 'PUT', + expect.stringMatching(updateRoutingUrlRegEx), + { input_id: 'input-test-id', stream_id: 'streamId1' }, + )); + + expect(await screen.findByRole('heading', { name: /Setting up Input.../i, hidden: true })).toBeInTheDocument(); + expect(await screen.findByText(/Routing set up!/i)).toBeInTheDocument(); + expect(await screen.findByText(/Input started sucessfully!/i)).toBeInTheDocument(); + }); + + describe('new stream', () => { + it('should show the progress for all steps', async () => { + renderWizard(); + await waitFor(() => createStream(true)); + goToStartInputStep(); + startInput(); + + expect(await screen.findByRole('heading', { name: /Setting up Input.../i, hidden: true })).toBeInTheDocument(); + expect(await screen.findByText(/Stream "Wingardium" created!/i)).toBeInTheDocument(); + expect(await screen.findByText(/Pipeline "Wingardium" created!/i)).toBeInTheDocument(); + expect(await screen.findByText(/Routing set up!/i)).toBeInTheDocument(); + expect(await screen.findByText(/Input started sucessfully!/i)).toBeInTheDocument(); + }); + + it('should start the input', async () => { + renderWizard(); + await waitFor(() => createStream()); + goToStartInputStep(); + startInput(); + + await waitFor(() => expect(InputStatesStore.start).toHaveBeenCalledWith(input)); + }); + + it('should create the new stream', async () => { + renderWizard(); + await waitFor(() => createStream()); + goToStartInputStep(); + startInput(); + + await waitFor(() => expect(fetch).toHaveBeenCalledWith( + 'POST', + expect.stringContaining(createStreamUrl), + newStreamConfig, + )); + }); + + it('should start the new stream', async () => { + asMock(fetch).mockImplementation(() => Promise.resolve({ stream_id: 1 })); + + renderWizard(); + await waitFor(() => createStream()); + goToStartInputStep(); + startInput(); + + await waitFor(() => expect(fetch).toHaveBeenCalledWith( + 'POST', + expect.stringContaining(startStreamUrl(1)), + )); + }); + + it('should create the new pipeline', async () => { + renderWizard(); + await waitFor(() => createStream(true)); + goToStartInputStep(); + startInput(); + + await waitFor(() => expect(fetch).toHaveBeenCalledWith( + 'POST', + expect.stringMatching(createPipelineUrlRegEx), + newPipelineConfig, + )); + }); + + it('create routing for the new stream', async () => { + renderWizard(); + await waitFor(() => createStream(true)); + goToStartInputStep(); + startInput(); + + await waitFor(() => expect(fetch).toHaveBeenCalledWith( + 'PUT', + expect.stringMatching(updateRoutingUrlRegEx), + { input_id: 'input-test-id', stream_id: 'streamId1' }, + )); + }); + }); +}); diff --git a/graylog2-web-interface/src/components/inputs/InputSetupWizard/InputSetupWizard.test.tsx b/graylog2-web-interface/src/components/inputs/InputSetupWizard/InputSetupWizard.test.tsx index d145198e2c4d..812dea79916c 100644 --- a/graylog2-web-interface/src/components/inputs/InputSetupWizard/InputSetupWizard.test.tsx +++ b/graylog2-web-interface/src/components/inputs/InputSetupWizard/InputSetupWizard.test.tsx @@ -15,35 +15,33 @@ * . */ import * as React from 'react'; -import { render, screen, fireEvent } from 'wrappedTestingLibrary'; +import { render, screen } from 'wrappedTestingLibrary'; -import { Button } from 'components/bootstrap'; import { asMock } from 'helpers/mocking'; import useStreams from 'components/streams/hooks/useStreams'; import usePipelinesConnectedStream from 'hooks/usePipelinesConnectedStream'; -import { useInputSetupWizard, InputSetupWizardProvider } from 'components/inputs/InputSetupWizard'; -import type { WizardData } from 'components/inputs/InputSetupWizard'; -import InputSetupWizard from './InputSetupWizard'; - -const OpenWizardTestButton = ({ wizardData } : { wizardData: WizardData}) => { - const { openWizard } = useInputSetupWizard(); - - return (); +import InputSetupWizardProvider from './contexts/InputSetupWizardProvider'; +import InputSetupWizard from './Wizard'; + +const input = { + id: 'inputId', + title: 'inputTitle', + type: 'type', + global: false, + name: 'inputName', + created_at: '', + creator_user_id: 'creatorId', + static_fields: { }, + attributes: { }, }; -const CloseWizardTestButton = () => { - const { closeWizard } = useInputSetupWizard(); - - return (); -}; +const onClose = jest.fn(); -const renderWizard = (wizardData: WizardData = {}) => ( +const renderWizard = () => ( render( - - - + , ) ); @@ -76,28 +74,8 @@ describe('InputSetupWizard', () => { it('should render the wizard and shows routing step as first step', async () => { renderWizard(); - const openButton = await screen.findByRole('button', { name: /Open Wizard!/ }); - - fireEvent.click(openButton); - - const wizard = await screen.findByText('Setup Routing'); - - expect(wizard).toBeInTheDocument(); - }); - - it('should close the wizard', async () => { - renderWizard(); - const openButton = await screen.findByRole('button', { name: /Open Wizard!/ }); - const closeButton = await screen.findByRole('button', { name: /Close Wizard!/ }); - - fireEvent.click(openButton); - const wizard = await screen.findByText('Setup Routing'); expect(wizard).toBeInTheDocument(); - - fireEvent.click(closeButton); - - expect(wizard).not.toBeInTheDocument(); }); }); diff --git a/graylog2-web-interface/src/components/inputs/InputSetupWizard/InputSetupWizard.tsx b/graylog2-web-interface/src/components/inputs/InputSetupWizard/InputSetupWizard.tsx index 089105e1c3f8..7aa54c5344e3 100644 --- a/graylog2-web-interface/src/components/inputs/InputSetupWizard/InputSetupWizard.tsx +++ b/graylog2-web-interface/src/components/inputs/InputSetupWizard/InputSetupWizard.tsx @@ -15,94 +15,21 @@ * . */ import * as React from 'react'; -import { useEffect, useCallback, useMemo } from 'react'; -import { PluginStore } from 'graylog-web-plugin/plugin'; -import { Modal } from 'components/bootstrap'; -import { Wizard } from 'components/common'; -import { INPUT_WIZARD_STEPS } from 'components/inputs/InputSetupWizard/types'; -import useInputSetupWizard from 'components/inputs/InputSetupWizard/hooks/useInputSetupWizard'; -import { getStepData } from 'components/inputs/InputSetupWizard/helpers/stepHelper'; - -import { InputDiagnosisStep, SetupRoutingStep, StartInputStep } from './steps'; - -const InputSetupWizard = () => { - const { activeStep, setActiveStep, show, orderedSteps, setOrderedSteps, stepsData, closeWizard } = useInputSetupWizard(); - const enterpriseSteps = PluginStore.exports('inputSetupWizard').find((plugin) => (!!plugin.steps))?.steps; - - const steps = useMemo(() => { - const defaultSteps = { - [INPUT_WIZARD_STEPS.SETUP_ROUTING]: { - key: INPUT_WIZARD_STEPS.SETUP_ROUTING, - title: ( - <> - Setup Routing - - ), - component: ( - - ), - disabled: false, - }, - [INPUT_WIZARD_STEPS.START_INPUT]: { - key: INPUT_WIZARD_STEPS.START_INPUT, - title: ( - <> - Start Input - - ), - component: ( - - ), - disabled: !getStepData(stepsData, INPUT_WIZARD_STEPS.START_INPUT, 'enabled'), - }, - [INPUT_WIZARD_STEPS.INPUT_DIAGNOSIS]: { - key: INPUT_WIZARD_STEPS.INPUT_DIAGNOSIS, - title: ( - <> - Input Diagnosis - - ), - component: ( - - ), - disabled: !getStepData(stepsData, INPUT_WIZARD_STEPS.INPUT_DIAGNOSIS, 'enabled'), - }, - }; - if (enterpriseSteps) return { ...defaultSteps, ...enterpriseSteps }; - - return defaultSteps; - }, [enterpriseSteps, stepsData]); - - const determineFirstStep = useCallback(() => { - setActiveStep(INPUT_WIZARD_STEPS.SETUP_ROUTING); - setOrderedSteps([INPUT_WIZARD_STEPS.SETUP_ROUTING, INPUT_WIZARD_STEPS.START_INPUT, INPUT_WIZARD_STEPS.INPUT_DIAGNOSIS]); - }, [setActiveStep, setOrderedSteps]); - - useEffect(() => { - if (!activeStep) { - determineFirstStep(); - } - - if (activeStep && orderedSteps.length < 1) { - setOrderedSteps([activeStep]); - } - }, [activeStep, determineFirstStep, orderedSteps, setOrderedSteps]); - - if (!show || orderedSteps.length < 1) return null; - - return ( - - - steps[step])} /> - - - ); -}; +import InputSetupWizardProvider from './contexts/InputSetupWizardProvider'; +import Wizard from './Wizard'; +import type { WizardData } from './types'; + +type Props = { + show: boolean, + input: WizardData['input'], + onClose: () => void, +} + +const InputSetupWizard = ({ show, input, onClose }: Props) => ( + + + +); export default InputSetupWizard; diff --git a/graylog2-web-interface/src/components/inputs/InputSetupWizard/Wizard.tsx b/graylog2-web-interface/src/components/inputs/InputSetupWizard/Wizard.tsx new file mode 100644 index 000000000000..9e117c250efe --- /dev/null +++ b/graylog2-web-interface/src/components/inputs/InputSetupWizard/Wizard.tsx @@ -0,0 +1,137 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +import * as React from 'react'; +import { useEffect, useCallback, useMemo } from 'react'; +import { PluginStore } from 'graylog-web-plugin/plugin'; + +import { Modal } from 'components/bootstrap'; +import { Wizard as CommonWizard } from 'components/common'; +import { INPUT_WIZARD_STEPS } from 'components/inputs/InputSetupWizard/types'; +import useInputSetupWizard from 'components/inputs/InputSetupWizard/hooks/useInputSetupWizard'; +import { getStepConfigOrData } from 'components/inputs/InputSetupWizard/helpers/stepHelper'; + +import InputSetupWizardStepsProvider from './contexts/InputSetupWizardStepsProvider'; +import type { WizardData } from './types'; +import { InputDiagnosisStep, SetupRoutingStep, StartInputStep } from './steps'; + +type Props = { + show: boolean, + input: WizardData['input'], + onClose: () => void, +} + +const Wizard = ({ show, input, onClose }: Props) => { + const { activeStep, setActiveStep, orderedSteps, setOrderedSteps, stepsConfig, setStepsConfig, setWizardData, wizardData } = useInputSetupWizard(); + + const enterpriseSteps = PluginStore.exports('inputSetupWizard').find((plugin) => (!!plugin.steps))?.steps; + + const initialStepsConfig = { + [INPUT_WIZARD_STEPS.SETUP_ROUTING]: { + enabled: true, + }, + [INPUT_WIZARD_STEPS.START_INPUT]: { + enabled: true, + }, + [INPUT_WIZARD_STEPS.INPUT_DIAGNOSIS]: { + enabled: true, + }, + }; + + useEffect(() => { + setStepsConfig(initialStepsConfig); + setWizardData({ ...wizardData, input }); // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); // Initial setup: intentionally ommiting dependencies to prevent from unneccesary rerenders + + const steps = useMemo(() => { + const defaultSteps = { + [INPUT_WIZARD_STEPS.SETUP_ROUTING]: { + key: INPUT_WIZARD_STEPS.SETUP_ROUTING, + title: ( + <> + Setup Routing + + ), + component: ( + + ), + disabled: !getStepConfigOrData(stepsConfig, INPUT_WIZARD_STEPS.SETUP_ROUTING, 'enabled'), + }, + [INPUT_WIZARD_STEPS.START_INPUT]: { + key: INPUT_WIZARD_STEPS.START_INPUT, + title: ( + <> + Start Input + + ), + component: ( + + ), + disabled: !getStepConfigOrData(stepsConfig, INPUT_WIZARD_STEPS.START_INPUT, 'enabled'), + }, + [INPUT_WIZARD_STEPS.INPUT_DIAGNOSIS]: { + key: INPUT_WIZARD_STEPS.INPUT_DIAGNOSIS, + title: ( + <> + Input Diagnosis + + ), + component: ( + + ), + disabled: !getStepConfigOrData(stepsConfig, INPUT_WIZARD_STEPS.INPUT_DIAGNOSIS, 'enabled'), + }, + }; + if (enterpriseSteps) return { ...defaultSteps, ...enterpriseSteps }; + + return defaultSteps; + }, [enterpriseSteps, stepsConfig]); + + const determineFirstStep = useCallback(() => { + setActiveStep(INPUT_WIZARD_STEPS.SETUP_ROUTING); + setOrderedSteps([INPUT_WIZARD_STEPS.SETUP_ROUTING, INPUT_WIZARD_STEPS.START_INPUT, INPUT_WIZARD_STEPS.INPUT_DIAGNOSIS]); + }, [setActiveStep, setOrderedSteps]); + + useEffect(() => { + if (!activeStep) { + determineFirstStep(); + } + + if (activeStep && orderedSteps.length < 1) { + setOrderedSteps([activeStep]); + } + }, [activeStep, determineFirstStep, orderedSteps, setOrderedSteps]); + + if (!show || orderedSteps.length < 1) return null; + + return ( + + Input Setup Wizard + + + steps[step])} /> + + + + ); +}; + +export default Wizard; diff --git a/graylog2-web-interface/src/components/inputs/InputSetupWizard/contexts/InputSetupWizardContext.tsx b/graylog2-web-interface/src/components/inputs/InputSetupWizard/contexts/InputSetupWizardContext.tsx index cc9bcdce7318..d55b0d68986f 100644 --- a/graylog2-web-interface/src/components/inputs/InputSetupWizard/contexts/InputSetupWizardContext.tsx +++ b/graylog2-web-interface/src/components/inputs/InputSetupWizard/contexts/InputSetupWizardContext.tsx @@ -18,22 +18,19 @@ import * as React from 'react'; import { singleton } from 'logic/singleton'; -import type { InputSetupWizardStep, WizardData, StepsData } from 'components/inputs/InputSetupWizard/types'; +import type { InputSetupWizardStep, WizardData, StepsConfig } from 'components/inputs/InputSetupWizard/types'; type InputSetupWizardContextType = { activeStep: InputSetupWizardStep | undefined, - setActiveStep: (InputSetupWizardStep) => void, - stepsData: StepsData, - setStepsData: (stepsData: StepsData) => void, + setActiveStep: (step: InputSetupWizardStep) => void, + stepsConfig: StepsConfig, + setStepsConfig: (stepsConfig: StepsConfig) => void, wizardData: WizardData, - updateWizardData: (key: keyof WizardData, value: WizardData[typeof key]) => void, + setWizardData: (wizardData: WizardData) => void; orderedSteps: Array, setOrderedSteps: (steps: Array) => void, - show: boolean, goToPreviousStep: () => void, goToNextStep: (step?: InputSetupWizardStep) => void, - openWizard: (data?: WizardData) => void, - closeWizard: () => void, }; const InputSetupWizardContext = React.createContext(undefined); diff --git a/graylog2-web-interface/src/components/inputs/InputSetupWizard/contexts/InputSetupWizardProvider.test.tsx b/graylog2-web-interface/src/components/inputs/InputSetupWizard/contexts/InputSetupWizardProvider.test.tsx deleted file mode 100644 index d17dccacb455..000000000000 --- a/graylog2-web-interface/src/components/inputs/InputSetupWizard/contexts/InputSetupWizardProvider.test.tsx +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (C) 2020 Graylog, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * . - */ -import * as React from 'react'; -import { render } from 'wrappedTestingLibrary'; - -import InputSetupWizardContext from './InputSetupWizardContext'; -import InputSetupWizardProvider from './InputSetupWizardProvider'; - -const contextData = { - activeStep: undefined, - stepsData: {}, - wizardData: {}, - orderedSteps: [], - show: false, -}; - -const renderSUT = () => { - const consume = jest.fn(); - - render( - - - {consume} - - , - ); - - return consume; -}; - -describe('', () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - - it('should render InputSetupWizardProvider', () => { - const consume = renderSUT(); - - expect(consume).toHaveBeenCalledWith(expect.objectContaining(contextData)); - }); -}); diff --git a/graylog2-web-interface/src/components/inputs/InputSetupWizard/contexts/InputSetupWizardProvider.tsx b/graylog2-web-interface/src/components/inputs/InputSetupWizard/contexts/InputSetupWizardProvider.tsx index 4a3ce87ad986..894398c62c46 100644 --- a/graylog2-web-interface/src/components/inputs/InputSetupWizard/contexts/InputSetupWizardProvider.tsx +++ b/graylog2-web-interface/src/components/inputs/InputSetupWizard/contexts/InputSetupWizardProvider.tsx @@ -19,42 +19,17 @@ import * as React from 'react'; import { useCallback, useMemo, useState } from 'react'; import InputSetupWizardContext from 'components/inputs/InputSetupWizard/contexts/InputSetupWizardContext'; -import type { InputSetupWizardStep, StepsData, WizardData } from 'components/inputs/InputSetupWizard/types'; +import type { InputSetupWizardStep, WizardData, StepsConfig } from 'components/inputs/InputSetupWizard/types'; import { addStepAfter, getNextStep, checkHasPreviousStep, checkIsNextStepDisabled } from 'components/inputs/InputSetupWizard/helpers/stepHelper'; const DEFAULT_ACTIVE_STEP = undefined; const DEFAULT_WIZARD_DATA = {}; -const DEFAULT_STEPS_DATA = {}; const InputSetupWizardProvider = ({ children = null }: React.PropsWithChildren<{}>) => { const [activeStep, setActiveStep] = useState(DEFAULT_ACTIVE_STEP); const [wizardData, setWizardData] = useState(DEFAULT_WIZARD_DATA); const [orderedSteps, setOrderedSteps] = useState>([]); - const [stepsData, setStepsData] = useState(DEFAULT_STEPS_DATA); - const [show, setShow] = useState(false); - - const updateWizardData = useCallback( - (key: keyof WizardData, value: WizardData[typeof key]) => { - setWizardData({ ...wizardData, [key]: value }); - }, - [wizardData], - ); - - const clearWizard = useCallback(() => { - setActiveStep(DEFAULT_ACTIVE_STEP); - setWizardData(DEFAULT_WIZARD_DATA); - setStepsData(DEFAULT_STEPS_DATA); - }, []); - - const closeWizard = useCallback(() => { - clearWizard(); - setShow(false); - }, [clearWizard]); - - const openWizard = useCallback((data: WizardData = {}) => { - setWizardData({ ...wizardData, ...data }); - setShow(true); - }, [wizardData]); + const [stepsConfig, setStepsConfig] = useState({}); const goToNextStep = useCallback((step?: InputSetupWizardStep) => { const nextStep = step ?? getNextStep(orderedSteps, activeStep); @@ -66,12 +41,12 @@ const InputSetupWizardProvider = ({ children = null }: React.PropsWithChildren<{ if (!nextStep) return; - if (checkIsNextStepDisabled(orderedSteps, activeStep, stepsData, nextStep)) return; + if (checkIsNextStepDisabled(orderedSteps, activeStep, stepsConfig, nextStep)) return; const nextStepIndex = orderedSteps.indexOf(nextStep); setActiveStep(orderedSteps[nextStepIndex]); - }, [activeStep, orderedSteps, stepsData]); + }, [activeStep, orderedSteps, stepsConfig]); const goToPreviousStep = useCallback(() => { if (!checkHasPreviousStep(orderedSteps, activeStep)) return; @@ -84,29 +59,21 @@ const InputSetupWizardProvider = ({ children = null }: React.PropsWithChildren<{ const value = useMemo(() => ({ setActiveStep, activeStep, - stepsData, - setStepsData, wizardData, - updateWizardData, - show, + setWizardData, orderedSteps, setOrderedSteps, + stepsConfig, + setStepsConfig, goToPreviousStep, goToNextStep, - openWizard, - closeWizard, }), [ activeStep, - stepsData, - setStepsData, wizardData, - updateWizardData, - show, orderedSteps, + stepsConfig, goToPreviousStep, goToNextStep, - openWizard, - closeWizard, ]); return ( diff --git a/graylog2-web-interface/src/components/inputs/InputSetupWizard/steps/components/CreateStream.tsx b/graylog2-web-interface/src/components/inputs/InputSetupWizard/contexts/InputSetupWizardStepsContext.tsx similarity index 59% rename from graylog2-web-interface/src/components/inputs/InputSetupWizard/steps/components/CreateStream.tsx rename to graylog2-web-interface/src/components/inputs/InputSetupWizard/contexts/InputSetupWizardStepsContext.tsx index a0e4ebf19ff3..eb11d67aa9f5 100644 --- a/graylog2-web-interface/src/components/inputs/InputSetupWizard/steps/components/CreateStream.tsx +++ b/graylog2-web-interface/src/components/inputs/InputSetupWizard/contexts/InputSetupWizardStepsContext.tsx @@ -14,18 +14,17 @@ * along with this program. If not, see * . */ + import * as React from 'react'; -import { Row, Col } from 'components/bootstrap'; +import { singleton } from 'logic/singleton'; +import type { StepsData } from 'components/inputs/InputSetupWizard/types'; + +type InputSetupWizardStepsContextType = { + stepsData: StepsData, + setStepsData: (stepsData: StepsData) => void, +}; -const CreateStream = () => ( - - -

- Create new stream -

- - -); +const InputSetupWizardStepsContext = React.createContext(undefined); -export default CreateStream; +export default singleton('contexts.InputSetupWizardStepsContext', () => InputSetupWizardStepsContext); diff --git a/graylog2-web-interface/src/components/inputs/InputSetupWizard/contexts/InputSetupWizardStepsProvider.tsx b/graylog2-web-interface/src/components/inputs/InputSetupWizard/contexts/InputSetupWizardStepsProvider.tsx new file mode 100644 index 000000000000..46b2d94a028a --- /dev/null +++ b/graylog2-web-interface/src/components/inputs/InputSetupWizard/contexts/InputSetupWizardStepsProvider.tsx @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ + +import * as React from 'react'; +import { useMemo, useState } from 'react'; + +import InputSetupWizardStepsContext from 'components/inputs/InputSetupWizard/contexts/InputSetupWizardStepsContext'; +import type { StepsData } from 'components/inputs/InputSetupWizard/types'; + +const DEFAULT_STEPS_DATA = {}; + +const InputSetupWizardStepsProvider = ({ children = null }: React.PropsWithChildren<{}>) => { + const [stepsData, setStepsData] = useState(DEFAULT_STEPS_DATA); + + const value = useMemo(() => ({ + stepsData, + setStepsData, + }), [ + stepsData, + ]); + + return ( + + {children} + + ); +}; + +export default InputSetupWizardStepsProvider; diff --git a/graylog2-web-interface/src/components/inputs/InputSetupWizard/helpers/stepHelper.test.ts b/graylog2-web-interface/src/components/inputs/InputSetupWizard/helpers/stepHelper.test.ts index eb9d942648cc..ff434d5d22b3 100644 --- a/graylog2-web-interface/src/components/inputs/InputSetupWizard/helpers/stepHelper.test.ts +++ b/graylog2-web-interface/src/components/inputs/InputSetupWizard/helpers/stepHelper.test.ts @@ -16,16 +16,16 @@ */ import { INPUT_WIZARD_STEPS } from 'components/inputs/InputSetupWizard/types'; -import type { StepData, StepsData } from 'components/inputs/InputSetupWizard/types'; +import type { StepsData } from 'components/inputs/InputSetupWizard/types'; import { - getStepData, + getStepConfigOrData, getNextStep, checkHasNextStep, checkHasPreviousStep, checkIsNextStepDisabled, addStepAfter, - updateStepData, + updateStepConfigOrData, enableNextStep, } from './stepHelper'; @@ -43,15 +43,15 @@ const stepsData = { const orderedSteps = [INPUT_WIZARD_STEPS.SELECT_CATEGORY, INPUT_WIZARD_STEPS.INPUT_DIAGNOSIS]; describe('stepHelper', () => { - describe('getStepData', () => { + describe('getStepConfigOrData', () => { it('returns data for specific step', () => { - expect(getStepData(stepsData as StepsData, INPUT_WIZARD_STEPS.INPUT_DIAGNOSIS)).toEqual( + expect(getStepConfigOrData(stepsData as StepsData, INPUT_WIZARD_STEPS.INPUT_DIAGNOSIS)).toEqual( stepsData[INPUT_WIZARD_STEPS.INPUT_DIAGNOSIS], ); }); it('returns undefined if no step data exists', () => { - expect(getStepData(stepsData as StepsData, INPUT_WIZARD_STEPS.SETUP_ROUTING)).toEqual( + expect(getStepConfigOrData(stepsData as StepsData, INPUT_WIZARD_STEPS.SETUP_ROUTING)).toEqual( undefined, ); }); @@ -221,7 +221,7 @@ describe('stepHelper', () => { }); }); - describe('updateStepData', () => { + describe('updateStepConfigOrData', () => { it('returns updated steps data with new attribute', () => { const testStepsData = { [INPUT_WIZARD_STEPS.INPUT_DIAGNOSIS]: { @@ -232,7 +232,7 @@ describe('stepHelper', () => { }, }; - expect(updateStepData(testStepsData as StepsData, INPUT_WIZARD_STEPS.INPUT_DIAGNOSIS, { foo: 'bar' } as StepData)).toEqual({ + expect(updateStepConfigOrData(testStepsData as StepsData, INPUT_WIZARD_STEPS.INPUT_DIAGNOSIS, { foo: 'bar' })).toEqual({ [INPUT_WIZARD_STEPS.INPUT_DIAGNOSIS]: { enabled: false, foo: 'bar', @@ -255,7 +255,7 @@ describe('stepHelper', () => { }, }; - expect(updateStepData(testStepsData as StepsData, INPUT_WIZARD_STEPS.SELECT_CATEGORY, { foo: 'bar' } as StepData)).toEqual({ + expect(updateStepConfigOrData(testStepsData as StepsData, INPUT_WIZARD_STEPS.SELECT_CATEGORY, { foo: 'bar' })).toEqual({ [INPUT_WIZARD_STEPS.INPUT_DIAGNOSIS]: { enabled: false, foo: 'foo', @@ -279,7 +279,7 @@ describe('stepHelper', () => { }, }; - expect(updateStepData(testStepsData as StepsData, INPUT_WIZARD_STEPS.SELECT_CATEGORY, { foo: 'bar' } as StepData, true)).toEqual({ + expect(updateStepConfigOrData(testStepsData as StepsData, INPUT_WIZARD_STEPS.SELECT_CATEGORY, { foo: 'bar' }, true)).toEqual({ [INPUT_WIZARD_STEPS.INPUT_DIAGNOSIS]: { enabled: false, foo: 'foo', @@ -302,7 +302,7 @@ describe('stepHelper', () => { }, }; - expect(updateStepData(testStepsData as StepsData, INPUT_WIZARD_STEPS.SELECT_CATEGORY, {} as StepData)).toEqual(testStepsData); + expect(updateStepConfigOrData(testStepsData as StepsData, INPUT_WIZARD_STEPS.SELECT_CATEGORY, {})).toEqual(testStepsData); }); it('returns updated steps data when no step data existed', () => { @@ -313,7 +313,7 @@ describe('stepHelper', () => { }, }; - expect(updateStepData(testStepsData as StepsData, INPUT_WIZARD_STEPS.SELECT_CATEGORY, { foo: 'bar' } as StepData)).toEqual({ + expect(updateStepConfigOrData(testStepsData as StepsData, INPUT_WIZARD_STEPS.SELECT_CATEGORY, { foo: 'bar' })).toEqual({ [INPUT_WIZARD_STEPS.INPUT_DIAGNOSIS]: { enabled: false, foo: 'foo', @@ -325,7 +325,7 @@ describe('stepHelper', () => { }); it('returns new steps data when no steps data existed', () => { - expect(updateStepData(undefined, INPUT_WIZARD_STEPS.SELECT_CATEGORY, { foo: 'bar' } as StepData)).toEqual({ + expect(updateStepConfigOrData(undefined, INPUT_WIZARD_STEPS.SELECT_CATEGORY, { foo: 'bar' })).toEqual({ [INPUT_WIZARD_STEPS.SELECT_CATEGORY]: { foo: 'bar', }, @@ -333,7 +333,7 @@ describe('stepHelper', () => { }); it('returns empty object when no step name is given', () => { - expect(updateStepData(undefined, undefined, { foo: 'bar' } as StepData)).toEqual({}); + expect(updateStepConfigOrData(undefined, undefined, { foo: 'bar' })).toEqual({}); }); }); diff --git a/graylog2-web-interface/src/components/inputs/InputSetupWizard/helpers/stepHelper.ts b/graylog2-web-interface/src/components/inputs/InputSetupWizard/helpers/stepHelper.ts index a7bebd4c1d53..0fda9a4eacd4 100644 --- a/graylog2-web-interface/src/components/inputs/InputSetupWizard/helpers/stepHelper.ts +++ b/graylog2-web-interface/src/components/inputs/InputSetupWizard/helpers/stepHelper.ts @@ -15,12 +15,12 @@ * . */ -import type { InputSetupWizardStep, StepsData, StepData } from 'components/inputs/InputSetupWizard/types'; +import type { InputSetupWizardStep, StepsConfig, StepsData } from 'components/inputs/InputSetupWizard/types'; -export const getStepData = (stepsData: StepsData, stepName: InputSetupWizardStep, key?: string) => { - if (key) return stepsData[stepName] ? stepsData[stepName][key] : undefined; +export const getStepConfigOrData = (configOrData: StepsConfig | StepsData, stepName: InputSetupWizardStep, key?: string) => { + if (key) return configOrData[stepName] ? configOrData[stepName][key] : undefined; - return stepsData[stepName]; + return configOrData[stepName]; }; export const getNextStep = (orderedSteps: Array, activeStep: InputSetupWizardStep) : InputSetupWizardStep | undefined => { @@ -31,10 +31,10 @@ export const getNextStep = (orderedSteps: Array, activeSte return orderedSteps[activeStepIndex + 1]; }; -export const checkIsNextStepDisabled = (orderedSteps: Array, activeStep: InputSetupWizardStep, stepsData: StepsData, step?: InputSetupWizardStep) => { +export const checkIsNextStepDisabled = (orderedSteps: Array, activeStep: InputSetupWizardStep, stepsConfig: StepsConfig, step?: InputSetupWizardStep) => { const nextStep = step ?? getNextStep(orderedSteps, activeStep); - return !stepsData[nextStep]?.enabled; + return !stepsConfig[nextStep]?.enabled; }; export const checkHasNextStep = (orderedSteps: Array, activeStep: InputSetupWizardStep) => { @@ -71,27 +71,40 @@ export const addStepAfter = (orderedSteps: Array, step: In return newOrderedSteps; }; -export const updateStepData = (stepsData: StepsData, stepName: InputSetupWizardStep, data: StepData = {}, override: boolean = false) : StepsData => { +export const updateStepConfigOrData = (configOrData: StepsConfig | StepsData, stepName: InputSetupWizardStep, data: object = {}, override: boolean = false) : StepsConfig | StepsData => { if (!stepName) return {}; - if (!stepsData) return { [stepName]: data }; + if (!configOrData) return { [stepName]: data }; if (override) { - return { ...stepsData, [stepName]: data }; + return { ...configOrData, [stepName]: data }; } - return { ...stepsData, [stepName]: { ...stepsData[stepName], ...data } }; + return { ...configOrData, [stepName]: { ...configOrData[stepName], ...data } }; }; export const enableNextStep = ( orderedSteps: Array, activeStep: InputSetupWizardStep, - stepsData: StepsData, + stepsConfig: StepsConfig, step?: InputSetupWizardStep, -) : StepsData => { +) : StepsConfig => { const nextStep = step ?? getNextStep(orderedSteps, activeStep); - if (!nextStep) return stepsData; + if (!nextStep) return stepsConfig; - return updateStepData(stepsData, nextStep, { enabled: true }); + return updateStepConfigOrData(stepsConfig, nextStep, { enabled: true }); +}; + +export const disableNextStep = ( + orderedSteps: Array, + activeStep: InputSetupWizardStep, + stepsConfig: StepsConfig, + step?: InputSetupWizardStep, +) : StepsConfig => { + const nextStep = step ?? getNextStep(orderedSteps, activeStep); + + if (!nextStep) return stepsConfig; + + return updateStepConfigOrData(stepsConfig, nextStep, { enabled: false }); }; diff --git a/graylog2-web-interface/src/components/inputs/InputSetupWizard/hooks/useInputSetupWizardSteps.ts b/graylog2-web-interface/src/components/inputs/InputSetupWizard/hooks/useInputSetupWizardSteps.ts new file mode 100644 index 000000000000..92edc59e81b3 --- /dev/null +++ b/graylog2-web-interface/src/components/inputs/InputSetupWizard/hooks/useInputSetupWizardSteps.ts @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +import { useContext } from 'react'; + +import InputSetupWizardStepsContext from 'components/inputs/InputSetupWizard/contexts/InputSetupWizardStepsContext'; + +const useInputSetupWizardSteps = () => { + const inputSetupWizardSteps = useContext(InputSetupWizardStepsContext); + + if (!inputSetupWizardSteps) { + throw new Error('useInputSetupWizardSteps hook needs to be used inside InputSetupWizardContextSteps.Provider'); + } + + return inputSetupWizardSteps; +}; + +export default useInputSetupWizardSteps; diff --git a/graylog2-web-interface/src/components/inputs/InputSetupWizard/hooks/useSetupInputMutations.ts b/graylog2-web-interface/src/components/inputs/InputSetupWizard/hooks/useSetupInputMutations.ts new file mode 100644 index 000000000000..575670b352dd --- /dev/null +++ b/graylog2-web-interface/src/components/inputs/InputSetupWizard/hooks/useSetupInputMutations.ts @@ -0,0 +1,104 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +import { useMutation } from '@tanstack/react-query'; + +import { qualifyUrl } from 'util/URLUtils'; +import fetch from 'logic/rest/FetchProvider'; +import ApiRoutes from 'routing/ApiRoutes'; +import type { Stream } from 'logic/streams/types'; +import type { PipelineType } from 'stores/pipelines/PipelinesStore'; + +export type RoutingParams = { + stream_id?: string, + input_id: string, +} + +export type StreamConfiguration = Pick & Partial> + +type PipelineConfiguration = Pick + +const createStream = async (stream: StreamConfiguration): Promise<{ stream_id: string }> => { + const url = qualifyUrl(ApiRoutes.StreamsApiController.create().url); + + return fetch('POST', url, stream); +}; + +const startStream = async (streamId) => { + const url = qualifyUrl(ApiRoutes.StreamsApiController.resume(streamId).url); + + return fetch('POST', url); +}; + +const createPipeline = (pipeline: PipelineConfiguration) : Promise => { + const url = qualifyUrl(ApiRoutes.PipelinesController.create().url); + + return fetch('POST', url, pipeline); +}; + +const updateRouting = async (params: RoutingParams): Promise <{ id: string }> => { + const url = qualifyUrl(ApiRoutes.PipelinesController.updateRouting().url); + + return fetch('PUT', url, params); +}; + +const deleteStream = async (streamId: string) => { + const url = qualifyUrl(ApiRoutes.StreamsApiController.delete(streamId).url); + + return fetch('DELETE', url); +}; + +const deletePipeline = async (pipelineId: string) => { + const url = qualifyUrl(ApiRoutes.PipelinesController.delete(pipelineId).url); + + return fetch('DELETE', url); +}; + +const deleteRoutingRule = async (ruleId: string) => { + const url = qualifyUrl(ApiRoutes.RulesController.delete(ruleId).url); + + return fetch('DELETE', url); +}; + +const usePipelineRoutingMutation = () => { + const createStreamMutation = useMutation(createStream); + const startStreamMutation = useMutation(startStream); + const createPipelineMutation = useMutation(createPipeline); + const updateRoutingMutation = useMutation(updateRouting); + const deleteStreamMutation = useMutation(deleteStream); + const deletePipelineMutation = useMutation(deletePipeline); + const deleteRoutingRuleMutation = useMutation(deleteRoutingRule); + + return ({ + createStreamMutation, + startStreamMutation, + createPipelineMutation, + updateRoutingMutation, + deleteStreamMutation, + deletePipelineMutation, + deleteRoutingRuleMutation, + }); +}; + +export default usePipelineRoutingMutation; diff --git a/graylog2-web-interface/src/components/inputs/InputSetupWizard/index.ts b/graylog2-web-interface/src/components/inputs/InputSetupWizard/index.ts index 08f93eb002ac..a92691102861 100644 --- a/graylog2-web-interface/src/components/inputs/InputSetupWizard/index.ts +++ b/graylog2-web-interface/src/components/inputs/InputSetupWizard/index.ts @@ -15,8 +15,6 @@ * . */ -export { default as InputSetupWizardProvider } from './contexts/InputSetupWizardProvider'; export { default as InputSetupWizard } from './InputSetupWizard'; -export { default as useInputSetupWizard } from './hooks/useInputSetupWizard'; export * from './types'; -export * from './constants'; +export { INPUT_SETUP_MODE_FEATURE_FLAG } from './constants'; diff --git a/graylog2-web-interface/src/components/inputs/InputSetupWizard/steps/SetupRoutingStep.tsx b/graylog2-web-interface/src/components/inputs/InputSetupWizard/steps/SetupRoutingStep.tsx index 8cec1fa08e48..e98db578c56b 100644 --- a/graylog2-web-interface/src/components/inputs/InputSetupWizard/steps/SetupRoutingStep.tsx +++ b/graylog2-web-interface/src/components/inputs/InputSetupWizard/steps/SetupRoutingStep.tsx @@ -15,18 +15,18 @@ * . */ import * as React from 'react'; -import { useEffect, useMemo, useState } from 'react'; +import { useEffect, useMemo, useState, useCallback } from 'react'; import styled, { css } from 'styled-components'; import { Alert, Button, Row, Col } from 'components/bootstrap'; import { Select } from 'components/common'; import useInputSetupWizard from 'components/inputs/InputSetupWizard/hooks/useInputSetupWizard'; +import useInputSetupWizardSteps from 'components/inputs/InputSetupWizard/hooks/useInputSetupWizardSteps'; import { defaultCompare } from 'logic/DefaultCompare'; -import type { StepData } from 'components/inputs/InputSetupWizard/types'; import { INPUT_WIZARD_STEPS } from 'components/inputs/InputSetupWizard/types'; import CreateStreamForm from 'components/inputs/InputSetupWizard/steps/components/CreateStreamForm'; -import type { FormValues as CreateStreamFormValues } from 'components/inputs/InputSetupWizard/steps/components/CreateStreamForm'; -import { checkHasPreviousStep, checkHasNextStep, checkIsNextStepDisabled, enableNextStep, updateStepData, getStepData } from 'components/inputs/InputSetupWizard/helpers/stepHelper'; +import type { StreamFormValues } from 'components/inputs/InputSetupWizard/steps/components/CreateStreamForm'; +import { checkHasPreviousStep, checkHasNextStep, checkIsNextStepDisabled, updateStepConfigOrData, getStepConfigOrData } from 'components/inputs/InputSetupWizard/helpers/stepHelper'; import useStreams from 'components/streams/hooks/useStreams'; import usePipelinesConnectedStream from 'hooks/usePipelinesConnectedStream'; @@ -67,30 +67,61 @@ const ConntectedPipelinesList = styled.ul` padding-left: 20px; `; -interface RoutingStepData extends StepData { +export type RoutingStepData = { streamId?: string, - newStream?: CreateStreamFormValues + newStream?: StreamFormValues, + shouldCreateNewPipeline?: boolean, + streamType: 'NEW' | 'EXISTING' | 'DEFAULT' } const SetupRoutingStep = () => { - const currentStepName = INPUT_WIZARD_STEPS.SETUP_ROUTING; - const { goToPreviousStep, goToNextStep, orderedSteps, activeStep, stepsData, setStepsData } = useInputSetupWizard(); - const newStream: CreateStreamFormValues = getStepData(stepsData, currentStepName, 'newStream'); + const currentStepName = useMemo(() => INPUT_WIZARD_STEPS.SETUP_ROUTING, []); + const { goToPreviousStep, goToNextStep, orderedSteps, activeStep, stepsConfig } = useInputSetupWizard(); + const { stepsData, setStepsData } = useInputSetupWizardSteps(); + const newStream: StreamFormValues = getStepConfigOrData(stepsData, currentStepName, 'newStream'); const [selectedStreamId, setSelectedStreamId] = useState(undefined); const [showCreateStream, setShowCreateStream] = useState(false); const hasPreviousStep = checkHasPreviousStep(orderedSteps, activeStep); const hasNextStep = checkHasNextStep(orderedSteps, activeStep); - const isNextStepDisabled = checkIsNextStepDisabled(orderedSteps, activeStep, stepsData); + const isNextStepDisabled = checkIsNextStepDisabled(orderedSteps, activeStep, stepsConfig); const { data: streamsData, isInitialLoading: isLoadingStreams } = useStreams({ query: '', page: 1, pageSize: 0, sort: { direction: 'asc', attributeId: 'title' } }); const streams = streamsData?.list; const { data: streamPipelinesData } = usePipelinesConnectedStream(selectedStreamId, !!selectedStreamId); + const defaultStepData: RoutingStepData = { streamType: 'DEFAULT' }; + + const isStepValid = useCallback(() => { + const stepData = getStepConfigOrData(stepsData, currentStepName); + if (!stepData) return false; + const { streamType, streamId } = stepData; + + if (showCreateStream && !newStream) return false; + + switch (streamType) { + case 'NEW': + if (!newStream) return false; + + return true; + case 'EXISTING': + if (!streamId) return false; + + return true; + case 'DEFAULT': + return true; + default: + return false; + } + }, [currentStepName, newStream, showCreateStream, stepsData]); + useEffect(() => { if (orderedSteps && activeStep && stepsData) { - const withNextStepEnabled = enableNextStep(orderedSteps, activeStep, stepsData); - setStepsData(withNextStepEnabled); + if (!getStepConfigOrData(stepsData, currentStepName)?.streamType) { + const withInitialStepsData = updateStepConfigOrData(stepsData, currentStepName, defaultStepData); + + setStepsData(withInitialStepsData); + } } // eslint-disable-next-line react-hooks/exhaustive-deps - }, []); + }, []); // Initial setup: intentionally ommiting dependencies to prevent from unneccesary rerenders const options = useMemo(() => { if (!streams) return []; @@ -103,19 +134,32 @@ const SetupRoutingStep = () => { const handleStreamSelect = (streamId: string) => { setSelectedStreamId(streamId); + + if (streamId) { + setStepsData( + updateStepConfigOrData(stepsData, currentStepName, { streamId, streamType: 'EXISTING' } as RoutingStepData), + ); + } else { + setStepsData( + updateStepConfigOrData(stepsData, currentStepName, { streamId: undefined, streamType: 'DEFAULT' } as RoutingStepData), + ); + } }; - const onNextStep = () => { - setStepsData( - updateStepData(stepsData, currentStepName, { streamId: selectedStreamId } as RoutingStepData), - ); + const handleCreateStream = () => { + setSelectedStreamId(undefined); + updateStepConfigOrData(stepsData, currentStepName, defaultStepData); + setShowCreateStream(true); + }; + + const onNextStep = () => { goToNextStep(); }; const handleBackClick = () => { setStepsData( - updateStepData(stepsData, currentStepName, {} as RoutingStepData, true), + updateStepConfigOrData(stepsData, currentStepName, defaultStepData, true), ); setShowCreateStream(false); @@ -125,9 +169,13 @@ const SetupRoutingStep = () => { const streamHasConnectedPipelines = streamPipelinesData && streamPipelinesData?.length > 0; - const submitStreamCreation = (values: CreateStreamFormValues) => { + const submitStreamCreation = ({ create_new_pipeline, ...stream }: StreamFormValues & { create_new_pipeline?: boolean }) => { setStepsData( - updateStepData(stepsData, currentStepName, { newStream: values } as RoutingStepData), + updateStepConfigOrData(stepsData, currentStepName, { + newStream: stream, + shouldCreateNewPipeline: create_new_pipeline ?? false, + streamType: 'NEW', + } as RoutingStepData), ); }; @@ -140,8 +188,8 @@ const SetupRoutingStep = () => {

- Choose a Destination Stream to Route Messages from this Input to. Messages that are not - routed to any streams will be sent to the "All Messages" Stream. + Choose a Destination Stream to route Messages from this Input to. Messages that are not + routed to any Streams will be sent to the "All Messages" Stream.

@@ -149,7 +197,7 @@ const SetupRoutingStep = () => { - The selected stream has existing pipelines connected to it: + The selected Stream has existing Pipelines connected to it: {streamPipelinesData.map((pipeline) =>
  • {pipeline.title}
    • )}     @@ -161,13 +209,13 @@ const SetupRoutingStep = () => { - Create new stream + Create new Stream {newStream ? ( <> -

    This input will use a new stream: "{newStream.title}".

    -

    Matches will {!newStream.remove_matches_from_default_stream && ('not ')}be removed from the Default stream.

    - {newStream.create_new_pipeline && (

    A new pipeline will be created.

    )} +

    This Input will use a new stream: "{newStream.title}".

    +

    Matches will {!newStream.remove_matches_from_default_stream && ('not ')}be removed from the Default Stream.

    + {getStepConfigOrData(stepsData, currentStepName, 'shouldCreateNewPipeline') && (

    A new Pipeline will be created.

    )} ) : ( @@ -190,7 +238,7 @@ const SetupRoutingStep = () => { Route to a new Stream - +     )} @@ -198,7 +246,7 @@ const SetupRoutingStep = () => { {(hasPreviousStep || showNewStreamSection) && ()} - {hasNextStep && ()} + {hasNextStep && ()} )} diff --git a/graylog2-web-interface/src/components/inputs/InputSetupWizard/steps/StartInputStep.tsx b/graylog2-web-interface/src/components/inputs/InputSetupWizard/steps/StartInputStep.tsx index af7aaae86783..2390fea0bdc6 100644 --- a/graylog2-web-interface/src/components/inputs/InputSetupWizard/steps/StartInputStep.tsx +++ b/graylog2-web-interface/src/components/inputs/InputSetupWizard/steps/StartInputStep.tsx @@ -15,11 +15,363 @@ * . */ import * as React from 'react'; +import { useEffect, useState, useMemo } from 'react'; +import styled, { css } from 'styled-components'; +import type { UseMutationResult } from '@tanstack/react-query'; -const StartInputStep = () => ( -
    - Start Input -
    -); +import useSetupInputMutations from 'components/inputs/InputSetupWizard/hooks/useSetupInputMutations'; +import { InputStatesStore } from 'stores/inputs/InputStatesStore'; +import { Button, Row, Col } from 'components/bootstrap'; +import useInputSetupWizard from 'components/inputs/InputSetupWizard/hooks/useInputSetupWizard'; +import useInputSetupWizardSteps from 'components/inputs/InputSetupWizard//hooks/useInputSetupWizardSteps'; +import { INPUT_WIZARD_STEPS } from 'components/inputs/InputSetupWizard/types'; +import { checkHasPreviousStep, checkHasNextStep, checkIsNextStepDisabled, getStepConfigOrData } from 'components/inputs/InputSetupWizard/helpers/stepHelper'; +import type { RoutingStepData } from 'components/inputs/InputSetupWizard/steps/SetupRoutingStep'; +import SourceGenerator from 'logic/pipelines/SourceGenerator'; +import type { StreamConfiguration } from 'components/inputs/InputSetupWizard/hooks/useSetupInputMutations'; +import ProgressMessage from 'components/inputs/InputSetupWizard/steps/components/ProgressMessage'; + +const StepCol = styled(Col)(({ theme }) => css` + padding-left: ${theme.spacings.lg}; + padding-right: ${theme.spacings.lg}; + padding-top: ${theme.spacings.sm}; +`); + +const DescriptionCol = styled(Col)(({ theme }) => css` + margin-bottom: ${theme.spacings.md}; +`); + +const StyledHeading = styled.h3(({ theme }) => css` + margin-bottom: ${theme.spacings.md}; +`); + +const ButtonCol = styled(Col)(({ theme }) => css` + display: flex; + justify-content: flex-end; + gap: ${theme.spacings.xs}; + margin-top: ${theme.spacings.lg}; +`); + +export type ProcessingSteps = 'createStream' | 'startStream' | 'createPipeline' | 'setupRouting' | 'deleteStream' | 'deletePipeline' | 'deleteRouting' | 'result'; + +const StartInputStep = () => { + const { goToPreviousStep, goToNextStep, orderedSteps, activeStep, wizardData, stepsConfig } = useInputSetupWizard(); + const { stepsData } = useInputSetupWizardSteps(); + const hasPreviousStep = checkHasPreviousStep(orderedSteps, activeStep); + const hasNextStep = checkHasNextStep(orderedSteps, activeStep); + const isNextStepDisabled = checkIsNextStepDisabled(orderedSteps, activeStep, stepsConfig); + const [startInputStatus, setStartInputStatus] = useState<'NOT_STARTED' | 'RUNNING' | 'SUCCESS' | 'FAILED' | 'ROLLED_BACK' | 'ROLLING_BACK'>('NOT_STARTED'); + const isRunning = startInputStatus === 'RUNNING' || startInputStatus === 'ROLLING_BACK'; + const hasBeenStarted = startInputStatus !== 'NOT_STARTED'; + const isRollback = startInputStatus === 'ROLLING_BACK' || startInputStatus === 'ROLLED_BACK'; + + const { + createStreamMutation, + startStreamMutation, + createPipelineMutation, + updateRoutingMutation, + deleteStreamMutation, + deletePipelineMutation, + deleteRoutingRuleMutation, + } = useSetupInputMutations(); + + const stepMutations = useMemo<{[key in ProcessingSteps]?: UseMutationResult}>(() => ({ + createStream: createStreamMutation, + startStream: startStreamMutation, + createPipeline: createPipelineMutation, + setupRouting: updateRoutingMutation, + }), [createStreamMutation, startStreamMutation, createPipelineMutation, updateRoutingMutation]); + + const rollBackMutations = useMemo<{[key in ProcessingSteps]?: UseMutationResult}>(() => ({ + deleteStream: deleteStreamMutation, + deletePipeline: deletePipelineMutation, + deleteRouting: deleteRoutingRuleMutation, + }), [deleteStreamMutation, deletePipelineMutation, deleteRoutingRuleMutation]); + + useEffect(() => { + if (!isRollback) { + const mutationsArray = Object.entries(stepMutations); + + const hasError = !!mutationsArray.find(([_, mutation]) => mutation.isError); + + const haveAllSucceeded = mutationsArray.every(([_, mutation]) => !mutation.isLoading && mutation.isSuccess); + + if (hasError) { + setStartInputStatus('FAILED'); + } else if (haveAllSucceeded) { + setStartInputStatus('SUCCESS'); + } + } + }, [stepMutations, isRollback]); + + const createPipeline = async (stream: StreamConfiguration) => { + const pipeline = { + title: stream.title, + description: `Pipeline for Stream: ${stream.title} created by the Input Setup Wizard.`, + }; + + const requestPipeline = { + ...pipeline, + source: SourceGenerator.generatePipeline({ ...pipeline, stages: [{ stage: 0, rules: [], match: '' }] }), + }; + + return createPipelineMutation.mutateAsync(requestPipeline); + }; + + const startInput = async () => { + const { input } = wizardData; + + if (!input) return; + + InputStatesStore.start(input) + .finally(() => { + setStartInputStatus('SUCCESS'); + }); + }; + + const stopInput = async () => { + const { input } = wizardData; + + if (!input) return; + + InputStatesStore.stop(input); + }; + + const setupInput = async () => { + const routingStepData = getStepConfigOrData(stepsData, INPUT_WIZARD_STEPS.SETUP_ROUTING) as RoutingStepData; + const { input } = wizardData; + const inputId = input?.id; + + if (!inputId || !routingStepData) return; + + switch (routingStepData.streamType) { + case 'NEW': + + if (routingStepData.shouldCreateNewPipeline) { + createPipeline(routingStepData.newStream); + } + + createStreamMutation.mutateAsync(routingStepData.newStream, { + onSuccess: (response) => { + startStreamMutation.mutateAsync(response.stream_id); + + updateRoutingMutation.mutateAsync({ input_id: inputId, stream_id: response.stream_id }).finally(() => { + startInput(); + }); + }, + }); + + break; + case 'EXISTING': + updateRoutingMutation.mutateAsync({ input_id: inputId, stream_id: routingStepData.streamId }).finally(() => { + startInput(); + }); + + break; + case 'DEFAULT': + startInput(); + break; + + default: + break; + } + }; + + const rollback = () => { + const routingStepData = getStepConfigOrData(stepsData, INPUT_WIZARD_STEPS.SETUP_ROUTING) as RoutingStepData; + const createdStreamId = createStreamMutation.data?.stream_id; + const createdPipelineId = createPipelineMutation.data?.id; + const routingRuleId = updateRoutingMutation.data?.id; + + switch (routingStepData.streamType) { + case 'NEW': + stopInput(); + + if (routingStepData.shouldCreateNewPipeline && createdPipelineId) { + deletePipelineMutation.mutateAsync(createdPipelineId); + } + + if (!createdStreamId) return; + + if (routingRuleId) { + deleteRoutingRuleMutation.mutateAsync(routingRuleId, { + }).finally(() => { + deleteStreamMutation.mutateAsync(createdStreamId).finally(() => { + setStartInputStatus('ROLLED_BACK'); + }); + }); + } else { + deleteStreamMutation.mutateAsync(createdStreamId).finally(() => { + setStartInputStatus('ROLLED_BACK'); + }); + } + + break; + case 'EXISTING': + stopInput(); + + if (!routingRuleId) return; + + deleteRoutingRuleMutation.mutateAsync(routingRuleId).finally( + () => { + setStartInputStatus('ROLLED_BACK'); + }); + + break; + case 'DEFAULT': + stopInput(); + + setStartInputStatus('ROLLED_BACK'); + + break; + + default: + break; + } + }; + + const handleStart = () => { + setStartInputStatus('RUNNING'); + setupInput(); + }; + + const handleRollback = () => { + setStartInputStatus('ROLLING_BACK'); + rollback(); + }; + + const onNextStep = () => { + goToNextStep(); + }; + + const handleBackClick = () => { + goToPreviousStep(); + }; + + const isInputStartable = () => { + const routingStepData = getStepConfigOrData(stepsData, INPUT_WIZARD_STEPS.SETUP_ROUTING) as RoutingStepData; + + if (!routingStepData) return false; + if (routingStepData.newStream || routingStepData.streamId || routingStepData.streamType === 'DEFAULT') return true; + + return false; + }; + + const getProgressEntityName = (stepName, mutations) => { + const mutation = mutations[stepName]; + + const routingStepData = getStepConfigOrData(stepsData, INPUT_WIZARD_STEPS.SETUP_ROUTING) as RoutingStepData; + + const name = mutation.data?.title ?? mutation.data?.name ?? undefined; + + switch (stepName) { + case 'createStream': + case 'deleteStream': + return routingStepData?.newStream.title ?? undefined; + + case 'startStream': + if (routingStepData.streamType === 'NEW') { + return routingStepData?.newStream.title ?? undefined; + } + + return name; + case 'createPipeline': + return routingStepData?.newStream.title ?? undefined; + default: + return name; + } + }; + + const renderProgressMessages = (mutations: {[key in ProcessingSteps]?: UseMutationResult}) => (Object.keys(mutations).map((stepName) => { + const mutation = mutations[stepName]; + + if (!mutation) return null; + if (mutation.isIdle) return null; + + const name = getProgressEntityName(stepName, mutations); + + return ( + + ); + }) + ); + + const renderNextButton = () => { + if (startInputStatus === 'NOT_STARTED' || startInputStatus === 'ROLLED_BACK') { + return ( + + ); + } + + if (startInputStatus === 'FAILED' || startInputStatus === 'ROLLING_BACK') { + return ( + + ); + } + + if (hasNextStep) { + return ( + + ); + } + + return null; + }; + + return ( + + + + +

    + Set up and start the Input according to the configuration made. +

    +     +     + + + {hasBeenStarted && ( + isRollback ? ( + <> + Rolling back Input... + {renderProgressMessages(rollBackMutations)} + + ) : ( + <> + Setting up Input... + {renderProgressMessages(stepMutations)} + {startInputStatus && ( + + )} + + ) + + )} + + {!hasBeenStarted && !isInputStartable() && (

    Your Input is not ready to be setup yet. Please complete the previous steps.

    )} + +     + + {(hasPreviousStep || hasNextStep) && ( + + + {(hasPreviousStep) && ()} + {renderNextButton()} + + + )} +     +     + ); +}; export default StartInputStep; diff --git a/graylog2-web-interface/src/components/inputs/InputSetupWizard/steps/components/CreateStreamForm.tsx b/graylog2-web-interface/src/components/inputs/InputSetupWizard/steps/components/CreateStreamForm.tsx index 507788813941..ac1334c6d4db 100644 --- a/graylog2-web-interface/src/components/inputs/InputSetupWizard/steps/components/CreateStreamForm.tsx +++ b/graylog2-web-interface/src/components/inputs/InputSetupWizard/steps/components/CreateStreamForm.tsx @@ -22,12 +22,14 @@ import Routes from 'routing/Routes'; import { Button, Col, Row } from 'components/bootstrap'; import { FormikInput, InputOptionalInfo, Spinner } from 'components/common'; import IndexSetSelect from 'components/streams/IndexSetSelect'; -import type { Stream } from 'stores/streams/StreamsStore'; import useIndexSetsList from 'components/indices/hooks/useIndexSetsList'; +import type { StreamConfiguration } from 'components/inputs/InputSetupWizard/hooks/useSetupInputMutations'; + +export type StreamFormValues = StreamConfiguration export type FormValues = { create_new_pipeline?: boolean - } & Partial> +} & StreamConfiguration type Props = { submitForm: (values: FormValues) => void diff --git a/graylog2-web-interface/src/components/inputs/InputSetupWizard/steps/components/ProgressMessage.tsx b/graylog2-web-interface/src/components/inputs/InputSetupWizard/steps/components/ProgressMessage.tsx new file mode 100644 index 000000000000..f9f02c73d6ef --- /dev/null +++ b/graylog2-web-interface/src/components/inputs/InputSetupWizard/steps/components/ProgressMessage.tsx @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +import React from 'react'; +import styled, { css } from 'styled-components'; + +import type FetchError from 'logic/errors/FetchError'; +import { Spinner, Icon } from 'components/common'; +import type { ProcessingSteps } from 'components/inputs/InputSetupWizard/steps/StartInputStep'; + +type Props = { + stepName: ProcessingSteps, + isLoading: boolean, + isSuccess: boolean, + name?: string, + isError: boolean, + errorMessage?: FetchError, +} + +const SuccessIcon = styled(Icon)(({ theme }) => css` + color: ${theme.colors.variant.success} +`); + +const ErrorIcon = styled(Icon)(({ theme }) => css` + color: ${theme.colors.variant.danger} +`); + +const ProgressMessage = ({ stepName, isLoading, isSuccess, isError, name = undefined, errorMessage = undefined } : Props) => { + const loadingText: {[key in ProcessingSteps]: (entityName?: string) => string} = { + createStream: (entityName) => `Creating Stream${entityName !== undefined ? (` "${entityName}"`) : ''}...`, + startStream: (entityName) => `Starting Stream${entityName !== undefined ? (` "${entityName}"`) : ''}...`, + createPipeline: (entityName) => `Creating Pipeline${entityName !== undefined ? (` "${entityName}"`) : ''}...`, + setupRouting: (_) => 'Setting up routing...', + result: (_) => '', + deleteStream: (entityName) => `Deleting Stream${entityName !== undefined ? (` "${entityName}"`) : ''}...`, + deletePipeline: (entityName) => `Deleting Pipeline${entityName !== undefined ? (` "${entityName}"`) : ''}...`, + deleteRouting: (_) => 'Removing routing...', + }; + + const errorText: {[key in ProcessingSteps]: (name?: string) => string} = { + createStream: (entityName) => `Creating Stream${entityName !== undefined ? (` "${entityName}"`) : ''} failed!`, + startStream: (entityName) => `Starting Stream${entityName !== undefined ? (` "${entityName}"`) : ''} failed!`, + createPipeline: (entityName) => `Creating Pipeline${entityName !== undefined ? (` "${entityName}"`) : ''} failed!`, + setupRouting: (_) => 'Setting up routing failed!', + result: (_) => 'Starting the Input has failed. Please roll it back to clean it up.', + deleteStream: (entityName) => `Deleting Stream${entityName !== undefined ? (` "${entityName}"`) : ''} failed!`, + deletePipeline: (entityName) => `Deleting Pipeline${entityName !== undefined ? (` "${entityName}"`) : ''} failed!`, + deleteRouting: (_) => 'Removing routing failed!', + }; + + const successText: {[key in ProcessingSteps]: (entityName?: string) => string} = { + createStream: (entityName) => `Stream${entityName !== undefined ? (` "${entityName}"`) : ''} created!`, + startStream: (entityName) => `Stream${entityName !== undefined ? (` "${entityName}"`) : ''} started!`, + createPipeline: (entityName) => `Pipeline${entityName !== undefined ? (` "${entityName}"`) : ''} created!`, + setupRouting: (_) => 'Routing set up!', + result: (_) => 'Input started sucessfully!', + deleteStream: (entityName) => `Stream${entityName !== undefined ? (` "${entityName}"`) : ''} deleted!`, + deletePipeline: (entityName) => `Pipeline${entityName !== undefined ? (` "${entityName}"`) : ''} deleted!`, + deleteRouting: (_) => 'Routing removed!', + }; + + if (isLoading) { + return

    ; + } + + if (isError) { + return ( + <> +

    {errorText[stepName](name)}

    + {errorMessage && (

    Details: {errorMessage.message}

    )} + + ); + } + + if (isSuccess) { + return ( +

    {successText[stepName](name)}

    + ); + } + + return null; +}; + +export default ProgressMessage; diff --git a/graylog2-web-interface/src/components/inputs/InputSetupWizard/types.ts b/graylog2-web-interface/src/components/inputs/InputSetupWizard/types.ts index bb7d7dda808d..6e01a8435ed4 100644 --- a/graylog2-web-interface/src/components/inputs/InputSetupWizard/types.ts +++ b/graylog2-web-interface/src/components/inputs/InputSetupWizard/types.ts @@ -14,6 +14,9 @@ * along with this program. If not, see * . */ + +import type { Input } from 'components/messageloaders/Types'; + export const INPUT_WIZARD_STEPS = { SELECT_CATEGORY: 'SELECT_CATEGORY', INPUT_DIAGNOSIS: 'INPUT_DIAGNOSIS', @@ -25,22 +28,22 @@ export const INPUT_WIZARD_CATEGORIES = { GENERIC: 'GENERIC', } as const; -export const INPUT_WIZARD_SUBCATEGORIES = { - GENERIC: 'GENERIC', -} as const; - export type InputSetupWizardStep = typeof INPUT_WIZARD_STEPS[keyof typeof INPUT_WIZARD_STEPS] export type InputSetupWizardCategory = typeof INPUT_WIZARD_CATEGORIES[keyof typeof INPUT_WIZARD_CATEGORIES] -export interface StepData { +export type StepConfig = { enabled?: boolean } +export type StepsConfig = { + [key in InputSetupWizardStep]?: StepConfig +} + export type StepsData = { - [key in InputSetupWizardStep]?: StepData + [key in InputSetupWizardStep]?: object } export type WizardData = { - inputId?: string, + input?: Input, category?: InputSetupWizardCategory } diff --git a/graylog2-web-interface/src/components/inputs/InputStateControl.tsx b/graylog2-web-interface/src/components/inputs/InputStateControl.tsx index 0842204955fa..4c2cd53474fb 100644 --- a/graylog2-web-interface/src/components/inputs/InputStateControl.tsx +++ b/graylog2-web-interface/src/components/inputs/InputStateControl.tsx @@ -28,18 +28,18 @@ import type { Input } from 'components/messageloaders/Types'; import { getPathnameWithoutId } from 'util/URLUtils'; import { TELEMETRY_EVENT_TYPE } from 'logic/telemetry/Constants'; import { Button } from 'components/bootstrap'; -import { INPUT_SETUP_MODE_FEATURE_FLAG, useInputSetupWizard } from 'components/inputs/InputSetupWizard'; +import { INPUT_SETUP_MODE_FEATURE_FLAG } from 'components/inputs/InputSetupWizard'; type Props = { - input: Input + input: Input, + openWizard: () => void, } -const InputStateControl = ({ input } : Props) => { +const InputStateControl = ({ input, openWizard } : Props) => { const sendTelemetry = useSendTelemetry(); const { pathname } = useLocation(); const [isLoading, setIsLoading] = useState(false); const { inputStates } = useStore(InputStatesStore) as { inputStates: InputStates }; - const { openWizard } = useInputSetupWizard(); const inputSetupFeatureFlagIsEnabled = useFeature(INPUT_SETUP_MODE_FEATURE_FLAG); const startInput = () => { @@ -76,7 +76,7 @@ const InputStateControl = ({ input } : Props) => { app_action_value: 'setup-input', }); - openWizard({ inputId: input.id }); + openWizard(); }; if (inputSetupFeatureFlagIsEnabled && isInputInSetupMode(inputStates, input.id)) { diff --git a/graylog2-web-interface/src/components/inputs/InputsList.tsx b/graylog2-web-interface/src/components/inputs/InputsList.tsx index d9fd4b438f50..f6aa7573da89 100644 --- a/graylog2-web-interface/src/components/inputs/InputsList.tsx +++ b/graylog2-web-interface/src/components/inputs/InputsList.tsx @@ -20,7 +20,6 @@ import styled from 'styled-components'; import { Row, Col } from 'components/bootstrap'; import { IfPermitted, Spinner, SearchForm } from 'components/common'; -import useFeature from 'hooks/useFeature'; import { naturalSortIgnoreCase } from 'util/SortUtils'; import EntityList from 'components/common/EntityList'; import { InputsActions, InputsStore } from 'stores/inputs/InputsStore'; @@ -30,7 +29,6 @@ import { useStore } from 'stores/connect'; import type { StoreState } from 'stores/StoreTypes'; import type { NodeInfo } from 'stores/nodes/NodesStore'; import type { Input } from 'components/messageloaders/Types'; -import { InputSetupWizardProvider, InputSetupWizard, INPUT_SETUP_MODE_FEATURE_FLAG } from 'components/inputs/InputSetupWizard'; import InputListItem from './InputListItem'; import CreateInputControl from './CreateInputControl'; @@ -103,8 +101,6 @@ const InputsList = ({ permissions, node }: Props) => { SingleNodeActions.get(); }, []); - const inputSetupFeatureFlagIsEnabled = useFeature(INPUT_SETUP_MODE_FEATURE_FLAG); - const currentNode = useStore(SingleNodeStore); const { globalInputs, localInputs } = useStore(InputsStore, (inputsStore) => _splitInputs(inputsStore, node)); const [filter, setFilter] = useState(); @@ -121,56 +117,51 @@ const InputsList = ({ permissions, node }: Props) => { return (
    - - {inputSetupFeatureFlagIsEnabled && ( - - )} - {!node && ( + {!node && ( - )} - - - - -
    -

    - Global inputs -   - {globalInputs.length} configured{nodeAffix} -

    - ( - - ))} /> -
    -
    -

    - Local inputs -   - {localInputs.length} configured{nodeAffix} -

    - ( - - ))} /> - -     -     + )} + + + + +
    +

    + Global inputs +   + {globalInputs.length} configured{nodeAffix} +

    + ( + + ))} /> +
    +
    +

    + Local inputs +   + {localInputs.length} configured{nodeAffix} +

    + ( + + ))} /> + +
    ); }; diff --git a/graylog2-web-interface/src/routing/ApiRoutes.ts b/graylog2-web-interface/src/routing/ApiRoutes.ts index 32ea05dd4848..bee735ea6008 100644 --- a/graylog2-web-interface/src/routing/ApiRoutes.ts +++ b/graylog2-web-interface/src/routing/ApiRoutes.ts @@ -421,6 +421,7 @@ const ApiRoutes = { update: (pipelineId: string) => ({ url: `/system/pipelines/pipeline/${pipelineId}` }), delete: (pipelineId: string) => ({ url: `/system/pipelines/pipeline/${pipelineId}` }), parse: () => ({ url: '/system/pipelines/pipeline/parse' }), + updateRouting: () => ({ url: '/system/pipelines/pipeline/routing' }), }, RulesController: { list: () => ({ url: '/system/pipelines/rule' }), diff --git a/graylog2-web-interface/src/stores/streams/StreamsStore.ts b/graylog2-web-interface/src/stores/streams/StreamsStore.ts index 4976201bfb70..993db388a142 100644 --- a/graylog2-web-interface/src/stores/streams/StreamsStore.ts +++ b/graylog2-web-interface/src/stores/streams/StreamsStore.ts @@ -58,6 +58,16 @@ type AlertConditionSummary = { title: string | null | undefined, }; +export type StreamConfiguration = Pick + type AlertReceiver = { emails: string[], users: string[], @@ -223,7 +233,7 @@ const StreamsStore = singletonStore('Streams', () => Reflux.createStore({ return promise; }, - save(stream: any, callback: ((streamId: string) => void)) { + save(stream: StreamConfiguration, callback: ((streamId: string) => void)) { const failCallback = (errorThrown) => { UserNotification.error(`Saving Stream failed with status: ${errorThrown}`, 'Could not save Stream'); From 1000827bdbc1b85b6dbe9ceb2b34726fafb5a53d Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Thu, 12 Dec 2024 14:37:00 +0100 Subject: [PATCH 5/7] Relax Maven version requirement (#21171) This fixes issues with backports where the Maven version in the project parent is newer than in the server. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d4f9961dbad9..32963a0618eb 100644 --- a/pom.xml +++ b/pom.xml @@ -724,7 +724,7 @@ - [3.9.9] + [3.9.6,3.99.99] [17.0,17.99] From b4e1e075f9a18382c55add16ad0bf1265c462fc3 Mon Sep 17 00:00:00 2001 From: Matthias Oesterheld <33032967+moesterheld@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:02:32 +0100 Subject: [PATCH 6/7] Refactor data node startup, add startup mechanism for new node types (#20181) * Initial reworked, simplified abstract node startup * use Graylog CmdLineTool in data node * remove unused pid file declaration * remove freshInstallation check as it doesn't make sense since it checks for fresh installation of the Graylog cluster * move mongo preflight check into regular preflight checks since it doesn't do the fresh installation detection any more * remove migration feature for data node * code cleanup * remove unused local flag, code cleaup * rename server to datanode for better clarification * fix feature flag test to use config for env/sys props * remove Graylog path configuration from data node configuration * temporary fix for excluding zstd library fix from data node * add requireexplicitbindings for new node types * remove unneeded settings bean * do not use `pluginLoaderConfig` for fixing zstd temp directory * do tls protocols configuration and setting of netty defaults only in Graylog node * remove ExampleCommand, replace with MinimalNode test * change forbidden method invocation * adjust log Co-authored-by: Bernd Ahlers * Change cmdline option description Co-authored-by: Bernd Ahlers * Change cmdline option description Co-authored-by: Bernd Ahlers * Change cmdline option description Co-authored-by: Bernd Ahlers * make usage of NodeIdFile configurable * adjust ServerStatus and provide common node command test * code cleanup * make configuration a field, remove unused constructor * code cleanup * use parent field * make Datanode and Server extend AbstractNodeCommand * change field names to camel case * remove redundant bindings, make journal commands extends AbstractNodeCommand * remove static field * remove redundant bindings * always applySecuritySettings * add javadocs to GraylogNodeModule --------- Co-authored-by: Tomas Dvorak Co-authored-by: Bernd Ahlers --- .../org/graylog/datanode/Configuration.java | 13 +- ...dings.java => DatanodeServerBindings.java} | 39 +- .../bindings/PreflightChecksBindings.java | 2 + .../datanode/bootstrap/CmdLineTool.java | 509 ------------------ ...rBootstrap.java => DatanodeBootstrap.java} | 132 +---- .../bootstrap/commands/MigrateCmd.java | 58 -- .../OpensearchDataDirCompatibilityCheck.java | 9 +- .../commands/{Server.java => Datanode.java} | 51 +- ...der.java => DatanodeCommandsProvider.java} | 4 +- ...org.graylog2.bootstrap.CliCommandsProvider | 2 +- .../ConfigurationDocumentationTest.java | 8 +- .../org/graylog2/CommonNodeConfiguration.java | 72 +++ .../main/java/org/graylog2/Configuration.java | 6 +- .../graylog2/GraylogNodeConfiguration.java | 73 +++ .../bindings/ConfigurationModule.java | 3 + .../graylog2/bindings/GraylogNodeModule.java | 95 ++++ .../org/graylog2/bindings/MongoDBModule.java | 5 - .../bindings/MongoDbConnectionModule.java | 43 ++ .../org/graylog2/bindings/ServerBindings.java | 3 - .../org/graylog2/bootstrap/CmdLineTool.java | 107 ++-- .../graylog2/bootstrap/ServerBootstrap.java | 19 +- .../commands/AbstractNodeCommand.java | 64 +++ .../java/org/graylog2/commands/Server.java | 12 +- .../journal/AbstractJournalCommand.java | 67 +-- .../commands/journal/JournalDecode.java | 9 +- .../configuration/EventBusConfiguration.java | 30 ++ .../featureflag/FeatureFlagsFactory.java | 11 +- .../ImmutableFeatureFlagsCollector.java | 16 +- .../graylog2/plugin/BaseConfiguration.java | 4 +- .../org/graylog2/plugin/PluginModule.java | 5 + .../org/graylog2/plugin/ServerStatus.java | 10 +- .../plugin/inject/Graylog2Module.java | 6 + .../shared/bindings/GenericBindings.java | 10 - .../shared/security/SecurityBindings.java | 2 - .../commands/CommonNodeCommandTest.java | 98 ++++ .../commands/MinimalNodeCommandTest.java | 130 +++++ .../ImmutableFeatureFlagsMetricsTest.java | 6 +- .../ImmutableFeatureFlagsTest.java | 8 +- .../org/graylog2/plugin/ServerStatusTest.java | 5 +- .../org/graylog2/commands/common-node.conf | 2 + .../org/graylog2/commands/minimal-node.conf | 1 + 41 files changed, 824 insertions(+), 925 deletions(-) rename data-node/src/main/java/org/graylog/datanode/bindings/{ServerBindings.java => DatanodeServerBindings.java} (53%) delete mode 100644 data-node/src/main/java/org/graylog/datanode/bootstrap/CmdLineTool.java rename data-node/src/main/java/org/graylog/datanode/bootstrap/{ServerBootstrap.java => DatanodeBootstrap.java} (54%) delete mode 100644 data-node/src/main/java/org/graylog/datanode/bootstrap/commands/MigrateCmd.java rename data-node/src/main/java/org/graylog/datanode/commands/{Server.java => Datanode.java} (78%) rename data-node/src/main/java/org/graylog/datanode/commands/{ServerCommandsProvider.java => DatanodeCommandsProvider.java} (88%) create mode 100644 graylog2-server/src/main/java/org/graylog2/CommonNodeConfiguration.java create mode 100644 graylog2-server/src/main/java/org/graylog2/GraylogNodeConfiguration.java create mode 100644 graylog2-server/src/main/java/org/graylog2/bindings/GraylogNodeModule.java create mode 100644 graylog2-server/src/main/java/org/graylog2/bindings/MongoDbConnectionModule.java create mode 100644 graylog2-server/src/main/java/org/graylog2/commands/AbstractNodeCommand.java create mode 100644 graylog2-server/src/main/java/org/graylog2/configuration/EventBusConfiguration.java create mode 100644 graylog2-server/src/test/java/org/graylog2/commands/CommonNodeCommandTest.java create mode 100644 graylog2-server/src/test/java/org/graylog2/commands/MinimalNodeCommandTest.java create mode 100644 graylog2-server/src/test/resources/org/graylog2/commands/common-node.conf create mode 100644 graylog2-server/src/test/resources/org/graylog2/commands/minimal-node.conf diff --git a/data-node/src/main/java/org/graylog/datanode/Configuration.java b/data-node/src/main/java/org/graylog/datanode/Configuration.java index 4595e4fde87c..fe0d7f93e7e9 100644 --- a/data-node/src/main/java/org/graylog/datanode/Configuration.java +++ b/data-node/src/main/java/org/graylog/datanode/Configuration.java @@ -32,6 +32,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.net.InetAddresses; import org.graylog.datanode.configuration.DatanodeDirectories; +import org.graylog2.CommonNodeConfiguration; import org.graylog2.Configuration.SafeClassesValidator; import org.graylog2.configuration.Documentation; import org.graylog2.plugin.Tools; @@ -57,7 +58,7 @@ * Helper class to hold configuration of DataNode */ @SuppressWarnings("FieldMayBeFinal") -public class Configuration { +public class Configuration implements CommonNodeConfiguration { private static final Logger LOG = LoggerFactory.getLogger(Configuration.class); public static final String TRANSPORT_CERTIFICATE_PASSWORD_PROPERTY = "transport_certificate_password"; public static final String HTTP_CERTIFICATE_PASSWORD_PROPERTY = "http_certificate_password"; @@ -659,4 +660,14 @@ public List getNodeRoles() { public String getOpensearchHeap() { return opensearchHeap; } + + @Override + public String getEnvironmentVariablePrefix() { + return "GRAYLOG_DATANODE_"; + } + + @Override + public String getSystemPropertyPrefix() { + return "graylog.datanode."; + } } diff --git a/data-node/src/main/java/org/graylog/datanode/bindings/ServerBindings.java b/data-node/src/main/java/org/graylog/datanode/bindings/DatanodeServerBindings.java similarity index 53% rename from data-node/src/main/java/org/graylog/datanode/bindings/ServerBindings.java rename to data-node/src/main/java/org/graylog/datanode/bindings/DatanodeServerBindings.java index 170de59a4468..58387dcb3426 100644 --- a/data-node/src/main/java/org/graylog/datanode/bindings/ServerBindings.java +++ b/data-node/src/main/java/org/graylog/datanode/bindings/DatanodeServerBindings.java @@ -16,36 +16,22 @@ */ package org.graylog.datanode.bindings; -import com.google.common.eventbus.EventBus; import com.google.inject.TypeLiteral; -import com.google.inject.multibindings.Multibinder; import com.google.inject.multibindings.OptionalBinder; -import jakarta.ws.rs.container.DynamicFeature; -import jakarta.ws.rs.ext.ExceptionMapper; -import org.graylog.datanode.Configuration; import org.graylog.datanode.shared.system.activities.DataNodeActivityWriter; -import org.graylog2.bindings.providers.ClusterEventBusProvider; import org.graylog2.cluster.ClusterConfigServiceImpl; import org.graylog2.cluster.nodes.DataNodeClusterService; import org.graylog2.cluster.nodes.DataNodeDto; import org.graylog2.cluster.nodes.NodeService; -import org.graylog2.events.ClusterEventBus; -import org.graylog2.jackson.InputConfigurationBeanDeserializerModifier; import org.graylog2.plugin.cluster.ClusterConfigService; import org.graylog2.plugin.cluster.ClusterIdFactory; import org.graylog2.plugin.cluster.RandomUUIDClusterIdFactory; import org.graylog2.plugin.inject.Graylog2Module; -import org.graylog2.shared.bindings.providers.EventBusProvider; import org.graylog2.shared.system.activities.ActivityWriter; -public class ServerBindings extends Graylog2Module { - private final Configuration configuration; - private final boolean isMigrationCommand; +public class DatanodeServerBindings extends Graylog2Module { - public ServerBindings(Configuration configuration, boolean isMigrationCommand) { - - this.configuration = configuration; - this.isMigrationCommand = isMigrationCommand; + public DatanodeServerBindings() { } @Override @@ -53,24 +39,10 @@ protected void configure() { bindInterfaces(); bindSingletons(); - bindProviders(); - bindFactoryModules(); bindDynamicFeatures(); bindExceptionMappers(); - bindAdditionalJerseyComponents(); -// install(new AuthenticatingRealmModule(configuration)); -// install(new AuthorizationOnlyRealmModule()); } - private void bindProviders() { - bind(ClusterEventBus.class).toProvider(ClusterEventBusProvider.class).asEagerSingleton(); - bind(EventBus.class).toProvider(EventBusProvider.class).asEagerSingleton(); - bind(InputConfigurationBeanDeserializerModifier.class).toInstance(InputConfigurationBeanDeserializerModifier.withoutConfig()); - } - - private void bindFactoryModules() { - // System Jobs - } private void bindSingletons() { bind(ClusterConfigService.class).to(ClusterConfigServiceImpl.class).asEagerSingleton(); @@ -83,14 +55,11 @@ private void bindInterfaces() { } private void bindDynamicFeatures() { - final Multibinder> dynamicFeatures = jerseyDynamicFeatureBinder(); + jerseyDynamicFeatureBinder(); } private void bindExceptionMappers() { - final Multibinder> exceptionMappers = jerseyExceptionMapperBinder(); + jerseyExceptionMapperBinder(); } - private void bindAdditionalJerseyComponents() { -// jerseyAdditionalComponentsBinder().addBinding().toInstance(GenericErrorCsvWriter.class); - } } diff --git a/data-node/src/main/java/org/graylog/datanode/bindings/PreflightChecksBindings.java b/data-node/src/main/java/org/graylog/datanode/bindings/PreflightChecksBindings.java index 857deedeccf9..0f47e8502069 100644 --- a/data-node/src/main/java/org/graylog/datanode/bindings/PreflightChecksBindings.java +++ b/data-node/src/main/java/org/graylog/datanode/bindings/PreflightChecksBindings.java @@ -27,6 +27,7 @@ import org.graylog.datanode.opensearch.CsrRequester; import org.graylog.datanode.opensearch.CsrRequesterImpl; import org.graylog2.bindings.providers.MongoConnectionProvider; +import org.graylog2.bootstrap.preflight.MongoDBPreflightCheck; import org.graylog2.bootstrap.preflight.PreflightCheck; import org.graylog2.cluster.certificates.CertificateExchange; import org.graylog2.cluster.certificates.CertificateExchangeImpl; @@ -40,6 +41,7 @@ protected void configure() { bind(CsrRequester.class).to(CsrRequesterImpl.class).asEagerSingleton(); bind(CertificateExchange.class).to(CertificateExchangeImpl.class); + addPreflightCheck(MongoDBPreflightCheck.class); addPreflightCheck(DatanodeDnsPreflightCheck.class); addPreflightCheck(OpensearchBinPreflightCheck.class); addPreflightCheck(DatanodeDirectoriesLockfileCheck.class); diff --git a/data-node/src/main/java/org/graylog/datanode/bootstrap/CmdLineTool.java b/data-node/src/main/java/org/graylog/datanode/bootstrap/CmdLineTool.java deleted file mode 100644 index 5bded69c587b..000000000000 --- a/data-node/src/main/java/org/graylog/datanode/bootstrap/CmdLineTool.java +++ /dev/null @@ -1,509 +0,0 @@ -/* - * Copyright (C) 2020 Graylog, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * . - */ -package org.graylog.datanode.bootstrap; - -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.jmx.JmxReporter; -import com.codahale.metrics.log4j2.InstrumentedAppender; -import com.github.joschi.jadconfig.JadConfig; -import com.github.joschi.jadconfig.ParameterException; -import com.github.joschi.jadconfig.Repository; -import com.github.joschi.jadconfig.RepositoryException; -import com.github.joschi.jadconfig.ValidationException; -import com.github.joschi.jadconfig.guava.GuavaConverterFactory; -import com.github.joschi.jadconfig.jodatime.JodaTimeConverterFactory; -import com.github.joschi.jadconfig.repositories.EnvironmentRepository; -import com.github.joschi.jadconfig.repositories.PropertiesRepository; -import com.github.joschi.jadconfig.repositories.SystemPropertiesRepository; -import com.github.rvesse.airline.annotations.Command; -import com.github.rvesse.airline.annotations.Option; -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Lists; -import com.google.inject.AbstractModule; -import com.google.inject.Binder; -import com.google.inject.CreationException; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.Module; -import com.google.inject.Stage; -import com.google.inject.name.Names; -import com.google.inject.spi.Message; -import io.netty.util.internal.logging.InternalLoggerFactory; -import io.netty.util.internal.logging.Slf4JLoggerFactory; -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.core.LoggerContext; -import org.bouncycastle.jce.provider.BouncyCastleProvider; -import org.graylog.datanode.Configuration; -import org.graylog.datanode.bootstrap.commands.MigrateCmd; -import org.graylog2.bindings.NamedConfigParametersOverrideModule; -import org.graylog2.bootstrap.CliCommand; -import org.graylog2.configuration.PathConfiguration; -import org.graylog2.configuration.TLSProtocolsConfiguration; -import org.graylog2.featureflag.FeatureFlags; -import org.graylog2.featureflag.FeatureFlagsFactory; -import org.graylog2.plugin.Plugin; -import org.graylog2.plugin.Tools; -import org.graylog2.plugin.Version; -import org.graylog2.shared.UI; -import org.graylog2.shared.bindings.GuiceInjectorHolder; -import org.graylog2.shared.bindings.IsDevelopmentBindings; -import org.graylog2.shared.metrics.MetricRegistryFactory; -import org.graylog2.shared.plugins.ChainingClassLoader; -import org.graylog2.shared.utilities.ExceptionUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.lang.management.ManagementFactory; -import java.nio.file.AccessDeniedException; -import java.security.Security; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static com.google.common.base.Strings.nullToEmpty; - -public abstract class CmdLineTool implements CliCommand { - - public static final String GRAYLOG_DATANODE_ENVIRONMENT_VAR_PREFIX = "GRAYLOG_DATANODE_"; - public static final String GRAYLOG_DATANODE_SYSTEM_PROP_PREFIX = "graylog.datanode."; - - static { - // Set up JDK Logging adapter, https://logging.apache.org/log4j/2.x/log4j-jul/index.html - System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); - } - - private static final Logger LOG = LoggerFactory.getLogger(CmdLineTool.class); - - protected static final Version version = Version.CURRENT_CLASSPATH; - protected static final String FILE_SEPARATOR = System.getProperty("file.separator"); - protected static final String TMPDIR = System.getProperty("java.io.tmpdir", "/tmp"); - - protected final JadConfig jadConfig; - protected final ChainingClassLoader chainingClassLoader; - - @Option(name = "--dump-config", description = "Show the effective Graylog DataNode configuration and exit") - protected boolean dumpConfig = false; - - @Option(name = "--dump-default-config", description = "Show the default configuration and exit") - protected boolean dumpDefaultConfig = false; - - @Option(name = {"-d", "--debug"}, description = "Run Graylog DataNode in debug mode") - private boolean debug = false; - - @Option(name = {"-f", "--configfile"}, description = "Configuration file for Graylog DataNode") - private String configFile = "/etc/graylog/datanode/datanode.conf"; - - @Option(name = {"-ff", "--featureflagfile"}, description = "Configuration file for Graylog DataNode feature flags") - private String customFeatureFlagFile = "/etc/graylog/datanode/feature-flag.conf"; - - protected String commandName = "command"; - - protected Injector injector; - protected Injector coreConfigInjector; - protected FeatureFlags featureFlags; - - protected CmdLineTool() { - this(null); - } - - protected CmdLineTool(String commandName) { - jadConfig = new JadConfig(); - jadConfig.addConverterFactory(new GuavaConverterFactory()); - jadConfig.addConverterFactory(new JodaTimeConverterFactory()); - - if (commandName == null) { - if (this.getClass().isAnnotationPresent(Command.class)) { - this.commandName = this.getClass().getAnnotation(Command.class).name(); - } else { - this.commandName = "tool"; - } - } else { - this.commandName = commandName; - } - this.chainingClassLoader = new ChainingClassLoader(this.getClass().getClassLoader()); - } - - - /** - * Validate the given configuration for this command. - * - * @return {@code true} if the configuration is valid, {@code false}. - */ - protected boolean validateConfiguration() { - return true; - } - - public boolean isDumpConfig() { - return dumpConfig; - } - - public boolean isDumpDefaultConfig() { - return dumpDefaultConfig; - } - - public boolean isDebug() { - return debug; - } - - protected abstract List getCommandBindings(FeatureFlags featureFlags); - - protected abstract List getCommandConfigurationBeans(); - - public boolean isMigrationCommand() { - return commandName.equals(MigrateCmd.MIGRATION_COMMAND); - } - - /** - * Things that have to run before the {@link #startCommand()} method is being called. - * Please note that this happens *before* the configuration file has been parsed. - */ - protected void beforeStart() { - } - - /** - * Things that have to run before the {@link #startCommand()} method is being called. - * Please note that this happens *before* the configuration file has been parsed. - */ - protected void beforeStart(TLSProtocolsConfiguration tlsProtocolsConfiguration, Configuration configuration) { - } - - /** - * Things that have to run before the guice injector is created. - * This call happens *after* the configuration file has been parsed. - * - * @param plugins The already loaded plugins - */ - protected void beforeInjectorCreation(Set plugins) { - } - - protected static void applySecuritySettings(TLSProtocolsConfiguration configuration) { - // Disable insecure TLS parameters and ciphers by default. - // Prevent attacks like LOGJAM, LUCKY13, et al. - setSystemPropertyIfEmpty("jdk.tls.ephemeralDHKeySize", "2048"); - setSystemPropertyIfEmpty("jdk.tls.rejectClientInitiatedRenegotiation", "true"); - - final Set tlsProtocols = configuration.getConfiguredTlsProtocols(); - final List disabledAlgorithms = Stream.of(Security.getProperty("jdk.tls.disabledAlgorithms").split(",")).map(String::trim).collect(Collectors.toList()); - - // Only restrict ciphers if insecure TLS protocols are explicitly enabled. - // c.f. https://github.com/Graylog2/graylog2-server/issues/10944 - if (tlsProtocols == null || !(tlsProtocols.isEmpty() || tlsProtocols.contains("TLSv1") || tlsProtocols.contains("TLSv1.1"))) { - disabledAlgorithms.addAll(ImmutableSet.of("CBC", "3DES")); - Security.setProperty("jdk.tls.disabledAlgorithms", String.join(", ", disabledAlgorithms)); - } else { - // Remove explicitly enabled legacy TLS protocols from the disabledAlgorithms filter - Set reEnabledTLSProtocols; - if (tlsProtocols.isEmpty()) { - reEnabledTLSProtocols = ImmutableSet.of("TLSv1", "TLSv1.1"); - } else { - reEnabledTLSProtocols = tlsProtocols; - } - final List updatedProperties = disabledAlgorithms.stream() - .filter(p -> !reEnabledTLSProtocols.contains(p)) - .collect(Collectors.toList()); - - Security.setProperty("jdk.tls.disabledAlgorithms", String.join(", ", updatedProperties)); - } - - // Explicitly register Bouncy Castle as security provider. - // This allows us to use more key formats than with JCE - Security.addProvider(new BouncyCastleProvider()); - } - - private static void setSystemPropertyIfEmpty(String key, String value) { - if (System.getProperty(key) == null) { - System.setProperty(key, value); - } - } - - @Override - public void run() { - // Setup logger first to ensure we can log any caught Throwable to the configured log file - final Level logLevel = setupLogger(); - try { - doRun(logLevel); - } catch (Throwable e) { - LOG.error("Startup error:", e); - throw e; - } - } - - public void doRun(Level logLevel) { - // This is holding all our metrics. - MetricRegistry metricRegistry = MetricRegistryFactory.create(); - featureFlags = getFeatureFlags(metricRegistry); - - if (isDumpDefaultConfig()) { - dumpDefaultConfigAndExit(); - } - - installConfigRepositories(); - installCommandConfig(); - - beforeStart(); - beforeStart(parseAndGetTLSConfiguration(), parseAndGetConfiguration(configFile)); - - processConfiguration(jadConfig); - - coreConfigInjector = setupCoreConfigInjector(); - - if (isDumpConfig()) { - dumpCurrentConfigAndExit(); - } - - if (!validateConfiguration()) { - LOG.error("Validating configuration file failed - exiting."); - System.exit(1); - } - - - final List arguments = ManagementFactory.getRuntimeMXBean().getInputArguments(); - LOG.info("Running with JVM arguments: {}", Joiner.on(' ').join(arguments)); - - - beforeInjectorCreation(Collections.emptySet()); - - injector = setupInjector( - new IsDevelopmentBindings(), - new NamedConfigParametersOverrideModule(jadConfig.getConfigurationBeans()), - binder -> binder.bind(MetricRegistry.class).toInstance(metricRegistry) - ); - - if (injector == null) { - LOG.error("Injector could not be created, exiting! (Please include the previous error messages in bug " + - "reports.)"); - System.exit(1); - } - - addInstrumentedAppender(metricRegistry, logLevel); - // Report metrics via JMX. - final JmxReporter reporter = JmxReporter.forRegistry(metricRegistry).build(); - reporter.start(); - - startCommand(); - } - - private Configuration parseAndGetConfiguration(String configFile) { - final Configuration configuration = new Configuration(); - processConfiguration(new JadConfig(getConfigRepositories(configFile), configuration)); - return configuration; - } - - // Parse only the TLSConfiguration bean - // to avoid triggering anything that might initialize the default SSLContext - private TLSProtocolsConfiguration parseAndGetTLSConfiguration() { - final JadConfig jadConfig = new JadConfig(); - jadConfig.setRepositories(getConfigRepositories(configFile)); - final TLSProtocolsConfiguration tlsConfiguration = new TLSProtocolsConfiguration(); - jadConfig.addConfigurationBean(tlsConfiguration); - processConfiguration(jadConfig); - - return tlsConfiguration; - } - - private PathConfiguration parseAndGetPathConfiguration(String configFile) { - final PathConfiguration pathConfiguration = new PathConfiguration(); - processConfiguration(new JadConfig(getConfigRepositories(configFile), pathConfiguration)); - return pathConfiguration; - } - - private void installCommandConfig() { - getCommandConfigurationBeans().forEach(jadConfig::addConfigurationBean); - } - - protected abstract void startCommand(); - - protected Level setupLogger() { - final Level logLevel; - if (isDebug()) { - LOG.info("Running in Debug mode"); - logLevel = Level.DEBUG; - - // Enable logging for Netty when running in debug mode. - InternalLoggerFactory.setDefaultFactory(Slf4JLoggerFactory.INSTANCE); - } else if (onlyLogErrors()) { - logLevel = Level.ERROR; - } else { - logLevel = Level.INFO; - } - - initializeLogging(logLevel); - - return logLevel; - } - - private void initializeLogging(final Level logLevel) { - final LoggerContext context = (LoggerContext) LogManager.getContext(false); - final org.apache.logging.log4j.core.config.Configuration config = context.getConfiguration(); - - config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME).setLevel(logLevel); - config.getLoggerConfig(Main.class.getPackage().getName()).setLevel(logLevel); - - context.updateLoggers(config); - } - - private void addInstrumentedAppender(final MetricRegistry metrics, final Level level) { - final InstrumentedAppender appender = new InstrumentedAppender(metrics, null, null, false); - appender.start(); - - final LoggerContext context = (LoggerContext) LogManager.getContext(false); - final org.apache.logging.log4j.core.config.Configuration config = context.getConfiguration(); - config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME).addAppender(appender, level, null); - context.updateLoggers(config); - } - - protected boolean onlyLogErrors() { - return false; - } - - private void dumpCurrentConfigAndExit() { - System.out.println(dumpConfiguration(jadConfig.dump())); - System.exit(0); - } - - private void dumpDefaultConfigAndExit() { - installCommandConfig(); - coreConfigInjector = setupCoreConfigInjector(); - dumpCurrentConfigAndExit(); - } - - private FeatureFlags getFeatureFlags(MetricRegistry metricRegistry) { - return new FeatureFlagsFactory().createImmutableFeatureFlags(customFeatureFlagFile, metricRegistry); - } - - protected Collection getConfigRepositories(String configFile) { - return Arrays.asList( - new EnvironmentRepository(GRAYLOG_DATANODE_ENVIRONMENT_VAR_PREFIX), - new SystemPropertiesRepository(GRAYLOG_DATANODE_SYSTEM_PROP_PREFIX), - new PropertiesRepository(configFile) - ); - } - - private String dumpConfiguration(final Map configMap) { - final StringBuilder sb = new StringBuilder(); - sb.append("# Configuration of graylog2-").append(commandName).append(" ").append(version).append(System.lineSeparator()); - sb.append("# Generated on ").append(Tools.nowUTC()).append(System.lineSeparator()); - - for (Map.Entry entry : configMap.entrySet()) { - sb.append(entry.getKey()).append('=').append(nullToEmpty(entry.getValue())).append(System.lineSeparator()); - } - - return sb.toString(); - } - - private void installConfigRepositories() { - jadConfig.setRepositories(getConfigRepositories(configFile)); - } - - protected void processConfiguration(JadConfig jadConfig) { - try { - jadConfig.processFailingLazily(); - } catch (RepositoryException e) { - LOG.error("Couldn't load configuration: {}", e.getMessage()); - System.exit(1); - } catch (ParameterException | ValidationException e) { - LOG.error("Invalid configuration", e); - System.exit(1); - } - } - - protected List getSharedBindingsModules() { - return Lists.newArrayList(); - } - - protected Injector setupInjector(Module... modules) { - try { - final ImmutableList.Builder builder = ImmutableList.builder(); - builder.addAll(getSharedBindingsModules()); - builder.addAll(getCommandBindings(featureFlags)); - builder.addAll(Arrays.asList(modules)); - builder.add(binder -> { - binder.bind(ChainingClassLoader.class).toInstance(chainingClassLoader); - featureFlagsBinding(binder); - binder.bind(String.class).annotatedWith(Names.named("BootstrapCommand")).toInstance(commandName); - }); - return GuiceInjectorHolder.createInjector(builder.build()); - } catch (CreationException e) { - annotateInjectorCreationException(e); - return null; - } catch (Exception e) { - LOG.error("Injector creation failed!", e); - return null; - } - } - - /** - * Set up a separate injector, containing only the core configuration bindings. It can be used to look up - * configuration values in modules at binding time. - */ - protected Injector setupCoreConfigInjector() { - final AbstractModule configModule = - new NamedConfigParametersOverrideModule(jadConfig.getConfigurationBeans()); - - Injector coreConfigInjector = null; - try { - coreConfigInjector = Guice.createInjector(Stage.PRODUCTION, ImmutableList.of(configModule, - (Module) Binder::requireExplicitBindings, this::featureFlagsBinding)); - } catch (CreationException e) { - annotateInjectorCreationException(e); - } catch (Exception e) { - LOG.error("Injector creation failed!", e); - } - - if (coreConfigInjector == null) { - LOG.error("Injector for core configuration could not be created, exiting! (Please include the previous " + - "error messages in bug reports.)"); - System.exit(1); - } - return coreConfigInjector; - } - - private void featureFlagsBinding(Binder binder) { - binder.bind(FeatureFlags.class).toInstance(featureFlags); - } - - protected void annotateInjectorCreationException(CreationException e) { - annotateInjectorExceptions(e.getErrorMessages()); - throw e; - } - - protected void annotateInjectorExceptions(Collection messages) { - for (Message message : messages) { - //noinspection ThrowableResultOfMethodCallIgnored - final Throwable rootCause = ExceptionUtils.getRootCause(message.getCause()); - if (rootCause instanceof AccessDeniedException) { - LOG.error(UI.wallString("Unable to access file " + rootCause.getMessage())); - System.exit(-2); - } else { - // other guice error, still print the raw messages - // TODO this could potentially print duplicate messages depending on what a subclass does... - LOG.error("Guice error (more detail on log level debug): {}", message.getMessage()); - if (rootCause != null) { - LOG.debug("Stacktrace:", rootCause); - } - } - } - } -} diff --git a/data-node/src/main/java/org/graylog/datanode/bootstrap/ServerBootstrap.java b/data-node/src/main/java/org/graylog/datanode/bootstrap/DatanodeBootstrap.java similarity index 54% rename from data-node/src/main/java/org/graylog/datanode/bootstrap/ServerBootstrap.java rename to data-node/src/main/java/org/graylog/datanode/bootstrap/DatanodeBootstrap.java index 88eb5d3a55b1..4e112b076704 100644 --- a/data-node/src/main/java/org/graylog/datanode/bootstrap/ServerBootstrap.java +++ b/data-node/src/main/java/org/graylog/datanode/bootstrap/DatanodeBootstrap.java @@ -16,9 +16,7 @@ */ package org.graylog.datanode.bootstrap; -import com.github.rvesse.airline.annotations.Option; import com.google.common.util.concurrent.ServiceManager; -import com.google.inject.AbstractModule; import com.google.inject.Binder; import com.google.inject.Guice; import com.google.inject.Injector; @@ -31,32 +29,19 @@ import org.graylog.datanode.bindings.GenericInitializerBindings; import org.graylog.datanode.bindings.OpensearchProcessBindings; import org.graylog.datanode.bindings.PreflightChecksBindings; -import org.graylog.datanode.bindings.SchedulerBindings; import org.graylog.datanode.bootstrap.preflight.PreflightClusterConfigurationModule; import org.graylog2.bindings.NamedConfigParametersOverrideModule; -import org.graylog2.bootstrap.preflight.MongoDBPreflightCheck; -import org.graylog2.bootstrap.preflight.PreflightCheckException; import org.graylog2.bootstrap.preflight.PreflightCheckService; -import org.graylog2.cluster.ClusterConfigServiceImpl; -import org.graylog2.configuration.TLSProtocolsConfiguration; +import org.graylog2.commands.AbstractNodeCommand; import org.graylog2.plugin.Plugin; import org.graylog2.plugin.Tools; -import org.graylog2.plugin.cluster.ClusterConfigService; -import org.graylog2.shared.bindings.FreshInstallDetectionModule; import org.graylog2.shared.bindings.IsDevelopmentBindings; -import org.graylog2.shared.plugins.ChainingClassLoader; import org.graylog2.shared.system.activities.Activity; import org.graylog2.shared.system.activities.ActivityWriter; import org.jsoftbiz.utils.OS; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; import java.util.Collection; import java.util.List; import java.util.Set; @@ -64,59 +49,18 @@ import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; -import static com.google.common.base.Strings.isNullOrEmpty; - -public abstract class ServerBootstrap extends CmdLineTool { - private static final Logger LOG = LoggerFactory.getLogger(ServerBootstrap.class); - private boolean isFreshInstallation; +public abstract class DatanodeBootstrap extends AbstractNodeCommand { + private static final Logger LOG = LoggerFactory.getLogger(DatanodeBootstrap.class); protected Configuration configuration; - protected ServerBootstrap(String commandName, Configuration configuration) { - super(commandName); + protected DatanodeBootstrap(String commandName, Configuration configuration) { + super(commandName, configuration); this.commandName = commandName; this.configuration = configuration; } - @Option(name = {"-p", "--pidfile"}, description = "File containing the PID of Graylog DataNode") - private String pidFile = TMPDIR + FILE_SEPARATOR + "datanode.pid"; - - @Option(name = {"-np", "--no-pid-file"}, description = "Do not write a PID file (overrides -p/--pidfile)") - private boolean noPidFile = false; - protected abstract void startNodeRegistration(Injector injector); - public String getPidFile() { - return pidFile; - } - - public boolean isNoPidFile() { - return noPidFile; - } - - private boolean isFreshInstallation() { - return isFreshInstallation; - } - - private void registerFreshInstallation() { - this.isFreshInstallation = true; - } - - @Override - protected void beforeStart(TLSProtocolsConfiguration tlsProtocolsConfiguration, Configuration configuration) { - super.beforeStart(tlsProtocolsConfiguration, configuration); - - // Do not use a PID file if the user requested not to - if (!isNoPidFile()) { - savePidFile(getPidFile()); - } - // This needs to run before the first SSLContext is instantiated, - // because it sets up the default SSLAlgorithmConstraints - applySecuritySettings(tlsProtocolsConfiguration); - - // Set these early in the startup because netty's NativeLibraryUtil uses a static initializer - setNettyNativeDefaults(configuration); - } - @Override protected void beforeInjectorCreation(Set plugins) { runPreFlightChecks(plugins); @@ -128,42 +72,12 @@ private void runPreFlightChecks(Set plugins) { return; } - runMongoPreflightCheck(); - final List preflightCheckModules = plugins.stream().map(Plugin::preflightCheckModules) .flatMap(Collection::stream).collect(Collectors.toList()); - preflightCheckModules.add(new FreshInstallDetectionModule(isFreshInstallation())); getPreflightInjector(preflightCheckModules).getInstance(PreflightCheckService.class).runChecks(); } - private void runMongoPreflightCheck() { - // The MongoDBPreflightCheck is not run via the PreflightCheckService, - // because it also detects whether we are running on a fresh Graylog installation - final Injector injector = getMongoPreFlightInjector(); - final MongoDBPreflightCheck mongoDBPreflightCheck = injector.getInstance(MongoDBPreflightCheck.class); - try { - mongoDBPreflightCheck.runCheck(); - } catch (PreflightCheckException e) { - LOG.error("Preflight check failed with error: {}", e.getLocalizedMessage()); - throw e; - } - - if (mongoDBPreflightCheck.isFreshInstallation()) { - registerFreshInstallation(); - } - } - - private Injector getMongoPreFlightInjector() { - return Guice.createInjector( - new IsDevelopmentBindings(), - new NamedConfigParametersOverrideModule(jadConfig.getConfigurationBeans()), - new ConfigurationModule(configuration), - new DatanodeConfigurationBindings() - - ); - } - private Injector getPreflightInjector(List preflightCheckModules) { return Guice.createInjector( new IsDevelopmentBindings(), @@ -180,24 +94,13 @@ public void configure(Binder binder) { }); } - private void setNettyNativeDefaults(Configuration configuration) { - // Give netty a better spot than /tmp to unpack its tcnative libraries - if (System.getProperty("io.netty.native.workdir") == null) { - System.setProperty("io.netty.native.workdir", configuration.getNativeLibDir().toAbsolutePath().toString()); - } - // Don't delete the native lib after unpacking, as this confuses needrestart(1) on some distributions - if (System.getProperty("io.netty.native.deleteLibAfterLoading") == null) { - System.setProperty("io.netty.native.deleteLibAfterLoading", "false"); - } - } - @Override protected void startCommand() { final String systemInformation = Tools.getSystemInformation(); final OS os = OS.getOs(); - LOG.info("Graylog {} {} starting up", commandName, version); + LOG.info("Graylog Data Node {} starting up (command: {})", version, commandName); LOG.info("JRE: {}", systemInformation); LOG.info("Deployment: {}", configuration.getInstallationSource()); LOG.info("OS: {}", os.getPlatformName()); @@ -247,33 +150,10 @@ protected void startCommand() { } } - public void runMigrations() { - LOG.info("Running {} migrations...", 0); - } - - protected void savePidFile(final String pidFile) { - final String pid = Tools.getPID(); - final Path pidFilePath = Paths.get(pidFile); - pidFilePath.toFile().deleteOnExit(); - - try { - if (isNullOrEmpty(pid) || "unknown".equals(pid)) { - throw new Exception("Could not determine PID."); - } - - Files.write(pidFilePath, pid.getBytes(StandardCharsets.UTF_8), StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW, LinkOption.NOFOLLOW_LINKS); - } catch (Exception e) { - LOG.error("Could not write PID file: " + e.getMessage(), e); - System.exit(1); - } - } - @Override protected List getSharedBindingsModules() { final List result = super.getSharedBindingsModules(); - result.add(new FreshInstallDetectionModule(isFreshInstallation())); result.add(new GenericBindings(isMigrationCommand())); - result.add(new SchedulerBindings()); result.add(new GenericInitializerBindings()); result.add(new OpensearchProcessBindings()); result.add(new DatanodeConfigurationBindings()); diff --git a/data-node/src/main/java/org/graylog/datanode/bootstrap/commands/MigrateCmd.java b/data-node/src/main/java/org/graylog/datanode/bootstrap/commands/MigrateCmd.java deleted file mode 100644 index 6e804d4c24e1..000000000000 --- a/data-node/src/main/java/org/graylog/datanode/bootstrap/commands/MigrateCmd.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (C) 2020 Graylog, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * . - */ -package org.graylog.datanode.bootstrap.commands; - -import com.github.rvesse.airline.annotations.Command; -import org.graylog.datanode.commands.Server; -import org.graylog2.plugin.Tools; -import org.graylog2.plugin.Version; -import org.jsoftbiz.utils.OS; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@Command(name = MigrateCmd.MIGRATION_COMMAND, description = "Run Graylog server migrations") -public class MigrateCmd extends Server { - public static final String MIGRATION_COMMAND = "migrate"; - - private static final Logger LOG = LoggerFactory.getLogger(MigrateCmd.class); - - private final Version version = Version.CURRENT_CLASSPATH; - - public MigrateCmd() { - super(MIGRATION_COMMAND); - } - - @Override - protected void startCommand() { - final OS os = OS.getOs(); - - LOG.info("Graylog {} {} migration command", commandName, version); - LOG.info("JRE: {}", Tools.getSystemInformation()); - LOG.info("Deployment: {}", configuration.getInstallationSource()); - LOG.info("OS: {}", os.getPlatformName()); - LOG.info("Arch: {}", os.getArch()); - - try { - runMigrations(); - } catch (Exception e) { - LOG.warn("Exception while running migrations", e); - System.exit(1); - } - - System.exit(0); - } -} diff --git a/data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/OpensearchDataDirCompatibilityCheck.java b/data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/OpensearchDataDirCompatibilityCheck.java index d7ecb96e9b20..2445cc2ffef6 100644 --- a/data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/OpensearchDataDirCompatibilityCheck.java +++ b/data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/OpensearchDataDirCompatibilityCheck.java @@ -18,7 +18,6 @@ import com.github.joschi.jadconfig.ValidationException; import jakarta.inject.Inject; -import jakarta.inject.Named; import org.graylog.datanode.DirectoryReadableValidator; import org.graylog.datanode.configuration.DatanodeConfiguration; import org.graylog.datanode.filesystem.index.IncompatibleIndexVersionException; @@ -38,15 +37,13 @@ public class OpensearchDataDirCompatibilityCheck implements PreflightCheck { private static final Logger LOG = LoggerFactory.getLogger(OpensearchDataDirCompatibilityCheck.class); - private final boolean isFreshInstallation; private final DatanodeConfiguration datanodeConfiguration; private final IndicesDirectoryParser indicesDirectoryParser; private final DirectoryReadableValidator directoryReadableValidator = new DirectoryReadableValidator(); @Inject - public OpensearchDataDirCompatibilityCheck(@Named("isFreshInstallation") boolean isFreshInstallation, DatanodeConfiguration datanodeConfiguration, IndicesDirectoryParser indicesDirectoryParser) { - this.isFreshInstallation = isFreshInstallation; + public OpensearchDataDirCompatibilityCheck(DatanodeConfiguration datanodeConfiguration, IndicesDirectoryParser indicesDirectoryParser) { this.datanodeConfiguration = datanodeConfiguration; this.indicesDirectoryParser = indicesDirectoryParser; } @@ -54,10 +51,6 @@ public OpensearchDataDirCompatibilityCheck(@Named("isFreshInstallation") boolean @Override public void runCheck() throws PreflightCheckException { - if (!isFreshInstallation) { - return; - } - final Path opensearchDataDir = datanodeConfiguration.datanodeDirectories().getDataTargetDir(); final String opensearchVersion = datanodeConfiguration.opensearchDistributionProvider().get().version(); diff --git a/data-node/src/main/java/org/graylog/datanode/commands/Server.java b/data-node/src/main/java/org/graylog/datanode/commands/Datanode.java similarity index 78% rename from data-node/src/main/java/org/graylog/datanode/commands/Server.java rename to data-node/src/main/java/org/graylog/datanode/commands/Datanode.java index a9fab6a7f1d4..1ffa355f9e5d 100644 --- a/data-node/src/main/java/org/graylog/datanode/commands/Server.java +++ b/data-node/src/main/java/org/graylog/datanode/commands/Datanode.java @@ -17,7 +17,6 @@ package org.graylog.datanode.commands; import com.github.rvesse.airline.annotations.Command; -import com.github.rvesse.airline.annotations.Option; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ServiceManager; import com.google.inject.Injector; @@ -25,13 +24,14 @@ import com.google.inject.Module; import com.google.inject.spi.Message; import com.mongodb.MongoException; +import jakarta.annotation.Nonnull; import jakarta.inject.Inject; import org.graylog.datanode.Configuration; import org.graylog.datanode.bindings.ConfigurationModule; +import org.graylog.datanode.bindings.DatanodeServerBindings; import org.graylog.datanode.bindings.PeriodicalBindings; -import org.graylog.datanode.bindings.ServerBindings; +import org.graylog.datanode.bootstrap.DatanodeBootstrap; import org.graylog.datanode.bootstrap.Main; -import org.graylog.datanode.bootstrap.ServerBootstrap; import org.graylog.datanode.configuration.DatanodeProvisioningBindings; import org.graylog.datanode.configuration.S3RepositoryConfiguration; import org.graylog.datanode.rest.RestBindings; @@ -40,13 +40,11 @@ import org.graylog2.cluster.nodes.DataNodeDto; import org.graylog2.cluster.nodes.DataNodeStatus; import org.graylog2.cluster.nodes.NodeService; -import org.graylog2.configuration.MongoDbConfiguration; import org.graylog2.configuration.TLSProtocolsConfiguration; import org.graylog2.featureflag.FeatureFlags; import org.graylog2.plugin.Tools; import org.graylog2.plugin.system.NodeId; import org.graylog2.shared.UI; -import org.graylog2.shared.bindings.ObjectMapperModule; import org.graylog2.shared.system.activities.Activity; import org.graylog2.shared.system.activities.ActivityWriter; import org.slf4j.Logger; @@ -57,53 +55,43 @@ import java.util.List; -@Command(name = "datanode", description = "Start the Graylog DataNode") -public class Server extends ServerBootstrap { - private static final Logger LOG = LoggerFactory.getLogger(Server.class); +@Command(name = "datanode", description = "Start Graylog Data Node") +public class Datanode extends DatanodeBootstrap { + private static final Logger LOG = LoggerFactory.getLogger(Datanode.class); - protected static final Configuration configuration = new Configuration(); private final S3RepositoryConfiguration s3RepositoryConfiguration = new S3RepositoryConfiguration(); - private final MongoDbConfiguration mongoDbConfiguration = new MongoDbConfiguration(); private final TLSProtocolsConfiguration tlsConfiguration = new TLSProtocolsConfiguration(); - public Server() { - super("datanode", configuration); - } - - public Server(String commandName) { - super(commandName, configuration); - } - - @Option(name = {"-l", "--local"}, description = "Run Graylog DataNode in local mode. Only interesting for Graylog developers.") - private boolean local = false; - - public boolean isLocal() { - return local; + public Datanode() { + super("datanode", new Configuration()); } @Override - protected List getCommandBindings(FeatureFlags featureFlags) { + protected @Nonnull List getNodeCommandBindings(FeatureFlags featureFlags) { final ImmutableList.Builder modules = ImmutableList.builder(); modules.add( new ConfigurationModule(configuration), new MongoDBModule(), - new ServerBindings(configuration, isMigrationCommand()), + new DatanodeServerBindings(), new RestBindings(), new DatanodeProvisioningBindings(), - new PeriodicalBindings(), - new ObjectMapperModule(chainingClassLoader) + new PeriodicalBindings() ); return modules.build(); } @Override - public List getCommandConfigurationBeans() { + public @Nonnull List getNodeCommandConfigurationBeans() { return Arrays.asList(configuration, - mongoDbConfiguration, tlsConfiguration, s3RepositoryConfiguration); } + @Override + protected Class shutdownHook() { + return ShutdownHook.class; + } + private static class ShutdownHook implements Runnable { private final ActivityWriter activityWriter; private final ServiceManager serviceManager; @@ -142,11 +130,6 @@ protected void startNodeRegistration(Injector injector) { .build()); } - @Override - protected Class shutdownHook() { - return ShutdownHook.class; - } - @Override protected void annotateInjectorExceptions(Collection messages) { super.annotateInjectorExceptions(messages); diff --git a/data-node/src/main/java/org/graylog/datanode/commands/ServerCommandsProvider.java b/data-node/src/main/java/org/graylog/datanode/commands/DatanodeCommandsProvider.java similarity index 88% rename from data-node/src/main/java/org/graylog/datanode/commands/ServerCommandsProvider.java rename to data-node/src/main/java/org/graylog/datanode/commands/DatanodeCommandsProvider.java index 7273400952da..17a8b77737e2 100644 --- a/data-node/src/main/java/org/graylog/datanode/commands/ServerCommandsProvider.java +++ b/data-node/src/main/java/org/graylog/datanode/commands/DatanodeCommandsProvider.java @@ -20,9 +20,9 @@ import org.graylog2.bootstrap.CliCommand; import org.graylog2.bootstrap.CliCommandsProvider; -public class ServerCommandsProvider implements CliCommandsProvider { +public class DatanodeCommandsProvider implements CliCommandsProvider { @Override public void addTopLevelCommandsOrGroups(CliBuilder builder) { - builder.withCommand(Server.class); + builder.withCommand(Datanode.class); } } diff --git a/data-node/src/main/resources/META-INF/services/org.graylog2.bootstrap.CliCommandsProvider b/data-node/src/main/resources/META-INF/services/org.graylog2.bootstrap.CliCommandsProvider index 4ccef3327922..1b38f84314c6 100644 --- a/data-node/src/main/resources/META-INF/services/org.graylog2.bootstrap.CliCommandsProvider +++ b/data-node/src/main/resources/META-INF/services/org.graylog2.bootstrap.CliCommandsProvider @@ -1 +1 @@ -org.graylog.datanode.commands.ServerCommandsProvider +org.graylog.datanode.commands.DatanodeCommandsProvider diff --git a/data-node/src/test/java/org/graylog/datanode/ConfigurationDocumentationTest.java b/data-node/src/test/java/org/graylog/datanode/ConfigurationDocumentationTest.java index 7581ee8ef69b..3b63f09e7c48 100644 --- a/data-node/src/test/java/org/graylog/datanode/ConfigurationDocumentationTest.java +++ b/data-node/src/test/java/org/graylog/datanode/ConfigurationDocumentationTest.java @@ -22,15 +22,13 @@ import org.apache.commons.csv.CSVPrinter; import org.apache.commons.lang3.ClassUtils; import org.bson.assertions.Assertions; -import org.graylog.datanode.commands.Server; +import org.graylog.datanode.commands.Datanode; import org.graylog2.configuration.Documentation; import org.junit.jupiter.api.Test; import java.io.IOException; import java.io.StringWriter; import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -40,7 +38,7 @@ class ConfigurationDocumentationTest { @Test void testAllFieldsAreDocumented() { - final List datanodeConfiguration = new Server().getCommandConfigurationBeans(); + final List datanodeConfiguration = new Datanode().getNodeCommandConfigurationBeans(); final List undocumentedFields = datanodeConfiguration.stream().flatMap(configurationBean -> { return Arrays.stream(configurationBean.getClass().getDeclaredFields()) .filter(f -> f.isAnnotationPresent(Parameter.class)) @@ -65,7 +63,7 @@ public static void main(String[] args) throws IOException { printer.printRecord("Parameter", "Type", "Required", "Default value", "Description"); - final List datanodeConfiguration = new Server().getCommandConfigurationBeans(); + final List datanodeConfiguration = new Datanode().getNodeCommandConfigurationBeans(); datanodeConfiguration.forEach(configurationBean -> { Arrays.stream(configurationBean.getClass().getDeclaredFields()) diff --git a/graylog2-server/src/main/java/org/graylog2/CommonNodeConfiguration.java b/graylog2-server/src/main/java/org/graylog2/CommonNodeConfiguration.java new file mode 100644 index 000000000000..9b2666b98524 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/CommonNodeConfiguration.java @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog2; + +import org.graylog2.plugin.ServerStatus; + +import java.util.Set; + +/** + * Helper class to hold configuration shared by all Graylog node types + */ +public interface CommonNodeConfiguration extends GraylogNodeConfiguration { + + @Override + default boolean withMongoDb() { + return true; + } + + @Override + default boolean withScheduler() { + return true; + } + + @Override + default boolean withEventBus() { + return true; + } + + @Override + default boolean withPlugins() { + return false; + } + + @Override + default boolean withNodeIdFile() { + return true; + } + + @Override + default Set withCapabilities() { + return Set.of(ServerStatus.Capability.SERVER); + } + + @Override + default boolean isMessageRecordingsEnabled() { + return false; + } + + @Override + default String getEnvironmentVariablePrefix() { + return "GRAYLOG_"; + } + + @Override + default String getSystemPropertyPrefix() { + return "graylog."; + } +} diff --git a/graylog2-server/src/main/java/org/graylog2/Configuration.java b/graylog2-server/src/main/java/org/graylog2/Configuration.java index cc01d2b40b09..2ac5763422bc 100644 --- a/graylog2-server/src/main/java/org/graylog2/Configuration.java +++ b/graylog2-server/src/main/java/org/graylog2/Configuration.java @@ -61,7 +61,7 @@ * Helper class to hold configuration of Graylog */ @SuppressWarnings("FieldMayBeFinal") -public class Configuration extends CaConfiguration { +public class Configuration extends CaConfiguration implements CommonNodeConfiguration { public static final String SAFE_CLASSES = "safe_classes"; public static final String CONTENT_PACKS_DIR = "content_packs_dir"; @@ -666,4 +666,8 @@ private static int defaultNumberOfOutputBufferProcessors() { return Math.round(Tools.availableProcessors() * 0.162f + 0.625f); } + @Override + public boolean withPlugins() { + return true; + } } diff --git a/graylog2-server/src/main/java/org/graylog2/GraylogNodeConfiguration.java b/graylog2-server/src/main/java/org/graylog2/GraylogNodeConfiguration.java new file mode 100644 index 000000000000..81022704c13f --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/GraylogNodeConfiguration.java @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog2; + +import org.graylog2.plugin.ServerStatus; + +import java.util.Set; + +/** + * Helper class to hold configuration shared by all Graylog node types + */ +public interface GraylogNodeConfiguration { + + /** + * This will load bindings required for basic database connection and mongojack infrastructure. + */ + boolean withMongoDb(); + + /** + * Binds the scheduled executors for daemon and non-daemon usage in the node. + */ + boolean withScheduler(); + + /** + * Binds event bus and cluster event bus. + */ + boolean withEventBus(); + + /** + * Configures node startup to load plugin configurations and plugins. + */ + boolean withPlugins(); + + /** + * Will bind NodeId to an id provided by FilePersistedNodeIdProvider. + * Falls back to use a dummy node id if set to 'false'. + */ + boolean withNodeIdFile(); + + /** + * Provides the {@link ServerStatus.Capability} to be used by ServerStatusBindings. + */ + Set withCapabilities(); + + /** + * Environment variable prefix to be used for this node (e.g. 
    GRAYLOG_
    for Graylog nodes). + */ + String getEnvironmentVariablePrefix(); + + /** + * System property prefix to be used for this node (e.g. 
    graylog.
    for Graylog nodes). + */ + String getSystemPropertyPrefix(); + + /** + * Enables message recording in ServerStatus. + */ + boolean isMessageRecordingsEnabled(); +} diff --git a/graylog2-server/src/main/java/org/graylog2/bindings/ConfigurationModule.java b/graylog2-server/src/main/java/org/graylog2/bindings/ConfigurationModule.java index 9c87885c50f0..b46423a001d4 100644 --- a/graylog2-server/src/main/java/org/graylog2/bindings/ConfigurationModule.java +++ b/graylog2-server/src/main/java/org/graylog2/bindings/ConfigurationModule.java @@ -19,6 +19,7 @@ import com.google.inject.Binder; import com.google.inject.Module; import org.graylog2.Configuration; +import org.graylog2.GraylogNodeConfiguration; import org.graylog2.plugin.BaseConfiguration; import static java.util.Objects.requireNonNull; @@ -34,5 +35,7 @@ public ConfigurationModule(Configuration configuration) { public void configure(Binder binder) { binder.bind(Configuration.class).toInstance(configuration); binder.bind(BaseConfiguration.class).toInstance(configuration); + // even though this is already bound in the GraylogNodeModule, this needs to be bound here for use in preflight + binder.bind(GraylogNodeConfiguration.class).toInstance(configuration); } } diff --git a/graylog2-server/src/main/java/org/graylog2/bindings/GraylogNodeModule.java b/graylog2-server/src/main/java/org/graylog2/bindings/GraylogNodeModule.java new file mode 100644 index 000000000000..4ed076768e4a --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/bindings/GraylogNodeModule.java @@ -0,0 +1,95 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog2.bindings; + +import com.google.common.eventbus.EventBus; +import com.google.inject.Scopes; +import org.glassfish.jersey.internal.guava.Sets; +import org.graylog2.GraylogNodeConfiguration; +import org.graylog2.audit.AuditBindings; +import org.graylog2.bindings.providers.ClusterEventBusProvider; +import org.graylog2.configuration.EventBusConfiguration; +import org.graylog2.configuration.MongoDbConfiguration; +import org.graylog2.events.ClusterEventBus; +import org.graylog2.jackson.InputConfigurationBeanDeserializerModifier; +import org.graylog2.plugin.LocalMetricRegistry; +import org.graylog2.plugin.inject.Graylog2Module; +import org.graylog2.plugin.system.FilePersistedNodeIdProvider; +import org.graylog2.plugin.system.NodeId; +import org.graylog2.plugin.system.SimpleNodeId; +import org.graylog2.security.encryption.EncryptedValueService; +import org.graylog2.shared.bindings.ObjectMapperModule; +import org.graylog2.shared.bindings.SchedulerBindings; +import org.graylog2.shared.bindings.ServerStatusBindings; +import org.graylog2.shared.bindings.providers.EventBusProvider; + +import java.util.Set; + +/** + *

    Guice module that contains all necessary bindings to start a basic node in a Graylog cluster.

    + *

    Bindings will be loaded conditionally based on the configuration in {@link GraylogNodeConfiguration}

    + */ +public class GraylogNodeModule extends Graylog2Module { + private final GraylogNodeConfiguration configuration; + + public GraylogNodeModule(final GraylogNodeConfiguration configuration) { + this.configuration = configuration; + } + + @Override + protected void configure() { + bind(GraylogNodeConfiguration.class).toInstance(configuration); + if (configuration.withMongoDb()) { + install(new MongoDbConnectionModule()); + install(new ObjectMapperModule()); + } + if (configuration.withScheduler()) { + install(new SchedulerBindings()); + } + install(new AuditBindings()); + + if (configuration.withEventBus()) { + bind(ClusterEventBus.class).toProvider(ClusterEventBusProvider.class).asEagerSingleton(); + bind(EventBus.class).toProvider(EventBusProvider.class).in(Scopes.SINGLETON); + } + // ensure we always create a new LocalMetricRegistry, they are meant to be separate from each other + bind(LocalMetricRegistry.class).in(Scopes.NO_SCOPE); + + if (configuration.withNodeIdFile()) { + bind(NodeId.class).toProvider(FilePersistedNodeIdProvider.class).asEagerSingleton(); + } else { + bind(NodeId.class).toInstance(new SimpleNodeId("dummy-nodeid")); + } + + install(new ServerStatusBindings(configuration.withCapabilities())); + + bind(EncryptedValueService.class).asEagerSingleton(); + bind(InputConfigurationBeanDeserializerModifier.class).toInstance(InputConfigurationBeanDeserializerModifier.withoutConfig()); + } + + public Set getConfigurationBeans() { + Set configurationBeans = Sets.newHashSet(); + configurationBeans.add(configuration); + if (configuration.withMongoDb()) { + configurationBeans.add(new MongoDbConfiguration()); + } + if (configuration.withEventBus()) { + configurationBeans.add(new EventBusConfiguration()); + } + return configurationBeans; + } +} diff --git a/graylog2-server/src/main/java/org/graylog2/bindings/MongoDBModule.java b/graylog2-server/src/main/java/org/graylog2/bindings/MongoDBModule.java index d6e3b60a0d5a..8a84108d529c 100644 --- a/graylog2-server/src/main/java/org/graylog2/bindings/MongoDBModule.java +++ b/graylog2-server/src/main/java/org/graylog2/bindings/MongoDBModule.java @@ -17,9 +17,6 @@ package org.graylog2.bindings; import com.google.inject.AbstractModule; -import org.graylog2.bindings.providers.MongoConnectionProvider; -import org.graylog2.database.MongoCollections; -import org.graylog2.database.MongoConnection; import org.graylog2.database.dbcatalog.DbEntitiesCatalog; import org.graylog2.database.dbcatalog.DbEntitiesScanner; @@ -27,7 +24,5 @@ public class MongoDBModule extends AbstractModule { @Override protected void configure() { bind(DbEntitiesCatalog.class).toProvider(DbEntitiesScanner.class).asEagerSingleton(); - bind(MongoConnection.class).toProvider(MongoConnectionProvider.class); - bind(MongoCollections.class).asEagerSingleton(); } } diff --git a/graylog2-server/src/main/java/org/graylog2/bindings/MongoDbConnectionModule.java b/graylog2-server/src/main/java/org/graylog2/bindings/MongoDbConnectionModule.java new file mode 100644 index 000000000000..f7beada37c2f --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/bindings/MongoDbConnectionModule.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog2.bindings; + +import org.graylog2.bindings.providers.MongoConnectionProvider; +import org.graylog2.bindings.providers.MongoJackObjectMapperProvider; +import org.graylog2.configuration.MongoDbConfiguration; +import org.graylog2.database.MongoCollections; +import org.graylog2.database.MongoConnection; +import org.graylog2.plugin.inject.Graylog2Module; + +import java.util.Set; + +/** + * Provides a basic MongoDB connection ready to be used with mongojack + */ +public class MongoDbConnectionModule extends Graylog2Module { + @Override + protected void configure() { + bind(MongoConnection.class).toProvider(MongoConnectionProvider.class); + bind(MongoCollections.class).asEagerSingleton(); + bind(MongoJackObjectMapperProvider.class); + } + + @Override + protected Set getConfigurationBeans() { + return Set.of(new MongoDbConfiguration()); + } +} diff --git a/graylog2-server/src/main/java/org/graylog2/bindings/ServerBindings.java b/graylog2-server/src/main/java/org/graylog2/bindings/ServerBindings.java index 237c3b6aa631..42ac0d3305a5 100644 --- a/graylog2-server/src/main/java/org/graylog2/bindings/ServerBindings.java +++ b/graylog2-server/src/main/java/org/graylog2/bindings/ServerBindings.java @@ -31,7 +31,6 @@ import org.graylog2.alerts.AlertSender; import org.graylog2.alerts.EmailRecipients; import org.graylog2.alerts.FormattedEmailAlertSender; -import org.graylog2.bindings.providers.ClusterEventBusProvider; import org.graylog2.bindings.providers.DefaultSecurityManagerProvider; import org.graylog2.bindings.providers.DefaultStreamProvider; import org.graylog2.bindings.providers.HtmlSafeJmteEngineProvider; @@ -45,7 +44,6 @@ import org.graylog2.cluster.leader.FakeLeaderElectionModule; import org.graylog2.cluster.leader.LeaderElectionModule; import org.graylog2.cluster.lock.LockServiceModule; -import org.graylog2.events.ClusterEventBus; import org.graylog2.grok.GrokModule; import org.graylog2.grok.GrokPatternRegistry; import org.graylog2.indexer.fieldtypes.FieldTypesModule; @@ -158,7 +156,6 @@ protected void configure() { } private void bindProviders() { - bind(ClusterEventBus.class).toProvider(ClusterEventBusProvider.class).asEagerSingleton(); bind(freemarker.template.Configuration.class).toProvider(SecureFreemarkerConfigProvider.class); } diff --git a/graylog2-server/src/main/java/org/graylog2/bootstrap/CmdLineTool.java b/graylog2-server/src/main/java/org/graylog2/bootstrap/CmdLineTool.java index 58347d903df7..b803ba7576d0 100644 --- a/graylog2-server/src/main/java/org/graylog2/bootstrap/CmdLineTool.java +++ b/graylog2-server/src/main/java/org/graylog2/bootstrap/CmdLineTool.java @@ -32,6 +32,7 @@ import com.github.luben.zstd.util.Native; import com.github.rvesse.airline.annotations.Command; import com.github.rvesse.airline.annotations.Option; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -55,6 +56,7 @@ import org.apache.logging.log4j.core.LoggerContext; import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.graylog2.Configuration; +import org.graylog2.GraylogNodeConfiguration; import org.graylog2.bindings.NamedConfigParametersOverrideModule; import org.graylog2.bootstrap.commands.MigrateCmd; import org.graylog2.configuration.PathConfiguration; @@ -102,10 +104,7 @@ import static com.google.common.base.Strings.nullToEmpty; -public abstract class CmdLineTool implements CliCommand { - - public static final String GRAYLOG_ENVIRONMENT_VAR_PREFIX = "GRAYLOG_"; - public static final String GRAYLOG_SYSTEM_PROP_PREFIX = "graylog."; +public abstract class CmdLineTool implements CliCommand { static { // Set up JDK Logging adapter, https://logging.apache.org/log4j/2.x/log4j-jul/index.html @@ -119,7 +118,7 @@ public abstract class CmdLineTool implements CliCommand { protected static final String TMPDIR = System.getProperty("java.io.tmpdir", "/tmp"); protected final JadConfig jadConfig; - protected final Configuration configuration; + protected final NodeConfiguration configuration; protected final ChainingClassLoader chainingClassLoader; @Option(name = "--dump-config", description = "Show the effective Graylog configuration and exit") @@ -132,10 +131,10 @@ public abstract class CmdLineTool implements CliCommand { private boolean debug = false; @Option(name = {"-f", "--configfile"}, description = "Configuration file for Graylog") - private String configFile = "/etc/graylog/server/server.conf"; + protected String configFile = "/etc/graylog/server/server.conf"; @Option(name = {"-ff", "--featureflagfile"}, description = "Configuration file for Graylog feature flags") - private String customFeatureFlagFile = "/etc/graylog/server/feature-flag.conf"; + protected String customFeatureFlagFile = "/etc/graylog/server/feature-flag.conf"; protected String commandName = "command"; @@ -144,11 +143,11 @@ public abstract class CmdLineTool implements CliCommand { protected FeatureFlags featureFlags; protected PluginLoader pluginLoader; - protected CmdLineTool(Configuration configuration) { + protected CmdLineTool(NodeConfiguration configuration) { this(null, configuration); } - protected CmdLineTool(String commandName, Configuration configuration) { + protected CmdLineTool(String commandName, NodeConfiguration configuration) { jadConfig = new JadConfig(); jadConfig.addConverterFactory(new GuavaConverterFactory()); jadConfig.addConverterFactory(new JodaTimeConverterFactory()); @@ -200,13 +199,9 @@ public boolean isMigrationCommand() { * Please note that this happens *before* the configuration file has been parsed. */ protected void beforeStart() { - } - - /** - * Things that have to run before the {@link #startCommand()} method is being called. - * Please note that this happens *before* the configuration file has been parsed. - */ - protected void beforeStart(TLSProtocolsConfiguration configuration, PathConfiguration pathConfiguration) { + // This needs to run before the first SSLContext is instantiated, + // because it sets up the default SSLAlgorithmConstraints + applySecuritySettings(parseAndGetTLSConfiguration(configFile)); } /** @@ -275,34 +270,40 @@ public void run() { } public void doRun(Level logLevel) { - final PluginLoaderConfig pluginLoaderConfig = getPluginLoaderConfig(configFile); - - // Move the zstd temp folder from /tmp to our native lib dir to avoid issues with noexec-mounted /tmp directories. - // See: https://github.com/Graylog2/graylog2-server/issues/17837 - // WARNING: This needs to be set before the first use of the zstd library. Our in-memory logger is using - // zstd library, so we need to set it before the first usage of the Logger instance. - // Setting it after the first library usage wouldn't have any effect. - if (Native.isLoaded()) { - LOG.warn("The zstd library is already loaded. Setting the ZstdTempFolder property doesn't have any effect!"); - } - final Path nativeLibPath = pluginLoaderConfig.getNativeLibDir().toAbsolutePath(); - try { - // We are very early in the startup process and the data_dir and native lib dir don't exist yet. Since the - // zstd library doesn't create its own temp directory, we have to do it to avoid errors on startup. - Files.createDirectories(nativeLibPath); - System.setProperty("ZstdTempFolder", nativeLibPath.toString()); - } catch (IOException e) { - LOG.warn("Couldn't create native lib dir <{}>. Unable to set ZstdTempFolder system property.", nativeLibPath, e); + if (configuration instanceof PathConfiguration) { + PathConfiguration pathConfiguration = parseAndGetPathConfiguration(configFile); + + // Move the zstd temp folder from /tmp to our native lib dir to avoid issues with noexec-mounted /tmp directories. + // See: https://github.com/Graylog2/graylog2-server/issues/17837 + // WARNING: This needs to be set before the first use of the zstd library. Our in-memory logger is using + // zstd library, so we need to set it before the first usage of the Logger instance. + // Setting it after the first library usage wouldn't have any effect. + if (Native.isLoaded()) { + LOG.warn("The zstd library is already loaded. Setting the ZstdTempFolder property doesn't have any effect!"); + } + final Path nativeLibPath = pathConfiguration.getNativeLibDir().toAbsolutePath(); + try { + // We are very early in the startup process and the data_dir and native lib dir don't exist yet. Since the + // zstd library doesn't create its own temp directory, we have to do it to avoid errors on startup. + Files.createDirectories(nativeLibPath); + System.setProperty("ZstdTempFolder", nativeLibPath.toString()); + } catch (IOException e) { + LOG.warn("Couldn't create native lib dir <{}>. Unable to set ZstdTempFolder system property.", nativeLibPath, e); + } } // This is holding all our metrics. MetricRegistry metricRegistry = MetricRegistryFactory.create(); featureFlags = getFeatureFlags(metricRegistry); - pluginLoader = getPluginLoader(pluginLoaderConfig, chainingClassLoader); + if (configuration.withPlugins()) { + pluginLoader = getPluginLoader(getPluginLoaderConfig(configFile), chainingClassLoader); + } installCommandConfig(); - installPluginBootstrapConfig(pluginLoader); + if (configuration.withPlugins()) { + installPluginBootstrapConfig(pluginLoader); + } if (isDumpDefaultConfig()) { dumpDefaultConfigAndExit(); @@ -311,16 +312,17 @@ public void doRun(Level logLevel) { installConfigRepositories(); beforeStart(); - beforeStart(parseAndGetTLSConfiguration(), parseAndGetPathConfiguration(configFile)); processConfiguration(jadConfig); bootstrapConfigInjector = setupBootstrapConfigInjector(); - final Set plugins = loadPlugins(); - - installPluginConfig(plugins); - processConfiguration(jadConfig); + Set plugins = new HashSet<>(); + if (configuration.withPlugins()) { + plugins = loadPlugins(); + installPluginConfig(plugins); + processConfiguration(jadConfig); + } if (isDumpConfig()) { dumpCurrentConfigAndExit(); @@ -372,7 +374,7 @@ private void installPluginBootstrapConfig(PluginLoader pluginLoader) { // Parse only the TLSConfiguration bean // to avoid triggering anything that might initialize the default SSLContext - private TLSProtocolsConfiguration parseAndGetTLSConfiguration() { + protected TLSProtocolsConfiguration parseAndGetTLSConfiguration(String configFile) { final JadConfig jadConfig = new JadConfig(); jadConfig.setRepositories(getConfigRepositories(configFile)); final TLSProtocolsConfiguration tlsConfiguration = new TLSProtocolsConfiguration(); @@ -382,7 +384,7 @@ private TLSProtocolsConfiguration parseAndGetTLSConfiguration() { return tlsConfiguration; } - private PathConfiguration parseAndGetPathConfiguration(String configFile) { + protected PathConfiguration parseAndGetPathConfiguration(String configFile) { final PathConfiguration pathConfiguration = new PathConfiguration(); processConfiguration(new JadConfig(getConfigRepositories(configFile), pathConfiguration)); return pathConfiguration; @@ -451,7 +453,9 @@ private void dumpCurrentConfigAndExit() { private void dumpDefaultConfigAndExit() { bootstrapConfigInjector = setupBootstrapConfigInjector(); - installPluginConfig(pluginLoader.loadPlugins(bootstrapConfigInjector)); + if (configuration.withPlugins()) { + installPluginConfig(pluginLoader.loadPlugins(bootstrapConfigInjector)); + } dumpCurrentConfigAndExit(); } @@ -463,7 +467,7 @@ private PluginLoaderConfig getPluginLoaderConfig(String configFile) { } private FeatureFlags getFeatureFlags(MetricRegistry metricRegistry) { - return new FeatureFlagsFactory().createImmutableFeatureFlags(customFeatureFlagFile, metricRegistry); + return new FeatureFlagsFactory().createImmutableFeatureFlags(customFeatureFlagFile, metricRegistry, configuration); } protected Set loadPlugins() { @@ -498,8 +502,8 @@ protected Set loadPlugins() { protected Collection getConfigRepositories(String configFile) { return Arrays.asList( - new EnvironmentRepository(GRAYLOG_ENVIRONMENT_VAR_PREFIX), - new SystemPropertiesRepository(GRAYLOG_SYSTEM_PROP_PREFIX), + new EnvironmentRepository(configuration.getEnvironmentVariablePrefix()), + new SystemPropertiesRepository(configuration.getSystemPropertyPrefix()), // Legacy prefixes new EnvironmentRepository("GRAYLOG2_"), new SystemPropertiesRepository("graylog2."), @@ -603,9 +607,9 @@ protected void annotateInjectorExceptions(Collection messages) { for (Message message : messages) { //noinspection ThrowableResultOfMethodCallIgnored final Throwable rootCause = ExceptionUtils.getRootCause(message.getCause()); - if (rootCause instanceof NodeIdPersistenceException) { + if (configuration.withNodeIdFile() && rootCause instanceof NodeIdPersistenceException) { LOG.error(UI.wallString( - "Unable to read or persist your NodeId file. This means your node id file (" + configuration.getNodeIdFile() + ") is not readable or writable by the current user. The following exception might give more information: " + message)); + "Unable to read or persist your NodeId file. This means your node id file is not readable or writable by the current user. The following exception might give more information: " + message)); System.exit(-1); } else if (rootCause instanceof AccessDeniedException) { LOG.error(UI.wallString("Unable to access file " + rootCause.getMessage())); @@ -631,4 +635,9 @@ protected void annotateInjectorExceptions(Collection messages) { protected Set capabilities() { return Collections.emptySet(); } + + @VisibleForTesting + protected void setConfigFile(String configFile) { + this.configFile = configFile; + } } diff --git a/graylog2-server/src/main/java/org/graylog2/bootstrap/ServerBootstrap.java b/graylog2-server/src/main/java/org/graylog2/bootstrap/ServerBootstrap.java index 40be8cdfc26c..c45bfbc45b50 100644 --- a/graylog2-server/src/main/java/org/graylog2/bootstrap/ServerBootstrap.java +++ b/graylog2-server/src/main/java/org/graylog2/bootstrap/ServerBootstrap.java @@ -45,9 +45,9 @@ import org.graylog2.bootstrap.preflight.web.PreflightBoot; import org.graylog2.cluster.leader.LeaderElectionService; import org.graylog2.cluster.preflight.GraylogServerProvisioningBindings; +import org.graylog2.commands.AbstractNodeCommand; import org.graylog2.configuration.IndexerDiscoveryModule; import org.graylog2.configuration.PathConfiguration; -import org.graylog2.configuration.TLSProtocolsConfiguration; import org.graylog2.migrations.Migration; import org.graylog2.migrations.MigrationType; import org.graylog2.plugin.MessageBindings; @@ -97,13 +97,16 @@ import static org.graylog2.audit.AuditEventTypes.NODE_STARTUP_INITIATE; import static org.graylog2.bootstrap.preflight.PreflightWebModule.FEATURE_FLAG_PREFLIGHT_WEB_ENABLED; -public abstract class ServerBootstrap extends CmdLineTool { +public abstract class ServerBootstrap extends AbstractNodeCommand { private static final Logger LOG = LoggerFactory.getLogger(ServerBootstrap.class); private boolean isFreshInstallation; + private final Configuration configuration; + protected ServerBootstrap(String commandName, Configuration configuration) { super(commandName, configuration); this.commandName = commandName; + this.configuration = configuration; } @Option(name = {"-p", "--pidfile"}, description = "File containing the PID of Graylog") @@ -131,20 +134,16 @@ private void registerFreshInstallation() { } @Override - protected void beforeStart(TLSProtocolsConfiguration tlsProtocolsConfiguration, PathConfiguration pathConfiguration) { - super.beforeStart(tlsProtocolsConfiguration, pathConfiguration); + protected void beforeStart() { + super.beforeStart(); // Do not use a PID file if the user requested not to if (!isNoPidFile()) { savePidFile(getPidFile()); } - // This needs to run before the first SSLContext is instantiated, - // because it sets up the default SSLAlgorithmConstraints - applySecuritySettings(tlsProtocolsConfiguration); // Set these early in the startup because netty's NativeLibraryUtil uses a static initializer - setNettyNativeDefaults(pathConfiguration); - + setNettyNativeDefaults(parseAndGetPathConfiguration(configFile)); } @Override @@ -446,10 +445,8 @@ protected List getSharedBindingsModules() { result.add(new GenericBindings(isMigrationCommand())); result.add(new MessageBindings()); result.add(new SecurityBindings()); - result.add(new ServerStatusBindings(capabilities())); result.add(new ValidatorModule()); result.add(new SharedPeriodicalBindings()); - result.add(new SchedulerBindings()); result.add(new GenericInitializerBindings()); result.add(new SystemStatsModule(configuration.isDisableNativeSystemStatsCollector())); result.add(new IndexerDiscoveryModule()); diff --git a/graylog2-server/src/main/java/org/graylog2/commands/AbstractNodeCommand.java b/graylog2-server/src/main/java/org/graylog2/commands/AbstractNodeCommand.java new file mode 100644 index 000000000000..d17473e58924 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/commands/AbstractNodeCommand.java @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog2.commands; + +import com.google.common.collect.Lists; +import com.google.inject.Module; +import jakarta.annotation.Nonnull; +import org.graylog2.GraylogNodeConfiguration; +import org.graylog2.bindings.GraylogNodeModule; +import org.graylog2.bootstrap.CmdLineTool; +import org.graylog2.featureflag.FeatureFlags; + +import java.util.ArrayList; +import java.util.List; + +/** + * Inherit from this command to create new standalone node types. + */ +public abstract class AbstractNodeCommand extends CmdLineTool { + + private final GraylogNodeModule nodeModule; + + public AbstractNodeCommand(final GraylogNodeConfiguration configuration) { + this(null, configuration); + } + + public AbstractNodeCommand(final String commandName, final GraylogNodeConfiguration configuration) { + super(commandName, configuration); + this.nodeModule = new GraylogNodeModule(configuration); + } + + @Override + protected List getCommandBindings(final FeatureFlags featureFlags) { + final List modules = Lists.newArrayList(nodeModule); + modules.addAll(getNodeCommandBindings(featureFlags)); + return modules; + } + + protected abstract @Nonnull List getNodeCommandBindings(final FeatureFlags featureFlags); + + @Override + protected List getCommandConfigurationBeans() { + final List configurationBeans = new ArrayList<>(nodeModule.getConfigurationBeans()); + configurationBeans.addAll(getNodeCommandConfigurationBeans()); + return configurationBeans; + } + + protected abstract @Nonnull List getNodeCommandConfigurationBeans(); + +} diff --git a/graylog2-server/src/main/java/org/graylog2/commands/Server.java b/graylog2-server/src/main/java/org/graylog2/commands/Server.java index 875b47250531..cd687c91134c 100644 --- a/graylog2-server/src/main/java/org/graylog2/commands/Server.java +++ b/graylog2-server/src/main/java/org/graylog2/commands/Server.java @@ -26,6 +26,7 @@ import com.google.inject.Module; import com.google.inject.spi.Message; import com.mongodb.MongoException; +import jakarta.annotation.Nonnull; import jakarta.inject.Inject; import org.graylog.enterprise.EnterpriseModule; import org.graylog.events.EventsModule; @@ -53,7 +54,6 @@ import org.graylog2.Configuration; import org.graylog2.alerts.AlertConditionBindings; import org.graylog2.audit.AuditActor; -import org.graylog2.audit.AuditBindings; import org.graylog2.audit.AuditEventSender; import org.graylog2.bindings.AlarmCallbackBindings; import org.graylog2.bindings.ConfigurationModule; @@ -76,7 +76,6 @@ import org.graylog2.configuration.ElasticsearchConfiguration; import org.graylog2.configuration.EmailConfiguration; import org.graylog2.configuration.HttpConfiguration; -import org.graylog2.configuration.MongoDbConfiguration; import org.graylog2.configuration.TLSProtocolsConfiguration; import org.graylog2.configuration.TelemetryConfiguration; import org.graylog2.configuration.VersionCheckConfiguration; @@ -102,7 +101,6 @@ import org.graylog2.rest.resources.system.ClusterConfigValidatorModule; import org.graylog2.shared.UI; import org.graylog2.shared.bindings.MessageInputBindings; -import org.graylog2.shared.bindings.ObjectMapperModule; import org.graylog2.shared.bindings.RestApiBindings; import org.graylog2.shared.journal.Journal; import org.graylog2.shared.system.activities.Activity; @@ -135,7 +133,6 @@ public class Server extends ServerBootstrap { private final ElasticsearchConfiguration elasticsearchConfiguration = new ElasticsearchConfiguration(); private final ElasticsearchClientConfiguration elasticsearchClientConfiguration = new ElasticsearchClientConfiguration(); private final EmailConfiguration emailConfiguration = new EmailConfiguration(); - private final MongoDbConfiguration mongoDbConfiguration = new MongoDbConfiguration(); private final VersionCheckConfiguration versionCheckConfiguration = new VersionCheckConfiguration(); private final KafkaJournalConfiguration kafkaJournalConfiguration = new KafkaJournalConfiguration(); private final NettyTransportConfiguration nettyTransportConfiguration = new NettyTransportConfiguration(); @@ -167,7 +164,7 @@ public boolean isLocal() { } @Override - protected List getCommandBindings(FeatureFlags featureFlags) { + protected @Nonnull List getNodeCommandBindings(FeatureFlags featureFlags) { final ImmutableList.Builder modules = ImmutableList.builder(); modules.add( new VersionAwareStorageModule(configuration), @@ -185,11 +182,9 @@ protected List getCommandBindings(FeatureFlags featureFlags) { new RotationStrategyBindings(elasticsearchConfiguration), new RetentionStrategyBindings(elasticsearchConfiguration), new PeriodicalBindings(), - new ObjectMapperModule(chainingClassLoader), new RestApiBindings(configuration), new PasswordAlgorithmBindings(), new DecoratorBindings(), - new AuditBindings(), new AlertConditionBindings(), new IndexerBindings(), new MigrationsModule(), @@ -224,13 +219,12 @@ protected List getCommandBindings(FeatureFlags featureFlags) { } @Override - protected List getCommandConfigurationBeans() { + protected @Nonnull List getNodeCommandConfigurationBeans() { return Arrays.asList(configuration, httpConfiguration, elasticsearchConfiguration, elasticsearchClientConfiguration, emailConfiguration, - mongoDbConfiguration, versionCheckConfiguration, kafkaJournalConfiguration, nettyTransportConfiguration, diff --git a/graylog2-server/src/main/java/org/graylog2/commands/journal/AbstractJournalCommand.java b/graylog2-server/src/main/java/org/graylog2/commands/journal/AbstractJournalCommand.java index 8d7849805d65..334dc0bc7a98 100644 --- a/graylog2-server/src/main/java/org/graylog2/commands/journal/AbstractJournalCommand.java +++ b/graylog2-server/src/main/java/org/graylog2/commands/journal/AbstractJournalCommand.java @@ -16,58 +16,38 @@ */ package org.graylog2.commands.journal; -import com.google.inject.AbstractModule; import com.google.inject.Module; +import jakarta.annotation.Nonnull; import org.graylog2.Configuration; -import org.graylog2.audit.AuditBindings; -import org.graylog2.bindings.ConfigurationModule; -import org.graylog2.bootstrap.CmdLineTool; +import org.graylog2.commands.AbstractNodeCommand; import org.graylog2.featureflag.FeatureFlags; import org.graylog2.plugin.KafkaJournalConfiguration; import org.graylog2.plugin.Plugin; -import org.graylog2.plugin.system.NodeId; -import org.graylog2.plugin.system.SimpleNodeId; -import org.graylog2.shared.bindings.SchedulerBindings; -import org.graylog2.shared.bindings.ServerStatusBindings; import org.graylog2.shared.journal.LocalKafkaJournal; import org.graylog2.shared.journal.LocalKafkaJournalModule; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Set; -public abstract class AbstractJournalCommand extends CmdLineTool { - protected static final Configuration configuration = new Configuration(); +public abstract class AbstractJournalCommand extends AbstractNodeCommand { protected final KafkaJournalConfiguration kafkaJournalConfiguration = new KafkaJournalConfiguration(); protected LocalKafkaJournal journal; - public AbstractJournalCommand() { - this(null); - } public AbstractJournalCommand(String commandName) { - super(commandName, configuration); + super(commandName, new JournalCommandConfiguration()); } @Override - protected List getCommandBindings(FeatureFlags featureFlags) { - return Arrays.asList( - new ConfigurationModule(configuration), - new AbstractModule() { - @Override - public void configure() { - bind(NodeId.class).toInstance(new SimpleNodeId("dummy-nodeid")); - } - }, - new ServerStatusBindings(capabilities()), - new SchedulerBindings(), - new LocalKafkaJournalModule(), - new AuditBindings()); + protected @Nonnull List getNodeCommandBindings(FeatureFlags featureFlags) { + return List.of( + new LocalKafkaJournalModule() + ); } @Override - protected List getCommandConfigurationBeans() { - return Arrays.asList(configuration, kafkaJournalConfiguration); + protected @Nonnull List getNodeCommandConfigurationBeans() { + return List.of(kafkaJournalConfiguration); } @Override @@ -98,4 +78,31 @@ protected void startCommand() { } protected abstract void runCommand(); + + static class JournalCommandConfiguration extends Configuration { + @Override + public boolean withNodeIdFile() { + return false; + } + + @Override + public boolean withScheduler() { + return true; + } + + @Override + public boolean withEventBus() { + return false; + } + + @Override + public boolean withPlugins() { + return false; + } + + @Override + public boolean withMongoDb() { + return false; + } + } } diff --git a/graylog2-server/src/main/java/org/graylog2/commands/journal/JournalDecode.java b/graylog2-server/src/main/java/org/graylog2/commands/journal/JournalDecode.java index 0c32bf55465f..03f1f5655976 100644 --- a/graylog2-server/src/main/java/org/graylog2/commands/journal/JournalDecode.java +++ b/graylog2-server/src/main/java/org/graylog2/commands/journal/JournalDecode.java @@ -25,14 +25,15 @@ import com.google.inject.Key; import com.google.inject.Module; import com.google.inject.TypeLiteral; +import jakarta.annotation.Nonnull; import org.graylog2.featureflag.FeatureFlags; import org.graylog2.inputs.codecs.CodecsModule; import org.graylog2.plugin.Message; +import org.graylog2.plugin.MessageBindings; import org.graylog2.plugin.ResolvableInetSocketAddress; import org.graylog2.plugin.inject.Graylog2Module; import org.graylog2.plugin.inputs.codecs.Codec; import org.graylog2.plugin.journal.RawMessage; -import org.graylog2.shared.bindings.ObjectMapperModule; import org.graylog2.shared.journal.Journal; import org.slf4j.helpers.MessageFormatter; @@ -51,11 +52,11 @@ public JournalDecode() { } @Override - protected List getCommandBindings(FeatureFlags featureFlags) { + protected @Nonnull List getNodeCommandBindings(FeatureFlags featureFlags) { return ImmutableList.builder() - .addAll(super.getCommandBindings(featureFlags)) + .addAll(super.getNodeCommandBindings(featureFlags)) .add(new CodecsModule()) - .add(new ObjectMapperModule(getClass().getClassLoader())) + .add(new MessageBindings()) .add(new Graylog2Module() { @Override protected void configure() { diff --git a/graylog2-server/src/main/java/org/graylog2/configuration/EventBusConfiguration.java b/graylog2-server/src/main/java/org/graylog2/configuration/EventBusConfiguration.java new file mode 100644 index 000000000000..f1e0d23af714 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/configuration/EventBusConfiguration.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog2.configuration; + +import com.github.joschi.jadconfig.Parameter; + +public class EventBusConfiguration { + + @Parameter(value = "async_eventbus_processors") + private final int asyncEventbusProcessors = 2; + + public int getAsyncEventbusProcessors() { + return asyncEventbusProcessors; + } + +} diff --git a/graylog2-server/src/main/java/org/graylog2/featureflag/FeatureFlagsFactory.java b/graylog2-server/src/main/java/org/graylog2/featureflag/FeatureFlagsFactory.java index 7a587616cc04..a8efd0f3255a 100644 --- a/graylog2-server/src/main/java/org/graylog2/featureflag/FeatureFlagsFactory.java +++ b/graylog2-server/src/main/java/org/graylog2/featureflag/FeatureFlagsFactory.java @@ -17,24 +17,27 @@ package org.graylog2.featureflag; import com.codahale.metrics.MetricRegistry; +import org.graylog2.GraylogNodeConfiguration; public class FeatureFlagsFactory { private static final String DEFAULT_PROPERTIES_FILE = "/org/graylog2/featureflag/feature-flag.config"; - public FeatureFlags createImmutableFeatureFlags(String customPropertiesFile, MetricRegistry metricRegistry) { + public FeatureFlags createImmutableFeatureFlags(String customPropertiesFile, MetricRegistry metricRegistry, GraylogNodeConfiguration configuration) { return createImmutableFeatureFlags( new FeatureFlagsResources(), DEFAULT_PROPERTIES_FILE, customPropertiesFile, - metricRegistry); + metricRegistry, + configuration); } public FeatureFlags createImmutableFeatureFlags(FeatureFlagsResources resources, String defaultPropertiesFile, String customPropertiesFile, - MetricRegistry metricRegistry) { + MetricRegistry metricRegistry, + GraylogNodeConfiguration configuration) { return new ImmutableFeatureFlags(new ImmutableFeatureFlagsCollector( - resources, defaultPropertiesFile, customPropertiesFile).toMap(), metricRegistry); + resources, defaultPropertiesFile, customPropertiesFile, configuration).toMap(), metricRegistry); } } diff --git a/graylog2-server/src/main/java/org/graylog2/featureflag/ImmutableFeatureFlagsCollector.java b/graylog2-server/src/main/java/org/graylog2/featureflag/ImmutableFeatureFlagsCollector.java index c143cbe03800..a21d4637e6ff 100644 --- a/graylog2-server/src/main/java/org/graylog2/featureflag/ImmutableFeatureFlagsCollector.java +++ b/graylog2-server/src/main/java/org/graylog2/featureflag/ImmutableFeatureFlagsCollector.java @@ -18,6 +18,7 @@ import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; +import org.graylog2.GraylogNodeConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,8 +33,6 @@ import java.util.stream.Collectors; import static java.util.stream.Collectors.toList; -import static org.graylog2.bootstrap.CmdLineTool.GRAYLOG_ENVIRONMENT_VAR_PREFIX; -import static org.graylog2.bootstrap.CmdLineTool.GRAYLOG_SYSTEM_PROP_PREFIX; import static org.graylog2.featureflag.FeatureFlagStringUtil.startsWithIgnoreCase; import static org.graylog2.featureflag.FeatureFlagStringUtil.stringFormat; import static org.graylog2.featureflag.FeatureFlagStringUtil.toUpperCase; @@ -42,18 +41,21 @@ class ImmutableFeatureFlagsCollector { private static final Logger LOG = LoggerFactory.getLogger(ImmutableFeatureFlagsCollector.class); - private static final String GRAYLOG_FF_ENVIRONMENT_VAR_PREFIX = GRAYLOG_ENVIRONMENT_VAR_PREFIX + "FEATURE_"; - private static final String GRAYLOG_FF_SYSTEM_PROP_PREFIX = GRAYLOG_SYSTEM_PROP_PREFIX + "feature."; + private final String featureFlagEnvPrefix; + private final String featureFlagSystemPropPrefix; private Map existingFlags = new HashMap<>(); private final FeatureFlagsResources resources; private final String defaultPropertiesFile; private final String customPropertiesFile; - public ImmutableFeatureFlagsCollector(FeatureFlagsResources resources, String defaultPropertiesFile, String customPropertiesFile) { + public ImmutableFeatureFlagsCollector(FeatureFlagsResources resources, String defaultPropertiesFile, + String customPropertiesFile, GraylogNodeConfiguration configuration) { this.resources = resources; this.defaultPropertiesFile = defaultPropertiesFile; this.customPropertiesFile = customPropertiesFile; + this.featureFlagEnvPrefix = configuration.getEnvironmentVariablePrefix() + "FEATURE_"; + this.featureFlagSystemPropPrefix = configuration.getSystemPropertyPrefix() + "feature."; } public Map toMap() { @@ -100,11 +102,11 @@ private void addCustomPropertiesFlags(String file) { } private void addSystemPropertiesFlags() { - addFlagsWithPrefix(GRAYLOG_FF_SYSTEM_PROP_PREFIX, resources.systemProperties(), "system properties"); + addFlagsWithPrefix(featureFlagSystemPropPrefix, resources.systemProperties(), "system properties"); } private void addEnvironmentVariableFlags() { - addFlagsWithPrefix(GRAYLOG_FF_ENVIRONMENT_VAR_PREFIX, resources.environmentVariables(), "environment variables"); + addFlagsWithPrefix(featureFlagEnvPrefix, resources.environmentVariables(), "environment variables"); } private void addFlagsWithPrefix(String prefix, Map newFlags, String resourceType) { diff --git a/graylog2-server/src/main/java/org/graylog2/plugin/BaseConfiguration.java b/graylog2-server/src/main/java/org/graylog2/plugin/BaseConfiguration.java index 031cf8b6acde..b08021246109 100644 --- a/graylog2-server/src/main/java/org/graylog2/plugin/BaseConfiguration.java +++ b/graylog2-server/src/main/java/org/graylog2/plugin/BaseConfiguration.java @@ -29,6 +29,7 @@ import com.lmax.disruptor.WaitStrategy; import com.lmax.disruptor.YieldingWaitStrategy; import org.apache.commons.lang3.StringUtils; +import org.graylog2.CommonNodeConfiguration; import org.graylog2.configuration.PathConfiguration; import org.graylog2.shared.messageq.MessageQueueModule; import org.graylog2.utilities.ProxyHostsPattern; @@ -42,7 +43,7 @@ import static org.graylog2.shared.messageq.MessageQueueModule.NOOP_JOURNAL_MODE; @SuppressWarnings("FieldMayBeFinal") -public abstract class BaseConfiguration extends PathConfiguration { +public abstract class BaseConfiguration extends PathConfiguration implements CommonNodeConfiguration { private static final Logger LOG = LoggerFactory.getLogger(BaseConfiguration.class); @Parameter(value = "shutdown_timeout", validator = PositiveIntegerValidator.class) @@ -169,6 +170,7 @@ public int getUdpRecvBufferSizes() { return udpRecvBufferSizes; } + @Override public boolean isMessageRecordingsEnabled() { return messageRecordingsEnable; } diff --git a/graylog2-server/src/main/java/org/graylog2/plugin/PluginModule.java b/graylog2-server/src/main/java/org/graylog2/plugin/PluginModule.java index 9cf18068b221..4b8b8f3d2939 100644 --- a/graylog2-server/src/main/java/org/graylog2/plugin/PluginModule.java +++ b/graylog2-server/src/main/java/org/graylog2/plugin/PluginModule.java @@ -85,6 +85,11 @@ public Set getConfigBeans() { return Collections.emptySet(); } + @Override + protected Set getConfigurationBeans() { + return Collections.singleton(getConfigBeans()); + } + protected void addMessageInput(Class messageInputClass) { installInput(inputsMapBinder(), messageInputClass); } diff --git a/graylog2-server/src/main/java/org/graylog2/plugin/ServerStatus.java b/graylog2-server/src/main/java/org/graylog2/plugin/ServerStatus.java index ba17650135a3..5c54888cce86 100644 --- a/graylog2-server/src/main/java/org/graylog2/plugin/ServerStatus.java +++ b/graylog2-server/src/main/java/org/graylog2/plugin/ServerStatus.java @@ -19,6 +19,10 @@ import com.google.common.collect.Sets; import com.google.common.eventbus.EventBus; import com.google.common.util.concurrent.Uninterruptibles; +import jakarta.inject.Inject; +import jakarta.inject.Provider; +import jakarta.inject.Singleton; +import org.graylog2.GraylogNodeConfiguration; import org.graylog2.audit.AuditActor; import org.graylog2.audit.AuditEventSender; import org.graylog2.cluster.leader.LeaderElectionService; @@ -30,10 +34,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import jakarta.inject.Inject; -import jakarta.inject.Provider; -import jakarta.inject.Singleton; - import java.util.Arrays; import java.util.Set; import java.util.concurrent.CountDownLatch; @@ -76,7 +76,7 @@ public enum Capability { private volatile Lifecycle lifecycle = Lifecycle.UNINITIALIZED; @Inject - public ServerStatus(BaseConfiguration configuration, Set capabilities, EventBus eventBus, Provider auditEventSenderProvider, final NodeId nodeId) { + public ServerStatus(GraylogNodeConfiguration configuration, Set capabilities, EventBus eventBus, Provider auditEventSenderProvider, final NodeId nodeId) { this.eventBus = eventBus; this.nodeId = nodeId; this.auditEventSenderProvider = auditEventSenderProvider; diff --git a/graylog2-server/src/main/java/org/graylog2/plugin/inject/Graylog2Module.java b/graylog2-server/src/main/java/org/graylog2/plugin/inject/Graylog2Module.java index d08f86dc10ce..73858eac9333 100644 --- a/graylog2-server/src/main/java/org/graylog2/plugin/inject/Graylog2Module.java +++ b/graylog2-server/src/main/java/org/graylog2/plugin/inject/Graylog2Module.java @@ -67,6 +67,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.lang.annotation.Annotation; +import java.util.Set; public abstract class Graylog2Module extends AbstractModule { private static final Logger LOG = LoggerFactory.getLogger(Graylog2Module.class); @@ -540,4 +541,9 @@ protected Multibinder staticReferencedSearchBinder() { protected Multibinder streamDeletionGuardBinder() { return Multibinder.newSetBinder(binder(), StreamDeletionGuard.class); } + + protected Set getConfigurationBeans() { + return Set.of(); + } + } diff --git a/graylog2-server/src/main/java/org/graylog2/shared/bindings/GenericBindings.java b/graylog2-server/src/main/java/org/graylog2/shared/bindings/GenericBindings.java index 914f47cc9780..299d7c61d4c2 100644 --- a/graylog2-server/src/main/java/org/graylog2/shared/bindings/GenericBindings.java +++ b/graylog2-server/src/main/java/org/graylog2/shared/bindings/GenericBindings.java @@ -16,7 +16,6 @@ */ package org.graylog2.shared.bindings; -import com.google.common.eventbus.EventBus; import com.google.common.util.concurrent.ServiceManager; import com.google.inject.Scopes; import com.google.inject.TypeLiteral; @@ -35,14 +34,10 @@ import org.graylog2.indexer.IndexTemplateProvider; import org.graylog2.indexer.MessageIndexTemplateProvider; import org.graylog2.plugin.IOState; -import org.graylog2.plugin.LocalMetricRegistry; import org.graylog2.plugin.buffers.InputBuffer; import org.graylog2.plugin.inject.Graylog2Module; import org.graylog2.plugin.inputs.MessageInput; import org.graylog2.plugin.inputs.util.ThroughputCounter; -import org.graylog2.plugin.system.FilePersistedNodeIdProvider; -import org.graylog2.plugin.system.NodeId; -import org.graylog2.shared.bindings.providers.EventBusProvider; import org.graylog2.shared.bindings.providers.OkHttpClientProvider; import org.graylog2.shared.bindings.providers.ProxiedRequestsExecutorService; import org.graylog2.shared.bindings.providers.ServiceManagerProvider; @@ -67,7 +62,6 @@ public GenericBindings(boolean isMigrationCommand) { @Override protected void configure() { - bind(LocalMetricRegistry.class).in(Scopes.NO_SCOPE); // must not be a singleton! install(new FactoryModuleBuilder().build(DecodingProcessor.Factory.class)); @@ -77,8 +71,6 @@ protected void configure() { } else { bind(InputBuffer.class).to(InputBufferImpl.class); } - bind(NodeId.class).toProvider(FilePersistedNodeIdProvider.class).asEagerSingleton(); - ; if (!isMigrationCommand) { bind(ServiceManager.class).toProvider(ServiceManagerProvider.class).asEagerSingleton(); @@ -87,8 +79,6 @@ protected void configure() { bind(ThroughputCounter.class); - bind(EventBus.class).toProvider(EventBusProvider.class).in(Scopes.SINGLETON); - bind(Semaphore.class).annotatedWith(Names.named("JournalSignal")).toInstance(new Semaphore(0)); install(new FactoryModuleBuilder().build(new TypeLiteral>() {})); diff --git a/graylog2-server/src/main/java/org/graylog2/shared/security/SecurityBindings.java b/graylog2-server/src/main/java/org/graylog2/shared/security/SecurityBindings.java index eda644059cb7..27f9e5e02300 100644 --- a/graylog2-server/src/main/java/org/graylog2/shared/security/SecurityBindings.java +++ b/graylog2-server/src/main/java/org/graylog2/shared/security/SecurityBindings.java @@ -26,7 +26,6 @@ import org.graylog2.security.DefaultX509TrustManager; import org.graylog2.security.TrustManagerProvider; import org.graylog2.security.UserSessionTerminationService; -import org.graylog2.security.encryption.EncryptedValueService; import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; @@ -34,7 +33,6 @@ public class SecurityBindings extends PluginModule { @Override protected void configure() { - bind(EncryptedValueService.class).asEagerSingleton(); bind(Permissions.class).asEagerSingleton(); bind(SessionCreator.class).in(Scopes.SINGLETON); addPermissions(RestPermissions.class); diff --git a/graylog2-server/src/test/java/org/graylog2/commands/CommonNodeCommandTest.java b/graylog2-server/src/test/java/org/graylog2/commands/CommonNodeCommandTest.java new file mode 100644 index 000000000000..05542967f3cc --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog2/commands/CommonNodeCommandTest.java @@ -0,0 +1,98 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog2.commands; + +import com.github.joschi.jadconfig.Parameter; +import com.google.inject.Module; +import jakarta.annotation.Nonnull; +import org.graylog.testing.mongodb.MongoDBExtension; +import org.graylog.testing.mongodb.MongoDBTestService; +import org.graylog2.CommonNodeConfiguration; +import org.graylog2.featureflag.FeatureFlags; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.net.URISyntaxException; +import java.net.URL; +import java.util.List; + +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +@ExtendWith(MockitoExtension.class) +@ExtendWith(MongoDBExtension.class) +public class CommonNodeCommandTest { + + static MongoDBTestService mongodb; + + @BeforeEach + void setUp(MongoDBTestService mongodb) { + System.setProperty("graylog.mongodb_uri", mongodb.uri()); + CommonNodeCommandTest.mongodb = mongodb; + } + + @Test + public void startCommonNode() { + CommonNode node = spy(new CommonNode()); + node.run(); + verify(node, times(1)).startCommand(); + } + + static class CommonNode extends AbstractNodeCommand { + + public CommonNode() { + super(new CommonNodeConfiguration() { + @Parameter("password_secret") + String passwordSecret; + @Parameter("node_id_file") + String nodeIdFile; + }); + URL resource = this.getClass().getResource("common-node.conf"); + if (resource == null) { + Assertions.fail("Cannot read configuration file"); + } + try { + setConfigFile(resource.toURI().getPath()); + } catch (URISyntaxException e) { + Assertions.fail("Cannot read configuration file"); + } + } + + @Override + protected @Nonnull List getNodeCommandBindings(FeatureFlags featureFlags) { + return List.of(); + } + + @Override + protected @Nonnull List getNodeCommandConfigurationBeans() { + return List.of(); + } + + @Override + protected void startCommand() { + + } + + + } + + +} diff --git a/graylog2-server/src/test/java/org/graylog2/commands/MinimalNodeCommandTest.java b/graylog2-server/src/test/java/org/graylog2/commands/MinimalNodeCommandTest.java new file mode 100644 index 000000000000..07454907a6ee --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog2/commands/MinimalNodeCommandTest.java @@ -0,0 +1,130 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog2.commands; + +import com.github.joschi.jadconfig.Parameter; +import com.google.inject.Module; +import jakarta.annotation.Nonnull; +import org.graylog2.GraylogNodeConfiguration; +import org.graylog2.featureflag.FeatureFlags; +import org.graylog2.plugin.ServerStatus; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.net.URISyntaxException; +import java.net.URL; +import java.util.List; +import java.util.Set; + +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +@ExtendWith(MockitoExtension.class) +public class MinimalNodeCommandTest { + + @Test + public void startMinimalNode() { + MinimalNode node = spy(new MinimalNode()); + node.run(); + verify(node, times(1)).startCommand(); + } + + static class MinimalNode extends AbstractNodeCommand { + + public MinimalNode() { + super(new MinimalNodeConfiguration()); + URL resource = this.getClass().getResource("minimal-node.conf"); + if (resource == null) { + Assertions.fail("Cannot read configuration file"); + } + try { + setConfigFile(resource.toURI().getPath()); + } catch (URISyntaxException e) { + Assertions.fail("Cannot read configuration file"); + } + } + + @Override + protected @Nonnull List getNodeCommandBindings(FeatureFlags featureFlags) { + return List.of(); + } + + @Override + protected @Nonnull List getNodeCommandConfigurationBeans() { + return List.of(); + } + + @Override + protected void startCommand() { + } + + static class MinimalNodeConfiguration implements GraylogNodeConfiguration { + @Parameter("password_secret") + String passwordSecret; + + @Override + public boolean withPlugins() { + return false; + } + + @Override + public boolean withEventBus() { + return false; + } + + @Override + public boolean withScheduler() { + return false; + } + + @Override + public boolean withMongoDb() { + return false; + } + + @Override + public boolean withNodeIdFile() { + return false; + } + + @Override + public Set withCapabilities() { + return Set.of(); + } + + @Override + public String getEnvironmentVariablePrefix() { + return "MINIMAL_"; + } + + @Override + public String getSystemPropertyPrefix() { + return "minimal."; + } + + @Override + public boolean isMessageRecordingsEnabled() { + return false; + } + } + } + + +} diff --git a/graylog2-server/src/test/java/org/graylog2/featureflag/ImmutableFeatureFlagsMetricsTest.java b/graylog2-server/src/test/java/org/graylog2/featureflag/ImmutableFeatureFlagsMetricsTest.java index 669f522d0188..96d54729bbcd 100644 --- a/graylog2-server/src/test/java/org/graylog2/featureflag/ImmutableFeatureFlagsMetricsTest.java +++ b/graylog2-server/src/test/java/org/graylog2/featureflag/ImmutableFeatureFlagsMetricsTest.java @@ -17,6 +17,8 @@ package org.graylog2.featureflag; import com.codahale.metrics.MetricRegistry; +import org.graylog2.CommonNodeConfiguration; +import org.graylog2.GraylogNodeConfiguration; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -41,6 +43,8 @@ public class ImmutableFeatureFlagsMetricsTest { @Mock FeatureFlagsResources resources; MetricRegistry metricRegistry; + GraylogNodeConfiguration nodeConfiguration = new CommonNodeConfiguration() { + }; @BeforeEach void setUp() { @@ -98,7 +102,7 @@ private FeatureFlags createFeatureFlags() throws IOException { private FeatureFlags createFeatureFlags(Map flags) throws IOException { given(resources.defaultProperties(any())).willReturn(flags); - return new FeatureFlagsFactory().createImmutableFeatureFlags(resources, "file", "file", metricRegistry); + return new FeatureFlagsFactory().createImmutableFeatureFlags(resources, "file", "file", metricRegistry, nodeConfiguration); } private long getFeatureFlagUsedCount() { diff --git a/graylog2-server/src/test/java/org/graylog2/featureflag/ImmutableFeatureFlagsTest.java b/graylog2-server/src/test/java/org/graylog2/featureflag/ImmutableFeatureFlagsTest.java index 26575a8ffd2a..5c81cd9b0292 100644 --- a/graylog2-server/src/test/java/org/graylog2/featureflag/ImmutableFeatureFlagsTest.java +++ b/graylog2-server/src/test/java/org/graylog2/featureflag/ImmutableFeatureFlagsTest.java @@ -17,6 +17,8 @@ package org.graylog2.featureflag; import com.codahale.metrics.MetricRegistry; +import org.graylog2.CommonNodeConfiguration; +import org.graylog2.GraylogNodeConfiguration; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; @@ -58,6 +60,8 @@ class ImmutableFeatureFlagsTest { MetricRegistry metricRegistry = new MetricRegistry(); + GraylogNodeConfiguration configuration = new CommonNodeConfiguration() {}; + @Test void testOverrideOrder() throws IOException { Map defaultProps = Map.of( @@ -125,7 +129,7 @@ void testFeatureFlagResourcesCouldBeRead() throws Exception { String file = Objects.requireNonNull(this.getClass() .getResource("/org/graylog2/featureflag/custom-feature-flag.config")).toURI().getPath(); - FeatureFlags flags = factory.createImmutableFeatureFlags(file, metricRegistry); + FeatureFlags flags = factory.createImmutableFeatureFlags(file, metricRegistry, configuration); assertThat(flags.getAll().keySet()).contains("feature1"); } @@ -188,7 +192,7 @@ private FeatureFlags create(Map defaultProperties, private FeatureFlags mockAndCreate(Action action) throws IOException { action.execute(); - return factory.createImmutableFeatureFlags(featureFlagsResources, FILE, FILE, metricRegistry); + return factory.createImmutableFeatureFlags(featureFlagsResources, FILE, FILE, metricRegistry, configuration); } interface Action { diff --git a/graylog2-server/src/test/java/org/graylog2/plugin/ServerStatusTest.java b/graylog2-server/src/test/java/org/graylog2/plugin/ServerStatusTest.java index 0cb4883ff9c6..510bc37ed15d 100644 --- a/graylog2-server/src/test/java/org/graylog2/plugin/ServerStatusTest.java +++ b/graylog2-server/src/test/java/org/graylog2/plugin/ServerStatusTest.java @@ -17,6 +17,7 @@ package org.graylog2.plugin; import com.google.common.eventbus.EventBus; +import org.graylog2.GraylogNodeConfiguration; import org.graylog2.audit.NullAuditEventSender; import org.graylog2.plugin.lifecycles.Lifecycle; import org.graylog2.plugin.system.FilePersistedNodeIdProvider; @@ -45,7 +46,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; public class ServerStatusTest { @Rule @@ -53,7 +53,8 @@ public class ServerStatusTest { @Rule public final MockitoRule mockitoRule = MockitoJUnit.rule(); - @Mock private BaseConfiguration config; + @Mock + private GraylogNodeConfiguration config; @Mock private EventBus eventBus; private ServerStatus status; diff --git a/graylog2-server/src/test/resources/org/graylog2/commands/common-node.conf b/graylog2-server/src/test/resources/org/graylog2/commands/common-node.conf new file mode 100644 index 000000000000..5c3f6509e90c --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog2/commands/common-node.conf @@ -0,0 +1,2 @@ +node_id_file=./node_id_file +password_secret=1234567890123456 diff --git a/graylog2-server/src/test/resources/org/graylog2/commands/minimal-node.conf b/graylog2-server/src/test/resources/org/graylog2/commands/minimal-node.conf new file mode 100644 index 000000000000..ed945d8f52ef --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog2/commands/minimal-node.conf @@ -0,0 +1 @@ +password_secret=1234567890123456 From feaaa76566a1bbcddfa3db8ada8d64d405b3805c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 09:31:02 +0100 Subject: [PATCH 7/7] Bump io.grpc:grpc-bom from 1.68.2 to 1.69.0 (#21160) Bumps [io.grpc:grpc-bom](https://github.com/grpc/grpc-java) from 1.68.2 to 1.69.0. - [Release notes](https://github.com/grpc/grpc-java/releases) - [Commits](https://github.com/grpc/grpc-java/compare/v1.68.2...v1.69.0) --- updated-dependencies: - dependency-name: io.grpc:grpc-bom dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 32963a0618eb..2bc915807a49 100644 --- a/pom.xml +++ b/pom.xml @@ -121,7 +121,7 @@ 1.5.1 4.2.1 0.1.9-graylog-3 - 1.68.2 + 1.69.0 2.0.0 33.3.1-jre 7.0.0