From b51cc856ef87f10597cdb56b7bda2f37eea222d3 Mon Sep 17 00:00:00 2001 From: Andrey Pleskach Date: Thu, 15 Feb 2024 10:41:39 +0100 Subject: [PATCH] Fix cluster default initialization Signed-off-by: Andrey Pleskach --- .../AbstractDefaultConfigurationTests.java | 86 +++++++ ...ultConfigurationMultiNodeClusterTests.java | 39 ++++ ...nMultiNodeClusterUseClusterStateTests.java | 42 ++++ ...ltConfigurationSingleNodeClusterTests.java | 44 ++++ ...SingleNodeClusterUseClusterStateTests.java | 42 ++++ .../security/DefaultConfigurationTests.java | 78 ------- .../SecurityConfigurationBootstrapTests.java | 3 +- .../security/OpenSearchSecurityPlugin.java | 35 ++- .../ConfigurationRepository.java | 110 ++++++--- .../security/securityconf/impl/CType.java | 59 +++-- .../impl/SecurityDynamicConfiguration.java | 5 + .../state/SecurityClusterStateListeners.java | 128 ++++++++++ .../security/state/SecurityMetadata.java | 98 ++++++++ .../security/support/ConfigConstants.java | 5 + .../security/support/ConfigHelper.java | 1 + .../security/support/ConfigReader.java | 108 +++++++++ .../support/SecurityIndexHandler.java | 221 ++++++++++++++++++ .../security/support/ConfigReaderTest.java | 79 +++++++ 18 files changed, 1054 insertions(+), 129 deletions(-) create mode 100644 src/integrationTest/java/org/opensearch/security/AbstractDefaultConfigurationTests.java create mode 100644 src/integrationTest/java/org/opensearch/security/DefaultConfigurationMultiNodeClusterTests.java create mode 100644 src/integrationTest/java/org/opensearch/security/DefaultConfigurationMultiNodeClusterUseClusterStateTests.java create mode 100644 src/integrationTest/java/org/opensearch/security/DefaultConfigurationSingleNodeClusterTests.java create mode 100644 src/integrationTest/java/org/opensearch/security/DefaultConfigurationSingleNodeClusterUseClusterStateTests.java delete mode 100644 src/integrationTest/java/org/opensearch/security/DefaultConfigurationTests.java create mode 100644 src/main/java/org/opensearch/security/state/SecurityClusterStateListeners.java create mode 100644 src/main/java/org/opensearch/security/state/SecurityMetadata.java create mode 100644 src/main/java/org/opensearch/security/support/ConfigReader.java create mode 100644 src/main/java/org/opensearch/security/support/SecurityIndexHandler.java create mode 100644 src/test/java/org/opensearch/security/support/ConfigReaderTest.java diff --git a/src/integrationTest/java/org/opensearch/security/AbstractDefaultConfigurationTests.java b/src/integrationTest/java/org/opensearch/security/AbstractDefaultConfigurationTests.java new file mode 100644 index 0000000000..a0b0d8785c --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/AbstractDefaultConfigurationTests.java @@ -0,0 +1,86 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ +package org.opensearch.security; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.apache.commons.io.FileUtils; +import org.awaitility.Awaitility; +import org.junit.AfterClass; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.security.state.SecurityMetadata; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasKey; +import static org.junit.Assert.assertTrue; + +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public abstract class AbstractDefaultConfigurationTests { + public final static Path configurationFolder = ConfigurationFiles.createConfigurationDirectory(); + public static final String ADMIN_USER_NAME = "admin"; + public static final String DEFAULT_PASSWORD = "secret"; + public static final String NEW_USER = "new-user"; + public static final String LIMITED_USER = "limited-user"; + + private final LocalCluster cluster; + + protected AbstractDefaultConfigurationTests(LocalCluster cluster) { + this.cluster = cluster; + } + + @AfterClass + public static void cleanConfigurationDirectory() throws IOException { + FileUtils.deleteDirectory(configurationFolder.toFile()); + } + + @Test + public void shouldLoadDefaultConfiguration() throws IOException { + try (TestRestClient client = cluster.getRestClient(NEW_USER, DEFAULT_PASSWORD)) { + Awaitility.waitAtMost(10, TimeUnit.SECONDS) + .await("Load default configuration") + .until(() -> client.getAuthInfo().getStatusCode(), equalTo(200)); + } + + try (TestRestClient client = cluster.getRestClient(ADMIN_USER_NAME, DEFAULT_PASSWORD)) { + client.confirmCorrectCredentials(ADMIN_USER_NAME); + TestRestClient.HttpResponse response = client.get("_plugins/_security/api/internalusers"); + response.assertStatusCode(200); + Map users = response.getBodyAs(Map.class); + assertThat(users, allOf(aMapWithSize(3), hasKey(ADMIN_USER_NAME), hasKey(NEW_USER), hasKey(LIMITED_USER))); + assertClusterState(client); + } + } + + void assertClusterState(final TestRestClient client) { + if (cluster.node().settings().getAsBoolean("plugins.security.allow_default_init_securityindex.use_cluster_state", false)) { + final TestRestClient.HttpResponse response = client.get("_cluster/state"); + response.assertStatusCode(200); + final var clusterState = response.getBodyAs(Map.class); + assertTrue(response.getBody(), clusterState.containsKey(SecurityMetadata.TYPE)); + @SuppressWarnings("unchecked") + final var securityClusterState = (Map) clusterState.get(SecurityMetadata.TYPE); + System.err.println(response.getBody()); + assertTrue(response.getBody(), (Boolean) securityClusterState.get(SecurityMetadata.SECURITY_CONFIGURATION_APPLIED_FIELD_NAME)); + } + } + +} diff --git a/src/integrationTest/java/org/opensearch/security/DefaultConfigurationMultiNodeClusterTests.java b/src/integrationTest/java/org/opensearch/security/DefaultConfigurationMultiNodeClusterTests.java new file mode 100644 index 0000000000..704e2c7255 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/DefaultConfigurationMultiNodeClusterTests.java @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ +package org.opensearch.security; + +import java.util.List; +import java.util.Map; + +import org.junit.ClassRule; + +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; + +public class DefaultConfigurationMultiNodeClusterTests extends AbstractDefaultConfigurationTests { + + @ClassRule + public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS) + .nodeSettings( + Map.of( + "plugins.security.allow_default_init_securityindex", + true, + "plugins.security.restapi.roles_enabled", + List.of("user_admin__all_access") + ) + ) + .defaultConfigurationInitDirectory(configurationFolder.toString()) + .loadConfigurationIntoIndex(false) + .build(); + + public DefaultConfigurationMultiNodeClusterTests() { + super(cluster); + } +} diff --git a/src/integrationTest/java/org/opensearch/security/DefaultConfigurationMultiNodeClusterUseClusterStateTests.java b/src/integrationTest/java/org/opensearch/security/DefaultConfigurationMultiNodeClusterUseClusterStateTests.java new file mode 100644 index 0000000000..8abffac9cf --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/DefaultConfigurationMultiNodeClusterUseClusterStateTests.java @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ +package org.opensearch.security; + +import java.util.List; +import java.util.Map; + +import org.junit.ClassRule; + +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; + +public class DefaultConfigurationMultiNodeClusterUseClusterStateTests extends AbstractDefaultConfigurationTests { + + @ClassRule + public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS) + .nodeSettings( + Map.of( + "plugins.security.allow_default_init_securityindex", + true, + "plugins.security.allow_default_init_securityindex.use_cluster_state", + true, + "plugins.security.restapi.roles_enabled", + List.of("user_admin__all_access") + ) + ) + .defaultConfigurationInitDirectory(configurationFolder.toString()) + .loadConfigurationIntoIndex(false) + .build(); + + public DefaultConfigurationMultiNodeClusterUseClusterStateTests() { + super(cluster); + } + +} diff --git a/src/integrationTest/java/org/opensearch/security/DefaultConfigurationSingleNodeClusterTests.java b/src/integrationTest/java/org/opensearch/security/DefaultConfigurationSingleNodeClusterTests.java new file mode 100644 index 0000000000..362245db5e --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/DefaultConfigurationSingleNodeClusterTests.java @@ -0,0 +1,44 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.security; + +import java.util.List; +import java.util.Map; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.junit.ClassRule; +import org.junit.runner.RunWith; + +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; + +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class DefaultConfigurationSingleNodeClusterTests extends AbstractDefaultConfigurationTests { + + @ClassRule + public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.SINGLENODE) + .nodeSettings( + Map.of( + "plugins.security.allow_default_init_securityindex", + true, + "plugins.security.restapi.roles_enabled", + List.of("user_admin__all_access") + ) + ) + .defaultConfigurationInitDirectory(configurationFolder.toString()) + .loadConfigurationIntoIndex(false) + .build(); + + public DefaultConfigurationSingleNodeClusterTests() { + super(cluster); + } + +} diff --git a/src/integrationTest/java/org/opensearch/security/DefaultConfigurationSingleNodeClusterUseClusterStateTests.java b/src/integrationTest/java/org/opensearch/security/DefaultConfigurationSingleNodeClusterUseClusterStateTests.java new file mode 100644 index 0000000000..e05005e912 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/DefaultConfigurationSingleNodeClusterUseClusterStateTests.java @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ +package org.opensearch.security; + +import java.util.List; +import java.util.Map; + +import org.junit.ClassRule; + +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; + +public class DefaultConfigurationSingleNodeClusterUseClusterStateTests extends AbstractDefaultConfigurationTests { + + @ClassRule + public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.SINGLENODE) + .nodeSettings( + Map.of( + "plugins.security.allow_default_init_securityindex", + true, + "plugins.security.allow_default_init_securityindex.use_cluster_state", + true, + "plugins.security.restapi.roles_enabled", + List.of("user_admin__all_access") + ) + ) + .defaultConfigurationInitDirectory(configurationFolder.toString()) + .loadConfigurationIntoIndex(false) + .build(); + + public DefaultConfigurationSingleNodeClusterUseClusterStateTests() { + super(cluster); + } + +} diff --git a/src/integrationTest/java/org/opensearch/security/DefaultConfigurationTests.java b/src/integrationTest/java/org/opensearch/security/DefaultConfigurationTests.java deleted file mode 100644 index 8bb5b96145..0000000000 --- a/src/integrationTest/java/org/opensearch/security/DefaultConfigurationTests.java +++ /dev/null @@ -1,78 +0,0 @@ -/* -* Copyright OpenSearch Contributors -* SPDX-License-Identifier: Apache-2.0 -* -* The OpenSearch Contributors require contributions made to -* this file be licensed under the Apache-2.0 license or a -* compatible open source license. -* -*/ -package org.opensearch.security; - -import java.io.IOException; -import java.nio.file.Path; -import java.util.List; -import java.util.Map; - -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; -import org.apache.commons.io.FileUtils; -import org.awaitility.Awaitility; -import org.junit.AfterClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.runner.RunWith; - -import org.opensearch.test.framework.cluster.ClusterManager; -import org.opensearch.test.framework.cluster.LocalCluster; -import org.opensearch.test.framework.cluster.TestRestClient; -import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.aMapWithSize; -import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasKey; - -@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) -@ThreadLeakScope(ThreadLeakScope.Scope.NONE) -public class DefaultConfigurationTests { - - private final static Path configurationFolder = ConfigurationFiles.createConfigurationDirectory(); - public static final String ADMIN_USER_NAME = "admin"; - public static final String DEFAULT_PASSWORD = "secret"; - public static final String NEW_USER = "new-user"; - public static final String LIMITED_USER = "limited-user"; - - @ClassRule - public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.SINGLENODE) - .nodeSettings( - Map.of( - "plugins.security.allow_default_init_securityindex", - true, - "plugins.security.restapi.roles_enabled", - List.of("user_admin__all_access") - ) - ) - .defaultConfigurationInitDirectory(configurationFolder.toString()) - .loadConfigurationIntoIndex(false) - .build(); - - @AfterClass - public static void cleanConfigurationDirectory() throws IOException { - FileUtils.deleteDirectory(configurationFolder.toFile()); - } - - @Test - public void shouldLoadDefaultConfiguration() { - try (TestRestClient client = cluster.getRestClient(NEW_USER, DEFAULT_PASSWORD)) { - Awaitility.await().alias("Load default configuration").until(() -> client.getAuthInfo().getStatusCode(), equalTo(200)); - } - try (TestRestClient client = cluster.getRestClient(ADMIN_USER_NAME, DEFAULT_PASSWORD)) { - client.confirmCorrectCredentials(ADMIN_USER_NAME); - HttpResponse response = client.get("_plugins/_security/api/internalusers"); - response.assertStatusCode(200); - Map users = response.getBodyAs(Map.class); - assertThat(users, allOf(aMapWithSize(3), hasKey(ADMIN_USER_NAME), hasKey(NEW_USER), hasKey(LIMITED_USER))); - } - } -} diff --git a/src/integrationTest/java/org/opensearch/security/SecurityConfigurationBootstrapTests.java b/src/integrationTest/java/org/opensearch/security/SecurityConfigurationBootstrapTests.java index 5b83e0d6d0..e6af5d58bb 100644 --- a/src/integrationTest/java/org/opensearch/security/SecurityConfigurationBootstrapTests.java +++ b/src/integrationTest/java/org/opensearch/security/SecurityConfigurationBootstrapTests.java @@ -124,6 +124,7 @@ public void shouldStillLoadSecurityConfigDuringBootstrapAndActiveConfigUpdateReq .put("action_groups.yml", CType.ACTIONGROUPS) .put("config.yml", CType.CONFIG) .put("roles.yml", CType.ROLES) + .put("roles_mapping.yml", CType.ROLESMAPPING) .put("tenants.yml", CType.TENANTS) .build(); @@ -146,7 +147,7 @@ public void shouldStillLoadSecurityConfigDuringBootstrapAndActiveConfigUpdateReq // After the configuration has been loaded, the rest clients should be able to connect successfully cluster.triggerConfigurationReloadForCTypes( internalNodeClient, - List.of(CType.ACTIONGROUPS, CType.CONFIG, CType.ROLES, CType.TENANTS), + List.of(CType.ACTIONGROUPS, CType.CONFIG, CType.ROLES, CType.ROLESMAPPING, CType.TENANTS), true ); try (final TestRestClient freshClient = cluster.getRestClient(USER_ADMIN)) { diff --git a/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java b/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java index 53493e7f6c..0db7743be0 100644 --- a/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java +++ b/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java @@ -72,6 +72,8 @@ import org.opensearch.action.search.SearchScrollAction; import org.opensearch.action.support.ActionFilter; import org.opensearch.client.Client; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.NamedDiff; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; @@ -173,6 +175,8 @@ import org.opensearch.security.ssl.transport.DefaultPrincipalExtractor; import org.opensearch.security.ssl.transport.SecuritySSLNettyTransport; import org.opensearch.security.ssl.util.SSLConfigConstants; +import org.opensearch.security.state.SecurityClusterStateListeners; +import org.opensearch.security.state.SecurityMetadata; import org.opensearch.security.support.ConfigConstants; import org.opensearch.security.support.GuardedSearchOperationWrapper; import org.opensearch.security.support.HeaderHelper; @@ -204,6 +208,8 @@ import static org.opensearch.security.dlic.rest.api.RestApiAdminPrivilegesEvaluator.ENDPOINTS_WITH_PERMISSIONS; import static org.opensearch.security.dlic.rest.api.RestApiAdminPrivilegesEvaluator.SECURITY_CONFIG_UPDATE; import static org.opensearch.security.setting.DeprecatedSettings.checkForDeprecatedSetting; +import static org.opensearch.security.support.ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX; +import static org.opensearch.security.support.ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_USE_CLUSTER_STATE; import static org.opensearch.security.support.ConfigConstants.SECURITY_UNSUPPORTED_RESTAPI_ALLOW_SECURITYCONFIG_MODIFICATION; // CS-ENFORCE-SINGLE @@ -284,6 +290,10 @@ private static boolean isDisabled(final Settings settings) { return settings.getAsBoolean(ConfigConstants.SECURITY_DISABLED, false); } + private static boolean useClusterStateToInitSecurityConfig(final Settings settings) { + return settings.getAsBoolean(SECURITY_ALLOW_DEFAULT_INIT_USE_CLUSTER_STATE, false); + } + /** * SSL Cert Reload will be enabled only if security is not disabled and not in we are not using sslOnly mode. * @param settings Elastic configuration settings @@ -1133,11 +1143,24 @@ public Collection createComponents( components.add(si); components.add(dcf); components.add(userService); - + final var allowDefaultInit = settings.getAsBoolean(SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, false); + final var useClusterSettings = useClusterStateToInitSecurityConfig(settings); + if (!SSLConfig.isSslOnlyMode() && !isDisabled(settings) && allowDefaultInit && useClusterSettings) { + clusterService.addListener(SecurityClusterStateListeners.onClusterManagerListener(clusterService, threadPool, cr)); + clusterService.addListener(SecurityClusterStateListeners.onNonClusterManagerListener(threadPool, cr)); + } return components; } + @Override + public List getNamedWriteables() { + return List.of( + new NamedWriteableRegistry.Entry(ClusterState.Custom.class, SecurityMetadata.TYPE, SecurityMetadata::new), + new NamedWriteableRegistry.Entry(NamedDiff.class, SecurityMetadata.TYPE, SecurityMetadata::readDiffFrom) + ); + } + @Override public Settings additionalSettings() { @@ -1278,9 +1301,8 @@ public List> getSettings() { settings.add( Setting.boolSetting(ConfigConstants.SECURITY_ALLOW_UNSAFE_DEMOCERTIFICATES, false, Property.NodeScope, Property.Filtered) ); - settings.add( - Setting.boolSetting(ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, false, Property.NodeScope, Property.Filtered) - ); + settings.add(Setting.boolSetting(SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, false, Property.NodeScope, Property.Filtered)); + settings.add(Setting.boolSetting(SECURITY_ALLOW_DEFAULT_INIT_USE_CLUSTER_STATE, false, Property.NodeScope, Property.Filtered)); settings.add( Setting.boolSetting( ConfigConstants.SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST, @@ -1876,11 +1898,10 @@ public List getSettingsFilter() { @Override public void onNodeStarted(DiscoveryNode localNode) { - log.info("Node started"); - if (!SSLConfig.isSslOnlyMode() && !client && !disabled) { + this.localNode.set(localNode); + if (!SSLConfig.isSslOnlyMode() && !client && !disabled && !useClusterStateToInitSecurityConfig(settings)) { cr.initOnNodeStart(); } - this.localNode.set(localNode); final Set securityModules = ReflectionHelper.getModulesLoaded(); log.info("{} OpenSearch Security modules loaded so far: {}", securityModules.size(), securityModules); } diff --git a/src/main/java/org/opensearch/security/configuration/ConfigurationRepository.java b/src/main/java/org/opensearch/security/configuration/ConfigurationRepository.java index dfbeb16cb3..b2bf45b22d 100644 --- a/src/main/java/org/opensearch/security/configuration/ConfigurationRepository.java +++ b/src/main/java/org/opensearch/security/configuration/ConfigurationRepository.java @@ -38,12 +38,14 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Supplier; +import java.util.stream.Collectors; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; @@ -77,9 +79,12 @@ import org.opensearch.security.ssl.util.ExceptionUtils; import org.opensearch.security.support.ConfigConstants; import org.opensearch.security.support.ConfigHelper; +import org.opensearch.security.support.SecurityIndexHandler; import org.opensearch.security.support.SecurityUtils; import org.opensearch.threadpool.ThreadPool; +import static org.opensearch.security.support.ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_USE_CLUSTER_STATE; + public class ConfigurationRepository { private static final Logger LOGGER = LogManager.getLogger(ConfigurationRepository.class); @@ -98,6 +103,12 @@ public class ConfigurationRepository { private final CompletableFuture initalizeConfigTask = new CompletableFuture<>(); private final boolean acceptInvalid; + private boolean auditHotReloadingEnabled = false; + + private final SecurityIndexHandler securityIndexHandler; + + private Map versions; + private ConfigurationRepository( Settings settings, final Path configPath, @@ -119,8 +130,18 @@ private ConfigurationRepository( this.configurationChangedListener = new ArrayList<>(); this.acceptInvalid = settings.getAsBoolean(ConfigConstants.SECURITY_UNSUPPORTED_ACCEPT_INVALID_CONFIG, false); cl = new ConfigurationLoaderSecurity7(client, threadPool, settings, clusterService); - configCache = CacheBuilder.newBuilder().build(); + this.securityIndexHandler = new SecurityIndexHandler(settings, this.securityIndex, client); + } + + public Map versions() { + return versions; + } + + private Path resolveConfigDir() { + return Optional.ofNullable(System.getProperty("security.default_init.dir")) + .map(Path::of) + .orElseGet(() -> new Environment(settings, configPath).configDir().resolve("opensearch-security/")); } private void initalizeClusterConfiguration(final boolean installDefaultConfig) { @@ -239,27 +260,7 @@ private void initalizeClusterConfiguration(final boolean installDefaultConfig) { } } } - - final Set deprecatedAuditKeysInSettings = AuditConfig.getDeprecatedKeys(settings); - if (!deprecatedAuditKeysInSettings.isEmpty()) { - LOGGER.warn( - "Following keys {} are deprecated in opensearch settings. They will be removed in plugin v2.0.0.0", - deprecatedAuditKeysInSettings - ); - } - final boolean isAuditConfigDocPresentInIndex = cl.isAuditConfigDocPresentInIndex(); - if (isAuditConfigDocPresentInIndex) { - if (!deprecatedAuditKeysInSettings.isEmpty()) { - LOGGER.warn("Audit configuration settings found in both index and opensearch settings (deprecated)"); - } - LOGGER.info("Hot-reloading of audit configuration is enabled"); - } else { - LOGGER.info( - "Hot-reloading of audit configuration is disabled. Using configuration with defaults from opensearch settings. Populate the configuration in index using audit.yml or securityadmin to enable it." - ); - auditLog.setConfig(AuditConfig.from(settings)); - } - + setupAuditConfigurationIfAny(cl.isAuditConfigDocPresentInIndex()); LOGGER.info("Node '{}' initialized", clusterService.localNode().getName()); } catch (Exception e) { @@ -267,6 +268,27 @@ private void initalizeClusterConfiguration(final boolean installDefaultConfig) { } } + private void setupAuditConfigurationIfAny(final boolean auditConfigDocPresent) { + final Set deprecatedAuditKeysInSettings = AuditConfig.getDeprecatedKeys(settings); + if (!deprecatedAuditKeysInSettings.isEmpty()) { + LOGGER.warn( + "Following keys {} are deprecated in opensearch settings. They will be removed in plugin v2.0.0.0", + deprecatedAuditKeysInSettings + ); + } + if (auditConfigDocPresent) { + if (!deprecatedAuditKeysInSettings.isEmpty()) { + LOGGER.warn("Audit configuration settings found in both index and opensearch settings (deprecated)"); + } + LOGGER.info("Hot-reloading of audit configuration is enabled"); + } else { + LOGGER.info( + "Hot-reloading of audit configuration is disabled. Using configuration with defaults from opensearch settings. Populate the configuration in index using audit.yml or securityadmin to enable it." + ); + auditLog.setConfig(AuditConfig.from(settings)); + } + } + private boolean createSecurityIndexIfAbsent() { try { final Map indexSettings = ImmutableMap.of("index.number_of_shards", 1, "index.auto_expand_replicas", "0-all"); @@ -299,7 +321,7 @@ private void waitForSecurityIndexToBeAtLeastYellow() { response == null ? "no response" : (response.isTimedOut() ? "timeout" : "other, maybe red cluster") ); try { - Thread.sleep(500); + TimeUnit.MILLISECONDS.sleep(500); } catch (InterruptedException e) { // ignore Thread.currentThread().interrupt(); @@ -312,6 +334,17 @@ private void waitForSecurityIndexToBeAtLeastYellow() { } } + public void initDefaultConfigurationOnManager() { + if (!clusterService.state().metadata().hasIndex(securityIndex)) { + securityIndexHandler.createIndex(); + } + if (!initalizeConfigTask.isDone()) { + securityIndexHandler.uploadSecurityConfiguration(resolveConfigDir()); + reloadConfiguration("initialization of default configuration on manager node", false, true); + } + } + + @Deprecated public CompletableFuture initOnNodeStart() { final boolean installDefaultConfig = settings.getAsBoolean(ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, false); @@ -328,13 +361,15 @@ public CompletableFuture initOnNodeStart() { return startInitialization.get(); } else if (settings.getAsBoolean(ConfigConstants.SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST, true)) { LOGGER.info( - "Will not attempt to create index {} and default configs if they are absent. Use securityadmin to initialize cluster", + "Will not attempt to create index {} and default configs if they are absent." + + " Use securityadmin to initialize cluster", securityIndex ); return startInitialization.get(); } else { LOGGER.info( - "Will not attempt to create index {} and default configs if they are absent. Will not perform background initialization", + "Will not attempt to create index {} and default configs if they are absent. " + + "Will not perform background initialization", securityIndex ); initalizeConfigTask.complete(null); @@ -347,7 +382,11 @@ public CompletableFuture initOnNodeStart() { } public boolean isAuditHotReloadingEnabled() { - return cl.isAuditConfigDocPresentInIndex(); + if (settings.getAsBoolean(SECURITY_ALLOW_DEFAULT_INIT_USE_CLUSTER_STATE, false)) { + return auditHotReloadingEnabled; + } else { + return cl.isAuditConfigDocPresentInIndex(); + } } public static ConfigurationRepository create( @@ -388,6 +427,21 @@ public SecurityDynamicConfiguration getConfiguration(CType configurationType) private final Lock LOCK = new ReentrantLock(); + public void reloadConfiguration(final String reason, final boolean refresh, final boolean verifyConfigVersion) { + if (!initalizeConfigTask.isDone()) { + LOGGER.info("Security configuration reloaded. Reason: {}", reason); + final var cTypeConfigs = securityIndexHandler.loadConfiguration(refresh, verifyConfigVersion); + configCache.putAll(cTypeConfigs); + notifyAboutChanges(cTypeConfigs); + versions = cTypeConfigs.entrySet() + .stream() + .collect(Collectors.toMap(e -> e.getKey().toLCString(), e -> e.getValue().getVersion())); + setupAuditConfigurationIfAny(cTypeConfigs.get(CType.AUDIT).notEmpty()); + auditHotReloadingEnabled = cTypeConfigs.get(CType.AUDIT).notEmpty(); + initalizeConfigTask.complete(null); + } + } + public boolean reloadConfiguration(final Collection configTypes) throws ConfigUpdateAlreadyInProgressException { return reloadConfiguration(configTypes, false); } @@ -398,6 +452,10 @@ private boolean reloadConfiguration(final Collection configTypes, final b LOGGER.warn("Unable to reload configuration, initalization thread has not yet completed."); return false; } + return loadConfigurationWithLock(configTypes); + } + + private boolean loadConfigurationWithLock(Collection configTypes) { try { if (LOCK.tryLock(60, TimeUnit.SECONDS)) { try { diff --git a/src/main/java/org/opensearch/security/securityconf/impl/CType.java b/src/main/java/org/opensearch/security/securityconf/impl/CType.java index 4e5e2de496..23158e5850 100644 --- a/src/main/java/org/opensearch/security/securityconf/impl/CType.java +++ b/src/main/java/org/opensearch/security/securityconf/impl/CType.java @@ -27,14 +27,17 @@ package org.opensearch.security.securityconf.impl; +import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; import java.util.stream.Collectors; +import com.google.common.collect.ImmutableMap; + import org.opensearch.security.auditlog.config.AuditConfig; import org.opensearch.security.securityconf.impl.v6.ActionGroupsV6; import org.opensearch.security.securityconf.impl.v6.ConfigV6; @@ -50,21 +53,39 @@ public enum CType { - INTERNALUSERS(toMap(1, InternalUserV6.class, 2, InternalUserV7.class)), - ACTIONGROUPS(toMap(0, List.class, 1, ActionGroupsV6.class, 2, ActionGroupsV7.class)), - CONFIG(toMap(1, ConfigV6.class, 2, ConfigV7.class)), - ROLES(toMap(1, RoleV6.class, 2, RoleV7.class)), - ROLESMAPPING(toMap(1, RoleMappingsV6.class, 2, RoleMappingsV7.class)), - TENANTS(toMap(2, TenantV7.class)), - NODESDN(toMap(1, NodesDn.class, 2, NodesDn.class)), - WHITELIST(toMap(1, WhitelistingSettings.class, 2, WhitelistingSettings.class)), - ALLOWLIST(toMap(1, AllowlistingSettings.class, 2, AllowlistingSettings.class)), - AUDIT(toMap(1, AuditConfig.class, 2, AuditConfig.class)); + ACTIONGROUPS(toMap(0, List.class, 1, ActionGroupsV6.class, 2, ActionGroupsV7.class), "action_groups.yml", false), + ALLOWLIST(toMap(1, AllowlistingSettings.class, 2, AllowlistingSettings.class), "allowlist.yml", true), + AUDIT(toMap(1, AuditConfig.class, 2, AuditConfig.class), "audit.yml", true), + CONFIG(toMap(1, ConfigV6.class, 2, ConfigV7.class), "config.yml", false), + INTERNALUSERS(toMap(1, InternalUserV6.class, 2, InternalUserV7.class), "internal_users.yml", false), + NODESDN(toMap(1, NodesDn.class, 2, NodesDn.class), "nodes_dn.yml", true), + ROLES(toMap(1, RoleV6.class, 2, RoleV7.class), "roles.yml", false), + ROLESMAPPING(toMap(1, RoleMappingsV6.class, 2, RoleMappingsV7.class), "roles_mapping.yml", false), + TENANTS(toMap(2, TenantV7.class), "tenants.yml", false), + WHITELIST(toMap(1, WhitelistingSettings.class, 2, WhitelistingSettings.class), "whitelist.yml", true); + + public static final List REQUIRED_CONFIG_FILES = Arrays.stream(CType.values()) + .filter(Predicate.not(CType::emptyIfMissing)) + .collect(Collectors.toList()); + + public static final List NOT_REQUIRED_CONFIG_FILES = Arrays.stream(CType.values()) + .filter(CType::emptyIfMissing) + .collect(Collectors.toList()); + + private final Map> implementations; + + private final String configFileName; - private Map> implementations; + private final boolean emptyIfMissing; - private CType(Map> implementations) { + private CType(Map> implementations, final String configFileName, final boolean emptyIfMissing) { this.implementations = implementations; + this.configFileName = configFileName; + this.emptyIfMissing = emptyIfMissing; + } + + public boolean emptyIfMissing() { + return emptyIfMissing; } public Map> getImplementationClass() { @@ -80,18 +101,22 @@ public String toLCString() { } public static Set lcStringValues() { - return Arrays.stream(CType.values()).map(c -> c.toLCString()).collect(Collectors.toSet()); + return Arrays.stream(CType.values()).map(CType::toLCString).collect(Collectors.toSet()); } public static Set fromStringValues(String[] strings) { - return Arrays.stream(strings).map(c -> CType.fromString(c)).collect(Collectors.toSet()); + return Arrays.stream(strings).map(CType::fromString).collect(Collectors.toSet()); + } + + public Path configFile(final Path configDir) { + return configDir.resolve(this.configFileName); } private static Map> toMap(Object... objects) { - final Map> map = new HashMap>(); + final ImmutableMap.Builder> map = ImmutableMap.builder(); for (int i = 0; i < objects.length; i = i + 2) { map.put((Integer) objects[i], (Class) objects[i + 1]); } - return Collections.unmodifiableMap(map); + return map.build(); } } diff --git a/src/main/java/org/opensearch/security/securityconf/impl/SecurityDynamicConfiguration.java b/src/main/java/org/opensearch/security/securityconf/impl/SecurityDynamicConfiguration.java index 90508840e7..83553f2de7 100644 --- a/src/main/java/org/opensearch/security/securityconf/impl/SecurityDynamicConfiguration.java +++ b/src/main/java/org/opensearch/security/securityconf/impl/SecurityDynamicConfiguration.java @@ -68,6 +68,11 @@ public static SecurityDynamicConfiguration empty() { return new SecurityDynamicConfiguration(); } + @JsonIgnore + public boolean notEmpty() { + return !centries.isEmpty(); + } + public static SecurityDynamicConfiguration fromJson(String json, CType ctype, int version, long seqNo, long primaryTerm) throws IOException { return fromJson(json, ctype, version, seqNo, primaryTerm, false); diff --git a/src/main/java/org/opensearch/security/state/SecurityClusterStateListeners.java b/src/main/java/org/opensearch/security/state/SecurityClusterStateListeners.java new file mode 100644 index 0000000000..f630183414 --- /dev/null +++ b/src/main/java/org/opensearch/security/state/SecurityClusterStateListeners.java @@ -0,0 +1,128 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ +package org.opensearch.security.state; + +import java.util.Map; +import java.util.concurrent.Future; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.opensearch.ExceptionsHelper; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.ClusterStateListener; +import org.opensearch.cluster.ClusterStateUpdateTask; +import org.opensearch.cluster.LocalNodeClusterManagerListener; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.Priority; +import org.opensearch.security.configuration.ConfigurationRepository; +import org.opensearch.threadpool.ThreadPool; + +public final class SecurityClusterStateListeners { + + private static final Logger LOGGER = LogManager.getLogger(ConfigurationRepository.class); + + public static class SecurityClusterStateUpdateTask extends ClusterStateUpdateTask { + + private final Map versions; + + public SecurityClusterStateUpdateTask(final Map versions) { + super(Priority.IMMEDIATE); + this.versions = versions; + } + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + final SecurityMetadata securityMetadata = currentState.custom(SecurityMetadata.TYPE, SecurityMetadata.DEFAULT); + if (!securityMetadata.initialSecurityConfigurationApplied()) { + return ClusterState.builder(currentState).putCustom(SecurityMetadata.TYPE, new SecurityMetadata(true, versions)).build(); + } + return currentState; + } + + @Override + public void onFailure(String s, Exception e) { + LOGGER.error(s, e); + } + + } + + public static ClusterStateListener onClusterManagerListener( + final ClusterService clusterService, + final ThreadPool threadPool, + final ConfigurationRepository configurationRepository + ) { + return new LocalNodeClusterManagerListener() { + + private Future initDefaultConfigFuture; + + @Override + public void onClusterManager() { + final var securityMetadata = clusterService.state().custom(SecurityMetadata.TYPE, SecurityMetadata.DEFAULT); + if (!securityMetadata.initialSecurityConfigurationApplied()) { + if (initDefaultConfigFuture == null) { + initDefaultConfigFuture = threadPool.generic().submit(() -> { + configurationRepository.initDefaultConfigurationOnManager(); + clusterService.submitStateUpdateTask( + "init-security-configuration", + new SecurityClusterStateUpdateTask(configurationRepository.versions()) + ); + return null; + }); + } + } + } + + @Override + public void offClusterManager() { + if (initDefaultConfigFuture != null) { + LOGGER.warn( + "Management node has been changed during configuration initialization. " + + "Default configuration will be re-init from scratch on the next selected management node." + ); + initDefaultConfigFuture.cancel(true); + initDefaultConfigFuture = null; + } + } + }; + } + + public static ClusterStateListener onNonClusterManagerListener( + final ThreadPool threadPool, + final ConfigurationRepository configurationRepository + ) { + return event -> { + if (event.localNodeClusterManager()) return; // skip management node + threadPool.generic().execute(() -> { + final var minNodeVersion = event.state().nodes().getMinNodeVersion(); + final var maxNodeVersion = event.state().nodes().getMaxNodeVersion(); + final var securityMetadata = event.state().custom(SecurityMetadata.TYPE, SecurityMetadata.DEFAULT); + try { + if (securityMetadata.initialSecurityConfigurationApplied()) { + configurationRepository.reloadConfiguration("default security configuration applied successfully", true, true); + } else if (minNodeVersion.before(maxNodeVersion)) { + configurationRepository.reloadConfiguration( + String.format( + "existing cluster has different nodes version. Min node version %s. Max node version %s. Loaded exiting configuration", + minNodeVersion, + maxNodeVersion + ), + true, + false + ); + } + } catch (Exception e) { + throw ExceptionsHelper.convertToOpenSearchException(e); + } + }); + }; + } + +} diff --git a/src/main/java/org/opensearch/security/state/SecurityMetadata.java b/src/main/java/org/opensearch/security/state/SecurityMetadata.java new file mode 100644 index 0000000000..434772d4f4 --- /dev/null +++ b/src/main/java/org/opensearch/security/state/SecurityMetadata.java @@ -0,0 +1,98 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ +package org.opensearch.security.state; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +import com.google.common.collect.ImmutableSortedMap; + +import org.opensearch.Version; +import org.opensearch.cluster.AbstractNamedDiffable; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.NamedDiff; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.XContentBuilder; + +public final class SecurityMetadata extends AbstractNamedDiffable implements ClusterState.Custom { + + public final static SecurityMetadata DEFAULT = new SecurityMetadata(false, Map.of()); + + public final static String TYPE = "security"; + + public final static String SECURITY_CONFIGURATION_APPLIED_FIELD_NAME = "initial_security_configuration_applied"; + + public final static String SECURITY_CONFIGURATION_VERSIONS = "versions"; + + private final boolean initialSecurityConfigurationApplied; + + private final Map versions; + + public SecurityMetadata(final boolean initialSecurityConfigurationApplied, final Map versions) { + this.initialSecurityConfigurationApplied = initialSecurityConfigurationApplied; + this.versions = Map.copyOf(versions); + } + + public SecurityMetadata(StreamInput in) throws IOException { + this.initialSecurityConfigurationApplied = in.readBoolean(); + this.versions = in.readMap(StreamInput::readString, StreamInput::readInt); + } + + public boolean initialSecurityConfigurationApplied() { + return initialSecurityConfigurationApplied; + } + + @Override + public Version getMinimalSupportedVersion() { + return Version.CURRENT.minimumCompatibilityVersion(); + } + + @Override + public String getWriteableName() { + return TYPE; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeBoolean(this.initialSecurityConfigurationApplied); + out.writeMap(versions, StreamOutput::writeString, StreamOutput::writeInt); + } + + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + final var builder = xContentBuilder.field(SECURITY_CONFIGURATION_APPLIED_FIELD_NAME, this.initialSecurityConfigurationApplied); + + final var sortedVersions = ImmutableSortedMap.copyOf(versions, String::compareTo); + builder.startObject(SECURITY_CONFIGURATION_VERSIONS); + for (final var e : sortedVersions.entrySet()) + builder.field(e.getKey(), e.getValue()); + builder.endObject(); + return builder; + } + + public static NamedDiff readDiffFrom(StreamInput in) throws IOException { + return readDiffFrom(ClusterState.Custom.class, TYPE, in); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SecurityMetadata that = (SecurityMetadata) o; + return initialSecurityConfigurationApplied == that.initialSecurityConfigurationApplied && Objects.equals(versions, that.versions); + } + + @Override + public int hashCode() { + return Objects.hash(initialSecurityConfigurationApplied, versions); + } +} diff --git a/src/main/java/org/opensearch/security/support/ConfigConstants.java b/src/main/java/org/opensearch/security/support/ConfigConstants.java index 3060e1b2dc..5169d02d20 100644 --- a/src/main/java/org/opensearch/security/support/ConfigConstants.java +++ b/src/main/java/org/opensearch/security/support/ConfigConstants.java @@ -220,9 +220,14 @@ public class ConfigConstants { public static final String SECURITY_NODES_DN = "plugins.security.nodes_dn"; public static final String SECURITY_NODES_DN_DYNAMIC_CONFIG_ENABLED = "plugins.security.nodes_dn_dynamic_config_enabled"; public static final String SECURITY_DISABLED = "plugins.security.disabled"; + public static final String SECURITY_CACHE_TTL_MINUTES = "plugins.security.cache.ttl_minutes"; public static final String SECURITY_ALLOW_UNSAFE_DEMOCERTIFICATES = "plugins.security.allow_unsafe_democertificates"; public static final String SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX = "plugins.security.allow_default_init_securityindex"; + + public static final String SECURITY_ALLOW_DEFAULT_INIT_USE_CLUSTER_STATE = + "plugins.security.allow_default_init_securityindex.use_cluster_state"; + public static final String SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST = "plugins.security.background_init_if_securityindex_not_exist"; diff --git a/src/main/java/org/opensearch/security/support/ConfigHelper.java b/src/main/java/org/opensearch/security/support/ConfigHelper.java index 4f310f6af7..e8526478f2 100644 --- a/src/main/java/org/opensearch/security/support/ConfigHelper.java +++ b/src/main/java/org/opensearch/security/support/ConfigHelper.java @@ -57,6 +57,7 @@ import static org.opensearch.core.xcontent.DeprecationHandler.THROW_UNSUPPORTED_OPERATION; +@Deprecated public class ConfigHelper { private static final Logger LOGGER = LogManager.getLogger(ConfigHelper.class); diff --git a/src/main/java/org/opensearch/security/support/ConfigReader.java b/src/main/java/org/opensearch/security/support/ConfigReader.java new file mode 100644 index 0000000000..07953e3983 --- /dev/null +++ b/src/main/java/org/opensearch/security/support/ConfigReader.java @@ -0,0 +1,108 @@ +/* + * Copyright 2015-2018 _floragunn_ GmbH + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ +package org.opensearch.security.support; + +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.io.StringReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.security.DefaultObjectMapper; +import org.opensearch.security.securityconf.impl.CType; +import org.opensearch.security.securityconf.impl.Meta; +import org.opensearch.security.securityconf.impl.SecurityDynamicConfiguration; + +import static org.opensearch.core.xcontent.DeprecationHandler.THROW_UNSUPPORTED_OPERATION; +import static org.opensearch.security.configuration.ConfigurationRepository.DEFAULT_CONFIG_VERSION; + +public final class ConfigReader { + + private static final Logger LOGGER = LogManager.getLogger(ConfigReader.class); + + public static BytesReference yamlContentFor(final CType cType, final Path configDir) throws IOException { + final var yamlXContent = XContentType.YAML.xContent(); + try ( + final var r = newReader(cType, configDir); + final var parser = yamlXContent.createParser(NamedXContentRegistry.EMPTY, THROW_UNSUPPORTED_OPERATION, r) + ) { + parser.nextToken(); + try (final var xContentBuilder = XContentFactory.jsonBuilder()) { + xContentBuilder.copyCurrentStructure(parser); + final var bytesRef = BytesReference.bytes(xContentBuilder); + validateYamlContent(cType, bytesRef.streamInput()); + return bytesRef; + } + } + } + + public static Reader newReader(final CType cType, final Path configDir) throws IOException { + final var cTypeFile = cType.configFile(configDir); + final var fileExists = Files.exists(cTypeFile); + if (!fileExists && !cType.emptyIfMissing()) { + throw new IOException("Couldn't find configuration file " + cTypeFile.getFileName()); + } + if (fileExists) { + LOGGER.info("Reading {} configuration from {}", cType, cTypeFile.getFileName()); + return new FileReader(cTypeFile.toFile(), StandardCharsets.UTF_8); + } else { + LOGGER.info("Reading empty {} configuration", cType); + return new StringReader(emptyYamlConfigFor(cType)); + } + } + + private static SecurityDynamicConfiguration emptyConfigFor(final CType cType) { + final var emptyConfiguration = SecurityDynamicConfiguration.empty(); + emptyConfiguration.setCType(cType); + emptyConfiguration.set_meta(new Meta()); + emptyConfiguration.get_meta().setConfig_version(DEFAULT_CONFIG_VERSION); + emptyConfiguration.get_meta().setType(cType.toLCString()); + return emptyConfiguration; + } + + public static String emptyJsonConfigFor(final CType cType) throws IOException { + return DefaultObjectMapper.writeValueAsString(emptyConfigFor(cType), false); + } + + public static String emptyYamlConfigFor(final CType cType) throws IOException { + return DefaultObjectMapper.YAML_MAPPER.writeValueAsString(emptyConfigFor(cType)); + } + + private static void validateYamlContent(final CType cType, final InputStream in) throws IOException { + SecurityDynamicConfiguration.fromNode(DefaultObjectMapper.YAML_MAPPER.readTree(in), cType, DEFAULT_CONFIG_VERSION, -1, -1); + } + +} diff --git a/src/main/java/org/opensearch/security/support/SecurityIndexHandler.java b/src/main/java/org/opensearch/security/support/SecurityIndexHandler.java new file mode 100644 index 0000000000..d8b4591071 --- /dev/null +++ b/src/main/java/org/opensearch/security/support/SecurityIndexHandler.java @@ -0,0 +1,221 @@ +/* + * Copyright 2015-2018 _floragunn_ GmbH + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ +package org.opensearch.security.support; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.Map; + +import com.google.common.collect.ImmutableMap; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.get.MultiGetRequest; +import org.opensearch.action.get.MultiGetResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.Client; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.security.DefaultObjectMapper; +import org.opensearch.security.securityconf.impl.CType; +import org.opensearch.security.securityconf.impl.SecurityDynamicConfiguration; + +import static org.opensearch.core.xcontent.DeprecationHandler.THROW_UNSUPPORTED_OPERATION; +import static org.opensearch.security.configuration.ConfigurationRepository.DEFAULT_CONFIG_VERSION; +import static org.opensearch.security.support.ConfigReader.emptyJsonConfigFor; +import static org.opensearch.security.support.ConfigReader.yamlContentFor; + +public final class SecurityIndexHandler { + + private static final Logger LOGGER = LogManager.getLogger(SecurityIndexHandler.class); + + private final Settings settings; + + private final Client client; + + private final String indexName; + + public SecurityIndexHandler(Settings settings, final String indexName, final Client client) { + this.settings = settings; + this.client = client; + this.indexName = indexName; + } + + private final static Map INDEX_SETTINGS = Map.of("index.number_of_shards", 1, "index.auto_expand_replicas", "0-all"); + + public void createIndex() { + try (final ThreadContext.StoredContext threadContext = client.threadPool().getThreadContext().stashContext()) { + final var created = client.admin() + .indices() + .create(new CreateIndexRequest(indexName).settings(INDEX_SETTINGS)) + .actionGet() + .isAcknowledged(); + if (created) { + LOGGER.info("Security index {} created", indexName); + } else { + throw new SecurityException("Couldn't create index"); + } + } + } + + public void uploadSecurityConfiguration(final Path configDir) { + try (final ThreadContext.StoredContext threadContext = client.threadPool().getThreadContext().stashContext()) { + AccessController.doPrivileged((PrivilegedAction) () -> { + try { + LOGGER.info("Uploading default security configuration from {}", configDir.toAbsolutePath()); + final var bulkRequest = new BulkRequest().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + for (final var cType : CType.values()) { + final var fileExists = Files.exists(cType.configFile(configDir)); + // Audit config is not packaged by default and while list is deprecated + if ((cType == CType.AUDIT || cType == CType.WHITELIST) && !fileExists) continue; + if (cType == CType.WHITELIST) { + LOGGER.warn( + "WHITELIST configuration type is deprecated and will be replaced with ALLOWLIST in the next major version" + ); + } + bulkRequest.add( + new IndexRequest(indexName).id(cType.toLCString()) + .opType(DocWriteRequest.OpType.INDEX) + .source(cType.toLCString(), yamlContentFor(cType, configDir)) + ); + } + final var response = client.bulk(bulkRequest).actionGet(); + if (response.hasFailures()) { + throw new SecurityException(response.buildFailureMessage()); + } + } catch (final IOException ioe) { + throw new SecurityException(ioe); + } + return null; + }); + } + } + + public Map> loadConfiguration(final boolean refresh, final boolean verifyVersion) { + try (final ThreadContext.StoredContext threadContext = client.threadPool().getThreadContext().stashContext()) { + client.threadPool().getThreadContext().putHeader(ConfigConstants.OPENDISTRO_SECURITY_CONF_REQUEST_HEADER, "true"); + + final var cTypeConfigsBuilder = ImmutableMap.>builderWithExpectedSize( + CType.values().length + ); + final var response = client.multiGet(newMultiGetRequest(refresh)).actionGet(); + for (final var responseItem : response.getResponses()) { + if (responseItem.isFailed()) { + throw new SecurityException(multiGetFailureMessage(responseItem.getId(), responseItem.getFailure())); + } + final var cTypeResponse = responseItem.getResponse(); + final var cType = CType.fromString(cTypeResponse.getId()); + if (cTypeResponse.isExists() && !cTypeResponse.isSourceEmpty()) { + final var config = buildDynamicConfiguration( + cType, + cTypeResponse.getSourceAsBytesRef(), + cTypeResponse.getSeqNo(), + cTypeResponse.getPrimaryTerm() + ); + if (verifyVersion) { + if (config.getVersion() != DEFAULT_CONFIG_VERSION) { + throw new SecurityException("Version " + config.getVersion() + " is not supported for " + cType.name()); + } + } + cTypeConfigsBuilder.put(cType, config); + } else { + if (!cType.emptyIfMissing()) { + throw new SecurityException("Missing required configuration for type: " + cType); + } + cTypeConfigsBuilder.put( + cType, + SecurityDynamicConfiguration.fromJson( + emptyJsonConfigFor(cType), + cType, + DEFAULT_CONFIG_VERSION, + cTypeResponse.getSeqNo(), + cTypeResponse.getPrimaryTerm() + ) + ); + } + } + return cTypeConfigsBuilder.build(); + } catch (Exception e) { + throw new SecurityException("Couldn't reload configuration", e); + } + } + + private MultiGetRequest newMultiGetRequest(final boolean refresh) { + final var request = new MultiGetRequest().realtime(true).refresh(refresh); + for (final var cType : CType.values()) { + request.add(indexName, cType.toLCString()); + } + return request; + } + + private SecurityDynamicConfiguration buildDynamicConfiguration( + final CType cType, + final BytesReference bytesRef, + final long seqNo, + final long primaryTerm + ) { + try { + final var source = SecurityUtils.replaceEnvVars(configTypeSource(bytesRef.streamInput()), settings); + final var jsonNode = DefaultObjectMapper.readTree(source); + var version = 1; + if (jsonNode.has("_meta")) { + if (jsonNode.get("_meta").has("config_version")) { + version = jsonNode.get("_meta").get("config_version").asInt(); + } + } + return SecurityDynamicConfiguration.fromJson(source, cType, version, seqNo, primaryTerm); + } catch (IOException e) { + throw new SecurityException("Couldn't parse content for " + cType, e); + } + } + + private String configTypeSource(final InputStream inputStream) throws IOException { + final var jsonContent = XContentType.JSON.xContent(); + try (final var parser = jsonContent.createParser(NamedXContentRegistry.EMPTY, THROW_UNSUPPORTED_OPERATION, inputStream)) { + parser.nextToken(); + parser.nextToken(); + parser.nextToken(); + return new String(parser.binaryValue(), StandardCharsets.UTF_8); + } + } + + private String multiGetFailureMessage(final String cTypeId, final MultiGetResponse.Failure failure) { + return String.format("Failure %s retrieving configuration for %s (index=%s)", failure, cTypeId, indexName); + } + +} diff --git a/src/test/java/org/opensearch/security/support/ConfigReaderTest.java b/src/test/java/org/opensearch/security/support/ConfigReaderTest.java new file mode 100644 index 0000000000..e6e1166a7b --- /dev/null +++ b/src/test/java/org/opensearch/security/support/ConfigReaderTest.java @@ -0,0 +1,79 @@ +/* + * Copyright 2015-2018 _floragunn_ GmbH + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ +package org.opensearch.security.support; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; + +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import org.opensearch.security.DefaultObjectMapper; +import org.opensearch.security.securityconf.impl.CType; + +import static org.opensearch.security.configuration.ConfigurationRepository.DEFAULT_CONFIG_VERSION; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +public class ConfigReaderTest { + + @ClassRule + public static TemporaryFolder folder = new TemporaryFolder(); + + private static File configDir; + + @BeforeClass + public static void createConfigFile() throws IOException { + configDir = folder.newFolder("config"); + } + + @Test + public void testThrowsIOExceptionForMandatoryCTypes() { + for (final var cType : CType.REQUIRED_CONFIG_FILES) { + assertThrows(IOException.class, () -> ConfigReader.newReader(cType, configDir.toPath())); + } + } + + @Test + public void testCreateReaderForNonMandatoryCTypes() throws IOException { + final var yamlMapper = DefaultObjectMapper.YAML_MAPPER; + for (final var cType : CType.NOT_REQUIRED_CONFIG_FILES) { + try (final var reader = new BufferedReader(ConfigReader.newReader(cType, configDir.toPath()))) { + final var emptyYaml = yamlMapper.readTree(reader); + assertTrue(emptyYaml.has("_meta")); + + final var meta = emptyYaml.get("_meta"); + assertEquals(cType.toLCString(), meta.get("type").asText()); + assertEquals(DEFAULT_CONFIG_VERSION, meta.get("config_version").asInt()); + } + } + } + +}