diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b4ae5a76a..3554591d15 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,7 @@ on: [push, pull_request] env: GRADLE_OPTS: -Dhttp.keepAlive=false + CI_ENVIRONMENT: normal jobs: generate-test-list: @@ -106,6 +107,73 @@ jobs: - run: ./gradlew clean build -Dbuild.snapshot=false -x test - uses: github/codeql-action/analyze@v1 + integration-tests: + name: integration-tests + strategy: + fail-fast: false + matrix: + jdk: [11] + platform: [ubuntu-latest] + runs-on: ${{ matrix.platform }} + + steps: + - name: Set up JDK for build and test + uses: actions/setup-java@v3 + with: + distribution: temurin # Temurin is a distribution of adoptium + java-version: ${{ matrix.jdk }} + + - name: Checkout security + uses: actions/checkout@v4 + + - name: Build and Test + uses: gradle/gradle-build-action@v2 + with: + cache-disabled: true + arguments: | + integrationTest -Dbuild.snapshot=false + + - uses: alehechka/upload-tartifact@v2 + if: always() + with: + name: integration-${{ matrix.platform }}-JDK${{ matrix.jdk }}-reports + path: | + ./build/reports/ + + resource-tests: + env: + CI_ENVIRONMENT: resource-test + strategy: + fail-fast: false + matrix: + jdk: [11] + platform: [ubuntu-latest] + runs-on: ${{ matrix.platform }} + + steps: + - name: Set up JDK for build and test + uses: actions/setup-java@v3 + with: + distribution: temurin # Temurin is a distribution of adoptium + java-version: ${{ matrix.jdk }} + + - name: Checkout security + uses: actions/checkout@v4 + + - name: Build and Test + uses: gradle/gradle-build-action@v2 + with: + cache-disabled: true + arguments: | + integrationTest -Dbuild.snapshot=false --tests org.opensearch.security.ResourceFocusedTests + + - uses: alehechka/upload-tartifact@v2 + if: always() + with: + name: resource-test-${{ matrix.platform }}-JDK${{ matrix.jdk }}-reports + path: | + ./build/reports/ + build-artifact-names: runs-on: ubuntu-latest steps: diff --git a/build.gradle b/build.gradle index 90587dab92..5409c32dfd 100644 --- a/build.gradle +++ b/build.gradle @@ -424,9 +424,78 @@ configurations.all { force "com.github.luben:zstd-jni:${versions.zstd}" force "org.xerial.snappy:snappy-java:1.1.10.1" force "com.google.guava:guava:${guava_version}" + + // For integrationTest + force "org.apache.httpcomponents:httpclient-cache:4.5.13" + force "org.apache.httpcomponents:httpclient:4.5.13" + force "org.apache.httpcomponents:fluent-hc:4.5.13" + force "org.apache.httpcomponents:httpcore:4.4.16" + force "org.apache.httpcomponents:httpcore-nio:4.4.16" + force "org.apache.httpcomponents:httpasyncclient:4.1.5" + } +} + +//create source set 'integrationTest' +//add classes from the main source set to the compilation and runtime classpaths of the integrationTest +sourceSets { + integrationTest { + java { + srcDir file ('src/integrationTest/java') + compileClasspath += sourceSets.main.output + runtimeClasspath += sourceSets.main.output + } + resources { + srcDir file('src/integrationTest/resources') + } + processIntegrationTestResources { + duplicatesStrategy(DuplicatesStrategy.INCLUDE) + } + } +} + +//add new task that runs integration tests +task integrationTest(type: Test) { + doFirst { + // Only run resources tests on resource-test CI environments or locally + if (System.getenv('CI_ENVIRONMENT') != 'resource-test' && System.getenv('CI_ENVIRONMENT') != null) { + exclude '**/ResourceFocusedTests.class' + } + // Only run with retries while in CI systems + if (System.getenv('CI_ENVIRONMENT') == 'normal') { + retry { + failOnPassedAfterRetry = false + maxRetries = 2 + maxFailures = 10 + } + } + } + description = 'Run integration tests.' + group = 'verification' + systemProperty "java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager" + testClassesDirs = sourceSets.integrationTest.output.classesDirs + classpath = sourceSets.integrationTest.runtimeClasspath + //run the integrationTest task after the test task + shouldRunAfter test + jacoco { + excludes = [ + "com.sun.jndi.dns.*", + "com.sun.security.sasl.gsskerb.*", + "java.sql.*", + "javax.script.*", + "org.jcp.xml.dsig.internal.dom.*", + "sun.nio.cs.ext.*", + "sun.security.ec.*", + "sun.security.jgss.*", + "sun.security.pkcs11.*", + "sun.security.smartcardio.*", + "sun.util.resources.provider.*" + ] } } +//run the integrationTest task before the check task +check.dependsOn integrationTest + dependencies { implementation "org.opensearch.plugin:transport-netty4-client:${opensearch_version}" implementation "org.opensearch.client:opensearch-rest-high-level-client:${opensearch_version}" @@ -562,6 +631,48 @@ dependencies { implementation('com.google.googlejavaformat:google-java-format:1.17.0') { exclude group: 'com.google.guava' } + integrationTestCompileOnly "org.opensearch:opensearch:${opensearch_version}" + integrationTestImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}" + integrationTestImplementation "org.opensearch:opensearch-ssl-config:${opensearch_version}" + integrationTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}" + integrationTestImplementation "org.opensearch.plugin:lang-mustache-client:${opensearch_version}" + integrationTestImplementation "org.opensearch.plugin:parent-join-client:${opensearch_version}" + integrationTestImplementation "org.opensearch.plugin:aggs-matrix-stats-client:${opensearch_version}" + integrationTestImplementation "com.google.guava:guava:32.1.1-jre" + integrationTestImplementation "org.apache.commons:commons-lang3:3.4" + integrationTestImplementation 'com.fasterxml.jackson.core:jackson-databind:2.14.1' + integrationTestImplementation 'io.jsonwebtoken:jjwt-api:0.10.8' + integrationTestImplementation "org.opensearch.client:opensearch-rest-high-level-client:${opensearch_version}" + integrationTestImplementation "org.opensearch.plugin:transport-netty4-client:${opensearch_version}" + + integrationTestRuntimeOnly 'org.greenrobot:eventbus:3.2.0' + integrationTestRuntimeOnly 'com.flipkart.zjsonpatch:zjsonpatch:0.4.4' + integrationTestRuntimeOnly 'commons-lang:commons-lang:2.4' + integrationTestRuntimeOnly 'org.ldaptive:ldaptive:1.2.3' + integrationTestRuntimeOnly 'commons-collections:commons-collections:3.2.2' + + //integration test framework: + integrationTestImplementation('com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.1') { + exclude(group: 'junit', module: 'junit') + } + integrationTestImplementation 'junit:junit:4.13.2' + integrationTestImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}" + integrationTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}" + integrationTestImplementation 'commons-io:commons-io:2.14.0' + integrationTestImplementation "org.apache.logging.log4j:log4j-core:2.17.1" + integrationTestImplementation "org.apache.logging.log4j:log4j-jul:2.17.1" + integrationTestImplementation 'org.hamcrest:hamcrest:2.2' + integrationTestImplementation "org.bouncycastle:bcpkix-jdk15to18:1.75" + integrationTestImplementation "org.bouncycastle:bcutil-jdk15to18:1.75" + integrationTestImplementation('org.awaitility:awaitility:4.2.0') { + exclude(group: 'org.hamcrest', module: 'hamcrest') + } + integrationTestImplementation 'com.unboundid:unboundid-ldapsdk:4.0.14' + integrationTestImplementation "org.apache.httpcomponents:httpclient-cache:4.5.14" + integrationTestImplementation "org.apache.httpcomponents:httpclient:4.5.14" + integrationTestImplementation "org.apache.httpcomponents:fluent-hc:4.5.13" + integrationTestImplementation "org.apache.httpcomponents:httpcore:4.4.16" + integrationTestImplementation "org.apache.httpcomponents:httpasyncclient:4.1.5" } jar { diff --git a/src/integrationTest/java/org/opensearch/common/logging/NodeAndClusterIdConverter.java b/src/integrationTest/java/org/opensearch/common/logging/NodeAndClusterIdConverter.java new file mode 100644 index 0000000000..4aba6c976b --- /dev/null +++ b/src/integrationTest/java/org/opensearch/common/logging/NodeAndClusterIdConverter.java @@ -0,0 +1,29 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.common.logging; + +/** +* Class uses to override OpenSearch NodeAndClusterIdConverter Log4j2 plugin in order to disable plugin and limit number of +* warn messages like "...ApplierService#updateTask][T#1] WARN ClusterApplierService:628 - failed to notify ClusterStateListener..." +* during tests execution. +* +* The class is rather a temporary solution and the real one should be developed in scope of: +* https://github.com/opensearch-project/OpenSearch/pull/4322 +*/ +import org.apache.logging.log4j.core.LogEvent; + +class NodeAndClusterIdConverter { + + public NodeAndClusterIdConverter() {} + + public static void setNodeIdAndClusterId(String nodeId, String clusterUUID) {} + + public void format(LogEvent event, StringBuilder toAppendTo) {} +} diff --git a/src/integrationTest/java/org/opensearch/node/PluginAwareNode.java b/src/integrationTest/java/org/opensearch/node/PluginAwareNode.java new file mode 100644 index 0000000000..191c32646a --- /dev/null +++ b/src/integrationTest/java/org/opensearch/node/PluginAwareNode.java @@ -0,0 +1,55 @@ +/* +* Copyright 2015-2018 _floragunn_ GmbH +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +/* +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ + +package org.opensearch.node; + +import java.util.Collection; +import java.util.Collections; + +import org.opensearch.common.settings.Settings; +import org.opensearch.plugins.Plugin; + +public class PluginAwareNode extends Node { + + private final boolean clusterManagerEligible; + + public PluginAwareNode( + boolean clusterManagerEligible, + final Settings preparedSettings, + final Collection> plugins + ) { + super( + InternalSettingsPreparer.prepareEnvironment(preparedSettings, Collections.emptyMap(), null, () -> System.getenv("HOSTNAME")), + plugins, + true + ); + this.clusterManagerEligible = clusterManagerEligible; + } + + public boolean isClusterManagerEligible() { + return clusterManagerEligible; + } +} diff --git a/src/integrationTest/java/org/opensearch/security/ConfigurationFiles.java b/src/integrationTest/java/org/opensearch/security/ConfigurationFiles.java new file mode 100644 index 0000000000..287bc139b1 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/ConfigurationFiles.java @@ -0,0 +1,60 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.security; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; + +class ConfigurationFiles { + + public static void createRoleMappingFile(File destination) { + String resource = "roles_mapping.yml"; + copyResourceToFile(resource, destination); + } + + public static Path createConfigurationDirectory() { + try { + Path tempDirectory = Files.createTempDirectory("test-security-config"); + String[] configurationFiles = { + "config.yml", + "action_groups.yml", + "config.yml", + "internal_users.yml", + "roles.yml", + "roles_mapping.yml", + "security_tenants.yml", + "tenants.yml" }; + for (String fileName : configurationFiles) { + Path configFileDestination = tempDirectory.resolve(fileName); + copyResourceToFile(fileName, configFileDestination.toFile()); + } + return tempDirectory.toAbsolutePath(); + } catch (IOException ex) { + throw new RuntimeException("Cannot create directory with security plugin configuration.", ex); + } + } + + private static void copyResourceToFile(String resource, File destination) { + try (InputStream input = ConfigurationFiles.class.getClassLoader().getResourceAsStream(resource)) { + Objects.requireNonNull(input, "Cannot find source resource " + resource); + try (OutputStream output = new FileOutputStream(destination)) { + input.transferTo(output); + } + } catch (IOException e) { + throw new RuntimeException("Cannot create file with security plugin configuration", e); + } + } +} diff --git a/src/integrationTest/java/org/opensearch/security/DlsTests.java b/src/integrationTest/java/org/opensearch/security/DlsTests.java new file mode 100644 index 0000000000..41da011bb7 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/DlsTests.java @@ -0,0 +1,284 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.security; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.OptionalDouble; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.concurrent.Callable; + + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; + +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.action.admin.indices.create.CreateIndexRequest; + +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.Client; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.test.framework.AsyncActions; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.either; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.oneOf; +import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; +import static org.opensearch.client.RequestOptions.DEFAULT; +import static org.opensearch.security.Song.SONGS; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.TestSecurityConfig.Role.ALL_ACCESS; + +//./gradlew integrationTest --tests org.opensearch.security.DlsTests +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class DlsTests { + + private enum TestRoles { + EMPTY_DLS, + DLS_ONLY_ROCK, + DLS_ONLY_JAZZ, + DLS_ONLY_ROCK_AND_JAZZ, + DLS_ONLY_LONG_VALUE; + } + + static final String INDEX_NAME_PREFIX = "test-index-"; + static final String ALL_INDICES_ALIAS = "_all"; + static final String READER_BACKEND_ROLE = "ber-reader"; + + static final TestSecurityConfig.User ADMIN_USER = new TestSecurityConfig.User("admin").roles(ALL_ACCESS); + + /** + * User who is allowed to see all fields on all indices. Values of the title and artist fields should be masked. + */ + static final TestSecurityConfig.User READER = new TestSecurityConfig.User("reader") + .roles( + new TestSecurityConfig.Role("read-everything") + .clusterPermissions("cluster_composite_ops_ro") + .indexPermissions("read") + .on("*") + ).backendRoles(READER_BACKEND_ROLE); + + @ClassRule + public static final LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS) + .anonymousAuth(false) + .nodeSettings( + Map.of("plugins.security.restapi.roles_enabled", List.of("user_" + ADMIN_USER.getName() + "__" + ALL_ACCESS.getName())) + ) + .roles(new TestSecurityConfig.Role(TestRoles.EMPTY_DLS.name()) + .clusterPermissions("cluster_composite_ops_ro") + .indexPermissions("read") + .dls("") + .on("*"), + new TestSecurityConfig.Role(TestRoles.DLS_ONLY_ROCK.name()) + .clusterPermissions("cluster_composite_ops_ro") + .indexPermissions("read") + .dls("{\"bool\":{\"must\":[{\"terms\":{\"genre.keyword\":[\"rock\"]}}]}}") + .on("*"), + new TestSecurityConfig.Role(TestRoles.DLS_ONLY_JAZZ.name()) + .clusterPermissions("cluster_composite_ops_ro") + .indexPermissions("read") + .dls("{\"bool\":{\"must\":[{\"terms\":{\"genre.keyword\":[\"jazz\"]}}]}}") + .on("*"), + new TestSecurityConfig.Role(TestRoles.DLS_ONLY_LONG_VALUE.name()) + .clusterPermissions("cluster_composite_ops_ro") + .indexPermissions("read") + .dls("{\"bool\":{\"must\":[{\"terms\":{\"genre.keyword\":[\"" + + "0123456789".repeat(100) // ==1000 characters + + "\"]}}]}}") + .on("*"), + new TestSecurityConfig.Role(TestRoles.DLS_ONLY_ROCK_AND_JAZZ.name()) + .clusterPermissions("cluster_composite_ops_ro") + .indexPermissions("read") + .dls("{\"bool\":{\"must\":[{\"terms\":{\"genre.keyword\":[\"jazz\"]}},{\"terms\":{\"genre.keyword\":[\"rock\"]}}]}}") + .on("*")) + .authc(AUTHC_HTTPBASIC_INTERNAL) + .users( + ADMIN_USER, + READER + ) + .build(); + + @BeforeClass + public static void createTestData() { + } + + @Before + public void setup() { + try (TestRestClient client = cluster.getRestClient(ADMIN_USER)) { + for (TestRoles role : TestRoles.values()) { + final String path = "_plugins/_security/api/rolesmapping/" + role.name(); + final HttpResponse response = client.delete(path); + assertThat(response.getStatusCode(), oneOf(200, 204, 404)); + } + } + // try (Client client = cluster.getInternalNodeClient()) { + // client.admin().indices() + // } + } + + @Test + public void testBaselinedDlsScenarios() throws Exception { + final Callable check = () -> { + final long startMs = System.currentTimeMillis(); + queryAndGetStats(ADMIN_USER); + queryAndGetStats(READER); + + attachRoleToReader(TestRoles.EMPTY_DLS); + queryAndGetStats(READER); + + attachRoleToReader(TestRoles.DLS_ONLY_ROCK); + queryAndGetStats(READER); + + attachRoleToReader(TestRoles.DLS_ONLY_JAZZ); + queryAndGetStats(READER); + final long endMs = System.currentTimeMillis() - startMs; + System.out.println("Finished checks in " + endMs + "ms"); + + return null; + }; + createIndices(5); + check.call(); + + setup(); + createIndices(50); + check.call(); + + setup(); + createIndices(100); + check.call(); + } + + @Test + public void testConsolidatedDlsScenarios() throws Exception { + final Callable check = () -> { + final long startMs = System.currentTimeMillis(); + queryAndGetStats(READER); + attachRoleToReader(TestRoles.DLS_ONLY_ROCK_AND_JAZZ); + queryAndGetStats(READER); + final long endMs = System.currentTimeMillis() - startMs; + System.out.println("Finished checks in " + endMs + "ms"); + return null; + }; + createIndices(5); + check.call(); + + setup(); + createIndices(50); + check.call(); + + setup(); + createIndices(100); + check.call(); + } + + @Test + public void testDlsLargerQueryScenarios() throws Exception { + final Callable check = () -> { + final long startMs = System.currentTimeMillis(); + queryAndGetStats(READER); + + attachRoleToReader(TestRoles.DLS_ONLY_LONG_VALUE); + queryAndGetStats(READER); + final long endMs = System.currentTimeMillis() - startMs; + System.out.println("Finished checks in " + endMs + "ms"); + return null; + }; + createIndices(5); + check.call(); + + setup(); + createIndices(50); + check.call(); + + setup(); + createIndices(100); + check.call(); + } + + private void attachRoleToReader(final TestRoles role) { + System.out.println("Attached READER with role " + role); + try (TestRestClient client = cluster.getRestClient(ADMIN_USER)) { + final String path = "_plugins/_security/api/rolesmapping/" + role; + final String body = String.format("{\"backend_roles\": [\"%s\"]}", READER_BACKEND_ROLE); + final HttpResponse response = client.putJson(path, body); + response.assertStatusCode(201); + } + } + + private void createIndices(final int count) throws IOException { + System.out.println("Creating " + count + " indices with 1 document"); + try (Client client = cluster.getInternalNodeClient()) { + final ExecutorService pool = Executors.newFixedThreadPool(25); + final List> futures = IntStream.range(1, count).mapToObj(n -> { + final String indexName = INDEX_NAME_PREFIX + n; + return CompletableFuture.runAsync(() -> client.prepareIndex().setIndex(indexName).setRefreshPolicy(IMMEDIATE).setSource(SONGS[0].asMap()).get(), pool); + }).collect(Collectors.toList()); + + final CompletableFuture futuresCompleted = CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + futuresCompleted.join(); + } + } + + private void queryAndGetStats(final TestSecurityConfig.User user) throws IOException { + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(user)) { + final int samplesToIgnore = 5; + final int samples = 100 + samplesToIgnore; + final List results = new ArrayList<>(); + for (int i = 0; i < samples; i++) { + final long start = System.currentTimeMillis(); + final SearchResponse response = restHighLevelClient.search(new SearchRequest(INDEX_NAME_PREFIX + "*"), DEFAULT); + final long endMs = System.currentTimeMillis() - start; + results.add(endMs); + } + // toss out inital samples + IntStream.range(0, samplesToIgnore).forEach(n -> results.remove(0)); + + System.out.println("User, Count, Avg, Max, Min, Std ms " + + user.getName() + + ", " + results.size() + + ", " + results.stream().mapToLong(a -> a).average().getAsDouble() + + ", " + results.stream().mapToLong(a -> a).max().getAsLong() + + ", " + results.stream().mapToLong(a -> a).min().getAsLong() + + ", " + String.format("%.2f", calcStd(results))); + } + } + + private static double calcStd(final List numbers) { + final Double mean= numbers.stream() + .mapToDouble(Long::doubleValue) + .average() + .orElse(0); + final double variance = numbers.stream() + .mapToDouble(i -> Math.pow(i - mean, 2)) + .average() + .orElse(0); + return Math.sqrt(variance); + } + +} diff --git a/src/integrationTest/java/org/opensearch/security/ResourceFocusedTests.java b/src/integrationTest/java/org/opensearch/security/ResourceFocusedTests.java new file mode 100644 index 0000000000..c1a79f94f9 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/ResourceFocusedTests.java @@ -0,0 +1,198 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.security; + +import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.TestSecurityConfig.Role.ALL_ACCESS; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.zip.GZIPOutputStream; + +import org.apache.http.HttpStatus; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.entity.ContentType; +import org.apache.http.message.BasicHeader; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.client.Client; +import org.opensearch.test.framework.AsyncActions; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.TestSecurityConfig.User; +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; + +@Ignore("Waiting on backport for https://github.com/opensearch-project/security/pull/3418") +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class ResourceFocusedTests { + private final static Logger LOG = LogManager.getLogger(AsyncActions.class); + private static final User ADMIN_USER = new User("admin").roles(ALL_ACCESS); + private static final User LIMITED_USER = new User("limited_user").roles( + new TestSecurityConfig.Role("limited-role").clusterPermissions( + "indices:data/read/mget", + "indices:data/read/msearch", + "indices:data/read/scroll", + "cluster:monitor/state", + "cluster:monitor/health" + ) + .indexPermissions( + "indices:data/read/search", + "indices:data/read/mget*", + "indices:monitor/settings/get", + "indices:monitor/stats" + ) + .on("*") + ); + + @ClassRule + public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS) + .authc(AUTHC_HTTPBASIC_INTERNAL) + .users(ADMIN_USER, LIMITED_USER) + .anonymousAuth(false) + .doNotFailOnForbidden(true) + .build(); + + @BeforeClass + public static void createTestData() { + try (Client client = cluster.getInternalNodeClient()) { + client.index(new IndexRequest().setRefreshPolicy(IMMEDIATE).index("document").source(Map.of("foo", "bar", "abc", "xyz"))) + .actionGet(); + } + } + + @Test + public void testUnauthenticatedFewBig() { + // Tweaks: + final RequestBodySize size = RequestBodySize.XLarge; + final String requestPath = "/*/_search"; + final int parrallelism = 5; + final int totalNumberOfRequests = 100; + + runResourceTest(size, requestPath, parrallelism, totalNumberOfRequests); + } + + @Test + public void testUnauthenticatedManyMedium() { + // Tweaks: + final RequestBodySize size = RequestBodySize.Medium; + final String requestPath = "/*/_search"; + final int parrallelism = 20; + final int totalNumberOfRequests = 10_000; + + runResourceTest(size, requestPath, parrallelism, totalNumberOfRequests); + } + + @Test + public void testUnauthenticatedTonsSmall() { + // Tweaks: + final RequestBodySize size = RequestBodySize.Small; + final String requestPath = "/*/_search"; + final int parrallelism = 100; + final int totalNumberOfRequests = 15_000; + + runResourceTest(size, requestPath, parrallelism, totalNumberOfRequests); + } + + private void runResourceTest( + final RequestBodySize size, + final String requestPath, + final int parrallelism, + final int totalNumberOfRequests + ) { + final byte[] compressedRequestBody = createCompressedRequestBody(size); + try (final TestRestClient client = cluster.getRestClient(new BasicHeader("Content-Encoding", "gzip"))) { + final List> requests = AsyncActions.generate(() -> { + final HttpPost post = new HttpPost(client.getHttpServerUri() + requestPath); + post.setEntity(new ByteArrayEntity(compressedRequestBody, ContentType.APPLICATION_JSON)); + return client.executeRequest(post); + }, parrallelism, totalNumberOfRequests); + + AsyncActions.getAll(requests, 2, TimeUnit.MINUTES) + .forEach((response) -> { response.assertStatusCode(HttpStatus.SC_UNAUTHORIZED); }); + } + } + + static enum RequestBodySize { + Small(1), + Medium(1_000), + XLarge(1_000_000); + + public final int elementCount; + + private RequestBodySize(final int elementCount) { + this.elementCount = elementCount; + } + } + + private byte[] createCompressedRequestBody(final RequestBodySize size) { + final int repeatCount = size.elementCount; + final String prefix = "{ \"items\": ["; + final String repeatedElement = IntStream.range(0, 20) + .mapToObj(n -> ('a' + n) + "") + .map(n -> '"' + n + '"' + ": 123") + .collect(Collectors.joining(",", "{", "}")); + final String postfix = "]}"; + long uncompressedBytesSize = 0; + + try ( + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + final GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream) + ) { + + final byte[] prefixBytes = prefix.getBytes(StandardCharsets.UTF_8); + final byte[] repeatedElementBytes = repeatedElement.getBytes(StandardCharsets.UTF_8); + final byte[] postfixBytes = postfix.getBytes(StandardCharsets.UTF_8); + + gzipOutputStream.write(prefixBytes); + uncompressedBytesSize = uncompressedBytesSize + prefixBytes.length; + for (int i = 0; i < repeatCount; i++) { + gzipOutputStream.write(repeatedElementBytes); + uncompressedBytesSize = uncompressedBytesSize + repeatedElementBytes.length; + } + gzipOutputStream.write(postfixBytes); + uncompressedBytesSize = uncompressedBytesSize + postfixBytes.length; + gzipOutputStream.finish(); + + final byte[] compressedRequestBody = byteArrayOutputStream.toByteArray(); + LOG.info( + String.format( + "Original size was %,d bytes, compressed to %,d bytes, ratio %,.2f", + uncompressedBytesSize, + compressedRequestBody.length, + ((double) uncompressedBytesSize / compressedRequestBody.length) + ) + ); + return compressedRequestBody; + } catch (final IOException ioe) { + throw new RuntimeException(ioe); + } + } +} diff --git a/src/integrationTest/java/org/opensearch/security/Song.java b/src/integrationTest/java/org/opensearch/security/Song.java new file mode 100644 index 0000000000..b7e6c4ef05 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/Song.java @@ -0,0 +1,98 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.security; + +import java.util.Map; +import java.util.Objects; + +public class Song { + + public static final String FIELD_TITLE = "title"; + public static final String FIELD_ARTIST = "artist"; + public static final String FIELD_LYRICS = "lyrics"; + public static final String FIELD_STARS = "stars"; + public static final String FIELD_GENRE = "genre"; + public static final String ARTIST_FIRST = "First artist"; + public static final String ARTIST_STRING = "String"; + public static final String ARTIST_TWINS = "Twins"; + public static final String TITLE_MAGNUM_OPUS = "Magnum Opus"; + public static final String TITLE_SONG_1_PLUS_1 = "Song 1+1"; + public static final String TITLE_NEXT_SONG = "Next song"; + public static final String ARTIST_NO = "No!"; + public static final String TITLE_POISON = "Poison"; + + public static final String ARTIST_YES = "yes"; + + public static final String TITLE_AFFIRMATIVE = "Affirmative"; + + public static final String ARTIST_UNKNOWN = "unknown"; + public static final String TITLE_CONFIDENTIAL = "confidential"; + + public static final String LYRICS_1 = "Very deep subject"; + public static final String LYRICS_2 = "Once upon a time"; + public static final String LYRICS_3 = "giant nonsense"; + public static final String LYRICS_4 = "Much too much"; + public static final String LYRICS_5 = "Little to little"; + public static final String LYRICS_6 = "confidential secret classified"; + + public static final String GENRE_ROCK = "rock"; + public static final String GENRE_JAZZ = "jazz"; + public static final String GENRE_BLUES = "blues"; + + public static final String QUERY_TITLE_NEXT_SONG = FIELD_TITLE + ":" + "\"" + TITLE_NEXT_SONG + "\""; + public static final String QUERY_TITLE_POISON = FIELD_TITLE + ":" + TITLE_POISON; + public static final String QUERY_TITLE_MAGNUM_OPUS = FIELD_TITLE + ":" + TITLE_MAGNUM_OPUS; + + public static final Song[] SONGS = { + new Song(ARTIST_FIRST, TITLE_MAGNUM_OPUS, LYRICS_1, 1, GENRE_ROCK), + new Song(ARTIST_STRING, TITLE_SONG_1_PLUS_1, LYRICS_2, 2, GENRE_BLUES), + new Song(ARTIST_TWINS, TITLE_NEXT_SONG, LYRICS_3, 3, GENRE_JAZZ), + new Song(ARTIST_NO, TITLE_POISON, LYRICS_4, 4, GENRE_ROCK), + new Song(ARTIST_YES, TITLE_AFFIRMATIVE, LYRICS_5, 5, GENRE_BLUES), + new Song(ARTIST_UNKNOWN, TITLE_CONFIDENTIAL, LYRICS_6, 6, GENRE_JAZZ) }; + + private final String artist; + private final String title; + private final String lyrics; + private final Integer stars; + private final String genre; + + public Song(String artist, String title, String lyrics, Integer stars, String genre) { + this.artist = Objects.requireNonNull(artist, "Artist is required"); + this.title = Objects.requireNonNull(title, "Title is required"); + this.lyrics = Objects.requireNonNull(lyrics, "Lyrics is required"); + this.stars = Objects.requireNonNull(stars, "Stars field is required"); + this.genre = Objects.requireNonNull(genre, "Genre field is required"); + } + + public String getArtist() { + return artist; + } + + public String getTitle() { + return title; + } + + public String getLyrics() { + return lyrics; + } + + public Integer getStars() { + return stars; + } + + public String getGenre() { + return genre; + } + + public Map asMap() { + return Map.of(FIELD_ARTIST, artist, FIELD_TITLE, title, FIELD_LYRICS, lyrics, FIELD_STARS, stars, FIELD_GENRE, genre); + } +} diff --git a/src/integrationTest/java/org/opensearch/security/SslOnlyTests.java b/src/integrationTest/java/org/opensearch/security/SslOnlyTests.java new file mode 100644 index 0000000000..b74953d557 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/SslOnlyTests.java @@ -0,0 +1,69 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.security; + +import java.util.Map; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.security.support.ConfigConstants; +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; + +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; + +/** +* Test related to SSL-only mode of security plugin. In this mode, the security plugin is responsible only for TLS/SSL encryption. +* Therefore, the plugin does not perform authentication and authorization. Moreover, the REST resources (e.g. /_plugins/_security/whoami, +* /_plugins/_security/authinfo, etc.) provided by the plugin are not available. +*/ +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class SslOnlyTests { + + @ClassRule + public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS) + .anonymousAuth(false) + .loadConfigurationIntoIndex(false) + .nodeSettings(Map.of(ConfigConstants.SECURITY_SSL_ONLY, true)) + .sslOnly(true) + .authc(AUTHC_HTTPBASIC_INTERNAL) + .build(); + + @Test + public void shouldNotLoadSecurityPluginResources() { + try (TestRestClient client = cluster.getRestClient()) { + + HttpResponse response = client.getAuthInfo(); + + // in SSL only mode the security plugin does not register a handler for resource /_plugins/_security/whoami. Therefore error + // response is returned. + response.assertStatusCode(404); + } + } + + @Test + public void shouldGetIndicesWithoutAuthentication() { + try (TestRestClient client = cluster.getRestClient()) { + + // request does not contains credential + HttpResponse response = client.get("/_cat/indices"); + + // successful response is returned because the security plugin in SSL only mode + // does not perform authentication and authorization + response.assertStatusCode(200); + } + } +} diff --git a/src/integrationTest/java/org/opensearch/security/TlsTests.java b/src/integrationTest/java/org/opensearch/security/TlsTests.java new file mode 100644 index 0000000000..91c81cf04c --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/TlsTests.java @@ -0,0 +1,106 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.security; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import javax.net.ssl.SSLHandshakeException; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.NoHttpResponseException; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.security.auditlog.impl.AuditCategory; +import org.opensearch.test.framework.AuditCompliance; +import org.opensearch.test.framework.AuditConfiguration; +import org.opensearch.test.framework.AuditFilters; +import org.opensearch.test.framework.TestSecurityConfig.User; +import org.opensearch.test.framework.audit.AuditLogsRule; +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.opensearch.security.auditlog.AuditLog.Origin.REST; +import static org.opensearch.security.ssl.util.SSLConfigConstants.SECURITY_SSL_HTTP_ENABLED_CIPHERS; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.TestSecurityConfig.Role.ALL_ACCESS; +import static org.opensearch.test.framework.audit.AuditMessagePredicate.auditPredicate; +import static org.opensearch.test.framework.cluster.TestRestClientConfiguration.getBasicAuthHeader; +import static org.opensearch.test.framework.matcher.ExceptionMatcherAssert.assertThatThrownBy; + +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class TlsTests { + + private static final User USER_ADMIN = new User("admin").roles(ALL_ACCESS); + + public static final String SUPPORTED_CIPHER_SUIT = "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"; + public static final String NOT_SUPPORTED_CIPHER_SUITE = "TLS_RSA_WITH_AES_128_CBC_SHA"; + public static final String AUTH_INFO_ENDPOINT = "/_opendistro/_security/authinfo?pretty"; + + @ClassRule + public static final LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS) + .anonymousAuth(false) + .nodeSettings(Map.of(SECURITY_SSL_HTTP_ENABLED_CIPHERS, List.of(SUPPORTED_CIPHER_SUIT))) + .authc(AUTHC_HTTPBASIC_INTERNAL) + .users(USER_ADMIN) + .audit( + new AuditConfiguration(true).compliance(new AuditCompliance().enabled(true)) + .filters(new AuditFilters().enabledRest(true).enabledTransport(true)) + ) + .build(); + + @Rule + public AuditLogsRule auditLogsRule = new AuditLogsRule(); + + @Test + public void shouldCreateAuditOnIncomingNonTlsConnection() throws IOException { + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpGet request = new HttpGet("http://localhost:" + cluster.getHttpPort()); + + assertThatThrownBy(() -> httpClient.execute(request), instanceOf(NoHttpResponseException.class)); + } + auditLogsRule.assertAtLeast(1, auditPredicate(AuditCategory.SSL_EXCEPTION).withLayer(REST)); + } + + @Test + public void shouldSupportClientCipherSuite_positive() throws IOException { + try (CloseableHttpClient client = cluster.getClosableHttpClient(new String[] { SUPPORTED_CIPHER_SUIT })) { + HttpGet httpGet = new HttpGet("https://localhost:" + cluster.getHttpPort() + AUTH_INFO_ENDPOINT); + httpGet.addHeader(getBasicAuthHeader(USER_ADMIN.getName(), USER_ADMIN.getPassword())); + + try (CloseableHttpResponse response = client.execute(httpGet)) { + + int responseStatusCode = response.getStatusLine().getStatusCode(); + assertThat(responseStatusCode, equalTo(200)); + } + } + } + + @Test + public void shouldSupportClientCipherSuite_negative() throws IOException { + try (CloseableHttpClient client = cluster.getClosableHttpClient(new String[] { NOT_SUPPORTED_CIPHER_SUITE })) { + HttpGet httpGet = new HttpGet("https://localhost:" + cluster.getHttpPort() + AUTH_INFO_ENDPOINT); + + assertThatThrownBy(() -> client.execute(httpGet), instanceOf(SSLHandshakeException.class)); + } + } +} diff --git a/src/integrationTest/java/org/opensearch/security/UserBruteForceAttacksPreventionTests.java b/src/integrationTest/java/org/opensearch/security/UserBruteForceAttacksPreventionTests.java new file mode 100644 index 0000000000..cb61950ada --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/UserBruteForceAttacksPreventionTests.java @@ -0,0 +1,133 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.security; + +import java.util.concurrent.TimeUnit; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.test.framework.AuthFailureListeners; +import org.opensearch.test.framework.RateLimiting; +import org.opensearch.test.framework.TestSecurityConfig.User; +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; +import org.opensearch.test.framework.log.LogsRule; + +import static org.apache.http.HttpStatus.SC_OK; +import static org.apache.http.HttpStatus.SC_UNAUTHORIZED; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.TestSecurityConfig.Role.ALL_ACCESS; + +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class UserBruteForceAttacksPreventionTests { + + private static final User USER_1 = new User("simple-user-1").roles(ALL_ACCESS); + private static final User USER_2 = new User("simple-user-2").roles(ALL_ACCESS); + private static final User USER_3 = new User("simple-user-3").roles(ALL_ACCESS); + private static final User USER_4 = new User("simple-user-4").roles(ALL_ACCESS); + private static final User USER_5 = new User("simple-user-5").roles(ALL_ACCESS); + + public static final int ALLOWED_TRIES = 3; + public static final int TIME_WINDOW_SECONDS = 3; + private static final AuthFailureListeners listener = new AuthFailureListeners().addRateLimit( + new RateLimiting("internal_authentication_backend_limiting").type("username") + .authenticationBackend("intern") + .allowedTries(ALLOWED_TRIES) + .timeWindowSeconds(TIME_WINDOW_SECONDS) + .blockExpirySeconds(2) + .maxBlockedClients(500) + .maxTrackedClients(500) + ); + + @ClassRule + public static final LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.SINGLENODE) + .anonymousAuth(false) + .authFailureListeners(listener) + .authc(AUTHC_HTTPBASIC_INTERNAL) + .users(USER_1, USER_2, USER_3, USER_4, USER_5) + .build(); + + @Rule + public LogsRule logsRule = new LogsRule("org.opensearch.security.auth.BackendRegistry"); + + @Test + public void shouldAuthenticateUserWhenBlockadeIsNotActive() { + try (TestRestClient client = cluster.getRestClient(USER_1)) { + + HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(SC_OK); + } + } + + @Test + public void shouldBlockUserWhenNumberOfFailureLoginAttemptIsEqualToLimit() { + authenticateUserWithIncorrectPassword(USER_2, ALLOWED_TRIES); + try (TestRestClient client = cluster.getRestClient(USER_2)) { + HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(SC_UNAUTHORIZED); + } + // Rejecting REST request because of blocked user: + logsRule.assertThatContain("Rejecting REST request because of blocked user: " + USER_2.getName()); + } + + @Test + public void shouldBlockUserWhenNumberOfFailureLoginAttemptIsGreaterThanLimit() { + authenticateUserWithIncorrectPassword(USER_3, ALLOWED_TRIES * 2); + try (TestRestClient client = cluster.getRestClient(USER_3)) { + HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(SC_UNAUTHORIZED); + } + logsRule.assertThatContain("Rejecting REST request because of blocked user: " + USER_3.getName()); + } + + @Test + public void shouldNotBlockUserWhenNumberOfLoginAttemptIsBelowLimit() { + authenticateUserWithIncorrectPassword(USER_4, ALLOWED_TRIES - 1); + try (TestRestClient client = cluster.getRestClient(USER_4)) { + HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(SC_OK); + } + } + + @Test + public void shouldReleaseLock() throws InterruptedException { + authenticateUserWithIncorrectPassword(USER_5, ALLOWED_TRIES); + try (TestRestClient client = cluster.getRestClient(USER_5)) { + HttpResponse response = client.getAuthInfo(); + response.assertStatusCode(SC_UNAUTHORIZED); + TimeUnit.SECONDS.sleep(TIME_WINDOW_SECONDS); + + response = client.getAuthInfo(); + + response.assertStatusCode(SC_OK); + } + logsRule.assertThatContain("Rejecting REST request because of blocked user: " + USER_5.getName()); + } + + private static void authenticateUserWithIncorrectPassword(User user, int numberOfAttempts) { + try (TestRestClient client = cluster.getRestClient(user.getName(), "incorrect password")) { + for (int i = 0; i < numberOfAttempts; ++i) { + HttpResponse response = client.getAuthInfo(); + response.assertStatusCode(SC_UNAUTHORIZED); + } + } + } +} diff --git a/src/integrationTest/java/org/opensearch/security/api/DashboardsInfoTest.java b/src/integrationTest/java/org/opensearch/security/api/DashboardsInfoTest.java index a8936765d2..0afe81591d 100644 --- a/src/integrationTest/java/org/opensearch/security/api/DashboardsInfoTest.java +++ b/src/integrationTest/java/org/opensearch/security/api/DashboardsInfoTest.java @@ -12,7 +12,6 @@ package org.opensearch.security.api; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; -import org.apache.hc.core5.http.HttpStatus; import org.junit.ClassRule; import org.junit.Test; import org.junit.runner.RunWith; @@ -48,7 +47,6 @@ public void testDashboardsInfoValidationMessage() throws Exception { try (TestRestClient client = cluster.getRestClient(DASHBOARDS_USER)) { TestRestClient.HttpResponse response = client.get("_plugins/_security/dashboardsinfo"); - assertThat(response.getStatusCode(), equalTo(HttpStatus.SC_OK)); assertThat(response.getBody(), containsString("password_validation_error_message")); assertThat(response.getBody(), containsString(DEFAULT_PASSWORD_MESSAGE)); } diff --git a/src/integrationTest/java/org/opensearch/security/api/DashboardsInfoWithSettingsTest.java b/src/integrationTest/java/org/opensearch/security/api/DashboardsInfoWithSettingsTest.java index 01654e17cd..d100bb7c57 100644 --- a/src/integrationTest/java/org/opensearch/security/api/DashboardsInfoWithSettingsTest.java +++ b/src/integrationTest/java/org/opensearch/security/api/DashboardsInfoWithSettingsTest.java @@ -12,7 +12,6 @@ package org.opensearch.security.api; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; -import org.apache.hc.core5.http.HttpStatus; import org.junit.ClassRule; import org.junit.Test; import org.junit.runner.RunWith; @@ -60,7 +59,6 @@ public void testDashboardsInfoValidationMessageWithCustomMessage() throws Except try (TestRestClient client = cluster.getRestClient(DASHBOARDS_USER)) { TestRestClient.HttpResponse response = client.get("_plugins/_security/dashboardsinfo"); - assertThat(response.getStatusCode(), equalTo(HttpStatus.SC_OK)); assertThat(response.getBody(), containsString("password_validation_error_message")); assertThat(response.getBody(), containsString(CUSTOM_PASSWORD_MESSAGE)); } diff --git a/src/integrationTest/java/org/opensearch/security/http/AnonymousAuthenticationTest.java b/src/integrationTest/java/org/opensearch/security/http/AnonymousAuthenticationTest.java new file mode 100644 index 0000000000..b1c13aeedc --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/http/AnonymousAuthenticationTest.java @@ -0,0 +1,129 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.security.http; + +import java.util.List; + +import com.carrotsearch.randomizedtesting.RandomizedRunner; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.test.framework.RolesMapping; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; + +@RunWith(RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class AnonymousAuthenticationTest { + + private static final String DEFAULT_ANONYMOUS_USER_NAME = "opendistro_security_anonymous"; + private static final String DEFAULT_ANONYMOUS_USER_BACKEND_ROLE_NAME = "opendistro_security_anonymous_backendrole"; + + /** + * Custom role assigned to the anonymous user via {@link #ANONYMOUS_USER_CUSTOM_ROLE_MAPPING} + */ + private static final TestSecurityConfig.Role ANONYMOUS_USER_CUSTOM_ROLE = new TestSecurityConfig.Role("anonymous_user_custom_role"); + + /** + * Maps {@link #ANONYMOUS_USER_CUSTOM_ROLE} to {@link #DEFAULT_ANONYMOUS_USER_BACKEND_ROLE_NAME} + */ + private static final RolesMapping ANONYMOUS_USER_CUSTOM_ROLE_MAPPING = new RolesMapping(ANONYMOUS_USER_CUSTOM_ROLE).backendRoles( + DEFAULT_ANONYMOUS_USER_BACKEND_ROLE_NAME + ); + + /** + * User who is stored in the internal user database and can authenticate + */ + private static final TestSecurityConfig.User EXISTING_USER = new TestSecurityConfig.User("existing_user").roles( + new TestSecurityConfig.Role("existing_user") + ); + + /** + * User who is not stored in the internal user database and can not authenticate + */ + private static final TestSecurityConfig.User NOT_EXISTING_USER = new TestSecurityConfig.User("not_existing_user").roles( + new TestSecurityConfig.Role("not_existing_user") + ); + + @ClassRule + public static final LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.SINGLENODE) + .anonymousAuth(true) + .authc(AUTHC_HTTPBASIC_INTERNAL) + .users(EXISTING_USER) + .roles(ANONYMOUS_USER_CUSTOM_ROLE) + .rolesMapping(ANONYMOUS_USER_CUSTOM_ROLE_MAPPING) + .build(); + + private static final String USER_NAME_POINTER = "/user_name"; + private static final String BACKEND_ROLES_POINTER = "/backend_roles"; + private static final String ROLES_POINTER = "/roles"; + + @Test + public void shouldAuthenticate_positive_anonymousUser() { + try (TestRestClient client = cluster.getRestClient()) { + + TestRestClient.HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(200); + + String username = response.getTextFromJsonBody(USER_NAME_POINTER); + assertThat(username, equalTo(DEFAULT_ANONYMOUS_USER_NAME)); + + List backendRoles = response.getTextArrayFromJsonBody(BACKEND_ROLES_POINTER); + assertThat(backendRoles, hasSize(1)); + assertThat(backendRoles, contains(DEFAULT_ANONYMOUS_USER_BACKEND_ROLE_NAME)); + + List roles = response.getTextArrayFromJsonBody(ROLES_POINTER); + assertThat(roles, hasSize(1)); + assertThat(roles, contains(ANONYMOUS_USER_CUSTOM_ROLE.getName())); + } + } + + @Test + public void shouldAuthenticate_positive_existingUser() { + try (TestRestClient client = cluster.getRestClient(EXISTING_USER)) { + + TestRestClient.HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(200); + + String username = response.getTextFromJsonBody(USER_NAME_POINTER); + assertThat(username, equalTo(EXISTING_USER.getName())); + + List backendRoles = response.getTextArrayFromJsonBody(BACKEND_ROLES_POINTER); + assertThat(backendRoles, hasSize(0)); + + List roles = response.getTextArrayFromJsonBody(ROLES_POINTER); + assertThat(roles, hasSize(EXISTING_USER.getRoleNames().size())); + assertThat(roles, containsInAnyOrder(EXISTING_USER.getRoleNames().toArray())); + } + } + + @Test + public void shouldAuthenticate_negative_notExistingUser() { + try (TestRestClient client = cluster.getRestClient(NOT_EXISTING_USER)) { + + TestRestClient.HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(401); + } + } +} diff --git a/src/integrationTest/java/org/opensearch/security/http/AuthInfo.java b/src/integrationTest/java/org/opensearch/security/http/AuthInfo.java new file mode 100644 index 0000000000..53ea6ab859 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/http/AuthInfo.java @@ -0,0 +1,30 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.security.http; + +import java.beans.ConstructorProperties; +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +@JsonIgnoreProperties(ignoreUnknown = true) +class AuthInfo { + + private final List customAttributeNames; + + @ConstructorProperties("custom_attribute_names") + public AuthInfo(List customAttributeNames) { + this.customAttributeNames = customAttributeNames; + } + + public List getCustomAttributeNames() { + return customAttributeNames; + } +} diff --git a/src/integrationTest/java/org/opensearch/security/http/BasicAuthTests.java b/src/integrationTest/java/org/opensearch/security/http/BasicAuthTests.java new file mode 100644 index 0000000000..1e424ab115 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/http/BasicAuthTests.java @@ -0,0 +1,146 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.security.http; + +import java.util.List; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.apache.http.HttpHeaders; +import org.hamcrest.Matchers; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain; +import org.opensearch.test.framework.TestSecurityConfig.User; +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; + +import static org.apache.http.HttpStatus.SC_OK; +import static org.apache.http.HttpStatus.SC_UNAUTHORIZED; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsStringIgnoringCase; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class BasicAuthTests { + static final User TEST_USER = new User("test_user").password("s3cret"); + + public static final String CUSTOM_ATTRIBUTE_NAME = "superhero"; + static final User SUPER_USER = new User("super-user").password("super-password").attr(CUSTOM_ATTRIBUTE_NAME, "true"); + public static final String NOT_EXISTING_USER = "not-existing-user"; + public static final String INVALID_PASSWORD = "secret-password"; + + public static final AuthcDomain AUTHC_DOMAIN = new AuthcDomain("basic", 0).httpAuthenticatorWithChallenge("basic").backend("internal"); + + @ClassRule + public static final LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.SINGLENODE) + .anonymousAuth(false) + .authc(AUTHC_DOMAIN) + .users(TEST_USER, SUPER_USER) + .build(); + + @Test + public void shouldRespondWith401WhenUserDoesNotExist() { + try (TestRestClient client = cluster.getRestClient(NOT_EXISTING_USER, INVALID_PASSWORD)) { + HttpResponse response = client.getAuthInfo(); + + assertThat(response, is(notNullValue())); + response.assertStatusCode(SC_UNAUTHORIZED); + } + } + + @Test + public void shouldRespondWith401WhenUserNameIsIncorrect() { + try (TestRestClient client = cluster.getRestClient(NOT_EXISTING_USER, TEST_USER.getPassword())) { + HttpResponse response = client.getAuthInfo(); + + assertThat(response, is(notNullValue())); + response.assertStatusCode(SC_UNAUTHORIZED); + } + } + + @Test + public void shouldRespondWith401WhenPasswordIsIncorrect() { + try (TestRestClient client = cluster.getRestClient(TEST_USER.getName(), INVALID_PASSWORD)) { + HttpResponse response = client.getAuthInfo(); + + assertThat(response, is(notNullValue())); + response.assertStatusCode(SC_UNAUTHORIZED); + } + } + + @Test + public void shouldRespondWith200WhenCredentialsAreCorrect() { + try (TestRestClient client = cluster.getRestClient(TEST_USER)) { + + HttpResponse response = client.getAuthInfo(); + + assertThat(response, is(notNullValue())); + response.assertStatusCode(SC_OK); + } + } + + @Test + public void testBrowserShouldRequestForCredentials() { + try (TestRestClient client = cluster.getRestClient()) { + + HttpResponse response = client.getAuthInfo(); + + assertThat(response, is(notNullValue())); + response.assertStatusCode(SC_UNAUTHORIZED); + assertThatBrowserAskUserForCredentials(response); + } + } + + @Test + public void testUserShouldNotHaveAssignedCustomAttributes() { + try (TestRestClient client = cluster.getRestClient(TEST_USER)) { + + HttpResponse response = client.getAuthInfo(); + + assertThat(response, is(notNullValue())); + response.assertStatusCode(SC_OK); + AuthInfo authInfo = response.getBodyAs(AuthInfo.class); + assertThat(authInfo, is(notNullValue())); + assertThat(authInfo.getCustomAttributeNames(), is(notNullValue())); + assertThat(authInfo.getCustomAttributeNames(), hasSize(0)); + } + } + + @Test + public void testUserShouldHaveAssignedCustomAttributes() { + try (TestRestClient client = cluster.getRestClient(SUPER_USER)) { + + HttpResponse response = client.getAuthInfo(); + + assertThat(response, is(notNullValue())); + response.assertStatusCode(SC_OK); + AuthInfo authInfo = response.getBodyAs(AuthInfo.class); + assertThat(authInfo, is(notNullValue())); + List customAttributeNames = authInfo.getCustomAttributeNames(); + assertThat(customAttributeNames, is(notNullValue())); + assertThat(customAttributeNames, hasSize(1)); + assertThat(customAttributeNames.get(0), Matchers.equalTo("attr.internal." + CUSTOM_ATTRIBUTE_NAME)); + } + } + + private void assertThatBrowserAskUserForCredentials(HttpResponse response) { + String reason = "Browser does not ask user for credentials"; + assertThat(reason, response.containHeader(HttpHeaders.WWW_AUTHENTICATE), equalTo(true)); + assertThat(response.getHeader(HttpHeaders.WWW_AUTHENTICATE).getValue(), containsStringIgnoringCase("basic")); + } +} diff --git a/src/integrationTest/java/org/opensearch/security/http/BasicAuthWithoutChallengeTests.java b/src/integrationTest/java/org/opensearch/security/http/BasicAuthWithoutChallengeTests.java new file mode 100644 index 0000000000..d602d0920b --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/http/BasicAuthWithoutChallengeTests.java @@ -0,0 +1,52 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.security.http; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.apache.http.HttpHeaders; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL_WITHOUT_CHALLENGE; + +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class BasicAuthWithoutChallengeTests { + + @ClassRule + public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.SINGLENODE) + .anonymousAuth(false) + .authc(AUTHC_HTTPBASIC_INTERNAL_WITHOUT_CHALLENGE) + .build(); + + @Test + public void browserShouldNotRequestUserForCredentials() { + try (TestRestClient client = cluster.getRestClient()) { + + HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(401); + assertThatBrowserDoesNotAskUserForCredentials(response); + } + } + + private void assertThatBrowserDoesNotAskUserForCredentials(HttpResponse response) { + String reason = "Browser asked user for credentials which is not expected"; + assertThat(reason, response.containHeader(HttpHeaders.WWW_AUTHENTICATE), equalTo(false)); + } +} diff --git a/src/integrationTest/java/org/opensearch/security/http/CertificateAuthenticationTest.java b/src/integrationTest/java/org/opensearch/security/http/CertificateAuthenticationTest.java new file mode 100644 index 0000000000..975ce25efb --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/http/CertificateAuthenticationTest.java @@ -0,0 +1,148 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.security.http; + +import java.util.List; +import java.util.Map; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.test.framework.RolesMapping; +import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain; +import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.HttpAuthenticator; +import org.opensearch.test.framework.TestSecurityConfig.Role; +import org.opensearch.test.framework.TestSecurityConfig.User; +import org.opensearch.test.framework.certificate.CertificateData; +import org.opensearch.test.framework.certificate.TestCertificates; +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; + +import static org.apache.http.HttpStatus.SC_OK; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.hasSize; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.TestSecurityConfig.Role.ALL_ACCESS; + +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class CertificateAuthenticationTest { + + private static final User USER_ADMIN = new User("admin").roles(ALL_ACCESS); + + public static final String POINTER_BACKEND_ROLES = "/backend_roles"; + public static final String POINTER_ROLES = "/roles"; + + private static final String USER_SPOCK = "spock"; + private static final String USER_KIRK = "kirk"; + + private static final String BACKEND_ROLE_BRIDGE = "bridge"; + private static final String BACKEND_ROLE_CAPTAIN = "captain"; + + private static final Role ROLE_ALL_INDEX_SEARCH = new Role("all-index-search").indexPermissions("indices:data/read/search").on("*"); + + private static final Map CERT_AUTH_CONFIG = Map.of("username_attribute", "cn", "roles_attribute", "ou"); + + @ClassRule + public static final LocalCluster cluster = new LocalCluster.Builder().nodeSettings( + Map.of("plugins.security.ssl.http.clientauth_mode", "OPTIONAL") + ) + .clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS) + .anonymousAuth(false) + .authc( + new AuthcDomain("clientcert_auth_domain", -1, true).httpAuthenticator( + new HttpAuthenticator("clientcert").challenge(false).config(CERT_AUTH_CONFIG) + ).backend("noop") + ) + .authc(AUTHC_HTTPBASIC_INTERNAL) + .roles(ROLE_ALL_INDEX_SEARCH) + .users(USER_ADMIN) + .rolesMapping(new RolesMapping(ROLE_ALL_INDEX_SEARCH).backendRoles(BACKEND_ROLE_BRIDGE)) + .build(); + + private static final TestCertificates TEST_CERTIFICATES = cluster.getTestCertificates(); + + @Test + public void shouldAuthenticateUserWithBasicAuthWhenCertificateAuthenticationIsConfigured() { + try (TestRestClient client = cluster.getRestClient(USER_ADMIN)) { + + HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(SC_OK); + } + } + + @Test + public void shouldAuthenticateUserWithCertificate_positiveUserSpoke() { + CertificateData userSpockCertificate = TEST_CERTIFICATES.issueUserCertificate(BACKEND_ROLE_BRIDGE, USER_SPOCK); + try (TestRestClient client = cluster.getRestClient(userSpockCertificate)) { + + client.confirmCorrectCredentials(USER_SPOCK); + } + } + + @Test + public void shouldAuthenticateUserWithCertificate_positiveUserKirk() { + CertificateData userSpockCertificate = TEST_CERTIFICATES.issueUserCertificate(BACKEND_ROLE_BRIDGE, USER_KIRK); + try (TestRestClient client = cluster.getRestClient(userSpockCertificate)) { + + client.confirmCorrectCredentials(USER_KIRK); + } + } + + @Test + public void shouldAuthenticateUserWithCertificate_negative() { + CertificateData untrustedUserCertificate = TEST_CERTIFICATES.createSelfSignedCertificate("CN=untrusted"); + try (TestRestClient client = cluster.getRestClient(untrustedUserCertificate)) { + + HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(401); + } + } + + @Test + public void shouldRetrieveBackendRoleFromCertificate_positiveRoleBridge() { + CertificateData userSpockCertificate = TEST_CERTIFICATES.issueUserCertificate(BACKEND_ROLE_BRIDGE, USER_KIRK); + try (TestRestClient client = cluster.getRestClient(userSpockCertificate)) { + + HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(200); + List backendRoles = response.getTextArrayFromJsonBody(POINTER_BACKEND_ROLES); + assertThat(backendRoles, hasSize(1)); + assertThat(backendRoles, containsInAnyOrder(BACKEND_ROLE_BRIDGE)); + List roles = response.getTextArrayFromJsonBody(POINTER_ROLES); + assertThat(roles, hasSize(1)); + assertThat(roles, containsInAnyOrder(ROLE_ALL_INDEX_SEARCH.getName())); + } + } + + @Test + public void shouldRetrieveBackendRoleFromCertificate_positiveRoleCaptain() { + CertificateData userSpockCertificate = TEST_CERTIFICATES.issueUserCertificate(BACKEND_ROLE_CAPTAIN, USER_KIRK); + try (TestRestClient client = cluster.getRestClient(userSpockCertificate)) { + + HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(200); + List backendRoles = response.getTextArrayFromJsonBody(POINTER_BACKEND_ROLES); + assertThat(backendRoles, hasSize(1)); + assertThat(backendRoles, containsInAnyOrder(BACKEND_ROLE_CAPTAIN)); + List roles = response.getTextArrayFromJsonBody(POINTER_ROLES); + assertThat(roles, hasSize(0)); + } + } +} diff --git a/src/integrationTest/java/org/opensearch/security/http/CommonProxyAuthenticationTests.java b/src/integrationTest/java/org/opensearch/security/http/CommonProxyAuthenticationTests.java new file mode 100644 index 0000000000..49ded4f2a9 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/http/CommonProxyAuthenticationTests.java @@ -0,0 +1,255 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.security.http; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.List; + +import org.opensearch.test.framework.RolesMapping; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.cluster.TestRestClientConfiguration; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.opensearch.test.framework.TestSecurityConfig.Role.ALL_ACCESS; + +/** +* Class defines common tests for proxy and extended-proxy authentication. Subclasses are used to run tests. +*/ +abstract class CommonProxyAuthenticationTests { + + protected static final String RESOURCE_AUTH_INFO = "/_opendistro/_security/authinfo"; + protected static final TestSecurityConfig.User USER_ADMIN = new TestSecurityConfig.User("admin").roles(ALL_ACCESS); + + protected static final String ATTRIBUTE_DEPARTMENT = "department"; + protected static final String ATTRIBUTE_SKILLS = "skills"; + + protected static final String USER_ATTRIBUTE_DEPARTMENT_NAME = "attr.proxy." + ATTRIBUTE_DEPARTMENT; + protected static final String USER_ATTRIBUTE_SKILLS_NAME = "attr.proxy." + ATTRIBUTE_SKILLS; + protected static final String USER_ATTRIBUTE_USERNAME_NAME = "attr.proxy.username"; + + protected static final String HEADER_PREFIX_CUSTOM_ATTRIBUTES = "x-custom-attr"; + protected static final String HEADER_PROXY_USER = "x-proxy-user"; + protected static final String HEADER_PROXY_ROLES = "x-proxy-roles"; + protected static final String HEADER_FORWARDED_FOR = "X-Forwarded-For"; + protected static final String HEADER_DEPARTMENT = HEADER_PREFIX_CUSTOM_ATTRIBUTES + ATTRIBUTE_DEPARTMENT; + protected static final String HEADER_SKILLS = HEADER_PREFIX_CUSTOM_ATTRIBUTES + ATTRIBUTE_SKILLS; + + protected static final String IP_PROXY = "127.0.0.10"; + protected static final String IP_NON_PROXY = "127.0.0.5"; + protected static final String IP_CLIENT = "127.0.0.1"; + + protected static final String USER_KIRK = "kirk"; + protected static final String USER_SPOCK = "spock"; + + protected static final String BACKEND_ROLE_FIRST_MATE = "firstMate"; + protected static final String BACKEND_ROLE_CAPTAIN = "captain"; + protected static final String DEPARTMENT_BRIDGE = "bridge"; + + protected static final String PERSONAL_INDEX_NAME_PATTERN = "personal-${" + + USER_ATTRIBUTE_DEPARTMENT_NAME + + "}-${" + + USER_ATTRIBUTE_USERNAME_NAME + + "}"; + protected static final String PERSONAL_INDEX_NAME_SPOCK = "personal-" + DEPARTMENT_BRIDGE + "-" + USER_SPOCK; + protected static final String PERSONAL_INDEX_NAME_KIRK = "personal-" + DEPARTMENT_BRIDGE + "-" + USER_KIRK; + + protected static final String POINTER_USERNAME = "/user_name"; + protected static final String POINTER_BACKEND_ROLES = "/backend_roles"; + protected static final String POINTER_ROLES = "/roles"; + protected static final String POINTER_CUSTOM_ATTRIBUTES = "/custom_attribute_names"; + protected static final String POINTER_TOTAL_HITS = "/hits/total/value"; + protected static final String POINTER_FIRST_DOCUMENT_ID = "/hits/hits/0/_id"; + protected static final String POINTER_FIRST_DOCUMENT_INDEX = "/hits/hits/0/_index"; + protected static final String POINTER_FIRST_DOCUMENT_SOURCE_TITLE = "/hits/hits/0/_source/title"; + + protected static final TestSecurityConfig.Role ROLE_ALL_INDEX_SEARCH = new TestSecurityConfig.Role("all-index-search").indexPermissions( + "indices:data/read/search" + ).on("*"); + + protected static final TestSecurityConfig.Role ROLE_PERSONAL_INDEX_SEARCH = new TestSecurityConfig.Role("personal-index-search") + .indexPermissions("indices:data/read/search") + .on(PERSONAL_INDEX_NAME_PATTERN); + + protected static final RolesMapping ROLES_MAPPING_CAPTAIN = new RolesMapping(ROLE_PERSONAL_INDEX_SEARCH).backendRoles( + BACKEND_ROLE_CAPTAIN + ); + + protected static final RolesMapping ROLES_MAPPING_FIRST_MATE = new RolesMapping(ROLE_ALL_INDEX_SEARCH).backendRoles( + BACKEND_ROLE_FIRST_MATE + ); + + protected abstract LocalCluster getCluster(); + + protected void shouldAuthenticateWithBasicAuthWhenProxyAuthenticationIsConfigured() { + try (TestRestClient client = getCluster().getRestClient(USER_ADMIN)) { + TestRestClient.HttpResponse response = client.get(RESOURCE_AUTH_INFO); + + response.assertStatusCode(200); + } + } + + protected void shouldAuthenticateWithProxy_positiveUserKirk() throws IOException { + TestRestClientConfiguration testRestClientConfiguration = new TestRestClientConfiguration().sourceInetAddress( + InetAddress.getByName(IP_PROXY) + ).header(HEADER_FORWARDED_FOR, IP_CLIENT).header(HEADER_PROXY_USER, USER_KIRK).header(HEADER_PROXY_ROLES, BACKEND_ROLE_CAPTAIN); + try (TestRestClient client = getCluster().createGenericClientRestClient(testRestClientConfiguration)) { + + TestRestClient.HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(200); + String username = response.getTextFromJsonBody(POINTER_USERNAME); + assertThat(username, equalTo(USER_KIRK)); + } + } + + protected void shouldAuthenticateWithProxy_positiveUserSpock() throws IOException { + TestRestClientConfiguration testRestClientConfiguration = new TestRestClientConfiguration().sourceInetAddress( + InetAddress.getByName(IP_PROXY) + ).header(HEADER_FORWARDED_FOR, IP_CLIENT).header(HEADER_PROXY_USER, USER_SPOCK).header(HEADER_PROXY_ROLES, BACKEND_ROLE_FIRST_MATE); + try (TestRestClient client = getCluster().createGenericClientRestClient(testRestClientConfiguration)) { + + TestRestClient.HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(200); + String username = response.getTextFromJsonBody(POINTER_USERNAME); + assertThat(username, equalTo(USER_SPOCK)); + } + } + + protected void shouldAuthenticateWithProxy_negativeWhenXffHeaderIsMissing() throws IOException { + TestRestClientConfiguration testRestClientConfiguration = new TestRestClientConfiguration().sourceInetAddress( + InetAddress.getByName(IP_PROXY) + ).header(HEADER_PROXY_USER, USER_KIRK).header(HEADER_PROXY_ROLES, BACKEND_ROLE_CAPTAIN); + try (TestRestClient client = getCluster().createGenericClientRestClient(testRestClientConfiguration)) { + + TestRestClient.HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(401); + } + } + + protected void shouldAuthenticateWithProxy_negativeWhenUserNameHeaderIsMissing() throws IOException { + TestRestClientConfiguration testRestClientConfiguration = new TestRestClientConfiguration().sourceInetAddress( + InetAddress.getByName(IP_PROXY) + ).header(HEADER_FORWARDED_FOR, IP_CLIENT).header(HEADER_PROXY_ROLES, BACKEND_ROLE_CAPTAIN); + try (TestRestClient client = getCluster().createGenericClientRestClient(testRestClientConfiguration)) { + + TestRestClient.HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(401); + } + } + + protected void shouldAuthenticateWithProxyWhenRolesHeaderIsMissing() throws IOException { + TestRestClientConfiguration testRestClientConfiguration = new TestRestClientConfiguration().sourceInetAddress( + InetAddress.getByName(IP_PROXY) + ).header(HEADER_FORWARDED_FOR, IP_CLIENT).header(HEADER_PROXY_USER, USER_KIRK); + try (TestRestClient client = getCluster().createGenericClientRestClient(testRestClientConfiguration)) { + + TestRestClient.HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(200); + String username = response.getTextFromJsonBody(POINTER_USERNAME); + assertThat(username, equalTo(USER_KIRK)); + } + } + + protected void shouldAuthenticateWithProxy_negativeWhenRequestWasNotSendByProxy() throws IOException { + TestRestClientConfiguration testRestClientConfiguration = new TestRestClientConfiguration().sourceInetAddress( + InetAddress.getByName(IP_NON_PROXY) + ).header(HEADER_FORWARDED_FOR, IP_CLIENT).header(HEADER_PROXY_USER, USER_KIRK); + try (TestRestClient client = getCluster().createGenericClientRestClient(testRestClientConfiguration)) { + + TestRestClient.HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(401); + } + } + + protected void shouldRetrieveEmptyListOfRoles() throws IOException { + TestRestClientConfiguration testRestClientConfiguration = new TestRestClientConfiguration().sourceInetAddress( + InetAddress.getByName(IP_PROXY) + ).header(HEADER_FORWARDED_FOR, IP_CLIENT).header(HEADER_PROXY_USER, USER_SPOCK); + try (TestRestClient client = getCluster().createGenericClientRestClient(testRestClientConfiguration)) { + + TestRestClient.HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(200); + List backendRoles = response.getTextArrayFromJsonBody(POINTER_BACKEND_ROLES); + assertThat(backendRoles, hasSize(0)); + List roles = response.getTextArrayFromJsonBody(POINTER_ROLES); + assertThat(roles, hasSize(0)); + } + } + + protected void shouldRetrieveSingleRoleFirstMate() throws IOException { + TestRestClientConfiguration testRestClientConfiguration = new TestRestClientConfiguration().sourceInetAddress( + InetAddress.getByName(IP_PROXY) + ).header(HEADER_FORWARDED_FOR, IP_CLIENT).header(HEADER_PROXY_USER, USER_SPOCK).header(HEADER_PROXY_ROLES, BACKEND_ROLE_FIRST_MATE); + try (TestRestClient client = getCluster().createGenericClientRestClient(testRestClientConfiguration)) { + + TestRestClient.HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(200); + List backendRoles = response.getTextArrayFromJsonBody(POINTER_BACKEND_ROLES); + assertThat(backendRoles, hasSize(1)); + assertThat(backendRoles, contains(BACKEND_ROLE_FIRST_MATE)); + List roles = response.getTextArrayFromJsonBody(POINTER_ROLES); + assertThat(roles, hasSize(1)); + assertThat(roles, contains(ROLE_ALL_INDEX_SEARCH.getName())); + } + } + + protected void shouldRetrieveSingleRoleCaptain() throws IOException { + TestRestClientConfiguration testRestClientConfiguration = new TestRestClientConfiguration().sourceInetAddress( + InetAddress.getByName(IP_PROXY) + ).header(HEADER_FORWARDED_FOR, IP_CLIENT).header(HEADER_PROXY_USER, USER_SPOCK).header(HEADER_PROXY_ROLES, BACKEND_ROLE_CAPTAIN); + try (TestRestClient client = getCluster().createGenericClientRestClient(testRestClientConfiguration)) { + + TestRestClient.HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(200); + List backendRoles = response.getTextArrayFromJsonBody(POINTER_BACKEND_ROLES); + assertThat(backendRoles, hasSize(1)); + assertThat(backendRoles, contains(BACKEND_ROLE_CAPTAIN)); + List roles = response.getTextArrayFromJsonBody(POINTER_ROLES); + assertThat(roles, hasSize(1)); + assertThat(roles, contains(ROLE_PERSONAL_INDEX_SEARCH.getName())); + } + } + + protected void shouldRetrieveMultipleRoles() throws IOException { + TestRestClientConfiguration testRestClientConfiguration = new TestRestClientConfiguration().sourceInetAddress( + InetAddress.getByName(IP_PROXY) + ) + .header(HEADER_FORWARDED_FOR, IP_CLIENT) + .header(HEADER_PROXY_USER, USER_SPOCK) + .header(HEADER_PROXY_ROLES, BACKEND_ROLE_CAPTAIN + "," + BACKEND_ROLE_FIRST_MATE); + try (TestRestClient client = getCluster().createGenericClientRestClient(testRestClientConfiguration)) { + + TestRestClient.HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(200); + List backendRoles = response.getTextArrayFromJsonBody(POINTER_BACKEND_ROLES); + assertThat(backendRoles, hasSize(2)); + assertThat(backendRoles, containsInAnyOrder(BACKEND_ROLE_CAPTAIN, BACKEND_ROLE_FIRST_MATE)); + List roles = response.getTextArrayFromJsonBody(POINTER_ROLES); + assertThat(roles, hasSize(2)); + assertThat(roles, containsInAnyOrder(ROLE_PERSONAL_INDEX_SEARCH.getName(), ROLE_ALL_INDEX_SEARCH.getName())); + } + } +} diff --git a/src/integrationTest/java/org/opensearch/security/http/DirectoryInformationTrees.java b/src/integrationTest/java/org/opensearch/security/http/DirectoryInformationTrees.java new file mode 100644 index 0000000000..3f9c220923 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/http/DirectoryInformationTrees.java @@ -0,0 +1,123 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.security.http; + +import org.opensearch.test.framework.ldap.LdifBuilder; +import org.opensearch.test.framework.ldap.LdifData; + +class DirectoryInformationTrees { + + public static final String DN_PEOPLE_TEST_ORG = "ou=people,o=test.org"; + public static final String DN_OPEN_SEARCH_PEOPLE_TEST_ORG = "cn=Open Search,ou=people,o=test.org"; + public static final String DN_CHRISTPHER_PEOPLE_TEST_ORG = "cn=Christpher,ou=people,o=test.org"; + public static final String DN_KIRK_PEOPLE_TEST_ORG = "cn=Kirk,ou=people,o=test.org"; + public static final String DN_CAPTAIN_SPOCK_PEOPLE_TEST_ORG = "cn=Captain Spock,ou=people,o=test.org"; + public static final String DN_LEONARD_PEOPLE_TEST_ORG = "cn=Leonard,ou=people,o=test.org"; + public static final String DN_JEAN_PEOPLE_TEST_ORG = "cn=Jean,ou=people,o=test.org"; + public static final String DN_GROUPS_TEST_ORG = "ou=groups,o=test.org"; + public static final String DN_BRIDGE_GROUPS_TEST_ORG = "cn=bridge,ou=groups,o=test.org"; + + public static final String USER_KIRK = "kirk"; + public static final String PASSWORD_KIRK = "kirk-secret"; + public static final String USER_SPOCK = "spock"; + public static final String PASSWORD_SPOCK = "spocksecret"; + public static final String USER_OPENS = "opens"; + public static final String PASSWORD_OPEN_SEARCH = "open_search-secret"; + public static final String USER_JEAN = "jean"; + public static final String PASSWORD_JEAN = "jeansecret"; + public static final String USER_LEONARD = "leonard"; + public static final String PASSWORD_LEONARD = "Leonard-secret"; + public static final String PASSWORD_CHRISTPHER = "christpher_secret"; + + public static final String CN_GROUP_ADMIN = "admin"; + public static final String CN_GROUP_CREW = "crew"; + public static final String CN_GROUP_BRIDGE = "bridge"; + + public static final String USER_SEARCH = "(uid={0})"; + public static final String USERNAME_ATTRIBUTE = "uid"; + + static final LdifData LDIF_DATA = new LdifBuilder().root("o=test.org") + .dc("TEST") + .classes("top", "domain") + .newRecord(DN_PEOPLE_TEST_ORG) + .ou("people") + .classes("organizationalUnit", "top") + .newRecord(DN_OPEN_SEARCH_PEOPLE_TEST_ORG) + .classes("inetOrgPerson") + .cn("Open Search") + .sn("Search") + .uid(USER_OPENS) + .userPassword(PASSWORD_OPEN_SEARCH) + .mail("open.search@example.com") + .ou("Human Resources") + .newRecord(DN_CAPTAIN_SPOCK_PEOPLE_TEST_ORG) + .classes("inetOrgPerson") + .cn("Captain Spock") + .sn(USER_SPOCK) + .uid(USER_SPOCK) + .userPassword(PASSWORD_SPOCK) + .mail("spock@example.com") + .ou("Human Resources") + .newRecord(DN_KIRK_PEOPLE_TEST_ORG) + .classes("inetOrgPerson") + .cn("Kirk") + .sn("Kirk") + .uid(USER_KIRK) + .userPassword(PASSWORD_KIRK) + .mail("spock@example.com") + .ou("Human Resources") + .newRecord(DN_CHRISTPHER_PEOPLE_TEST_ORG) + .classes("inetOrgPerson") + .cn("Christpher") + .sn("Christpher") + .uid("christpher") + .userPassword(PASSWORD_CHRISTPHER) + .mail("christpher@example.com") + .ou("Human Resources") + .newRecord(DN_LEONARD_PEOPLE_TEST_ORG) + .classes("inetOrgPerson") + .cn("Leonard") + .sn("Leonard") + .uid(USER_LEONARD) + .userPassword(PASSWORD_LEONARD) + .mail("leonard@example.com") + .ou("Human Resources") + .newRecord(DN_JEAN_PEOPLE_TEST_ORG) + .classes("inetOrgPerson") + .cn("Jean") + .sn("Jean") + .uid(USER_JEAN) + .userPassword(PASSWORD_JEAN) + .mail("jean@example.com") + .ou("Human Resources") + .newRecord(DN_GROUPS_TEST_ORG) + .ou("groups") + .cn("groupsRoot") + .classes("groupofuniquenames", "top") + .newRecord("cn=admin,ou=groups,o=test.org") + .ou("groups") + .cn(CN_GROUP_ADMIN) + .uniqueMember(DN_KIRK_PEOPLE_TEST_ORG) + .classes("groupofuniquenames", "top") + .newRecord("cn=crew,ou=groups,o=test.org") + .ou("groups") + .cn(CN_GROUP_CREW) + .uniqueMember(DN_CAPTAIN_SPOCK_PEOPLE_TEST_ORG) + .uniqueMember(DN_CHRISTPHER_PEOPLE_TEST_ORG) + .uniqueMember(DN_BRIDGE_GROUPS_TEST_ORG) + .classes("groupofuniquenames", "top") + .newRecord(DN_BRIDGE_GROUPS_TEST_ORG) + .ou("groups") + .cn(CN_GROUP_BRIDGE) + .uniqueMember(DN_JEAN_PEOPLE_TEST_ORG) + .classes("groupofuniquenames", "top") + .buildRecord() + .buildLdif(); +} diff --git a/src/integrationTest/java/org/opensearch/security/http/JwtAuthorizationHeaderFactory.java b/src/integrationTest/java/org/opensearch/security/http/JwtAuthorizationHeaderFactory.java new file mode 100644 index 0000000000..dd3a165d92 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/http/JwtAuthorizationHeaderFactory.java @@ -0,0 +1,141 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.security.http; + +import java.security.PrivateKey; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; + +import com.google.common.collect.ImmutableMap; +import io.jsonwebtoken.Jwts; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.Header; +import org.apache.http.message.BasicHeader; + +import static io.jsonwebtoken.SignatureAlgorithm.RS256; +import static java.util.Objects.requireNonNull; + +class JwtAuthorizationHeaderFactory { + public static final String AUDIENCE = "OpenSearch"; + public static final String ISSUER = "test-code"; + private final PrivateKey privateKey; + + private final String usernameClaimName; + + private final String rolesClaimName; + + private final String headerName; + + public JwtAuthorizationHeaderFactory(PrivateKey privateKey, String usernameClaimName, String rolesClaimName, String headerName) { + this.privateKey = requireNonNull(privateKey, "Private key is required"); + this.usernameClaimName = requireNonNull(usernameClaimName, "Username claim name is required"); + this.rolesClaimName = requireNonNull(rolesClaimName, "Roles claim name is required."); + this.headerName = requireNonNull(headerName, "Header name is required"); + } + + Header generateValidToken(String username, String... roles) { + requireNonNull(username, "Username is required"); + Date now = new Date(); + String token = Jwts.builder() + .setClaims(customClaimsMap(username, roles)) + .setIssuer(ISSUER) + .setSubject(subject(username)) + .setAudience(AUDIENCE) + .setIssuedAt(now) + .setExpiration(new Date(now.getTime() + 3600 * 1000)) + .signWith(privateKey, RS256) + .compact(); + return toHeader(token); + } + + private Map customClaimsMap(String username, String[] roles) { + ImmutableMap.Builder builder = new ImmutableMap.Builder(); + if (StringUtils.isNoneEmpty(username)) { + builder.put(usernameClaimName, username); + } + if ((roles != null) && (roles.length > 0)) { + builder.put(rolesClaimName, Arrays.stream(roles).collect(Collectors.joining(","))); + } + return builder.build(); + } + + Header generateValidTokenWithCustomClaims(String username, String[] roles, Map additionalClaims) { + requireNonNull(username, "Username is required"); + requireNonNull(additionalClaims, "Custom claims are required"); + Map claims = new HashMap<>(customClaimsMap(username, roles)); + claims.putAll(additionalClaims); + Date now = new Date(); + String token = Jwts.builder() + .setClaims(claims) + .setIssuer(ISSUER) + .setSubject(subject(username)) + .setAudience(AUDIENCE) + .setIssuedAt(now) + .setExpiration(new Date(now.getTime() + 3600 * 1000)) + .signWith(privateKey, RS256) + .compact(); + return toHeader(token); + } + + private BasicHeader toHeader(String token) { + return new BasicHeader(headerName, token); + } + + Header generateTokenWithoutPreferredUsername(String username) { + requireNonNull(username, "Username is required"); + Date now = new Date(); + String token = Jwts.builder() + .setIssuer(ISSUER) + .setSubject(username) + .setIssuedAt(now) + .setExpiration(new Date(now.getTime() + 3600 * 1000)) + .signWith(privateKey, RS256) + .compact(); + return toHeader(token); + } + + public Header generateExpiredToken(String username) { + requireNonNull(username, "Username is required"); + Date now = new Date(1000); + String token = Jwts.builder() + .setClaims(Map.of(usernameClaimName, username)) + .setIssuer(ISSUER) + .setSubject(subject(username)) + .setAudience(AUDIENCE) + .setIssuedAt(now) + .setExpiration(new Date(now.getTime() + 3600 * 1000)) + .signWith(privateKey, RS256) + .compact(); + return toHeader(token); + } + + public Header generateTokenSignedWithKey(PrivateKey key, String username) { + requireNonNull(key, "Private key is required"); + requireNonNull(username, "Username is required"); + Date now = new Date(); + String token = Jwts.builder() + .setClaims(Map.of(usernameClaimName, username)) + .setIssuer(ISSUER) + .setSubject(subject(username)) + .setAudience(AUDIENCE) + .setIssuedAt(now) + .setExpiration(new Date(now.getTime() + 3600 * 1000)) + .signWith(key, RS256) + .compact(); + return toHeader(token); + } + + private static String subject(String username) { + return "subject-" + username; + } +} diff --git a/src/integrationTest/java/org/opensearch/security/http/LdapAuthenticationTest.java b/src/integrationTest/java/org/opensearch/security/http/LdapAuthenticationTest.java new file mode 100644 index 0000000000..299b2cc7d2 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/http/LdapAuthenticationTest.java @@ -0,0 +1,120 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.security.http; + +import java.util.List; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.RuleChain; +import org.junit.runner.RunWith; + +import org.opensearch.test.framework.LdapAuthenticationConfigBuilder; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain; +import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AuthenticationBackend; +import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.HttpAuthenticator; +import org.opensearch.test.framework.certificate.TestCertificates; +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.ldap.EmbeddedLDAPServer; +import org.opensearch.test.framework.log.LogsRule; + +import static org.opensearch.security.http.DirectoryInformationTrees.DN_CAPTAIN_SPOCK_PEOPLE_TEST_ORG; +import static org.opensearch.security.http.DirectoryInformationTrees.DN_OPEN_SEARCH_PEOPLE_TEST_ORG; +import static org.opensearch.security.http.DirectoryInformationTrees.DN_PEOPLE_TEST_ORG; +import static org.opensearch.security.http.DirectoryInformationTrees.LDIF_DATA; +import static org.opensearch.security.http.DirectoryInformationTrees.PASSWORD_OPEN_SEARCH; +import static org.opensearch.security.http.DirectoryInformationTrees.PASSWORD_SPOCK; +import static org.opensearch.security.http.DirectoryInformationTrees.USERNAME_ATTRIBUTE; +import static org.opensearch.security.http.DirectoryInformationTrees.USER_SEARCH; +import static org.opensearch.security.http.DirectoryInformationTrees.USER_SPOCK; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.BASIC_AUTH_DOMAIN_ORDER; +import static org.opensearch.test.framework.TestSecurityConfig.Role.ALL_ACCESS; + +/** +* Test uses plain (non TLS) connection between OpenSearch and LDAP server. +*/ +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class LdapAuthenticationTest { + + private static final Logger log = LogManager.getLogger(LdapAuthenticationTest.class); + + private static final TestSecurityConfig.User ADMIN_USER = new TestSecurityConfig.User("admin").roles(ALL_ACCESS); + + private static final TestCertificates TEST_CERTIFICATES = new TestCertificates(); + + public static final EmbeddedLDAPServer embeddedLDAPServer = new EmbeddedLDAPServer( + TEST_CERTIFICATES.getRootCertificateData(), + TEST_CERTIFICATES.getLdapCertificateData(), + LDIF_DATA + ); + + public static LocalCluster cluster = new LocalCluster.Builder().testCertificates(TEST_CERTIFICATES) + .clusterManager(ClusterManager.SINGLENODE) + .anonymousAuth(false) + .authc( + new AuthcDomain("ldap", BASIC_AUTH_DOMAIN_ORDER + 1, true).httpAuthenticator(new HttpAuthenticator("basic").challenge(false)) + .backend( + new AuthenticationBackend("ldap").config( + () -> LdapAuthenticationConfigBuilder.config() + // this port is available when embeddedLDAPServer is already started, therefore Supplier interface is used to + // postpone + // execution of the code in this block. + .enableSsl(false) + .enableStartTls(false) + .hosts(List.of("localhost:" + embeddedLDAPServer.getLdapNonTlsPort())) + .bindDn(DN_OPEN_SEARCH_PEOPLE_TEST_ORG) + .password(PASSWORD_OPEN_SEARCH) + .userBase(DN_PEOPLE_TEST_ORG) + .userSearch(USER_SEARCH) + .usernameAttribute(USERNAME_ATTRIBUTE) + .build() + ) + ) + ) + .authc(AUTHC_HTTPBASIC_INTERNAL) + .users(ADMIN_USER) + .build(); + + @ClassRule + public static RuleChain ruleChain = RuleChain.outerRule(embeddedLDAPServer).around(cluster); + + @Rule + public LogsRule logsRule = new LogsRule("com.amazon.dlic.auth.ldap.backend.LDAPAuthenticationBackend"); + + @Test + public void shouldAuthenticateUserWithLdap_positive() { + try (TestRestClient client = cluster.getRestClient(USER_SPOCK, PASSWORD_SPOCK)) { + TestRestClient.HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(200); + } + } + + @Test + public void shouldAuthenticateUserWithLdap_negativeWhenIncorrectPassword() { + try (TestRestClient client = cluster.getRestClient(USER_SPOCK, "incorrect password")) { + TestRestClient.HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(401); + String expectedStackTraceFragment = "Unable to bind as user '".concat(DN_CAPTAIN_SPOCK_PEOPLE_TEST_ORG) + .concat("' because the provided password was incorrect."); + logsRule.assertThatStackTraceContain(expectedStackTraceFragment); + } + } +} diff --git a/src/integrationTest/java/org/opensearch/security/http/LdapStartTlsAuthenticationTest.java b/src/integrationTest/java/org/opensearch/security/http/LdapStartTlsAuthenticationTest.java new file mode 100644 index 0000000000..395467897d --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/http/LdapStartTlsAuthenticationTest.java @@ -0,0 +1,118 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.security.http; + +import java.util.List; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.RuleChain; +import org.junit.runner.RunWith; + +import org.opensearch.test.framework.LdapAuthenticationConfigBuilder; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain; +import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AuthenticationBackend; +import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.HttpAuthenticator; +import org.opensearch.test.framework.certificate.TestCertificates; +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.ldap.EmbeddedLDAPServer; +import org.opensearch.test.framework.log.LogsRule; + +import static org.opensearch.security.http.DirectoryInformationTrees.DN_CAPTAIN_SPOCK_PEOPLE_TEST_ORG; +import static org.opensearch.security.http.DirectoryInformationTrees.DN_OPEN_SEARCH_PEOPLE_TEST_ORG; +import static org.opensearch.security.http.DirectoryInformationTrees.DN_PEOPLE_TEST_ORG; +import static org.opensearch.security.http.DirectoryInformationTrees.LDIF_DATA; +import static org.opensearch.security.http.DirectoryInformationTrees.PASSWORD_OPEN_SEARCH; +import static org.opensearch.security.http.DirectoryInformationTrees.PASSWORD_SPOCK; +import static org.opensearch.security.http.DirectoryInformationTrees.USERNAME_ATTRIBUTE; +import static org.opensearch.security.http.DirectoryInformationTrees.USER_SEARCH; +import static org.opensearch.security.http.DirectoryInformationTrees.USER_SPOCK; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.BASIC_AUTH_DOMAIN_ORDER; +import static org.opensearch.test.framework.TestSecurityConfig.Role.ALL_ACCESS; + +/** +* Test initiates plain (non-TLS) connection between OpenSearch and LDAP server and then in the course of the test connection is upgraded +* to TLS. +*/ +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class LdapStartTlsAuthenticationTest { + + private static final TestSecurityConfig.User ADMIN_USER = new TestSecurityConfig.User("admin").roles(ALL_ACCESS); + + private static final TestCertificates TEST_CERTIFICATES = new TestCertificates(); + + public static final EmbeddedLDAPServer embeddedLDAPServer = new EmbeddedLDAPServer( + TEST_CERTIFICATES.getRootCertificateData(), + TEST_CERTIFICATES.getLdapCertificateData(), + LDIF_DATA + ); + + public static LocalCluster cluster = new LocalCluster.Builder().testCertificates(TEST_CERTIFICATES) + .clusterManager(ClusterManager.SINGLENODE) + .anonymousAuth(false) + .authc( + new AuthcDomain("ldap-config-id", BASIC_AUTH_DOMAIN_ORDER + 1, true).httpAuthenticator( + new HttpAuthenticator("basic").challenge(false) + ) + .backend( + new AuthenticationBackend("ldap").config( + () -> LdapAuthenticationConfigBuilder.config() + // this port is available when embeddedLDAPServer is already started, therefore Supplier interface is used + .hosts(List.of("localhost:" + embeddedLDAPServer.getLdapNonTlsPort())) + .enableSsl(false) + .enableStartTls(true) + .bindDn(DN_OPEN_SEARCH_PEOPLE_TEST_ORG) + .password(PASSWORD_OPEN_SEARCH) + .userBase(DN_PEOPLE_TEST_ORG) + .userSearch(USER_SEARCH) + .usernameAttribute(USERNAME_ATTRIBUTE) + .penTrustedCasFilePath(TEST_CERTIFICATES.getRootCertificate().getAbsolutePath()) + .build() + ) + ) + ) + .authc(AUTHC_HTTPBASIC_INTERNAL) + .users(ADMIN_USER) + .build(); + + @ClassRule + public static RuleChain ruleChain = RuleChain.outerRule(embeddedLDAPServer).around(cluster); + + @Rule + public LogsRule logsRule = new LogsRule("com.amazon.dlic.auth.ldap.backend.LDAPAuthenticationBackend"); + + @Test + public void shouldAuthenticateUserWithLdap_positive() { + try (TestRestClient client = cluster.getRestClient(USER_SPOCK, PASSWORD_SPOCK)) { + TestRestClient.HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(200); + } + } + + @Test + public void shouldAuthenticateUserWithLdap_negativeWhenIncorrectPassword() { + try (TestRestClient client = cluster.getRestClient(USER_SPOCK, "incorrect password")) { + TestRestClient.HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(401); + String expectedStackTraceFragment = "Unable to bind as user '".concat(DN_CAPTAIN_SPOCK_PEOPLE_TEST_ORG) + .concat("' because the provided password was incorrect."); + logsRule.assertThatStackTraceContain(expectedStackTraceFragment); + } + } +} diff --git a/src/integrationTest/java/org/opensearch/security/http/ProxyAuthenticationTest.java b/src/integrationTest/java/org/opensearch/security/http/ProxyAuthenticationTest.java new file mode 100644 index 0000000000..8d9ede8e5a --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/http/ProxyAuthenticationTest.java @@ -0,0 +1,128 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.security.http; + +import java.io.IOException; +import java.util.Map; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain; +import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AuthenticationBackend; +import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.HttpAuthenticator; +import org.opensearch.test.framework.XffConfig; +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; + +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; + +/** +* Class used to run tests defined in the supper class against OpenSearch cluster with configured proxy authentication. +*/ +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class ProxyAuthenticationTest extends CommonProxyAuthenticationTests { + + private static final Map PROXY_AUTHENTICATOR_CONFIG = Map.of( + "user_header", + HEADER_PROXY_USER, + "roles_header", + HEADER_PROXY_ROLES + ); + + @ClassRule + public static final LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.SINGLENODE) + .anonymousAuth(false) + .xff(new XffConfig(true).internalProxiesRegexp("127\\.0\\.0\\.10")) + .authc( + new AuthcDomain("proxy_auth_domain", -5, true).httpAuthenticator( + new HttpAuthenticator("proxy").challenge(false).config(PROXY_AUTHENTICATOR_CONFIG) + ).backend(new AuthenticationBackend("noop")) + ) + .authc(AUTHC_HTTPBASIC_INTERNAL) + .users(USER_ADMIN) + .roles(ROLE_ALL_INDEX_SEARCH, ROLE_PERSONAL_INDEX_SEARCH) + .rolesMapping(ROLES_MAPPING_CAPTAIN, ROLES_MAPPING_FIRST_MATE) + .build(); + + @Override + protected LocalCluster getCluster() { + return cluster; + } + + @Test + @Override + public void shouldAuthenticateWithBasicAuthWhenProxyAuthenticationIsConfigured() { + super.shouldAuthenticateWithBasicAuthWhenProxyAuthenticationIsConfigured(); + } + + @Test + @Override + public void shouldAuthenticateWithProxy_positiveUserKirk() throws IOException { + super.shouldAuthenticateWithProxy_positiveUserKirk(); + } + + @Test + @Override + public void shouldAuthenticateWithProxy_positiveUserSpock() throws IOException { + super.shouldAuthenticateWithProxy_positiveUserSpock(); + } + + @Test + @Override + public void shouldAuthenticateWithProxy_negativeWhenXffHeaderIsMissing() throws IOException { + super.shouldAuthenticateWithProxy_negativeWhenXffHeaderIsMissing(); + } + + @Test + @Override + public void shouldAuthenticateWithProxy_negativeWhenUserNameHeaderIsMissing() throws IOException { + super.shouldAuthenticateWithProxy_negativeWhenUserNameHeaderIsMissing(); + } + + @Test + @Override + public void shouldAuthenticateWithProxyWhenRolesHeaderIsMissing() throws IOException { + super.shouldAuthenticateWithProxyWhenRolesHeaderIsMissing(); + } + + @Test + @Override + public void shouldAuthenticateWithProxy_negativeWhenRequestWasNotSendByProxy() throws IOException { + super.shouldAuthenticateWithProxy_negativeWhenRequestWasNotSendByProxy(); + } + + @Test + @Override + public void shouldRetrieveEmptyListOfRoles() throws IOException { + super.shouldRetrieveEmptyListOfRoles(); + } + + @Test + @Override + public void shouldRetrieveSingleRoleFirstMate() throws IOException { + super.shouldRetrieveSingleRoleFirstMate(); + } + + @Test + @Override + public void shouldRetrieveSingleRoleCaptain() throws IOException { + super.shouldRetrieveSingleRoleCaptain(); + } + + @Test + @Override + public void shouldRetrieveMultipleRoles() throws IOException { + super.shouldRetrieveMultipleRoles(); + } +} diff --git a/src/integrationTest/java/org/opensearch/security/http/UntrustedLdapServerCertificateTest.java b/src/integrationTest/java/org/opensearch/security/http/UntrustedLdapServerCertificateTest.java new file mode 100644 index 0000000000..10e3f0853f --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/http/UntrustedLdapServerCertificateTest.java @@ -0,0 +1,104 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.security.http; + +import java.util.List; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.RuleChain; +import org.junit.runner.RunWith; + +import org.opensearch.test.framework.LdapAuthenticationConfigBuilder; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain; +import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AuthenticationBackend; +import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.HttpAuthenticator; +import org.opensearch.test.framework.certificate.TestCertificates; +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.ldap.EmbeddedLDAPServer; +import org.opensearch.test.framework.log.LogsRule; + +import static org.opensearch.security.http.DirectoryInformationTrees.DN_OPEN_SEARCH_PEOPLE_TEST_ORG; +import static org.opensearch.security.http.DirectoryInformationTrees.DN_PEOPLE_TEST_ORG; +import static org.opensearch.security.http.DirectoryInformationTrees.LDIF_DATA; +import static org.opensearch.security.http.DirectoryInformationTrees.PASSWORD_OPEN_SEARCH; +import static org.opensearch.security.http.DirectoryInformationTrees.PASSWORD_SPOCK; +import static org.opensearch.security.http.DirectoryInformationTrees.USERNAME_ATTRIBUTE; +import static org.opensearch.security.http.DirectoryInformationTrees.USER_SEARCH; +import static org.opensearch.security.http.DirectoryInformationTrees.USER_SPOCK; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.BASIC_AUTH_DOMAIN_ORDER; +import static org.opensearch.test.framework.TestSecurityConfig.Role.ALL_ACCESS; + +/** +* Negative test case related to LDAP server certificate. Connection between OpenSearch and LDAP server should not be established if +* OpenSearch "does not trust" LDAP server certificate. +*/ +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class UntrustedLdapServerCertificateTest { + + private static final TestSecurityConfig.User ADMIN_USER = new TestSecurityConfig.User("admin").roles(ALL_ACCESS); + + private static final TestCertificates TEST_CERTIFICATES = new TestCertificates(); + + public static final EmbeddedLDAPServer embeddedLDAPServer = new EmbeddedLDAPServer( + TEST_CERTIFICATES.getRootCertificateData(), + TEST_CERTIFICATES.createSelfSignedCertificate("CN=untrusted"), + LDIF_DATA + ); + + public static LocalCluster cluster = new LocalCluster.Builder().testCertificates(TEST_CERTIFICATES) + .clusterManager(ClusterManager.SINGLENODE) + .anonymousAuth(false) + .authc( + new AuthcDomain("ldap", BASIC_AUTH_DOMAIN_ORDER + 1, true).httpAuthenticator(new HttpAuthenticator("basic").challenge(false)) + .backend( + new AuthenticationBackend("ldap").config( + () -> LdapAuthenticationConfigBuilder.config() + // this port is available when embeddedLDAPServer is already started, therefore Supplier interface is used + .hosts(List.of("localhost:" + embeddedLDAPServer.getLdapTlsPort())) + .enableSsl(true) + .bindDn(DN_OPEN_SEARCH_PEOPLE_TEST_ORG) + .password(PASSWORD_OPEN_SEARCH) + .userBase(DN_PEOPLE_TEST_ORG) + .userSearch(USER_SEARCH) + .usernameAttribute(USERNAME_ATTRIBUTE) + .penTrustedCasFilePath(TEST_CERTIFICATES.getRootCertificate().getAbsolutePath()) + .build() + ) + ) + ) + .authc(AUTHC_HTTPBASIC_INTERNAL) + .users(ADMIN_USER) + .build(); + + @ClassRule + public static RuleChain ruleChain = RuleChain.outerRule(embeddedLDAPServer).around(cluster); + + @Rule + public LogsRule logsRule = new LogsRule("com.amazon.dlic.auth.ldap.backend.LDAPAuthenticationBackend"); + + @Test + public void shouldNotAuthenticateUserWithLdap() { + try (TestRestClient client = cluster.getRestClient(USER_SPOCK, PASSWORD_SPOCK)) { + TestRestClient.HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(401); + } + logsRule.assertThatStackTraceContain("javax.net.ssl.SSLHandshakeException"); + } + +} diff --git a/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java b/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java new file mode 100644 index 0000000000..9551db8f52 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java @@ -0,0 +1,103 @@ +/* +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ + +package org.opensearch.security.privileges; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.apache.http.HttpStatus; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.script.mustache.MustachePlugin; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.TestSecurityConfig.Role; +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; + +/** +* This is a port for the test +* org.opensearch.security.privileges.PrivilegesEvaluatorTest to the new test +* framework for direct comparison +*/ +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class PrivilegesEvaluatorTest { + + protected final static TestSecurityConfig.User NEGATIVE_LOOKAHEAD = new TestSecurityConfig.User("negative_lookahead_user").roles( + new Role("negative_lookahead_role").indexPermissions("read").on("/^(?!t.*).*/").clusterPermissions("cluster_composite_ops") + ); + + protected final static TestSecurityConfig.User NEGATED_REGEX = new TestSecurityConfig.User("negated_regex_user").roles( + new Role("negated_regex_role").indexPermissions("read").on("/^[a-z].*/").clusterPermissions("cluster_composite_ops") + ); + + protected final static TestSecurityConfig.User SEARCH_TEMPLATE = new TestSecurityConfig.User("search_template_user").roles( + new Role("search_template_role").indexPermissions("read").on("services").clusterPermissions("cluster_composite_ops") + ); + + private String TEST_QUERY = + "{\"source\":{\"query\":{\"match\":{\"service\":\"{{service_name}}\"}}},\"params\":{\"service_name\":\"Oracle\"}}"; + + private String TEST_DOC = "{\"source\": {\"title\": \"Spirited Away\"}}"; + + @ClassRule + public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS) + .authc(AUTHC_HTTPBASIC_INTERNAL) + .users(NEGATIVE_LOOKAHEAD, NEGATED_REGEX, SEARCH_TEMPLATE, TestSecurityConfig.User.USER_ADMIN) + .plugin(MustachePlugin.class) + .build(); + + @Test + public void testNegativeLookaheadPattern() throws Exception { + + try (TestRestClient client = cluster.getRestClient(NEGATIVE_LOOKAHEAD)) { + assertThat(client.get("*/_search").getStatusCode(), equalTo(HttpStatus.SC_FORBIDDEN)); + assertThat(client.get("r*/_search").getStatusCode(), equalTo(HttpStatus.SC_OK)); + } + } + + @Test + public void testRegexPattern() throws Exception { + + try (TestRestClient client = cluster.getRestClient(NEGATED_REGEX)) { + assertThat(client.get("*/_search").getStatusCode(), equalTo(HttpStatus.SC_FORBIDDEN)); + assertThat(client.get("r*/_search").getStatusCode(), equalTo(HttpStatus.SC_OK)); + } + + } + + @Test + public void testSearchTemplateRequestUnauthorizedIndex() { + try (TestRestClient client = cluster.getRestClient(SEARCH_TEMPLATE)) { + final String searchTemplateOnMoviesIndex = "movies/_search/template"; + final TestRestClient.HttpResponse searchTemplateOnUnauthorizedIndexResponse = client.getWithJsonBody( + searchTemplateOnMoviesIndex, + TEST_QUERY + ); + assertThat(searchTemplateOnUnauthorizedIndexResponse.getStatusCode(), equalTo(HttpStatus.SC_FORBIDDEN)); + } + } + + @Test + public void testSearchTemplateRequestUnauthorizedAllIndices() { + try (TestRestClient client = cluster.getRestClient(SEARCH_TEMPLATE)) { + final String searchTemplateOnAllIndices = "_search/template"; + final TestRestClient.HttpResponse searchOnAllIndicesResponse = client.getWithJsonBody(searchTemplateOnAllIndices, TEST_QUERY); + assertThat(searchOnAllIndicesResponse.getStatusCode(), equalTo(HttpStatus.SC_FORBIDDEN)); + } + } +} diff --git a/src/integrationTest/java/org/opensearch/security/rest/CompressionTests.java b/src/integrationTest/java/org/opensearch/security/rest/CompressionTests.java new file mode 100644 index 0000000000..ecc49d93c1 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/rest/CompressionTests.java @@ -0,0 +1,134 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.security.rest; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.apache.http.HttpStatus; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.entity.ContentType; +import org.apache.http.message.BasicHeader; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.test.framework.AsyncActions; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.zip.GZIPOutputStream; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.TestSecurityConfig.Role.ALL_ACCESS; +import static org.opensearch.test.framework.cluster.TestRestClientConfiguration.getBasicAuthHeader; + +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class CompressionTests { + private static final TestSecurityConfig.User ADMIN_USER = new TestSecurityConfig.User("admin").roles(ALL_ACCESS); + + @ClassRule + public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS) + .authc(AUTHC_HTTPBASIC_INTERNAL) + .users(ADMIN_USER) + .anonymousAuth(false) + .build(); + + @Test + public void testAuthenticatedGzippedRequests() { + final String requestPath = "/*/_search"; + final int parallelism = 10; + final int totalNumberOfRequests = 100; + + final String rawBody = "{ \"query\": { \"match\": { \"foo\": \"bar\" }}}"; + + final byte[] compressedRequestBody = createCompressedRequestBody(rawBody); + try (final TestRestClient client = cluster.getRestClient(ADMIN_USER, new BasicHeader("Content-Encoding", "gzip"))) { + final List> requests = AsyncActions.generate(() -> { + final HttpPost post = new HttpPost(client.getHttpServerUri() + requestPath); + post.setEntity(new ByteArrayEntity(compressedRequestBody, ContentType.APPLICATION_JSON)); + return client.executeRequest(post); + }, parallelism, totalNumberOfRequests); + + AsyncActions.getAll(requests, 30, TimeUnit.SECONDS).forEach((response) -> { response.assertStatusCode(HttpStatus.SC_OK); }); + } + } + + @Test + public void testMixOfAuthenticatedAndUnauthenticatedGzippedRequests() throws Exception { + final String requestPath = "/*/_search"; + final int parallelism = 10; + final int totalNumberOfRequests = 50; + + final String rawBody = "{ \"query\": { \"match\": { \"foo\": \"bar\" }}}"; + + final byte[] compressedRequestBody = createCompressedRequestBody(rawBody); + try (final TestRestClient client = cluster.getRestClient(new BasicHeader("Content-Encoding", "gzip"))) { + final CountDownLatch countDownLatch = new CountDownLatch(1); + + final List> authorizedRequests = AsyncActions.generate(() -> { + countDownLatch.await(); + System.err.println("Generation triggered authorizedRequests"); + final HttpPost post = new HttpPost(client.getHttpServerUri() + requestPath); + post.setEntity(new ByteArrayEntity(compressedRequestBody, ContentType.APPLICATION_JSON)); + return client.executeRequest(post, getBasicAuthHeader(ADMIN_USER.getName(), ADMIN_USER.getPassword())); + }, parallelism, totalNumberOfRequests); + + final List> unauthorizedRequests = AsyncActions.generate(() -> { + countDownLatch.await(); + System.err.println("Generation triggered unauthorizedRequests"); + final HttpPost post = new HttpPost(client.getHttpServerUri() + requestPath); + post.setEntity(new ByteArrayEntity(compressedRequestBody, ContentType.APPLICATION_JSON)); + return client.executeRequest(post); + }, parallelism, totalNumberOfRequests); + + // Make sure all requests start at the same time + countDownLatch.countDown(); + + AsyncActions.getAll(authorizedRequests, 30, TimeUnit.SECONDS).forEach((response) -> { + assertThat(response.getStatusCode(), equalTo(HttpStatus.SC_OK)); + }); + AsyncActions.getAll(unauthorizedRequests, 30, TimeUnit.SECONDS).forEach((response) -> { + assertThat(response.getBody(), not(containsString("json_parse_exception"))); + assertThat(response.getStatusCode(), equalTo(HttpStatus.SC_UNAUTHORIZED)); + }); + } + } + + static byte[] createCompressedRequestBody(final String rawBody) { + try ( + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + final GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream) + ) { + gzipOutputStream.write(rawBody.getBytes(StandardCharsets.UTF_8)); + gzipOutputStream.finish(); + + final byte[] compressedRequestBody = byteArrayOutputStream.toByteArray(); + return compressedRequestBody; + } catch (final IOException ioe) { + throw new RuntimeException(ioe); + } + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/AsyncActions.java b/src/integrationTest/java/org/opensearch/test/framework/AsyncActions.java new file mode 100644 index 0000000000..409aa5a416 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/AsyncActions.java @@ -0,0 +1,99 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.test.framework; + +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class AsyncActions { + private final static Logger LOG = LogManager.getLogger(AsyncActions.class); + + /** + * Using the provided generator create a list of completable futures. + * @param parrallelism How many calls to the generator should be done at the same time. + * @param generationCount The total number of calls to the generator to conduct. + * @return The list of completable futures running on the fork join thread pool. + */ + public static List> generate(final Callable generator, final int parrallelism, final int generationCount) { + final ForkJoinPool forkJoinPool = new ForkJoinPool(parrallelism); + return IntStream.rangeClosed(1, generationCount).boxed().map(i -> CompletableFuture.supplyAsync(() -> { + try { + return generator.call(); + } catch (final Exception ex) { + throw new RuntimeException(ex); + } + }, forkJoinPool)).collect(Collectors.toList()); + } + + /** + * Waits for futures for a time period and then returns them a list + * @param futures Futures to wait for completion with a result + * @param n Amount of time to wait + * @param unit Time associated with those units + * @return Completed results from the futures + */ + public static List getAll(final List> futures, final int n, final TimeUnit unit) { + LOG.info("Starting to wait for " + futures.size() + " futures to complete in " + unit.toSeconds(n) + " seconds."); + final long startTimeMs = System.currentTimeMillis(); + final CompletableFuture futuresCompleted = CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + try { + futuresCompleted.get(n, unit); + } catch (final Exception ex) { + final long completedFuturesCount = futures.stream().filter(CompletableFuture::isDone).count(); + final String perfReport = calculatePerfReport(startTimeMs, completedFuturesCount); + throw new RuntimeException( + "Unable to wait for all futures to complete, of " + + futures.size() + + " futures " + + completedFuturesCount + + " have finished." + + perfReport + ); + } + final long completedFuturesCount = futures.stream().filter(CompletableFuture::isDone).count(); + final String perfReport = calculatePerfReport(startTimeMs, completedFuturesCount); + LOG.info(perfReport); + + final long elapsedTimeMs = System.currentTimeMillis() - startTimeMs; + final long expectedMs = unit.toMillis(n); + if (elapsedTimeMs > .75 * expectedMs) { + LOG.warn("Completion time was within 25% of the expected time, more than this threshold is recommended."); + } + + return futures.stream().map(future -> { + try { + return future.get(); + } catch (final Exception ex) { + throw new RuntimeException(ex); + } + }).collect(Collectors.toList()); + } + + private static String calculatePerfReport(final long startTimeMs, final long completedFuturesCount) { + final long elapsedTimeMs = System.currentTimeMillis() - startTimeMs; + final double avgTimePerFutureMs = (double) elapsedTimeMs / completedFuturesCount; + final double futuresPerSecond = 1000 / avgTimePerFutureMs; + return String.format( + "Waited for %d seconds, completion speed was on average %.2fms per future %.2fx per second.", + TimeUnit.MILLISECONDS.toSeconds(elapsedTimeMs), + avgTimePerFutureMs, + futuresPerSecond + ); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/AuditCompliance.java b/src/integrationTest/java/org/opensearch/test/framework/AuditCompliance.java new file mode 100644 index 0000000000..d75fc0e4e5 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/AuditCompliance.java @@ -0,0 +1,107 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +public class AuditCompliance implements ToXContentObject { + + private boolean enabled = false; + + private Boolean writeLogDiffs; + + private List readIgnoreUsers; + + private List writeWatchedIndices; + + private List writeIgnoreUsers; + + private Boolean readMetadataOnly; + + private Boolean writeMetadataOnly; + + private Boolean externalConfig; + + private Boolean internalConfig; + + public AuditCompliance enabled(boolean enabled) { + this.enabled = enabled; + this.writeLogDiffs = false; + this.readIgnoreUsers = Collections.emptyList(); + this.writeWatchedIndices = Collections.emptyList(); + this.writeIgnoreUsers = Collections.emptyList(); + this.readMetadataOnly = false; + this.writeMetadataOnly = false; + this.externalConfig = false; + this.internalConfig = false; + return this; + } + + public AuditCompliance writeLogDiffs(boolean writeLogDiffs) { + this.writeLogDiffs = writeLogDiffs; + return this; + } + + public AuditCompliance readIgnoreUsers(List list) { + this.readIgnoreUsers = list; + return this; + } + + public AuditCompliance writeWatchedIndices(List list) { + this.writeWatchedIndices = list; + return this; + } + + public AuditCompliance writeIgnoreUsers(List list) { + this.writeIgnoreUsers = list; + return this; + } + + public AuditCompliance readMetadataOnly(boolean readMetadataOnly) { + this.readMetadataOnly = readMetadataOnly; + return this; + } + + public AuditCompliance writeMetadataOnly(boolean writeMetadataOnly) { + this.writeMetadataOnly = writeMetadataOnly; + return this; + } + + public AuditCompliance externalConfig(boolean externalConfig) { + this.externalConfig = externalConfig; + return this; + } + + public AuditCompliance internalConfig(boolean internalConfig) { + this.internalConfig = internalConfig; + return this; + } + + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + xContentBuilder.startObject(); + xContentBuilder.field("enabled", enabled); + xContentBuilder.field("write_log_diffs", writeLogDiffs); + xContentBuilder.field("read_ignore_users", readIgnoreUsers); + xContentBuilder.field("write_watched_indices", writeWatchedIndices); + xContentBuilder.field("write_ignore_users", writeIgnoreUsers); + xContentBuilder.field("read_metadata_only", readMetadataOnly); + xContentBuilder.field("write_metadata_only", writeMetadataOnly); + xContentBuilder.field("external_config", externalConfig); + xContentBuilder.field("internal_config", internalConfig); + xContentBuilder.endObject(); + return xContentBuilder; + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/AuditConfiguration.java b/src/integrationTest/java/org/opensearch/test/framework/AuditConfiguration.java new file mode 100644 index 0000000000..1b3f11cc83 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/AuditConfiguration.java @@ -0,0 +1,56 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework; + +import java.io.IOException; + +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +public class AuditConfiguration implements ToXContentObject { + private final boolean enabled; + + private AuditFilters filters; + + private AuditCompliance compliance; + + public AuditConfiguration(boolean enabled) { + this.filters = new AuditFilters(); + this.compliance = new AuditCompliance(); + this.enabled = enabled; + } + + public boolean isEnabled() { + return enabled; + } + + public AuditConfiguration filters(AuditFilters filters) { + this.filters = filters; + return this; + } + + public AuditConfiguration compliance(AuditCompliance auditCompliance) { + this.compliance = auditCompliance; + return this; + } + + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + // json built here must be deserialized to org.opensearch.security.auditlog.config.AuditConfig + xContentBuilder.startObject(); + xContentBuilder.field("enabled", enabled); + + xContentBuilder.field("audit", filters); + xContentBuilder.field("compliance", compliance); + + xContentBuilder.endObject(); + return xContentBuilder; + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/AuditFilters.java b/src/integrationTest/java/org/opensearch/test/framework/AuditFilters.java new file mode 100644 index 0000000000..f984becefa --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/AuditFilters.java @@ -0,0 +1,122 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +public class AuditFilters implements ToXContentObject { + + private Boolean enabledRest; + + private Boolean enabledTransport; + + private Boolean logRequestBody; + + private Boolean resolveIndices; + + private Boolean resolveBulkRequests; + + private Boolean excludeSensitiveHeaders; + + private List ignoreUsers; + + private List ignoreRequests; + + private List disabledRestCategories; + + private List disabledTransportCategories; + + public AuditFilters() { + this.enabledRest = false; + this.enabledTransport = false; + + this.logRequestBody = true; + this.resolveIndices = true; + this.resolveBulkRequests = false; + this.excludeSensitiveHeaders = true; + + this.ignoreUsers = Collections.emptyList(); + this.ignoreRequests = Collections.emptyList(); + this.disabledRestCategories = Collections.emptyList(); + this.disabledTransportCategories = Collections.emptyList(); + } + + public AuditFilters enabledRest(boolean enabled) { + this.enabledRest = enabled; + return this; + } + + public AuditFilters enabledTransport(boolean enabled) { + this.enabledTransport = enabled; + return this; + } + + public AuditFilters logRequestBody(boolean logRequestBody) { + this.logRequestBody = logRequestBody; + return this; + } + + public AuditFilters resolveIndices(boolean resolveIndices) { + this.resolveIndices = resolveIndices; + return this; + } + + public AuditFilters resolveBulkRequests(boolean resolveBulkRequests) { + this.resolveBulkRequests = resolveBulkRequests; + return this; + } + + public AuditFilters excludeSensitiveHeaders(boolean excludeSensitiveHeaders) { + this.excludeSensitiveHeaders = excludeSensitiveHeaders; + return this; + } + + public AuditFilters ignoreUsers(List ignoreUsers) { + this.ignoreUsers = ignoreUsers; + return this; + } + + public AuditFilters ignoreRequests(List ignoreRequests) { + this.ignoreRequests = ignoreRequests; + return this; + } + + public AuditFilters disabledRestCategories(List disabledRestCategories) { + this.disabledRestCategories = disabledRestCategories; + return this; + } + + public AuditFilters disabledTransportCategories(List disabledTransportCategories) { + this.disabledTransportCategories = disabledTransportCategories; + return this; + } + + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + xContentBuilder.startObject(); + xContentBuilder.field("enable_rest", enabledRest); + xContentBuilder.field("enable_transport", enabledTransport); + xContentBuilder.field("resolve_indices", resolveIndices); + xContentBuilder.field("log_request_body", logRequestBody); + xContentBuilder.field("resolve_bulk_requests", resolveBulkRequests); + xContentBuilder.field("exclude_sensitive_headers", excludeSensitiveHeaders); + xContentBuilder.field("ignore_users", ignoreUsers); + xContentBuilder.field("ignore_requests", ignoreRequests); + xContentBuilder.field("disabled_rest_categories", disabledRestCategories); + xContentBuilder.field("disabled_transport_categories", disabledTransportCategories); + xContentBuilder.endObject(); + return xContentBuilder; + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/AuthFailureListeners.java b/src/integrationTest/java/org/opensearch/test/framework/AuthFailureListeners.java new file mode 100644 index 0000000000..472d3d8d08 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/AuthFailureListeners.java @@ -0,0 +1,39 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; + +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +public class AuthFailureListeners implements ToXContentObject { + + private Map limits = new LinkedHashMap<>(); + + public AuthFailureListeners addRateLimit(RateLimiting rateLimiting) { + Objects.requireNonNull(rateLimiting, "Rate limiting is required"); + limits.put(rateLimiting.getName(), rateLimiting); + return this; + } + + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + xContentBuilder.startObject(); + for (Map.Entry entry : limits.entrySet()) { + xContentBuilder.field(entry.getKey(), entry.getValue()); + } + xContentBuilder.endObject(); + return xContentBuilder; + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/AuthorizationBackend.java b/src/integrationTest/java/org/opensearch/test/framework/AuthorizationBackend.java new file mode 100644 index 0000000000..521d35ed46 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/AuthorizationBackend.java @@ -0,0 +1,45 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; +import java.util.function.Supplier; + +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +public class AuthorizationBackend implements ToXContentObject { + private final String type; + private Supplier> config; + + public AuthorizationBackend(String type) { + this.type = type; + } + + public AuthorizationBackend config(Map ldapConfig) { + return config(() -> ldapConfig); + } + + public AuthorizationBackend config(Supplier> ldapConfigSupplier) { + this.config = Objects.requireNonNull(ldapConfigSupplier, "Configuration supplier is required"); + return this; + } + + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + xContentBuilder.startObject(); + xContentBuilder.field("type", type); + xContentBuilder.field("config", config.get()); + xContentBuilder.endObject(); + return xContentBuilder; + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/AuthzDomain.java b/src/integrationTest/java/org/opensearch/test/framework/AuthzDomain.java new file mode 100644 index 0000000000..5ccf1f9ee0 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/AuthzDomain.java @@ -0,0 +1,70 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework; + +import java.io.IOException; + +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +/** +* The class represents authorization domain +*/ +public class AuthzDomain implements ToXContentObject { + + private final String id; + + private String description; + + private boolean httpEnabled; + + private boolean transportEnabled; + + private AuthorizationBackend authorizationBackend; + + public AuthzDomain(String id) { + this.id = id; + } + + public String getId() { + return id; + } + + public AuthzDomain description(String description) { + this.description = description; + return this; + } + + public AuthzDomain httpEnabled(boolean httpEnabled) { + this.httpEnabled = httpEnabled; + return this; + } + + public AuthzDomain authorizationBackend(AuthorizationBackend authorizationBackend) { + this.authorizationBackend = authorizationBackend; + return this; + } + + public AuthzDomain transportEnabled(boolean transportEnabled) { + this.transportEnabled = transportEnabled; + return this; + } + + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + xContentBuilder.startObject(); + xContentBuilder.field("description", description); + xContentBuilder.field("http_enabled", httpEnabled); + xContentBuilder.field("transport_enabled", transportEnabled); + xContentBuilder.field("authorization_backend", authorizationBackend); + xContentBuilder.endObject(); + return xContentBuilder; + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/JwtConfigBuilder.java b/src/integrationTest/java/org/opensearch/test/framework/JwtConfigBuilder.java new file mode 100644 index 0000000000..48dfa128e0 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/JwtConfigBuilder.java @@ -0,0 +1,62 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework; + +import java.util.Map; +import java.util.Objects; + +import com.google.common.collect.ImmutableMap.Builder; + +import static org.apache.commons.lang3.StringUtils.isNoneBlank; + +public class JwtConfigBuilder { + private String jwtHeader; + private String signingKey; + private String subjectKey; + private String rolesKey; + + public JwtConfigBuilder jwtHeader(String jwtHeader) { + this.jwtHeader = jwtHeader; + return this; + } + + public JwtConfigBuilder signingKey(String signingKey) { + this.signingKey = signingKey; + return this; + } + + public JwtConfigBuilder subjectKey(String subjectKey) { + this.subjectKey = subjectKey; + return this; + } + + public JwtConfigBuilder rolesKey(String rolesKey) { + this.rolesKey = rolesKey; + return this; + } + + public Map build() { + Builder builder = new Builder<>(); + if (Objects.isNull(signingKey)) { + throw new IllegalStateException("Signing key is required."); + } + builder.put("signing_key", signingKey); + if (isNoneBlank(jwtHeader)) { + builder.put("jwt_header", jwtHeader); + } + if (isNoneBlank(subjectKey)) { + builder.put("subject_key", subjectKey); + } + if (isNoneBlank(rolesKey)) { + builder.put("roles_key", rolesKey); + } + return builder.build(); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/LdapAuthenticationConfigBuilder.java b/src/integrationTest/java/org/opensearch/test/framework/LdapAuthenticationConfigBuilder.java new file mode 100644 index 0000000000..07f1836b59 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/LdapAuthenticationConfigBuilder.java @@ -0,0 +1,119 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +/** +* @param is related to subclasses thus method defined in the class LdapAuthenticationConfigBuilder return proper subclass +* type so that all method defined in subclass are available in one of builder superclass method is invoked. Please see +* {@link LdapAuthorizationConfigBuilder} +*/ +public class LdapAuthenticationConfigBuilder { + private boolean enableSsl = false; + private boolean enableStartTls = false; + private boolean enableSslClientAuth = false; + private boolean verifyHostnames = false; + private List hosts; + private String bindDn; + private String password; + private String userBase; + private String userSearch; + private String usernameAttribute; + + private String penTrustedCasFilePath; + + /** + * Subclass of this + */ + private final T builderSubclass; + + protected LdapAuthenticationConfigBuilder(Function thisCastFunction) { + this.builderSubclass = thisCastFunction.apply(this); + } + + public static LdapAuthenticationConfigBuilder config() { + return new LdapAuthenticationConfigBuilder<>(Function.identity()); + } + + public T enableSsl(boolean enableSsl) { + this.enableSsl = enableSsl; + return builderSubclass; + } + + public T enableStartTls(boolean enableStartTls) { + this.enableStartTls = enableStartTls; + return builderSubclass; + } + + public T enableSslClientAuth(boolean enableSslClientAuth) { + this.enableSslClientAuth = enableSslClientAuth; + return builderSubclass; + } + + public T verifyHostnames(boolean verifyHostnames) { + this.verifyHostnames = verifyHostnames; + return builderSubclass; + } + + public T hosts(List hosts) { + this.hosts = hosts; + return builderSubclass; + } + + public T bindDn(String bindDn) { + this.bindDn = bindDn; + return builderSubclass; + } + + public T password(String password) { + this.password = password; + return builderSubclass; + } + + public T userBase(String userBase) { + this.userBase = userBase; + return builderSubclass; + } + + public T userSearch(String userSearch) { + this.userSearch = userSearch; + return builderSubclass; + } + + public T usernameAttribute(String usernameAttribute) { + this.usernameAttribute = usernameAttribute; + return builderSubclass; + } + + public T penTrustedCasFilePath(String penTrustedCasFilePath) { + this.penTrustedCasFilePath = penTrustedCasFilePath; + return builderSubclass; + } + + public Map build() { + HashMap config = new HashMap<>(); + config.put("enable_ssl", enableSsl); + config.put("enable_start_tls", enableStartTls); + config.put("enable_ssl_client_auth", enableSslClientAuth); + config.put("verify_hostnames", verifyHostnames); + config.put("hosts", hosts); + config.put("bind_dn", bindDn); + config.put("password", password); + config.put("userbase", userBase); + config.put("usersearch", userSearch); + config.put("username_attribute", usernameAttribute); + config.put("pemtrustedcas_filepath", penTrustedCasFilePath); + return config; + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/LdapAuthorizationConfigBuilder.java b/src/integrationTest/java/org/opensearch/test/framework/LdapAuthorizationConfigBuilder.java new file mode 100644 index 0000000000..9f2a0abd83 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/LdapAuthorizationConfigBuilder.java @@ -0,0 +1,75 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework; + +import java.util.List; +import java.util.Map; + +public class LdapAuthorizationConfigBuilder extends LdapAuthenticationConfigBuilder { + private List skipUsers; + private String roleBase; + private String roleSearch; + private String userRoleAttribute; + private String userRoleName; + private String roleName; + private boolean resolveNestedRoles; + + public LdapAuthorizationConfigBuilder() { + super(LdapAuthorizationConfigBuilder.class::cast); + } + + public LdapAuthorizationConfigBuilder skipUsers(List skipUsers) { + this.skipUsers = skipUsers; + return this; + } + + public LdapAuthorizationConfigBuilder roleBase(String roleBase) { + this.roleBase = roleBase; + return this; + } + + public LdapAuthorizationConfigBuilder roleSearch(String roleSearch) { + this.roleSearch = roleSearch; + return this; + } + + public LdapAuthorizationConfigBuilder userRoleAttribute(String userRoleAttribute) { + this.userRoleAttribute = userRoleAttribute; + return this; + } + + public LdapAuthorizationConfigBuilder userRoleName(String userRoleName) { + this.userRoleName = userRoleName; + return this; + } + + public LdapAuthorizationConfigBuilder roleName(String roleName) { + this.roleName = roleName; + return this; + } + + public LdapAuthorizationConfigBuilder resolveNestedRoles(boolean resolveNestedRoles) { + this.resolveNestedRoles = resolveNestedRoles; + return this; + } + + @Override + public Map build() { + Map map = super.build(); + map.put("skip_users", skipUsers); + map.put("rolebase", roleBase); + map.put("rolesearch", roleSearch); + map.put("userroleattribute", userRoleAttribute); + map.put("userrolename", userRoleName); + map.put("rolename", roleName); + map.put("resolve_nested_roles", resolveNestedRoles); + return map; + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/OnBehalfOfConfig.java b/src/integrationTest/java/org/opensearch/test/framework/OnBehalfOfConfig.java new file mode 100644 index 0000000000..63e1544f98 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/OnBehalfOfConfig.java @@ -0,0 +1,51 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ +package org.opensearch.test.framework; + +import java.io.IOException; + +import org.apache.commons.lang3.StringUtils; + +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +public class OnBehalfOfConfig implements ToXContentObject { + private Boolean oboEnabled; + private String signing_key; + private String encryption_key; + + public OnBehalfOfConfig oboEnabled(Boolean oboEnabled) { + this.oboEnabled = oboEnabled; + return this; + } + + public OnBehalfOfConfig signingKey(String signing_key) { + this.signing_key = signing_key; + return this; + } + + public OnBehalfOfConfig encryptionKey(String encryption_key) { + this.encryption_key = encryption_key; + return this; + } + + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, ToXContent.Params params) throws IOException { + xContentBuilder.startObject(); + xContentBuilder.field("enabled", oboEnabled); + xContentBuilder.field("signing_key", signing_key); + if (StringUtils.isNoneBlank(encryption_key)) { + xContentBuilder.field("encryption_key", encryption_key); + } + xContentBuilder.endObject(); + return xContentBuilder; + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/RateLimiting.java b/src/integrationTest/java/org/opensearch/test/framework/RateLimiting.java new file mode 100644 index 0000000000..bd38aac1e5 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/RateLimiting.java @@ -0,0 +1,85 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework; + +import java.io.IOException; +import java.util.Objects; + +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +public class RateLimiting implements ToXContentObject { + + private final String name; + private String type; + private String authenticationBackend; + private Integer allowedTries; + private Integer timeWindowSeconds; + private Integer blockExpirySeconds; + private Integer maxBlockedClients; + private Integer maxTrackedClients; + + public String getName() { + return name; + } + + public RateLimiting(String name) { + this.name = Objects.requireNonNull(name, "Rate limit name is required."); + } + + public RateLimiting type(String type) { + this.type = type; + return this; + } + + public RateLimiting authenticationBackend(String authenticationBackend) { + this.authenticationBackend = authenticationBackend; + return this; + } + + public RateLimiting allowedTries(Integer allowedTries) { + this.allowedTries = allowedTries; + return this; + } + + public RateLimiting timeWindowSeconds(Integer timeWindowSeconds) { + this.timeWindowSeconds = timeWindowSeconds; + return this; + } + + public RateLimiting blockExpirySeconds(Integer blockExpirySeconds) { + this.blockExpirySeconds = blockExpirySeconds; + return this; + } + + public RateLimiting maxBlockedClients(Integer maxBlockedClients) { + this.maxBlockedClients = maxBlockedClients; + return this; + } + + public RateLimiting maxTrackedClients(Integer maxTrackedClients) { + this.maxTrackedClients = maxTrackedClients; + return this; + } + + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + xContentBuilder.startObject(); + xContentBuilder.field("type", type); + xContentBuilder.field("authentication_backend", authenticationBackend); + xContentBuilder.field("allowed_tries", allowedTries); + xContentBuilder.field("time_window_seconds", timeWindowSeconds); + xContentBuilder.field("block_expiry_seconds", blockExpirySeconds); + xContentBuilder.field("max_blocked_clients", maxBlockedClients); + xContentBuilder.field("max_tracked_clients", maxTrackedClients); + xContentBuilder.endObject(); + return xContentBuilder; + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/RolesMapping.java b/src/integrationTest/java/org/opensearch/test/framework/RolesMapping.java new file mode 100644 index 0000000000..997e7e128b --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/RolesMapping.java @@ -0,0 +1,108 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.test.framework.TestSecurityConfig.Role; + +import static java.util.Objects.requireNonNull; + +/** +* The class represents mapping between backend roles {@link #backendRoles} to OpenSearch role defined by field {@link #roleName}. The +* class provides convenient builder-like methods and can be serialized to JSON. Serialization to JSON is required to store the class +* in an OpenSearch index which contains Security plugin configuration. +*/ +public class RolesMapping implements ToXContentObject { + + /** + * OpenSearch role name + */ + private String roleName; + + /** + * Backend role names + */ + private List backendRoles; + private List hostIPs; + + private boolean reserved = false; + + /** + * Creates roles mapping to OpenSearch role defined by parameter role + * @param role OpenSearch role, must not be null. + */ + public RolesMapping(Role role) { + requireNonNull(role); + this.roleName = requireNonNull(role.getName()); + this.backendRoles = new ArrayList<>(); + this.hostIPs = new ArrayList<>(); + } + + /** + * Defines backend role names + * @param backendRoles backend roles names + * @return current {@link RolesMapping} instance + */ + public RolesMapping backendRoles(String... backendRoles) { + this.backendRoles.addAll(Arrays.asList(backendRoles)); + return this; + } + + /** + * Defines host IP address + * @param hostIPs host IP address + * @return current {@link RolesMapping} instance + */ + public RolesMapping hostIPs(String... hostIPs) { + this.hostIPs.addAll(Arrays.asList(hostIPs)); + return this; + } + + /** + * Determines if role is reserved + * @param reserved true for reserved roles + * @return current {@link RolesMapping} instance + */ + public RolesMapping reserved(boolean reserved) { + this.reserved = reserved; + return this; + } + + /** + * Returns OpenSearch role name + * @return role name + */ + public String getRoleName() { + return roleName; + } + + /** + * Controls serialization to JSON + * @param xContentBuilder must not be null + * @param params not used parameter, but required by the interface {@link ToXContentObject} + * @return builder form parameter xContentBuilder + * @throws IOException denotes error during serialization to JSON + */ + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + xContentBuilder.startObject(); + xContentBuilder.field("reserved", reserved); + xContentBuilder.field("backend_roles", backendRoles); + xContentBuilder.field("hosts", hostIPs); + xContentBuilder.endObject(); + return xContentBuilder; + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestIndex.java b/src/integrationTest/java/org/opensearch/test/framework/TestIndex.java new file mode 100644 index 0000000000..6f6bd935a5 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/TestIndex.java @@ -0,0 +1,83 @@ +/* +* Copyright 2021-2022 floragunn GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*/ + +/* +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ + +package org.opensearch.test.framework; + +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.client.Client; +import org.opensearch.common.settings.Settings; + +public class TestIndex { + + private final String name; + private final Settings settings; + + public TestIndex(String name, Settings settings) { + this.name = name; + this.settings = settings; + + } + + public void create(Client client) { + client.admin().indices().create(new CreateIndexRequest(name).settings(settings)).actionGet(); + } + + public String getName() { + return name; + } + + public static Builder name(String name) { + return new Builder().name(name); + } + + public static class Builder { + private String name; + private Settings.Builder settings = Settings.builder(); + + public Builder name(String name) { + this.name = name; + return this; + } + + public Builder setting(String name, int value) { + settings.put(name, value); + return this; + } + + public Builder shards(int value) { + settings.put("index.number_of_shards", 5); + return this; + } + + public TestIndex build() { + return new TestIndex(name, settings.build()); + } + + } + +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java new file mode 100644 index 0000000000..cbbc4e2905 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java @@ -0,0 +1,742 @@ +/* +* Copyright 2021 floragunn GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*/ + +/* +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ + +package org.opensearch.test.framework; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bouncycastle.crypto.generators.OpenBSDBCrypt; + +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.update.UpdateRequest; +import org.opensearch.client.Client; +import org.opensearch.common.Strings; +import org.opensearch.common.bytes.BytesReference; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.security.securityconf.impl.CType; +import org.opensearch.test.framework.cluster.OpenSearchClientProvider.UserCredentialsHolder; + +import static org.apache.http.HttpHeaders.AUTHORIZATION; +import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; + +/** +* This class allows the declarative specification of the security configuration; in particular: +* +* - config.yml +* - internal_users.yml +* - roles.yml +* - roles_mapping.yml +* +* The class does the whole round-trip, i.e., the configuration is serialized to YAML/JSON and then written to +* the configuration index of the security plugin. +*/ +public class TestSecurityConfig { + + private static final Logger log = LogManager.getLogger(TestSecurityConfig.class); + + private Config config = new Config(); + private Map internalUsers = new LinkedHashMap<>(); + private Map roles = new LinkedHashMap<>(); + private AuditConfiguration auditConfiguration; + private Map rolesMapping = new LinkedHashMap<>(); + + private String indexName = ".opendistro_security"; + + public TestSecurityConfig() { + + } + + public TestSecurityConfig configIndexName(String configIndexName) { + this.indexName = configIndexName; + return this; + } + + public TestSecurityConfig authFailureListeners(AuthFailureListeners listener) { + config.authFailureListeners(listener); + return this; + } + + public TestSecurityConfig anonymousAuth(boolean anonymousAuthEnabled) { + config.anonymousAuth(anonymousAuthEnabled); + return this; + } + + public TestSecurityConfig doNotFailOnForbidden(boolean doNotFailOnForbidden) { + config.doNotFailOnForbidden(doNotFailOnForbidden); + return this; + } + + public TestSecurityConfig xff(XffConfig xffConfig) { + config.xffConfig(xffConfig); + return this; + } + + public TestSecurityConfig onBehalfOf(OnBehalfOfConfig onBehalfOfConfig) { + config.onBehalfOfConfig(onBehalfOfConfig); + return this; + } + + public TestSecurityConfig authc(AuthcDomain authcDomain) { + config.authc(authcDomain); + return this; + } + + public TestSecurityConfig authz(AuthzDomain authzDomain) { + config.authz(authzDomain); + return this; + } + + public TestSecurityConfig user(User user) { + this.internalUsers.put(user.name, user); + + for (Role role : user.roles) { + this.roles.put(role.name, role); + } + + return this; + } + + public List getUsers() { + return new ArrayList<>(internalUsers.values()); + } + + public TestSecurityConfig roles(Role... roles) { + for (Role role : roles) { + if (this.roles.containsKey(role.name)) { + throw new IllegalStateException("Role with name " + role.name + " is already defined"); + } + this.roles.put(role.name, role); + } + + return this; + } + + public TestSecurityConfig audit(AuditConfiguration auditConfiguration) { + this.auditConfiguration = auditConfiguration; + return this; + } + + public TestSecurityConfig rolesMapping(RolesMapping... mappings) { + for (RolesMapping mapping : mappings) { + String roleName = mapping.getRoleName(); + if (rolesMapping.containsKey(roleName)) { + throw new IllegalArgumentException("Role mapping " + roleName + " already exists"); + } + this.rolesMapping.put(roleName, mapping); + } + return this; + } + + public static class Config implements ToXContentObject { + private boolean anonymousAuth; + + private Boolean doNotFailOnForbidden; + private XffConfig xffConfig; + private OnBehalfOfConfig onBehalfOfConfig; + private Map authcDomainMap = new LinkedHashMap<>(); + + private AuthFailureListeners authFailureListeners; + private Map authzDomainMap = new LinkedHashMap<>(); + + public Config anonymousAuth(boolean anonymousAuth) { + this.anonymousAuth = anonymousAuth; + return this; + } + + public Config doNotFailOnForbidden(Boolean doNotFailOnForbidden) { + this.doNotFailOnForbidden = doNotFailOnForbidden; + return this; + } + + public Config xffConfig(XffConfig xffConfig) { + this.xffConfig = xffConfig; + return this; + } + + public Config onBehalfOfConfig(OnBehalfOfConfig onBehalfOfConfig) { + this.onBehalfOfConfig = onBehalfOfConfig; + return this; + } + + public Config authc(AuthcDomain authcDomain) { + authcDomainMap.put(authcDomain.id, authcDomain); + return this; + } + + public Config authFailureListeners(AuthFailureListeners authFailureListeners) { + this.authFailureListeners = authFailureListeners; + return this; + } + + public Config authz(AuthzDomain authzDomain) { + authzDomainMap.put(authzDomain.getId(), authzDomain); + return this; + } + + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + xContentBuilder.startObject(); + xContentBuilder.startObject("dynamic"); + + if (onBehalfOfConfig != null) { + xContentBuilder.field("on_behalf_of", onBehalfOfConfig); + } + + if (anonymousAuth || (xffConfig != null)) { + xContentBuilder.startObject("http"); + xContentBuilder.field("anonymous_auth_enabled", anonymousAuth); + if (xffConfig != null) { + xContentBuilder.field("xff", xffConfig); + } + xContentBuilder.endObject(); + } + if (doNotFailOnForbidden != null) { + xContentBuilder.field("do_not_fail_on_forbidden", doNotFailOnForbidden); + } + + xContentBuilder.field("authc", authcDomainMap); + if (authzDomainMap.isEmpty() == false) { + xContentBuilder.field("authz", authzDomainMap); + } + + if (authFailureListeners != null) { + xContentBuilder.field("auth_failure_listeners", authFailureListeners); + } + + xContentBuilder.endObject(); + xContentBuilder.endObject(); + return xContentBuilder; + } + } + + public static class User implements UserCredentialsHolder, ToXContentObject { + + public final static TestSecurityConfig.User USER_ADMIN = new TestSecurityConfig.User("admin").roles( + new Role("allaccess").indexPermissions("*").on("*").clusterPermissions("*") + ); + + String name; + private String password; + List roles = new ArrayList<>(); + List backendRoles = new ArrayList<>(); + String requestedTenant; + private Map attributes = new HashMap<>(); + + public User(String name) { + this.name = name; + this.password = "secret"; + } + + public User password(String password) { + this.password = password; + return this; + } + + public User roles(Role... roles) { + // We scope the role names by user to keep tests free of potential side effects + String roleNamePrefix = "user_" + this.getName() + "__"; + this.roles.addAll( + Arrays.asList(roles).stream().map((r) -> r.clone().name(roleNamePrefix + r.getName())).collect(Collectors.toSet()) + ); + return this; + } + + public User backendRoles(String... backendRoles) { + this.backendRoles.addAll(Arrays.asList(backendRoles)); + return this; + } + + public User attr(String key, String value) { + this.attributes.put(key, value); + return this; + } + + public String getName() { + return name; + } + + public String getPassword() { + return password; + } + + public Set getRoleNames() { + return roles.stream().map(Role::getName).collect(Collectors.toSet()); + } + + public Object getAttribute(String attributeName) { + return attributes.get(attributeName); + } + + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + xContentBuilder.startObject(); + + xContentBuilder.field("hash", hash(password.toCharArray())); + + Set roleNames = getRoleNames(); + + if (!roleNames.isEmpty()) { + xContentBuilder.field("opendistro_security_roles", roleNames); + } + + if (!backendRoles.isEmpty()) { + xContentBuilder.field("backend_roles", backendRoles); + } + + if (attributes != null && attributes.size() != 0) { + xContentBuilder.field("attributes", attributes); + } + + xContentBuilder.endObject(); + return xContentBuilder; + } + } + + public static class Role implements ToXContentObject { + public static Role ALL_ACCESS = new Role("all_access").clusterPermissions("*").indexPermissions("*").on("*"); + + private String name; + private List clusterPermissions = new ArrayList<>(); + + private List indexPermissions = new ArrayList<>(); + + public Role(String name) { + this.name = name; + } + + public Role clusterPermissions(String... clusterPermissions) { + this.clusterPermissions.addAll(Arrays.asList(clusterPermissions)); + return this; + } + + public IndexPermission indexPermissions(String... indexPermissions) { + return new IndexPermission(this, indexPermissions); + } + + public Role name(String name) { + this.name = name; + return this; + } + + public String getName() { + return name; + } + + public Role clone() { + Role role = new Role(this.name); + role.clusterPermissions.addAll(this.clusterPermissions); + role.indexPermissions.addAll(this.indexPermissions); + return role; + } + + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + xContentBuilder.startObject(); + + if (!clusterPermissions.isEmpty()) { + xContentBuilder.field("cluster_permissions", clusterPermissions); + } + + if (!indexPermissions.isEmpty()) { + xContentBuilder.field("index_permissions", indexPermissions); + } + + xContentBuilder.endObject(); + return xContentBuilder; + } + } + + public static class IndexPermission implements ToXContentObject { + private List allowedActions; + private List indexPatterns; + private Role role; + private String dlsQuery; + private List fls; + private List maskedFields; + + IndexPermission(Role role, String... allowedActions) { + this.allowedActions = Arrays.asList(allowedActions); + this.role = role; + } + + public IndexPermission dls(String dlsQuery) { + this.dlsQuery = dlsQuery; + return this; + } + + public IndexPermission fls(String... fls) { + this.fls = Arrays.asList(fls); + return this; + } + + public IndexPermission maskedFields(String... maskedFields) { + this.maskedFields = Arrays.asList(maskedFields); + return this; + } + + public Role on(String... indexPatterns) { + this.indexPatterns = Arrays.asList(indexPatterns); + this.role.indexPermissions.add(this); + return this.role; + } + + public Role on(TestIndex... testindices) { + this.indexPatterns = Arrays.asList(testindices).stream().map(TestIndex::getName).collect(Collectors.toList()); + this.role.indexPermissions.add(this); + return this.role; + } + + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + xContentBuilder.startObject(); + + xContentBuilder.field("index_patterns", indexPatterns); + xContentBuilder.field("allowed_actions", allowedActions); + + if (dlsQuery != null) { + xContentBuilder.field("dls", dlsQuery); + } + + if (fls != null) { + xContentBuilder.field("fls", fls); + } + + if (maskedFields != null) { + xContentBuilder.field("masked_fields", maskedFields); + } + + xContentBuilder.endObject(); + return xContentBuilder; + } + } + + public static class AuthcDomain implements ToXContentObject { + + private static String PUBLIC_KEY = + "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoqZbjLUAWc+DZTkinQAdvy1GFjPHPnxheU89hSiWoDD3NOW76H3u3T7cCDdOah2msdxSlBmCBH6wik8qLYkcV8owWukQg3PQmbEhrdPaKo0QCgomWs4nLgtmEYqcZ+QQldd82MdTlQ1QmoQmI9Uxqs1SuaKZASp3Gy19y8su5CV+FZ6BruUw9HELK055sAwl3X7j5ouabXGbcib2goBF3P52LkvbJLuWr5HDZEOeSkwIeqSeMojASM96K5SdotD+HwEyjaTjzRPL2Aa1BEQFWOQ6CFJLyLH7ZStDuPM1mJU1VxIVfMbZrhsUBjAnIhRynmWxML7YlNqkP9j6jyOIYQIDAQAB"; + + public static final int BASIC_AUTH_DOMAIN_ORDER = 0; + public final static AuthcDomain AUTHC_HTTPBASIC_INTERNAL = new TestSecurityConfig.AuthcDomain("basic", BASIC_AUTH_DOMAIN_ORDER) + .httpAuthenticatorWithChallenge("basic") + .backend("internal"); + + public final static AuthcDomain AUTHC_HTTPBASIC_INTERNAL_WITHOUT_CHALLENGE = new TestSecurityConfig.AuthcDomain( + "basic", + BASIC_AUTH_DOMAIN_ORDER + ).httpAuthenticator("basic").backend("internal"); + + public final static AuthcDomain DISABLED_AUTHC_HTTPBASIC_INTERNAL = new TestSecurityConfig.AuthcDomain( + "basic", + BASIC_AUTH_DOMAIN_ORDER, + false + ).httpAuthenticator("basic").backend("internal"); + + public final static AuthcDomain JWT_AUTH_DOMAIN = new TestSecurityConfig.AuthcDomain("jwt", 1).jwtHttpAuthenticator( + new JwtConfigBuilder().jwtHeader(AUTHORIZATION).signingKey(PUBLIC_KEY) + ).backend("noop"); + + private final String id; + private boolean enabled = true; + private int order; + private List skipUsers = new ArrayList<>(); + private HttpAuthenticator httpAuthenticator; + private AuthenticationBackend authenticationBackend; + + public AuthcDomain(String id, int order, boolean enabled) { + this.id = id; + this.order = order; + this.enabled = enabled; + } + + public AuthcDomain(String id, int order) { + this(id, order, true); + } + + public AuthcDomain httpAuthenticator(String type) { + this.httpAuthenticator = new HttpAuthenticator(type); + return this; + } + + public AuthcDomain jwtHttpAuthenticator(JwtConfigBuilder builder) { + this.httpAuthenticator = new HttpAuthenticator("jwt").challenge(false).config(builder.build()); + return this; + } + + public AuthcDomain httpAuthenticatorWithChallenge(String type) { + this.httpAuthenticator = new HttpAuthenticator(type).challenge(true); + return this; + } + + public AuthcDomain httpAuthenticator(HttpAuthenticator httpAuthenticator) { + this.httpAuthenticator = httpAuthenticator; + return this; + } + + public AuthcDomain backend(String type) { + this.authenticationBackend = new AuthenticationBackend(type); + return this; + } + + public AuthcDomain backend(AuthenticationBackend authenticationBackend) { + this.authenticationBackend = authenticationBackend; + return this; + } + + public AuthcDomain skipUsers(String... users) { + this.skipUsers.addAll(Arrays.asList(users)); + return this; + } + + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + xContentBuilder.startObject(); + + xContentBuilder.field("http_enabled", enabled); + xContentBuilder.field("order", order); + + if (httpAuthenticator != null) { + xContentBuilder.field("http_authenticator", httpAuthenticator); + } + + if (authenticationBackend != null) { + xContentBuilder.field("authentication_backend", authenticationBackend); + } + + if (skipUsers != null && skipUsers.size() > 0) { + xContentBuilder.field("skip_users", skipUsers); + } + + xContentBuilder.endObject(); + return xContentBuilder; + } + + public static class HttpAuthenticator implements ToXContentObject { + private final String type; + private boolean challenge; + private Map config = new HashMap(); + + public HttpAuthenticator(String type) { + this.type = type; + } + + public HttpAuthenticator challenge(boolean challenge) { + this.challenge = challenge; + return this; + } + + public HttpAuthenticator config(Map config) { + this.config.putAll(config); + return this; + } + + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + xContentBuilder.startObject(); + + xContentBuilder.field("type", type); + xContentBuilder.field("challenge", challenge); + xContentBuilder.field("config", config); + + xContentBuilder.endObject(); + return xContentBuilder; + } + } + + public static class AuthenticationBackend implements ToXContentObject { + private final String type; + private Supplier> config = () -> new HashMap(); + + public AuthenticationBackend(String type) { + this.type = type; + } + + public AuthenticationBackend config(Map config) { + Map configCopy = new HashMap<>(config); + this.config = () -> configCopy; + return this; + } + + public AuthenticationBackend config(Supplier> configSupplier) { + this.config = configSupplier; + return this; + } + + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + xContentBuilder.startObject(); + + xContentBuilder.field("type", type); + xContentBuilder.field("config", config.get()); + + xContentBuilder.endObject(); + return xContentBuilder; + } + } + } + + public void initIndex(Client client) { + Map settings = new HashMap<>(); + if (indexName.startsWith(".")) { + settings.put("index.hidden", true); + } + client.admin().indices().create(new CreateIndexRequest(indexName).settings(settings)).actionGet(); + + writeSingleEntryConfigToIndex(client, CType.CONFIG, config); + if (auditConfiguration != null) { + writeSingleEntryConfigToIndex(client, CType.AUDIT, "config", auditConfiguration); + } + writeConfigToIndex(client, CType.ROLES, roles); + writeConfigToIndex(client, CType.INTERNALUSERS, internalUsers); + writeConfigToIndex(client, CType.ROLESMAPPING, rolesMapping); + writeEmptyConfigToIndex(client, CType.ACTIONGROUPS); + writeEmptyConfigToIndex(client, CType.TENANTS); + } + + public void updateInternalUsersConfiguration(Client client, List users) { + Map userMap = new HashMap<>(); + for (User user : users) { + userMap.put(user.getName(), user); + } + updateConfigInIndex(client, CType.INTERNALUSERS, userMap); + } + + static String hash(final char[] clearTextPassword) { + final byte[] salt = new byte[16]; + new SecureRandom().nextBytes(salt); + final String hash = OpenBSDBCrypt.generate((Objects.requireNonNull(clearTextPassword)), salt, 12); + Arrays.fill(salt, (byte) 0); + Arrays.fill(clearTextPassword, '\0'); + return hash; + } + + private void writeEmptyConfigToIndex(Client client, CType configType) { + writeConfigToIndex(client, configType, Collections.emptyMap()); + } + + private void writeConfigToIndex(Client client, CType configType, Map config) { + try { + String json = configToJson(configType, config); + + log.info("Writing security configuration into index " + configType + ":\n" + json); + + BytesReference bytesReference = toByteReference(json); + client.index( + new IndexRequest(indexName).id(configType.toLCString()) + .setRefreshPolicy(IMMEDIATE) + .source(configType.toLCString(), bytesReference) + ).actionGet(); + } catch (Exception e) { + throw new RuntimeException("Error while initializing config for " + indexName, e); + } + } + + private static BytesReference toByteReference(String string) throws UnsupportedEncodingException { + return BytesReference.fromByteBuffer(ByteBuffer.wrap(string.getBytes("utf-8"))); + } + + private void updateConfigInIndex(Client client, CType configType, Map config) { + try { + String json = configToJson(configType, config); + BytesReference bytesReference = toByteReference(json); + log.info("Update configuration of type '{}' in index '{}', new value '{}'.", configType, indexName, json); + UpdateRequest upsert = new UpdateRequest(indexName, configType.toLCString()).doc(configType.toLCString(), bytesReference) + .setRefreshPolicy(IMMEDIATE); + client.update(upsert).actionGet(); + } catch (Exception e) { + throw new RuntimeException("Error while updating config for " + indexName, e); + } + } + + private static String configToJson(CType configType, Map config) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + + builder.startObject(); + builder.startObject("_meta"); + builder.field("type", configType.toLCString()); + builder.field("config_version", 2); + builder.endObject(); + + for (Map.Entry entry : config.entrySet()) { + builder.field(entry.getKey(), entry.getValue()); + } + + builder.endObject(); + return Strings.toString(builder); + } + + private void writeSingleEntryConfigToIndex(Client client, CType configType, ToXContentObject config) { + writeSingleEntryConfigToIndex(client, configType, configType.toLCString(), config); + } + + private void writeSingleEntryConfigToIndex(Client client, CType configType, String configurationRoot, ToXContentObject config) { + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + + builder.startObject(); + builder.startObject("_meta"); + builder.field("type", configType.toLCString()); + builder.field("config_version", 2); + builder.endObject(); + + builder.field(configurationRoot, config); + + builder.endObject(); + + String json = Strings.toString(builder); + + log.info("Writing security plugin configuration into index " + configType + ":\n" + json); + + client.index( + new IndexRequest(indexName).id(configType.toLCString()) + .setRefreshPolicy(IMMEDIATE) + .source(configType.toLCString(), toByteReference(json)) + ).actionGet(); + } catch (Exception e) { + throw new RuntimeException("Error while initializing config for " + indexName, e); + } + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/XffConfig.java b/src/integrationTest/java/org/opensearch/test/framework/XffConfig.java new file mode 100644 index 0000000000..b1c10bfd73 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/XffConfig.java @@ -0,0 +1,82 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework; + +import java.io.IOException; + +import org.apache.commons.lang3.StringUtils; + +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +/** +*

+* XFF is an abbreviation of X-Forwarded-For. X-Forwarded-For is an HTTP header which contains client source IP address +* and additionally IP addresses of proxies which forward the request. +* The X-Forwarded-For header is used by HTTP authentication of type +*

+*
    +*
  1. proxy defined by class {@link org.opensearch.security.http.HTTPProxyAuthenticator}
    2. +*
  2. extended-proxy defined by the class {@link org.opensearch.security.http.proxy.HTTPExtendedProxyAuthenticator}
    3. +*
+* +*

+* The above authenticators use the X-Forwarded-For to determine if an HTTP request comes from trusted proxies. The trusted proxies +* are defined by a regular expression {@link #internalProxiesRegexp}. The proxy authentication can be applied only to HTTP requests +* which were forwarded by trusted HTTP proxies. +*

+* +*

+* The class can be serialized to JSON and then stored in an OpenSearch index which contains security plugin configuration. +*

+*/ +public class XffConfig implements ToXContentObject { + + private final boolean enabled; + + /** + * Regular expression used to determine if HTTP proxy is trusted or not. IP address of trusted proxies must match the regular + * expression defined by the below field. + */ + private String internalProxiesRegexp; + + private String remoteIpHeader; + + public XffConfig(boolean enabled) { + this.enabled = enabled; + } + + /** + * Builder-like method used to set value of the field {@link #internalProxiesRegexp} + * @param internalProxiesRegexp regular expression which matches IP address of a HTTP proxies if the proxies are trusted. + * @return builder + */ + public XffConfig internalProxiesRegexp(String internalProxiesRegexp) { + this.internalProxiesRegexp = internalProxiesRegexp; + return this; + } + + public XffConfig remoteIpHeader(String remoteIpHeader) { + this.remoteIpHeader = remoteIpHeader; + return this; + } + + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + xContentBuilder.startObject(); + xContentBuilder.field("enabled", enabled); + xContentBuilder.field("internalProxies", internalProxiesRegexp); + if (StringUtils.isNoneBlank(remoteIpHeader)) { + xContentBuilder.field("remoteIpHeader", remoteIpHeader); + } + xContentBuilder.endObject(); + return xContentBuilder; + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/audit/AuditLogsRule.java b/src/integrationTest/java/org/opensearch/test/framework/audit/AuditLogsRule.java new file mode 100644 index 0000000000..3d13d731eb --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/audit/AuditLogsRule.java @@ -0,0 +1,137 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.audit; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; +import org.hamcrest.Matcher; +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; + +import org.opensearch.security.auditlog.impl.AuditMessage; + +import static java.util.Collections.synchronizedList; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.opensearch.test.framework.matcher.AuditMessageMatchers.atLeastCertainNumberOfAuditsFulfillPredicate; +import static org.opensearch.test.framework.matcher.AuditMessageMatchers.exactNumberOfAuditsFulfillPredicate; + +public class AuditLogsRule implements TestRule { + + private static final Logger log = LogManager.getLogger(AuditLogsRule.class); + + private List currentTestAuditMessages; + + public List getCurrentTestAuditMessages() { + return currentTestAuditMessages; + } + + public void waitForAuditLogs() { + try { + TimeUnit.SECONDS.sleep(3); + afterWaitingForAuditLogs(); + } catch (InterruptedException e) { + throw new RuntimeException("Waiting for audit logs interrupted.", e); + } + } + + private void afterWaitingForAuditLogs() { + if (log.isDebugEnabled()) { + log.debug("Audit records captured during test:\n{}", auditMessagesToString(currentTestAuditMessages)); + } + } + + public void assertExactlyOne(Predicate predicate) { + assertExactly(1, predicate); + } + + public void assertAuditLogsCount(int from, int to) { + int actualCount = currentTestAuditMessages.size(); + String message = "Expected audit log count is between " + from + " and " + to + " but was " + actualCount; + assertThat(message, actualCount, allOf(greaterThanOrEqualTo(from), lessThanOrEqualTo(to))); + } + + public void assertExactly(long expectedNumberOfAuditMessages, Predicate predicate) { + assertExactly(exactNumberOfAuditsFulfillPredicate(expectedNumberOfAuditMessages, predicate)); + } + + private void assertExactly(Matcher> matcher) { + // pollDelay - initial delay before first evaluation + Awaitility.await("Await for audit logs") + .atMost(3, TimeUnit.SECONDS) + .pollDelay(0, TimeUnit.MICROSECONDS) + .until(() -> new ArrayList<>(currentTestAuditMessages), matcher); + } + + public void assertAtLeast(long minCount, Predicate predicate) { + assertExactly(atLeastCertainNumberOfAuditsFulfillPredicate(minCount, predicate)); + } + + private static String auditMessagesToString(List audits) { + return audits.stream().map(AuditMessage::toString).collect(Collectors.joining(",\n")); + } + + @Override + public Statement apply(Statement statement, Description description) { + return new Statement() { + @Override + public void evaluate() throws Throwable { + String methodName = description.getMethodName(); + beforeTest(methodName); + try { + statement.evaluate(); + } catch (ConditionTimeoutException ex) { + whenTimeoutOccurs(methodName); + throw ex; + } finally { + afterTest(); + } + } + }; + } + + private void whenTimeoutOccurs(String methodName) { + List copy = new ArrayList<>(currentTestAuditMessages); + String auditMessages = auditMessagesToString(copy); + log.error( + "Timeout occured due to insufficient number ('{}') of captured audit messages during test '{}'\n{}", + copy.size(), + methodName, + auditMessages + ); + } + + private void afterTest() { + TestRuleAuditLogSink.unregisterListener(); + this.currentTestAuditMessages = null; + } + + private void beforeTest(String methodName) { + log.info("Start collecting audit logs before test {}", methodName); + this.currentTestAuditMessages = synchronizedList(new ArrayList<>()); + TestRuleAuditLogSink.registerListener(this); + } + + public void onAuditMessage(AuditMessage auditMessage) { + currentTestAuditMessages.add(auditMessage); + log.debug("New audit message received '{}', total number of audit messages '{}'.", auditMessage, currentTestAuditMessages.size()); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/audit/AuditMessagePredicate.java b/src/integrationTest/java/org/opensearch/test/framework/audit/AuditMessagePredicate.java new file mode 100644 index 0000000000..4935bf0387 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/audit/AuditMessagePredicate.java @@ -0,0 +1,278 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.audit; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import org.opensearch.rest.RestRequest.Method; +import org.opensearch.security.auditlog.AuditLog.Origin; +import org.opensearch.security.auditlog.impl.AuditCategory; +import org.opensearch.security.auditlog.impl.AuditMessage; +import org.opensearch.test.framework.TestSecurityConfig.User; + +import static org.opensearch.security.auditlog.impl.AuditCategory.AUTHENTICATED; +import static org.opensearch.security.auditlog.impl.AuditCategory.GRANTED_PRIVILEGES; +import static org.opensearch.security.auditlog.impl.AuditCategory.MISSING_PRIVILEGES; +import static org.opensearch.security.auditlog.impl.AuditMessage.REQUEST_LAYER; +import static org.opensearch.security.auditlog.impl.AuditMessage.RESOLVED_INDICES; +import static org.opensearch.security.auditlog.impl.AuditMessage.REST_REQUEST_PATH; + +public class AuditMessagePredicate implements Predicate { + + private final AuditCategory category; + private final Origin requestLayer; + private final String restRequestPath; + private final String initiatingUser; + private final Method requestMethod; + private final String transportRequestType; + private final String effectiveUser; + private final String index; + private final String privilege; + + private AuditMessagePredicate( + AuditCategory category, + Origin requestLayer, + String restRequestPath, + String initiatingUser, + Method requestMethod, + String transportRequestType, + String effectiveUser, + String index, + String privilege + ) { + this.category = category; + this.requestLayer = requestLayer; + this.restRequestPath = restRequestPath; + this.initiatingUser = initiatingUser; + this.requestMethod = requestMethod; + this.transportRequestType = transportRequestType; + this.effectiveUser = effectiveUser; + this.index = index; + this.privilege = privilege; + } + + private AuditMessagePredicate(AuditCategory category) { + this(category, null, null, null, null, null, null, null, null); + } + + public static AuditMessagePredicate auditPredicate(AuditCategory category) { + return new AuditMessagePredicate(category); + } + + public static AuditMessagePredicate userAuthenticated(User user) { + return auditPredicate(AUTHENTICATED).withInitiatingUser(user); + } + + public static AuditMessagePredicate grantedPrivilege(User user, String requestType) { + return auditPredicate(GRANTED_PRIVILEGES).withLayer(Origin.TRANSPORT).withEffectiveUser(user).withTransportRequestType(requestType); + } + + public static AuditMessagePredicate missingPrivilege(User user, String requestType) { + return auditPredicate(MISSING_PRIVILEGES).withLayer(Origin.TRANSPORT).withEffectiveUser(user).withTransportRequestType(requestType); + } + + public static AuditMessagePredicate privilegePredicateTransportLayer( + AuditCategory category, + User user, + String requestType, + String privilege + ) { + return auditPredicate(category).withLayer(Origin.TRANSPORT) + .withEffectiveUser(user) + .withPrivilege(privilege) + .withTransportRequestType(requestType); + } + + public static AuditMessagePredicate privilegePredicateRESTLayer(AuditCategory category, User user, Method method, String endpoint) { + return auditPredicate(category).withLayer(Origin.REST).withEffectiveUser(user).withRestRequest(method, endpoint); + } + + public static AuditMessagePredicate userAuthenticatedPredicate(User user, Method method, String endpoint) { + return userAuthenticated(user).withLayer(Origin.REST).withRestRequest(method, endpoint).withInitiatingUser(user); + } + + public AuditMessagePredicate withLayer(Origin layer) { + return new AuditMessagePredicate( + category, + layer, + restRequestPath, + initiatingUser, + requestMethod, + transportRequestType, + effectiveUser, + index, + privilege + ); + } + + public AuditMessagePredicate withRequestPath(String path) { + return new AuditMessagePredicate( + category, + requestLayer, + path, + initiatingUser, + requestMethod, + transportRequestType, + effectiveUser, + index, + privilege + ); + } + + public AuditMessagePredicate withInitiatingUser(String user) { + return new AuditMessagePredicate( + category, + requestLayer, + restRequestPath, + user, + requestMethod, + transportRequestType, + effectiveUser, + index, + privilege + ); + } + + public AuditMessagePredicate withInitiatingUser(User user) { + return withInitiatingUser(user.getName()); + } + + public AuditMessagePredicate withRestMethod(Method method) { + return new AuditMessagePredicate( + category, + requestLayer, + restRequestPath, + initiatingUser, + method, + transportRequestType, + effectiveUser, + index, + privilege + ); + } + + public AuditMessagePredicate withTransportRequestType(String type) { + return new AuditMessagePredicate( + category, + requestLayer, + restRequestPath, + initiatingUser, + requestMethod, + type, + effectiveUser, + index, + privilege + ); + } + + public AuditMessagePredicate withEffectiveUser(String user) { + return new AuditMessagePredicate( + category, + requestLayer, + restRequestPath, + initiatingUser, + requestMethod, + transportRequestType, + user, + index, + privilege + ); + } + + public AuditMessagePredicate withEffectiveUser(User user) { + return withEffectiveUser(user.getName()); + } + + public AuditMessagePredicate withRestRequest(Method method, String path) { + return this.withLayer(Origin.REST).withRestMethod(method).withRequestPath(path); + } + + public AuditMessagePredicate withIndex(String indexName) { + return new AuditMessagePredicate( + category, + requestLayer, + restRequestPath, + initiatingUser, + requestMethod, + transportRequestType, + effectiveUser, + indexName, + privilege + ); + } + + public AuditMessagePredicate withPrivilege(String privilegeAction) { + return new AuditMessagePredicate( + category, + requestLayer, + restRequestPath, + initiatingUser, + requestMethod, + transportRequestType, + effectiveUser, + index, + privilegeAction + ); + } + + @Override + public boolean test(AuditMessage auditMessage) { + List> predicates = new ArrayList<>(); + predicates.add(audit -> Objects.isNull(category) || category.equals(audit.getCategory())); + predicates.add(audit -> Objects.isNull(requestLayer) || requestLayer.equals(audit.getAsMap().get(REQUEST_LAYER))); + predicates.add(audit -> Objects.isNull(restRequestPath) || restRequestPath.equals(audit.getAsMap().get(REST_REQUEST_PATH))); + predicates.add(audit -> Objects.isNull(initiatingUser) || initiatingUser.equals(audit.getInitiatingUser())); + predicates.add(audit -> Objects.isNull(requestMethod) || requestMethod.equals(audit.getRequestMethod())); + predicates.add(audit -> Objects.isNull(transportRequestType) || transportRequestType.equals(audit.getRequestType())); + predicates.add(audit -> Objects.isNull(effectiveUser) || effectiveUser.equals(audit.getEffectiveUser())); + predicates.add(audit -> Objects.isNull(index) || containIndex(audit, index)); + predicates.add(audit -> Objects.isNull(privilege) || privilege.equals(audit.getPrivilege())); + return predicates.stream().reduce(Predicate::and).orElseThrow().test(auditMessage); + } + + private boolean containIndex(AuditMessage auditMessage, String indexName) { + Map audit = auditMessage.getAsMap(); + return Optional.ofNullable(audit.get(RESOLVED_INDICES)) + .filter(String[].class::isInstance) + .map(String[].class::cast) + .stream() + .flatMap(Arrays::stream) + .collect(Collectors.toSet()) + .contains(indexName); + } + + @Override + public String toString() { + return "AuditMessagePredicate{" + + "category=" + + category + + ", requestLayer=" + + requestLayer + + ", restRequestPath='" + + restRequestPath + + '\'' + + ", requestInitiatingUser='" + + initiatingUser + + '\'' + + ", requestMethod=" + + requestMethod + + ", transportRequestType='" + + transportRequestType + + '\'' + + '}'; + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/audit/TestRuleAuditLogSink.java b/src/integrationTest/java/org/opensearch/test/framework/audit/TestRuleAuditLogSink.java new file mode 100644 index 0000000000..c73d57d23c --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/audit/TestRuleAuditLogSink.java @@ -0,0 +1,51 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.audit; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.opensearch.common.settings.Settings; +import org.opensearch.security.auditlog.impl.AuditMessage; +import org.opensearch.security.auditlog.sink.AuditLogSink; + +public class TestRuleAuditLogSink extends AuditLogSink { + private static final Logger log = LogManager.getLogger(TestRuleAuditLogSink.class); + + private static volatile AuditLogsRule listener; + + public TestRuleAuditLogSink(String name, Settings settings, String settingsPrefix, AuditLogSink fallbackSink) { + super(name, settings, settingsPrefix, fallbackSink); + log.info("Test rule audit log sink created"); + } + + @Override + protected boolean doStore(AuditMessage auditMessage) { + log.debug("New audit message received '{}'.", auditMessage); + AuditLogsRule currentListener = listener; + if (currentListener != null) { + currentListener.onAuditMessage(auditMessage); + } + return true; + } + + public static void registerListener(AuditLogsRule auditLogsRule) { + listener = auditLogsRule; + } + + public static void unregisterListener() { + listener = null; + } + + @Override + public boolean isHandlingBackpressure() { + return true; + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/certificate/AlgorithmKit.java b/src/integrationTest/java/org/opensearch/test/framework/certificate/AlgorithmKit.java new file mode 100644 index 0000000000..60ae56410c --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/certificate/AlgorithmKit.java @@ -0,0 +1,147 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +/* +* Copyright 2021 floragunn GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*/ + +package org.opensearch.test.framework.certificate; + +import java.security.InvalidAlgorithmParameterException; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.NoSuchAlgorithmException; +import java.security.Provider; +import java.security.spec.ECGenParameterSpec; +import java.util.function.Supplier; + +import com.google.common.base.Strings; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import static java.util.Objects.requireNonNull; + +/** +* The class determines cryptographic algorithms used for certificate creation. To create certificate it is necessary to generate public +* and private key, so-called key pair. The class encapsulates the process of key pairs creation ({@link #generateKeyPair()}), +* thus determines algorithm used for key pair creation. Additionally, class defines also algorithms used to digitally sign a certificate. +* Please see {@link #getSignatureAlgorithmName()} +*/ +class AlgorithmKit { + + private static final Logger log = LogManager.getLogger(AlgorithmKit.class); + public static final String SIGNATURE_ALGORITHM_SHA_256_WITH_RSA = "SHA256withRSA"; + public static final String SIGNATURE_ALGORITHM_SHA_256_WITH_ECDSA = "SHA256withECDSA"; + + private final String signatureAlgorithmName; + private final Supplier keyPairSupplier; + + private AlgorithmKit(String signatureAlgorithmName, Supplier keyPairSupplier) { + notEmptyAlgorithmName(signatureAlgorithmName); + this.signatureAlgorithmName = signatureAlgorithmName; + this.keyPairSupplier = requireNonNull(keyPairSupplier, "Key pair supplier is required."); + } + + private static void notEmptyAlgorithmName(String signatureAlgorithmName) { + if (Strings.isNullOrEmpty(signatureAlgorithmName)) { + throw new RuntimeException("Algorithm name is required."); + } + } + + /** + * Static factory method. ECDSA algorithm used for key pair creation. Signature algorithm is defined by field + * {@link #SIGNATURE_ALGORITHM_SHA_256_WITH_ECDSA} + * + * @param securityProvider determines cryptographic algorithm implementation + * @param ellipticCurve + * @return new instance of class {@link AlgorithmKit} + */ + public static AlgorithmKit ecdsaSha256withEcdsa(Provider securityProvider, String ellipticCurve) { + notEmptyAlgorithmName(ellipticCurve); + Supplier supplier = ecdsaKeyPairSupplier(requireNonNull(securityProvider, "Security provider is required"), ellipticCurve); + return new AlgorithmKit(SIGNATURE_ALGORITHM_SHA_256_WITH_ECDSA, supplier); + } + + /** + * Static factory method. It creates object of {@link AlgorithmKit} which enforces usage of RSA algorithm for key pair generation. + * Signature algorithm is defined by {@link #SIGNATURE_ALGORITHM_SHA_256_WITH_RSA} + * + * @param securityProvider determines cryptographic algorithm implementation + * @param keySize defines key size for RSA algorithm + * @return new instance of class {@link AlgorithmKit} + */ + public static AlgorithmKit rsaSha256withRsa(Provider securityProvider, int keySize) { + positiveKeySize(keySize); + Supplier supplier = rsaKeyPairSupplier(securityProvider, keySize); + return new AlgorithmKit(SIGNATURE_ALGORITHM_SHA_256_WITH_RSA, supplier); + } + + private static void positiveKeySize(int keySize) { + if (keySize <= 0) { + throw new RuntimeException("Key size must be a positive integer value, provided: " + keySize); + } + } + + /** + * It determines algorithm used for digital signature + * @return algorithm name + */ + public String getSignatureAlgorithmName() { + return signatureAlgorithmName; + } + + /** + * It creates new private and public key pair + * @return new pair of keys + */ + public KeyPair generateKeyPair() { + return keyPairSupplier.get(); + } + + private static Supplier rsaKeyPairSupplier(Provider securityProvider, int keySize) { + try { + KeyPairGenerator generator = KeyPairGenerator.getInstance("RSA", securityProvider); + log.info("Initialize key pair generator with keySize: {}", keySize); + generator.initialize(keySize); + return generator::generateKeyPair; + } catch (NoSuchAlgorithmException e) { + String message = "Error while initializing RSA asymmetric key generator."; + log.error(message, e); + throw new RuntimeException(message, e); + } + } + + private static Supplier ecdsaKeyPairSupplier(Provider securityProvider, String ellipticCurve) { + try { + KeyPairGenerator generator = KeyPairGenerator.getInstance("EC", securityProvider); + log.info("Initialize key pair generator with elliptic curve: {}", ellipticCurve); + ECGenParameterSpec ecsp = new ECGenParameterSpec(ellipticCurve); + generator.initialize(ecsp); + return generator::generateKeyPair; + } catch (NoSuchAlgorithmException | InvalidAlgorithmParameterException e) { + String message = "Error while initializing ECDSA asymmetric key generator."; + log.error(message, e); + throw new RuntimeException(message, e); + } + } + +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/certificate/CertificateData.java b/src/integrationTest/java/org/opensearch/test/framework/certificate/CertificateData.java new file mode 100644 index 0000000000..09d0f931e6 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/certificate/CertificateData.java @@ -0,0 +1,89 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +/* +* Copyright 2021 floragunn GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*/ + +package org.opensearch.test.framework.certificate; + +import java.security.Key; +import java.security.KeyPair; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; + +import org.bouncycastle.asn1.x500.X500Name; +import org.bouncycastle.cert.X509CertificateHolder; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; + +/** +* The class contains all data related to Certificate including private key which is considered to be a secret. +*/ +public class CertificateData { + + private final X509CertificateHolder certificate; + private final KeyPair keyPair; + + public CertificateData(X509CertificateHolder certificate, KeyPair keyPair) { + this.certificate = certificate; + this.keyPair = keyPair; + } + + /** + * The method returns X.509 certificate encoded in PEM format. PEM format is defined by + * RFC 1421. + * @return Certificate in PEM format + */ + public String certificateInPemFormat() { + return PemConverter.toPem(certificate); + } + + public X509Certificate certificate() { + try { + return new JcaX509CertificateConverter().getCertificate(certificate); + } catch (CertificateException e) { + throw new RuntimeException("Cannot retrieve certificate", e); + } + } + + /** + * It returns the private key associated with certificate encoded in PEM format. PEM format is defined by + * RFC 1421. + * @param privateKeyPassword password used for private key encryption. null for unencrypted key. + * @return private key encoded in PEM format + */ + public String privateKeyInPemFormat(String privateKeyPassword) { + return PemConverter.toPem(keyPair.getPrivate(), privateKeyPassword); + } + + X500Name getCertificateSubject() { + return certificate.getSubject(); + } + + KeyPair getKeyPair() { + return keyPair; + } + + public Key getKey() { + return keyPair.getPrivate(); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/certificate/CertificateMetadata.java b/src/integrationTest/java/org/opensearch/test/framework/certificate/CertificateMetadata.java new file mode 100644 index 0000000000..cc94621f72 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/certificate/CertificateMetadata.java @@ -0,0 +1,220 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.certificate; + +// CS-SUPPRESS-SINGLE: RegexpSingleline Extension is used to refer to certificate extensions, keeping this rule disable for the whole file +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.List; +import java.util.Set; + +import com.google.common.base.Strings; +import org.bouncycastle.asn1.ASN1Encodable; +import org.bouncycastle.asn1.DERSequence; +import org.bouncycastle.asn1.x509.ExtendedKeyUsage; +import org.bouncycastle.asn1.x509.GeneralName; +import org.bouncycastle.asn1.x509.KeyPurposeId; +import org.bouncycastle.asn1.x509.KeyUsage; + +import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptySet; +import static java.util.Objects.requireNonNull; + +/** +*

+* The class represents metadata which should be embedded in certificate to describe a certificate subject (person, company, web server, +* IoT device). The class contains some basic metadata and metadata which should be placed in certificate extensions. +*

+* +*

+* The class is immutable. +*

+* +*/ +class CertificateMetadata { + /** + * Certification subject (person, company, web server, IoT device). The subject of certificate is an owner of the certificate + * (simplification). The format of this field must adhere to RFC 4514. + * @see RFC 4514 + */ + private final String subject; + + /** + * It describes certificate expiration date + */ + private final int validityDays; + + /** + * Optionally used by Open Search to indicate that the certificate can be used by Open Search node to confirm the node identity. The + * value becomes a part of + * SAN (Subject Alternative Name) extension + * + * @see #dnsNames + * @see SAN (Subject Alternative Name) extension + */ + private final String nodeOid; + + /** + * The certificate contains only one {@link #subject}. This is a common limitation when a certificate is used by a web server which is + * associated with a few domains. To overcome this limitation SAN (Subject Alternative Name) extension was introduced. + * The field contains additional subject names which enables creation of so called multi-domain certificates. The extension is defined + * in section 4.2.1.6 of RFC 5280 + * + * @see RFC 5280 + */ + private final List dnsNames; + + /** + * Similar to {@link #dnsNames} but contains IP addresses instead of domains. + */ + private final List ipAddresses; + + /** + * If a private key associated with certificate is used to sign other certificate then this field has to be true. + */ + private final boolean basicConstrainIsCa; + + /** + * Allowed usages for public key associated with certificate + */ + private final Set keyUsages; + + private CertificateMetadata( + String subject, + int validityDays, + String nodeOid, + List dnsNames, + List ipAddresses, + boolean basicConstrainIsCa, + Set keyUsages + ) { + this.subject = subject; + this.validityDays = validityDays; + this.nodeOid = nodeOid; + this.dnsNames = requireNonNull(dnsNames, "List of dns names must not be null."); + this.ipAddresses = requireNonNull(ipAddresses, "List of IP addresses must not be null"); + this.basicConstrainIsCa = basicConstrainIsCa; + this.keyUsages = requireNonNull(keyUsages, "Key usage set must not be null."); + } + + /** + * Static factory method. It creates metadata which contains only basic information. + * @param subjectName please see {@link #subject} + * @param validityDays please see {@link #validityDays} + * @return new instance of {@link CertificateMetadata} + */ + public static CertificateMetadata basicMetadata(String subjectName, int validityDays) { + return new CertificateMetadata(subjectName, validityDays, null, emptyList(), emptyList(), false, emptySet()); + } + + /** + * It is related to private key associated with certificate. It specifies metadata related to allowed private key usage. + * @param basicConstrainIsCa {@link #basicConstrainIsCa} + * @param keyUsages {@link #keyUsages} + * @return returns newly created instance of {@link CertificateData} + */ + public CertificateMetadata withKeyUsage(boolean basicConstrainIsCa, PublicKeyUsage... keyUsages) { + Set usages = arrayToEnumSet(keyUsages); + return new CertificateMetadata(subject, validityDays, nodeOid, dnsNames, ipAddresses, basicConstrainIsCa, usages); + } + + private > Set arrayToEnumSet(T[] enumArray) { + if ((enumArray == null) || (enumArray.length == 0)) { + return Collections.emptySet(); + } + return EnumSet.copyOf(asList(enumArray)); + } + + /** + * The method defines metadata related to SAN (Subject Alternative Name) extension. + * @param nodeOid {@link #nodeOid} + * @param dnsNames {@link #dnsNames} + * @param ipAddresses {@link #ipAddresses} + * @return new instance of {@link CertificateMetadata} + * @see SAN (Subject Alternative Name) extension + */ + public CertificateMetadata withSubjectAlternativeName(String nodeOid, List dnsNames, String... ipAddresses) { + return new CertificateMetadata(subject, validityDays, nodeOid, dnsNames, asList(ipAddresses), basicConstrainIsCa, keyUsages); + } + + /** + * {@link #subject} + * @return Subject name + */ + public String getSubject() { + return subject; + } + + /** + * {@link #validityDays} + * @return determines certificate expiration date + */ + public int getValidityDays() { + return validityDays; + } + + /** + * {@link #basicConstrainIsCa} + * @return Determines if another certificate can be derived from certificate. + */ + public boolean isBasicConstrainIsCa() { + return basicConstrainIsCa; + } + + KeyUsage asKeyUsage() { + Integer keyUsageBitMask = keyUsages.stream() + .filter(PublicKeyUsage::isNotExtendedUsage) + .map(PublicKeyUsage::asInt) + .reduce(0, (accumulator, currentValue) -> accumulator | currentValue); + return new KeyUsage(keyUsageBitMask); + } + + boolean hasSubjectAlternativeNameExtension() { + return ((ipAddresses.size() + dnsNames.size()) > 0) || (Strings.isNullOrEmpty(nodeOid) == false); + } + + DERSequence createSubjectAlternativeNames() { + List subjectAlternativeNameList = new ArrayList<>(); + if (!Strings.isNullOrEmpty(nodeOid)) { + subjectAlternativeNameList.add(new GeneralName(GeneralName.registeredID, nodeOid)); + } + if (isNotEmpty(dnsNames)) { + for (String dnsName : dnsNames) { + subjectAlternativeNameList.add(new GeneralName(GeneralName.dNSName, dnsName)); + } + } + if (isNotEmpty(ipAddresses)) { + for (String ip : ipAddresses) { + subjectAlternativeNameList.add(new GeneralName(GeneralName.iPAddress, ip)); + } + } + return new DERSequence(subjectAlternativeNameList.toArray(ASN1Encodable[]::new)); + } + + private static boolean isNotEmpty(Collection collection) { + return (collection != null) && (!collection.isEmpty()); + } + + boolean hasExtendedKeyUsage() { + return keyUsages.stream().anyMatch(PublicKeyUsage::isNotExtendedUsage); + } + + ExtendedKeyUsage getExtendedKeyUsage() { + KeyPurposeId[] usages = keyUsages.stream() + .filter(PublicKeyUsage::isExtendedUsage) + .map(PublicKeyUsage::getKeyPurposeId) + .toArray(KeyPurposeId[]::new); + return new ExtendedKeyUsage(usages); + } +} +// CS-ENFORCE-SINGLE diff --git a/src/integrationTest/java/org/opensearch/test/framework/certificate/CertificatesIssuer.java b/src/integrationTest/java/org/opensearch/test/framework/certificate/CertificatesIssuer.java new file mode 100644 index 0000000000..877539cc0c --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/certificate/CertificatesIssuer.java @@ -0,0 +1,241 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +/* +* Copyright 2021 floragunn GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*/ + +package org.opensearch.test.framework.certificate; + +// CS-SUPPRESS-SINGLE: RegexpSingleline Extension is used to refer to certificate extensions, keeping this rule disable for the whole file +import java.math.BigInteger; +import java.security.KeyPair; +import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.security.Provider; +import java.security.PublicKey; +import java.util.Calendar; +import java.util.Date; +import java.util.concurrent.atomic.AtomicLong; + +import com.google.common.base.Strings; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bouncycastle.asn1.DERSequence; +import org.bouncycastle.asn1.x500.X500Name; +import org.bouncycastle.asn1.x500.style.RFC4519Style; +import org.bouncycastle.asn1.x509.BasicConstraints; +import org.bouncycastle.asn1.x509.Extension; +import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo; +import org.bouncycastle.cert.CertIOException; +import org.bouncycastle.cert.X509CertificateHolder; +import org.bouncycastle.cert.X509v3CertificateBuilder; +import org.bouncycastle.cert.jcajce.JcaX509ExtensionUtils; +import org.bouncycastle.operator.ContentSigner; +import org.bouncycastle.operator.OperatorCreationException; +import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; + +import static java.util.Objects.requireNonNull; + +/** +*

+* The class is used to generate public key certificate. The class hides low level details related to certificate creation and +* usage of underlying Bouncy Castle library. +*

+*

+* The public key certificate according to its name contains a public key and some metadata. The metadata describes an entity (human, +* company, web server, IoT device, etc.) which is an owner of private key associated with the certificate (private key is not included +* into certificate and is a kind of secret). The responsibility of the class is to issue a certificate. To issue a certificate it is +* necessary to provide metadata which is embedded in the certificates. The metadata is represented by the class +* {@link CertificateMetadata}. Furthermore, the class needs a public key which also must be embedded in the certificate. To obtain public +* and private key pair the class uses {@link AlgorithmKit}. The result of creating certificate is data structure {@link CertificateData}. +* The class {@link CertificateData} contains entire information which is necessary to use the certificate by its owner, that is: +* certificate and private key. +*

+* +*

+* The class is able to create self-signed certificates or certificates signed by some entity. To create a self signed certificate +* the method {@link #issueSignedCertificate(CertificateMetadata, CertificateData)} is used, whereas to create signed certificates +* the method {@link #issueSignedCertificate(CertificateMetadata, CertificateData)} is employed. +*

+*

+* The instance of the class can be obtained by invocation of static method defined in class {@link CertificatesIssuerFactory}. +*

+*/ +class CertificatesIssuer { + + private static final Logger log = LogManager.getLogger(CertificatesIssuer.class); + + private static final AtomicLong ID_COUNTER = new AtomicLong(System.currentTimeMillis()); + + private final Provider securityProvider; + private final AlgorithmKit algorithmKit; + private final JcaX509ExtensionUtils extUtils; + + CertificatesIssuer(Provider securityProvider, AlgorithmKit algorithmKit) { + this.securityProvider = securityProvider; + this.algorithmKit = algorithmKit; + this.extUtils = getExtUtils(); + } + + /** + * The method creates a certificate with provided metadata and public key obtained from {@link #algorithmKit}. The result of invocation + * contains required data to use a certificate by its owner. + * + * @param certificateMetadata metadata which should be embedded into created certificate + * @return {@link CertificateData} which contain certificate and private key associated with the certificate. + */ + public CertificateData issueSelfSignedCertificate(CertificateMetadata certificateMetadata) { + try { + KeyPair publicAndPrivateKey = algorithmKit.generateKeyPair(); + X500Name issuerName = stringToX500Name(requireNonNull(certificateMetadata.getSubject(), "Certificate metadata are required.")); + X509CertificateHolder x509CertificateHolder = buildCertificateHolder( + certificateMetadata, + issuerName, + publicAndPrivateKey.getPublic(), + publicAndPrivateKey + ); + return new CertificateData(x509CertificateHolder, publicAndPrivateKey); + } catch (OperatorCreationException | CertIOException e) { + log.error("Error while generating certificate", e); + throw new RuntimeException("Error while generating self signed certificate", e); + } + } + + /** + * The method is similar to {@link #issueSignedCertificate(CertificateMetadata, CertificateData)} but additionally it signs created + * certificate using data from parentCertificateData. + * + * @param metadata metadata which should be embedded into created certificate + * @param parentCertificateData data required to signe a newly issued certificate (private key among others things). + * @return {@link CertificateData} which contain certificate and private key associated with the certificate. + */ + public CertificateData issueSignedCertificate(CertificateMetadata metadata, CertificateData parentCertificateData) { + try { + KeyPair publicAndPrivateKey = algorithmKit.generateKeyPair(); + KeyPair parentKeyPair = requireNonNull(parentCertificateData, "Issuer certificate data are required").getKeyPair(); + X500Name issuerName = parentCertificateData.getCertificateSubject(); + X509CertificateHolder x509CertificateHolder = buildCertificateHolder( + requireNonNull(metadata, "Certificate metadata are required"), + issuerName, + publicAndPrivateKey.getPublic(), + parentKeyPair + ); + return new CertificateData(x509CertificateHolder, publicAndPrivateKey); + } catch (OperatorCreationException | CertIOException e) { + log.error("Error while generating signed certificate", e); + throw new RuntimeException("Error while generating signed certificate", e); + } + } + + private X509CertificateHolder buildCertificateHolder( + CertificateMetadata certificateMetadata, + X500Name issuerName, + PublicKey certificatePublicKey, + KeyPair parentKeyPair + ) throws CertIOException, OperatorCreationException { + X509v3CertificateBuilder builder = builderWithBasicExtensions( + certificateMetadata, + issuerName, + certificatePublicKey, + parentKeyPair.getPublic() + ); + addSubjectAlternativeNameExtension(builder, certificateMetadata); + addExtendedKeyUsageExtension(builder, certificateMetadata); + return builder.build(createContentSigner(parentKeyPair.getPrivate())); + } + + private ContentSigner createContentSigner(PrivateKey privateKey) throws OperatorCreationException { + return new JcaContentSignerBuilder(algorithmKit.getSignatureAlgorithmName()).setProvider(securityProvider).build(privateKey); + } + + private void addExtendedKeyUsageExtension(X509v3CertificateBuilder builder, CertificateMetadata certificateMetadata) + throws CertIOException { + if (certificateMetadata.hasExtendedKeyUsage()) { + builder.addExtension(Extension.extendedKeyUsage, true, certificateMetadata.getExtendedKeyUsage()); + } + } + + private X509v3CertificateBuilder builderWithBasicExtensions( + CertificateMetadata certificateMetadata, + X500Name issuerName, + PublicKey certificatePublicKey, + PublicKey parentPublicKey + ) throws CertIOException { + X500Name subjectName = stringToX500Name(certificateMetadata.getSubject()); + Date validityStartDate = new Date(System.currentTimeMillis() - (24 * 3600 * 1000)); + Date validityEndDate = getEndDate(validityStartDate, certificateMetadata.getValidityDays()); + + BigInteger certificateSerialNumber = generateNextCertificateSerialNumber(); + return new X509v3CertificateBuilder( + issuerName, + certificateSerialNumber, + validityStartDate, + validityEndDate, + subjectName, + SubjectPublicKeyInfo.getInstance(certificatePublicKey.getEncoded()) + ).addExtension(Extension.basicConstraints, true, new BasicConstraints(certificateMetadata.isBasicConstrainIsCa())) + .addExtension(Extension.authorityKeyIdentifier, false, extUtils.createAuthorityKeyIdentifier(parentPublicKey)) + .addExtension(Extension.subjectKeyIdentifier, false, extUtils.createSubjectKeyIdentifier(certificatePublicKey)) + .addExtension(Extension.keyUsage, true, certificateMetadata.asKeyUsage()); + } + + private void addSubjectAlternativeNameExtension(X509v3CertificateBuilder builder, CertificateMetadata metadata) throws CertIOException { + if (metadata.hasSubjectAlternativeNameExtension()) { + DERSequence subjectAlternativeNames = metadata.createSubjectAlternativeNames(); + builder.addExtension(Extension.subjectAlternativeName, false, subjectAlternativeNames); + } + } + + private Date getEndDate(Date startDate, int validityDays) { + Calendar calendar = Calendar.getInstance(); + calendar.setTime(startDate); + calendar.add(Calendar.DATE, validityDays); + return calendar.getTime(); + } + + private static JcaX509ExtensionUtils getExtUtils() { + try { + return new JcaX509ExtensionUtils(); + } catch (NoSuchAlgorithmException e) { + log.error("Getting certificate extension utils failed", e); + throw new RuntimeException("Getting certificate extension utils failed", e); + } + } + + private X500Name stringToX500Name(String distinguishedName) { + if (Strings.isNullOrEmpty(distinguishedName)) { + throw new RuntimeException("No DN (distinguished name) must not be null or empty"); + } + try { + return new X500Name(RFC4519Style.INSTANCE, distinguishedName); + } catch (IllegalArgumentException e) { + String message = String.format("Invalid DN (distinguished name) specified for %s certificate.", distinguishedName); + throw new RuntimeException(message, e); + } + } + + private BigInteger generateNextCertificateSerialNumber() { + return BigInteger.valueOf(ID_COUNTER.incrementAndGet()); + } +} +// CS-ENFORCE-SINGLE diff --git a/src/integrationTest/java/org/opensearch/test/framework/certificate/CertificatesIssuerFactory.java b/src/integrationTest/java/org/opensearch/test/framework/certificate/CertificatesIssuerFactory.java new file mode 100644 index 0000000000..f68ccf6022 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/certificate/CertificatesIssuerFactory.java @@ -0,0 +1,68 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.certificate; + +import java.security.Provider; +import java.util.Optional; + +import org.bouncycastle.jce.provider.BouncyCastleProvider; + +import static org.opensearch.test.framework.certificate.AlgorithmKit.ecdsaSha256withEcdsa; +import static org.opensearch.test.framework.certificate.AlgorithmKit.rsaSha256withRsa; + +/** +* The class defines static factory method for class {@link CertificatesIssuer}. Object of class {@link CertificatesIssuer} created by +* various factory methods differs in terms of cryptographic algorithms used for certificates creation. +* +*/ +class CertificatesIssuerFactory { + + private static final int KEY_SIZE = 2048; + + private CertificatesIssuerFactory() { + + } + + private static final Provider DEFAULT_SECURITY_PROVIDER = new BouncyCastleProvider(); + + /** + * @see {@link #rsaBaseCertificateIssuer(Provider)} + */ + public static CertificatesIssuer rsaBaseCertificateIssuer() { + return rsaBaseCertificateIssuer(null); + } + + /** + * The method creates {@link CertificatesIssuer} which uses RSA algorithm for certificate creation. + * @param securityProvider determines cryptographic algorithm implementation, can be null. + * @return new instance of {@link CertificatesIssuer} + */ + public static CertificatesIssuer rsaBaseCertificateIssuer(Provider securityProvider) { + Provider provider = Optional.ofNullable(securityProvider).orElse(DEFAULT_SECURITY_PROVIDER); + return new CertificatesIssuer(provider, rsaSha256withRsa(provider, KEY_SIZE)); + } + + /** + * {@link #rsaBaseCertificateIssuer(Provider)} + */ + public static CertificatesIssuer ecdsaBaseCertificatesIssuer() { + return ecdsaBaseCertificatesIssuer(null); + } + + /** + * It creates {@link CertificatesIssuer} which uses asymmetric cryptography algorithm which relays on elliptic curves. + * @param securityProvider determines cryptographic algorithm implementation, can be null. + * @return new instance of {@link CertificatesIssuer} + */ + public static CertificatesIssuer ecdsaBaseCertificatesIssuer(Provider securityProvider) { + Provider provider = Optional.ofNullable(securityProvider).orElse(DEFAULT_SECURITY_PROVIDER); + return new CertificatesIssuer(provider, ecdsaSha256withEcdsa(securityProvider, "P-384")); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/certificate/PemConverter.java b/src/integrationTest/java/org/opensearch/test/framework/certificate/PemConverter.java new file mode 100644 index 0000000000..749ab232bc --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/certificate/PemConverter.java @@ -0,0 +1,119 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +/* +* Copyright 2021 floragunn GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*/ + +package org.opensearch.test.framework.certificate; + +import java.io.IOException; +import java.io.StringWriter; +import java.io.Writer; +import java.security.PrivateKey; +import java.security.SecureRandom; + +import com.google.common.base.Strings; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bouncycastle.asn1.pkcs.PrivateKeyInfo; +import org.bouncycastle.cert.X509CertificateHolder; +import org.bouncycastle.openssl.PKCS8Generator; +import org.bouncycastle.openssl.jcajce.JcaPEMWriter; +import org.bouncycastle.openssl.jcajce.JceOpenSSLPKCS8EncryptorBuilder; +import org.bouncycastle.operator.OperatorCreationException; +import org.bouncycastle.operator.OutputEncryptor; +import org.bouncycastle.util.io.pem.PemGenerationException; +import org.bouncycastle.util.io.pem.PemObject; + +import static java.util.Objects.requireNonNull; + +/** +* The class provides a method useful for converting certificate and private key into PEM format +* @see RFC 1421 +*/ +class PemConverter { + + private PemConverter() {} + + private static final Logger log = LogManager.getLogger(PemConverter.class); + private static final SecureRandom secureRandom = new SecureRandom(); + + /** + * It converts certificate represented by {@link X509CertificateHolder} object to PEM format + * @param certificate is a certificate to convert + * @return {@link String} which contains PEM encoded certificate + */ + public static String toPem(X509CertificateHolder certificate) { + StringWriter stringWriter = new StringWriter(); + try (JcaPEMWriter writer = new JcaPEMWriter(stringWriter)) { + writer.writeObject(requireNonNull(certificate, "Certificate is required.")); + } catch (Exception e) { + throw new RuntimeException("Cannot write certificate in PEM format", e); + } + return stringWriter.toString(); + } + + /** + * It converts private key represented by class {@link PrivateKey} to PEM format. + * @param privateKey is a private key, cannot be null + * @param privateKeyPassword is a password used to encode private key, null for unencrypted private key + * @return {@link String} which contains PEM encoded private key + */ + public static String toPem(PrivateKey privateKey, String privateKeyPassword) { + try (StringWriter stringWriter = new StringWriter()) { + savePrivateKey(stringWriter, requireNonNull(privateKey, "Private key is required."), privateKeyPassword); + return stringWriter.toString(); + } catch (IOException e) { + throw new RuntimeException("Cannot convert private key into PEM format.", e); + } + } + + private static void savePrivateKey(Writer out, PrivateKey privateKey, String privateKeyPassword) { + try (JcaPEMWriter writer = new JcaPEMWriter(out)) { + writer.writeObject(createPkcs8PrivateKeyPem(privateKey, privateKeyPassword)); + } catch (Exception e) { + log.error("Error while writing private key.", e); + throw new RuntimeException("Error while writing private key ", e); + } + } + + private static PemObject createPkcs8PrivateKeyPem(PrivateKey privateKey, String password) { + try { + OutputEncryptor outputEncryptor = password == null ? null : getPasswordEncryptor(password); + return new PKCS8Generator(PrivateKeyInfo.getInstance(privateKey.getEncoded()), outputEncryptor).generate(); + } catch (PemGenerationException | OperatorCreationException e) { + log.error("Creating PKCS8 private key failed", e); + throw new RuntimeException("Creating PKCS8 private key failed", e); + } + } + + private static OutputEncryptor getPasswordEncryptor(String password) throws OperatorCreationException { + if (!Strings.isNullOrEmpty(password)) { + JceOpenSSLPKCS8EncryptorBuilder encryptorBuilder = new JceOpenSSLPKCS8EncryptorBuilder(PKCS8Generator.PBE_SHA1_3DES); + encryptorBuilder.setRandom(secureRandom); + encryptorBuilder.setPassword(password.toCharArray()); + return encryptorBuilder.build(); + } + return null; + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/certificate/PublicKeyUsage.java b/src/integrationTest/java/org/opensearch/test/framework/certificate/PublicKeyUsage.java new file mode 100644 index 0000000000..af37c66001 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/certificate/PublicKeyUsage.java @@ -0,0 +1,75 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.certificate; + +import java.util.Objects; + +import org.bouncycastle.asn1.x509.KeyPurposeId; +import org.bouncycastle.asn1.x509.KeyUsage; + +// CS-SUPPRESS-SINGLE: RegexpSingleline Extension is used to refer to certificate extensions +/** +* The class is associated with certificate extensions related to key usages. These extensions are defined by +* RFC 5280 and describes allowed usage of public kay which is embedded in +* certificate. The class is related to the following extensions: +*
    +*
  1. Key Usage, defined in section 4.2.1.3
    2. +*
  2. Extended Key Usage, defined in section 4.2.1.12
    3. +*
+* +* @see RFC 5280 +*/ +// CS-ENFORCE-SINGLE +enum PublicKeyUsage { + DIGITAL_SIGNATURE(KeyUsage.digitalSignature), + KEY_CERT_SIGN(KeyUsage.keyCertSign), + CRL_SIGN(KeyUsage.cRLSign), + NON_REPUDIATION(KeyUsage.nonRepudiation), + KEY_ENCIPHERMENT(KeyUsage.keyEncipherment), + + SERVER_AUTH(KeyPurposeId.id_kp_serverAuth), + + CLIENT_AUTH(KeyPurposeId.id_kp_clientAuth); + + private final int keyUsage; + private final KeyPurposeId id; + + PublicKeyUsage(int keyUsage) { + this.keyUsage = keyUsage; + this.id = null; + } + + PublicKeyUsage(KeyPurposeId id) { + this.id = Objects.requireNonNull(id, "Key purpose id is required."); + this.keyUsage = 0; + } + + boolean isExtendedUsage() { + return this.id != null; + } + + boolean isNotExtendedUsage() { + return this.id == null; + } + + int asInt() { + if (isExtendedUsage()) { + throw new RuntimeException("Integer value is not available for extended key usage"); + } + return keyUsage; + } + + KeyPurposeId getKeyPurposeId() { + if (isExtendedUsage() == false) { + throw new RuntimeException("Key purpose id is not available."); + } + return id; + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/certificate/TestCertificates.java b/src/integrationTest/java/org/opensearch/test/framework/certificate/TestCertificates.java new file mode 100644 index 0000000000..2dd1dd5eea --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/certificate/TestCertificates.java @@ -0,0 +1,214 @@ +/* +* Copyright 2021 floragunn GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*/ + +/* +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ + +package org.opensearch.test.framework.certificate; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import static org.opensearch.test.framework.certificate.PublicKeyUsage.CLIENT_AUTH; +import static org.opensearch.test.framework.certificate.PublicKeyUsage.CRL_SIGN; +import static org.opensearch.test.framework.certificate.PublicKeyUsage.DIGITAL_SIGNATURE; +import static org.opensearch.test.framework.certificate.PublicKeyUsage.KEY_CERT_SIGN; +import static org.opensearch.test.framework.certificate.PublicKeyUsage.KEY_ENCIPHERMENT; +import static org.opensearch.test.framework.certificate.PublicKeyUsage.NON_REPUDIATION; +import static org.opensearch.test.framework.certificate.PublicKeyUsage.SERVER_AUTH; + +/** +* It provides TLS certificates required in test cases. The certificates are generated during process of creation objects of the class. +* The class exposes method which can be used to write certificates and private keys in temporally files. +*/ +public class TestCertificates { + + private static final Logger log = LogManager.getLogger(TestCertificates.class); + + public static final Integer MAX_NUMBER_OF_NODE_CERTIFICATES = 3; + + private static final String CA_SUBJECT = "DC=com,DC=example,O=Example Com Inc.,OU=Example Com Inc. Root CA,CN=Example Com Inc. Root CA"; + private static final String ADMIN_DN = "CN=kirk,OU=client,O=client,L=test,C=de"; + private static final int CERTIFICATE_VALIDITY_DAYS = 365; + private static final String CERTIFICATE_FILE_EXT = ".cert"; + private static final String KEY_FILE_EXT = ".key"; + private final CertificateData caCertificate; + private final CertificateData adminCertificate; + private final List nodeCertificates; + + private final CertificateData ldapCertificate; + + public TestCertificates() { + this.caCertificate = createCaCertificate(); + this.nodeCertificates = IntStream.range(0, MAX_NUMBER_OF_NODE_CERTIFICATES) + .mapToObj(this::createNodeCertificate) + .collect(Collectors.toList()); + this.ldapCertificate = createLdapCertificate(); + this.adminCertificate = createAdminCertificate(ADMIN_DN); + log.info("Test certificates successfully generated"); + } + + private CertificateData createCaCertificate() { + CertificateMetadata metadata = CertificateMetadata.basicMetadata(CA_SUBJECT, CERTIFICATE_VALIDITY_DAYS) + .withKeyUsage(true, DIGITAL_SIGNATURE, KEY_CERT_SIGN, CRL_SIGN); + return CertificatesIssuerFactory.rsaBaseCertificateIssuer().issueSelfSignedCertificate(metadata); + } + + public CertificateData createAdminCertificate(String adminDn) { + CertificateMetadata metadata = CertificateMetadata.basicMetadata(adminDn, CERTIFICATE_VALIDITY_DAYS) + .withKeyUsage(false, DIGITAL_SIGNATURE, NON_REPUDIATION, KEY_ENCIPHERMENT, CLIENT_AUTH); + return CertificatesIssuerFactory.rsaBaseCertificateIssuer().issueSignedCertificate(metadata, caCertificate); + } + + public CertificateData createSelfSignedCertificate(String distinguishedName) { + CertificateMetadata metadata = CertificateMetadata.basicMetadata(distinguishedName, CERTIFICATE_VALIDITY_DAYS); + return CertificatesIssuerFactory.rsaBaseCertificateIssuer().issueSelfSignedCertificate(metadata); + } + + /** + * It returns the most trusted certificate. Certificates for nodes and users are derived from this certificate. + * @return file which contains certificate in PEM format, defined by RFC 1421 + */ + public File getRootCertificate() { + return createTempFile("root", CERTIFICATE_FILE_EXT, caCertificate.certificateInPemFormat()); + } + + public CertificateData getRootCertificateData() { + return caCertificate; + } + + /** + * Certificate for Open Search node. The certificate is derived from root certificate, returned by method {@link #getRootCertificate()} + * @param node is a node index. It has to be less than {@link #MAX_NUMBER_OF_NODE_CERTIFICATES} + * @return file which contains certificate in PEM format, defined by RFC 1421 + */ + public File getNodeCertificate(int node) { + CertificateData certificateData = getNodeCertificateData(node); + return createTempFile("node-" + node, CERTIFICATE_FILE_EXT, certificateData.certificateInPemFormat()); + } + + public CertificateData getNodeCertificateData(int node) { + isCorrectNodeNumber(node); + return nodeCertificates.get(node); + } + + private void isCorrectNodeNumber(int node) { + if (node >= MAX_NUMBER_OF_NODE_CERTIFICATES) { + String message = String.format( + "Cannot get certificate for node %d, number of created certificates for nodes is %d", + node, + MAX_NUMBER_OF_NODE_CERTIFICATES + ); + throw new RuntimeException(message); + } + } + + private CertificateData createNodeCertificate(Integer node) { + String subject = String.format("DC=de,L=test,O=node,OU=node,CN=node-%d.example.com", node); + String domain = String.format("node-%d.example.com", node); + CertificateMetadata metadata = CertificateMetadata.basicMetadata(subject, CERTIFICATE_VALIDITY_DAYS) + .withKeyUsage(false, DIGITAL_SIGNATURE, NON_REPUDIATION, KEY_ENCIPHERMENT, CLIENT_AUTH, SERVER_AUTH) + .withSubjectAlternativeName("1.2.3.4.5.5", List.of(domain, "localhost"), "127.0.0.1"); + return CertificatesIssuerFactory.rsaBaseCertificateIssuer().issueSignedCertificate(metadata, caCertificate); + } + + public CertificateData issueUserCertificate(String organizationUnit, String username) { + String subject = String.format("DC=de,L=test,O=users,OU=%s,CN=%s", organizationUnit, username); + CertificateMetadata metadata = CertificateMetadata.basicMetadata(subject, CERTIFICATE_VALIDITY_DAYS) + .withKeyUsage(false, DIGITAL_SIGNATURE, NON_REPUDIATION, KEY_ENCIPHERMENT, CLIENT_AUTH, SERVER_AUTH); + return CertificatesIssuerFactory.rsaBaseCertificateIssuer().issueSignedCertificate(metadata, caCertificate); + } + + private CertificateData createLdapCertificate() { + String subject = "DC=de,L=test,O=node,OU=node,CN=ldap.example.com"; + CertificateMetadata metadata = CertificateMetadata.basicMetadata(subject, CERTIFICATE_VALIDITY_DAYS) + .withKeyUsage(false, DIGITAL_SIGNATURE, NON_REPUDIATION, KEY_ENCIPHERMENT, CLIENT_AUTH, SERVER_AUTH) + .withSubjectAlternativeName(null, List.of("localhost"), "127.0.0.1"); + return CertificatesIssuerFactory.rsaBaseCertificateIssuer().issueSignedCertificate(metadata, caCertificate); + } + + public CertificateData getLdapCertificateData() { + return ldapCertificate; + } + + /** + * It returns private key associated with node certificate returned by method {@link #getNodeCertificate(int)} + * + * @param node is a node index. It has to be less than {@link #MAX_NUMBER_OF_NODE_CERTIFICATES} + * @param privateKeyPassword is a password used to encode private key, can be null to retrieve unencrypted key. + * @return file which contains private key encoded in PEM format, defined + * by RFC 1421 + */ + public File getNodeKey(int node, String privateKeyPassword) { + CertificateData certificateData = nodeCertificates.get(node); + return createTempFile("node-" + node, KEY_FILE_EXT, certificateData.privateKeyInPemFormat(privateKeyPassword)); + } + + /** + * Certificate which proofs admin user identity. Certificate is derived from root certificate returned by + * method {@link #getRootCertificate()} + * @return file which contains certificate in PEM format, defined by RFC 1421 + */ + public File getAdminCertificate() { + return createTempFile("admin", CERTIFICATE_FILE_EXT, adminCertificate.certificateInPemFormat()); + } + + public CertificateData getAdminCertificateData() { + return adminCertificate; + } + + /** + * It returns private key associated with admin certificate returned by {@link #getAdminCertificate()}. + * + * @param privateKeyPassword is a password used to encode private key, can be null to retrieve unencrypted key. + * @return file which contains private key encoded in PEM format, defined + * by RFC 1421 + */ + public File getAdminKey(String privateKeyPassword) { + return createTempFile("admin", KEY_FILE_EXT, adminCertificate.privateKeyInPemFormat(privateKeyPassword)); + } + + public String[] getAdminDNs() { + return new String[] { ADMIN_DN }; + } + + private File createTempFile(String name, String suffix, String contents) { + try { + Path path = Files.createTempFile(name, suffix); + Files.writeString(path, contents); + return path.toFile(); + } catch (IOException ex) { + throw new RuntimeException("Cannot create temp file with name " + name + " and suffix " + suffix); + } + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/CloseableHttpClientFactory.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/CloseableHttpClientFactory.java new file mode 100644 index 0000000000..a6a0324b27 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/CloseableHttpClientFactory.java @@ -0,0 +1,81 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.cluster; + +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +import javax.net.ssl.SSLContext; + +import org.apache.http.client.config.RequestConfig; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.HttpClientConnectionManager; +import org.apache.http.conn.routing.HttpRoutePlanner; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; + +class CloseableHttpClientFactory { + + private final SSLContext sslContext; + + private final RequestConfig requestConfig; + + private final HttpRoutePlanner routePlanner; + + private final String[] supportedCipherSuites; + + public CloseableHttpClientFactory( + SSLContext sslContext, + RequestConfig requestConfig, + HttpRoutePlanner routePlanner, + String[] supportedCipherSuit + ) { + this.sslContext = Objects.requireNonNull(sslContext, "SSL context is required."); + this.requestConfig = requestConfig; + this.routePlanner = routePlanner; + this.supportedCipherSuites = supportedCipherSuit; + } + + public CloseableHttpClient getHTTPClient() { + + final HttpClientBuilder hcb = HttpClients.custom(); + + final SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory( + this.sslContext, + /* Uses default supported protocals */ null, + supportedCipherSuites, + NoopHostnameVerifier.INSTANCE + ); + + final HttpClientConnectionManager cm = new PoolingHttpClientConnectionManager( + RegistryBuilder.create().register("https", sslsf).build(), + /* Uses default connnction factory */ null, + /* Uses default scheme port resolver */ null, + /* Uses default dns resolver */ null, + 60, + TimeUnit.SECONDS + ); + hcb.setConnectionManager(cm); + if (routePlanner != null) { + hcb.setRoutePlanner(routePlanner); + } + + if (requestConfig != null) { + hcb.setDefaultRequestConfig(requestConfig); + } + + return hcb.build(); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java new file mode 100644 index 0000000000..0bf50c7a4d --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java @@ -0,0 +1,172 @@ +/* +* Copyright 2015-2017 floragunn GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*/ + +/* +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ + +package org.opensearch.test.framework.cluster; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import org.opensearch.index.reindex.ReindexPlugin; +import org.opensearch.join.ParentJoinPlugin; +import org.opensearch.percolator.PercolatorPlugin; +import org.opensearch.plugins.Plugin; +import org.opensearch.search.aggregations.matrix.MatrixAggregationPlugin; +import org.opensearch.security.OpenSearchSecurityPlugin; +import org.opensearch.transport.Netty4Plugin; + +import static java.util.Collections.unmodifiableList; +import static org.opensearch.test.framework.cluster.NodeType.CLIENT; +import static org.opensearch.test.framework.cluster.NodeType.CLUSTER_MANAGER; +import static org.opensearch.test.framework.cluster.NodeType.DATA; + +public enum ClusterManager { + // 3 nodes (1m, 2d) + DEFAULT(new NodeSettings(NodeRole.CLUSTER_MANAGER), new NodeSettings(NodeRole.DATA), new NodeSettings(NodeRole.DATA)), + + // 1 node (1md) + SINGLENODE(new NodeSettings(NodeRole.CLUSTER_MANAGER, NodeRole.DATA)), + + SINGLE_REMOTE_CLIENT(new NodeSettings(NodeRole.CLUSTER_MANAGER, NodeRole.DATA, NodeRole.REMOTE_CLUSTER_CLIENT)), + + // 4 node (1m, 2d, 1c) + CLIENTNODE( + new NodeSettings(NodeRole.CLUSTER_MANAGER), + new NodeSettings(NodeRole.DATA), + new NodeSettings(NodeRole.DATA), + new NodeSettings() + ), + + THREE_CLUSTER_MANAGERS( + new NodeSettings(NodeRole.CLUSTER_MANAGER), + new NodeSettings(NodeRole.CLUSTER_MANAGER), + new NodeSettings(NodeRole.CLUSTER_MANAGER), + new NodeSettings(NodeRole.DATA), + new NodeSettings(NodeRole.DATA) + ); + + private List nodeSettings = new LinkedList<>(); + + private ClusterManager(NodeSettings... settings) { + nodeSettings.addAll(Arrays.asList(settings)); + } + + public List getNodeSettings() { + return unmodifiableList(nodeSettings); + } + + public List getClusterManagerNodeSettings() { + return unmodifiableList(nodeSettings.stream().filter(a -> a.containRole(NodeRole.CLUSTER_MANAGER)).collect(Collectors.toList())); + } + + public List getNonClusterManagerNodeSettings() { + return unmodifiableList(nodeSettings.stream().filter(a -> !a.containRole(NodeRole.CLUSTER_MANAGER)).collect(Collectors.toList())); + } + + public int getNodes() { + return nodeSettings.size(); + } + + public int getClusterManagerNodes() { + return (int) nodeSettings.stream().filter(a -> a.containRole(NodeRole.CLUSTER_MANAGER)).count(); + } + + public int getDataNodes() { + return (int) nodeSettings.stream().filter(a -> a.containRole(NodeRole.DATA)).count(); + } + + public int getClientNodes() { + return (int) nodeSettings.stream().filter(a -> a.isClientNode()).count(); + } + + public static class NodeSettings { + + private final static List> DEFAULT_PLUGINS = List.of( + Netty4Plugin.class, + OpenSearchSecurityPlugin.class, + MatrixAggregationPlugin.class, + ParentJoinPlugin.class, + PercolatorPlugin.class, + ReindexPlugin.class + ); + + private final Set roles; + public final List> plugins; + + public NodeSettings(NodeRole... roles) { + this(roles.length == 0 ? Collections.emptySet() : EnumSet.copyOf(Arrays.asList(roles)), Collections.emptyList()); + } + + public NodeSettings(Set roles, List> additionalPlugins) { + super(); + this.roles = Objects.requireNonNull(roles, "Node roles set must not be null"); + this.plugins = mergePlugins(additionalPlugins, DEFAULT_PLUGINS); + } + + public boolean containRole(NodeRole nodeRole) { + return roles.contains(nodeRole); + } + + public boolean isClientNode() { + return (roles.contains(NodeRole.DATA) == false) && (roles.contains(NodeRole.CLUSTER_MANAGER)); + } + + NodeType recognizeNodeType() { + if (roles.contains(NodeRole.CLUSTER_MANAGER)) { + return CLUSTER_MANAGER; + } else if (roles.contains(NodeRole.DATA)) { + return DATA; + } else { + return CLIENT; + } + } + + private List> mergePlugins(Collection>... plugins) { + List> mergedPlugins = Arrays.stream(plugins) + .filter(Objects::nonNull) + .flatMap(Collection::stream) + .collect(Collectors.toList()); + return unmodifiableList(mergedPlugins); + } + + @SuppressWarnings("unchecked") + public Class[] getPlugins() { + return plugins.toArray(new Class[0]); + } + + public Collection> pluginsWithAddition(List> additionalPlugins) { + return mergePlugins(plugins, additionalPlugins); + } + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java new file mode 100644 index 0000000000..2b05807fa2 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java @@ -0,0 +1,55 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.cluster; + +import java.util.Collections; +import java.util.Map; + +import org.opensearch.action.ActionListener; +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionResponse; +import org.opensearch.action.ActionType; +import org.opensearch.action.support.ContextPreservingActionListener; +import org.opensearch.client.Client; +import org.opensearch.client.FilterClient; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.common.util.concurrent.ThreadContext.StoredContext; + +/** +* The class adds provided headers into context before sending request via wrapped {@link Client} +*/ +public class ContextHeaderDecoratorClient extends FilterClient { + + private Map headers; + + public ContextHeaderDecoratorClient(Client in, Map headers) { + super(in); + this.headers = headers != null ? headers : Collections.emptyMap(); + } + + @Override + protected void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { + + ThreadContext threadContext = threadPool().getThreadContext(); + ContextPreservingActionListener wrappedListener = new ContextPreservingActionListener<>( + threadContext.newRestorableContext(true), + listener + ); + + try (StoredContext ctx = threadContext.stashContext()) { + threadContext.putHeader(this.headers); + super.doExecute(action, request, wrappedListener); + } + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalAddressRoutePlanner.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalAddressRoutePlanner.java new file mode 100644 index 0000000000..09d8b2b6de --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalAddressRoutePlanner.java @@ -0,0 +1,56 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.cluster; + +import java.net.InetAddress; +import java.util.Objects; + +import org.apache.http.HttpException; +import org.apache.http.HttpHost; +import org.apache.http.HttpRequest; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.conn.routing.HttpRoute; +import org.apache.http.impl.conn.DefaultRoutePlanner; +import org.apache.http.impl.conn.DefaultSchemePortResolver; +import org.apache.http.protocol.HttpContext; + +/** +* Class which can be used to bind Apache HTTP client to a particular network interface or its IP address so that the IP address of +* network interface is used as a source IP address of HTTP request. +*/ +class LocalAddressRoutePlanner extends DefaultRoutePlanner { + + /** + * IP address of one of the local network interfaces. + */ + private final InetAddress localAddress; + + /** + * Creates {@link LocalAddressRoutePlanner} + * @param localAddress IP address of one of the local network interfaces. Client socket used by Apache HTTP client will be bind to + * address from this parameter. The parameter must not be null. + */ + public LocalAddressRoutePlanner(InetAddress localAddress) { + super(DefaultSchemePortResolver.INSTANCE); + this.localAddress = Objects.requireNonNull(localAddress); + } + + @Override + public HttpRoute determineRoute(final HttpHost host, final HttpRequest request, final HttpContext context) throws HttpException { + final HttpClientContext clientContext = HttpClientContext.adapt(context); + final RequestConfig localRequsetConfig = RequestConfig.copy(clientContext.getRequestConfig()) + .setLocalAddress(this.localAddress) + .build(); + clientContext.setRequestConfig(localRequsetConfig); + + return super.determineRoute(host, request, clientContext); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java new file mode 100644 index 0000000000..64207ead5b --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java @@ -0,0 +1,534 @@ +/* +* Copyright 2015-2021 floragunn GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*/ + +/* +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ + +package org.opensearch.test.framework.cluster; + +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.rules.ExternalResource; + +import org.opensearch.client.Client; +import org.opensearch.common.settings.Settings; +import org.opensearch.node.PluginAwareNode; +import org.opensearch.plugins.Plugin; +import org.opensearch.security.action.configupdate.ConfigUpdateAction; +import org.opensearch.security.action.configupdate.ConfigUpdateRequest; +import org.opensearch.security.action.configupdate.ConfigUpdateResponse; +import org.opensearch.security.securityconf.impl.CType; +import org.opensearch.security.support.ConfigConstants; +import org.opensearch.test.framework.AuditConfiguration; +import org.opensearch.test.framework.AuthFailureListeners; +import org.opensearch.test.framework.AuthzDomain; +import org.opensearch.test.framework.OnBehalfOfConfig; +import org.opensearch.test.framework.RolesMapping; +import org.opensearch.test.framework.TestIndex; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.TestSecurityConfig.Role; +import org.opensearch.test.framework.XffConfig; +import org.opensearch.test.framework.audit.TestRuleAuditLogSink; +import org.opensearch.test.framework.certificate.CertificateData; +import org.opensearch.test.framework.certificate.TestCertificates; + +/** +* This class allows to you start and manage a local cluster in an integration test. In contrast to the +* OpenSearchIntegTestCase class, this class can be used in a composite way and allows the specification +* of the security plugin configuration. +* +* This class can be both used as a JUnit @ClassRule (preferred) or in a try-with-resources block. The latter way should +* be only sparingly used, as starting a cluster is not a particularly fast operation. +*/ +public class LocalCluster extends ExternalResource implements AutoCloseable, OpenSearchClientProvider { + + private static final Logger log = LogManager.getLogger(LocalCluster.class); + + public static final String INIT_CONFIGURATION_DIR = "security.default_init.dir"; + + protected static final AtomicLong num = new AtomicLong(); + + private boolean sslOnly; + + private final List> plugins; + private final ClusterManager clusterManager; + private final TestSecurityConfig testSecurityConfig; + private Settings nodeOverride; + private final String clusterName; + private final MinimumSecuritySettingsSupplierFactory minimumOpenSearchSettingsSupplierFactory; + private final TestCertificates testCertificates; + private final List clusterDependencies; + private final Map remotes; + private volatile LocalOpenSearchCluster localOpenSearchCluster; + private final List testIndices; + + private boolean loadConfigurationIntoIndex; + + private LocalCluster( + String clusterName, + TestSecurityConfig testSgConfig, + boolean sslOnly, + Settings nodeOverride, + ClusterManager clusterManager, + List> plugins, + TestCertificates testCertificates, + List clusterDependencies, + Map remotes, + List testIndices, + boolean loadConfigurationIntoIndex, + String defaultConfigurationInitDirectory + ) { + this.plugins = plugins; + this.testCertificates = testCertificates; + this.clusterManager = clusterManager; + this.testSecurityConfig = testSgConfig; + this.sslOnly = sslOnly; + this.nodeOverride = nodeOverride; + this.clusterName = clusterName; + this.minimumOpenSearchSettingsSupplierFactory = new MinimumSecuritySettingsSupplierFactory(testCertificates); + this.remotes = remotes; + this.clusterDependencies = clusterDependencies; + this.testIndices = testIndices; + this.loadConfigurationIntoIndex = loadConfigurationIntoIndex; + if (StringUtils.isNoneBlank(defaultConfigurationInitDirectory)) { + System.setProperty(INIT_CONFIGURATION_DIR, defaultConfigurationInitDirectory); + } + } + + public String getSnapshotDirPath() { + return localOpenSearchCluster.getSnapshotDirPath(); + } + + @Override + public void before() throws Throwable { + if (localOpenSearchCluster == null) { + for (LocalCluster dependency : clusterDependencies) { + if (!dependency.isStarted()) { + dependency.before(); + } + } + + for (Map.Entry entry : remotes.entrySet()) { + @SuppressWarnings("resource") + InetSocketAddress transportAddress = entry.getValue().localOpenSearchCluster.clusterManagerNode().getTransportAddress(); + String key = "cluster.remote." + entry.getKey() + ".seeds"; + String value = transportAddress.getHostString() + ":" + transportAddress.getPort(); + log.info("Remote cluster '{}' added to configuration with the following seed '{}'", key, value); + nodeOverride = Settings.builder().put(nodeOverride).putList(key, value).build(); + } + start(); + } + } + + @Override + protected void after() { + System.clearProperty(INIT_CONFIGURATION_DIR); + close(); + } + + @Override + public void close() { + if (localOpenSearchCluster != null && localOpenSearchCluster.isStarted()) { + try { + localOpenSearchCluster.destroy(); + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + localOpenSearchCluster = null; + } + } + } + + @Override + public String getClusterName() { + return clusterName; + } + + @Override + public InetSocketAddress getHttpAddress() { + return localOpenSearchCluster.clientNode().getHttpAddress(); + } + + public int getHttpPort() { + return getHttpAddress().getPort(); + } + + @Override + public InetSocketAddress getTransportAddress() { + return localOpenSearchCluster.clientNode().getTransportAddress(); + } + + /** + * Returns a Client object that performs cluster-internal requests. As these requests are regard as cluster-internal, + * no authentication is performed and no user-information is attached to these requests. Thus, this client should + * be only used for preparing test environments, but not as a test subject. + */ + public Client getInternalNodeClient() { + return localOpenSearchCluster.clientNode().getInternalNodeClient(); + } + + /** + * Returns a random node of this cluster. + */ + public PluginAwareNode node() { + return this.localOpenSearchCluster.clusterManagerNode().esNode(); + } + + /** + * Returns all nodes of this cluster. + */ + public List nodes() { + return this.localOpenSearchCluster.getNodes(); + } + + public LocalOpenSearchCluster.Node getNodeByName(String name) { + return this.localOpenSearchCluster.getNodeByName(name); + } + + public boolean isStarted() { + return localOpenSearchCluster != null; + } + + public List getConfiguredUsers() { + return testSecurityConfig.getUsers(); + } + + public Random getRandom() { + return localOpenSearchCluster.getRandom(); + } + + private void start() { + try { + NodeSettingsSupplier nodeSettingsSupplier = minimumOpenSearchSettingsSupplierFactory.minimumOpenSearchSettings( + sslOnly, + nodeOverride + ); + localOpenSearchCluster = new LocalOpenSearchCluster( + clusterName, + clusterManager, + nodeSettingsSupplier, + plugins, + testCertificates + ); + + localOpenSearchCluster.start(); + + if (loadConfigurationIntoIndex) { + initSecurityIndex(testSecurityConfig); + } + + try (Client client = getInternalNodeClient()) { + for (TestIndex index : this.testIndices) { + index.create(client); + } + } + + } catch (Exception e) { + log.error("Local ES cluster start failed", e); + throw new RuntimeException(e); + } + } + + private void initSecurityIndex(TestSecurityConfig testSecurityConfig) { + log.info("Initializing OpenSearch Security index"); + try ( + Client client = new ContextHeaderDecoratorClient( + this.getInternalNodeClient(), + Map.of(ConfigConstants.OPENDISTRO_SECURITY_CONF_REQUEST_HEADER, "true") + ) + ) { + testSecurityConfig.initIndex(client); + triggerConfigurationReload(client); + } + } + + public void updateUserConfiguration(List users) { + try ( + Client client = new ContextHeaderDecoratorClient( + this.getInternalNodeClient(), + Map.of(ConfigConstants.OPENDISTRO_SECURITY_CONF_REQUEST_HEADER, "true") + ) + ) { + testSecurityConfig.updateInternalUsersConfiguration(client, users); + triggerConfigurationReload(client); + } + } + + private static void triggerConfigurationReload(Client client) { + ConfigUpdateResponse configUpdateResponse = client.execute( + ConfigUpdateAction.INSTANCE, + new ConfigUpdateRequest(CType.lcStringValues().toArray(new String[0])) + ).actionGet(); + if (configUpdateResponse.hasFailures()) { + throw new RuntimeException("ConfigUpdateResponse produced failures: " + configUpdateResponse.failures()); + } + } + + public CertificateData getAdminCertificate() { + return testCertificates.getAdminCertificateData(); + } + + public static class Builder { + + private final Settings.Builder nodeOverrideSettingsBuilder = Settings.builder(); + + private boolean sslOnly = false; + private final List> plugins = new ArrayList<>(); + private Map remoteClusters = new HashMap<>(); + private List clusterDependencies = new ArrayList<>(); + private List testIndices = new ArrayList<>(); + private ClusterManager clusterManager = ClusterManager.DEFAULT; + private TestSecurityConfig testSecurityConfig = new TestSecurityConfig(); + private String clusterName = "local_cluster"; + private TestCertificates testCertificates; + + private boolean loadConfigurationIntoIndex = true; + + private String defaultConfigurationInitDirectory = null; + + public Builder() {} + + public Builder dependsOn(Object object) { + // We just want to make sure that the object is already done + if (object == null) { + throw new IllegalStateException("Dependency not fulfilled"); + } + return this; + } + + public Builder clusterManager(ClusterManager clusterManager) { + this.clusterManager = clusterManager; + return this; + } + + /** + * Starts a cluster with only one node and thus saves some resources during startup. This shall be only used + * for tests where the node interactions are not relevant to the test. An example for this would be + * authentication tests, as authentication is always done on the directly connected node. + */ + public Builder singleNode() { + this.clusterManager = ClusterManager.SINGLENODE; + return this; + } + + /** + * Specifies the configuration of the security plugin that shall be used by this cluster. + */ + public Builder config(TestSecurityConfig testSecurityConfig) { + this.testSecurityConfig = testSecurityConfig; + return this; + } + + public Builder sslOnly(boolean sslOnly) { + this.sslOnly = sslOnly; + return this; + } + + public Builder nodeSettings(Map settings) { + settings.forEach((key, value) -> { + if (value instanceof List) { + List values = ((List) value).stream().map(String::valueOf).collect(Collectors.toList()); + nodeOverrideSettingsBuilder.putList(key, values); + } else { + nodeOverrideSettingsBuilder.put(key, String.valueOf(value)); + } + }); + + return this; + } + + /** + * Adds additional plugins to the cluster + */ + public Builder plugin(Class plugin) { + this.plugins.add(plugin); + + return this; + } + + public Builder authFailureListeners(AuthFailureListeners listener) { + testSecurityConfig.authFailureListeners(listener); + return this; + } + + /** + * Specifies a remote cluster and its name. The remote cluster can be then used in Cross Cluster Search + * operations with the specified name. + */ + public Builder remote(String name, LocalCluster anotherCluster) { + remoteClusters.put(name, anotherCluster); + + clusterDependencies.add(anotherCluster); + + return this; + } + + /** + * Specifies test indices that shall be created upon startup of the cluster. + */ + public Builder indices(TestIndex... indices) { + this.testIndices.addAll(Arrays.asList(indices)); + return this; + } + + public Builder users(TestSecurityConfig.User... users) { + for (TestSecurityConfig.User user : users) { + testSecurityConfig.user(user); + } + return this; + } + + public Builder audit(AuditConfiguration auditConfiguration) { + if (auditConfiguration != null) { + testSecurityConfig.audit(auditConfiguration); + } + if (auditConfiguration.isEnabled()) { + nodeOverrideSettingsBuilder.put("plugins.security.audit.type", TestRuleAuditLogSink.class.getName()); + } else { + nodeOverrideSettingsBuilder.put("plugins.security.audit.type", "noop"); + } + return this; + } + + public List getUsers() { + return testSecurityConfig.getUsers(); + } + + public Builder roles(Role... roles) { + testSecurityConfig.roles(roles); + return this; + } + + public Builder rolesMapping(RolesMapping... mappings) { + testSecurityConfig.rolesMapping(mappings); + return this; + } + + public Builder authc(TestSecurityConfig.AuthcDomain authc) { + testSecurityConfig.authc(authc); + return this; + } + + public Builder authz(AuthzDomain authzDomain) { + testSecurityConfig.authz(authzDomain); + return this; + } + + public Builder clusterName(String clusterName) { + this.clusterName = clusterName; + return this; + } + + public Builder configIndexName(String configIndexName) { + testSecurityConfig.configIndexName(configIndexName); + return this; + } + + public Builder testCertificates(TestCertificates certificates) { + this.testCertificates = certificates; + return this; + } + + public Builder anonymousAuth(boolean anonAuthEnabled) { + testSecurityConfig.anonymousAuth(anonAuthEnabled); + return this; + } + + public Builder xff(XffConfig xffConfig) { + testSecurityConfig.xff(xffConfig); + return this; + } + + public Builder onBehalfOf(OnBehalfOfConfig onBehalfOfConfig) { + testSecurityConfig.onBehalfOf(onBehalfOfConfig); + return this; + } + + public Builder loadConfigurationIntoIndex(boolean loadConfigurationIntoIndex) { + this.loadConfigurationIntoIndex = loadConfigurationIntoIndex; + return this; + } + + public Builder certificates(TestCertificates certificates) { + this.testCertificates = certificates; + return this; + } + + public Builder doNotFailOnForbidden(boolean doNotFailOnForbidden) { + testSecurityConfig.doNotFailOnForbidden(doNotFailOnForbidden); + return this; + } + + public Builder defaultConfigurationInitDirectory(String defaultConfigurationInitDirectory) { + this.defaultConfigurationInitDirectory = defaultConfigurationInitDirectory; + return this; + } + + public LocalCluster build() { + try { + if (testCertificates == null) { + testCertificates = new TestCertificates(); + } + clusterName += "_" + num.incrementAndGet(); + Settings settings = nodeOverrideSettingsBuilder.build(); + return new LocalCluster( + clusterName, + testSecurityConfig, + sslOnly, + settings, + clusterManager, + plugins, + testCertificates, + clusterDependencies, + remoteClusters, + testIndices, + loadConfigurationIntoIndex, + defaultConfigurationInitDirectory + ); + } catch (Exception e) { + log.error("Failed to build LocalCluster", e); + throw new RuntimeException(e); + } + } + + } + + @Override + public TestCertificates getTestCertificates() { + return testCertificates; + } + +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java new file mode 100644 index 0000000000..f69fd82c83 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java @@ -0,0 +1,576 @@ +/* +* Copyright 2015-2021 floragunn GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*/ + +/* +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ + +package org.opensearch.test.framework.cluster; + +import java.io.File; +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Random; +import java.util.Set; +import java.util.SortedSet; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import javax.net.ssl.SSLContext; + +import com.google.common.collect.ImmutableList; +import com.google.common.net.InetAddresses; +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; +import org.opensearch.client.AdminClient; +import org.opensearch.client.Client; +import org.opensearch.cluster.health.ClusterHealthStatus; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.common.Strings; +import org.opensearch.http.BindHttpException; +import org.opensearch.node.PluginAwareNode; +import org.opensearch.plugins.Plugin; +import org.opensearch.test.framework.certificate.CertificateData; +import org.opensearch.test.framework.certificate.TestCertificates; +import org.opensearch.test.framework.cluster.ClusterManager.NodeSettings; +import org.opensearch.transport.BindTransportException; + +import static java.util.Objects.requireNonNull; +import static org.junit.Assert.assertEquals; +import static org.opensearch.test.framework.cluster.NodeType.CLIENT; +import static org.opensearch.test.framework.cluster.NodeType.CLUSTER_MANAGER; +import static org.opensearch.test.framework.cluster.NodeType.DATA; +import static org.opensearch.test.framework.cluster.PortAllocator.TCP; + +/** +* Encapsulates all the logic to start a local OpenSearch cluster - without any configuration of the security plugin. +* +* The security plugin configuration is the job of LocalCluster, which uses this class under the hood. Thus, test code +* for the security plugin should always use LocalCluster. +*/ +public class LocalOpenSearchCluster { + + static { + System.setProperty("opensearch.enforce.bootstrap.checks", "true"); + } + + private static final Logger log = LogManager.getLogger(LocalOpenSearchCluster.class); + + private final String clusterName; + private final ClusterManager clusterManager; + private final NodeSettingsSupplier nodeSettingsSupplier; + private final List> additionalPlugins; + private final List nodes = new ArrayList<>(); + private final TestCertificates testCertificates; + + private File clusterHomeDir; + private List seedHosts; + private List initialClusterManagerHosts; + private int retry = 0; + private boolean started; + private Random random = new Random(); + + private File snapshotDir; + + public LocalOpenSearchCluster( + String clusterName, + ClusterManager clusterManager, + NodeSettingsSupplier nodeSettingsSupplier, + List> additionalPlugins, + TestCertificates testCertificates + ) { + this.clusterName = clusterName; + this.clusterManager = clusterManager; + this.nodeSettingsSupplier = nodeSettingsSupplier; + this.additionalPlugins = additionalPlugins; + this.testCertificates = testCertificates; + try { + createClusterDirectory(clusterName); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + public String getSnapshotDirPath() { + return snapshotDir.getAbsolutePath(); + } + + private void createClusterDirectory(String clusterName) throws IOException { + this.clusterHomeDir = Files.createTempDirectory("local_cluster_" + clusterName).toFile(); + log.debug("Cluster home directory '{}'.", clusterHomeDir.getAbsolutePath()); + this.snapshotDir = new File(this.clusterHomeDir, "snapshots"); + this.snapshotDir.mkdir(); + } + + private List getNodesByType(NodeType nodeType) { + return nodes.stream().filter(currentNode -> currentNode.hasAssignedType(nodeType)).collect(Collectors.toList()); + } + + private long countNodesByType(NodeType nodeType) { + return getNodesByType(nodeType).stream().count(); + } + + public void start() throws Exception { + log.info("Starting {}", clusterName); + + int clusterManagerNodeCount = clusterManager.getClusterManagerNodes(); + int nonClusterManagerNodeCount = clusterManager.getDataNodes() + clusterManager.getClientNodes(); + + SortedSet clusterManagerNodeTransportPorts = TCP.allocate( + clusterName, + Math.max(clusterManagerNodeCount, 4), + 5000 + 42 * 1000 + 300 + ); + SortedSet clusterManagerNodeHttpPorts = TCP.allocate(clusterName, clusterManagerNodeCount, 5000 + 42 * 1000 + 200); + + this.seedHosts = toHostList(clusterManagerNodeTransportPorts); + Set clusterManagerPorts = clusterManagerNodeTransportPorts.stream() + .limit(clusterManagerNodeCount) + .collect(Collectors.toSet()); + this.initialClusterManagerHosts = toHostList(clusterManagerPorts); + + started = true; + + CompletableFuture clusterManagerNodeFuture = startNodes( + clusterManager.getClusterManagerNodeSettings(), + clusterManagerNodeTransportPorts, + clusterManagerNodeHttpPorts + ); + + SortedSet nonClusterManagerNodeTransportPorts = TCP.allocate( + clusterName, + nonClusterManagerNodeCount, + 5000 + 42 * 1000 + 310 + ); + SortedSet nonClusterManagerNodeHttpPorts = TCP.allocate(clusterName, nonClusterManagerNodeCount, 5000 + 42 * 1000 + 210); + + CompletableFuture nonClusterManagerNodeFuture = startNodes( + clusterManager.getNonClusterManagerNodeSettings(), + nonClusterManagerNodeTransportPorts, + nonClusterManagerNodeHttpPorts + ); + + CompletableFuture.allOf(clusterManagerNodeFuture, nonClusterManagerNodeFuture).join(); + + if (isNodeFailedWithPortCollision()) { + log.info("Detected port collision for cluster manager node. Retrying."); + + retry(); + return; + } + + log.info("Startup finished. Waiting for GREEN"); + + waitForCluster(ClusterHealthStatus.GREEN, TimeValue.timeValueSeconds(10), nodes.size()); + + log.info("Started: {}", this); + + } + + public String getClusterName() { + return clusterName; + } + + public boolean isStarted() { + return started; + } + + public void stop() { + List> stopFutures = new ArrayList<>(); + for (Node node : nodes) { + stopFutures.add(node.stop(2, TimeUnit.SECONDS)); + } + CompletableFuture.allOf(stopFutures.toArray(size -> new CompletableFuture[size])).join(); + } + + public void destroy() { + stop(); + nodes.clear(); + + try { + FileUtils.deleteDirectory(clusterHomeDir); + } catch (IOException e) { + log.warn("Error while deleting " + clusterHomeDir, e); + } + } + + public Node clientNode() { + return findRunningNode(getNodesByType(CLIENT), getNodesByType(DATA), getNodesByType(CLUSTER_MANAGER)); + } + + public Node clusterManagerNode() { + return findRunningNode(getNodesByType(CLUSTER_MANAGER)); + } + + public List getNodes() { + return Collections.unmodifiableList(nodes); + } + + public Node getNodeByName(String name) { + return nodes.stream() + .filter(node -> node.getNodeName().equals(name)) + .findAny() + .orElseThrow( + () -> new RuntimeException( + "No such node with name: " + name + "; available: " + nodes.stream().map(Node::getNodeName).collect(Collectors.toList()) + ) + ); + } + + private boolean isNodeFailedWithPortCollision() { + return nodes.stream().anyMatch(Node::isPortCollision); + } + + private void retry() throws Exception { + retry++; + + if (retry > 10) { + throw new RuntimeException("Detected port collisions for cluster manager node. Giving up."); + } + + stop(); + + this.nodes.clear(); + this.seedHosts = null; + this.initialClusterManagerHosts = null; + createClusterDirectory("local_cluster_" + clusterName + "_retry_" + retry); + start(); + } + + @SafeVarargs + private final Node findRunningNode(List nodes, List... moreNodes) { + for (Node node : nodes) { + if (node.isRunning()) { + return node; + } + } + + if (moreNodes != null && moreNodes.length > 0) { + for (List nodesList : moreNodes) { + for (Node node : nodesList) { + if (node.isRunning()) { + return node; + } + } + } + } + + return null; + } + + private CompletableFuture startNodes( + List nodeSettingList, + SortedSet transportPorts, + SortedSet httpPorts + ) { + Iterator transportPortIterator = transportPorts.iterator(); + Iterator httpPortIterator = httpPorts.iterator(); + List> futures = new ArrayList<>(); + + for (NodeSettings nodeSettings : nodeSettingList) { + Node node = new Node(nodeSettings, transportPortIterator.next(), httpPortIterator.next()); + futures.add(node.start()); + } + + return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + } + + public void waitForCluster(ClusterHealthStatus status, TimeValue timeout, int expectedNodeCount) throws IOException { + Client client = clientNode().getInternalNodeClient(); + + log.debug("waiting for cluster state {} and {} nodes", status.name(), expectedNodeCount); + AdminClient adminClient = client.admin(); + + final ClusterHealthResponse healthResponse = adminClient.cluster() + .prepareHealth() + .setWaitForStatus(status) + .setTimeout(timeout) + .setWaitForNodes("" + expectedNodeCount) + .execute() + .actionGet(); + + if (log.isDebugEnabled()) { + log.debug("Current ClusterState:\n{}", XContentHelper.toXContent(healthResponse, XContentType.JSON, true)); + } + + if (healthResponse.isTimedOut()) { + throw new IOException( + "cluster state is " + healthResponse.getStatus().name() + " with " + healthResponse.getNumberOfNodes() + " nodes" + ); + } else { + log.debug("... cluster state ok {} with {} nodes", healthResponse.getStatus().name(), healthResponse.getNumberOfNodes()); + } + + assertEquals(expectedNodeCount, healthResponse.getNumberOfNodes()); + + } + + @Override + public String toString() { + String clusterManagerNodes = nodeByTypeToString(CLUSTER_MANAGER); + String dataNodes = nodeByTypeToString(DATA); + String clientNodes = nodeByTypeToString(CLIENT); + return "\nOS Cluster " + + clusterName + + "\ncluster manager nodes: " + + clusterManagerNodes + + "\n data nodes: " + + dataNodes + + "\nclient nodes: " + + clientNodes + + "\n"; + } + + private String nodeByTypeToString(NodeType type) { + return getNodesByType(type).stream().map(Objects::toString).collect(Collectors.joining(", ")); + } + + private static List toHostList(Collection ports) { + return ports.stream().map(port -> "127.0.0.1:" + port).collect(Collectors.toList()); + } + + private String createNextNodeName(NodeSettings nodeSettings) { + NodeType type = nodeSettings.recognizeNodeType(); + long nodeTypeCount = countNodesByType(type); + String nodeType = type.name().toLowerCase(Locale.ROOT); + return nodeType + "_" + nodeTypeCount; + } + + public class Node implements OpenSearchClientProvider { + private final NodeType nodeType; + private final String nodeName; + private final NodeSettings nodeSettings; + private final File nodeHomeDir; + private final File dataDir; + private final File logsDir; + private final int transportPort; + private final int httpPort; + private final InetSocketAddress httpAddress; + private final InetSocketAddress transportAddress; + private PluginAwareNode node; + private boolean running = false; + private boolean portCollision = false; + + Node(NodeSettings nodeSettings, int transportPort, int httpPort) { + this.nodeName = createNextNodeName(requireNonNull(nodeSettings, "Node settings are required.")); + this.nodeSettings = nodeSettings; + this.nodeHomeDir = new File(clusterHomeDir, nodeName); + this.dataDir = new File(this.nodeHomeDir, "data"); + this.logsDir = new File(this.nodeHomeDir, "logs"); + this.transportPort = transportPort; + this.httpPort = httpPort; + InetAddress hostAddress = InetAddresses.forString("127.0.0.1"); + this.httpAddress = new InetSocketAddress(hostAddress, httpPort); + this.transportAddress = new InetSocketAddress(hostAddress, transportPort); + + this.nodeType = nodeSettings.recognizeNodeType(); + nodes.add(this); + } + + boolean hasAssignedType(NodeType type) { + return requireNonNull(type, "Node type is required.").equals(this.nodeType); + } + + CompletableFuture start() { + CompletableFuture completableFuture = new CompletableFuture<>(); + final Collection> mergedPlugins = nodeSettings.pluginsWithAddition(additionalPlugins); + this.node = new PluginAwareNode(nodeSettings.containRole(NodeRole.CLUSTER_MANAGER), getOpenSearchSettings(), mergedPlugins); + + new Thread(new Runnable() { + + @Override + public void run() { + try { + node.start(); + running = true; + completableFuture.complete(StartStage.INITIALIZED); + } catch (BindTransportException | BindHttpException e) { + log.warn("Port collision detected for {}", this, e); + portCollision = true; + try { + node.close(); + } catch (IOException e1) { + log.error(e1); + } + + node = null; + TCP.reserve(transportPort, httpPort); + + completableFuture.complete(StartStage.RETRY); + + } catch (Throwable e) { + log.error("Unable to start {}", this, e); + node = null; + completableFuture.completeExceptionally(e); + } + } + }).start(); + + return completableFuture; + } + + public Client getInternalNodeClient() { + return node.client(); + } + + public PluginAwareNode esNode() { + return node; + } + + public boolean isRunning() { + return running; + } + + public X getInjectable(Class clazz) { + return node.injector().getInstance(clazz); + } + + public CompletableFuture stop(long timeout, TimeUnit timeUnit) { + return CompletableFuture.supplyAsync(() -> { + try { + log.info("Stopping {}", this); + + running = false; + + if (node != null) { + node.close(); + boolean stopped = node.awaitClose(timeout, timeUnit); + node = null; + return stopped; + } else { + return false; + } + } catch (Throwable e) { + String message = "Error while stopping " + this; + log.warn(message, e); + throw new RuntimeException(message, e); + } + }); + } + + @Override + public String toString() { + String state = running ? "RUNNING" : node != null ? "INITIALIZING" : "STOPPED"; + + return nodeName + " " + state + " [" + transportPort + ", " + httpPort + "]"; + } + + public boolean isPortCollision() { + return portCollision; + } + + public String getNodeName() { + return nodeName; + } + + @Override + public InetSocketAddress getHttpAddress() { + return httpAddress; + } + + @Override + public InetSocketAddress getTransportAddress() { + return transportAddress; + } + + private Settings getOpenSearchSettings() { + Settings settings = Settings.builder() + .put(getMinimalOpenSearchSettings()) + .putList("path.repo", List.of(getSnapshotDirPath())) + .build(); + + if (nodeSettingsSupplier != null) { + // TODO node number + return Settings.builder().put(settings).put(nodeSettingsSupplier.get(0)).build(); + } + return settings; + } + + private Settings getMinimalOpenSearchSettings() { + return Settings.builder() + .put("node.name", nodeName) + .putList("node.roles", createNodeRolesSettings()) + .put("cluster.name", clusterName) + .put("path.home", nodeHomeDir.toPath()) + .put("path.data", dataDir.toPath()) + .put("path.logs", logsDir.toPath()) + .putList("cluster.initial_master_nodes", initialClusterManagerHosts) + .put("discovery.initial_state_timeout", "8s") + .putList("discovery.seed_hosts", seedHosts) + .put("transport.tcp.port", transportPort) + .put("http.port", httpPort) + .put("cluster.routing.allocation.disk.threshold_enabled", false) + .put("discovery.probe.connect_timeout", "10s") + .put("discovery.probe.handshake_timeout", "10s") + .put("http.cors.enabled", true) + .put("gateway.auto_import_dangling_indices", "true") + .build(); + } + + private List createNodeRolesSettings() { + final ImmutableList.Builder nodeRolesBuilder = ImmutableList.builder(); + if (nodeSettings.containRole(NodeRole.DATA)) { + nodeRolesBuilder.add("data"); + } + if (nodeSettings.containRole(NodeRole.CLUSTER_MANAGER)) { + nodeRolesBuilder.add("master"); + } + if (nodeSettings.containRole(NodeRole.REMOTE_CLUSTER_CLIENT)) { + nodeRolesBuilder.add("remote_cluster_client"); + } + return nodeRolesBuilder.build(); + } + + @Override + public String getClusterName() { + return clusterName; + } + + @Override + public TestCertificates getTestCertificates() { + return testCertificates; + } + } + + public Random getRandom() { + return random; + } + +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java new file mode 100644 index 0000000000..4ad5f8420e --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java @@ -0,0 +1,84 @@ +/* +* Copyright 2021 floragunn GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*/ + +/* +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ + +package org.opensearch.test.framework.cluster; + +import org.opensearch.common.settings.Settings; +import org.opensearch.security.support.ConfigConstants; +import org.opensearch.test.framework.certificate.TestCertificates; + +public class MinimumSecuritySettingsSupplierFactory { + + private final String PRIVATE_KEY_HTTP_PASSWORD = "aWVV63OJ4qzZyPrBwl2MFny4ZV8lQRZchjL"; + private final String PRIVATE_KEY_TRANSPORT_PASSWORD = "iWbUv9w79sbd5tcxvSJNfHXS9GhcPCvdw9x"; + + private TestCertificates testCertificates; + + public MinimumSecuritySettingsSupplierFactory(TestCertificates testCertificates) { + if (testCertificates == null) { + throw new IllegalArgumentException("certificates must not be null"); + } + this.testCertificates = testCertificates; + + } + + public NodeSettingsSupplier minimumOpenSearchSettings(boolean sslOnly, Settings other) { + return i -> minimumOpenSearchSettingsBuilder(i, sslOnly).put(other).build(); + } + + private Settings.Builder minimumOpenSearchSettingsBuilder(int node, boolean sslOnly) { + + Settings.Builder builder = Settings.builder(); + + builder.put("plugins.security.ssl.transport.pemtrustedcas_filepath", testCertificates.getRootCertificate().getAbsolutePath()); + builder.put("plugins.security.ssl.transport.pemcert_filepath", testCertificates.getNodeCertificate(node).getAbsolutePath()); + builder.put( + "plugins.security.ssl.transport.pemkey_filepath", + testCertificates.getNodeKey(node, PRIVATE_KEY_TRANSPORT_PASSWORD).getAbsolutePath() + ); + builder.put("plugins.security.ssl.transport.pemkey_password", PRIVATE_KEY_TRANSPORT_PASSWORD); + + builder.put("plugins.security.ssl.http.enabled", true); + builder.put("plugins.security.ssl.http.pemtrustedcas_filepath", testCertificates.getRootCertificate().getAbsolutePath()); + builder.put("plugins.security.ssl.http.pemcert_filepath", testCertificates.getNodeCertificate(node).getAbsolutePath()); + builder.put( + "plugins.security.ssl.http.pemkey_filepath", + testCertificates.getNodeKey(node, PRIVATE_KEY_HTTP_PASSWORD).getAbsolutePath() + ); + builder.put("plugins.security.ssl.http.pemkey_password", PRIVATE_KEY_HTTP_PASSWORD); + if (sslOnly == false) { + builder.put(ConfigConstants.SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST, false); + builder.putList("plugins.security.authcz.admin_dn", testCertificates.getAdminDNs()); + builder.put("plugins.security.compliance.salt", "1234567890123456"); + builder.put("plugins.security.audit.type", "noop"); + builder.put("plugins.security.background_init_if_securityindex_not_exist", "false"); + } + return builder; + + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeRole.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeRole.java new file mode 100644 index 0000000000..0d465fa119 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeRole.java @@ -0,0 +1,16 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.cluster; + +enum NodeRole { + DATA, + CLUSTER_MANAGER, + REMOTE_CLUSTER_CLIENT +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeSettingsSupplier.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeSettingsSupplier.java new file mode 100644 index 0000000000..cab3a760ca --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeSettingsSupplier.java @@ -0,0 +1,34 @@ +/* +* Copyright 2015-2018 _floragunn_ GmbH +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +/* +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ + +package org.opensearch.test.framework.cluster; + +import org.opensearch.common.settings.Settings; + +@FunctionalInterface +public interface NodeSettingsSupplier { + Settings get(int i); +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeType.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeType.java new file mode 100644 index 0000000000..8ae8941e8d --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeType.java @@ -0,0 +1,17 @@ +/* +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ +package org.opensearch.test.framework.cluster; + +enum NodeType { + CLIENT, + DATA, + CLUSTER_MANAGER +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java new file mode 100644 index 0000000000..0eebcd1545 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java @@ -0,0 +1,261 @@ +/* +* Copyright 2020 floragunn GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*/ + +/* +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ + +package org.opensearch.test.framework.cluster; + +import static org.opensearch.test.framework.cluster.TestRestClientConfiguration.getBasicAuthHeader; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.URI; +import java.security.KeyStore; +import java.security.cert.Certificate; +import java.security.cert.X509Certificate; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import javax.net.ssl.KeyManager; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManagerFactory; + +import org.apache.http.Header; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; +import org.opensearch.client.RestClient; +import org.opensearch.client.RestClientBuilder; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.security.support.PemKeyReader; +import org.opensearch.test.framework.certificate.CertificateData; +import org.opensearch.test.framework.certificate.TestCertificates; + +/** +* OpenSearchClientProvider provides methods to get a REST client for an underlying cluster or node. +* +* This interface is implemented by both LocalCluster and LocalOpenSearchCluster.Node. Thus, it is possible to get a +* REST client for a whole cluster (without choosing the node it is operating on) or to get a REST client for a specific +* node. +*/ +public interface OpenSearchClientProvider { + + String getClusterName(); + + TestCertificates getTestCertificates(); + + InetSocketAddress getHttpAddress(); + + InetSocketAddress getTransportAddress(); + + default URI getHttpAddressAsURI() { + InetSocketAddress address = getHttpAddress(); + return URI.create("https://" + address.getHostString() + ":" + address.getPort()); + } + + /** + * Returns a REST client that sends requests with basic authentication for the specified User object. Optionally, + * additional HTTP headers can be specified which will be sent with each request. + * + * This method should be usually preferred. The other getRestClient() methods shall be only used for specific + * situations. + */ + default TestRestClient getRestClient(UserCredentialsHolder user, CertificateData useCertificateData, Header... headers) { + return getRestClient(user.getName(), user.getPassword(), useCertificateData, headers); + } + + default TestRestClient getRestClient(UserCredentialsHolder user, Header... headers) { + return getRestClient(user.getName(), user.getPassword(), null, headers); + } + + default RestHighLevelClient getRestHighLevelClient(String username, String password, Header... headers) { + return getRestHighLevelClient(new UserCredentialsHolder() { + @Override + public String getName() { + return username; + } + + @Override + public String getPassword() { + return password; + } + }, Arrays.asList(headers)); + } + + default RestHighLevelClient getRestHighLevelClient(UserCredentialsHolder user) { + return getRestHighLevelClient(user, Collections.emptySet()); + } + + default RestHighLevelClient getRestHighLevelClient(UserCredentialsHolder user, Collection defaultHeaders) { + + BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(new AuthScope(null, -1), new UsernamePasswordCredentials(user.getName(), user.getPassword())); + + return getRestHighLevelClient(credentialsProvider, defaultHeaders); + } + + default RestHighLevelClient getRestHighLevelClient(Collection defaultHeaders) { + return getRestHighLevelClient((BasicCredentialsProvider) null, defaultHeaders); + } + + default RestHighLevelClient getRestHighLevelClient( + BasicCredentialsProvider credentialsProvider, + Collection defaultHeaders + ) { + RestClientBuilder.HttpClientConfigCallback configCallback = httpClientBuilder -> { + Optional.ofNullable(credentialsProvider).ifPresent(httpClientBuilder::setDefaultCredentialsProvider); + httpClientBuilder.setSSLStrategy( + new SSLIOSessionStrategy( + getSSLContext(), + /* Use default supported protocols */ null, + /* Use default supported cipher suites */ null, + NoopHostnameVerifier.INSTANCE + ) + ); + httpClientBuilder.setDefaultHeaders(defaultHeaders); + return httpClientBuilder; + }; + + InetSocketAddress httpAddress = getHttpAddress(); + RestClientBuilder builder = RestClient.builder(new HttpHost(httpAddress.getHostString(), httpAddress.getPort(), "https")) + .setHttpClientConfigCallback(configCallback); + + return new RestHighLevelClient(builder); + } + + default org.apache.http.impl.client.CloseableHttpClient getClosableHttpClient(String[] supportedCipherSuit) { + CloseableHttpClientFactory factory = new CloseableHttpClientFactory(getSSLContext(), null, null, supportedCipherSuit); + return factory.getHTTPClient(); + } + + /** + * Returns a REST client that sends requests with basic authentication for the specified user name and password. Optionally, + * additional HTTP headers can be specified which will be sent with each request. + * + * Normally, you should use the method with the User object argument instead. Use this only if you need more + * control over username and password - for example, when you want to send a wrong password. + */ + default TestRestClient getRestClient(String user, String password, Header... headers) { + return createGenericClientRestClient(new TestRestClientConfiguration().username(user).password(password).headers(headers)); + } + + default TestRestClient getRestClient(String user, String password, CertificateData useCertificateData, Header... headers) { + Header basicAuthHeader = getBasicAuthHeader(user, password); + if (headers != null && headers.length > 0) { + List
concatenatedHeaders = Stream.concat(Stream.of(basicAuthHeader), Stream.of(headers)).collect(Collectors.toList()); + return getRestClient(concatenatedHeaders, useCertificateData); + } + return getRestClient(useCertificateData, basicAuthHeader); + } + + /** + * Returns a REST client. You can specify additional HTTP headers that will be sent with each request. Use this + * method to test non-basic authentication, such as JWT bearer authentication. + */ + default TestRestClient getRestClient(CertificateData useCertificateData, Header... headers) { + return getRestClient(Arrays.asList(headers), useCertificateData); + } + + default TestRestClient getRestClient(Header... headers) { + return getRestClient((CertificateData) null, headers); + } + + default TestRestClient getRestClient(List
headers) { + return createGenericClientRestClient(new TestRestClientConfiguration().headers(headers)); + + } + + default TestRestClient getRestClient(List
headers, CertificateData useCertificateData) { + return createGenericClientRestClient(headers, useCertificateData, null); + } + + default TestRestClient createGenericClientRestClient( + List
headers, + CertificateData useCertificateData, + InetAddress sourceInetAddress + ) { + return new TestRestClient(getHttpAddress(), headers, getSSLContext(useCertificateData), sourceInetAddress); + } + + default TestRestClient createGenericClientRestClient(TestRestClientConfiguration configuration) { + return new TestRestClient(getHttpAddress(), configuration.getHeaders(), getSSLContext(), configuration.getSourceInetAddress()); + } + + default SSLContext getSSLContext() { + return getSSLContext(null); + } + + default SSLContext getSSLContext(CertificateData useCertificateData) { + X509Certificate[] trustCertificates; + + try { + trustCertificates = PemKeyReader.loadCertificatesFromFile(getTestCertificates().getRootCertificate().getAbsolutePath()); + + TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType()); + + ks.load(null); + + for (int i = 0; i < trustCertificates.length; i++) { + ks.setCertificateEntry("caCert-" + i, trustCertificates[i]); + } + KeyManager[] keyManagers = null; + if (useCertificateData != null) { + Certificate[] chainOfTrust = { useCertificateData.certificate() }; + ks.setKeyEntry("admin-certificate", useCertificateData.getKey(), null, chainOfTrust); + KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + keyManagerFactory.init(ks, null); + keyManagers = keyManagerFactory.getKeyManagers(); + } + + tmf.init(ks); + + SSLContext sslContext = SSLContext.getInstance("TLS"); + + sslContext.init(keyManagers, tmf.getTrustManagers(), null); + return sslContext; + + } catch (Exception e) { + throw new RuntimeException("Error loading root CA ", e); + } + } + + public interface UserCredentialsHolder { + String getName(); + + String getPassword(); + } + +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/PortAllocator.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/PortAllocator.java new file mode 100644 index 0000000000..139378fd22 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/PortAllocator.java @@ -0,0 +1,165 @@ +/* +* Copyright 2021 floragunn GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*/ + +/* +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ + +package org.opensearch.test.framework.cluster; + +import java.time.Duration; +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.opensearch.test.framework.cluster.SocketUtils.SocketType; + +/** +* Helper class that allows you to allocate ports. This helps with avoiding port conflicts when running tests. +* +* NOTE: This class shall be only considered as a heuristic; ports allocated by this class are just likely to be unused; +* however, there is no guarantee that these will be unused. Thus, you still need to be prepared for port-conflicts +* and retry the procedure in such a case. If you notice a port conflict, you can use the method reserve() to mark the +* port as used. +*/ +public class PortAllocator { + + public static final PortAllocator TCP = new PortAllocator(SocketType.TCP, Duration.ofSeconds(100)); + public static final PortAllocator UDP = new PortAllocator(SocketType.UDP, Duration.ofSeconds(100)); + + private final SocketType socketType; + private final Duration timeoutDuration; + private final Map allocatedPorts = new HashMap<>(); + + PortAllocator(SocketType socketType, Duration timeoutDuration) { + this.socketType = socketType; + this.timeoutDuration = timeoutDuration; + } + + public SortedSet allocate(String clientName, int numRequested, int minPort) { + + int startPort = minPort; + + while (!isAvailable(startPort)) { + startPort += 10; + } + + SortedSet foundPorts = new TreeSet<>(); + + for (int currentPort = startPort; foundPorts.size() < numRequested + && currentPort < SocketUtils.PORT_RANGE_MAX + && (currentPort - startPort) < 10000; currentPort++) { + if (allocate(clientName, currentPort)) { + foundPorts.add(currentPort); + } + } + + if (foundPorts.size() < numRequested) { + throw new IllegalStateException("Could not find " + numRequested + " free ports starting at " + minPort + " for " + clientName); + } + + return foundPorts; + } + + public int allocateSingle(String clientName, int minPort) { + + int startPort = minPort; + + for (int currentPort = startPort; currentPort < SocketUtils.PORT_RANGE_MAX && (currentPort - startPort) < 10000; currentPort++) { + if (allocate(clientName, currentPort)) { + return currentPort; + } + } + + throw new IllegalStateException("Could not find free port starting at " + minPort + " for " + clientName); + + } + + public void reserve(int... ports) { + + for (int port : ports) { + allocate("reserved", port); + } + } + + private boolean isInUse(int port) { + boolean result = !this.socketType.isPortAvailable(port); + + if (result) { + synchronized (this) { + allocatedPorts.put(port, new AllocatedPort("external")); + } + } + + return result; + } + + private boolean isAvailable(int port) { + return !isAllocated(port) && !isInUse(port); + } + + private synchronized boolean isAllocated(int port) { + AllocatedPort allocatedPort = this.allocatedPorts.get(port); + + return allocatedPort != null && !allocatedPort.isTimedOut(); + } + + private synchronized boolean allocate(String clientName, int port) { + + AllocatedPort allocatedPort = allocatedPorts.get(port); + + if (allocatedPort != null && allocatedPort.isTimedOut()) { + allocatedPort = null; + allocatedPorts.remove(port); + } + + if (allocatedPort == null && !isInUse(port)) { + allocatedPorts.put(port, new AllocatedPort(clientName)); + return true; + } else { + return false; + } + } + + private class AllocatedPort { + final String client; + final Instant allocatedAt; + + AllocatedPort(String client) { + this.client = client; + this.allocatedAt = Instant.now(); + } + + boolean isTimedOut() { + return allocatedAt.plus(timeoutDuration).isBefore(Instant.now()); + } + + @Override + public String toString() { + return "AllocatedPort [client=" + client + ", allocatedAt=" + allocatedAt + "]"; + } + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/RestClientException.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/RestClientException.java new file mode 100644 index 0000000000..0023d65e98 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/RestClientException.java @@ -0,0 +1,16 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.cluster; + +public class RestClientException extends RuntimeException { + RestClientException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/SearchRequestFactory.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/SearchRequestFactory.java new file mode 100644 index 0000000000..b40aa9cfcb --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/SearchRequestFactory.java @@ -0,0 +1,104 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.cluster; + +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.search.SearchScrollRequest; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.sort.FieldSortBuilder; +import org.opensearch.search.sort.SortOrder; + +import static java.util.concurrent.TimeUnit.MINUTES; + +public final class SearchRequestFactory { + + private SearchRequestFactory() { + + } + + public static SearchRequest queryByIdsRequest(String indexName, String... ids) { + SearchRequest searchRequest = new SearchRequest(indexName); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.idsQuery().addIds(ids)); + searchRequest.source(searchSourceBuilder); + return searchRequest; + } + + public static SearchRequest queryStringQueryRequest(String indexName, String queryString) { + SearchRequest searchRequest = new SearchRequest(indexName); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.queryStringQuery(queryString)); + searchRequest.source(searchSourceBuilder); + return searchRequest; + } + + public static SearchRequest queryStringQueryRequest(String[] indicesNames, String queryString) { + SearchRequest searchRequest = new SearchRequest(indicesNames); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.queryStringQuery(queryString)); + searchRequest.source(searchSourceBuilder); + return searchRequest; + } + + public static SearchRequest queryStringQueryRequest(String queryString) { + SearchRequest searchRequest = new SearchRequest(); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.queryStringQuery(queryString)); + searchRequest.source(searchSourceBuilder); + return searchRequest; + } + + public static SearchRequest searchRequestWithScroll(String indexName, int pageSize) { + SearchRequest searchRequest = new SearchRequest(indexName); + searchRequest.scroll(new TimeValue(1, MINUTES)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.matchAllQuery()); + searchSourceBuilder.sort(new FieldSortBuilder("_id").order(SortOrder.ASC)); + searchSourceBuilder.size(pageSize); + searchRequest.source(searchSourceBuilder); + return searchRequest; + } + + public static SearchRequest searchAll(String... indexNames) { + SearchRequest searchRequest = new SearchRequest(indexNames); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.matchAllQuery()); + searchRequest.source(searchSourceBuilder); + return searchRequest; + } + + public static SearchScrollRequest getSearchScrollRequest(SearchResponse searchResponse) { + SearchScrollRequest scrollRequest = new SearchScrollRequest(searchResponse.getScrollId()); + scrollRequest.scroll(new TimeValue(1, MINUTES)); + return scrollRequest; + } + + public static SearchRequest averageAggregationRequest(String indexName, String aggregationName, String fieldName) { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.aggregation(AggregationBuilders.avg(aggregationName).field(fieldName)); + searchSourceBuilder.size(0); + SearchRequest searchRequest = new SearchRequest(indexName); + searchRequest.source(searchSourceBuilder); + return searchRequest; + } + + public static SearchRequest statsAggregationRequest(String indexName, String aggregationName, String fieldName) { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.aggregation(AggregationBuilders.stats(aggregationName).field(fieldName)); + searchSourceBuilder.size(0); + SearchRequest searchRequest = new SearchRequest(indexName); + searchRequest.source(searchSourceBuilder); + return searchRequest; + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtils.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtils.java new file mode 100644 index 0000000000..5895829243 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtils.java @@ -0,0 +1,311 @@ +/* +* Copyright 2002-2017 the original author or authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +/* +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ + +package org.opensearch.test.framework.cluster; + +import java.net.DatagramSocket; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.util.Random; +import java.util.SortedSet; +import java.util.TreeSet; + +import javax.net.ServerSocketFactory; + +/** +* Simple utility methods for working with network sockets — for example, +* for finding available ports on {@code localhost}. +* +*

Within this class, a TCP port refers to a port for a {@link ServerSocket}; +* whereas, a UDP port refers to a port for a {@link DatagramSocket}. +* +* @author Sam Brannen +* @author Ben Hale +* @author Arjen Poutsma +* @author Gunnar Hillert +* @author Gary Russell +* @since 4.0 +*/ +public class SocketUtils { + + /** + * The default minimum value for port ranges used when finding an available + * socket port. + */ + public static final int PORT_RANGE_MIN = 1024; + + /** + * The default maximum value for port ranges used when finding an available + * socket port. + */ + public static final int PORT_RANGE_MAX = 65535; + + private static final Random random = new Random(System.currentTimeMillis()); + + /** + * Although {@code SocketUtils} consists solely of static utility methods, + * this constructor is intentionally {@code public}. + *

Rationale

+ *

Static methods from this class may be invoked from within XML + * configuration files using the Spring Expression Language (SpEL) and the + * following syntax. + * 

<bean id="bean1" ... p:port="#{T(org.springframework.util.SocketUtils).findAvailableTcpPort(12000)}" />
+ * If this constructor were {@code private}, you would be required to supply + * the fully qualified class name to SpEL's {@code T()} function for each usage. + * Thus, the fact that this constructor is {@code public} allows you to reduce + * boilerplate configuration with SpEL as can be seen in the following example. + * 
<bean id="socketUtils" class="org.springframework.util.SocketUtils" />
+    * <bean id="bean1" ... p:port="#{socketUtils.findAvailableTcpPort(12000)}" />
+    * <bean id="bean2" ... p:port="#{socketUtils.findAvailableTcpPort(30000)}" />
+ */ + public SocketUtils() { + /* no-op */ + } + + /** + * Find an available TCP port randomly selected from the range + * [{@value #PORT_RANGE_MIN}, {@value #PORT_RANGE_MAX}]. + * @return an available TCP port number + * @throws IllegalStateException if no available port could be found + */ + public static int findAvailableTcpPort() { + return findAvailableTcpPort(PORT_RANGE_MIN); + } + + /** + * Find an available TCP port randomly selected from the range + * [{@code minPort}, {@value #PORT_RANGE_MAX}]. + * @param minPort the minimum port number + * @return an available TCP port number + * @throws IllegalStateException if no available port could be found + */ + public static int findAvailableTcpPort(int minPort) { + return findAvailableTcpPort(minPort, PORT_RANGE_MAX); + } + + /** + * Find an available TCP port randomly selected from the range + * [{@code minPort}, {@code maxPort}]. + * @param minPort the minimum port number + * @param maxPort the maximum port number + * @return an available TCP port number + * @throws IllegalStateException if no available port could be found + */ + public static int findAvailableTcpPort(int minPort, int maxPort) { + return SocketType.TCP.findAvailablePort(minPort, maxPort); + } + + /** + * Find the requested number of available TCP ports, each randomly selected + * from the range [{@value #PORT_RANGE_MIN}, {@value #PORT_RANGE_MAX}]. + * @param numRequested the number of available ports to find + * @return a sorted set of available TCP port numbers + * @throws IllegalStateException if the requested number of available ports could not be found + */ + public static SortedSet findAvailableTcpPorts(int numRequested) { + return findAvailableTcpPorts(numRequested, PORT_RANGE_MIN, PORT_RANGE_MAX); + } + + /** + * Find the requested number of available TCP ports, each randomly selected + * from the range [{@code minPort}, {@code maxPort}]. + * @param numRequested the number of available ports to find + * @param minPort the minimum port number + * @param maxPort the maximum port number + * @return a sorted set of available TCP port numbers + * @throws IllegalStateException if the requested number of available ports could not be found + */ + public static SortedSet findAvailableTcpPorts(int numRequested, int minPort, int maxPort) { + return SocketType.TCP.findAvailablePorts(numRequested, minPort, maxPort); + } + + /** + * Find an available UDP port randomly selected from the range + * [{@value #PORT_RANGE_MIN}, {@value #PORT_RANGE_MAX}]. + * @return an available UDP port number + * @throws IllegalStateException if no available port could be found + */ + public static int findAvailableUdpPort() { + return findAvailableUdpPort(PORT_RANGE_MIN); + } + + /** + * Find an available UDP port randomly selected from the range + * [{@code minPort}, {@value #PORT_RANGE_MAX}]. + * @param minPort the minimum port number + * @return an available UDP port number + * @throws IllegalStateException if no available port could be found + */ + public static int findAvailableUdpPort(int minPort) { + return findAvailableUdpPort(minPort, PORT_RANGE_MAX); + } + + /** + * Find an available UDP port randomly selected from the range + * [{@code minPort}, {@code maxPort}]. + * @param minPort the minimum port number + * @param maxPort the maximum port number + * @return an available UDP port number + * @throws IllegalStateException if no available port could be found + */ + public static int findAvailableUdpPort(int minPort, int maxPort) { + return SocketType.UDP.findAvailablePort(minPort, maxPort); + } + + /** + * Find the requested number of available UDP ports, each randomly selected + * from the range [{@value #PORT_RANGE_MIN}, {@value #PORT_RANGE_MAX}]. + * @param numRequested the number of available ports to find + * @return a sorted set of available UDP port numbers + * @throws IllegalStateException if the requested number of available ports could not be found + */ + public static SortedSet findAvailableUdpPorts(int numRequested) { + return findAvailableUdpPorts(numRequested, PORT_RANGE_MIN, PORT_RANGE_MAX); + } + + /** + * Find the requested number of available UDP ports, each randomly selected + * from the range [{@code minPort}, {@code maxPort}]. + * @param numRequested the number of available ports to find + * @param minPort the minimum port number + * @param maxPort the maximum port number + * @return a sorted set of available UDP port numbers + * @throws IllegalStateException if the requested number of available ports could not be found + */ + public static SortedSet findAvailableUdpPorts(int numRequested, int minPort, int maxPort) { + return SocketType.UDP.findAvailablePorts(numRequested, minPort, maxPort); + } + + public enum SocketType { + + TCP { + @Override + protected boolean isPortAvailable(int port) { + try { + ServerSocket serverSocket = ServerSocketFactory.getDefault() + .createServerSocket(port, 1, InetAddress.getByName("localhost")); + serverSocket.close(); + return true; + } catch (Exception ex) { + return false; + } + } + }, + + UDP { + @Override + protected boolean isPortAvailable(int port) { + try { + DatagramSocket socket = new DatagramSocket(port, InetAddress.getByName("localhost")); + socket.close(); + return true; + } catch (Exception ex) { + return false; + } + } + }; + + /** + * Determine if the specified port for this {@code SocketType} is + * currently available on {@code localhost}. + */ + protected abstract boolean isPortAvailable(int port); + + /** + * Find a pseudo-random port number within the range + * [{@code minPort}, {@code maxPort}]. + * @param minPort the minimum port number + * @param maxPort the maximum port number + * @return a random port number within the specified range + */ + private int findRandomPort(int minPort, int maxPort) { + int portRange = maxPort - minPort; + return minPort + random.nextInt(portRange + 1); + } + + /** + * Find an available port for this {@code SocketType}, randomly selected + * from the range [{@code minPort}, {@code maxPort}]. + * @param minPort the minimum port number + * @param maxPort the maximum port number + * @return an available port number for this socket type + * @throws IllegalStateException if no available port could be found + */ + int findAvailablePort(int minPort, int maxPort) { + // Assert.assertTrue(minPort > 0, "'minPort' must be greater than 0"); + // Assert.isTrue(maxPort >= minPort, "'maxPort' must be greater than or equal to 'minPort'"); + // Assert.isTrue(maxPort <= PORT_RANGE_MAX, "'maxPort' must be less than or equal to " + PORT_RANGE_MAX); + + int portRange = maxPort - minPort; + int candidatePort; + int searchCounter = 0; + do { + if (searchCounter > portRange) { + throw new IllegalStateException( + String.format( + "Could not find an available %s port in the range [%d, %d] after %d attempts", + name(), + minPort, + maxPort, + searchCounter + ) + ); + } + candidatePort = findRandomPort(minPort, maxPort); + searchCounter++; + } while (!isPortAvailable(candidatePort)); + + return candidatePort; + } + + /** + * Find the requested number of available ports for this {@code SocketType}, + * each randomly selected from the range [{@code minPort}, {@code maxPort}]. + * @param numRequested the number of available ports to find + * @param minPort the minimum port number + * @param maxPort the maximum port number + * @return a sorted set of available port numbers for this socket type + * @throws IllegalStateException if the requested number of available ports could not be found + */ + SortedSet findAvailablePorts(int numRequested, int minPort, int maxPort) { + SortedSet availablePorts = new TreeSet<>(); + int attemptCount = 0; + while ((++attemptCount <= numRequested + 100) && availablePorts.size() < numRequested) { + availablePorts.add(findAvailablePort(minPort, maxPort)); + } + + if (availablePorts.size() != numRequested) { + throw new IllegalStateException( + String.format("Could not find %d available %s ports in the range [%d, %d]", numRequested, name(), minPort, maxPort) + ); + } + + return availablePorts; + } + } + +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtilsTests.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtilsTests.java new file mode 100644 index 0000000000..fb298c5283 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtilsTests.java @@ -0,0 +1,207 @@ +/* +* Copyright 2002-2020 the original author or authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* https://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +/* +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ + +package org.opensearch.test.framework.cluster; + +import java.net.DatagramSocket; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.util.SortedSet; + +import javax.net.ServerSocketFactory; + +import org.junit.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.startsWith; +import static org.junit.Assert.assertThrows; +import static org.opensearch.test.framework.cluster.SocketUtils.PORT_RANGE_MAX; +import static org.opensearch.test.framework.cluster.SocketUtils.PORT_RANGE_MIN; + +/** +* Unit tests for {@link SocketUtils}. +* +* @author Sam Brannen +* @author Gary Russell +*/ +public class SocketUtilsTests { + + // TCP + + @Test + public void findAvailableTcpPort() { + int port = SocketUtils.findAvailableTcpPort(); + assertPortInRange(port, PORT_RANGE_MIN, PORT_RANGE_MAX); + } + + @Test + public void findAvailableTcpPortWithMinPortEqualToMaxPort() { + int minMaxPort = SocketUtils.findAvailableTcpPort(); + int port = SocketUtils.findAvailableTcpPort(minMaxPort, minMaxPort); + assertThat(port, equalTo(minMaxPort)); + } + + @Test + public void findAvailableTcpPortWhenPortOnLoopbackInterfaceIsNotAvailable() throws Exception { + int port = SocketUtils.findAvailableTcpPort(); + try (ServerSocket socket = ServerSocketFactory.getDefault().createServerSocket(port, 1, InetAddress.getByName("localhost"))) { + assertThat(socket, notNullValue()); + // will only look for the exact port + IllegalStateException exception = assertThrows(IllegalStateException.class, () -> SocketUtils.findAvailableTcpPort(port, port)); + assertThat(exception.getMessage(), startsWith("Could not find an available TCP port")); + assertThat(exception.getMessage(), endsWith("after 1 attempts")); + } + } + + @Test + public void findAvailableTcpPortWithMin() { + int port = SocketUtils.findAvailableTcpPort(50000); + assertPortInRange(port, 50000, PORT_RANGE_MAX); + } + + @Test + public void findAvailableTcpPortInRange() { + int minPort = 20000; + int maxPort = minPort + 1000; + int port = SocketUtils.findAvailableTcpPort(minPort, maxPort); + assertPortInRange(port, minPort, maxPort); + } + + @Test + public void find4AvailableTcpPorts() { + findAvailableTcpPorts(4); + } + + @Test + public void find50AvailableTcpPorts() { + findAvailableTcpPorts(50); + } + + @Test + public void find4AvailableTcpPortsInRange() { + findAvailableTcpPorts(4, 30000, 35000); + } + + @Test + public void find50AvailableTcpPortsInRange() { + findAvailableTcpPorts(50, 40000, 45000); + } + + // UDP + + @Test + public void findAvailableUdpPort() { + int port = SocketUtils.findAvailableUdpPort(); + assertPortInRange(port, PORT_RANGE_MIN, PORT_RANGE_MAX); + } + + @Test + public void findAvailableUdpPortWhenPortOnLoopbackInterfaceIsNotAvailable() throws Exception { + int port = SocketUtils.findAvailableUdpPort(); + try (DatagramSocket socket = new DatagramSocket(port, InetAddress.getByName("localhost"))) { + assertThat(socket, notNullValue()); + // will only look for the exact port + IllegalStateException exception = assertThrows(IllegalStateException.class, () -> SocketUtils.findAvailableUdpPort(port, port)); + assertThat(exception.getMessage(), startsWith("Could not find an available UDP port")); + assertThat(exception.getMessage(), endsWith("after 1 attempts")); + } + } + + @Test + public void findAvailableUdpPortWithMin() { + int port = SocketUtils.findAvailableUdpPort(50000); + assertPortInRange(port, 50000, PORT_RANGE_MAX); + } + + @Test + public void findAvailableUdpPortInRange() { + int minPort = 20000; + int maxPort = minPort + 1000; + int port = SocketUtils.findAvailableUdpPort(minPort, maxPort); + assertPortInRange(port, minPort, maxPort); + } + + @Test + public void find4AvailableUdpPorts() { + findAvailableUdpPorts(4); + } + + @Test + public void find50AvailableUdpPorts() { + findAvailableUdpPorts(50); + } + + @Test + public void find4AvailableUdpPortsInRange() { + findAvailableUdpPorts(4, 30000, 35000); + } + + @Test + public void find50AvailableUdpPortsInRange() { + findAvailableUdpPorts(50, 40000, 45000); + } + + // Helpers + + private void findAvailableTcpPorts(int numRequested) { + SortedSet ports = SocketUtils.findAvailableTcpPorts(numRequested); + assertAvailablePorts(ports, numRequested, PORT_RANGE_MIN, PORT_RANGE_MAX); + } + + private void findAvailableTcpPorts(int numRequested, int minPort, int maxPort) { + SortedSet ports = SocketUtils.findAvailableTcpPorts(numRequested, minPort, maxPort); + assertAvailablePorts(ports, numRequested, minPort, maxPort); + } + + private void findAvailableUdpPorts(int numRequested) { + SortedSet ports = SocketUtils.findAvailableUdpPorts(numRequested); + assertAvailablePorts(ports, numRequested, PORT_RANGE_MIN, PORT_RANGE_MAX); + } + + private void findAvailableUdpPorts(int numRequested, int minPort, int maxPort) { + SortedSet ports = SocketUtils.findAvailableUdpPorts(numRequested, minPort, maxPort); + assertAvailablePorts(ports, numRequested, minPort, maxPort); + } + + private void assertPortInRange(int port, int minPort, int maxPort) { + assertThat("port [" + port + "] >= " + minPort, port, greaterThanOrEqualTo(minPort)); + assertThat("port [" + port + "] <= " + maxPort, port, lessThanOrEqualTo(maxPort)); + } + + private void assertAvailablePorts(SortedSet ports, int numRequested, int minPort, int maxPort) { + assertThat("number of ports requested", ports.size(), equalTo(numRequested)); + for (int port : ports) { + assertPortInRange(port, minPort, maxPort); + } + } + +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/StartStage.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/StartStage.java new file mode 100644 index 0000000000..d5dce0056a --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/StartStage.java @@ -0,0 +1,15 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.cluster; + +enum StartStage { + INITIALIZED, + RETRY +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java new file mode 100644 index 0000000000..722c64758a --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java @@ -0,0 +1,457 @@ +/* +* Copyright 2021 floragunn GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*/ + +/* +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ + +package org.opensearch.test.framework.cluster; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +import javax.net.ssl.SSLContext; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.commons.io.IOUtils; +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.NameValuePair; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpHead; +import org.apache.http.client.methods.HttpOptions; +import org.apache.http.client.methods.HttpPatch; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.conn.routing.HttpRoutePlanner; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.message.BasicHeader; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.common.Strings; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.security.DefaultObjectMapper; + +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; + +/** +* A OpenSearch REST client, which is tailored towards use in integration tests. Instances of this class can be +* obtained via the OpenSearchClientProvider interface, which is implemented by LocalCluster and Node. +* +* Usually, an instance of this class sends constant authentication headers which are defined when obtaining the +* instance from OpenSearchClientProvider. +*/ +public class TestRestClient implements AutoCloseable { + + private static final Logger log = LogManager.getLogger(TestRestClient.class); + + private boolean enableHTTPClientSSL = true; + private boolean sendHTTPClientCertificate = false; + private InetSocketAddress nodeHttpAddress; + private RequestConfig requestConfig; + private List
headers = new ArrayList<>(); + private Header CONTENT_TYPE_JSON = new BasicHeader("Content-Type", "application/json"); + private SSLContext sslContext; + + private final InetAddress sourceInetAddress; + + public TestRestClient(InetSocketAddress nodeHttpAddress, List
headers, SSLContext sslContext, InetAddress sourceInetAddress) { + this.nodeHttpAddress = nodeHttpAddress; + this.headers.addAll(headers); + this.sslContext = sslContext; + this.sourceInetAddress = sourceInetAddress; + } + + public HttpResponse get(String path, List queryParameters, Header... headers) { + try { + URI uri = new URIBuilder(getHttpServerUri()).setPath(path).addParameters(queryParameters).build(); + return executeRequest(new HttpGet(uri), headers); + } catch (URISyntaxException ex) { + throw new RuntimeException("Incorrect URI syntax", ex); + } + } + + public HttpResponse get(String path, Header... headers) { + return get(path, Collections.emptyList(), headers); + } + + public HttpResponse getAuthInfo(Header... headers) { + return executeRequest(new HttpGet(getHttpServerUri() + "/_opendistro/_security/authinfo?pretty"), headers); + } + + public void confirmCorrectCredentials(String expectedUserName) { + HttpResponse response = getAuthInfo(); + assertThat(response, notNullValue()); + response.assertStatusCode(200); + String username = response.getTextFromJsonBody("/user_name"); + String message = String.format("Expected user name is '%s', but was '%s'", expectedUserName, username); + assertThat(message, username, equalTo(expectedUserName)); + } + + public HttpResponse head(String path, Header... headers) { + return executeRequest(new HttpHead(getHttpServerUri() + "/" + path), headers); + } + + public HttpResponse options(String path, Header... headers) { + return executeRequest(new HttpOptions(getHttpServerUri() + "/" + path), headers); + } + + public HttpResponse putJson(String path, String body, Header... headers) { + HttpPut uriRequest = new HttpPut(getHttpServerUri() + "/" + path); + uriRequest.setEntity(toStringEntity(body)); + return executeRequest(uriRequest, mergeHeaders(CONTENT_TYPE_JSON, headers)); + } + + public HttpResponse getWithJsonBody(String path, String body, Header... headers) { + // Clever workaround to get support for GET with body https://stackoverflow.com/a/25019452/533057 + HttpPost uriRequest = new HttpPost(getHttpServerUri() + "/" + path) { + @Override + public String getMethod() { + return "GET"; + } + }; + uriRequest.setEntity(toStringEntity(body)); + return executeRequest(uriRequest, mergeHeaders(CONTENT_TYPE_JSON, headers)); + } + + private StringEntity toStringEntity(String body) { + try { + return new StringEntity(body); + } catch (UnsupportedEncodingException uee) { + throw new RuntimeException(uee); + } + } + + public HttpResponse putJson(String path, ToXContentObject body) { + try { + return putJson(path, XContentHelper.toXContent(body, XContentType.JSON, true).toString()); + } catch (final IOException ioe) { + throw new RuntimeException(ioe); + } + } + + public HttpResponse put(String path) { + HttpPut uriRequest = new HttpPut(getHttpServerUri() + "/" + path); + return executeRequest(uriRequest); + } + + public HttpResponse delete(String path, Header... headers) { + return executeRequest(new HttpDelete(getHttpServerUri() + "/" + path), headers); + } + + public HttpResponse postJson(String path, String body, Header... headers) { + HttpPost uriRequest = new HttpPost(getHttpServerUri() + "/" + path); + uriRequest.setEntity(toStringEntity(body)); + return executeRequest(uriRequest, mergeHeaders(CONTENT_TYPE_JSON, headers)); + } + + public HttpResponse postJson(String path, ToXContentObject body) { + try { + return postJson(path, XContentHelper.toXContent(body, XContentType.JSON, true).toString()); + } catch (final IOException ioe) { + throw new RuntimeException(ioe); + } + } + + public HttpResponse post(String path) { + HttpPost uriRequest = new HttpPost(getHttpServerUri() + "/" + path); + return executeRequest(uriRequest); + } + + public HttpResponse patch(String path, String body) { + HttpPatch uriRequest = new HttpPatch(getHttpServerUri() + "/" + path); + uriRequest.setEntity(toStringEntity(body)); + return executeRequest(uriRequest, CONTENT_TYPE_JSON); + } + + public HttpResponse assignRoleToUser(String username, String roleName) { + Objects.requireNonNull(roleName, "Role name is required"); + Objects.requireNonNull(username, "User name is required"); + String body = String.format("[{\"op\":\"add\",\"path\":\"/opendistro_security_roles\",\"value\":[\"%s\"]}]", roleName); + return patch("_plugins/_security/api/internalusers/" + username, body); + } + + public HttpResponse createRole(String roleName, ToXContentObject role) { + Objects.requireNonNull(roleName, "Role name is required"); + Objects.requireNonNull(role, "Role is required"); + return putJson("_plugins/_security/api/roles/" + roleName, role); + } + + public HttpResponse createUser(String userName, ToXContentObject user) { + Objects.requireNonNull(userName, "User name is required"); + Objects.requireNonNull(user, "User is required"); + return putJson("_plugins/_security/api/internalusers/" + userName, user); + } + + public HttpResponse executeRequest(HttpUriRequest uriRequest, Header... requestSpecificHeaders) { + try (CloseableHttpClient httpClient = getHTTPClient()) { + + if (requestSpecificHeaders != null && requestSpecificHeaders.length > 0) { + for (int i = 0; i < requestSpecificHeaders.length; i++) { + Header h = requestSpecificHeaders[i]; + uriRequest.addHeader(h); + } + } + + for (Header header : headers) { + uriRequest.addHeader(header); + } + + HttpResponse res = new HttpResponse(httpClient.execute(uriRequest)); + log.debug(res.getBody()); + return res; + } catch (IOException e) { + throw new RestClientException("Error occured during HTTP request execution", e); + } + } + + public void createRoleMapping(String backendRoleName, String roleName) { + requireNonNull(backendRoleName, "Backend role name is required"); + requireNonNull(roleName, "Role name is required"); + String path = "_plugins/_security/api/rolesmapping/" + roleName; + String body = String.format("{\"backend_roles\": [\"%s\"]}", backendRoleName); + HttpResponse response = putJson(path, body); + response.assertStatusCode(201); + } + + public final String getHttpServerUri() { + return "http" + (enableHTTPClientSSL ? "s" : "") + "://" + nodeHttpAddress.getHostString() + ":" + nodeHttpAddress.getPort(); + } + + protected final CloseableHttpClient getHTTPClient() { + HttpRoutePlanner routePlanner = Optional.ofNullable(sourceInetAddress).map(LocalAddressRoutePlanner::new).orElse(null); + CloseableHttpClientFactory factory = new CloseableHttpClientFactory(sslContext, requestConfig, routePlanner, null); + return factory.getHTTPClient(); + } + + private Header[] mergeHeaders(Header header, Header... headers) { + + if (headers == null || headers.length == 0) { + return new Header[] { header }; + } else { + Header[] result = new Header[headers.length + 1]; + result[0] = header; + System.arraycopy(headers, 0, result, 1, headers.length); + return result; + } + } + + public static class HttpResponse { + private final CloseableHttpResponse inner; + private final String body; + private final Header[] header; + private final int statusCode; + private final String statusReason; + + public HttpResponse(CloseableHttpResponse inner) throws IllegalStateException, IOException { + super(); + this.inner = inner; + final HttpEntity entity = inner.getEntity(); + if (entity == null) { // head request does not have a entity + this.body = ""; + } else { + this.body = IOUtils.toString(entity.getContent(), StandardCharsets.UTF_8); + } + this.header = inner.getAllHeaders(); + this.statusCode = inner.getStatusLine().getStatusCode(); + this.statusReason = inner.getStatusLine().getReasonPhrase(); + inner.close(); + } + + public String getContentType() { + Header h = getInner().getFirstHeader("content-type"); + if (h != null) { + return h.getValue(); + } + return null; + } + + public boolean isJsonContentType() { + String ct = getContentType(); + if (ct == null) { + return false; + } + return ct.contains("application/json"); + } + + public CloseableHttpResponse getInner() { + return inner; + } + + public String getBody() { + return body; + } + + public Header[] getHeader() { + return header; + } + + public Optional
findHeader(String name) { + return Arrays.stream(header) + .filter(header -> requireNonNull(name, "Header name is mandatory.").equalsIgnoreCase(header.getName())) + .findFirst(); + } + + public Header getHeader(String name) { + return findHeader(name).orElseThrow(); + } + + public boolean containHeader(String name) { + return findHeader(name).isPresent(); + } + + public int getStatusCode() { + return statusCode; + } + + public String getStatusReason() { + return statusReason; + } + + public List
getHeaders() { + return header == null ? Collections.emptyList() : Arrays.asList(header); + } + + public String getTextFromJsonBody(String jsonPointer) { + return getJsonNodeAt(jsonPointer).asText(); + } + + public List getTextArrayFromJsonBody(String jsonPointer) { + return StreamSupport.stream(getJsonNodeAt(jsonPointer).spliterator(), false) + .map(JsonNode::textValue) + .collect(Collectors.toList()); + } + + public int getIntFromJsonBody(String jsonPointer) { + return getJsonNodeAt(jsonPointer).asInt(); + } + + public Boolean getBooleanFromJsonBody(String jsonPointer) { + return getJsonNodeAt(jsonPointer).asBoolean(); + } + + public Double getDoubleFromJsonBody(String jsonPointer) { + return getJsonNodeAt(jsonPointer).asDouble(); + } + + public Long getLongFromJsonBody(String jsonPointer) { + return getJsonNodeAt(jsonPointer).asLong(); + } + + private JsonNode getJsonNodeAt(String jsonPointer) { + try { + return toJsonNode().at(jsonPointer); + } catch (IOException e) { + throw new IllegalArgumentException("Cound not convert response body to JSON node '" + getBody() + "'", e); + } + } + + private JsonNode toJsonNode() throws JsonProcessingException, IOException { + return DefaultObjectMapper.objectMapper.readTree(getBody()); + } + + @Override + public String toString() { + return "HttpResponse [inner=" + + inner + + ", body=" + + body + + ", header=" + + Arrays.toString(header) + + ", statusCode=" + + statusCode + + ", statusReason=" + + statusReason + + "]"; + } + + public T getBodyAs(Class authInfoClass) { + try { + return DefaultObjectMapper.readValue(getBody(), authInfoClass); + } catch (IOException e) { + throw new RuntimeException("Cannot parse response body", e); + } + } + + public void assertStatusCode(int expectedHttpStatus) { + String reason = format("Expected status code is '%d', but was '%d'. Response body '%s'.", expectedHttpStatus, statusCode, body); + assertThat(reason, statusCode, equalTo(expectedHttpStatus)); + } + } + + @Override + public String toString() { + return "TestRestClient [server=" + getHttpServerUri() + ", node=" + nodeHttpAddress + "]"; + } + + public RequestConfig getRequestConfig() { + return requestConfig; + } + + public void setRequestConfig(RequestConfig requestConfig) { + this.requestConfig = requestConfig; + } + + public boolean isSendHTTPClientCertificate() { + return sendHTTPClientCertificate; + } + + public void setSendHTTPClientCertificate(boolean sendHTTPClientCertificate) { + this.sendHTTPClientCertificate = sendHTTPClientCertificate; + } + + @Override + public void close() { + // TODO: Is there anything to clean up here? + } + +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClientConfiguration.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClientConfiguration.java new file mode 100644 index 0000000000..02028a5432 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClientConfiguration.java @@ -0,0 +1,174 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.cluster; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.commons.lang3.StringUtils; +import org.apache.http.Header; +import org.apache.http.message.BasicHeader; + +import org.opensearch.test.framework.cluster.OpenSearchClientProvider.UserCredentialsHolder; + +import static java.util.Objects.requireNonNull; + +/** +* Object which groups some parameters needed for {@link TestRestClient} creation. The class was created to reduce number of parameters +* of methods which are used to create {@link TestRestClient} . The class provides convenient builder-like methods. All fields of a class +* are nullable. +*/ +public class TestRestClientConfiguration { + + /** + * Username + */ + private String username; + /** + * Password + */ + private String password; + /** + * HTTP headers which should be attached to each HTTP request which is sent by {@link TestRestClient} + */ + private final List
headers = new ArrayList<>(); + /** + * IP address of client socket of {@link TestRestClient} + */ + private InetAddress sourceInetAddress; + + /** + * Set username + * @param username username + * @return builder + */ + public TestRestClientConfiguration username(String username) { + this.username = username; + return this; + } + + /** + * Set user's password + * @param password password + * @return builder + */ + public TestRestClientConfiguration password(String password) { + this.password = password; + return this; + } + + /** + * The method sets username and password read form userCredentialsHolder + * @param userCredentialsHolder source of credentials + * @return builder + */ + public TestRestClientConfiguration credentials(UserCredentialsHolder userCredentialsHolder) { + Objects.requireNonNull(userCredentialsHolder, "User credential holder is required."); + this.username = userCredentialsHolder.getName(); + this.password = userCredentialsHolder.getPassword(); + return this; + } + + /** + * Add HTTP headers which are attached to each HTTP request + * @param headers headers + * @return builder + */ + public TestRestClientConfiguration header(final String headerName, final String headerValue) { + this.headers.add( + new BasicHeader( + Objects.requireNonNull(headerName, "Header names are required"), + Objects.requireNonNull(headerValue, "Header values are required") + ) + ); + return this; + } + + /** + * Add HTTP headers which are attached to each HTTP request + * @param headers headers + * @return builder + */ + public TestRestClientConfiguration headers(Header... headers) { + this.headers.addAll(Arrays.asList(Objects.requireNonNull(headers, "Headers are required"))); + return this; + } + + /** + * Add HTTP headers which are attached to each HTTP request + * @param headers list of headers + * @return builder + */ + public TestRestClientConfiguration headers(List
headers) { + this.headers.addAll(Objects.requireNonNull(headers, "Cannot add null headers")); + return this; + } + + /** + * Set IP address of client socket used by {@link TestRestClient} + * @param sourceInetAddress IP address + * @return builder + */ + public TestRestClientConfiguration sourceInetAddress(InetAddress sourceInetAddress) { + this.sourceInetAddress = sourceInetAddress; + return this; + } + + public TestRestClientConfiguration sourceInetAddress(String sourceInetAddress) { + try { + this.sourceInetAddress = InetAddress.getByName(sourceInetAddress); + return this; + } catch (UnknownHostException e) { + throw new RuntimeException("Cannot get IP address for string " + sourceInetAddress, e); + } + } + + public static TestRestClientConfiguration userWithSourceIp(UserCredentialsHolder credentials, String sourceIpAddress) { + return new TestRestClientConfiguration().credentials(credentials).sourceInetAddress(sourceIpAddress); + } + + /** + * Return complete header list. Basic authentication header is created using fields {@link #username} and {@link #password} + * @return header list + */ + List
getHeaders() { + return Stream.concat(createBasicAuthHeader().stream(), headers.stream()).collect(Collectors.toList()); + } + + private Optional
createBasicAuthHeader() { + if (containsCredentials()) { + return Optional.of(getBasicAuthHeader(username, password)); + } + return Optional.empty(); + } + + private boolean containsCredentials() { + return StringUtils.isNoneBlank(username) && StringUtils.isNoneBlank(password); + } + + InetAddress getSourceInetAddress() { + return sourceInetAddress; + } + + public static Header getBasicAuthHeader(String user, String password) { + String value = "Basic " + + Base64.getEncoder().encodeToString((user + ":" + requireNonNull(password)).getBytes(StandardCharsets.UTF_8)); + return new BasicHeader("Authorization", value); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/ldap/EmbeddedLDAPServer.java b/src/integrationTest/java/org/opensearch/test/framework/ldap/EmbeddedLDAPServer.java new file mode 100755 index 0000000000..583a0cdaeb --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/ldap/EmbeddedLDAPServer.java @@ -0,0 +1,56 @@ +/* +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ + +package org.opensearch.test.framework.ldap; + +import java.util.Objects; + +import org.junit.rules.ExternalResource; + +import org.opensearch.test.framework.certificate.CertificateData; + +public class EmbeddedLDAPServer extends ExternalResource { + + private final LdapServer server; + + private final LdifData ldifData; + + public EmbeddedLDAPServer(CertificateData trustAnchor, CertificateData ldapCertificate, LdifData ldifData) { + this.ldifData = Objects.requireNonNull(ldifData, "Ldif data is required"); + this.server = new LdapServer(trustAnchor, ldapCertificate); + } + + @Override + protected void before() { + try { + server.start(ldifData); + } catch (Exception e) { + throw new RuntimeException("Cannot start ldap server", e); + } + } + + @Override + protected void after() { + try { + server.stop(); + } catch (InterruptedException e) { + throw new RuntimeException("Cannot stop LDAP server.", e); + } + } + + public int getLdapNonTlsPort() { + return server.getLdapNonTlsPort(); + } + + public int getLdapTlsPort() { + return server.getLdapsTlsPort(); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/ldap/LdapServer.java b/src/integrationTest/java/org/opensearch/test/framework/ldap/LdapServer.java new file mode 100644 index 0000000000..18a14242cc --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/ldap/LdapServer.java @@ -0,0 +1,226 @@ +/* +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ + +package org.opensearch.test.framework.ldap; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.StringReader; +import java.net.BindException; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.UnrecoverableKeyException; +import java.security.cert.Certificate; +import java.security.cert.CertificateException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReentrantLock; + +import javax.net.ssl.KeyManager; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; + +import com.unboundid.ldap.listener.InMemoryDirectoryServer; +import com.unboundid.ldap.listener.InMemoryDirectoryServerConfig; +import com.unboundid.ldap.listener.InMemoryListenerConfig; +import com.unboundid.ldap.sdk.DN; +import com.unboundid.ldap.sdk.Entry; +import com.unboundid.ldap.sdk.LDAPException; +import com.unboundid.ldap.sdk.schema.Schema; +import com.unboundid.ldif.LDIFReader; +import com.unboundid.util.ssl.SSLUtil; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.opensearch.test.framework.certificate.CertificateData; +import org.opensearch.test.framework.cluster.SocketUtils; + +/** +* Based on class com.amazon.dlic.auth.ldap.srv.LdapServer from older tests +*/ +final class LdapServer { + private static final Logger log = LogManager.getLogger(LdapServer.class); + + private static final int LOCK_TIMEOUT = 60; + private static final TimeUnit TIME_UNIT = TimeUnit.SECONDS; + + private static final String LOCK_TIMEOUT_MSG = "Unable to obtain lock due to timeout after " + + LOCK_TIMEOUT + + " " + + TIME_UNIT.toString(); + private static final String SERVER_NOT_STARTED = "The LDAP server is not started."; + private static final String SERVER_ALREADY_STARTED = "The LDAP server is already started."; + + private final CertificateData trustAnchor; + + private final CertificateData ldapCertificate; + + private InMemoryDirectoryServer server; + private final AtomicBoolean isStarted = new AtomicBoolean(Boolean.FALSE); + private final ReentrantLock serverStateLock = new ReentrantLock(); + + private int ldapNonTlsPort = -1; + private int ldapTlsPort = -1; + + public LdapServer(CertificateData trustAnchor, CertificateData ldapCertificate) { + this.trustAnchor = trustAnchor; + this.ldapCertificate = ldapCertificate; + } + + public boolean isStarted() { + return this.isStarted.get(); + } + + public int getLdapNonTlsPort() { + return ldapNonTlsPort; + } + + public int getLdapsTlsPort() { + return ldapTlsPort; + } + + public void start(LdifData ldifData) throws Exception { + Objects.requireNonNull(ldifData, "Ldif data is required"); + boolean hasLock = false; + try { + hasLock = serverStateLock.tryLock(LdapServer.LOCK_TIMEOUT, LdapServer.TIME_UNIT); + if (hasLock) { + doStart(ldifData); + this.isStarted.set(Boolean.TRUE); + } else { + throw new IllegalStateException(LdapServer.LOCK_TIMEOUT_MSG); + } + } catch (InterruptedException ioe) { + // lock interrupted + log.error("LDAP server start lock interrupted", ioe); + throw ioe; + } finally { + if (hasLock) { + serverStateLock.unlock(); + } + } + } + + private void doStart(LdifData ldifData) throws Exception { + if (isStarted.get()) { + throw new IllegalStateException(LdapServer.SERVER_ALREADY_STARTED); + } + configureAndStartServer(ldifData); + } + + private Collection getInMemoryListenerConfigs() throws Exception { + KeyStore keyStore = createEmptyKeyStore(); + addLdapCertificatesToKeystore(keyStore); + final SSLUtil sslUtil = new SSLUtil(createKeyManager(keyStore), createTrustManagers(keyStore)); + + ldapNonTlsPort = SocketUtils.findAvailableTcpPort(); + ldapTlsPort = SocketUtils.findAvailableTcpPort(); + + Collection listenerConfigs = new ArrayList<>(); + listenerConfigs.add(InMemoryListenerConfig.createLDAPConfig("ldap", null, ldapNonTlsPort, sslUtil.createSSLSocketFactory())); + listenerConfigs.add(InMemoryListenerConfig.createLDAPSConfig("ldaps", ldapTlsPort, sslUtil.createSSLServerSocketFactory())); + return listenerConfigs; + } + + private static KeyManager[] createKeyManager(KeyStore keyStore) throws NoSuchAlgorithmException, KeyStoreException, + UnrecoverableKeyException { + KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + keyManagerFactory.init(keyStore, null); + return keyManagerFactory.getKeyManagers(); + } + + private static TrustManager[] createTrustManagers(KeyStore keyStore) throws NoSuchAlgorithmException, KeyStoreException { + TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trustManagerFactory.init(keyStore); + return trustManagerFactory.getTrustManagers(); + } + + private void addLdapCertificatesToKeystore(KeyStore keyStore) throws KeyStoreException { + keyStore.setCertificateEntry("trustAnchor", trustAnchor.certificate()); + keyStore.setKeyEntry("ldap-key", ldapCertificate.getKey(), null, new Certificate[] { ldapCertificate.certificate() }); + } + + private static KeyStore createEmptyKeyStore() throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException { + KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); + keyStore.load(null); + return keyStore; + } + + private synchronized void configureAndStartServer(LdifData ldifData) throws Exception { + Collection listenerConfigs = getInMemoryListenerConfigs(); + + Schema schema = Schema.getDefaultStandardSchema(); + + final String rootObjectDN = ldifData.getRootDistinguishedName(); + InMemoryDirectoryServerConfig config = new InMemoryDirectoryServerConfig(new DN(rootObjectDN)); + + config.setSchema(schema); // schema can be set on the rootDN too, per javadoc. + config.setListenerConfigs(listenerConfigs); + config.setEnforceAttributeSyntaxCompliance(false); + config.setEnforceSingleStructuralObjectClass(false); + + server = new InMemoryDirectoryServer(config); + + try { + /* Clear entries from server. */ + server.clear(); + server.startListening(); + loadLdifData(ldifData); + } catch (LDAPException ldape) { + if (ldape.getMessage().contains("java.net.BindException")) { + throw new BindException(ldape.getMessage()); + } + throw ldape; + } + + } + + public void stop() throws InterruptedException { + boolean hasLock = false; + try { + hasLock = serverStateLock.tryLock(LdapServer.LOCK_TIMEOUT, LdapServer.TIME_UNIT); + if (hasLock) { + if (!isStarted.get()) { + throw new IllegalStateException(LdapServer.SERVER_NOT_STARTED); + } + log.info("Shutting down in-Memory Ldap Server."); + server.shutDown(true); + } else { + throw new IllegalStateException(LdapServer.LOCK_TIMEOUT_MSG); + } + } catch (InterruptedException ioe) { + // lock interrupted + log.error("Canot stop LDAP server due to interruption", ioe); + throw ioe; + } finally { + if (hasLock) { + serverStateLock.unlock(); + } + } + } + + private void loadLdifData(LdifData ldifData) throws Exception { + try (LDIFReader r = new LDIFReader(new BufferedReader(new StringReader(ldifData.getContent())))) { + Entry entry; + while ((entry = r.readEntry()) != null) { + server.add(entry); + } + } catch (Exception e) { + log.error("Cannot load data into LDAP server", e); + throw e; + } + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/ldap/LdifBuilder.java b/src/integrationTest/java/org/opensearch/test/framework/ldap/LdifBuilder.java new file mode 100644 index 0000000000..87f01a2bbc --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/ldap/LdifBuilder.java @@ -0,0 +1,66 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.ldap; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class LdifBuilder { + + private static final Logger log = LogManager.getLogger(LdifBuilder.class); + + private final List records; + + private Record root; + + public LdifBuilder() { + this.records = new ArrayList<>(); + } + + public RecordBuilder root(String distinguishedName) { + if (root != null) { + throw new IllegalStateException("Root object is already defined"); + } + return new RecordBuilder(this, distinguishedName); + } + + RecordBuilder newRecord(String distinguishedName) { + if (root == null) { + throw new IllegalStateException("Define root object first"); + } + return new RecordBuilder(this, distinguishedName); + } + + void addRecord(Record record) { + Objects.requireNonNull(record, "Cannot add null record"); + if (records.isEmpty()) { + this.root = record; + } + records.add(Objects.requireNonNull(record, "Cannot add null record")); + } + + public LdifData buildLdif() { + String ldif = records.stream().map(record -> record.toLdifRepresentation()).collect(Collectors.joining("\n##########\n")); + log.debug("Built ldif file: \n{}", ldif); + return new LdifData(getRootDistinguishedName(), ldif); + } + + private String getRootDistinguishedName() { + if (root == null) { + throw new IllegalStateException("Root object is not present."); + } + return root.getDistinguishedName(); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/ldap/LdifData.java b/src/integrationTest/java/org/opensearch/test/framework/ldap/LdifData.java new file mode 100644 index 0000000000..4a1af5bc08 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/ldap/LdifData.java @@ -0,0 +1,48 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.ldap; + +import org.apache.commons.lang3.StringUtils; + +/** +* Value object which represents LDIF file data and some metadata. Ensure type safety. +*/ +public class LdifData { + + private final String rootDistinguishedName; + + private final String content; + + LdifData(String rootDistinguishedName, String content) { + this.rootDistinguishedName = requireNotBlank(rootDistinguishedName, "Root distinguished name is required"); + this.content = requireNotBlank(content, "Ldif file content is required"); + + } + + private static String requireNotBlank(String string, String message) { + if (StringUtils.isBlank(string)) { + throw new IllegalArgumentException(message); + } + return string; + } + + String getContent() { + return content; + } + + String getRootDistinguishedName() { + return rootDistinguishedName; + } + + @Override + public String toString() { + return "LdifData{" + "content='" + content + '\'' + '}'; + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/ldap/Record.java b/src/integrationTest/java/org/opensearch/test/framework/ldap/Record.java new file mode 100644 index 0000000000..48e7484777 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/ldap/Record.java @@ -0,0 +1,67 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.ldap; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.tuple.Pair; + +class Record { + + private final String distinguishedName; + + private final List classes; + private final List> attributes; + + public Record(String distinguishedName) { + this.distinguishedName = Objects.requireNonNull(distinguishedName, "Distinguished name is required"); + this.classes = new ArrayList<>(); + this.attributes = new ArrayList<>(); + } + + public String getDistinguishedName() { + return distinguishedName; + } + + public void addClass(String clazz) { + classes.add(Objects.requireNonNull(clazz, "Object class is required.")); + } + + public void addAttribute(String name, String value) { + Objects.requireNonNull(name, "Attribute name is required"); + Objects.requireNonNull(value, "Attribute value is required"); + attributes.add(Pair.of(name, value)); + } + + boolean isValid() { + return classes.size() > 0; + } + + String toLdifRepresentation() { + return new StringBuilder("dn: ").append(distinguishedName) + .append("\n") + .append(formattedClasses()) + .append("\n") + .append(formattedAttributes()) + .append("\n") + .toString(); + } + + private String formattedAttributes() { + return attributes.stream().map(pair -> pair.getKey() + ": " + pair.getValue()).collect(Collectors.joining("\n")); + } + + private String formattedClasses() { + return classes.stream().map(clazz -> "objectClass: " + clazz).collect(Collectors.joining("\n")); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/ldap/RecordBuilder.java b/src/integrationTest/java/org/opensearch/test/framework/ldap/RecordBuilder.java new file mode 100644 index 0000000000..1df27c72fe --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/ldap/RecordBuilder.java @@ -0,0 +1,92 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.ldap; + +import java.util.Objects; + +public class RecordBuilder { + + private final LdifBuilder builder; + private final Record record; + + RecordBuilder(LdifBuilder builder, String distinguishedName) { + this.builder = Objects.requireNonNull(builder, "LdifBuilder is required"); + this.record = new Record(distinguishedName); + } + + public RecordBuilder classes(String... classes) { + for (String clazz : classes) { + this.record.addClass(clazz); + } + return this; + } + + public RecordBuilder dn(String distinguishedName) { + record.addAttribute("dn", distinguishedName); + return this; + } + + public RecordBuilder dc(String domainComponent) { + record.addAttribute("dc", domainComponent); + return this; + } + + public RecordBuilder ou(String organizationUnit) { + record.addAttribute("ou", organizationUnit); + return this; + } + + public RecordBuilder cn(String commonName) { + record.addAttribute("cn", commonName); + return this; + } + + public RecordBuilder sn(String surname) { + record.addAttribute("sn", surname); + return this; + } + + public RecordBuilder uid(String userId) { + record.addAttribute("uid", userId); + return this; + } + + public RecordBuilder userPassword(String password) { + record.addAttribute("userpassword", password); + return this; + } + + public RecordBuilder mail(String emailAddress) { + record.addAttribute("mail", emailAddress); + return this; + } + + public RecordBuilder uniqueMember(String userDistinguishedName) { + record.addAttribute("uniquemember", userDistinguishedName); + return this; + } + + public RecordBuilder attribute(String name, String value) { + record.addAttribute(name, value); + return this; + } + + public LdifBuilder buildRecord() { + if (record.isValid() == false) { + throw new IllegalStateException("Record is invalid"); + } + builder.addRecord(record); + return builder; + } + + public RecordBuilder newRecord(String distinguishedName) { + return buildRecord().newRecord(distinguishedName); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/log/LogCapturingAppender.java b/src/integrationTest/java/org/opensearch/test/framework/log/LogCapturingAppender.java new file mode 100644 index 0000000000..5673f1bd3e --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/log/LogCapturingAppender.java @@ -0,0 +1,134 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.log; + +import com.google.common.collect.EvictingQueue; +import com.google.common.collect.Queues; +import org.apache.logging.log4j.core.Appender; +import org.apache.logging.log4j.core.Core; +import org.apache.logging.log4j.core.Filter; +import org.apache.logging.log4j.core.Layout; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.appender.AbstractAppender; +import org.apache.logging.log4j.core.config.Property; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +import static org.opensearch.test.framework.log.LogCapturingAppender.PLUGIN_NAME; + +/** +*

The class acts as Log4j2 appender with a special purpose. The appender is used to capture logs which are generated during tests and +* then test can examine logs. To use the appender it is necessary to:

+*
    +*
  1. Add package with appender to log4j2 package scan in Log4j2 configuration file
    2. +*
  2. Create appender in log4j2 configuration
    3. +*
  3. Assign required loggers to appender
    4. +*
  4. Enable appender for certain classes with method {@link #enable(String...)}. Each test can enable appender for distinct classes
    5. +*
+*/ +@Plugin(name = PLUGIN_NAME, category = Core.CATEGORY_NAME, elementType = Appender.ELEMENT_TYPE, printObject = true) +public class LogCapturingAppender extends AbstractAppender { + + public final static String PLUGIN_NAME = "LogCapturingAppender"; + /** + * Appender stores only last MAX_SIZE messages to avoid excessive RAM memory usage. + */ + public static final int MAX_SIZE = 100; + + /** + * Buffer for captured log messages + */ + private static final Queue messages = Queues.synchronizedQueue(EvictingQueue.create(MAX_SIZE)); + + /** + * Log messages are stored in buffer {@link #messages} only for classes which are added to the {@link #activeLoggers} set. + */ + private static final Set activeLoggers = ConcurrentHashMap.newKeySet(); + + protected LogCapturingAppender( + String name, + Filter filter, + Layout layout, + boolean ignoreExceptions, + Property[] properties + ) { + super(name, filter, layout, ignoreExceptions, properties); + } + + /** + * Method used by Log4j2 to create appender + * @param name appender name from Log4j2 configuration + * @return newly created appender + */ + @PluginFactory + public static LogCapturingAppender createAppender( + @PluginAttribute(value = "name", defaultString = "logCapturingAppender") String name + ) { + return new LogCapturingAppender(name, null, null, true, Property.EMPTY_ARRAY); + } + + /** + * Method invoked by Log4j2 to append log events + * @param event The LogEvent, represents log message. + */ + @Override + public void append(LogEvent event) { + String loggerName = event.getLoggerName(); + boolean loggable = activeLoggers.contains(loggerName); + if (loggable) { + event.getThrown(); + messages.add(new LogMessage(event.getMessage().getFormattedMessage(), event.getThrown())); + } + } + + /** + * To collect log messages form given logger the logger name must be passed to {@link #enable(String...)} method. + * @param loggerNames logger names + */ + public static void enable(String... loggerNames) { + disable(); + activeLoggers.addAll(Arrays.asList(loggerNames)); + } + + /** + * Invocation cause that appender stops collecting log messages. Additionally, memory used by collected messages so far is released. + */ + public static void disable() { + activeLoggers.clear(); + messages.clear(); + } + + /** + * Is used to obtain gathered log messages + * @return Log messages + */ + public static List getLogMessages() { + return new ArrayList<>(messages); + } + + public static List getLogMessagesAsString() { + return getLogMessages().stream().map(LogMessage::getMessage).collect(Collectors.toList()); + } + + @Override + public String toString() { + return "LogCapturingAppender{}"; + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/log/LogMessage.java b/src/integrationTest/java/org/opensearch/test/framework/log/LogMessage.java new file mode 100644 index 0000000000..9342c7ee30 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/log/LogMessage.java @@ -0,0 +1,40 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.log; + +import java.util.Objects; +import java.util.Optional; + +import org.apache.commons.lang3.exception.ExceptionUtils; + +class LogMessage { + + private final String message; + private final String stackTrace; + + public LogMessage(String message, Throwable throwable) { + this.message = message; + this.stackTrace = Optional.ofNullable(throwable).map(ExceptionUtils::getStackTrace).orElse(""); + } + + public boolean containMessage(String expectedMessage) { + Objects.requireNonNull(expectedMessage, "Expected message must not be null."); + return expectedMessage.equals(message); + } + + public boolean stackTraceContains(String stackTraceFragment) { + Objects.requireNonNull(stackTraceFragment, "Stack trace fragment is required."); + return stackTrace.contains(stackTraceFragment); + } + + public String getMessage() { + return message; + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/log/LogsRule.java b/src/integrationTest/java/org/opensearch/test/framework/log/LogsRule.java new file mode 100644 index 0000000000..46fa252df4 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/log/LogsRule.java @@ -0,0 +1,93 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.log; + +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import org.junit.rules.ExternalResource; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasItem; + +/** +* The class is a JUnit 4 rule and enables developers to write assertion related to log messages generated in the course of test. To use +* {@link LogsRule} appender {@link LogCapturingAppender} must be properly configured. The rule also manages {@link LogCapturingAppender} +* so that memory occupied by gathered log messages is released after each test. +*/ +public class LogsRule extends ExternalResource { + + private final String[] loggerNames; + + /** + * Constructor used to start gathering log messages from certain loggers + * @param loggerNames Loggers names. Log messages are collected only if the log message is associated with the logger with a name which + * is present in loggerNames parameter. + */ + public LogsRule(String... loggerNames) { + this.loggerNames = Objects.requireNonNull(loggerNames, "Logger names are required"); + } + + @Override + protected void before() { + LogCapturingAppender.enable(loggerNames); + } + + @Override + protected void after() { + LogCapturingAppender.disable(); + } + + /** + * Check if during the tests certain log message was logged + * @param expectedLogMessage expected log message + */ + public void assertThatContainExactly(String expectedLogMessage) { + List messages = LogCapturingAppender.getLogMessagesAsString(); + String reason = reasonMessage(expectedLogMessage, messages); + assertThat(reason, messages, hasItem(expectedLogMessage)); + } + + /** + * Check if during the tests certain log message was logged + * @param messageFragment expected log message fragment + */ + public void assertThatContain(String messageFragment) { + List messages = LogCapturingAppender.getLogMessagesAsString(); + ; + String reason = reasonMessage(messageFragment, messages); + assertThat(reason, messages, hasItem(containsString(messageFragment))); + } + + /** + * Check if during the tests a stack trace was logged which contain given fragment + * @param stackTraceFragment stack trace fragment + */ + public void assertThatStackTraceContain(String stackTraceFragment) { + long count = LogCapturingAppender.getLogMessages() + .stream() + .filter(logMessage -> logMessage.stackTraceContains(stackTraceFragment)) + .count(); + String reason = "Stack trace does not contain element " + stackTraceFragment; + assertThat(reason, count, greaterThan(0L)); + } + + private static String reasonMessage(String expectedLogMessage, List messages) { + String concatenatedLogMessages = messages.stream().map(message -> String.format("'%s'", message)).collect(Collectors.joining(", ")); + return String.format( + "Expected message '%s' has not been found in logs. All captured log messages: %s", + expectedLogMessage, + concatenatedLogMessages + ); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/AtLeastCertainNumberOfAuditsFulfillPredicateMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/AtLeastCertainNumberOfAuditsFulfillPredicateMatcher.java new file mode 100644 index 0000000000..ba7feed4c3 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/AtLeastCertainNumberOfAuditsFulfillPredicateMatcher.java @@ -0,0 +1,48 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import java.util.List; +import java.util.function.Predicate; + +import org.hamcrest.Description; + +import org.opensearch.security.auditlog.impl.AuditMessage; + +class AtLeastCertainNumberOfAuditsFulfillPredicateMatcher extends AuditsFulfillPredicateMatcher { + + private final long minimumNumberOfAudits; + + public AtLeastCertainNumberOfAuditsFulfillPredicateMatcher(Predicate predicate, long minimumNumberOfAudits) { + super(predicate); + this.minimumNumberOfAudits = minimumNumberOfAudits; + } + + @Override + protected boolean matchesSafely(List audits, Description mismatchDescription) { + long count = countAuditsWhichMatchPredicate(audits); + if (count < minimumNumberOfAudits) { + mismatchDescription.appendText(" only ") + .appendValue(count) + .appendText(" match predicate. Examined audit logs ") + .appendText(auditMessagesToString(audits)); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Al least ") + .appendValue(minimumNumberOfAudits) + .appendText(" audits records should match predicate ") + .appendValue(predicate); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/AuditMessageMatchers.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/AuditMessageMatchers.java new file mode 100644 index 0000000000..080d4473e1 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/AuditMessageMatchers.java @@ -0,0 +1,38 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import java.util.List; +import java.util.function.Predicate; + +import org.hamcrest.Matcher; + +import org.opensearch.security.auditlog.impl.AuditMessage; + +public class AuditMessageMatchers { + + private AuditMessageMatchers() { + + } + + public static Matcher> exactNumberOfAuditsFulfillPredicate( + long exactNumberOfAuditMessages, + Predicate predicate + ) { + return new ExactNumberOfAuditsFulfillPredicateMatcher(exactNumberOfAuditMessages, predicate); + } + + public static Matcher> atLeastCertainNumberOfAuditsFulfillPredicate( + long minimumNumberOfAudits, + Predicate predicate + ) { + return new AtLeastCertainNumberOfAuditsFulfillPredicateMatcher(predicate, minimumNumberOfAudits); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/AuditsFulfillPredicateMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/AuditsFulfillPredicateMatcher.java new file mode 100644 index 0000000000..2864c1df81 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/AuditsFulfillPredicateMatcher.java @@ -0,0 +1,36 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import java.util.List; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.security.auditlog.impl.AuditMessage; + +abstract class AuditsFulfillPredicateMatcher extends TypeSafeDiagnosingMatcher> { + + protected final Predicate predicate; + + public AuditsFulfillPredicateMatcher(Predicate predicate) { + this.predicate = predicate; + } + + protected String auditMessagesToString(List audits) { + return audits.stream().map(AuditMessage::toString).collect(Collectors.joining(",\n")); + } + + protected long countAuditsWhichMatchPredicate(List audits) { + return audits.stream().filter(predicate).count(); + } + +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/BulkResponseContainExceptionsAtIndexMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/BulkResponseContainExceptionsAtIndexMatcher.java new file mode 100644 index 0000000000..2fb2cc5e74 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/BulkResponseContainExceptionsAtIndexMatcher.java @@ -0,0 +1,76 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.bulk.BulkItemResponse; +import org.opensearch.action.bulk.BulkResponse; + +import static java.util.Objects.requireNonNull; + +class BulkResponseContainExceptionsAtIndexMatcher extends TypeSafeDiagnosingMatcher { + + private final int errorIndex; + private final Matcher exceptionMatcher; + + public BulkResponseContainExceptionsAtIndexMatcher(int errorIndex, Matcher exceptionMatcher) { + this.errorIndex = errorIndex; + this.exceptionMatcher = requireNonNull(exceptionMatcher, "Exception matcher is required."); + } + + @Override + protected boolean matchesSafely(BulkResponse response, Description mismatchDescription) { + if (response.hasFailures() == false) { + mismatchDescription.appendText("received successful bulk response what is not expected."); + return false; + } + BulkItemResponse[] items = response.getItems(); + if ((items == null) || (items.length == 0) || (errorIndex >= items.length)) { + mismatchDescription.appendText("bulk response does not contain item with index ").appendValue(errorIndex); + return false; + } + BulkItemResponse item = items[errorIndex]; + if (item == null) { + mismatchDescription.appendText("bulk item response with index ").appendValue(errorIndex).appendText(" is null."); + return false; + } + BulkItemResponse.Failure failure = item.getFailure(); + if (failure == null) { + mismatchDescription.appendText("bulk response item with index ") + .appendValue(errorIndex) + .appendText(" does not contain failure"); + return false; + } + Exception exception = failure.getCause(); + if (exception == null) { + mismatchDescription.appendText("bulk response item with index ") + .appendValue(errorIndex) + .appendText(" does not contain exception."); + return false; + } + if (exceptionMatcher.matches(exception) == false) { + mismatchDescription.appendText("bulk response item with index ") + .appendValue(errorIndex) + .appendText(" contains incorrect exception which is ") + .appendValue(exception); + return false; + } + + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("bulk response should contain exceptions which indicate failure"); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/BulkResponseContainExceptionsMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/BulkResponseContainExceptionsMatcher.java new file mode 100644 index 0000000000..b5e46ba9e2 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/BulkResponseContainExceptionsMatcher.java @@ -0,0 +1,71 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.bulk.BulkItemResponse; +import org.opensearch.action.bulk.BulkResponse; + +import static java.util.Objects.requireNonNull; + +class BulkResponseContainExceptionsMatcher extends TypeSafeDiagnosingMatcher { + + private final Matcher exceptionMatcher; + + public BulkResponseContainExceptionsMatcher(Matcher exceptionMatcher) { + this.exceptionMatcher = requireNonNull(exceptionMatcher, "Exception matcher is required."); + } + + @Override + protected boolean matchesSafely(BulkResponse response, Description mismatchDescription) { + if (response.hasFailures() == false) { + mismatchDescription.appendText("received successful bulk response what is not expected."); + return false; + } + BulkItemResponse[] items = response.getItems(); + if ((items == null) || (items.length == 0)) { + mismatchDescription.appendText("bulk response does not contain items ").appendValue(items); + return false; + } + for (int i = 0; i < items.length; ++i) { + BulkItemResponse item = items[i]; + if (item == null) { + mismatchDescription.appendText("bulk item response with index ").appendValue(i).appendText(" is null."); + return false; + } + BulkItemResponse.Failure failure = item.getFailure(); + if (failure == null) { + mismatchDescription.appendText("bulk response item with index ").appendValue(i).appendText(" does not contain failure"); + return false; + } + Exception exception = failure.getCause(); + if (exception == null) { + mismatchDescription.appendText("bulk response item with index ").appendValue(i).appendText(" does not contain exception."); + return false; + } + if (exceptionMatcher.matches(exception) == false) { + mismatchDescription.appendText("bulk response item with index ") + .appendValue(i) + .appendText(" contains incorrect exception which is ") + .appendValue(exception); + return false; + } + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("bulk response should contain exceptions which indicate failure"); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/BulkResponseMatchers.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/BulkResponseMatchers.java new file mode 100644 index 0000000000..eedcd3a3a0 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/BulkResponseMatchers.java @@ -0,0 +1,37 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Matcher; + +import org.opensearch.action.bulk.BulkResponse; + +public class BulkResponseMatchers { + + private BulkResponseMatchers() { + + } + + public static Matcher successBulkResponse() { + return new SuccessBulkResponseMatcher(); + } + + public static Matcher failureBulkResponse() { + return new FailureBulkResponseMatcher(); + } + + public static Matcher bulkResponseContainExceptions(Matcher exceptionMatcher) { + return new BulkResponseContainExceptionsMatcher(exceptionMatcher); + } + + public static Matcher bulkResponseContainExceptions(int index, Matcher exceptionMatcher) { + return new BulkResponseContainExceptionsAtIndexMatcher(index, exceptionMatcher); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/ClusterContainDocumentCountIndexMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/ClusterContainDocumentCountIndexMatcher.java new file mode 100644 index 0000000000..91d2a28ef8 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/ClusterContainDocumentCountIndexMatcher.java @@ -0,0 +1,43 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.admin.indices.get.GetIndexResponse; +import org.opensearch.client.Client; +import org.opensearch.test.framework.cluster.LocalCluster; + +import static java.util.Objects.requireNonNull; + +class ClusterContainDocumentCountIndexMatcher extends TypeSafeDiagnosingMatcher { + + private final String indexName; + private final int expectedDocumentCount; + + public ClusterContainDocumentCountIndexMatcher(String indexName, int expectedDocumentCount) { + this.indexName = requireNonNull(indexName, "Index name is required."); + this.expectedDocumentCount = expectedDocumentCount; + } + + @Override + protected boolean matchesSafely(LocalCluster cluster, Description mismatchDescription) { + try (Client client = cluster.getInternalNodeClient()) { + GetIndexResponse response = client.admin().indices().getIndex(null).actionGet(); + } + return false; + } + + @Override + public void describeTo(Description description) { + description.appendText("contains ").appendValue(expectedDocumentCount).appendText(" in index ").appendText(indexName); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/ClusterContainSuccessSnapshotMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/ClusterContainSuccessSnapshotMatcher.java new file mode 100644 index 0000000000..362663e07b --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/ClusterContainSuccessSnapshotMatcher.java @@ -0,0 +1,69 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import java.util.stream.Collectors; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; +import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; +import org.opensearch.client.Client; +import org.opensearch.snapshots.SnapshotMissingException; +import org.opensearch.snapshots.SnapshotState; + +import static java.util.Objects.requireNonNull; + +class ClusterContainSuccessSnapshotMatcher extends TypeSafeDiagnosingMatcher { + + private final String repositoryName; + private final String snapshotName; + + public ClusterContainSuccessSnapshotMatcher(String repositoryName, String snapshotName) { + this.repositoryName = requireNonNull(repositoryName, "Snapshot repository name is required."); + this.snapshotName = requireNonNull(snapshotName, "Snapshot name is required."); + } + + @Override + protected boolean matchesSafely(Client client, Description mismatchDescription) { + try { + GetSnapshotsRequest request = new GetSnapshotsRequest(repositoryName, new String[] { snapshotName }); + GetSnapshotsResponse response = client.admin().cluster().getSnapshots(request).actionGet(); + long count = response.getSnapshots() + .stream() + .map(snapshot -> snapshot.state()) + .filter(status -> SnapshotState.SUCCESS.equals(status)) + .count(); + if (count != 1) { + String snapshotStatuses = response.getSnapshots() + .stream() + .map(info -> String.format("%s %s", info.snapshotId().getName(), info.state())) + .collect(Collectors.joining(", ")); + mismatchDescription.appendText("snapshot is not present or has incorrect state, snapshots statuses ") + .appendValue(snapshotStatuses); + return false; + } + } catch (SnapshotMissingException e) { + mismatchDescription.appendText(" snapshot does not exist"); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Cluster contain snapshot ") + .appendValue(snapshotName) + .appendText(" in repository ") + .appendValue(repositoryName) + .appendText(" with success status"); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/ClusterContainsDocumentMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/ClusterContainsDocumentMatcher.java new file mode 100644 index 0000000000..3153214213 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/ClusterContainsDocumentMatcher.java @@ -0,0 +1,57 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import java.util.concurrent.ExecutionException; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.get.GetRequest; +import org.opensearch.action.get.GetResponse; +import org.opensearch.client.Client; + +import static java.util.Objects.requireNonNull; + +class ClusterContainsDocumentMatcher extends TypeSafeDiagnosingMatcher { + + private static final Logger log = LogManager.getLogger(ClusterContainsDocumentMatcher.class); + + private final String indexName; + private final String documentId; + + ClusterContainsDocumentMatcher(String indexName, String documentId) { + this.indexName = requireNonNull(indexName, "Index name is required."); + this.documentId = requireNonNull(documentId, "Document id is required."); + } + + @Override + protected boolean matchesSafely(Client client, Description mismatchDescription) { + try { + GetResponse response = client.get(new GetRequest(indexName, documentId)).get(); + if (response.isExists() == false) { + mismatchDescription.appendText("Document does not exists"); + return false; + } + } catch (InterruptedException | ExecutionException e) { + log.error("Cannot verify if cluster contains document '{}' in index '{}'.", documentId, indexName, e); + mismatchDescription.appendText("Exception occured during verification if cluster contain document").appendValue(e); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Cluster contain document in index ").appendValue(indexName).appendText(" with id ").appendValue(documentId); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/ClusterContainsDocumentWithFieldValueMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/ClusterContainsDocumentWithFieldValueMatcher.java new file mode 100644 index 0000000000..21f062fabf --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/ClusterContainsDocumentWithFieldValueMatcher.java @@ -0,0 +1,85 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import java.util.Map; +import java.util.concurrent.ExecutionException; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.get.GetRequest; +import org.opensearch.action.get.GetResponse; +import org.opensearch.client.Client; + +import static java.util.Objects.requireNonNull; + +class ClusterContainsDocumentWithFieldValueMatcher extends TypeSafeDiagnosingMatcher { + + private static final Logger log = LogManager.getLogger(ClusterContainsDocumentWithFieldValueMatcher.class); + + private final String indexName; + private final String documentId; + + private final String fieldName; + + private final Object fieldValue; + + ClusterContainsDocumentWithFieldValueMatcher(String indexName, String documentId, String fieldName, Object fieldValue) { + this.indexName = requireNonNull(indexName, "Index name is required."); + this.documentId = requireNonNull(documentId, "Document id is required."); + this.fieldName = requireNonNull(fieldName, "Field name is required."); + this.fieldValue = requireNonNull(fieldValue, "Field value is required."); + } + + @Override + protected boolean matchesSafely(Client client, Description mismatchDescription) { + try { + GetResponse response = client.get(new GetRequest(indexName, documentId)).get(); + if (response.isExists() == false) { + mismatchDescription.appendText("Document does not exists"); + return false; + } + Map source = response.getSource(); + if (source == null) { + mismatchDescription.appendText("Cannot retrieve document source"); + return false; + } + if (source.containsKey(fieldName) == false) { + mismatchDescription.appendText("document does not contain field ").appendValue(fieldName); + return false; + } + Object actualFieldValue = source.get(fieldName); + if (fieldValue.equals(actualFieldValue) == false) { + mismatchDescription.appendText(" document contain incorrect field value ").appendValue(actualFieldValue); + return false; + } + } catch (InterruptedException | ExecutionException e) { + log.error("Cannot verify if cluster contains document '{}' in index '{}'.", documentId, indexName, e); + mismatchDescription.appendText("Exception occured during verification if cluster contain document").appendValue(e); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Cluster contain document in index ") + .appendValue(indexName) + .appendText(" with id ") + .appendValue(documentId) + .appendText(" with field ") + .appendValue(fieldName) + .appendText(" which is equal to ") + .appendValue(fieldValue); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/ClusterContainsSnapshotRepositoryMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/ClusterContainsSnapshotRepositoryMatcher.java new file mode 100644 index 0000000000..fcff8bd6cf --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/ClusterContainsSnapshotRepositoryMatcher.java @@ -0,0 +1,67 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import java.util.Set; +import java.util.stream.Collectors; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesRequest; +import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse; +import org.opensearch.client.Client; +import org.opensearch.client.ClusterAdminClient; +import org.opensearch.repositories.RepositoryMissingException; + +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.joining; + +class ClusterContainsSnapshotRepositoryMatcher extends TypeSafeDiagnosingMatcher { + + private final String repositoryName; + + public ClusterContainsSnapshotRepositoryMatcher(String repositoryName) { + this.repositoryName = requireNonNull(repositoryName, "Repository name is required."); + } + + @Override + protected boolean matchesSafely(Client client, Description mismatchDescription) { + try { + ClusterAdminClient adminClient = client.admin().cluster(); + GetRepositoriesRequest request = new GetRepositoriesRequest(new String[] { "*" }); + GetRepositoriesResponse response = adminClient.getRepositories(request).actionGet(); + if (response == null) { + mismatchDescription.appendText("Cannot check if cluster contain repository"); + return false; + } + Set actualRepositoryNames = response.repositories() + .stream() + .map(metadata -> metadata.name()) + .collect(Collectors.toSet()); + if (actualRepositoryNames.contains(repositoryName) == false) { + mismatchDescription.appendText("Cluster does not contain snapshot repository ") + .appendValue(repositoryName) + .appendText(", but the following repositories are defined in the cluster ") + .appendValue(actualRepositoryNames.stream().collect(joining(", "))); + return false; + } + } catch (RepositoryMissingException e) { + mismatchDescription.appendText(" cluster does not contain any repository."); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Cluster contain snapshot repository with name ").appendValue(repositoryName); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/ClusterMatchers.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/ClusterMatchers.java new file mode 100644 index 0000000000..33a7eb7787 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/ClusterMatchers.java @@ -0,0 +1,59 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import java.util.Map; + +import org.hamcrest.Matcher; + +import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.test.framework.cluster.LocalCluster; + +public class ClusterMatchers { + + private ClusterMatchers() { + + } + + public static Matcher clusterContainsDocument(String indexName, String documentId) { + return new ClusterContainsDocumentMatcher(indexName, documentId); + } + + public static Matcher clusterContainsDocumentWithFieldValue( + String indexName, + String documentId, + String fieldName, + Object fieldValue + ) { + return new ClusterContainsDocumentWithFieldValueMatcher(indexName, documentId, fieldName, fieldValue); + } + + public static Matcher clusterContainsSnapshotRepository(String repositoryName) { + return new ClusterContainsSnapshotRepositoryMatcher(repositoryName); + } + + public static Matcher clusterContainSuccessSnapshot(String repositoryName, String snapshotName) { + return new ClusterContainSuccessSnapshotMatcher(repositoryName, snapshotName); + } + + public static Matcher snapshotInClusterDoesNotExists(String repositoryName, String snapshotName) { + return new SnapshotInClusterDoesNotExist(repositoryName, snapshotName); + } + + public static Matcher indexExists(String expectedIndexName) { + return new IndexExistsMatcher(expectedIndexName); + } + + public static Matcher indexSettingsContainValues(String expectedIndexName, Settings expectedSettings) { + return new IndexSettingsContainValuesMatcher(expectedIndexName, expectedSettings); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/ContainNotEmptyScrollingIdMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/ContainNotEmptyScrollingIdMatcher.java new file mode 100644 index 0000000000..03b9b6bab8 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/ContainNotEmptyScrollingIdMatcher.java @@ -0,0 +1,34 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.apache.commons.lang3.StringUtils; +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.search.SearchResponse; + +class ContainNotEmptyScrollingIdMatcher extends TypeSafeDiagnosingMatcher { + + @Override + protected boolean matchesSafely(SearchResponse searchResponse, Description mismatchDescription) { + String scrollId = searchResponse.getScrollId(); + if (StringUtils.isEmpty(scrollId)) { + mismatchDescription.appendText("scrolling id is null or empty"); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Search response should contain scrolling id."); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/ContainsAggregationWithNameAndTypeMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/ContainsAggregationWithNameAndTypeMatcher.java new file mode 100644 index 0000000000..b1ef21c922 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/ContainsAggregationWithNameAndTypeMatcher.java @@ -0,0 +1,57 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.search.SearchResponse; +import org.opensearch.search.aggregations.Aggregation; +import org.opensearch.search.aggregations.Aggregations; + +import static java.util.Objects.requireNonNull; + +class ContainsAggregationWithNameAndTypeMatcher extends TypeSafeDiagnosingMatcher { + + private final String expectedAggregationName; + private final String expectedAggregationType; + + public ContainsAggregationWithNameAndTypeMatcher(String expectedAggregationName, String expectedAggregationType) { + this.expectedAggregationName = requireNonNull(expectedAggregationName, "Aggregation name is required"); + this.expectedAggregationType = requireNonNull(expectedAggregationType, "Expected aggregation type is required."); + } + + @Override + protected boolean matchesSafely(SearchResponse response, Description mismatchDescription) { + Aggregations aggregations = response.getAggregations(); + if (aggregations == null) { + mismatchDescription.appendText("search response does not contain aggregations"); + return false; + } + Aggregation aggregation = aggregations.get(expectedAggregationName); + if (aggregation == null) { + mismatchDescription.appendText("Response does not contain aggregation with name ").appendValue(expectedAggregationName); + return false; + } + if (expectedAggregationType.equals(aggregation.getType()) == false) { + mismatchDescription.appendText("Aggregation contain incorrect type which is ").appendValue(aggregation.getType()); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Search response should contains aggregation results with name ") + .appendValue(expectedAggregationName) + .appendText(" and type ") + .appendValue(expectedAggregationType); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/ContainsExactlyIndicesMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/ContainsExactlyIndicesMatcher.java new file mode 100644 index 0000000000..9b597ad1c8 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/ContainsExactlyIndicesMatcher.java @@ -0,0 +1,46 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import java.util.Set; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.fieldcaps.FieldCapabilitiesResponse; + +import static java.util.Objects.isNull; + +class ContainsExactlyIndicesMatcher extends TypeSafeDiagnosingMatcher { + + private final Set expectedIndices; + + ContainsExactlyIndicesMatcher(String... expectedIndices) { + if (isNull(expectedIndices) || expectedIndices.length == 0) { + throw new IllegalArgumentException("expectedIndices cannot be null or empty"); + } + this.expectedIndices = Set.of(expectedIndices); + } + + @Override + protected boolean matchesSafely(FieldCapabilitiesResponse response, Description mismatchDescription) { + Set actualIndices = Set.of(response.getIndices()); + if (!expectedIndices.equals(actualIndices)) { + mismatchDescription.appendText("Actual indices: ").appendValue(actualIndices); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Response contains indices: ").appendValue(expectedIndices); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/ContainsFieldWithTypeMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/ContainsFieldWithTypeMatcher.java new file mode 100644 index 0000000000..cd6b4b05cc --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/ContainsFieldWithTypeMatcher.java @@ -0,0 +1,55 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import java.util.Map; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.fieldcaps.FieldCapabilities; +import org.opensearch.action.fieldcaps.FieldCapabilitiesResponse; + +import static java.util.Objects.requireNonNull; + +class ContainsFieldWithTypeMatcher extends TypeSafeDiagnosingMatcher { + + private final String expectedFieldName; + private final String expectedFieldType; + + ContainsFieldWithTypeMatcher(String expectedFieldName, String expectedFieldType) { + this.expectedFieldName = requireNonNull(expectedFieldName, "Field name is required"); + ; + this.expectedFieldType = requireNonNull(expectedFieldType, "Field type is required"); + ; + } + + @Override + protected boolean matchesSafely(FieldCapabilitiesResponse response, Description mismatchDescription) { + Map> fieldCapabilitiesMap = response.get(); + if (!fieldCapabilitiesMap.containsKey(expectedFieldName)) { + mismatchDescription.appendText("Response does not contain field with name ").appendText(expectedFieldName); + return false; + } + if (!fieldCapabilitiesMap.get(expectedFieldName).containsKey(expectedFieldType)) { + mismatchDescription.appendText("Field type does not match ").appendText(expectedFieldType); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Response contains field with name ") + .appendValue(expectedFieldName) + .appendText(" and type ") + .appendValue(expectedFieldType); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/DeleteResponseMatchers.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/DeleteResponseMatchers.java new file mode 100644 index 0000000000..4112d0bab8 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/DeleteResponseMatchers.java @@ -0,0 +1,23 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Matcher; + +import org.opensearch.action.delete.DeleteResponse; + +public class DeleteResponseMatchers { + + private DeleteResponseMatchers() {} + + public static Matcher isSuccessfulDeleteResponse() { + return new SuccessfulDeleteResponseMatcher(); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/ExactNumberOfAuditsFulfillPredicateMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/ExactNumberOfAuditsFulfillPredicateMatcher.java new file mode 100644 index 0000000000..9bbe966e07 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/ExactNumberOfAuditsFulfillPredicateMatcher.java @@ -0,0 +1,45 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import java.util.List; +import java.util.function.Predicate; + +import org.hamcrest.Description; + +import org.opensearch.security.auditlog.impl.AuditMessage; + +class ExactNumberOfAuditsFulfillPredicateMatcher extends AuditsFulfillPredicateMatcher { + + private final long exactNumberOfAuditMessages; + + public ExactNumberOfAuditsFulfillPredicateMatcher(long exactNumberOfAuditMessages, Predicate predicate) { + super(predicate); + this.exactNumberOfAuditMessages = exactNumberOfAuditMessages; + } + + @Override + protected boolean matchesSafely(List audits, Description mismatchDescription) { + long count = countAuditsWhichMatchPredicate(audits); + if (exactNumberOfAuditMessages != count) { + mismatchDescription.appendText(" only ") + .appendValue(count) + .appendText(" match predicate. Examined audit logs ") + .appendText(auditMessagesToString(audits)); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendValue(exactNumberOfAuditMessages).appendText(" audit records should match predicate ").appendValue(predicate); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/ExceptionErrorMessageMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/ExceptionErrorMessageMatcher.java new file mode 100644 index 0000000000..2e07bd0ec5 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/ExceptionErrorMessageMatcher.java @@ -0,0 +1,43 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import static java.util.Objects.requireNonNull; + +class ExceptionErrorMessageMatcher extends TypeSafeDiagnosingMatcher { + + private final Matcher errorMessageMatcher; + + public ExceptionErrorMessageMatcher(Matcher errorMessageMatcher) { + this.errorMessageMatcher = requireNonNull(errorMessageMatcher, "Error message matcher is required"); + } + + @Override + protected boolean matchesSafely(Throwable ex, Description mismatchDescription) { + boolean matches = errorMessageMatcher.matches(ex.getMessage()); + if (matches == false) { + mismatchDescription.appendText("Exception of class ") + .appendValue(ex.getClass().getCanonicalName()) + .appendText("contains unexpected error message which is ") + .appendValue(ex.getMessage()); + } + return matches; + + } + + @Override + public void describeTo(Description description) { + description.appendText("Error message in exception matches").appendValue(errorMessageMatcher); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/ExceptionHasCauseMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/ExceptionHasCauseMatcher.java new file mode 100644 index 0000000000..ed1b203898 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/ExceptionHasCauseMatcher.java @@ -0,0 +1,43 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import java.util.Objects; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +class ExceptionHasCauseMatcher extends TypeSafeDiagnosingMatcher { + + private final Class expectedCauseType; + + public ExceptionHasCauseMatcher(Class expectedCauseType) { + this.expectedCauseType = Objects.requireNonNull(expectedCauseType, "Exception cause type is required"); + } + + @Override + protected boolean matchesSafely(Throwable throwable, Description mismatchDescription) { + Throwable cause = throwable.getCause(); + if (cause == null) { + mismatchDescription.appendText("exception cause is null"); + return false; + } + if (expectedCauseType.isInstance(cause) == false) { + mismatchDescription.appendText(" cause is instance of ").appendValue(cause.getClass()); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Exception cause is instance of ").appendValue(expectedCauseType); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/ExceptionMatcherAssert.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/ExceptionMatcherAssert.java new file mode 100644 index 0000000000..671f22b8e3 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/ExceptionMatcherAssert.java @@ -0,0 +1,40 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Matcher; + +import static java.util.Objects.requireNonNull; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.notNullValue; + +public class ExceptionMatcherAssert { + + @FunctionalInterface + public interface ThrowingCallable { + void call() throws Exception; + } + + public static void assertThatThrownBy(ThrowingCallable throwingCallable, Matcher matcher) { + Throwable expectedException = catchThrowable(throwingCallable); + assertThat("Expected exception was not thrown", expectedException, notNullValue()); + assertThat(expectedException, matcher); + } + + public static Throwable catchThrowable(ThrowingCallable throwingCallable) { + Throwable expectedException = null; + try { + requireNonNull(throwingCallable, "ThrowingCallable must not be null.").call(); + } catch (Throwable e) { + expectedException = e; + } + return expectedException; + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/FailureBulkResponseMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/FailureBulkResponseMatcher.java new file mode 100644 index 0000000000..3d912e0283 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/FailureBulkResponseMatcher.java @@ -0,0 +1,32 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.bulk.BulkResponse; + +class FailureBulkResponseMatcher extends TypeSafeDiagnosingMatcher { + + @Override + protected boolean matchesSafely(BulkResponse response, Description mismatchDescription) { + if (response.hasFailures() == false) { + mismatchDescription.appendText(" bulk operation was executed correctly what is not expected."); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("bulk operation failure"); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/FieldCapabilitiesResponseMatchers.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/FieldCapabilitiesResponseMatchers.java new file mode 100644 index 0000000000..2a78c7b71c --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/FieldCapabilitiesResponseMatchers.java @@ -0,0 +1,32 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Matcher; + +import org.opensearch.action.fieldcaps.FieldCapabilitiesResponse; + +public class FieldCapabilitiesResponseMatchers { + + private FieldCapabilitiesResponseMatchers() {} + + public static Matcher containsExactlyIndices(String... expectedIndices) { + return new ContainsExactlyIndicesMatcher(expectedIndices); + } + + public static Matcher containsFieldWithNameAndType(String expectedFieldName, String expectedFieldType) { + return new ContainsFieldWithTypeMatcher(expectedFieldName, expectedFieldType); + } + + public static Matcher numberOfFieldsIsEqualTo(int expectedNumberOfFields) { + return new NumberOfFieldsIsEqualToMatcher(expectedNumberOfFields); + } + +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/GetIndexResponseContainsIndicesMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/GetIndexResponseContainsIndicesMatcher.java new file mode 100644 index 0000000000..20f02b1319 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/GetIndexResponseContainsIndicesMatcher.java @@ -0,0 +1,49 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import java.util.Arrays; +import java.util.List; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.client.indices.GetIndexResponse; + +import static java.util.Objects.isNull; + +class GetIndexResponseContainsIndicesMatcher extends TypeSafeDiagnosingMatcher { + + private final String[] expectedIndices; + + GetIndexResponseContainsIndicesMatcher(String[] expectedIndices) { + if (isNull(expectedIndices) || 0 == expectedIndices.length) { + throw new IllegalArgumentException("expectedIndices cannot be null or empty"); + } + this.expectedIndices = expectedIndices; + } + + @Override + protected boolean matchesSafely(GetIndexResponse response, Description mismatchDescription) { + List actual = Arrays.asList(response.getIndices()); + for (String index : expectedIndices) { + if (!actual.contains(index)) { + mismatchDescription.appendText("Actual indices: ").appendValue(response.getIndices()); + return false; + } + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Response should contain indices: ").appendValue(expectedIndices); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/GetMappingsResponseContainsIndicesMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/GetMappingsResponseContainsIndicesMatcher.java new file mode 100644 index 0000000000..8e2b3da097 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/GetMappingsResponseContainsIndicesMatcher.java @@ -0,0 +1,49 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import java.util.Map; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.client.indices.GetMappingsResponse; +import org.opensearch.cluster.metadata.MappingMetadata; + +import static java.util.Objects.isNull; + +class GetMappingsResponseContainsIndicesMatcher extends TypeSafeDiagnosingMatcher { + + private final String[] expectedIndices; + + GetMappingsResponseContainsIndicesMatcher(String[] expectedIndices) { + if (isNull(expectedIndices) || 0 == expectedIndices.length) { + throw new IllegalArgumentException("expectedIndices cannot be null or empty"); + } + this.expectedIndices = expectedIndices; + } + + @Override + protected boolean matchesSafely(GetMappingsResponse response, Description mismatchDescription) { + Map indicesMappings = response.mappings(); + for (String index : expectedIndices) { + if (!indicesMappings.containsKey(index)) { + mismatchDescription.appendText("Response contains mappings of indices: ").appendValue(indicesMappings.keySet()); + return false; + } + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Response should contain mappings of indices: ").appendValue(expectedIndices); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/GetResponseContainOnlyDocumentIdMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/GetResponseContainOnlyDocumentIdMatcher.java new file mode 100644 index 0000000000..e6d7f02c6e --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/GetResponseContainOnlyDocumentIdMatcher.java @@ -0,0 +1,54 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.get.GetResponse; + +import static java.util.Objects.requireNonNull; + +class GetResponseContainOnlyDocumentIdMatcher extends TypeSafeDiagnosingMatcher { + + private final String indexName; + private final String documentId; + + public GetResponseContainOnlyDocumentIdMatcher(String indexName, String documentId) { + this.indexName = requireNonNull(indexName, "Index name is required"); + this.documentId = requireNonNull(documentId, "Document id is required"); + } + + @Override + protected boolean matchesSafely(GetResponse response, Description mismatchDescription) { + if (indexName.equals(response.getIndex()) == false) { + mismatchDescription.appendText(" index name ").appendValue(response.getIndex()).appendText(" is incorrect "); + return false; + } + if (documentId.equals(response.getId()) == false) { + mismatchDescription.appendText(" id ").appendValue(response.getId()).appendText(" is incorrect "); + return false; + } + if (response.isExists()) { + mismatchDescription.appendText(" document exist what is not desired "); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Response should contain document id from index ") + .appendValue(indexName) + .appendText(" with id ") + .appendValue(documentId) + .appendText(" but document should not be present "); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/GetResponseContainsDocumentWithIdMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/GetResponseContainsDocumentWithIdMatcher.java new file mode 100644 index 0000000000..aa9d702243 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/GetResponseContainsDocumentWithIdMatcher.java @@ -0,0 +1,57 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.get.GetResponse; + +import static java.util.Objects.requireNonNull; + +class GetResponseContainsDocumentWithIdMatcher extends TypeSafeDiagnosingMatcher { + + private final String indexName; + private final String documentId; + + public GetResponseContainsDocumentWithIdMatcher(String indexName, String documentId) { + this.indexName = requireNonNull(indexName, "Index name is required"); + this.documentId = requireNonNull(documentId, "Document id is required"); + } + + @Override + protected boolean matchesSafely(GetResponse response, Description mismatchDescription) { + if (indexName.equals(response.getIndex()) == false) { + mismatchDescription.appendText("Document should not belong to index ").appendValue(response.getIndex()); + return false; + } + if (documentId.equals(response.getId()) == false) { + mismatchDescription.appendText("Document contain incorrect id which is ").appendValue(response.getId()); + return false; + } + if (response.isExists() == false) { + mismatchDescription.appendText("Document does not exist or is inaccessible"); + return false; + } + if (response.isSourceEmpty()) { + mismatchDescription.appendText("Document source is empty"); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Response should contain document from index ") + .appendValue(indexName) + .appendText(" with id ") + .appendValue(documentId); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/GetResponseDocumentContainsExactlyFieldsWithNamesMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/GetResponseDocumentContainsExactlyFieldsWithNamesMatcher.java new file mode 100644 index 0000000000..66bdc0a9b7 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/GetResponseDocumentContainsExactlyFieldsWithNamesMatcher.java @@ -0,0 +1,51 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import java.util.Map; +import java.util.Set; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.get.GetResponse; + +import static java.util.Objects.isNull; + +class GetResponseDocumentContainsExactlyFieldsWithNamesMatcher extends TypeSafeDiagnosingMatcher { + + private final Set expectedFieldsNames; + + GetResponseDocumentContainsExactlyFieldsWithNamesMatcher(String... expectedFieldsNames) { + if (isNull(expectedFieldsNames) || expectedFieldsNames.length == 0) { + throw new IllegalArgumentException("expectedFieldsNames cannot be null or empty"); + } + this.expectedFieldsNames = Set.of(expectedFieldsNames); + } + + @Override + protected boolean matchesSafely(GetResponse response, Description mismatchDescription) { + Map sourceMap = response.getSourceAsMap(); + Set actualFieldsNames = sourceMap.keySet(); + if (!expectedFieldsNames.equals(actualFieldsNames)) { + mismatchDescription.appendValue("Document with id ") + .appendValue(response.getId()) + .appendText(" contains fields with names: ") + .appendValue(actualFieldsNames); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Document contain exactly fields with names: ").appendValue(expectedFieldsNames); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/GetResponseDocumentDoesNotContainFieldMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/GetResponseDocumentDoesNotContainFieldMatcher.java new file mode 100644 index 0000000000..508e5b8b61 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/GetResponseDocumentDoesNotContainFieldMatcher.java @@ -0,0 +1,47 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import java.util.Map; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.get.GetResponse; + +import static java.util.Objects.requireNonNull; + +class GetResponseDocumentDoesNotContainFieldMatcher extends TypeSafeDiagnosingMatcher { + + private final String fieldName; + + public GetResponseDocumentDoesNotContainFieldMatcher(String fieldName) { + this.fieldName = requireNonNull(fieldName, "Field name is required."); + } + + @Override + protected boolean matchesSafely(GetResponse response, Description mismatchDescription) { + Map source = response.getSource(); + if (source == null) { + mismatchDescription.appendText("Source is not available in search results"); + return false; + } + if (source.containsKey(fieldName)) { + mismatchDescription.appendText("Document contains field ").appendValue(fieldName); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Document does not contain field ").appendValue(fieldName); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/GetResponseDocumentFieldValueMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/GetResponseDocumentFieldValueMatcher.java new file mode 100644 index 0000000000..78bcae5494 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/GetResponseDocumentFieldValueMatcher.java @@ -0,0 +1,57 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import java.util.Map; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.get.GetResponse; + +import static java.util.Objects.requireNonNull; + +class GetResponseDocumentFieldValueMatcher extends TypeSafeDiagnosingMatcher { + + private final String fieldName; + private final Object fieldValue; + + public GetResponseDocumentFieldValueMatcher(String fieldName, Object fieldValue) { + this.fieldName = requireNonNull(fieldName, "Field name is required."); + this.fieldValue = requireNonNull(fieldValue, "Field value is required."); + } + + @Override + protected boolean matchesSafely(GetResponse response, Description mismatchDescription) { + Map source = response.getSource(); + if (source == null) { + mismatchDescription.appendText("Source is not available in search results"); + return false; + } + if (source.containsKey(fieldName) == false) { + mismatchDescription.appendText("Document does not contain field ").appendValue(fieldName); + return false; + } + Object actualFieldValue = source.get(fieldName); + if (fieldValue.equals(actualFieldValue) == false) { + mismatchDescription.appendText("Field ") + .appendValue(fieldName) + .appendText(" has incorrect value ") + .appendValue(actualFieldValue); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Document contain field ").appendValue(fieldName).appendText(" with value ").appendValue(fieldValue); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/GetResponseMatchers.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/GetResponseMatchers.java new file mode 100644 index 0000000000..89c183fc34 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/GetResponseMatchers.java @@ -0,0 +1,39 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Matcher; + +import org.opensearch.action.get.GetResponse; + +public class GetResponseMatchers { + + private GetResponseMatchers() {} + + public static Matcher containDocument(String indexName, String documentId) { + return new GetResponseContainsDocumentWithIdMatcher(indexName, documentId); + } + + public static Matcher containOnlyDocumentId(String indexName, String documentId) { + return new GetResponseContainOnlyDocumentIdMatcher(indexName, documentId); + } + + public static Matcher documentContainField(String fieldName, Object fieldValue) { + return new GetResponseDocumentFieldValueMatcher(fieldName, fieldValue); + } + + public static Matcher documentDoesNotContainField(String fieldName) { + return new GetResponseDocumentDoesNotContainFieldMatcher(fieldName); + } + + public static Matcher documentContainsExactlyFieldsWithNames(String... expectedFieldsNames) { + return new GetResponseDocumentContainsExactlyFieldsWithNamesMatcher(expectedFieldsNames); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/IndexExistsMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/IndexExistsMatcher.java new file mode 100644 index 0000000000..aab3d426d2 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/IndexExistsMatcher.java @@ -0,0 +1,49 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest; +import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse; +import org.opensearch.client.Client; +import org.opensearch.test.framework.cluster.LocalCluster; + +import static java.util.Objects.requireNonNull; + +class IndexExistsMatcher extends TypeSafeDiagnosingMatcher { + + private final String expectedIndexName; + + IndexExistsMatcher(String expectedIndexName) { + this.expectedIndexName = requireNonNull(expectedIndexName); + } + + @Override + protected boolean matchesSafely(LocalCluster cluster, Description mismatchDescription) { + try (Client client = cluster.getInternalNodeClient()) { + IndicesExistsResponse indicesExistsResponse = client.admin() + .indices() + .exists(new IndicesExistsRequest(expectedIndexName)) + .actionGet(); + if (!indicesExistsResponse.isExists()) { + mismatchDescription.appendText("Index ").appendValue(expectedIndexName).appendValue(" does not exist"); + return false; + } + return true; + } + } + + @Override + public void describeTo(Description description) { + description.appendText("Index ").appendValue(expectedIndexName).appendText(" exists"); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/IndexSettingsContainValuesMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/IndexSettingsContainValuesMatcher.java new file mode 100644 index 0000000000..6bb89c6cae --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/IndexSettingsContainValuesMatcher.java @@ -0,0 +1,75 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.opensearch.client.Client; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.test.framework.cluster.LocalCluster; + +import static java.util.Objects.isNull; +import static java.util.Objects.requireNonNull; + +class IndexSettingsContainValuesMatcher extends TypeSafeDiagnosingMatcher { + + private final String expectedIndexName; + private final Settings expectedSettings; + + IndexSettingsContainValuesMatcher(String expectedIndexName, Settings expectedSettings) { + this.expectedIndexName = requireNonNull(expectedIndexName); + if (isNull(expectedSettings) || expectedSettings.isEmpty()) { + throw new IllegalArgumentException("expectedSettings cannot be null or empty"); + } + this.expectedSettings = expectedSettings; + } + + @Override + protected boolean matchesSafely(LocalCluster cluster, Description mismatchDescription) { + try (Client client = cluster.getInternalNodeClient()) { + GetSettingsResponse response = client.admin() + .indices() + .getSettings(new GetSettingsRequest().indices(expectedIndexName)) + .actionGet(); + + Settings actualSettings = response.getIndexToSettings().get(expectedIndexName); + + for (String setting : expectedSettings.keySet()) { + if (isNull(actualSettings.get(setting))) { + mismatchDescription.appendValue("Value of ").appendValue(setting).appendText(" property is missing"); + return false; + } + if (!expectedSettings.get(setting).equals(actualSettings.get(setting))) { + mismatchDescription.appendText("Actual value of `") + .appendValue(setting) + .appendText("` property: ") + .appendValue(actualSettings.get(setting)); + return false; + } + } + return true; + } catch (IndexNotFoundException e) { + mismatchDescription.appendText("Index: ").appendValue(expectedIndexName).appendText(" does not exist"); + return false; + } + } + + @Override + public void describeTo(Description description) { + description.appendText("Settings of index ") + .appendValue(expectedIndexName) + .appendText(" should contain values: ") + .appendValue(expectedSettings); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/MultiGetResponseMatchers.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/MultiGetResponseMatchers.java new file mode 100644 index 0000000000..c2e86b1310 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/MultiGetResponseMatchers.java @@ -0,0 +1,28 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Matcher; + +import org.opensearch.action.get.MultiGetResponse; + +public class MultiGetResponseMatchers { + + private MultiGetResponseMatchers() {} + + public static Matcher isSuccessfulMultiGetResponse() { + return new SuccessfulMultiGetResponseMatcher(); + } + + public static Matcher numberOfGetItemResponsesIsEqualTo(int expectedNumberOfResponses) { + return new NumberOfGetItemResponsesIsEqualToMatcher(expectedNumberOfResponses); + } + +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/MultiSearchResponseMatchers.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/MultiSearchResponseMatchers.java new file mode 100644 index 0000000000..9709249d11 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/MultiSearchResponseMatchers.java @@ -0,0 +1,28 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Matcher; + +import org.opensearch.action.search.MultiSearchResponse; + +public class MultiSearchResponseMatchers { + + private MultiSearchResponseMatchers() {} + + public static Matcher isSuccessfulMultiSearchResponse() { + return new SuccessfulMultiSearchResponseMatcher(); + } + + public static Matcher numberOfSearchItemResponsesIsEqualTo(int expectedNumberOfResponses) { + return new NumberOfSearchItemResponsesIsEqualToMatcher(expectedNumberOfResponses); + } + +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/NumberOfFieldsIsEqualToMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/NumberOfFieldsIsEqualToMatcher.java new file mode 100644 index 0000000000..ad8e9725c3 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/NumberOfFieldsIsEqualToMatcher.java @@ -0,0 +1,38 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.fieldcaps.FieldCapabilitiesResponse; + +class NumberOfFieldsIsEqualToMatcher extends TypeSafeDiagnosingMatcher { + + private final int expectedNumberOfFields; + + NumberOfFieldsIsEqualToMatcher(int expectedNumberOfFields) { + this.expectedNumberOfFields = expectedNumberOfFields; + } + + @Override + protected boolean matchesSafely(FieldCapabilitiesResponse response, Description mismatchDescription) { + if (expectedNumberOfFields != response.get().size()) { + mismatchDescription.appendText("Actual number of fields: ").appendValue(response.get().size()); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Response contains information about ").appendValue(expectedNumberOfFields).appendText(" fields"); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/NumberOfGetItemResponsesIsEqualToMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/NumberOfGetItemResponsesIsEqualToMatcher.java new file mode 100644 index 0000000000..38cfaeb130 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/NumberOfGetItemResponsesIsEqualToMatcher.java @@ -0,0 +1,38 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.get.MultiGetResponse; + +class NumberOfGetItemResponsesIsEqualToMatcher extends TypeSafeDiagnosingMatcher { + + private final int expectedNumberOfResponses; + + NumberOfGetItemResponsesIsEqualToMatcher(int expectedNumberOfResponses) { + this.expectedNumberOfResponses = expectedNumberOfResponses; + } + + @Override + protected boolean matchesSafely(MultiGetResponse response, Description mismatchDescription) { + if (expectedNumberOfResponses != response.getResponses().length) { + mismatchDescription.appendText("Actual number of responses: ").appendValue(response.getResponses().length); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Multi get response contains: ").appendValue(expectedNumberOfResponses).appendText(" item responses"); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/NumberOfHitsInPageIsEqualToMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/NumberOfHitsInPageIsEqualToMatcher.java new file mode 100644 index 0000000000..8a25a336f3 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/NumberOfHitsInPageIsEqualToMatcher.java @@ -0,0 +1,45 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.search.SearchResponse; +import org.opensearch.search.SearchHits; + +class NumberOfHitsInPageIsEqualToMatcher extends TypeSafeDiagnosingMatcher { + + private final int expectedNumberOfHits; + + public NumberOfHitsInPageIsEqualToMatcher(int expectedNumberOfHits) { + this.expectedNumberOfHits = expectedNumberOfHits; + } + + @Override + protected boolean matchesSafely(SearchResponse searchResponse, Description mismatchDescription) { + SearchHits hits = searchResponse.getHits(); + if ((hits == null) || (hits.getHits() == null)) { + mismatchDescription.appendText("contains null hits"); + return false; + } + int actualNumberOfHits = hits.getHits().length; + if (expectedNumberOfHits != actualNumberOfHits) { + mismatchDescription.appendText("actual number of hits is equal to ").appendValue(actualNumberOfHits); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Number of hits on current page should be equal to ").appendValue(expectedNumberOfHits); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/NumberOfSearchItemResponsesIsEqualToMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/NumberOfSearchItemResponsesIsEqualToMatcher.java new file mode 100644 index 0000000000..54bb83cba7 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/NumberOfSearchItemResponsesIsEqualToMatcher.java @@ -0,0 +1,39 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.search.MultiSearchResponse; + +class NumberOfSearchItemResponsesIsEqualToMatcher extends TypeSafeDiagnosingMatcher { + + private final int expectedNumberOfResponses; + + NumberOfSearchItemResponsesIsEqualToMatcher(int expectedNumberOfResponses) { + this.expectedNumberOfResponses = expectedNumberOfResponses; + } + + @Override + protected boolean matchesSafely(MultiSearchResponse response, Description mismatchDescription) { + if (expectedNumberOfResponses != response.getResponses().length) { + mismatchDescription.appendText("Actual number of responses: ").appendValue(response.getResponses().length); + return false; + } + + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Multi search response contains: ").appendValue(expectedNumberOfResponses).appendText(" item responses"); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/NumberOfTotalHitsIsEqualToMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/NumberOfTotalHitsIsEqualToMatcher.java new file mode 100644 index 0000000000..3f2b379498 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/NumberOfTotalHitsIsEqualToMatcher.java @@ -0,0 +1,59 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import java.util.Arrays; +import java.util.stream.Collectors; + +import org.apache.lucene.search.TotalHits; +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.search.SearchResponse; +import org.opensearch.search.SearchHits; + +class NumberOfTotalHitsIsEqualToMatcher extends TypeSafeDiagnosingMatcher { + + private final int expectedNumberOfHits; + + NumberOfTotalHitsIsEqualToMatcher(int expectedNumberOfHits) { + this.expectedNumberOfHits = expectedNumberOfHits; + } + + @Override + protected boolean matchesSafely(SearchResponse searchResponse, Description mismatchDescription) { + SearchHits hits = searchResponse.getHits(); + if (hits == null) { + mismatchDescription.appendText("contains null hits"); + return false; + } + TotalHits totalHits = hits.getTotalHits(); + if (totalHits == null) { + mismatchDescription.appendText("Total hits number is null."); + return false; + } + if (expectedNumberOfHits != totalHits.value) { + String documentIds = Arrays.stream(searchResponse.getHits().getHits()) + .map(hit -> hit.getIndex() + "/" + hit.getId()) + .collect(Collectors.joining(",")); + mismatchDescription.appendText("contains ") + .appendValue(hits.getHits().length) + .appendText(" hits, found document ids ") + .appendValue(documentIds); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Search response should contains ").appendValue(expectedNumberOfHits).appendText(" hits"); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/OpenSearchExceptionMatchers.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/OpenSearchExceptionMatchers.java new file mode 100644 index 0000000000..8627e4bda2 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/OpenSearchExceptionMatchers.java @@ -0,0 +1,37 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Matcher; + +import org.opensearch.rest.RestStatus; + +import static org.hamcrest.Matchers.containsString; + +public class OpenSearchExceptionMatchers { + + private OpenSearchExceptionMatchers() {} + + public static Matcher statusException(RestStatus expectedRestStatus) { + return new OpenSearchStatusExceptionMatcher(expectedRestStatus); + } + + public static Matcher errorMessage(Matcher errorMessageMatcher) { + return new ExceptionErrorMessageMatcher(errorMessageMatcher); + } + + public static Matcher errorMessageContain(String errorMessage) { + return errorMessage(containsString(errorMessage)); + } + + public static Matcher hasCause(Class clazz) { + return new ExceptionHasCauseMatcher(clazz); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/OpenSearchStatusExceptionMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/OpenSearchStatusExceptionMatcher.java new file mode 100644 index 0000000000..118251ec04 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/OpenSearchStatusExceptionMatcher.java @@ -0,0 +1,52 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.OpenSearchException; +import org.opensearch.rest.RestStatus; + +import static java.util.Objects.requireNonNull; + +class OpenSearchStatusExceptionMatcher extends TypeSafeDiagnosingMatcher { + + private final RestStatus expectedRestStatus; + + public OpenSearchStatusExceptionMatcher(RestStatus expectedRestStatus) { + this.expectedRestStatus = requireNonNull(expectedRestStatus, "Expected rest status is required."); + } + + @Override + protected boolean matchesSafely(Throwable throwable, Description mismatchDescription) { + if ((throwable instanceof OpenSearchException) == false) { + mismatchDescription.appendText("actual exception type is ") + .appendValue(throwable.getClass().getCanonicalName()) + .appendText(", error message ") + .appendValue(throwable.getMessage()); + return false; + } + OpenSearchException openSearchException = (OpenSearchException) throwable; + if (expectedRestStatus.equals(openSearchException.status()) == false) { + mismatchDescription.appendText("actual status code is ") + .appendValue(openSearchException.status()) + .appendText(", error message ") + .appendValue(throwable.getMessage()); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("OpenSearchException with status code ").appendValue(expectedRestStatus); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchHitContainsFieldWithValueMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchHitContainsFieldWithValueMatcher.java new file mode 100644 index 0000000000..c92924ebfe --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchHitContainsFieldWithValueMatcher.java @@ -0,0 +1,74 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import java.util.Map; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.search.SearchResponse; +import org.opensearch.search.SearchHit; + +import static org.opensearch.test.framework.matcher.SearchResponseMatchers.readTotalHits; + +class SearchHitContainsFieldWithValueMatcher extends TypeSafeDiagnosingMatcher { + + private final int hitIndex; + + private final String fieldName; + + private final T expectedValue; + + SearchHitContainsFieldWithValueMatcher(int hitIndex, String fieldName, T expectedValue) { + this.hitIndex = hitIndex; + this.fieldName = fieldName; + this.expectedValue = expectedValue; + } + + @Override + protected boolean matchesSafely(SearchResponse searchResponse, Description mismatchDescription) { + Long numberOfHits = readTotalHits(searchResponse); + if (numberOfHits == null) { + mismatchDescription.appendText("Total number of hits is unknown."); + return false; + } + if (hitIndex >= numberOfHits) { + mismatchDescription.appendText("Search result contain only ").appendValue(numberOfHits).appendText(" hits"); + return false; + } + SearchHit searchHit = searchResponse.getHits().getAt(hitIndex); + Map source = searchHit.getSourceAsMap(); + if (source == null) { + mismatchDescription.appendText("Source document is null, is fetch source option set to true?"); + return false; + } + if (source.containsKey(fieldName) == false) { + mismatchDescription.appendText("Document does not contain field ").appendValue(fieldName); + return false; + } + Object actualValue = source.get(fieldName); + if (!expectedValue.equals(actualValue)) { + mismatchDescription.appendText("Field value is equal to ").appendValue(actualValue); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Search hit with index ") + .appendValue(hitIndex) + .appendText(" should contain field ") + .appendValue(fieldName) + .appendValue(" with value equal to ") + .appendValue(expectedValue); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchHitDoesNotContainFieldMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchHitDoesNotContainFieldMatcher.java new file mode 100644 index 0000000000..0562acdcbb --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchHitDoesNotContainFieldMatcher.java @@ -0,0 +1,65 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import java.util.Map; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.search.SearchResponse; +import org.opensearch.search.SearchHit; + +import static java.util.Objects.requireNonNull; +import static org.opensearch.test.framework.matcher.SearchResponseMatchers.readTotalHits; + +class SearchHitDoesNotContainFieldMatcher extends TypeSafeDiagnosingMatcher { + + private final int hitIndex; + + private final String fieldName; + + public SearchHitDoesNotContainFieldMatcher(int hitIndex, String fieldName) { + this.hitIndex = hitIndex; + this.fieldName = requireNonNull(fieldName, "Field name is required."); + } + + @Override + protected boolean matchesSafely(SearchResponse searchResponse, Description mismatchDescription) { + Long numberOfHits = readTotalHits(searchResponse); + if (numberOfHits == null) { + mismatchDescription.appendText("Total number of hits is unknown."); + return false; + } + if (hitIndex >= numberOfHits) { + mismatchDescription.appendText("Search result contain only ").appendValue(numberOfHits).appendText(" hits"); + return false; + } + SearchHit searchHit = searchResponse.getHits().getAt(hitIndex); + Map source = searchHit.getSourceAsMap(); + if (source == null) { + mismatchDescription.appendText("Source document is null, is fetch source option set to true?"); + return false; + } + if (source.containsKey(fieldName)) { + mismatchDescription.appendText(" document contains field ").appendValue(fieldName); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("search hit with index ") + .appendValue(hitIndex) + .appendText(" does not contain field ") + .appendValue(fieldName); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchHitsContainDocumentWithIdMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchHitsContainDocumentWithIdMatcher.java new file mode 100644 index 0000000000..d21df7a578 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchHitsContainDocumentWithIdMatcher.java @@ -0,0 +1,64 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.search.SearchResponse; +import org.opensearch.search.SearchHit; + +import static org.opensearch.test.framework.matcher.SearchResponseMatchers.readTotalHits; + +class SearchHitsContainDocumentWithIdMatcher extends TypeSafeDiagnosingMatcher { + + private final int hitIndex; + private final String indexName; + private final String id; + + public SearchHitsContainDocumentWithIdMatcher(int hitIndex, String indexName, String id) { + this.hitIndex = hitIndex; + this.indexName = indexName; + this.id = id; + } + + @Override + protected boolean matchesSafely(SearchResponse searchResponse, Description mismatchDescription) { + Long numberOfHits = readTotalHits(searchResponse); + if (numberOfHits == null) { + mismatchDescription.appendText("Number of total hits is unknown."); + return false; + } + if (hitIndex >= numberOfHits) { + mismatchDescription.appendText("Search result contain only ").appendValue(numberOfHits).appendText(" hits"); + return false; + } + SearchHit searchHit = searchResponse.getHits().getAt(hitIndex); + if (indexName.equals(searchHit.getIndex()) == false) { + mismatchDescription.appendText("document is part of another index ").appendValue(indexName); + return false; + } + if (id.equals(searchHit.getId()) == false) { + mismatchDescription.appendText("Document has another id which is ").appendValue(searchHit.getId()); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Search hit with index ") + .appendValue(hitIndex) + .appendText(" should contains document which is part of index ") + .appendValue(indexName) + .appendValue(" and has id ") + .appendValue(id); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchHitsContainDocumentsInAnyOrderMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchHitsContainDocumentsInAnyOrderMatcher.java new file mode 100644 index 0000000000..28bf13e321 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchHitsContainDocumentsInAnyOrderMatcher.java @@ -0,0 +1,76 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.tuple.Pair; +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.search.SearchResponse; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; + +import static java.util.Objects.requireNonNull; + +class SearchHitsContainDocumentsInAnyOrderMatcher extends TypeSafeDiagnosingMatcher { + + /** + * Pair contain index name and document id + */ + private final List> documentIds; + + /** + * + * @param documentIds Pair contain index name and document id + */ + public SearchHitsContainDocumentsInAnyOrderMatcher(List> documentIds) { + this.documentIds = requireNonNull(documentIds, "Document ids are required."); + } + + @Override + protected boolean matchesSafely(SearchResponse response, Description mismatchDescription) { + SearchHits hits = response.getHits(); + if (hits == null) { + mismatchDescription.appendText("Search response does not contains hits (null)."); + return false; + } + SearchHit[] hitsArray = hits.getHits(); + if (hitsArray == null) { + mismatchDescription.appendText("Search hits array is null"); + return false; + } + Set> actualDocumentIds = Arrays.stream(hitsArray) + .map(result -> Pair.of(result.getIndex(), result.getId())) + .collect(Collectors.toSet()); + for (Pair desiredDocumentId : documentIds) { + if (actualDocumentIds.contains(desiredDocumentId) == false) { + mismatchDescription.appendText("search result does not contain document with id ") + .appendValue(desiredDocumentId.getKey()) + .appendText("/") + .appendValue(desiredDocumentId.getValue()); + return false; + } + } + return true; + } + + @Override + public void describeTo(Description description) { + String documentIdsString = documentIds.stream() + .map(pair -> pair.getKey() + "/" + pair.getValue()) + .collect(Collectors.joining(", ")); + description.appendText("Search response should contains following documents ").appendValue(documentIdsString); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchResponseMatchers.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchResponseMatchers.java new file mode 100644 index 0000000000..c0a484b47c --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchResponseMatchers.java @@ -0,0 +1,87 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +import org.apache.commons.lang3.tuple.Pair; +import org.hamcrest.Matcher; + +import org.opensearch.action.search.SearchResponse; +import org.opensearch.rest.RestStatus; +import org.opensearch.search.SearchHits; + +public class SearchResponseMatchers { + + private SearchResponseMatchers() {} + + public static Matcher isSuccessfulSearchResponse() { + return new SuccessfulSearchResponseMatcher(); + } + + public static Matcher numberOfTotalHitsIsEqualTo(int expectedNumberOfHits) { + return new NumberOfTotalHitsIsEqualToMatcher(expectedNumberOfHits); + } + + public static Matcher numberOfHitsInPageIsEqualTo(int expectedNumberOfHits) { + return new NumberOfHitsInPageIsEqualToMatcher(expectedNumberOfHits); + } + + public static Matcher searchHitContainsFieldWithValue(int hitIndex, String fieldName, T expectedValue) { + return new SearchHitContainsFieldWithValueMatcher<>(hitIndex, fieldName, expectedValue); + } + + public static Matcher searchHitDoesNotContainField(int hitIndex, String fieldName) { + return new SearchHitDoesNotContainFieldMatcher(hitIndex, fieldName); + } + + public static Matcher searchHitsContainDocumentWithId(int hitIndex, String indexName, String documentId) { + return new SearchHitsContainDocumentWithIdMatcher(hitIndex, indexName, documentId); + } + + public static Matcher restStatusIs(RestStatus expectedRestStatus) { + return new SearchResponseWithStatusCodeMatcher(expectedRestStatus); + } + + public static Matcher containNotEmptyScrollingId() { + return new ContainNotEmptyScrollingIdMatcher(); + } + + public static Matcher containAggregationWithNameAndType( + String expectedAggregationName, + String expectedAggregationType + ) { + return new ContainsAggregationWithNameAndTypeMatcher(expectedAggregationName, expectedAggregationType); + } + + /** + * Matcher checks if search result contains all expected documents + * + * @param documentIds Pair contain index name and document id + * @return matcher + */ + public static Matcher searchHitsContainDocumentsInAnyOrder(List> documentIds) { + return new SearchHitsContainDocumentsInAnyOrderMatcher(documentIds); + } + + public static Matcher searchHitsContainDocumentsInAnyOrder(Pair... documentIds) { + return new SearchHitsContainDocumentsInAnyOrderMatcher(Arrays.asList(documentIds)); + } + + static Long readTotalHits(SearchResponse searchResponse) { + return Optional.ofNullable(searchResponse) + .map(SearchResponse::getHits) + .map(SearchHits::getTotalHits) + .map(totalHits -> totalHits.value) + .orElse(null); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchResponseWithStatusCodeMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchResponseWithStatusCodeMatcher.java new file mode 100644 index 0000000000..ede13bcb56 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchResponseWithStatusCodeMatcher.java @@ -0,0 +1,39 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.search.SearchResponse; +import org.opensearch.rest.RestStatus; + +class SearchResponseWithStatusCodeMatcher extends TypeSafeDiagnosingMatcher { + + private final RestStatus expectedRestStatus; + + public SearchResponseWithStatusCodeMatcher(RestStatus expectedRestStatus) { + this.expectedRestStatus = expectedRestStatus; + } + + @Override + protected boolean matchesSafely(SearchResponse searchResponse, Description mismatchDescription) { + if (expectedRestStatus.equals(searchResponse.status()) == false) { + mismatchDescription.appendText("actual response status is ").appendValue(searchResponse.status()); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Expected response status is ").appendValue(expectedRestStatus); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/SnapshotInClusterDoesNotExist.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/SnapshotInClusterDoesNotExist.java new file mode 100644 index 0000000000..36e50143f0 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/SnapshotInClusterDoesNotExist.java @@ -0,0 +1,49 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; +import org.opensearch.client.Client; +import org.opensearch.snapshots.SnapshotMissingException; + +import static java.util.Objects.requireNonNull; + +class SnapshotInClusterDoesNotExist extends TypeSafeDiagnosingMatcher { + private final String repositoryName; + private final String snapshotName; + + public SnapshotInClusterDoesNotExist(String repositoryName, String snapshotName) { + this.repositoryName = requireNonNull(repositoryName, "Snapshot repository name is required."); + this.snapshotName = requireNonNull(snapshotName, "Snapshot name is required."); + } + + @Override + protected boolean matchesSafely(Client client, Description mismatchDescription) { + try { + GetSnapshotsRequest request = new GetSnapshotsRequest(repositoryName, new String[] { snapshotName }); + client.admin().cluster().getSnapshots(request).actionGet(); + mismatchDescription.appendText("snapshot exists"); + return false; + } catch (SnapshotMissingException e) { + return true; + } + } + + @Override + public void describeTo(Description description) { + description.appendText("Snapshot ") + .appendValue(snapshotName) + .appendText(" does not exist in repository ") + .appendValue(repositoryName); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessBulkResponseMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessBulkResponseMatcher.java new file mode 100644 index 0000000000..7bf4993783 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessBulkResponseMatcher.java @@ -0,0 +1,47 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import java.util.Arrays; +import java.util.stream.Collectors; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.bulk.BulkItemResponse; +import org.opensearch.action.bulk.BulkResponse; +import org.opensearch.rest.RestStatus; + +class SuccessBulkResponseMatcher extends TypeSafeDiagnosingMatcher { + + @Override + protected boolean matchesSafely(BulkResponse response, Description mismatchDescription) { + RestStatus status = response.status(); + if (RestStatus.OK.equals(status) == false) { + mismatchDescription.appendText("incorrect response status ").appendValue(status); + return false; + } + if (response.hasFailures()) { + String failureDescription = Arrays.stream(response.getItems()) + .filter(BulkItemResponse::isFailed) + .map(BulkItemResponse::getFailure) + .map(Object::toString) + .collect(Collectors.joining(",\n")); + mismatchDescription.appendText("bulk response contains failures ").appendValue(failureDescription); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("success bulk response"); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulClearIndicesCacheResponseMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulClearIndicesCacheResponseMatcher.java new file mode 100644 index 0000000000..e27ee17ef6 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulClearIndicesCacheResponseMatcher.java @@ -0,0 +1,37 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse; +import org.opensearch.rest.RestStatus; + +class SuccessfulClearIndicesCacheResponseMatcher extends TypeSafeDiagnosingMatcher { + + @Override + protected boolean matchesSafely(ClearIndicesCacheResponse response, Description mismatchDescription) { + if (!RestStatus.OK.equals(response.getStatus())) { + mismatchDescription.appendText("Status is equal to ").appendValue(response.getStatus()); + return false; + } + if (response.getShardFailures().length != 0) { + mismatchDescription.appendText("Contains ").appendValue(response.getShardFailures().length).appendText(" shard failures"); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Successful clear index cache response"); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulCloseIndexResponseMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulCloseIndexResponseMatcher.java new file mode 100644 index 0000000000..480b7845e9 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulCloseIndexResponseMatcher.java @@ -0,0 +1,36 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.client.indices.CloseIndexResponse; + +class SuccessfulCloseIndexResponseMatcher extends TypeSafeDiagnosingMatcher { + + @Override + protected boolean matchesSafely(CloseIndexResponse response, Description mismatchDescription) { + if (!response.isShardsAcknowledged()) { + mismatchDescription.appendText("shardsAcknowledged is equal to ").appendValue(response.isShardsAcknowledged()); + return false; + } + if (!response.isAcknowledged()) { + mismatchDescription.appendText("acknowledged is equal to ").appendValue(response.isShardsAcknowledged()); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Successful close index response"); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulCreateIndexResponseMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulCreateIndexResponseMatcher.java new file mode 100644 index 0000000000..810c93e034 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulCreateIndexResponseMatcher.java @@ -0,0 +1,51 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.client.indices.CreateIndexResponse; + +import static java.util.Objects.requireNonNull; + +class SuccessfulCreateIndexResponseMatcher extends TypeSafeDiagnosingMatcher { + + private final String expectedIndexName; + + SuccessfulCreateIndexResponseMatcher(String expectedIndexName) { + this.expectedIndexName = requireNonNull(expectedIndexName); + } + + @Override + protected boolean matchesSafely(CreateIndexResponse response, Description mismatchDescription) { + if (!expectedIndexName.equals(response.index())) { + mismatchDescription.appendText("Index name ") + .appendValue(response.index()) + .appendText(" does not match expected index name ") + .appendValue(expectedIndexName); + return false; + } + if (!response.isShardsAcknowledged()) { + mismatchDescription.appendText("shardsAcknowledged is equal to ").appendValue(response.isShardsAcknowledged()); + return false; + } + if (!response.isAcknowledged()) { + mismatchDescription.appendText("acknowledged is equal to ").appendValue(response.isShardsAcknowledged()); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Successful create index response"); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulDeleteResponseMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulDeleteResponseMatcher.java new file mode 100644 index 0000000000..eb4b1fc064 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulDeleteResponseMatcher.java @@ -0,0 +1,39 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.delete.DeleteResponse; +import org.opensearch.rest.RestStatus; + +class SuccessfulDeleteResponseMatcher extends TypeSafeDiagnosingMatcher { + + @Override + protected boolean matchesSafely(DeleteResponse response, Description mismatchDescription) { + if (!RestStatus.OK.equals(response.status())) { + mismatchDescription.appendText("has status ").appendValue(response.status()).appendText(" which denotes failure."); + return false; + } + if (response.getShardInfo().getFailures().length != 0) { + mismatchDescription.appendText("contains ") + .appendValue(response.getShardInfo().getFailures().length) + .appendText(" shard failures"); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Successful delete response"); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulMultiGetResponseMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulMultiGetResponseMatcher.java new file mode 100644 index 0000000000..fce5d1201c --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulMultiGetResponseMatcher.java @@ -0,0 +1,39 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.get.MultiGetItemResponse; +import org.opensearch.action.get.MultiGetResponse; + +class SuccessfulMultiGetResponseMatcher extends TypeSafeDiagnosingMatcher { + + @Override + protected boolean matchesSafely(MultiGetResponse response, Description mismatchDescription) { + for (MultiGetItemResponse getItemResponse : response.getResponses()) { + if (getItemResponse.isFailed()) { + mismatchDescription.appendValue("Get an item from index: ") + .appendValue(getItemResponse.getIndex()) + .appendText(" failed: ") + .appendValue(getItemResponse.getFailure().getMessage()); + return false; + } + } + + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Successful multi get response"); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulMultiSearchResponseMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulMultiSearchResponseMatcher.java new file mode 100644 index 0000000000..e601f16e8e --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulMultiSearchResponseMatcher.java @@ -0,0 +1,35 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.search.MultiSearchResponse; + +class SuccessfulMultiSearchResponseMatcher extends TypeSafeDiagnosingMatcher { + + @Override + protected boolean matchesSafely(MultiSearchResponse response, Description mismatchDescription) { + for (MultiSearchResponse.Item itemResponse : response.getResponses()) { + if (itemResponse.isFailure()) { + mismatchDescription.appendValue("Get an item failed: ").appendValue(itemResponse.getFailureMessage()); + return false; + } + } + + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Successful multi search response"); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulOpenIndexResponseMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulOpenIndexResponseMatcher.java new file mode 100644 index 0000000000..68389979b1 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulOpenIndexResponseMatcher.java @@ -0,0 +1,36 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.admin.indices.open.OpenIndexResponse; + +class SuccessfulOpenIndexResponseMatcher extends TypeSafeDiagnosingMatcher { + + @Override + protected boolean matchesSafely(OpenIndexResponse response, Description mismatchDescription) { + if (!response.isShardsAcknowledged()) { + mismatchDescription.appendText("shardsAcknowledged is equal to ").appendValue(response.isShardsAcknowledged()); + return false; + } + if (!response.isAcknowledged()) { + mismatchDescription.appendText("acknowledged is equal to ").appendValue(response.isShardsAcknowledged()); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Successful open index response"); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulResizeResponseMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulResizeResponseMatcher.java new file mode 100644 index 0000000000..915a0f39bb --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulResizeResponseMatcher.java @@ -0,0 +1,51 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.client.indices.ResizeResponse; + +import static java.util.Objects.requireNonNull; + +class SuccessfulResizeResponseMatcher extends TypeSafeDiagnosingMatcher { + + private final String expectedIndexName; + + SuccessfulResizeResponseMatcher(String expectedIndexName) { + this.expectedIndexName = requireNonNull(expectedIndexName); + } + + @Override + protected boolean matchesSafely(ResizeResponse response, Description mismatchDescription) { + if (!expectedIndexName.equals(response.index())) { + mismatchDescription.appendText("Index name ") + .appendValue(response.index()) + .appendText(" does not match expected index name ") + .appendValue(expectedIndexName); + return false; + } + if (!response.isShardsAcknowledged()) { + mismatchDescription.appendText("shardsAcknowledged is equal to ").appendValue(response.isShardsAcknowledged()); + return false; + } + if (!response.isAcknowledged()) { + mismatchDescription.appendText("acknowledged is equal to ").appendValue(response.isShardsAcknowledged()); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Successful create index response"); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulSearchResponseMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulSearchResponseMatcher.java new file mode 100644 index 0000000000..fe2acff9b9 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulSearchResponseMatcher.java @@ -0,0 +1,37 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.search.SearchResponse; +import org.opensearch.rest.RestStatus; + +class SuccessfulSearchResponseMatcher extends TypeSafeDiagnosingMatcher { + + @Override + protected boolean matchesSafely(SearchResponse searchResponse, Description mismatchDescription) { + if (RestStatus.OK.equals(searchResponse.status()) == false) { + mismatchDescription.appendText("has status ").appendValue(searchResponse.status()).appendText(" which denotes failure."); + return false; + } + if (searchResponse.getShardFailures().length != 0) { + mismatchDescription.appendText("contains ").appendValue(searchResponse.getShardFailures().length).appendText(" shard failures"); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Successful search response"); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulUpdateResponseMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulUpdateResponseMatcher.java new file mode 100644 index 0000000000..dcff052f01 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/SuccessfulUpdateResponseMatcher.java @@ -0,0 +1,39 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.update.UpdateResponse; +import org.opensearch.rest.RestStatus; + +class SuccessfulUpdateResponseMatcher extends TypeSafeDiagnosingMatcher { + + @Override + protected boolean matchesSafely(UpdateResponse response, Description mismatchDescription) { + if (!RestStatus.OK.equals(response.status())) { + mismatchDescription.appendText("has status ").appendValue(response.status()).appendText(" which denotes failure."); + return false; + } + if (response.getShardInfo().getFailures().length != 0) { + mismatchDescription.appendText("contains ") + .appendValue(response.getShardInfo().getFailures().length) + .appendText(" shard failures"); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Successful update response"); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/UpdateResponseMatchers.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/UpdateResponseMatchers.java new file mode 100644 index 0000000000..ee01fabced --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/UpdateResponseMatchers.java @@ -0,0 +1,23 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import org.hamcrest.Matcher; + +import org.opensearch.action.update.UpdateResponse; + +public class UpdateResponseMatchers { + + private UpdateResponseMatchers() {} + + public static Matcher isSuccessfulUpdateResponse() { + return new SuccessfulUpdateResponseMatcher(); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/testplugins/AbstractRestHandler.java b/src/integrationTest/java/org/opensearch/test/framework/testplugins/AbstractRestHandler.java new file mode 100644 index 0000000000..ac93a57b2a --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/testplugins/AbstractRestHandler.java @@ -0,0 +1,62 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.testplugins; + +import org.opensearch.ExceptionsHelper; +import org.opensearch.client.node.NodeClient; +import org.opensearch.rest.RestStatus; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestChannel; +import org.opensearch.rest.RestRequest; + +import java.io.IOException; + +public class AbstractRestHandler extends BaseRestHandler { + + @Override + public String getName() { + return getClass().getSimpleName(); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + switch (request.method()) { + case GET: + return channel -> handleGet(channel, request, client); + case POST: + return channel -> handlePost(channel, request, client); + default: + throw new IllegalArgumentException(request.method() + " not supported"); + } + } + + private void notImplemented(RestChannel channel, RestRequest.Method method) { + try { + final XContentBuilder builder = channel.newBuilder(); + builder.startObject(); + builder.field("status", RestStatus.NOT_IMPLEMENTED.name()); + builder.field("message", "Method " + method + " not implemented."); + builder.endObject(); + channel.sendResponse(new BytesRestResponse(RestStatus.NOT_IMPLEMENTED, builder)); + } catch (IOException e) { + throw ExceptionsHelper.convertToOpenSearchException(e); + } + } + + public void handlePost(RestChannel channel, RestRequest request, NodeClient client) { + notImplemented(channel, request.method()); + } + + public void handleGet(RestChannel channel, RestRequest request, NodeClient client) { + notImplemented(channel, request.method()); + } +} diff --git a/src/integrationTest/resources/action_groups.yml b/src/integrationTest/resources/action_groups.yml new file mode 100644 index 0000000000..32188f69d0 --- /dev/null +++ b/src/integrationTest/resources/action_groups.yml @@ -0,0 +1,4 @@ +--- +_meta: + type: "actiongroups" + config_version: 2 diff --git a/src/integrationTest/resources/allowlist.yml b/src/integrationTest/resources/allowlist.yml new file mode 100644 index 0000000000..d1b4540d6d --- /dev/null +++ b/src/integrationTest/resources/allowlist.yml @@ -0,0 +1,4 @@ +--- +_meta: + type: "allowlist" + config_version: 2 diff --git a/src/integrationTest/resources/config.yml b/src/integrationTest/resources/config.yml new file mode 100644 index 0000000000..17aeb1881d --- /dev/null +++ b/src/integrationTest/resources/config.yml @@ -0,0 +1,22 @@ +--- +_meta: + type: "config" + config_version: 2 +config: + dynamic: + authc: + basic: + http_enabled: true + order: 0 + http_authenticator: + type: "basic" + challenge: true + config: {} + authentication_backend: + type: "internal" + config: {} + on_behalf_of: + # The decoded signing key is: This is the jwt signing key for an on behalf of token authentication backend for testing of extensions + # The decoded encryption key is: encryptionKey + signing_key: "VGhpcyBpcyB0aGUgand0IHNpZ25pbmcga2V5IGZvciBhbiBvbiBiZWhhbGYgb2YgdG9rZW4gYXV0aGVudGljYXRpb24gYmFja2VuZCBmb3IgdGVzdGluZyBvZiBleHRlbnNpb25z" + encryption_key: "ZW5jcnlwdGlvbktleQ==" diff --git a/src/integrationTest/resources/internal_users.yml b/src/integrationTest/resources/internal_users.yml new file mode 100644 index 0000000000..866a879165 --- /dev/null +++ b/src/integrationTest/resources/internal_users.yml @@ -0,0 +1,14 @@ +--- +_meta: + type: "internalusers" + config_version: 2 +new-user: + hash: "$2y$12$d2KAKcGE9qoywfu.c.hV/.pHigC7HTZFp2yJzBo8z2w.585t7XDWO" +limited-user: + hash: "$2y$12$fOJAMx0U7e7M4OObVPzm6eUTnAyN/Gtpzfv34M6PL1bfusae43a52" + opendistro_security_roles: + - "user_limited-user__limited-role" +admin: + hash: "$2y$12$53iW.RRy.uumsmU7lrlp7OUCPdxz40Z5uIJo1WcCC2GNFwEWNiTD6" + opendistro_security_roles: + - "user_admin__all_access" diff --git a/src/integrationTest/resources/log4j2-test.properties b/src/integrationTest/resources/log4j2-test.properties new file mode 100644 index 0000000000..0b865b46b3 --- /dev/null +++ b/src/integrationTest/resources/log4j2-test.properties @@ -0,0 +1,47 @@ +status = info +name = Integration test logging configuration +packages = org.opensearch.test.framework.log + +appender.console.type = Console +appender.console.name = consoleAppender +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %threadName %-5p %c{1}:%L - %m%n +appender.console.filter.prerelease.type=RegexFilter +appender.console.filter.prerelease.regex=.+\\Qis a pre-release version of OpenSearch and is not suitable for production\\E +appender.console.filter.prerelease.onMatch=DENY +appender.console.filter.prerelease.onMismatch=NEUTRAL + +appender.capturing.type = LogCapturingAppender +appender.capturing.name = logCapturingAppender + +rootLogger.level = info +rootLogger.appenderRef.stdout.ref = consoleAppender + +logger.testsecconfig.name = org.opensearch.test.framework.TestSecurityConfig +logger.testsecconfig.level = info +logger.localopensearchcluster.name=org.opensearch.test.framework.cluster.LocalOpenSearchCluster +logger.localopensearchcluster.level = info + +logger.auditlogs.name=org.opensearch.test.framework.audit +logger.auditlogs.level = info + +# Logger required by test org.opensearch.security.http.JwtAuthenticationTests +logger.httpjwtauthenticator.name = com.amazon.dlic.auth.http.jwt.HTTPJwtAuthenticator +logger.httpjwtauthenticator.level = debug +logger.backendreg.additivity = false +logger.httpjwtauthenticator.appenderRef.capturing.ref = logCapturingAppender + +#Required by tests: +# org.opensearch.security.IpBruteForceAttacksPreventionTests +# org.opensearch.security.UserBruteForceAttacksPreventionTests +logger.backendreg.name = org.opensearch.security.auth.BackendRegistry +logger.backendreg.level = debug +logger.backendreg.additivity = false +logger.backendreg.appenderRef.capturing.ref = logCapturingAppender + +#com.amazon.dlic.auth.ldap +#logger.ldap.name=com.amazon.dlic.auth.ldap.backend.LDAPAuthenticationBackend +logger.ldap.name=com.amazon.dlic.auth.ldap.backend +logger.ldap.level=TRACE +logger.backendreg.additivity = false +logger.ldap.appenderRef.capturing.ref = logCapturingAppender diff --git a/src/integrationTest/resources/nodes_dn.yml b/src/integrationTest/resources/nodes_dn.yml new file mode 100644 index 0000000000..437583b160 --- /dev/null +++ b/src/integrationTest/resources/nodes_dn.yml @@ -0,0 +1,4 @@ +--- +_meta: + type: "nodesdn" + config_version: 2 diff --git a/src/integrationTest/resources/roles.yml b/src/integrationTest/resources/roles.yml new file mode 100644 index 0000000000..02de9bf3d5 --- /dev/null +++ b/src/integrationTest/resources/roles.yml @@ -0,0 +1,19 @@ +--- +_meta: + type: "roles" + config_version: 2 +user_admin__all_access: + cluster_permissions: + - "*" + index_permissions: + - index_patterns: + - "*" + allowed_actions: + - "*" +user_limited-user__limited-role: + index_permissions: + - index_patterns: + - "user-${user.name}" + allowed_actions: + - "indices:data/read/get" + - "indices:data/read/search" diff --git a/src/integrationTest/resources/roles_mapping.yml b/src/integrationTest/resources/roles_mapping.yml new file mode 100644 index 0000000000..193f999176 --- /dev/null +++ b/src/integrationTest/resources/roles_mapping.yml @@ -0,0 +1,9 @@ +--- +_meta: + type: "rolesmapping" + config_version: 2 + +readall: + reserved: false + backend_roles: + - "readall" diff --git a/src/integrationTest/resources/security_tenants.yml b/src/integrationTest/resources/security_tenants.yml new file mode 100644 index 0000000000..93b510dd16 --- /dev/null +++ b/src/integrationTest/resources/security_tenants.yml @@ -0,0 +1,4 @@ +--- +_meta: + type: "tenants" + config_version: 2 diff --git a/src/integrationTest/resources/tenants.yml b/src/integrationTest/resources/tenants.yml new file mode 100644 index 0000000000..add18ebd54 --- /dev/null +++ b/src/integrationTest/resources/tenants.yml @@ -0,0 +1,8 @@ +--- +_meta: + type: "tenants" + config_version: 2 + +admin_tenant: + reserved: false + description: "Test tenant for admin user" diff --git a/src/integrationTest/resources/whitelist.yml b/src/integrationTest/resources/whitelist.yml new file mode 100644 index 0000000000..866ffe9eb3 --- /dev/null +++ b/src/integrationTest/resources/whitelist.yml @@ -0,0 +1,4 @@ +--- +_meta: + type: "whitelist" + config_version: 2 diff --git a/src/main/java/org/opensearch/security/configuration/DlsFlsValveImpl.java b/src/main/java/org/opensearch/security/configuration/DlsFlsValveImpl.java index 81fe9e255f..b932df8f3f 100644 --- a/src/main/java/org/opensearch/security/configuration/DlsFlsValveImpl.java +++ b/src/main/java/org/opensearch/security/configuration/DlsFlsValveImpl.java @@ -428,7 +428,10 @@ private static List mergeBuckets( private void setDlsHeaders(EvaluatedDlsFlsConfig dlsFls, ActionRequest request) { if (!dlsFls.getDlsQueriesByIndex().isEmpty()) { + final long startgetDlsQueriesByIndexMs = System.currentTimeMillis(); Map> dlsQueries = dlsFls.getDlsQueriesByIndex(); + final long endgetDlsQueriesByIndexMs = System.currentTimeMillis() - startgetDlsQueriesByIndexMs; + // log.error("$$$$ Timeto compute dls queries by index, {}ms", endgetDlsQueriesByIndexMs); if (request instanceof ClusterSearchShardsRequest && HeaderHelper.isTrustedClusterRequest(threadContext)) { threadContext.addResponseHeader( @@ -448,6 +451,7 @@ private void setDlsHeaders(EvaluatedDlsFlsConfig dlsFls, ActionRequest request) ConfigConstants.OPENDISTRO_SECURITY_DLS_QUERY_HEADER + " does not match (SG 900D)" ); } + // log.error("$$$$ {}ms found dls query header: {}", endMs, deserializedDlsQueries); } else { threadContext.putHeader( ConfigConstants.OPENDISTRO_SECURITY_DLS_QUERY_HEADER, diff --git a/src/main/java/org/opensearch/security/dlic/rest/api/InternalUsersApiAction.java b/src/main/java/org/opensearch/security/dlic/rest/api/InternalUsersApiAction.java index 532032891d..eae48455e0 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/api/InternalUsersApiAction.java +++ b/src/main/java/org/opensearch/security/dlic/rest/api/InternalUsersApiAction.java @@ -14,6 +14,7 @@ import java.io.IOException; import java.nio.file.Path; import java.util.List; +import java.util.stream.Collectors; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; diff --git a/src/main/java/org/opensearch/security/dlic/rest/api/RolesApiAction.java b/src/main/java/org/opensearch/security/dlic/rest/api/RolesApiAction.java index 65dbaac05e..bfca235a31 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/api/RolesApiAction.java +++ b/src/main/java/org/opensearch/security/dlic/rest/api/RolesApiAction.java @@ -113,5 +113,4 @@ protected boolean isReadOnly(SecurityDynamicConfiguration existingConfigurati return super.isReadOnly(existingConfiguration, name); } } - } diff --git a/src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/HTTPJwtKeyByOpenIdConnectAuthenticatorTest.java b/src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/HTTPJwtKeyByOpenIdConnectAuthenticatorTest.java index 0efba2ad7e..1f05c4f75b 100644 --- a/src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/HTTPJwtKeyByOpenIdConnectAuthenticatorTest.java +++ b/src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/HTTPJwtKeyByOpenIdConnectAuthenticatorTest.java @@ -67,7 +67,6 @@ public void basicTest() { @Test public void jwksUriTest() { - Settings settings = Settings.builder() .put("jwks_uri", mockIdpServer.getJwksUri()) .put("required_issuer", TestJwts.TEST_ISSUER) .put("required_audience", TestJwts.TEST_AUDIENCE) @@ -75,27 +74,6 @@ public void jwksUriTest() { HTTPJwtKeyByOpenIdConnectAuthenticator jwtAuth = new HTTPJwtKeyByOpenIdConnectAuthenticator(settings, null); - AuthCredentials creds = jwtAuth.extractCredentials( - new FakeRestRequest(ImmutableMap.of("Authorization", TestJwts.MC_COY_SIGNED_OCT_2), new HashMap<>()), - null - ); - - Assert.assertNotNull(creds); - Assert.assertEquals(TestJwts.MCCOY_SUBJECT, creds.getUsername()); - Assert.assertEquals(TestJwts.TEST_AUDIENCE, creds.getAttributes().get("attr.jwt.aud")); - Assert.assertEquals(0, creds.getBackendRoles().size()); - Assert.assertEquals(4, creds.getAttributes().size()); - } - - @Test - public void jwksMissingRequiredIssuerInClaimTest() { - Settings settings = Settings.builder() - .put("jwks_uri", mockIdpServer.getJwksUri()) - .put("required_issuer", TestJwts.TEST_ISSUER) - .build(); - - HTTPJwtKeyByOpenIdConnectAuthenticator jwtAuth = new HTTPJwtKeyByOpenIdConnectAuthenticator(settings, null); - AuthCredentials creds = jwtAuth.extractCredentials( new FakeRestRequest(ImmutableMap.of("Authorization", TestJwts.MC_COY_SIGNED_NO_ISSUER_OCT_1), new HashMap<>()), null diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/ActionGroupsApiTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/ActionGroupsApiTest.java index 611109e9fa..7a90b5aeb0 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/ActionGroupsApiTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/ActionGroupsApiTest.java @@ -260,13 +260,8 @@ void verifyPatchForSuperAdmin(final Header[] header, final boolean userAdminCert response = rh.executePatchRequest( ENDPOINT, "[{ \"op\": \"add\", \"path\": \"/BULKNEW1\", \"value\": {\"allowed_actions\": [\"BULKNEW1\"] } }," - + "{ \"op\": \"add\", \"path\": \"/BULKNEW2\", \"value\": {\"allowed_actions\": [\"READ_UT\"] } }]", header ); - Assert.assertEquals(HttpStatus.SC_BAD_REQUEST, response.getStatusCode()); - Assert.assertTrue(response.getBody().contains("BULKNEW1 cannot be an allowed_action of itself")); - - // PATCH hidden resource, must be not found, can be found by superadmin, but fails with no path exist error rh.sendAdminCertificate = userAdminCert; response = rh.executePatchRequest( ENDPOINT + "/INTERNAL", diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/NodesDnApiTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/NodesDnApiTest.java index 50294c3c1a..b98eeae08c 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/NodesDnApiTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/NodesDnApiTest.java @@ -203,6 +203,88 @@ public void testNodesDnApi() throws Exception { } } + + @Test + public void testNodesDnApiWithPermissions() throws Exception { + Settings settings = + Settings.builder() + .put(ConfigConstants.SECURITY_NODES_DN_DYNAMIC_CONFIG_ENABLED, true) + .put(SECURITY_RESTAPI_ADMIN_ENABLED, true) + .build(); + setupWithRestRoles(settings); + final Header restApiAdminHeader = encodeBasicHeader("rest_api_admin_user", "rest_api_admin_user"); + final Header restApiNodesDnHeader = encodeBasicHeader("rest_api_admin_nodesdn", "rest_api_admin_nodesdn"); + final Header restApiUserHeader = encodeBasicHeader("test", "test"); + //full access admin + { + rh.sendAdminCertificate = false; + response = rh.executeGetRequest( + ENDPOINT + "/nodesdn", restApiAdminHeader); + assertThat(response.getBody(), response.getStatusCode(), equalTo(HttpStatus.SC_OK)); + + response = rh.executePutRequest( + ENDPOINT + "/nodesdn/c1", "{\"nodes_dn\": [\"cn=popeye\"]}", + restApiAdminHeader); + assertThat(response.getBody(), response.getStatusCode(), equalTo(HttpStatus.SC_CREATED)); + + response = rh.executePatchRequest( + ENDPOINT + "/nodesdn/c1", + "[{ \"op\": \"add\", \"path\": \"/nodes_dn/-\", \"value\": \"bluto\" }]", + restApiAdminHeader + ); + assertThat(response.getBody(), response.getStatusCode(), equalTo(HttpStatus.SC_OK)); + + response = rh.executeDeleteRequest(ENDPOINT + "/nodesdn/c1", restApiAdminHeader); + assertThat(response.getBody(), response.getStatusCode(), equalTo(HttpStatus.SC_OK)); + } + //NodesDN only + { + rh.sendAdminCertificate = false; + response = rh.executeGetRequest(ENDPOINT + "/nodesdn", restApiNodesDnHeader); + assertThat(response.getBody(), response.getStatusCode(), equalTo(HttpStatus.SC_OK)); + + response = rh.executePutRequest( + ENDPOINT + "/nodesdn/c1", "{\"nodes_dn\": [\"cn=popeye\"]}", + restApiNodesDnHeader); + assertThat(response.getBody(), response.getStatusCode(), equalTo(HttpStatus.SC_CREATED)); + + response = rh.executePatchRequest( + ENDPOINT + "/nodesdn/c1", + "[{ \"op\": \"add\", \"path\": \"/nodes_dn/-\", \"value\": \"bluto\" }]", + restApiNodesDnHeader + ); + assertThat(response.getBody(), response.getStatusCode(), equalTo(HttpStatus.SC_OK)); + + response = rh.executeDeleteRequest(ENDPOINT + "/nodesdn/c1", restApiNodesDnHeader); + assertThat(response.getBody(), response.getStatusCode(), equalTo(HttpStatus.SC_OK)); + + response = rh.executeGetRequest(ENDPOINT + "/actiongroups", restApiNodesDnHeader); + assertThat(response.getBody(), response.getStatusCode(), equalTo(HttpStatus.SC_OK)); + } + //rest api user + { + rh.sendAdminCertificate = false; + response = rh.executeGetRequest(ENDPOINT + "/nodesdn", restApiUserHeader); + assertThat(response.getBody(), response.getStatusCode(), equalTo(HttpStatus.SC_FORBIDDEN)); + + response = rh.executePutRequest( + ENDPOINT + "/nodesdn/c1", "{\"nodes_dn\": [\"cn=popeye\"]}", + restApiUserHeader); + assertThat(response.getBody(), response.getStatusCode(), equalTo(HttpStatus.SC_FORBIDDEN)); + + response = rh.executePatchRequest( + ENDPOINT + "/nodesdn/c1", + "[{ \"op\": \"add\", \"path\": \"/nodes_dn/-\", \"value\": \"bluto\" }]", + restApiUserHeader + ); + assertThat(response.getBody(), response.getStatusCode(), equalTo(HttpStatus.SC_FORBIDDEN)); + + response = rh.executeDeleteRequest(ENDPOINT + "/nodesdn/c1", restApiUserHeader); + assertThat(response.getBody(), response.getStatusCode(), equalTo(HttpStatus.SC_FORBIDDEN)); + } + + } + @Test public void testNodesDnApiWithPermissions() throws Exception { Settings settings = Settings.builder() diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/RolesMappingApiTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/RolesMappingApiTest.java index 5274e34b48..85711a2683 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/RolesMappingApiTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/RolesMappingApiTest.java @@ -293,12 +293,7 @@ void verifyPutForSuperAdmin(final Header[] header) throws Exception { ENDPOINT + "/rolesmapping/opendistro_security_role_starfleet_captains", FileHelper.loadFile("restapi/rolesmapping_users_picard_single_wrong_datatype.json"), header - ); settings = Settings.builder().loadFromSource(response.getBody(), XContentType.JSON).build(); - Assert.assertEquals(HttpStatus.SC_BAD_REQUEST, response.getStatusCode()); - Assert.assertEquals(AbstractConfigurationValidator.ErrorType.WRONG_DATATYPE.getMessage(), settings.get("reason")); - Assert.assertTrue(settings.get("hosts").equals("Array expected")); - Assert.assertTrue(settings.get("users").equals("Array expected")); Assert.assertTrue(settings.get("backend_roles").equals("Array expected")); // Read only role mapping