diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ce9d15f79556..1fbea2ec284e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,13 +13,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Switch from `buildSrc/version.properties` to Gradle version catalog (`gradle/libs.versions.toml`) to enable dependabot to perform automated upgrades on common libs ([#16284](https://github.com/opensearch-project/OpenSearch/pull/16284)) - Increase segrep pressure checkpoint default limit to 30 ([#16577](https://github.com/opensearch-project/OpenSearch/pull/16577/files)) - Add dynamic setting allowing size > 0 requests to be cached in the request cache ([#16483](https://github.com/opensearch-project/OpenSearch/pull/16483)) +- Support installing plugin SNAPSHOTs with SNASPHOT distribution ([#16581](https://github.com/opensearch-project/OpenSearch/pull/16581)) - Make IndexStoreListener a pluggable interface ([#16583](https://github.com/opensearch-project/OpenSearch/pull/16583)) - Support for keyword fields in star-tree index ([#16233](https://github.com/opensearch-project/OpenSearch/pull/16233)) - Add a flag in QueryShardContext to differentiate inner hit query ([#16600](https://github.com/opensearch-project/OpenSearch/pull/16600)) - Add vertical scaling and SoftReference for snapshot repository data cache ([#16489](https://github.com/opensearch-project/OpenSearch/pull/16489)) +- Support prefix list for remote repository attributes([#16271](https://github.com/opensearch-project/OpenSearch/pull/16271)) - Add new configuration setting `synonym_analyzer`, to the `synonym` and `synonym_graph` filters, enabling the specification of a custom analyzer for reading the synonym file ([#16488](https://github.com/opensearch-project/OpenSearch/pull/16488)). ### Dependencies +- Bump `com.google.cloud:google-cloud-core-http` from 2.23.0 to 2.47.0 ([#16504](https://github.com/opensearch-project/OpenSearch/pull/16504)) +- Bump `google-auth-library-oauth2-http` from 1.7.0 to 1.29.0 in /plugins/repository-gcs ([#16520](https://github.com/opensearch-project/OpenSearch/pull/16520)) - Bump `com.azure:azure-storage-common` from 12.25.1 to 12.27.1 ([#16521](https://github.com/opensearch-project/OpenSearch/pull/16521)) - Bump `com.google.apis:google-api-services-compute` from v1-rev20240407-2.0.0 to v1-rev20241021-2.0.0 ([#16502](https://github.com/opensearch-project/OpenSearch/pull/16502), [#16548](https://github.com/opensearch-project/OpenSearch/pull/16548)) - Bump `com.azure:azure-storage-blob` from 12.23.0 to 12.28.1 ([#16501](https://github.com/opensearch-project/OpenSearch/pull/16501)) @@ -28,6 +32,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.nimbusds:nimbus-jose-jwt` from 9.41.1 to 9.46 ([#16611](https://github.com/opensearch-project/OpenSearch/pull/16611)) - Bump `lycheeverse/lychee-action` from 2.0.2 to 2.1.0 ([#16610](https://github.com/opensearch-project/OpenSearch/pull/16610)) - Bump `me.champeau.gradle.japicmp` from 0.4.4 to 0.4.5 ([#16614](https://github.com/opensearch-project/OpenSearch/pull/16614)) +- Bump `mockito` from 5.14.1 to 5.14.2, `objenesis` from 3.2 to 3.3 and `bytebuddy` from 1.15.4 to 1.15.10 ([#16655](https://github.com/opensearch-project/OpenSearch/pull/16655)) +- Bump `Netty` from 4.1.114.Final to 4.1.115.Final ([#16661](https://github.com/opensearch-project/OpenSearch/pull/16661)) +- Bump `org.xerial.snappy:snappy-java` from 1.1.10.6 to 1.1.10.7 ([#16665](https://github.com/opensearch-project/OpenSearch/pull/16665)) ### Changed @@ -43,6 +50,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Support retrieving doc values of unsigned long field ([#16543](https://github.com/opensearch-project/OpenSearch/pull/16543)) - Fix rollover alias supports restored searchable snapshot index([#16483](https://github.com/opensearch-project/OpenSearch/pull/16483)) - Fix permissions error on scripted query against remote snapshot ([#16544](https://github.com/opensearch-project/OpenSearch/pull/16544)) +- Fix `doc_values` only (`index:false`) IP field searching for masks ([#16628](https://github.com/opensearch-project/OpenSearch/pull/16628)) - Fix max request cache size settings not working properly with pluggable caching ([#16636](https://github.com/opensearch-project/OpenSearch/pull/16636)) ### Security diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 03cd189aa911e..b40f5d9b3f21a 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -2,9 +2,7 @@ - [Getting Started](#getting-started) - [Git Clone OpenSearch Repo](#git-clone-opensearch-repo) - [Install Prerequisites](#install-prerequisites) - - [JDK 11](#jdk-11) - - [JDK 14](#jdk-14) - - [JDK 17](#jdk-17) + - [JDK](#jdk) - [Custom Runtime JDK](#custom-runtime-jdk) - [Windows](#windows) - [Docker](#docker) @@ -76,35 +74,24 @@ Fork [opensearch-project/OpenSearch](https://github.com/opensearch-project/OpenS ### Install Prerequisites -#### JDK 11 +#### JDK -OpenSearch builds using Java 11 at a minimum, using the Adoptium distribution. This means you must have a JDK 11 installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK 11 installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-11`. This is configured in [buildSrc/build.gradle](buildSrc/build.gradle) and [distribution/tools/java-version-checker/build.gradle](distribution/tools/java-version-checker/build.gradle). +OpenSearch recommends building with the [Temurin/Adoptium](https://adoptium.net/temurin/releases/) distribution. JDK 11 is the minimum supported, and JDK-23 is the newest supported. You must have a supported JDK installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-21`. -``` -allprojects { - targetCompatibility = JavaVersion.VERSION_11 - sourceCompatibility = JavaVersion.VERSION_11 -} -``` +Download Java 11 from [here](https://adoptium.net/releases.html?variant=openjdk11). -``` -sourceCompatibility = JavaVersion.VERSION_11 -targetCompatibility = JavaVersion.VERSION_11 -``` -Download Java 11 from [here](https://adoptium.net/releases.html?variant=openjdk11). +In addition, certain backward compatibility tests check out and compile the previous major version of OpenSearch, and therefore require installing [JDK 11](https://adoptium.net/temurin/releases/?version=11) and [JDK 17](https://adoptium.net/temurin/releases/?version=17) and setting the `JAVA11_HOME` and `JAVA17_HOME` environment variables. More to that, since 8.10 release, Gradle has deprecated the usage of the any JDKs below JDK-16. For smooth development experience, the recommendation is to install at least [JDK 17](https://adoptium.net/temurin/releases/?version=17) or [JDK 21](https://adoptium.net/temurin/releases/?version=21). If you still want to build with JDK-11 only, please add `-Dorg.gradle.warning.mode=none` when invoking any Gradle build task from command line, for example: -#### JDK 14 - -To run the full suite of tests, download and install [JDK 14](https://jdk.java.net/archive/) and set `JAVA11_HOME`, and `JAVA14_HOME`. They are required by the [backwards compatibility test](./TESTING.md#testing-backwards-compatibility). - -#### JDK 17 +``` +./gradlew check -Dorg.gradle.warning.mode=none +``` -By default, the test tasks use bundled JDK runtime, configured in [buildSrc/version.properties](buildSrc/version.properties), and set to JDK 17 (LTS). +By default, the test tasks use bundled JDK runtime, configured in version catalog [gradle/libs.versions.toml](gradle/libs.versions.toml), and set to JDK 23 (non-LTS). ``` bundled_jdk_vendor = adoptium -bundled_jdk = 17.0.2+8 +bundled_jdk = 23.0.1+11 ``` #### Custom Runtime JDK diff --git a/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/InstallPluginCommand.java b/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/InstallPluginCommand.java index 838d6e22a37bd..511d6974085aa 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/InstallPluginCommand.java +++ b/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/InstallPluginCommand.java @@ -137,8 +137,6 @@ */ class InstallPluginCommand extends EnvironmentAwareCommand { - private static final String PROPERTY_STAGING_ID = "opensearch.plugins.staging"; - // exit codes for install /** A plugin with the same name is already installed. */ static final int PLUGIN_EXISTS = 1; @@ -307,14 +305,7 @@ void execute(Terminal terminal, List pluginIds, boolean isBatch, Environ private Path download(Terminal terminal, String pluginId, Path tmpDir, boolean isBatch) throws Exception { if (OFFICIAL_PLUGINS.contains(pluginId)) { - final String url = getOpenSearchUrl( - terminal, - getStagingHash(), - Version.CURRENT, - isSnapshot(), - pluginId, - Platforms.PLATFORM_NAME - ); + final String url = getOpenSearchUrl(terminal, Version.CURRENT, isSnapshot(), pluginId, Platforms.PLATFORM_NAME); terminal.println("-> Downloading " + pluginId + " from opensearch"); return downloadAndValidate(terminal, url, tmpDir, true, isBatch); } @@ -341,11 +332,6 @@ private Path download(Terminal terminal, String pluginId, Path tmpDir, boolean i return downloadZip(terminal, pluginId, tmpDir, isBatch); } - // pkg private so tests can override - String getStagingHash() { - return System.getProperty(PROPERTY_STAGING_ID); - } - boolean isSnapshot() { return Build.CURRENT.isSnapshot(); } @@ -353,26 +339,18 @@ boolean isSnapshot() { /** Returns the url for an official opensearch plugin. */ private String getOpenSearchUrl( final Terminal terminal, - final String stagingHash, final Version version, final boolean isSnapshot, final String pluginId, final String platform ) throws IOException, UserException { final String baseUrl; - if (isSnapshot && stagingHash == null) { - throw new UserException( - ExitCodes.CONFIG, - "attempted to install release build of official plugin on snapshot build of OpenSearch" - ); - } - if (stagingHash != null) { + if (isSnapshot == true) { baseUrl = String.format( Locale.ROOT, - "https://artifacts.opensearch.org/snapshots/plugins/%s/%s-%s", + "https://artifacts.opensearch.org/snapshots/plugins/%s/%s", pluginId, - version, - stagingHash + Build.CURRENT.getQualifiedVersion() ); } else { baseUrl = String.format( diff --git a/distribution/tools/plugin-cli/src/test/java/org/opensearch/plugins/InstallPluginCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/opensearch/plugins/InstallPluginCommandTests.java index c264788df20e8..e6c1070f36066 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/opensearch/plugins/InstallPluginCommandTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/opensearch/plugins/InstallPluginCommandTests.java @@ -990,7 +990,6 @@ void assertInstallPluginFromUrl( final String pluginId, final String name, final String url, - final String stagingHash, final boolean isSnapshot, final String shaExtension, final Function shaCalculator, @@ -1065,11 +1064,6 @@ boolean urlExists(Terminal terminal, String urlString) throws IOException { return urlString.equals(url); } - @Override - String getStagingHash() { - return stagingHash; - } - @Override boolean isSnapshot() { return isSnapshot; @@ -1084,19 +1078,13 @@ void jarHellCheck(PluginInfo candidateInfo, Path candidate, Path pluginsDir, Pat assertPlugin(name, pluginDir, env.v2()); } - public void assertInstallPluginFromUrl( - final String pluginId, - final String name, - final String url, - final String stagingHash, - boolean isSnapshot - ) throws Exception { + public void assertInstallPluginFromUrl(final String pluginId, final String name, final String url, boolean isSnapshot) + throws Exception { final MessageDigest digest = MessageDigest.getInstance("SHA-512"); assertInstallPluginFromUrl( pluginId, name, url, - stagingHash, isSnapshot, ".sha512", checksumAndFilename(digest, url), @@ -1111,42 +1099,17 @@ public void testOfficialPlugin() throws Exception { + "/analysis-icu-" + Build.CURRENT.getQualifiedVersion() + ".zip"; - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, false); + assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, false); } public void testOfficialPluginSnapshot() throws Exception { String url = String.format( Locale.ROOT, - "https://artifacts.opensearch.org/snapshots/plugins/analysis-icu/%s-abc123/analysis-icu-%s.zip", - Version.CURRENT, - Build.CURRENT.getQualifiedVersion() - ); - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123", true); - } - - public void testInstallReleaseBuildOfPluginOnSnapshotBuild() { - String url = String.format( - Locale.ROOT, - "https://artifacts.opensearch.org/snapshots/plugins/analysis-icu/%s-abc123/analysis-icu-%s.zip", + "https://artifacts.opensearch.org/snapshots/plugins/analysis-icu/%s-SNAPSHOT/analysis-icu-%s.zip", Version.CURRENT, Build.CURRENT.getQualifiedVersion() ); - // attemping to install a release build of a plugin (no staging ID) on a snapshot build should throw a user exception - final UserException e = expectThrows( - UserException.class, - () -> assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, true) - ); - assertThat(e.exitCode, equalTo(ExitCodes.CONFIG)); - assertThat(e, hasToString(containsString("attempted to install release build of official plugin on snapshot build of OpenSearch"))); - } - - public void testOfficialPluginStaging() throws Exception { - String url = "https://artifacts.opensearch.org/snapshots/plugins/analysis-icu/" - + Version.CURRENT - + "-abc123/analysis-icu-" - + Build.CURRENT.getQualifiedVersion() - + ".zip"; - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123", false); + assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, true); } public void testOfficialPlatformPlugin() throws Exception { @@ -1157,62 +1120,30 @@ public void testOfficialPlatformPlugin() throws Exception { + "-" + Build.CURRENT.getQualifiedVersion() + ".zip"; - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, false); - } - - public void testOfficialPlatformPluginSnapshot() throws Exception { - String url = String.format( - Locale.ROOT, - "https://artifacts.opensearch.org/snapshots/plugins/analysis-icu/%s-abc123/analysis-icu-%s-%s.zip", - Version.CURRENT, - Platforms.PLATFORM_NAME, - Build.CURRENT.getQualifiedVersion() - ); - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123", true); - } - - public void testOfficialPlatformPluginStaging() throws Exception { - String url = "https://artifacts.opensearch.org/snapshots/plugins/analysis-icu/" - + Version.CURRENT - + "-abc123/analysis-icu-" - + Platforms.PLATFORM_NAME - + "-" - + Build.CURRENT.getQualifiedVersion() - + ".zip"; - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123", false); + assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, false); } public void testMavenPlugin() throws Exception { String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip"; - assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, false); + assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, false); } public void testMavenPlatformPlugin() throws Exception { String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-" + Platforms.PLATFORM_NAME + "-1.0.0.zip"; - assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, false); + assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, false); } public void testMavenSha1Backcompat() throws Exception { String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip"; MessageDigest digest = MessageDigest.getInstance("SHA-1"); - assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, false, ".sha1", checksum(digest), null, (b, p) -> null); + assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, false, ".sha1", checksum(digest), null, (b, p) -> null); assertTrue(terminal.getOutput(), terminal.getOutput().contains("sha512 not found, falling back to sha1")); } public void testMavenChecksumWithoutFilename() throws Exception { String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip"; MessageDigest digest = MessageDigest.getInstance("SHA-512"); - assertInstallPluginFromUrl( - "mygroup:myplugin:1.0.0", - "myplugin", - url, - null, - false, - ".sha512", - checksum(digest), - null, - (b, p) -> null - ); + assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, false, ".sha512", checksum(digest), null, (b, p) -> null); } public void testOfficialChecksumWithoutFilename() throws Exception { @@ -1224,17 +1155,7 @@ public void testOfficialChecksumWithoutFilename() throws Exception { MessageDigest digest = MessageDigest.getInstance("SHA-512"); UserException e = expectThrows( UserException.class, - () -> assertInstallPluginFromUrl( - "analysis-icu", - "analysis-icu", - url, - null, - false, - ".sha512", - checksum(digest), - null, - (b, p) -> null - ) + () -> assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, false, ".sha512", checksum(digest), null, (b, p) -> null) ); assertEquals(ExitCodes.IO_ERROR, e.exitCode); assertThat(e.getMessage(), startsWith("Invalid checksum file")); @@ -1249,17 +1170,7 @@ public void testOfficialShaMissing() throws Exception { MessageDigest digest = MessageDigest.getInstance("SHA-1"); UserException e = expectThrows( UserException.class, - () -> assertInstallPluginFromUrl( - "analysis-icu", - "analysis-icu", - url, - null, - false, - ".sha1", - checksum(digest), - null, - (b, p) -> null - ) + () -> assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, false, ".sha1", checksum(digest), null, (b, p) -> null) ); assertEquals(ExitCodes.IO_ERROR, e.exitCode); assertEquals("Plugin checksum missing: " + url + ".sha512", e.getMessage()); @@ -1269,17 +1180,7 @@ public void testMavenShaMissing() throws Exception { String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip"; UserException e = expectThrows( UserException.class, - () -> assertInstallPluginFromUrl( - "mygroup:myplugin:1.0.0", - "myplugin", - url, - null, - false, - ".dne", - bytes -> null, - null, - (b, p) -> null - ) + () -> assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, false, ".dne", bytes -> null, null, (b, p) -> null) ); assertEquals(ExitCodes.IO_ERROR, e.exitCode); assertEquals("Plugin checksum missing: " + url + ".sha1", e.getMessage()); @@ -1294,17 +1195,7 @@ public void testInvalidShaFileMissingFilename() throws Exception { MessageDigest digest = MessageDigest.getInstance("SHA-512"); UserException e = expectThrows( UserException.class, - () -> assertInstallPluginFromUrl( - "analysis-icu", - "analysis-icu", - url, - null, - false, - ".sha512", - checksum(digest), - null, - (b, p) -> null - ) + () -> assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, false, ".sha512", checksum(digest), null, (b, p) -> null) ); assertEquals(ExitCodes.IO_ERROR, e.exitCode); assertTrue(e.getMessage(), e.getMessage().startsWith("Invalid checksum file")); @@ -1323,7 +1214,6 @@ public void testInvalidShaFileMismatchFilename() throws Exception { "analysis-icu", "analysis-icu", url, - null, false, ".sha512", checksumAndString(digest, " repository-s3-" + Build.CURRENT.getQualifiedVersion() + ".zip"), @@ -1348,7 +1238,6 @@ public void testInvalidShaFileContainingExtraLine() throws Exception { "analysis-icu", "analysis-icu", url, - null, false, ".sha512", checksumAndString(digest, " analysis-icu-" + Build.CURRENT.getQualifiedVersion() + ".zip\nfoobar"), @@ -1372,7 +1261,6 @@ public void testSha512Mismatch() throws Exception { "analysis-icu", "analysis-icu", url, - null, false, ".sha512", bytes -> "foobar analysis-icu-" + Build.CURRENT.getQualifiedVersion() + ".zip", @@ -1392,7 +1280,6 @@ public void testSha1Mismatch() throws Exception { "mygroup:myplugin:1.0.0", "myplugin", url, - null, false, ".sha1", bytes -> "foobar", @@ -1426,17 +1313,7 @@ public void testPublicKeyIdMismatchToExpectedPublicKeyId() throws Exception { final String expectedID = Long.toHexString(verifyingKey.getKeyID()).toUpperCase(Locale.ROOT); final IllegalStateException e = expectThrows( IllegalStateException.class, - () -> assertInstallPluginFromUrl( - icu, - icu, - url, - null, - false, - ".sha512", - checksumAndFilename(digest, url), - verifyingKey, - signature - ) + () -> assertInstallPluginFromUrl(icu, icu, url, false, ".sha512", checksumAndFilename(digest, url), verifyingKey, signature) ); assertThat(e, hasToString(containsString("key id [" + actualID + "] does not match expected key id [" + expectedID + "]"))); } @@ -1463,17 +1340,7 @@ public void testFailedSignatureVerification() throws Exception { }; final IllegalStateException e = expectThrows( IllegalStateException.class, - () -> assertInstallPluginFromUrl( - icu, - icu, - url, - null, - false, - ".sha512", - checksumAndFilename(digest, url), - newSecretKey(), - signature - ) + () -> assertInstallPluginFromUrl(icu, icu, url, false, ".sha512", checksumAndFilename(digest, url), newSecretKey(), signature) ); assertThat(e, hasToString(equalTo("java.lang.IllegalStateException: signature verification for [" + url + "] failed"))); } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 8ff3f6e45397d..55e338efc1cca 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -24,6 +24,7 @@ guava = "32.1.1-jre" protobuf = "3.25.5" jakarta_annotation = "1.3.5" google_http_client = "1.44.1" +google_auth = "1.29.0" tdigest = "3.3" hdrhistogram = "2.2.2" grpc = "1.68.0" @@ -31,7 +32,7 @@ grpc = "1.68.0" # when updating the JNA version, also update the version in buildSrc/build.gradle jna = "5.13.0" -netty = "4.1.114.Final" +netty = "4.1.115.Final" joda = "2.12.7" # project reactor @@ -61,9 +62,9 @@ bouncycastle="1.78" randomizedrunner = "2.7.1" junit = "4.13.2" hamcrest = "2.1" -mockito = "5.14.1" -objenesis = "3.2" -bytebuddy = "1.15.4" +mockito = "5.14.2" +objenesis = "3.3" +bytebuddy = "1.15.10" # benchmark dependencies jmh = "1.35" diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle index a8a165df637a2..cdaf8350055f0 100644 --- a/modules/transport-netty4/build.gradle +++ b/modules/transport-netty4/build.gradle @@ -240,6 +240,9 @@ thirdPartyAudit { 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField', 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField', 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueConsumerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerLimitField', 'io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess', 'io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess', 'io.netty.util.internal.shaded.org.jctools.util.UnsafeLongArrayAccess', diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 deleted file mode 100644 index 8cb83fc367d78..0000000000000 --- a/modules/transport-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f1d77d15c0b781cd9395a2a956262766fd0c7602 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.115.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..825b91b71601d --- /dev/null +++ b/modules/transport-netty4/licenses/netty-buffer-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d5daf1030e5c36d198caf7562da2441a97ec0df6 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 deleted file mode 100644 index 1be26fee34d46..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5a49dfa2828d64bf756f670e63259115332744cf \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.115.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..e39392f923f7b --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d326bf3a4c785b272da3db6941779a1bd5448378 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 deleted file mode 100644 index e683773245716..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fbce5a53884275662e68aaad70f88bf7e5d04164 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.115.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..ce4bdc323b2a0 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-http-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +80f0dece29a2c0269217e8dd1b6db6ff9710781f \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 deleted file mode 100644 index 2c18924e33c62..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -19ae07fdf99142a70338f8cda70a3d2edbc8e80a \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..f718fc9bf1622 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +0bc474c27c96e3a309da73160fbcfe0bd3aa85bc \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 deleted file mode 100644 index 74ce939dc6190..0000000000000 --- a/modules/transport-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -862712e292b162c8ccaa7847a6a54df8178f77e5 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-common-4.1.115.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..2ff84dc14147d --- /dev/null +++ b/modules/transport-netty4/licenses/netty-common-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +9da10a9f72e3f87e181d91b525174007a6fc4f11 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 deleted file mode 100644 index c431976b6fbd2..0000000000000 --- a/modules/transport-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e56fbde4b9aa628eed15a5dbfbeb97877db88146 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.115.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..5c58e02209691 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-handler-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d54dbf68b9d88a98240107758c6b63da5e46e23a \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 deleted file mode 100644 index e8080a5b2acb1..0000000000000 --- a/modules/transport-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -10b23784b23d6a948930f52ba82874f1291b5873 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.115.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..b1d9f1f679966 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-resolver-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +e33b4d476c03975957f5d8d0319d592bf2bc5e96 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 deleted file mode 100644 index fb2d518789a18..0000000000000 --- a/modules/transport-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e0225a575f487904be8517092cbd74e01913533c \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.115.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..aeeda9c282b30 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-transport-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +39cef77c1a25908ac1abf4960c2e789f0bf70ff9 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 deleted file mode 100644 index a80b9e51be74b..0000000000000 --- a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d1171bb99411f282068f49d780cedf8c9adeabfd \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..41996121685f0 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +dc96c67d06cd6b5eb677f2728f27bf2e3d9a7284 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 deleted file mode 100644 index 04a48547adb05..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -35798729ba06670fb4fcd02db98d9577e363992d \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.115.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..fc735d97148ef --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-dns-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d39b9866939cbbb8ae3a1af5c1df5ddf93656d47 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 deleted file mode 100644 index 2c18924e33c62..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -19ae07fdf99142a70338f8cda70a3d2edbc8e80a \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..f718fc9bf1622 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +0bc474c27c96e3a309da73160fbcfe0bd3aa85bc \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.114.Final.jar.sha1 deleted file mode 100644 index acbab117e4c15..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-socks-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -42b1159cac3d196f6bdbd528e29f0fab9dbaae06 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.115.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..8a617d4d857cf --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-socks-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +955faadc8f5b1ca41881398043b62ce1a245800c \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.114.Final.jar.sha1 deleted file mode 100644 index 891ed0a444dc2..0000000000000 --- a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a01071edffb4812009312b461ce5f160cdec9b75 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.115.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..13156914a5fd6 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +5f61f44704a09b373167f1c0b1854bdcd880a8cb \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 deleted file mode 100644 index 7df7a05cd7345..0000000000000 --- a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2fed36ff50059ded641fa5064963c4b4313512f3 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.115.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..e443fa3fa383f --- /dev/null +++ b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +33b0636c6fe36c7f41d9da204aa1ad94ff244ac7 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 deleted file mode 100644 index a80b9e51be74b..0000000000000 --- a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d1171bb99411f282068f49d780cedf8c9adeabfd \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..41996121685f0 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +dc96c67d06cd6b5eb677f2728f27bf2e3d9a7284 \ No newline at end of file diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index b90bcc7f822d1..97ae88aac5485 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -47,10 +47,6 @@ opensearchplugin { classname 'org.opensearch.repositories.gcs.GoogleCloudStoragePlugin' } -versions << [ - 'google_auth': '1.7.0' -] - dependencies { api 'com.google.api:api-common:1.8.1' api 'com.google.api:gax:2.35.0' @@ -67,7 +63,7 @@ dependencies { api "com.google.auth:google-auth-library-oauth2-http:${versions.google_auth}" api 'com.google.cloud:google-cloud-core:2.30.0' - api 'com.google.cloud:google-cloud-core-http:2.23.0' + api 'com.google.cloud:google-cloud-core-http:2.47.0' api 'com.google.cloud:google-cloud-storage:1.113.1' api 'com.google.code.gson:gson:2.11.0' @@ -148,7 +144,6 @@ thirdPartyAudit { 'com.google.appengine.api.urlfetch.HTTPResponse', 'com.google.appengine.api.urlfetch.URLFetchService', 'com.google.appengine.api.urlfetch.URLFetchServiceFactory', - 'com.google.auth.oauth2.GdchCredentials', 'com.google.protobuf.util.JsonFormat', 'com.google.protobuf.util.JsonFormat$Parser', 'com.google.protobuf.util.JsonFormat$Printer', @@ -304,6 +299,13 @@ testClusters { all testClustersConfiguration } +/** + * Used for testing getting credentials from GCE + */ +test { + environment 'NO_GCE_CHECK', 'true' +} + /* * We only use a small amount of data in these tests, which means that the resumable upload path is not tested. We add * an additional test that forces the large blob threshold to be small to exercise the resumable upload path. diff --git a/plugins/repository-gcs/licenses/google-auth-library-credentials-1.29.0.jar.sha1 b/plugins/repository-gcs/licenses/google-auth-library-credentials-1.29.0.jar.sha1 new file mode 100644 index 0000000000000..e2f931a1e876f --- /dev/null +++ b/plugins/repository-gcs/licenses/google-auth-library-credentials-1.29.0.jar.sha1 @@ -0,0 +1 @@ +19af4907301816d9328c1eb1fcc6dd05c8a0b544 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-auth-library-credentials-1.7.0.jar.sha1 b/plugins/repository-gcs/licenses/google-auth-library-credentials-1.7.0.jar.sha1 deleted file mode 100644 index f2e9a4f7283bf..0000000000000 --- a/plugins/repository-gcs/licenses/google-auth-library-credentials-1.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b29af5a9ea94e9e7f86bded11e39f5afda5b17e8 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-1.29.0.jar.sha1 b/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-1.29.0.jar.sha1 new file mode 100644 index 0000000000000..98d0d1beda43d --- /dev/null +++ b/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-1.29.0.jar.sha1 @@ -0,0 +1 @@ +2a42aead6cdc5d2cd22cdda1b9d7922e6135240f \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-1.7.0.jar.sha1 b/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-1.7.0.jar.sha1 deleted file mode 100644 index 738645d6b8c7b..0000000000000 --- a/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-1.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -985d183303dbd4b7ceb348056e41e59677f6f74f \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-cloud-core-http-2.23.0.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-core-http-2.23.0.jar.sha1 deleted file mode 100644 index 9db3cbcbec35b..0000000000000 --- a/plugins/repository-gcs/licenses/google-cloud-core-http-2.23.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9913d0806fcfbfbc4a775f29865126ed8465464b \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-cloud-core-http-2.47.0.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-core-http-2.47.0.jar.sha1 new file mode 100644 index 0000000000000..224893caeaafb --- /dev/null +++ b/plugins/repository-gcs/licenses/google-cloud-core-http-2.47.0.jar.sha1 @@ -0,0 +1 @@ +bfc8c587e8f2f1f1158cf36b0e515ef84f9e0a95 \ No newline at end of file diff --git a/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageServiceTests.java b/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageServiceTests.java index 58e412684ed5a..b620f212df413 100644 --- a/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageServiceTests.java +++ b/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageServiceTests.java @@ -242,7 +242,7 @@ public void testApplicationDefaultCredentialsWhenNoSettingProvided() throws Exce Exception exception = assertThrows(IOException.class, GoogleCredentials::getApplicationDefault); assertNotNull(storageOptions); assertNull(storageOptions.getCredentials()); - MatcherAssert.assertThat(exception.getMessage(), containsString("The Application Default Credentials are not available")); + MatcherAssert.assertThat(exception.getMessage(), containsString("Your default credentials were not found")); } /** @@ -254,7 +254,7 @@ public void testDefaultCredentialsThrowsExceptionWithoutGCStorageService() { GoogleCredentials credentials = googleApplicationDefaultCredentials.get(); assertNull(credentials); Exception exception = assertThrows(IOException.class, GoogleCredentials::getApplicationDefault); - MatcherAssert.assertThat(exception.getMessage(), containsString("The Application Default Credentials are not available")); + MatcherAssert.assertThat(exception.getMessage(), containsString("Your default credentials were not found")); } /** diff --git a/plugins/repository-hdfs/licenses/netty-all-4.1.114.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.1.114.Final.jar.sha1 deleted file mode 100644 index 32022e1b2eaa4..0000000000000 --- a/plugins/repository-hdfs/licenses/netty-all-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -08134b298d48eec5ddf2e3674a978b52e4667304 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/netty-all-4.1.115.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..4491099ace714 --- /dev/null +++ b/plugins/repository-hdfs/licenses/netty-all-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +1d65f327e8406f80f744060e10135dd5f61a369a \ No newline at end of file diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 22aa151c92003..398611a016ed2 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -583,6 +583,9 @@ thirdPartyAudit { 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField', 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField', 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueConsumerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerLimitField', 'io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess', 'io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess', 'io.netty.util.internal.shaded.org.jctools.util.UnsafeLongArrayAccess', diff --git a/plugins/repository-s3/licenses/netty-buffer-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-buffer-4.1.114.Final.jar.sha1 deleted file mode 100644 index 8cb83fc367d78..0000000000000 --- a/plugins/repository-s3/licenses/netty-buffer-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f1d77d15c0b781cd9395a2a956262766fd0c7602 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-buffer-4.1.115.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-buffer-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..825b91b71601d --- /dev/null +++ b/plugins/repository-s3/licenses/netty-buffer-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d5daf1030e5c36d198caf7562da2441a97ec0df6 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-4.1.114.Final.jar.sha1 deleted file mode 100644 index 1be26fee34d46..0000000000000 --- a/plugins/repository-s3/licenses/netty-codec-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5a49dfa2828d64bf756f670e63259115332744cf \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-4.1.115.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..e39392f923f7b --- /dev/null +++ b/plugins/repository-s3/licenses/netty-codec-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d326bf3a4c785b272da3db6941779a1bd5448378 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http-4.1.114.Final.jar.sha1 deleted file mode 100644 index e683773245716..0000000000000 --- a/plugins/repository-s3/licenses/netty-codec-http-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fbce5a53884275662e68aaad70f88bf7e5d04164 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http-4.1.115.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..ce4bdc323b2a0 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-codec-http-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +80f0dece29a2c0269217e8dd1b6db6ff9710781f \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 deleted file mode 100644 index 2c18924e33c62..0000000000000 --- a/plugins/repository-s3/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -19ae07fdf99142a70338f8cda70a3d2edbc8e80a \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..f718fc9bf1622 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +0bc474c27c96e3a309da73160fbcfe0bd3aa85bc \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-common-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-common-4.1.114.Final.jar.sha1 deleted file mode 100644 index 74ce939dc6190..0000000000000 --- a/plugins/repository-s3/licenses/netty-common-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -862712e292b162c8ccaa7847a6a54df8178f77e5 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-common-4.1.115.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-common-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..2ff84dc14147d --- /dev/null +++ b/plugins/repository-s3/licenses/netty-common-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +9da10a9f72e3f87e181d91b525174007a6fc4f11 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-handler-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-handler-4.1.114.Final.jar.sha1 deleted file mode 100644 index c431976b6fbd2..0000000000000 --- a/plugins/repository-s3/licenses/netty-handler-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e56fbde4b9aa628eed15a5dbfbeb97877db88146 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-handler-4.1.115.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-handler-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..5c58e02209691 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-handler-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d54dbf68b9d88a98240107758c6b63da5e46e23a \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-resolver-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-resolver-4.1.114.Final.jar.sha1 deleted file mode 100644 index e8080a5b2acb1..0000000000000 --- a/plugins/repository-s3/licenses/netty-resolver-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -10b23784b23d6a948930f52ba82874f1291b5873 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-resolver-4.1.115.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-resolver-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..b1d9f1f679966 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-resolver-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +e33b4d476c03975957f5d8d0319d592bf2bc5e96 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-4.1.114.Final.jar.sha1 deleted file mode 100644 index fb2d518789a18..0000000000000 --- a/plugins/repository-s3/licenses/netty-transport-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e0225a575f487904be8517092cbd74e01913533c \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-4.1.115.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..aeeda9c282b30 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-transport-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +39cef77c1a25908ac1abf4960c2e789f0bf70ff9 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.114.Final.jar.sha1 deleted file mode 100644 index 7120cd93e5c0d..0000000000000 --- a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f442c794e6fe89e6974f058bf393353e01fb927d \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.115.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..aea007d98763e --- /dev/null +++ b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +11fea00408ecbd8b8d1f0698d708e37db4a01841 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 deleted file mode 100644 index a80b9e51be74b..0000000000000 --- a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d1171bb99411f282068f49d780cedf8c9adeabfd \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..41996121685f0 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +dc96c67d06cd6b5eb677f2728f27bf2e3d9a7284 \ No newline at end of file diff --git a/plugins/transport-nio/build.gradle b/plugins/transport-nio/build.gradle index c0f0150378434..7132c97864238 100644 --- a/plugins/transport-nio/build.gradle +++ b/plugins/transport-nio/build.gradle @@ -175,6 +175,9 @@ thirdPartyAudit { 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField', 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField', 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueConsumerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerLimitField', 'io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess', 'io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess', 'io.netty.util.internal.shaded.org.jctools.util.UnsafeLongArrayAccess', diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.114.Final.jar.sha1 deleted file mode 100644 index 8cb83fc367d78..0000000000000 --- a/plugins/transport-nio/licenses/netty-buffer-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f1d77d15c0b781cd9395a2a956262766fd0c7602 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.115.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..825b91b71601d --- /dev/null +++ b/plugins/transport-nio/licenses/netty-buffer-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d5daf1030e5c36d198caf7562da2441a97ec0df6 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.114.Final.jar.sha1 deleted file mode 100644 index 1be26fee34d46..0000000000000 --- a/plugins/transport-nio/licenses/netty-codec-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5a49dfa2828d64bf756f670e63259115332744cf \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.115.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..e39392f923f7b --- /dev/null +++ b/plugins/transport-nio/licenses/netty-codec-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d326bf3a4c785b272da3db6941779a1bd5448378 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.114.Final.jar.sha1 deleted file mode 100644 index e683773245716..0000000000000 --- a/plugins/transport-nio/licenses/netty-codec-http-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fbce5a53884275662e68aaad70f88bf7e5d04164 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.115.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..ce4bdc323b2a0 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-codec-http-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +80f0dece29a2c0269217e8dd1b6db6ff9710781f \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-common-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.114.Final.jar.sha1 deleted file mode 100644 index 74ce939dc6190..0000000000000 --- a/plugins/transport-nio/licenses/netty-common-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -862712e292b162c8ccaa7847a6a54df8178f77e5 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-common-4.1.115.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..2ff84dc14147d --- /dev/null +++ b/plugins/transport-nio/licenses/netty-common-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +9da10a9f72e3f87e181d91b525174007a6fc4f11 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.114.Final.jar.sha1 deleted file mode 100644 index c431976b6fbd2..0000000000000 --- a/plugins/transport-nio/licenses/netty-handler-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e56fbde4b9aa628eed15a5dbfbeb97877db88146 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.115.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..5c58e02209691 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-handler-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d54dbf68b9d88a98240107758c6b63da5e46e23a \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.114.Final.jar.sha1 deleted file mode 100644 index e8080a5b2acb1..0000000000000 --- a/plugins/transport-nio/licenses/netty-resolver-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -10b23784b23d6a948930f52ba82874f1291b5873 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.115.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..b1d9f1f679966 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-resolver-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +e33b4d476c03975957f5d8d0319d592bf2bc5e96 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.114.Final.jar.sha1 deleted file mode 100644 index fb2d518789a18..0000000000000 --- a/plugins/transport-nio/licenses/netty-transport-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e0225a575f487904be8517092cbd74e01913533c \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.115.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..aeeda9c282b30 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-transport-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +39cef77c1a25908ac1abf4960c2e789f0bf70ff9 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 deleted file mode 100644 index a80b9e51be74b..0000000000000 --- a/plugins/transport-nio/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d1171bb99411f282068f49d780cedf8c9adeabfd \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..41996121685f0 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +dc96c67d06cd6b5eb677f2728f27bf2e3d9a7284 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/build.gradle b/plugins/transport-reactor-netty4/build.gradle index 089e57f062a9f..1e76d1a29efc1 100644 --- a/plugins/transport-reactor-netty4/build.gradle +++ b/plugins/transport-reactor-netty4/build.gradle @@ -258,6 +258,9 @@ thirdPartyAudit { 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField', 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField', 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueConsumerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerLimitField', 'io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess', 'io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess', 'io.netty.util.internal.shaded.org.jctools.util.UnsafeLongArrayAccess', diff --git a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 deleted file mode 100644 index 8cb83fc367d78..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f1d77d15c0b781cd9395a2a956262766fd0c7602 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.115.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..825b91b71601d --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d5daf1030e5c36d198caf7562da2441a97ec0df6 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 deleted file mode 100644 index 1be26fee34d46..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5a49dfa2828d64bf756f670e63259115332744cf \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.115.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..e39392f923f7b --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d326bf3a4c785b272da3db6941779a1bd5448378 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 deleted file mode 100644 index 04a48547adb05..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -35798729ba06670fb4fcd02db98d9577e363992d \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.115.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..fc735d97148ef --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d39b9866939cbbb8ae3a1af5c1df5ddf93656d47 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 deleted file mode 100644 index e683773245716..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fbce5a53884275662e68aaad70f88bf7e5d04164 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.115.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..ce4bdc323b2a0 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +80f0dece29a2c0269217e8dd1b6db6ff9710781f \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 deleted file mode 100644 index 2c18924e33c62..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -19ae07fdf99142a70338f8cda70a3d2edbc8e80a \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..f718fc9bf1622 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +0bc474c27c96e3a309da73160fbcfe0bd3aa85bc \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 deleted file mode 100644 index 74ce939dc6190..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -862712e292b162c8ccaa7847a6a54df8178f77e5 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-common-4.1.115.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-common-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..2ff84dc14147d --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-common-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +9da10a9f72e3f87e181d91b525174007a6fc4f11 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 deleted file mode 100644 index c431976b6fbd2..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e56fbde4b9aa628eed15a5dbfbeb97877db88146 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.115.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..5c58e02209691 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d54dbf68b9d88a98240107758c6b63da5e46e23a \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 deleted file mode 100644 index e8080a5b2acb1..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -10b23784b23d6a948930f52ba82874f1291b5873 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.115.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..b1d9f1f679966 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +e33b4d476c03975957f5d8d0319d592bf2bc5e96 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 deleted file mode 100644 index 7df7a05cd7345..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2fed36ff50059ded641fa5064963c4b4313512f3 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.115.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..e443fa3fa383f --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +33b0636c6fe36c7f41d9da204aa1ad94ff244ac7 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 deleted file mode 100644 index fb2d518789a18..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e0225a575f487904be8517092cbd74e01913533c \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.115.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..aeeda9c282b30 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +39cef77c1a25908ac1abf4960c2e789f0bf70ff9 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 deleted file mode 100644 index a80b9e51be74b..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d1171bb99411f282068f49d780cedf8c9adeabfd \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..41996121685f0 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +dc96c67d06cd6b5eb677f2728f27bf2e3d9a7284 \ No newline at end of file diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml index 408e3c9fe0f9f..68d2a899d46cb 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml @@ -512,8 +512,8 @@ setup: --- "Composite aggregation with filtered nested parent": - skip: - version: " - 2.99.99" - reason: fixed in 3.0.0 + version: " - 2.12.99" + reason: fixed in 2.13.0 - do: search: rest_total_hits_as_int: true @@ -582,8 +582,8 @@ setup: --- "Composite aggregation with filtered reverse nested parent": - skip: - version: " - 2.99.99" - reason: fixed in 3.0.0 + version: " - 2.12.99" + reason: fixed in 2.13.0 - do: search: rest_total_hits_as_int: true diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml index a133060f07c6f..647aaf2c9088b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml @@ -449,6 +449,28 @@ - match: { hits.total: 2 } + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + term: + ip_field: "192.168.0.1/24" + + - match: { hits.total: 3 } + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + term: + ip_field: "192.168.0.1/31" + + - match: { hits.total: 1 } + - do: search: rest_total_hits_as_int: true @@ -987,6 +1009,28 @@ - match: { hits.total: 2 } + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + term: + ip_field: "192.168.0.1/24" + + - match: { hits.total: 3 } + + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + term: + ip_field: "192.168.0.1/31" + + - match: { hits.total: 1 } + - do: search: rest_total_hits_as_int: true @@ -1077,8 +1121,8 @@ "search on fields with only doc_values enabled": - skip: features: [ "headers" ] - version: " - 2.99.99" - reason: "searching with only doc_values was added in 3.0.0" + version: " - 2.18.99" + reason: "searching with only doc_values was finally added in 2.19.0" - do: indices.create: index: test-doc-values @@ -1372,6 +1416,28 @@ - match: { hits.total: 2 } + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + terms: + ip_field: ["192.168.0.1", "192.168.0.2"] + + - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + terms: + ip_field: ["192.168.0.1/31", "192.168.0.3"] + + - match: { hits.total: 2 } + - do: search: rest_total_hits_as_int: true @@ -1516,6 +1582,28 @@ - match: { hits.total: 2 } + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + term: + ip_field: "192.168.0.1/31" + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + term: + ip_field: "192.168.0.1/24" + + - match: { hits.total: 3 } + - do: search: rest_total_hits_as_int: true diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java index 57bf9eccbf5b4..1b5d924fa0b62 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java @@ -8,34 +8,9 @@ package org.opensearch.gateway.remote; -import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; -import org.opensearch.common.settings.Settings; -import org.opensearch.plugins.Plugin; -import org.opensearch.remotemigration.MigrationBaseTestCase; -import org.opensearch.remotestore.multipart.mocks.MockFsRepositoryPlugin; -import org.opensearch.repositories.blobstore.BlobStoreRepository; -import org.opensearch.repositories.fs.FsRepository; -import org.opensearch.repositories.fs.ReloadableFsRepository; -import org.opensearch.test.InternalSettingsPlugin; import org.opensearch.test.OpenSearchIntegTestCase; -import org.opensearch.test.transport.MockTransportService; -import org.junit.Assert; import org.junit.Before; -import java.util.Collection; -import java.util.Locale; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING; -import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; - /** * Tests the compatibility between types of nodes based on the configured repositories * Non Remote node [No Repositories configured] @@ -44,260 +19,15 @@ * Remote Node With Routing Table [Cluster State + Segment + Translog + Routing Table] */ @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) -public class RemotePublicationConfigurationIT extends MigrationBaseTestCase { - private final String REMOTE_PRI_DOCREP_REP = "remote-primary-docrep-replica"; - - @Override - protected Collection> nodePlugins() { - /* Adding the following mock plugins: - - InternalSettingsPlugin : To override default intervals of retention lease and global ckp sync - - MockFsRepositoryPlugin and MockTransportService.TestPlugin: To ensure remote interactions are not no-op and retention leases are properly propagated - */ - return Stream.concat( - super.nodePlugins().stream(), - Stream.of(InternalSettingsPlugin.class, MockFsRepositoryPlugin.class, MockTransportService.TestPlugin.class) - ).collect(Collectors.toList()); - } - +public class RemotePublicationConfigurationIT extends RemoteRepositoryConfigurationIT { @Before public void setUp() throws Exception { if (segmentRepoPath == null || translogRepoPath == null) { segmentRepoPath = randomRepoPath().toAbsolutePath(); translogRepoPath = randomRepoPath().toAbsolutePath(); } + super.remoteRepoPrefix = "remote_publication"; super.setUp(); } - public Settings.Builder remotePublishConfiguredNodeSetting() { - String stateRepoSettingsAttributeKeyPrefix = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, - REPOSITORY_NAME - ); - String prefixModeVerificationSuffix = BlobStoreRepository.PREFIX_MODE_VERIFICATION_SETTING.getKey(); - String stateRepoTypeAttributeKey = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, - REPOSITORY_NAME - ); - String routingTableRepoTypeAttributeKey = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, - ROUTING_TABLE_REPO_NAME - ); - String routingTableRepoSettingsAttributeKeyPrefix = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, - ROUTING_TABLE_REPO_NAME - ); - - Settings.Builder builder = Settings.builder() - .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, REPOSITORY_NAME) - .put(stateRepoTypeAttributeKey, ReloadableFsRepository.TYPE) - .put(stateRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath) - .put(stateRepoSettingsAttributeKeyPrefix + prefixModeVerificationSuffix, true) - .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) - .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, ROUTING_TABLE_REPO_NAME) - .put(routingTableRepoTypeAttributeKey, FsRepository.TYPE) - .put(routingTableRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath); - return builder; - } - - public Settings.Builder remoteWithRoutingTableNodeSetting() { - // Remote Cluster with Routing table - - return Settings.builder() - .put( - remoteStoreClusterSettings( - REPOSITORY_NAME, - segmentRepoPath, - ReloadableFsRepository.TYPE, - REPOSITORY_2_NAME, - translogRepoPath, - ReloadableFsRepository.TYPE, - REPOSITORY_NAME, - segmentRepoPath, - ReloadableFsRepository.TYPE - ) - ) - .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true); - } - - public void testRemoteClusterStateServiceNotInitialized_WhenNodeAttributesNotPresent() { - internalCluster().startClusterManagerOnlyNode(); - internalCluster().startDataOnlyNodes(2); - - ensureStableCluster(3); - ensureGreen(); - - internalCluster().getDataOrClusterManagerNodeInstances(RemoteClusterStateService.class).forEach(Assert::assertNull); - } - - public void testServiceInitialized_WhenNodeAttributesPresent() { - internalCluster().startClusterManagerOnlyNode( - buildRemoteStateNodeAttributes(REPOSITORY_NAME, segmentRepoPath, ReloadableFsRepository.TYPE) - ); - internalCluster().startDataOnlyNodes( - 2, - buildRemoteStateNodeAttributes(REPOSITORY_NAME, segmentRepoPath, ReloadableFsRepository.TYPE) - ); - - ensureStableCluster(3); - ensureGreen(); - - internalCluster().getDataOrClusterManagerNodeInstances(RemoteClusterStateService.class).forEach(Assert::assertNotNull); - } - - public void testRemotePublishConfigNodeJoinNonRemoteCluster() throws Exception { - internalCluster().startClusterManagerOnlyNode(); - internalCluster().startDataOnlyNodes(2); - - Settings.Builder build = remotePublishConfiguredNodeSetting(); - internalCluster().startClusterManagerOnlyNode(build.build()); - internalCluster().startDataOnlyNodes(2, build.build()); - - ensureStableCluster(6); - ensureGreen(); - } - - public void testRemotePublishConfigNodeJoinRemoteCluster() throws Exception { - // Remote Cluster without Routing table - setAddRemote(true); - internalCluster().startClusterManagerOnlyNode(); - internalCluster().startDataOnlyNodes(2); - setAddRemote(false); - - ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); - updateSettingsRequest.persistentSettings( - Settings.builder() - .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") - .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") - ); - assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); - Settings.Builder build = remotePublishConfiguredNodeSetting(); - internalCluster().startClusterManagerOnlyNode(build.build()); - ensureStableCluster(4); - ensureGreen(); - } - - public void testRemoteNodeWithRoutingTableJoinRemoteCluster() throws Exception { - setAddRemote(true); - internalCluster().startClusterManagerOnlyNode(); - internalCluster().startDataOnlyNodes(2); - setAddRemote(false); - - ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); - updateSettingsRequest.persistentSettings( - Settings.builder() - .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") - .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") - ); - assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); - - // Remote Repo with Routing table - Settings settings = remoteWithRoutingTableNodeSetting().build(); - internalCluster().startClusterManagerOnlyNode(settings); - ensureStableCluster(4); - ensureGreen(); - } - - public void testNonRemoteNodeJoinRemoteWithRoutingCluster() throws Exception { - Settings settings = remoteWithRoutingTableNodeSetting().build(); - internalCluster().startClusterManagerOnlyNode(settings); - internalCluster().startDataOnlyNodes(2, settings); - - ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); - updateSettingsRequest.persistentSettings( - Settings.builder() - .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") - .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") - ); - assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); - - internalCluster().startClusterManagerOnlyNode(); - ensureStableCluster(4); - ensureGreen(); - } - - public void testRemotePublishConfigNodeJoinRemoteWithRoutingCluster() throws Exception { - Settings settings = remoteWithRoutingTableNodeSetting().build(); - internalCluster().startClusterManagerOnlyNode(settings); - internalCluster().startDataOnlyNodes(2, settings); - - ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); - updateSettingsRequest.persistentSettings( - Settings.builder() - .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") - .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") - ); - assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); - - internalCluster().startClusterManagerOnlyNode(remotePublishConfiguredNodeSetting().build()); - - ensureStableCluster(4); - ensureGreen(); - } - - public void testNonRemoteNodeJoiningPublishConfigCluster() throws Exception { - Settings.Builder build = remotePublishConfiguredNodeSetting(); - internalCluster().startClusterManagerOnlyNode(build.build()); - internalCluster().startDataOnlyNodes(2, build.build()); - - internalCluster().startClusterManagerOnlyNode(); - - ensureStableCluster(4); - ensureGreen(); - } - - public void testRemoteNodeJoiningPublishConfigCluster() throws Exception { - Settings.Builder build = remotePublishConfiguredNodeSetting(); - internalCluster().startClusterManagerOnlyNode(build.build()); - internalCluster().startDataOnlyNodes(2, build.build()); - - ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); - updateSettingsRequest.persistentSettings( - Settings.builder() - .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") - .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") - ); - assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); - - setAddRemote(true); - internalCluster().startClusterManagerOnlyNode(); - ensureStableCluster(4); - ensureGreen(); - } - - public void testRemoteNodeWithRoutingTableJoiningPublishConfigCluster() throws Exception { - Settings.Builder build = remotePublishConfiguredNodeSetting(); - internalCluster().startClusterManagerOnlyNode(build.build()); - internalCluster().startDataOnlyNodes(2, build.build()); - - ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); - updateSettingsRequest.persistentSettings( - Settings.builder() - .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") - .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") - ); - - Settings settings = Settings.builder() - .put( - buildRemoteStoreNodeAttributes( - REPOSITORY_NAME, - segmentRepoPath, - REPOSITORY_2_NAME, - translogRepoPath, - ROUTING_TABLE_REPO_NAME, - segmentRepoPath, - false - ) - ) - .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) - .build(); - assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); - internalCluster().startClusterManagerOnlyNode(settings); - - ensureStableCluster(4); - ensureGreen(); - } } diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRepositoryConfigurationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRepositoryConfigurationIT.java new file mode 100644 index 0000000000000..48afa85dc5691 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRepositoryConfigurationIT.java @@ -0,0 +1,308 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; +import org.opensearch.common.settings.Settings; +import org.opensearch.plugins.Plugin; +import org.opensearch.remotemigration.MigrationBaseTestCase; +import org.opensearch.remotestore.multipart.mocks.MockFsRepositoryPlugin; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.repositories.fs.FsRepository; +import org.opensearch.repositories.fs.ReloadableFsRepository; +import org.opensearch.test.InternalSettingsPlugin; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.transport.MockTransportService; +import org.junit.Assert; +import org.junit.Before; + +import java.util.Collection; +import java.util.Locale; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING; +import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; + +/** + * Tests the compatibility between types of nodes based on the configured repositories + * Non Remote node [No Repositories configured] + * Remote Publish Configured Node [Cluster State + Routing Table] + * Remote Node [Cluster State + Segment + Translog] + * Remote Node With Routing Table [Cluster State + Segment + Translog + Routing Table] + */ +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +public class RemoteRepositoryConfigurationIT extends MigrationBaseTestCase { + private final String REMOTE_PRI_DOCREP_REP = "remote-primary-docrep-replica"; + + protected String remoteRepoPrefix = "remote_store"; + + @Override + protected Collection> nodePlugins() { + /* Adding the following mock plugins: + - InternalSettingsPlugin : To override default intervals of retention lease and global ckp sync + - MockFsRepositoryPlugin and MockTransportService.TestPlugin: To ensure remote interactions are not no-op and retention leases are properly propagated + */ + return Stream.concat( + super.nodePlugins().stream(), + Stream.of(InternalSettingsPlugin.class, MockFsRepositoryPlugin.class, MockTransportService.TestPlugin.class) + ).collect(Collectors.toList()); + } + + @Before + public void setUp() throws Exception { + if (segmentRepoPath == null || translogRepoPath == null) { + segmentRepoPath = randomRepoPath().toAbsolutePath(); + translogRepoPath = randomRepoPath().toAbsolutePath(); + } + super.setUp(); + } + + public Settings.Builder remotePublishConfiguredNodeSetting() { + String stateRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + remoteRepoPrefix, + REPOSITORY_NAME + ); + String prefixModeVerificationSuffix = BlobStoreRepository.PREFIX_MODE_VERIFICATION_SETTING.getKey(); + String stateRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + remoteRepoPrefix, + REPOSITORY_NAME + ); + String routingTableRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + remoteRepoPrefix, + ROUTING_TABLE_REPO_NAME + ); + String routingTableRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + remoteRepoPrefix, + ROUTING_TABLE_REPO_NAME + ); + + Settings.Builder builder = Settings.builder() + .put("node.attr." + remoteRepoPrefix + ".state.repository", REPOSITORY_NAME) + .put(stateRepoTypeAttributeKey, ReloadableFsRepository.TYPE) + .put(stateRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath) + .put(stateRepoSettingsAttributeKeyPrefix + prefixModeVerificationSuffix, true) + .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) + .put("node.attr." + remoteRepoPrefix + ".routing_table.repository", ROUTING_TABLE_REPO_NAME) + .put(routingTableRepoTypeAttributeKey, ReloadableFsRepository.TYPE) + .put(routingTableRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath); + return builder; + } + + public Settings.Builder remoteWithRoutingTableNodeSetting() { + // Remote Cluster with Routing table + return Settings.builder() + .put( + buildRemoteStoreNodeAttributes( + REPOSITORY_NAME, + segmentRepoPath, + REPOSITORY_2_NAME, + translogRepoPath, + ROUTING_TABLE_REPO_NAME, + segmentRepoPath, + false + ) + ) + .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true); + } + + public void testRemoteClusterStateServiceNotInitialized_WhenNodeAttributesNotPresent() { + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNodes(2); + + ensureStableCluster(3); + ensureGreen(); + + internalCluster().getDataOrClusterManagerNodeInstances(RemoteClusterStateService.class).forEach(Assert::assertNull); + } + + public void testServiceInitialized_WhenNodeAttributesPresent() { + internalCluster().startClusterManagerOnlyNode( + buildRemoteStateNodeAttributes(REPOSITORY_NAME, segmentRepoPath, ReloadableFsRepository.TYPE) + ); + internalCluster().startDataOnlyNodes( + 2, + buildRemoteStateNodeAttributes(REPOSITORY_NAME, segmentRepoPath, ReloadableFsRepository.TYPE) + ); + + ensureStableCluster(3); + ensureGreen(); + + internalCluster().getDataOrClusterManagerNodeInstances(RemoteClusterStateService.class).forEach(Assert::assertNotNull); + } + + public void testRemotePublishConfigNodeJoinNonRemoteCluster() throws Exception { + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNodes(2); + + Settings.Builder build = remotePublishConfiguredNodeSetting(); + internalCluster().startClusterManagerOnlyNode(build.build()); + internalCluster().startDataOnlyNodes(2, build.build()); + + ensureStableCluster(6); + ensureGreen(); + } + + public void testRemotePublishConfigNodeJoinRemoteCluster() throws Exception { + // Remote Cluster without Routing table + setAddRemote(true); + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNodes(2); + setAddRemote(false); + + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings( + Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + ); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + Settings.Builder build = remotePublishConfiguredNodeSetting(); + internalCluster().startClusterManagerOnlyNode(build.build()); + ensureStableCluster(4); + ensureGreen(); + } + + public void testRemoteNodeWithRoutingTableJoinRemoteCluster() throws Exception { + setAddRemote(true); + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNodes(2); + setAddRemote(false); + + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings( + Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + ); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + + // Remote Repo with Routing table + Settings settings = remoteWithRoutingTableNodeSetting().build(); + + internalCluster().startClusterManagerOnlyNode(settings); + ensureStableCluster(4); + ensureGreen(); + } + + public void testNonRemoteNodeJoinRemoteWithRoutingCluster() throws Exception { + Settings settings = remoteWithRoutingTableNodeSetting().build(); + internalCluster().startClusterManagerOnlyNode(settings); + internalCluster().startDataOnlyNodes(2, settings); + + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings( + Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + ); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + + internalCluster().startClusterManagerOnlyNode(); + ensureStableCluster(4); + ensureGreen(); + } + + public void testRemotePublishConfigNodeJoinRemoteWithRoutingCluster() throws Exception { + Settings settings = remoteWithRoutingTableNodeSetting().build(); + internalCluster().startClusterManagerOnlyNode(settings); + internalCluster().startDataOnlyNodes(2, settings); + + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings( + Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + ); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + + internalCluster().startClusterManagerOnlyNode(remotePublishConfiguredNodeSetting().build()); + + ensureStableCluster(4); + ensureGreen(); + } + + public void testNonRemoteNodeJoiningPublishConfigCluster() throws Exception { + Settings.Builder build = remotePublishConfiguredNodeSetting(); + internalCluster().startClusterManagerOnlyNode(build.build()); + internalCluster().startDataOnlyNodes(2, build.build()); + + internalCluster().startClusterManagerOnlyNode(); + + ensureStableCluster(4); + ensureGreen(); + } + + public void testRemoteNodeJoiningPublishConfigCluster() throws Exception { + Settings.Builder build = remotePublishConfiguredNodeSetting(); + internalCluster().startClusterManagerOnlyNode(build.build()); + internalCluster().startDataOnlyNodes(2, build.build()); + + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings( + Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + ); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + + setAddRemote(true); + internalCluster().startClusterManagerOnlyNode(); + ensureStableCluster(4); + ensureGreen(); + } + + public void testRemoteNodeWithRoutingTableJoiningPublishConfigCluster() throws Exception { + Settings.Builder build = remotePublishConfiguredNodeSetting(); + internalCluster().startClusterManagerOnlyNode(build.build()); + internalCluster().startDataOnlyNodes(2, build.build()); + + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings( + Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + ); + + Settings settings = Settings.builder() + .put( + buildRemoteStoreNodeAttributes( + REPOSITORY_NAME, + segmentRepoPath, + ReloadableFsRepository.TYPE, + REPOSITORY_2_NAME, + translogRepoPath, + FsRepository.TYPE, + ROUTING_TABLE_REPO_NAME, + segmentRepoPath, + ReloadableFsRepository.TYPE, + false + ) + ) + .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) + .build(); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + internalCluster().startClusterManagerOnlyNode(settings); + + ensureStableCluster(4); + ensureGreen(); + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java index 26b30af4c2c50..85d81761ea4a0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java @@ -45,6 +45,37 @@ public class DeleteSnapshotIT extends AbstractSnapshotIntegTestCase { private static final String REMOTE_REPO_NAME = "remote-store-repo-name"; + public void testStaleIndexDeletion() throws Exception { + String indexName1 = ".testindex1"; + String repoName = "test-restore-snapshot-repo"; + String snapshotName1 = "test-restore-snapshot1"; + Path absolutePath = randomRepoPath().toAbsolutePath(); + logger.info("Path [{}]", absolutePath); + + Client client = client(); + // Write a document + String docId = Integer.toString(randomInt()); + index(indexName1, "_doc", docId, "value", "expected"); + createRepository(repoName, "fs", absolutePath); + + logger.info("--> snapshot"); + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(repoName, snapshotName1) + .setWaitForCompletion(true) + .setIndices(indexName1) + .get(); + assertTrue(createSnapshotResponse.getSnapshotInfo().successfulShards() > 0); + assertEquals(createSnapshotResponse.getSnapshotInfo().totalShards(), createSnapshotResponse.getSnapshotInfo().successfulShards()); + assertEquals(SnapshotState.SUCCESS, createSnapshotResponse.getSnapshotInfo().state()); + + assertAcked(startDeleteSnapshot(repoName, snapshotName1).get()); + assertBusy(() -> assertEquals(0, RemoteStoreBaseIntegTestCase.getFileCount(absolutePath.resolve(BlobStoreRepository.INDICES_DIR)))); + assertBusy(() -> assertEquals(0, RemoteStoreBaseIntegTestCase.getFileCount(absolutePath.resolve(SnapshotShardPaths.DIR)))); + // At the end there are 2 files that exists - index-N and index.latest + assertBusy(() -> assertEquals(2, RemoteStoreBaseIntegTestCase.getFileCount(absolutePath))); + } + public void testDeleteSnapshot() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); final Path remoteStoreRepoPath = randomRepoPath(); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index d597b51c32ccd..ec30496a3f7ad 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -55,6 +55,7 @@ import org.opensearch.persistent.PersistentTasksCustomMetadata; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -69,7 +70,8 @@ import static org.opensearch.cluster.decommission.DecommissionHelper.nodeCommissioned; import static org.opensearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_CLUSTER_PUBLICATION_REPO_NAME_ATTRIBUTES; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.getClusterStateRepoName; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.getRoutingTableRepoName; import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode; import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode.MIXED; import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode.STRICT; @@ -539,7 +541,12 @@ private static void ensureRemoteClusterStateNodesCompatibility(DiscoveryNode joi .findFirst(); if (remotePublicationNode.isPresent() && joiningNode.isRemoteStatePublicationEnabled()) { - ensureRepositoryCompatibility(joiningNode, remotePublicationNode.get(), REMOTE_CLUSTER_PUBLICATION_REPO_NAME_ATTRIBUTES); + List repos = Arrays.asList( + getClusterStateRepoName(remotePublicationNode.get().getAttributes()), + getRoutingTableRepoName(remotePublicationNode.get().getAttributes()) + ); + + ensureRepositoryCompatibility(joiningNode, remotePublicationNode.get(), repos); } } @@ -568,16 +575,12 @@ private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNod List reposToSkip = new ArrayList<>(1); // find a remote node which has routing table configured Optional remoteRoutingTableNode = existingNodes.stream() - .filter( - node -> node.isRemoteStoreNode() - && node.getAttributes().get(RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY) != null - ) + .filter(node -> node.isRemoteStoreNode() && RemoteStoreNodeAttribute.getRoutingTableRepoName(node.getAttributes()) != null) .findFirst(); // If none of the existing nodes have routing table repo, then we skip this repo check if present in joining node. // This ensures a new node with remote routing table repo is able to join the cluster. if (remoteRoutingTableNode.isEmpty()) { - String joiningNodeRepoName = joiningNode.getAttributes() - .get(RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY); + String joiningNodeRepoName = getRoutingTableRepoName(joiningNode.getAttributes()); if (joiningNodeRepoName != null) { reposToSkip.add(joiningNodeRepoName); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index 727a08b615050..232201d18ba13 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -1178,12 +1178,8 @@ public static void updateRemoteStoreSettings( .findFirst(); if (remoteNode.isPresent()) { - translogRepo = remoteNode.get() - .getAttributes() - .get(RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY); - segmentRepo = remoteNode.get() - .getAttributes() - .get(RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY); + translogRepo = RemoteStoreNodeAttribute.getTranslogRepoName(remoteNode.get().getAttributes()); + segmentRepo = RemoteStoreNodeAttribute.getSegmentRepoName(remoteNode.get().getAttributes()); if (segmentRepo != null && translogRepo != null) { settingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, true) .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, segmentRepo) diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index 8c9a37a767ede..d84fb794c5e4f 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -45,6 +45,7 @@ import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.node.Node; +import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import java.io.IOException; import java.util.Collections; @@ -62,10 +63,9 @@ import java.util.stream.Stream; import static org.opensearch.node.NodeRoleSettings.NODE_ROLES_SETTING; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isClusterStateRepoConfigured; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRoutingTableRepoConfigured; /** * A discovery node represents a node that is part of the cluster. @@ -510,8 +510,7 @@ public boolean isSearchNode() { * @return true if the node contains remote store node attributes, false otherwise */ public boolean isRemoteStoreNode() { - return this.getAttributes().keySet().stream().anyMatch(key -> key.equals(REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY)) - && this.getAttributes().keySet().stream().anyMatch(key -> key.equals(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY)); + return isClusterStateRepoConfigured(this.getAttributes()) && RemoteStoreNodeAttribute.isSegmentRepoConfigured(this.getAttributes()); } /** @@ -519,11 +518,7 @@ public boolean isRemoteStoreNode() { * @return true if the node contains remote cluster state node attribute and remote routing table node attribute */ public boolean isRemoteStatePublicationEnabled() { - return this.getAttributes() - .keySet() - .stream() - .anyMatch(key -> (key.equals(REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY))) - && this.getAttributes().keySet().stream().anyMatch(key -> key.equals(REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY)); + return isClusterStateRepoConfigured(this.getAttributes()) && isRoutingTableRepoConfigured(this.getAttributes()); } /** @@ -587,13 +582,16 @@ public String toString() { sb.append('}'); } if (!attributes.isEmpty()) { - sb.append( - attributes.entrySet() - .stream() - .filter(entry -> !entry.getKey().startsWith(REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX)) // filter remote_store attributes - // from logging to reduce noise. - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) - ); + sb.append(attributes.entrySet().stream().filter(entry -> { + for (String prefix : REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX) { + if (entry.getKey().startsWith(prefix)) { + return false; + } + } + return true; + }) // filter remote_store attributes + // from logging to reduce noise. + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); } return sb.toString(); } diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java index ea8f980c14972..eafbe05faf76f 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java @@ -33,7 +33,6 @@ import org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable; import org.opensearch.gateway.remote.routingtable.RemoteRoutingTableDiff; import org.opensearch.index.translog.transfer.BlobStoreTransferService; -import org.opensearch.node.Node; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; @@ -235,9 +234,7 @@ protected void doClose() throws IOException { @Override protected void doStart() { assert isRemoteRoutingTableConfigured(settings) == true : "Remote routing table is not enabled"; - final String remoteStoreRepo = settings.get( - Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY - ); + final String remoteStoreRepo = RemoteStoreNodeAttribute.getRoutingTableRepoName(settings); assert remoteStoreRepo != null : "Remote routing table repository is not configured"; final Repository repository = repositoriesService.get().repository(remoteStoreRepo); assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index dc41189afc3cb..e4f4bae9bef7c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -61,7 +61,6 @@ import org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata; import org.opensearch.gateway.remote.routingtable.RemoteRoutingTableDiff; import org.opensearch.index.translog.transfer.BlobStoreTransferService; -import org.opensearch.node.Node; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; @@ -1065,9 +1064,8 @@ public void close() throws IOException { public void start() { assert isRemoteClusterStateConfigured(settings) == true : "Remote cluster state is not enabled"; - final String remoteStoreRepo = settings.get( - Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY - ); + final String remoteStoreRepo = RemoteStoreNodeAttribute.getClusterStateRepoName(settings); + assert remoteStoreRepo != null : "Remote Cluster State repository is not configured"; final Repository repository = repositoriesService.get().repository(remoteStoreRepo); assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; diff --git a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java index db8da8a949d6f..c51cada9f3143 100644 --- a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java @@ -64,6 +64,7 @@ import java.util.Map; import java.util.function.BiFunction; import java.util.function.Supplier; +import java.util.stream.Collectors; /** * A {@link FieldMapper} for ip addresses. @@ -225,9 +226,9 @@ protected Object parseSourceValue(Object value) { @Override public Query termQuery(Object value, @Nullable QueryShardContext context) { failIfNotIndexedAndNoDocValues(); - Query query; + final PointRangeQuery pointQuery; if (value instanceof InetAddress) { - query = InetAddressPoint.newExactQuery(name(), (InetAddress) value); + pointQuery = (PointRangeQuery) InetAddressPoint.newExactQuery(name(), (InetAddress) value); } else { if (value instanceof BytesRef) { value = ((BytesRef) value).utf8ToString(); @@ -235,32 +236,27 @@ public Query termQuery(Object value, @Nullable QueryShardContext context) { String term = value.toString(); if (term.contains("/")) { final Tuple cidr = InetAddresses.parseCidr(term); - query = InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2()); + pointQuery = (PointRangeQuery) InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2()); } else { InetAddress address = InetAddresses.forString(term); - query = InetAddressPoint.newExactQuery(name(), address); + pointQuery = (PointRangeQuery) InetAddressPoint.newExactQuery(name(), address); } } - if (isSearchable() && hasDocValues()) { - String term = value.toString(); - if (term.contains("/")) { - final Tuple cidr = InetAddresses.parseCidr(term); - return InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2()); - } - return new IndexOrDocValuesQuery( - query, - SortedSetDocValuesField.newSlowExactQuery(name(), new BytesRef(((PointRangeQuery) query).getLowerPoint())) + Query dvQuery = null; + if (hasDocValues()) { + dvQuery = SortedSetDocValuesField.newSlowRangeQuery( + name(), + new BytesRef(pointQuery.getLowerPoint()), + new BytesRef(pointQuery.getUpperPoint()), + true, + true ); } - if (hasDocValues()) { - String term = value.toString(); - if (term.contains("/")) { - final Tuple cidr = InetAddresses.parseCidr(term); - return InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2()); - } - return SortedSetDocValuesField.newSlowExactQuery(name(), new BytesRef(((PointRangeQuery) query).getLowerPoint())); + if (isSearchable() && hasDocValues()) { + return new IndexOrDocValuesQuery(pointQuery, dvQuery); + } else { + return isSearchable() ? pointQuery : dvQuery; } - return query; } @Override @@ -285,36 +281,46 @@ public Query termsQuery(List values, QueryShardContext context) { } addresses[i++] = address; } - return InetAddressPoint.newSetQuery(name(), addresses); + Query dvQuery = null; + if (hasDocValues()) { + List bytesRefs = Arrays.stream(addresses) + .distinct() + .map(InetAddressPoint::encode) + .map(BytesRef::new) + .collect(Collectors.toList()); + dvQuery = SortedSetDocValuesField.newSlowSetQuery(name(), bytesRefs); + } + Query pointQuery = null; + if (isSearchable()) { + pointQuery = InetAddressPoint.newSetQuery(name(), addresses); + } + if (isSearchable() && hasDocValues()) { + return new IndexOrDocValuesQuery(pointQuery, dvQuery); + } else { + return isSearchable() ? pointQuery : dvQuery; + } } @Override public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) { failIfNotIndexedAndNoDocValues(); return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, (lower, upper) -> { - Query query = InetAddressPoint.newRangeQuery(name(), lower, upper); - if (isSearchable() && hasDocValues()) { - return new IndexOrDocValuesQuery( - query, - SortedSetDocValuesField.newSlowRangeQuery( - ((PointRangeQuery) query).getField(), - new BytesRef(((PointRangeQuery) query).getLowerPoint()), - new BytesRef(((PointRangeQuery) query).getUpperPoint()), - true, - true - ) - ); - } + PointRangeQuery pointQuery = (PointRangeQuery) InetAddressPoint.newRangeQuery(name(), lower, upper); + Query dvQuery = null; if (hasDocValues()) { - return SortedSetDocValuesField.newSlowRangeQuery( - ((PointRangeQuery) query).getField(), - new BytesRef(((PointRangeQuery) query).getLowerPoint()), - new BytesRef(((PointRangeQuery) query).getUpperPoint()), + dvQuery = SortedSetDocValuesField.newSlowRangeQuery( + pointQuery.getField(), + new BytesRef(pointQuery.getLowerPoint()), + new BytesRef(pointQuery.getUpperPoint()), true, true ); } - return query; + if (isSearchable() && hasDocValues()) { + return new IndexOrDocValuesQuery(pointQuery, dvQuery); + } else { + return isSearchable() ? pointQuery : dvQuery; + } }); } diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java index 2a76a5b966884..18b6d6184d1b0 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java @@ -26,7 +26,6 @@ import org.opensearch.gateway.remote.RemoteStateTransferException; import org.opensearch.index.remote.RemoteStoreEnums.PathType; import org.opensearch.indices.RemoteStoreSettings; -import org.opensearch.node.Node; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; @@ -70,11 +69,6 @@ public class RemoteIndexPathUploader extends IndexMetadataUploadListener { private static final String TIMEOUT_EXCEPTION_MSG = "Timed out waiting while uploading remote index path file for indexes=%s"; private static final String UPLOAD_EXCEPTION_MSG = "Exception occurred while uploading remote index paths for indexes=%s"; - static final String TRANSLOG_REPO_NAME_KEY = Node.NODE_ATTRIBUTES.getKey() - + RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; - static final String SEGMENT_REPO_NAME_KEY = Node.NODE_ATTRIBUTES.getKey() - + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; - private static final Logger logger = LogManager.getLogger(RemoteIndexPathUploader.class); private final Settings settings; @@ -226,9 +220,8 @@ private void writePathToRemoteStore( } } - private Repository validateAndGetRepository(String repoSetting) { - final String repo = settings.get(repoSetting); - assert repo != null : "Remote " + repoSetting + " repository is not configured"; + private Repository validateAndGetRepository(String repo) { + assert repo != null : "Remote repository is not configured"; final Repository repository = repositoriesService.get().repository(repo); assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; return repository; @@ -240,15 +233,16 @@ public void start() { // If remote store data attributes are not present than we skip this. return; } - translogRepository = (BlobStoreRepository) validateAndGetRepository(TRANSLOG_REPO_NAME_KEY); - segmentRepository = (BlobStoreRepository) validateAndGetRepository(SEGMENT_REPO_NAME_KEY); + + translogRepository = (BlobStoreRepository) validateAndGetRepository(RemoteStoreNodeAttribute.getRemoteStoreTranslogRepo(settings)); + segmentRepository = (BlobStoreRepository) validateAndGetRepository(RemoteStoreNodeAttribute.getRemoteStoreSegmentRepo(settings)); } private boolean isTranslogSegmentRepoSame() { // TODO - The current comparison checks the repository name. But it is also possible that the repository are same // by attributes, but different by name. We need to handle this. - String translogRepoName = settings.get(TRANSLOG_REPO_NAME_KEY); - String segmentRepoName = settings.get(SEGMENT_REPO_NAME_KEY); + String translogRepoName = RemoteStoreNodeAttribute.getRemoteStoreTranslogRepo(settings); + String segmentRepoName = RemoteStoreNodeAttribute.getRemoteStoreSegmentRepo(settings); return Objects.equals(translogRepoName, segmentRepoName); } diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteMigrationIndexMetadataUpdater.java b/server/src/main/java/org/opensearch/index/remote/RemoteMigrationIndexMetadataUpdater.java index cc51fcd2f18f6..1f9ffca4460b7 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteMigrationIndexMetadataUpdater.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteMigrationIndexMetadataUpdater.java @@ -18,6 +18,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.index.remote.RemoteStoreEnums.PathType; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import java.util.List; import java.util.Map; @@ -30,8 +31,6 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.index.remote.RemoteStoreUtils.determineRemoteStoreCustomMetadataDuringMigration; import static org.opensearch.index.remote.RemoteStoreUtils.getRemoteStoreRepoName; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; /** * Utils for checking and mutating cluster state during remote migration @@ -74,8 +73,9 @@ public void maybeAddRemoteIndexSettings(IndexMetadata.Builder indexMetadataBuild index ); Map remoteRepoNames = getRemoteStoreRepoName(discoveryNodes); - String segmentRepoName = remoteRepoNames.get(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY); - String tlogRepoName = remoteRepoNames.get(REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY); + String segmentRepoName = RemoteStoreNodeAttribute.getSegmentRepoName(remoteRepoNames); + String tlogRepoName = RemoteStoreNodeAttribute.getTranslogRepoName(remoteRepoNames); + assert Objects.nonNull(segmentRepoName) && Objects.nonNull(tlogRepoName) : "Remote repo names cannot be null"; Settings.Builder indexSettingsBuilder = Settings.builder().put(currentIndexSettings); updateRemoteStoreSettings(indexSettingsBuilder, segmentRepoName, tlogRepoName); diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index b1b6259e4ca18..89d06753063b7 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -12,12 +12,14 @@ import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.node.Node; import org.opensearch.repositories.blobstore.BlobStoreRepository; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -35,10 +37,28 @@ */ public class RemoteStoreNodeAttribute { - public static final String REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX = "remote_store"; + public static final List REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX = List.of("remote_store", "remote_publication"); + + // TO-DO the string constants are used only for tests and can be moved to test package + public static final String REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.state.repository"; public static final String REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.segment.repository"; public static final String REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.translog.repository"; - public static final String REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.state.repository"; + public static final String REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.routing_table.repository"; + + public static final List REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS = REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX.stream() + .map(prefix -> prefix + ".state.repository") + .collect(Collectors.toList()); + + public static final List REMOTE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEYS = REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX.stream() + .map(prefix -> prefix + ".routing_table.repository") + .collect(Collectors.toList()); + public static final List REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS = REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX.stream() + .map(prefix -> prefix + ".segment.repository") + .collect(Collectors.toList()); + public static final List REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS = REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX.stream() + .map(prefix -> prefix + ".translog.repository") + .collect(Collectors.toList()); + public static final String REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT = "remote_store.repository.%s.type"; public static final String REMOTE_STORE_REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT = "remote_store.repository.%s." + CryptoMetadata.CRYPTO_METADATA_KEY; @@ -46,18 +66,19 @@ public class RemoteStoreNodeAttribute { + "." + CryptoMetadata.SETTINGS_KEY; public static final String REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX = "remote_store.repository.%s.settings."; - public static final String REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.routing_table.repository"; - private final RepositoriesMetadata repositoriesMetadata; + public static final String REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT = "%s.repository.%s.type"; + public static final String REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT = "%s.repository.%s." + CryptoMetadata.CRYPTO_METADATA_KEY; + public static final String REPOSITORY_CRYPTO_SETTINGS_PREFIX = REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT + + "." + + CryptoMetadata.SETTINGS_KEY; + public static final String REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX = "%s.repository.%s.settings."; - public static List SUPPORTED_DATA_REPO_NAME_ATTRIBUTES = List.of( - REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, - REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY - ); + private final RepositoriesMetadata repositoriesMetadata; - public static List REMOTE_CLUSTER_PUBLICATION_REPO_NAME_ATTRIBUTES = List.of( - REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, - REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY + public static List> SUPPORTED_DATA_REPO_NAME_ATTRIBUTES = Arrays.asList( + REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS, + REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS ); /** @@ -76,8 +97,17 @@ private String validateAttributeNonNull(DiscoveryNode node, String attributeKey) return attributeValue; } - private CryptoMetadata buildCryptoMetadata(DiscoveryNode node, String repositoryName) { - String metadataKey = String.format(Locale.getDefault(), REMOTE_STORE_REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT, repositoryName); + private Tuple validateAttributeNonNull(DiscoveryNode node, List attributeKeys) { + Tuple attributeValue = getValue(node.getAttributes(), attributeKeys); + if (attributeValue == null || attributeValue.v1() == null || attributeValue.v1().isEmpty()) { + throw new IllegalStateException("joining node [" + node + "] doesn't have the node attribute [" + attributeKeys.get(0) + "]"); + } + + return attributeValue; + } + + private CryptoMetadata buildCryptoMetadata(DiscoveryNode node, String repositoryName, String prefix) { + String metadataKey = String.format(Locale.getDefault(), REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT, prefix, repositoryName); boolean isRepoEncrypted = node.getAttributes().keySet().stream().anyMatch(key -> key.startsWith(metadataKey)); if (isRepoEncrypted == false) { return null; @@ -86,11 +116,7 @@ private CryptoMetadata buildCryptoMetadata(DiscoveryNode node, String repository String keyProviderName = validateAttributeNonNull(node, metadataKey + "." + CryptoMetadata.KEY_PROVIDER_NAME_KEY); String keyProviderType = validateAttributeNonNull(node, metadataKey + "." + CryptoMetadata.KEY_PROVIDER_TYPE_KEY); - String settingsAttributeKeyPrefix = String.format( - Locale.getDefault(), - REMOTE_STORE_REPOSITORY_CRYPTO_SETTINGS_PREFIX, - repositoryName - ); + String settingsAttributeKeyPrefix = String.format(Locale.getDefault(), REPOSITORY_CRYPTO_SETTINGS_PREFIX, prefix, repositoryName); Map settingsMap = node.getAttributes() .keySet() @@ -104,10 +130,11 @@ private CryptoMetadata buildCryptoMetadata(DiscoveryNode node, String repository return new CryptoMetadata(keyProviderName, keyProviderType, settings.build()); } - private Map validateSettingsAttributesNonNull(DiscoveryNode node, String repositoryName) { + private Map validateSettingsAttributesNonNull(DiscoveryNode node, String repositoryName, String prefix) { String settingsAttributeKeyPrefix = String.format( Locale.getDefault(), - REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + prefix, repositoryName ); Map settingsMap = node.getAttributes() @@ -125,17 +152,17 @@ private Map validateSettingsAttributesNonNull(DiscoveryNode node return settingsMap; } - private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String name) { + private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String name, String prefix) { String type = validateAttributeNonNull( node, - String.format(Locale.getDefault(), REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, name) + String.format(Locale.getDefault(), REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, prefix, name) ); - Map settingsMap = validateSettingsAttributesNonNull(node, name); + Map settingsMap = validateSettingsAttributesNonNull(node, name, prefix); Settings.Builder settings = Settings.builder(); settingsMap.forEach(settings::put); - CryptoMetadata cryptoMetadata = buildCryptoMetadata(node, name); + CryptoMetadata cryptoMetadata = buildCryptoMetadata(node, name, prefix); // Repository metadata built here will always be for a system repository. settings.put(BlobStoreRepository.SYSTEM_REPOSITORY_SETTING.getKey(), true); @@ -144,53 +171,104 @@ private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String na } private RepositoriesMetadata buildRepositoriesMetadata(DiscoveryNode node) { - Set repositoryNames = getValidatedRepositoryNames(node); + Map repositoryNamesWithPrefix = getValidatedRepositoryNames(node); List repositoryMetadataList = new ArrayList<>(); - for (String repositoryName : repositoryNames) { - repositoryMetadataList.add(buildRepositoryMetadata(node, repositoryName)); + for (Map.Entry repository : repositoryNamesWithPrefix.entrySet()) { + repositoryMetadataList.add(buildRepositoryMetadata(node, repository.getKey(), repository.getValue())); } return new RepositoriesMetadata(repositoryMetadataList); } - private Set getValidatedRepositoryNames(DiscoveryNode node) { - Set repositoryNames = new HashSet<>(); - if (node.getAttributes().containsKey(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY) - || node.getAttributes().containsKey(REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY)) { - repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY)); - repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY)); - repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY)); - } else if (node.getAttributes().containsKey(REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY)) { - repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY)); + private static Tuple getValue(Map attributes, List keys) { + for (String key : keys) { + if (attributes.containsKey(key)) { + return new Tuple<>(attributes.get(key), key); + } } - if (node.getAttributes().containsKey(REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY)) { - repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY)); + return null; + } + + private Map getValidatedRepositoryNames(DiscoveryNode node) { + Set> repositoryNames = new HashSet<>(); + if (containsKey(node.getAttributes(), REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS) + || containsKey(node.getAttributes(), REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS)) { + repositoryNames.add(validateAttributeNonNull(node, REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS)); + repositoryNames.add(validateAttributeNonNull(node, REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS)); + repositoryNames.add(validateAttributeNonNull(node, REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS)); + } else if (containsKey(node.getAttributes(), REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS)) { + repositoryNames.add(validateAttributeNonNull(node, REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS)); + } + if (containsKey(node.getAttributes(), REMOTE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEYS)) { + repositoryNames.add(validateAttributeNonNull(node, REMOTE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEYS)); } - return repositoryNames; + Map repoNamesWithPrefix = new HashMap<>(); + repositoryNames.forEach(t -> { + String[] attrKeyParts = t.v2().split("\\."); + repoNamesWithPrefix.put(t.v1(), attrKeyParts[0]); + }); + + return repoNamesWithPrefix; } public static boolean isRemoteStoreAttributePresent(Settings settings) { - return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX).isEmpty() == false; + for (String prefix : REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX) { + if (settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + prefix).isEmpty() == false) { + return true; + } + } + return false; } public static boolean isRemoteDataAttributePresent(Settings settings) { - return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY).isEmpty() == false - || settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY).isEmpty() == false; + return isSegmentRepoConfigured(settings) || isTranslogRepoConfigured(settings); + } + + public static boolean isSegmentRepoConfigured(Settings settings) { + for (String prefix : REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS) { + if (settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + prefix).isEmpty() == false) { + return true; + } + } + return false; + } + + public static boolean isTranslogRepoConfigured(Settings settings) { + for (String prefix : REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS) { + if (settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + prefix).isEmpty() == false) { + return true; + } + } + return false; } public static boolean isRemoteClusterStateConfigured(Settings settings) { - return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY) - .isEmpty() == false; + for (String prefix : REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS) { + if (settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + prefix).isEmpty() == false) { + return true; + } + } + return false; } public static String getRemoteStoreSegmentRepo(Settings settings) { - return settings.get(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY); + for (String prefix : REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS) { + if (settings.get(Node.NODE_ATTRIBUTES.getKey() + prefix) != null) { + return settings.get(Node.NODE_ATTRIBUTES.getKey() + prefix); + } + } + return null; } public static String getRemoteStoreTranslogRepo(Settings settings) { - return settings.get(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY); + for (String prefix : REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS) { + if (settings.get(Node.NODE_ATTRIBUTES.getKey() + prefix) != null) { + return settings.get(Node.NODE_ATTRIBUTES.getKey() + prefix); + } + } + return null; } public static boolean isRemoteStoreClusterStateEnabled(Settings settings) { @@ -198,8 +276,12 @@ public static boolean isRemoteStoreClusterStateEnabled(Settings settings) { } private static boolean isRemoteRoutingTableAttributePresent(Settings settings) { - return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY) - .isEmpty() == false; + for (String prefix : REMOTE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEYS) { + if (settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + prefix).isEmpty() == false) { + return true; + } + } + return false; } public static boolean isRemoteRoutingTableConfigured(Settings settings) { @@ -219,21 +301,83 @@ public RepositoriesMetadata getRepositoriesMetadata() { public static Map getDataRepoNames(DiscoveryNode node) { assert remoteDataAttributesPresent(node.getAttributes()); Map dataRepoNames = new HashMap<>(); - for (String supportedRepoAttribute : SUPPORTED_DATA_REPO_NAME_ATTRIBUTES) { - dataRepoNames.put(supportedRepoAttribute, node.getAttributes().get(supportedRepoAttribute)); + for (List supportedRepoAttribute : SUPPORTED_DATA_REPO_NAME_ATTRIBUTES) { + Tuple value = getValue(node.getAttributes(), supportedRepoAttribute); + if (value != null && value.v1() != null) { + dataRepoNames.put(value.v2(), value.v1()); + } } return dataRepoNames; } private static boolean remoteDataAttributesPresent(Map nodeAttrs) { - for (String supportedRepoAttributes : SUPPORTED_DATA_REPO_NAME_ATTRIBUTES) { - if (nodeAttrs.get(supportedRepoAttributes) == null || nodeAttrs.get(supportedRepoAttributes).isEmpty()) { + for (List supportedRepoAttribute : SUPPORTED_DATA_REPO_NAME_ATTRIBUTES) { + Tuple value = getValue(nodeAttrs, supportedRepoAttribute); + if (value == null || value.v1() == null) { return false; } } return true; } + public static String getClusterStateRepoName(Map repos) { + return getValueFromAnyKey(repos, REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS); + } + + public static String getRoutingTableRepoName(Map repos) { + return getValueFromAnyKey(repos, REMOTE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEYS); + } + + public static String getSegmentRepoName(Map repos) { + return getValueFromAnyKey(repos, REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS); + } + + public static String getTranslogRepoName(Map repos) { + return getValueFromAnyKey(repos, REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS); + } + + private static String getValueFromAnyKey(Map repos, List keys) { + for (String key : keys) { + if (repos.get(key) != null) { + return repos.get(key); + } + } + return null; + } + + public static String getClusterStateRepoName(Settings settings) { + return getValueFromAnyKey(settings, REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS); + } + + public static String getRoutingTableRepoName(Settings settings) { + return getValueFromAnyKey(settings, REMOTE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEYS); + } + + private static String getValueFromAnyKey(Settings settings, List keys) { + for (String key : keys) { + if (settings.get(Node.NODE_ATTRIBUTES.getKey() + key) != null) { + return settings.get(Node.NODE_ATTRIBUTES.getKey() + key); + } + } + return null; + } + + public static boolean isClusterStateRepoConfigured(Map attributes) { + return containsKey(attributes, REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS); + } + + public static boolean isRoutingTableRepoConfigured(Map attributes) { + return containsKey(attributes, REMOTE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEYS); + } + + public static boolean isSegmentRepoConfigured(Map attributes) { + return containsKey(attributes, REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS); + } + + private static boolean containsKey(Map attributes, List keys) { + return keys.stream().filter(k -> attributes.containsKey(k)).findFirst().isPresent(); + } + @Override public int hashCode() { // The hashCode is generated by computing the hash of all the repositoryMetadata present in diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java index 98fcad0e6c496..f5b372ddd9b80 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java @@ -21,7 +21,6 @@ import org.opensearch.common.util.concurrent.AbstractAsyncTask; import org.opensearch.core.action.ActionListener; import org.opensearch.indices.RemoteStoreSettings; -import org.opensearch.node.Node; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; import org.opensearch.repositories.blobstore.BlobStoreRepository; @@ -42,6 +41,8 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.getRemoteStoreSegmentRepo; + /** * Service for managing pinned timestamps in a remote store. * This service handles pinning and unpinning of timestamps, as well as periodic updates of the pinned timestamps set. @@ -86,9 +87,7 @@ public void start() { } private static BlobContainer validateAndCreateBlobContainer(Settings settings, RepositoriesService repositoriesService) { - final String remoteStoreRepo = settings.get( - Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY - ); + final String remoteStoreRepo = getRemoteStoreSegmentRepo(settings); assert remoteStoreRepo != null : "Remote Segment Store repository is not configured"; final Repository repository = repositoriesService.repository(remoteStoreRepo); assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index c1305fa563b16..a82c05dab0b44 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -2383,11 +2383,23 @@ private List findMatchingShardPaths(String indexId, Map findHighestGenerationShardPaths(List matchingShardPaths) { - return matchingShardPaths.stream() - .map(s -> s.split("\\" + SnapshotShardPaths.DELIMITER)) - .sorted((a, b) -> Integer.parseInt(b[2]) - Integer.parseInt(a[2])) - .map(parts -> String.join(SnapshotShardPaths.DELIMITER, parts)) - .findFirst(); + if (matchingShardPaths.isEmpty()) { + return Optional.empty(); + } + + int maxGen = Integer.MIN_VALUE; + String maxGenShardPath = null; + + for (String shardPath : matchingShardPaths) { + String[] parts = shardPath.split("\\" + SnapshotShardPaths.DELIMITER); + int shardCount = Integer.parseInt(parts[parts.length - 3]); + if (shardCount > maxGen) { + maxGen = shardCount; + maxGenShardPath = shardPath; + } + } + assert maxGenShardPath != null : "Valid maxGenShardPath should be present"; + return Optional.of(maxGenShardPath); } /** @@ -2625,22 +2637,28 @@ public void finalizeSnapshot( * on account of new indexes by same index name being snapshotted that exists already in the repository's snapshots. */ private void cleanupRedundantSnapshotShardPaths(Set updatedShardPathsIndexIds) { - Set updatedIndexIds = updatedShardPathsIndexIds.stream() - .map(s -> getIndexId(s.split("\\" + SnapshotShardPaths.DELIMITER)[0])) - .collect(Collectors.toSet()); - Set indexIdShardPaths = getSnapshotShardPaths().keySet(); - List staleShardPaths = indexIdShardPaths.stream().filter(s -> updatedShardPathsIndexIds.contains(s) == false).filter(s -> { - String indexId = getIndexId(s.split("\\" + SnapshotShardPaths.DELIMITER)[0]); - return updatedIndexIds.contains(indexId); - }).collect(Collectors.toList()); try { + Set updatedIndexIds = updatedShardPathsIndexIds.stream() + .map(s -> getIndexId(s.split("\\" + SnapshotShardPaths.DELIMITER)[0])) + .collect(Collectors.toSet()); + logger.debug(new ParameterizedMessage("updatedIndexIds={}", updatedIndexIds)); + Set indexIdShardPaths = getSnapshotShardPaths().keySet(); + logger.debug(new ParameterizedMessage("indexIdShardPaths={}", indexIdShardPaths)); + List staleShardPaths = indexIdShardPaths.stream() + .filter(s -> updatedShardPathsIndexIds.contains(s) == false) + .filter(s -> { + String indexId = getIndexId(s.split("\\" + SnapshotShardPaths.DELIMITER)[0]); + return updatedIndexIds.contains(indexId); + }) + .collect(Collectors.toList()); + logger.debug(new ParameterizedMessage("staleShardPaths={}", staleShardPaths)); deleteFromContainer(snapshotShardPathBlobContainer(), staleShardPaths); - } catch (IOException e) { + } catch (Exception e) { logger.warn( new ParameterizedMessage( - "Repository [{}] Exception during snapshot stale index deletion {}", + "Repository [{}] Exception during snapshot stale index deletion for updatedIndexIds {}", metadata.name(), - staleShardPaths + updatedShardPathsIndexIds ), e ); diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotShardPaths.java b/server/src/main/java/org/opensearch/snapshots/SnapshotShardPaths.java index 878c2baba4ce9..dd0b67ca9bfaa 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotShardPaths.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotShardPaths.java @@ -92,18 +92,25 @@ public static SnapshotShardPaths fromXContent(XContentParser ignored) { * Parses a shard path string and extracts relevant shard information. * * @param shardPath The shard path string to parse. Expected format is: - * [index_id]#[index_name]#[shard_count]#[path_type_code]#[path_hash_algorithm_code] + * snapshot_path_[index_id].[index_name].[shard_count].[path_type_code].[path_hash_algorithm_code] * @return A {@link ShardInfo} object containing the parsed index ID and shard count. * @throws IllegalArgumentException if the shard path format is invalid or cannot be parsed. */ public static ShardInfo parseShardPath(String shardPath) { String[] parts = shardPath.split("\\" + SnapshotShardPaths.DELIMITER); - if (parts.length != 5) { + int len = parts.length; + if (len < 5) { throw new IllegalArgumentException("Invalid shard path format: " + shardPath); } try { - IndexId indexId = new IndexId(parts[1], getIndexId(parts[0]), Integer.parseInt(parts[3])); - int shardCount = Integer.parseInt(parts[2]); + String indexName = shardPath.substring( + // First separator after index id + shardPath.indexOf(DELIMITER) + 1, + // Since we know there are exactly 3 fields at the end + shardPath.lastIndexOf(DELIMITER, shardPath.lastIndexOf(DELIMITER, shardPath.lastIndexOf(DELIMITER) - 1) - 1) + ); + IndexId indexId = new IndexId(indexName, getIndexId(parts[0]), Integer.parseInt(parts[len - 2])); + int shardCount = Integer.parseInt(parts[len - 3]); return new ShardInfo(indexId, shardCount); } catch (NumberFormatException e) { throw new IllegalArgumentException("Invalid shard path format: " + shardPath, e); diff --git a/server/src/main/resources/org/opensearch/bootstrap/test-framework.policy b/server/src/main/resources/org/opensearch/bootstrap/test-framework.policy index 19f8adbe003ca..c62adda511140 100644 --- a/server/src/main/resources/org/opensearch/bootstrap/test-framework.policy +++ b/server/src/main/resources/org/opensearch/bootstrap/test-framework.policy @@ -45,6 +45,13 @@ grant codeBase "${codebase.mockito-core}" { permission java.lang.RuntimePermission "accessDeclaredMembers"; permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; permission java.lang.RuntimePermission "getClassLoader"; + + permission java.lang.RuntimePermission "createClassLoader"; + permission java.lang.RuntimePermission "net.bytebuddy.createJavaDispatcher"; + permission java.lang.reflect.ReflectPermission "newProxyInPackage.net.bytebuddy.utility"; + permission java.lang.reflect.ReflectPermission "newProxyInPackage.net.bytebuddy.dynamic.loading"; + permission java.lang.reflect.ReflectPermission "newProxyInPackage.net.bytebuddy.description.type"; + permission java.lang.reflect.ReflectPermission "newProxyInPackage.net.bytebuddy.description.method"; }; grant codeBase "${codebase.objenesis}" { diff --git a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java index 525a53f3e6158..6550ed39e8042 100644 --- a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java +++ b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java @@ -99,7 +99,7 @@ public void testRemoteStoreRedactionInToString() { roles, Version.CURRENT ); - assertFalse(node.toString().contains(RemoteStoreNodeAttribute.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX)); + assertFalse(node.toString().contains(RemoteStoreNodeAttribute.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX.get(0))); } public void testDiscoveryNodeIsCreatedWithHostFromInetAddress() throws Exception { diff --git a/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java index 0a2435553b19e..a5403ef81481f 100644 --- a/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java @@ -50,6 +50,8 @@ import java.net.InetAddress; import java.util.Arrays; import java.util.Collections; +import java.util.List; +import java.util.Objects; public class IpFieldTypeTests extends FieldTypeTestCase { @@ -76,7 +78,7 @@ public void testValueForSearch() { } public void testTermQuery() { - MappedFieldType ft = new IpFieldMapper.IpFieldType("field"); + MappedFieldType ft = new IpFieldMapper.IpFieldType("field", true, false, true, null, Collections.emptyMap()); String ip = "2001:db8::2:1"; @@ -104,20 +106,94 @@ public void testTermQuery() { String prefix = ip + "/64"; query = InetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 64); - assertEquals(query, ft.termQuery(prefix, null)); + assertEquals( + new IndexOrDocValuesQuery( + query, + SortedSetDocValuesField.newSlowRangeQuery( + "field", + ipToByteRef("2001:db8:0:0:0:0:0:0"), + ipToByteRef("2001:db8:0:0:ffff:ffff:ffff:ffff"), + true, + true + ) + ), + ft.termQuery(prefix, null) + ); ip = "192.168.1.7"; prefix = ip + "/16"; query = InetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 16); - assertEquals(query, ft.termQuery(prefix, null)); + assertEquals( + new IndexOrDocValuesQuery( + query, + SortedSetDocValuesField.newSlowRangeQuery( + "field", + ipToByteRef("::ffff:192.168.0.0"), + ipToByteRef("::ffff:192.168.255.255"), + true, + true + ) + ), + ft.termQuery(prefix, null) + ); MappedFieldType unsearchable = new IpFieldMapper.IpFieldType("field", false, false, false, null, Collections.emptyMap()); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("::1", null)); assertEquals("Cannot search on field [field] since it is both not indexed, and does not have doc_values enabled.", e.getMessage()); } + public void testDvOnlyTermQuery() { + IpFieldMapper.IpFieldType dvOnly = new IpFieldMapper.IpFieldType("field", false, false, true, null, Collections.emptyMap()); + String ip = "2001:db8::2:1"; + + Query query = InetAddressPoint.newExactQuery("field", InetAddresses.forString(ip)); + + assertEquals( + SortedSetDocValuesField.newSlowExactQuery("field", new BytesRef(((PointRangeQuery) query).getLowerPoint())), + dvOnly.termQuery(ip, null) + ); + + ip = "192.168.1.7"; + query = InetAddressPoint.newExactQuery("field", InetAddresses.forString(ip)); + assertEquals( + SortedSetDocValuesField.newSlowExactQuery("field", new BytesRef(((PointRangeQuery) query).getLowerPoint())), + dvOnly.termQuery(ip, null) + ); + + ip = "2001:db8::2:1"; + String prefix = ip + "/64"; + + assertEquals( + SortedSetDocValuesField.newSlowRangeQuery( + "field", + ipToByteRef("2001:db8:0:0:0:0:0:0"), + ipToByteRef("2001:db8:0:0:ffff:ffff:ffff:ffff"), + true, + true + ), + dvOnly.termQuery(prefix, null) + ); + + ip = "192.168.1.7"; + prefix = ip + "/16"; + assertEquals( + SortedSetDocValuesField.newSlowRangeQuery( + "field", + ipToByteRef("::ffff:192.168.0.0"), + ipToByteRef("::ffff:192.168.255.255"), + true, + true + ), + dvOnly.termQuery(prefix, null) + ); + } + + private static BytesRef ipToByteRef(String ipString) { + return new BytesRef(Objects.requireNonNull(InetAddresses.ipStringToBytes(ipString))); + } + public void testTermsQuery() { - MappedFieldType ft = new IpFieldMapper.IpFieldType("field"); + MappedFieldType ft = new IpFieldMapper.IpFieldType("field", true, false, false, null, Collections.emptyMap()); assertEquals( InetAddressPoint.newSetQuery("field", InetAddresses.forString("::2"), InetAddresses.forString("::5")), @@ -139,6 +215,29 @@ public void testTermsQuery() { ); } + public void testDvOnlyTermsQuery() { + MappedFieldType dvOnly = new IpFieldMapper.IpFieldType("field", false, false, true, null, Collections.emptyMap()); + + assertEquals( + SortedSetDocValuesField.newSlowSetQuery("field", List.of(ipToByteRef("::2"), ipToByteRef("::5"))), + dvOnly.termsQuery(Arrays.asList(InetAddresses.forString("::2"), InetAddresses.forString("::5")), null) + ); + assertEquals( + SortedSetDocValuesField.newSlowSetQuery("field", List.of(ipToByteRef("::2"), ipToByteRef("::5"))), + dvOnly.termsQuery(Arrays.asList("::2", "::5"), null) + ); + + // if the list includes a prefix query we fallback to a bool query + assertEquals( + new ConstantScoreQuery( + new BooleanQuery.Builder().add(dvOnly.termQuery("::42", null), Occur.SHOULD) + .add(dvOnly.termQuery("::2/16", null), Occur.SHOULD) + .build() + ), + dvOnly.termsQuery(Arrays.asList("::42", "::2/16"), null) + ); + } + public void testRangeQuery() { MappedFieldType ft = new IpFieldMapper.IpFieldType("field"); Query query = InetAddressPoint.newRangeQuery("field", InetAddresses.forString("::"), InetAddressPoint.MAX_VALUE); diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathUploaderTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathUploaderTests.java index d6519d9db8ee6..2e6523a4a64a0 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathUploaderTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathUploaderTests.java @@ -80,11 +80,16 @@ public class RemoteIndexPathUploaderTests extends OpenSearchTestCase { private final AtomicLong successCount = new AtomicLong(); private final AtomicLong failureCount = new AtomicLong(); + static final String TRANSLOG_REPO_NAME_KEY = Node.NODE_ATTRIBUTES.getKey() + + RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; + static final String SEGMENT_REPO_NAME_KEY = Node.NODE_ATTRIBUTES.getKey() + + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; + @Before public void setup() { settings = Settings.builder() - .put(RemoteIndexPathUploader.TRANSLOG_REPO_NAME_KEY, TRANSLOG_REPO_NAME) - .put(RemoteIndexPathUploader.SEGMENT_REPO_NAME_KEY, TRANSLOG_REPO_NAME) + .put(TRANSLOG_REPO_NAME_KEY, TRANSLOG_REPO_NAME) + .put(SEGMENT_REPO_NAME_KEY, TRANSLOG_REPO_NAME) .put(CLUSTER_STATE_REPO_KEY, TRANSLOG_REPO_NAME) .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) .build(); @@ -247,10 +252,7 @@ public void testInterceptWithSameRepo() throws IOException { } public void testInterceptWithDifferentRepo() throws IOException { - Settings settings = Settings.builder() - .put(this.settings) - .put(RemoteIndexPathUploader.SEGMENT_REPO_NAME_KEY, SEGMENT_REPO_NAME) - .build(); + Settings settings = Settings.builder().put(this.settings).put(SEGMENT_REPO_NAME_KEY, SEGMENT_REPO_NAME).build(); when(repositoriesService.repository(SEGMENT_REPO_NAME)).thenReturn(repository); RemoteIndexPathUploader remoteIndexPathUploader = new RemoteIndexPathUploader( threadPool, diff --git a/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java b/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java index de7f8977686a7..537a5a5739b75 100644 --- a/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java +++ b/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java @@ -32,12 +32,66 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REPOSITORY_CRYPTO_SETTINGS_PREFIX; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.test.RemoteStoreAttributeConstants.REMOTE_PUBLICATION_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.test.RemoteStoreAttributeConstants.REMOTE_PUBLICATION_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.test.RemoteStoreAttributeConstants.REMOTE_PUBLICATION_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; public class RemoteStoreNodeAttributeTests extends OpenSearchTestCase { static private final String KEY_ARN = "arn:aws:kms:us-east-1:123456789:key/6e9aa906-2cc3-4924-8ded-f385c78d9dcf"; static private final String REGION = "us-east-1"; + public void testCryptoMetadataForPublication() throws UnknownHostException { + String repoName = "remote-store-A"; + String prefix = "remote_publication"; + String repoTypeSettingKey = String.format(Locale.ROOT, REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, prefix, repoName); + String repoSettingsKey = String.format(Locale.ROOT, REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, prefix, repoName); + String repoCryptoMetadataKey = String.format(Locale.ROOT, REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT, prefix, repoName); + String repoCryptoMetadataSettingsKey = String.format(Locale.ROOT, REPOSITORY_CRYPTO_SETTINGS_PREFIX, prefix, repoName); + Map attr = Map.of( + REMOTE_PUBLICATION_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, + repoName, + REMOTE_PUBLICATION_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, + repoName, + REMOTE_PUBLICATION_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, + repoName, + repoTypeSettingKey, + "s3", + repoSettingsKey, + "abc", + repoSettingsKey + "base_path", + "xyz", + repoCryptoMetadataKey + ".key_provider_name", + "store-test", + repoCryptoMetadataKey + ".key_provider_type", + "aws-kms", + repoCryptoMetadataSettingsKey + ".region", + REGION, + repoCryptoMetadataSettingsKey + ".key_arn", + KEY_ARN + ); + DiscoveryNode node = new DiscoveryNode( + "C", + new TransportAddress(InetAddress.getByName("localhost"), 9876), + attr, + emptySet(), + Version.CURRENT + ); + + RemoteStoreNodeAttribute remoteStoreNodeAttribute = new RemoteStoreNodeAttribute(node); + assertEquals(remoteStoreNodeAttribute.getRepositoriesMetadata().repositories().size(), 1); + RepositoryMetadata repositoryMetadata = remoteStoreNodeAttribute.getRepositoriesMetadata().repositories().get(0); + Settings.Builder settings = Settings.builder(); + settings.put("region", REGION); + settings.put("key_arn", KEY_ARN); + CryptoMetadata cryptoMetadata = new CryptoMetadata("store-test", "aws-kms", settings.build()); + assertEquals(cryptoMetadata, repositoryMetadata.cryptoMetadata()); + } + public void testCryptoMetadata() throws UnknownHostException { String repoName = "remote-store-A"; String repoTypeSettingKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, repoName); diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index bfa6c65ee6ec8..55e27c7d6169b 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -89,5 +89,5 @@ dependencies { exclude group: "com.squareup.okio" } runtimeOnly "com.squareup.okio:okio:3.9.1" - runtimeOnly "org.xerial.snappy:snappy-java:1.1.10.6" + runtimeOnly "org.xerial.snappy:snappy-java:1.1.10.7" } diff --git a/test/framework/build.gradle b/test/framework/build.gradle index c65bf51c6af36..84a536fdf62c8 100644 --- a/test/framework/build.gradle +++ b/test/framework/build.gradle @@ -80,7 +80,8 @@ thirdPartyAudit.ignoreMissingClasses( 'org.apache.log4j.Priority', 'org.mockito.internal.creation.bytebuddy.inject.MockMethodDispatcher', 'org.opentest4j.AssertionFailedError', - 'net.bytebuddy.agent.ByteBuddyAgent' + 'net.bytebuddy.agent.ByteBuddyAgent', + 'net.bytebuddy.agent.Installer' ) // TODO - OpenSearch remove this violation. Issue: https://github.com/opensearch-project/OpenSearch/issues/420 thirdPartyAudit.ignoreViolations( diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index e27ff311c06f6..1ee856d3092f0 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -2838,7 +2838,7 @@ private static Settings buildRemoteStoreNodeAttributes( ); } - private static Settings buildRemoteStoreNodeAttributes( + protected static Settings buildRemoteStoreNodeAttributes( String segmentRepoName, Path segmentRepoPath, String segmentRepoType, diff --git a/test/framework/src/main/java/org/opensearch/test/RemoteStoreAttributeConstants.java b/test/framework/src/main/java/org/opensearch/test/RemoteStoreAttributeConstants.java new file mode 100644 index 0000000000000..0e7ebb9f871f6 --- /dev/null +++ b/test/framework/src/main/java/org/opensearch/test/RemoteStoreAttributeConstants.java @@ -0,0 +1,19 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.test; + +public class RemoteStoreAttributeConstants { + + public static final String REMOTE_PUBLICATION_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_publication.state.repository"; + public static final String REMOTE_PUBLICATION_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_publication.segment.repository"; + public static final String REMOTE_PUBLICATION_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_publication.translog.repository"; + public static final String REMOTE_PUBLICATION_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY = + "remote_publication.routing_table.repository"; + +}