Merge branch 'main' into keyword_terms

Signed-off-by: Harsha Vamsi Kalluri <[email protected]>
harshavamsi · Jul 31, 2024 · 9829e9c · 9829e9c
2 parents b3b17a6 + 0324edd
commit 9829e9c
Show file tree

Hide file tree

Showing 106 changed files with 5,039 additions and 376 deletions.
diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md
@@ -43,6 +43,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Remove LegacyESVersion.V_7_10_ Constants ([#5018](https://github.com/opensearch-project/OpenSearch/pull/5018))
 - Remove Version.V_1_ Constants ([#5021](https://github.com/opensearch-project/OpenSearch/pull/5021))
 - Remove custom Map, List and Set collection classes ([#6871](https://github.com/opensearch-project/OpenSearch/pull/6871))
+- Remove `index.store.hybrid.mmap.extensions` setting in favor of `index.store.hybrid.nio.extensions` setting ([#9392](https://github.com/opensearch-project/OpenSearch/pull/9392))
 
 ### Fixed
 - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827))

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ## [Unreleased 2.x]
 ### Added
 - Fix for hasInitiatedFetching to fix allocation explain and manual reroute APIs (([#14972](https://github.com/opensearch-project/OpenSearch/pull/14972))
+- [Workload Management] Add queryGroupId to Task ([14708](https://github.com/opensearch-project/OpenSearch/pull/14708))
+- Add setting to ignore throttling nodes for allocation of unassigned primaries in remote restore ([#14991](https://github.com/opensearch-project/OpenSearch/pull/14991))
+- [Streaming Indexing] Enhance RestClient with a new streaming API support ([#14437](https://github.com/opensearch-project/OpenSearch/pull/14437))
+- Add basic aggregation support for derived fields ([#14618](https://github.com/opensearch-project/OpenSearch/pull/14618))
+- Add ThreadContextPermission for markAsSystemContext and allow core to perform the method ([#15016](https://github.com/opensearch-project/OpenSearch/pull/15016))
+- Add ThreadContextPermission for stashAndMergeHeaders and stashWithOrigin ([#15039](https://github.com/opensearch-project/OpenSearch/pull/15039))
 
 ### Dependencies
 - Bump `org.apache.commons:commons-lang3` from 3.14.0 to 3.15.0 ([#14861](https://github.com/opensearch-project/OpenSearch/pull/14861))
@@ -14,13 +20,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `actions/github-script` from 6 to 7 ([#14997](https://github.com/opensearch-project/OpenSearch/pull/14997))
 
 ### Changed
+- Add lower limit for primary and replica batch allocators timeout ([#14979](https://github.com/opensearch-project/OpenSearch/pull/14979))
 - Updated `termsQuery` in `KeywordField` to set custom rewrite_override before executing query
 
 ### Deprecated
 
 ### Removed
 
 ### Fixed
+- Fix constraint bug which allows more primary shards than average primary shards per index ([#14908](https://github.com/opensearch-project/OpenSearch/pull/14908))
+- Fix missing value of FieldSort for unsigned_long ([#14963](https://github.com/opensearch-project/OpenSearch/pull/14963))
 
 ### Security
 

diff --git a/...dSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionBwcSetupPlugin.java b/...dSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionBwcSetupPlugin.java
@@ -158,7 +158,17 @@ private static List<DistributionProject> resolveArchiveProjects(File checkoutDir
         projects.addAll(asList("deb", "rpm"));
 
         if (bwcVersion.onOrAfter("7.0.0")) { // starting with 7.0 we bundle a jdk which means we have platform-specific archives
-            projects.addAll(asList("darwin-tar", "linux-tar", "windows-zip"));
+            projects.addAll(
+                asList(
+                    "darwin-tar",
+                    "darwin-arm64-tar",
+                    "linux-tar",
+                    "linux-arm64-tar",
+                    "linux-ppc64le-tar",
+                    "linux-s390x-tar",
+                    "windows-zip"
+                )
+            );
         } else { // prior to 7.0 we published only a single zip and tar archives
             projects.addAll(asList("zip", "tar"));
         }

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -37,8 +37,8 @@ reactor_netty     = 1.1.21
 reactor           = 3.5.19
 
 # client dependencies
-httpclient5       = 5.2.1
-httpcore5         = 5.2.2
+httpclient5       = 5.2.3
+httpcore5         = 5.2.5
 httpclient        = 4.5.14
 httpcore          = 4.4.16
 httpasyncclient   = 4.1.5

diff --git a/client/rest/build.gradle b/client/rest/build.gradle
@@ -47,10 +47,15 @@ dependencies {
   api "org.apache.httpcomponents.client5:httpclient5:${versions.httpclient5}"
   api "org.apache.httpcomponents.core5:httpcore5:${versions.httpcore5}"
   api "org.apache.httpcomponents.core5:httpcore5-h2:${versions.httpcore5}"
+  api "org.apache.httpcomponents.core5:httpcore5-reactive:${versions.httpcore5}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
   api "commons-logging:commons-logging:${versions.commonslogging}"
   api "org.slf4j:slf4j-api:${versions.slf4j}"
 
+  // reactor
+  api "io.projectreactor:reactor-core:${versions.reactor}"
+  api "org.reactivestreams:reactive-streams:${versions.reactivestreams}"
+
   testImplementation project(":client:test")
   testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testImplementation "junit:junit:${versions.junit}"
@@ -93,22 +98,52 @@ testingConventions {
   }
 }
 
-thirdPartyAudit.ignoreMissingClasses(
-  'org.conscrypt.Conscrypt',
-  'org.slf4j.impl.StaticLoggerBinder',
-  'org.slf4j.impl.StaticMDCBinder',
-  'org.slf4j.impl.StaticMarkerBinder',
-  //commons-logging optional dependencies
-  'org.apache.avalon.framework.logger.Logger',
-  'org.apache.log.Hierarchy',
-  'org.apache.log.Logger',
-  'org.apache.log4j.Level',
-  'org.apache.log4j.Logger',
-  'org.apache.log4j.Priority',
-  //commons-logging provided dependencies
-  'javax.servlet.ServletContextEvent',
-  'javax.servlet.ServletContextListener'
-)
+thirdPartyAudit {
+  ignoreMissingClasses(
+      'org.conscrypt.Conscrypt',
+      'org.slf4j.impl.StaticLoggerBinder',
+      'org.slf4j.impl.StaticMDCBinder',
+      'org.slf4j.impl.StaticMarkerBinder',
+      //commons-logging optional dependencies
+      'org.apache.avalon.framework.logger.Logger',
+      'org.apache.log.Hierarchy',
+      'org.apache.log.Logger',
+      'org.apache.log4j.Level',
+      'org.apache.log4j.Logger',
+      'org.apache.log4j.Priority',
+      //commons-logging provided dependencies
+      'javax.servlet.ServletContextEvent',
+      'javax.servlet.ServletContextListener',
+      'io.micrometer.context.ContextAccessor',
+      'io.micrometer.context.ContextRegistry',
+      'io.micrometer.context.ContextSnapshot',
+      'io.micrometer.context.ContextSnapshot$Scope',
+      'io.micrometer.context.ContextSnapshotFactory',
+      'io.micrometer.context.ContextSnapshotFactory$Builder',
+      'io.micrometer.context.ThreadLocalAccessor',
+      'io.micrometer.core.instrument.Clock',
+      'io.micrometer.core.instrument.Counter',
+      'io.micrometer.core.instrument.Counter$Builder',
+      'io.micrometer.core.instrument.DistributionSummary',
+      'io.micrometer.core.instrument.DistributionSummary$Builder',
+      'io.micrometer.core.instrument.Meter',
+      'io.micrometer.core.instrument.MeterRegistry',
+      'io.micrometer.core.instrument.Metrics',
+      'io.micrometer.core.instrument.Tag',
+      'io.micrometer.core.instrument.Tags',
+      'io.micrometer.core.instrument.Timer',
+      'io.micrometer.core.instrument.Timer$Builder',
+      'io.micrometer.core.instrument.Timer$Sample',
+      'io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics',
+      'io.micrometer.core.instrument.composite.CompositeMeterRegistry',
+      'io.micrometer.core.instrument.search.Search',
+      'reactor.blockhound.BlockHound$Builder',
+      'reactor.blockhound.integration.BlockHoundIntegration'
+   )
+   ignoreViolations(
+      'reactor.core.publisher.Traces$SharedSecretsCallSiteSupplierFactory$TracingException'
+   )
+}
 
 tasks.withType(JavaCompile) {
   // Suppressing '[options] target value 8 is obsolete and will be removed in a future release'

diff --git a/client/rest/licenses/httpclient5-5.2.1.jar.sha1 b/client/rest/licenses/httpclient5-5.2.1.jar.sha1
diff --git a/client/rest/licenses/httpclient5-5.2.3.jar.sha1 b/client/rest/licenses/httpclient5-5.2.3.jar.sha1
@@ -0,0 +1 @@
+5d753a99d299756998a08c488f2efdf9cf26198e
diff --git a/client/rest/licenses/httpcore5-5.2.2.jar.sha1 b/client/rest/licenses/httpcore5-5.2.2.jar.sha1
diff --git a/client/rest/licenses/httpcore5-5.2.5.jar.sha1 b/client/rest/licenses/httpcore5-5.2.5.jar.sha1
@@ -0,0 +1 @@
+dab1e18842971a45ca8942491ce005ab86a028d7
diff --git a/client/rest/licenses/httpcore5-h2-5.2.2.jar.sha1 b/client/rest/licenses/httpcore5-h2-5.2.2.jar.sha1
diff --git a/client/rest/licenses/httpcore5-h2-5.2.5.jar.sha1 b/client/rest/licenses/httpcore5-h2-5.2.5.jar.sha1
@@ -0,0 +1 @@
+09425df4d1365cee86a8e031a036bdca4343da4b
diff --git a/client/rest/licenses/httpcore5-reactive-5.2.5.jar.sha1 b/client/rest/licenses/httpcore5-reactive-5.2.5.jar.sha1
@@ -0,0 +1 @@
+f68949965075b957c12b4c1ef89fd4bab2a0fdb1