Merge remote-tracking branch 'origin/main' into unset-dynamic-mapper

Signed-off-by: Peter Nied <[email protected]>

.github/dependabot.yml

@@ -711,6 +711,14 @@ updates:
     labels:
       - "dependabot"
       - "dependencies"
+  - directory: /modules/crypto/
+    open-pull-requests-limit: 1
+    package-ecosystem: gradle
+    schedule:
+      interval: weekly
+    labels:
+      - "dependabot"
+      - "dependencies"
   - directory: /plugins/
     open-pull-requests-limit: 1
     package-ecosystem: gradle

.github/workflows/add-untriaged.yml

@@ -9,7 +9,7 @@ jobs:
     if: github.repository == 'opensearch-project/OpenSearch'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/github-script@v6
+      - uses: actions/github-script@v7
         with:
           script: |
             github.rest.issues.addLabels({

CHANGELOG.md

@@ -20,7 +20,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote cluster state] Restore global metadata from remote store when local state is lost after quorum loss ([#10404](https://github.com/opensearch-project/OpenSearch/pull/10404))
 - [AdmissionControl] Added changes for AdmissionControl Interceptor and AdmissionControlService for RateLimiting ([#9286](https://github.com/opensearch-project/OpenSearch/pull/9286))
 - GHA to verify checklist items completion in PR descriptions ([#10800](https://github.com/opensearch-project/OpenSearch/pull/10800))
+- Allow to pass the list settings through environment variables (like [], ["a", "b", "c"], ...) ([#10625](https://github.com/opensearch-project/OpenSearch/pull/10625))
 - [Remote cluster state] Restore cluster state version during remote state auto restore ([#10853](https://github.com/opensearch-project/OpenSearch/pull/10853))
+- Add back half_float BKD based sort query optimization ([#11024](https://github.com/opensearch-project/OpenSearch/pull/11024))
+- Make number of segment metadata files in remote segment store configurable ([#11329](https://github.com/opensearch-project/OpenSearch/pull/11329))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0
@@ -53,6 +56,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bumps jetty version to 9.4.52.v20230823 to fix GMS-2023-1857 ([#9822](https://github.com/opensearch-project/OpenSearch/pull/9822))
 - Bump `org.eclipse.jgit` from 6.5.0 to 6.7.0 ([#10147](https://github.com/opensearch-project/OpenSearch/pull/10147))
 - Bump OpenTelemetry from 1.30.1 to 1.31.0 ([#10617](https://github.com/opensearch-project/OpenSearch/pull/10617))
+- Bump OpenTelemetry from 1.31.0 to 1.32.0 and OpenTelemetry Semconv from 1.21.0-alpha to 1.23.1-alpha ([#11305](https://github.com/opensearch-project/OpenSearch/pull/11305))
 
 ### Changed
 - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
@@ -88,12 +92,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827))
 - Fix compression support for h2c protocol ([#4944](https://github.com/opensearch-project/OpenSearch/pull/4944))
 - Don't over-allocate in HeapBufferedAsyncEntityConsumer in order to consume the response ([#9993](https://github.com/opensearch-project/OpenSearch/pull/9993))
+- [BUG] Fix the thread context that is not properly cleared and messes up the traces ([#10873](https://github.com/opensearch-project/OpenSearch/pull/10873))
+- Handle canMatchSearchAfter for frozen context scenario ([#11249](https://github.com/opensearch-project/OpenSearch/pull/11249))
 
 ### Security
 
 ## [Unreleased 2.x]
 ### Added
 - Per request phase latency ([#10351](https://github.com/opensearch-project/OpenSearch/issues/10351))
+- Request level coordinator slow logs ([#10650](https://github.com/opensearch-project/OpenSearch/pull/10650))
 - [Remote Store] Add repository stats for remote store([#10567](https://github.com/opensearch-project/OpenSearch/pull/10567))
 - Add search query categorizer ([#10255](https://github.com/opensearch-project/OpenSearch/pull/10255))
 - Introduce ConcurrentQueryProfiler to profile query using concurrent segment search path and support concurrency during rewrite and create weight ([10352](https://github.com/opensearch-project/OpenSearch/pull/10352))
@@ -102,6 +109,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add cluster state stats ([#10670](https://github.com/opensearch-project/OpenSearch/pull/10670))
 - Adding slf4j license header to LoggerMessageFormat.java ([#11069](https://github.com/opensearch-project/OpenSearch/pull/11069))
 - [Streaming Indexing] Introduce new experimental server HTTP transport based on Netty 4 and Project Reactor (Reactor Netty) ([#9672](https://github.com/opensearch-project/OpenSearch/pull/9672))
+- Add template snippets support for field and target_field in KV ingest processor ([#10040](https://github.com/opensearch-project/OpenSearch/pull/10040))
+- Allowing pipeline processors to access index mapping info by passing ingest service ref as part of the processor factory parameters ([#10307](https://github.com/opensearch-project/OpenSearch/pull/10307))
 
 ### Dependencies
 - Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.1 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208), [#10298](https://github.com/opensearch-project/OpenSearch/pull/10298))
@@ -114,9 +123,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.codehaus.woodstox:stax2-api` from 4.2.1 to 4.2.2 ([#10639](https://github.com/opensearch-project/OpenSearch/pull/10639))
 - Bump `com.google.http-client:google-http-client` from 1.43.2 to 1.43.3 ([#10635](https://github.com/opensearch-project/OpenSearch/pull/10635))
 - Bump `com.squareup.okio:okio` from 3.5.0 to 3.6.0 ([#10637](https://github.com/opensearch-project/OpenSearch/pull/10637))
-- Bump `org.apache.logging.log4j:log4j-core` from 2.20.0 to 2.21.1 ([#10858](https://github.com/opensearch-project/OpenSearch/pull/10858), [#11000](https://github.com/opensearch-project/OpenSearch/pull/11000))
+- Bump `org.apache.logging.log4j:log4j-core` from 2.20.0 to 2.22.0 ([#10858](https://github.com/opensearch-project/OpenSearch/pull/10858), [#11000](https://github.com/opensearch-project/OpenSearch/pull/11000), [#11270](https://github.com/opensearch-project/OpenSearch/pull/11270))
 - Bump `aws-actions/configure-aws-credentials` from 2 to 4 ([#10504](https://github.com/opensearch-project/OpenSearch/pull/10504))
 - Bump `stefanzweifel/git-auto-commit-action` from 4 to 5 ([#11171](https://github.com/opensearch-project/OpenSearch/pull/11171))
+- Bump `actions/github-script` from 6 to 7 ([#11271](https://github.com/opensearch-project/OpenSearch/pull/11271))
+- Bump `jackson` and `jackson_databind` from 2.15.2 to 2.16.0 ([#11273](https://github.com/opensearch-project/OpenSearch/pull/11273))
+- Bump `netty` from 4.1.100.Final to 4.1.101.Final ([#11294](https://github.com/opensearch-project/OpenSearch/pull/11294))
+- Bump `com.avast.gradle:gradle-docker-compose-plugin` from 0.16.12 to 0.17.5 ([#10163](https://github.com/opensearch-project/OpenSearch/pull/10163))
+- Bump `com.squareup.okhttp3:okhttp` from 4.11.0 to 4.12.0 ([#10861](https://github.com/opensearch-project/OpenSearch/pull/10861))
+- Bump `com.gradle.enterprise` from 3.14.1 to 3.15.1 ([#11339](https://github.com/opensearch-project/OpenSearch/pull/11339))
+- Bump `org.apache.commons:commons-text` from 1.10.0 to 1.11.0 ([#11344](https://github.com/opensearch-project/OpenSearch/pull/11344))
 
 ### Changed
 - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840))
@@ -127,11 +143,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [BUG] Fix java.lang.SecurityException in repository-gcs plugin ([#10642](https://github.com/opensearch-project/OpenSearch/pull/10642))
 - Add telemetry tracer/metric enable flag and integ test. ([#10395](https://github.com/opensearch-project/OpenSearch/pull/10395))
 - Add instrumentation for indexing in transport bulk action and transport shard bulk action. ([#10273](https://github.com/opensearch-project/OpenSearch/pull/10273))
-- [BUG] Disable sort optimization for HALF_FLOAT ([#10999](https://github.com/opensearch-project/OpenSearch/pull/10999))
+- Disallow removing some metadata fields by remove ingest processor ([#10895](https://github.com/opensearch-project/OpenSearch/pull/10895))
+- Refactor common parts from the Rounding class into a separate 'round' package ([#11023](https://github.com/opensearch-project/OpenSearch/issues/11023))
 - Performance improvement for MultiTerm Queries on Keyword fields ([#7057](https://github.com/opensearch-project/OpenSearch/issues/7057))
+- Performance improvement for date histogram aggregations without sub-aggregations ([#11083](https://github.com/opensearch-project/OpenSearch/pull/11083))
 - Disable concurrent aggs for Diversified Sampler and Sampler aggs ([#11087](https://github.com/opensearch-project/OpenSearch/issues/11087))
 - Made leader/follower check timeout setting dynamic ([#10528](https://github.com/opensearch-project/OpenSearch/pull/10528))
-- Use iterative approach to evaluate Regex.simpleMatch ([#11060](https://github.com/opensearch-project/OpenSearch/pull/11060))
+- Improve boolean parsing performance ([#11308](https://github.com/opensearch-project/OpenSearch/pull/11308))
 
 ### Deprecated
 
@@ -146,7 +164,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix passing wrong parameter when calling newConfigurationException() in DotExpanderProcessor ([#10737](https://github.com/opensearch-project/OpenSearch/pull/10737))
 - Fix SuggestSearch.testSkipDuplicates by forceing refresh when indexing its test documents ([#11068](https://github.com/opensearch-project/OpenSearch/pull/11068))
 - Adding version condition while adding geoshape doc values to the index, to ensure backward compatibility.([#11095](https://github.com/opensearch-project/OpenSearch/pull/11095))
-- Allow unsetting of index.mapper.dynamic for better backwards compatibility ([#11193](https://github.com/opensearch-project/OpenSearch/pull/11193))
+- Delegating CachingWeightWrapper#count to internal weight object ([#10543](https://github.com/opensearch-project/OpenSearch/pull/10543))
+- Fix per request latency last phase not tracked ([#10934](https://github.com/opensearch-project/OpenSearch/pull/10934))
+- Fix for stuck update action in a bulk with `retry_on_conflict` property ([#11152](https://github.com/opensearch-project/OpenSearch/issues/11152))
+- Warn about deprecated and ignored index.mapper.dynamic index setting ([#11193](https://github.com/opensearch-project/OpenSearch/pull/11193))
 
 ### Security
 

benchmarks/src/main/java/org/opensearch/common/ArrayRoundingBenchmark.java → benchmarks/src/main/java/org/opensearch/common/round/RoundableBenchmark.java

@@ -6,7 +6,7 @@
  * compatible open source license.
  */
 
-package org.opensearch.common;
+package org.opensearch.common.round;
 
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -27,13 +27,13 @@
 @Warmup(iterations = 3, time = 1)
 @Measurement(iterations = 1, time = 1)
 @BenchmarkMode(Mode.Throughput)
-public class ArrayRoundingBenchmark {
+public class RoundableBenchmark {
 
     @Benchmark
-    public void round(Blackhole bh, Options opts) {
-        Rounding.Prepared rounding = opts.supplier.get();
+    public void floor(Blackhole bh, Options opts) {
+        Roundable roundable = opts.supplier.get();
         for (long key : opts.queries) {
-            bh.consume(rounding.round(key));
+            bh.consume(roundable.floor(key));
         }
     }
 
@@ -90,7 +90,7 @@ public static class Options {
         public String distribution;
 
         public long[] queries;
-        public Supplier<Rounding.Prepared> supplier;
+        public Supplier<Roundable> supplier;
 
         @Setup
         public void setup() {
@@ -130,10 +130,10 @@ public void setup() {
 
             switch (type) {
                 case "binary":
-                    supplier = () -> new Rounding.BinarySearchArrayRounding(values, size, null);
+                    supplier = () -> new BinarySearcher(values, size);
                     break;
                 case "linear":
-                    supplier = () -> new Rounding.BidirectionalLinearSearchArrayRounding(values, size, null);
+                    supplier = () -> new BidirectionalLinearSearcher(values, size);
                     break;
                 default:
                     throw new IllegalArgumentException("invalid type: " + type);

buildSrc/build.gradle

@@ -115,7 +115,7 @@ dependencies {
   api 'org.jdom:jdom2:2.0.6.1'
   api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}"
   api 'de.thetaphi:forbiddenapis:3.6'
-  api 'com.avast.gradle:gradle-docker-compose-plugin:0.16.12'
+  api 'com.avast.gradle:gradle-docker-compose-plugin:0.17.5'
   api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}"
   api 'org.apache.maven:maven-model:3.9.4'
   api 'com.networknt:json-schema-validator:1.0.86'

buildSrc/src/main/java/org/opensearch/gradle/testfixtures/TestFixturesPlugin.java

@@ -170,6 +170,7 @@ public void execute(Task task) {
                 .findFirst();
 
             composeExtension.getExecutable().set(dockerCompose.isPresent() ? dockerCompose.get() : "/usr/bin/docker");
+            composeExtension.getUseDockerComposeV2().set(false);
 
             tasks.named("composeUp").configure(t -> {
                 // Avoid running docker-compose tasks in parallel in CI due to some issues on certain Linux distributions

buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle

@@ -17,7 +17,7 @@ repositories {
 }
 
 dependencies {
-  implementation "org.apache.logging.log4j:log4j-core:2.21.1"
+  implementation "org.apache.logging.log4j:log4j-core:2.22.0"
 }
 
 ["0.0.1", "0.0.2"].forEach { v ->

buildSrc/version.properties

@@ -7,8 +7,8 @@ bundled_jdk = 21.0.1+12
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0
-jackson           = 2.15.2
-jackson_databind  = 2.15.2
+jackson           = 2.16.0
+jackson_databind  = 2.16.0
 snakeyaml         = 2.1
 icu4j             = 70.1
 supercsv          = 2.4.0
@@ -26,7 +26,7 @@ jakarta_annotation = 1.3.5
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.13.0
 
-netty             = 4.1.100.Final
+netty             = 4.1.101.Final
 joda              = 2.12.2
 
 # project reactor
@@ -70,5 +70,5 @@ jzlib             = 1.1.3
 resteasy          = 6.2.4.Final
 
 # opentelemetry dependencies
-opentelemetry         = 1.31.0
-opentelemetrysemconv  = 1.21.0-alpha
+opentelemetry         = 1.32.0
+opentelemetrysemconv  = 1.23.1-alpha

client/sniffer/licenses/jackson-core-2.16.0.jar.sha1

@@ -0,0 +1 @@
+899e5cf01be55fbf094ad72b2edb0c5df99111ee

distribution/docker/src/docker/config/log4j2.properties

@@ -34,6 +34,16 @@ logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling
 logger.deprecation.appenderRef.header_warning.ref = header_warning
 logger.deprecation.additivity = false
 
+appender.search_request_slowlog_json_appender.type = Console
+appender.search_request_slowlog_json_appender.name = search_request_slowlog_json_appender
+appender.search_request_slowlog_json_appender.layout.type = OpenSearchJsonLayout
+appender.search_request_slowlog_json_appender.layout.type_name = search_request_slowlog
+
+logger.search_request_slowlog_logger.name = cluster.search.request.slowlog
+logger.search_request_slowlog_logger.level = trace
+logger.search_request_slowlog_logger.appenderRef.search_request_slowlog_json_appender.ref = search_request_slowlog_json_appender
+logger.search_request_slowlog_logger.additivity = false
+
 appender.index_search_slowlog_rolling.type = Console
 appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
 appender.index_search_slowlog_rolling.layout.type = OpenSearchJsonLayout

distribution/src/config/log4j2.properties

@@ -113,6 +113,47 @@ logger.deprecation.appenderRef.deprecation_rolling_old.ref = deprecation_rolling
 logger.deprecation.appenderRef.header_warning.ref = header_warning
 logger.deprecation.additivity = false
 
+######## Search Request Slowlog JSON ####################
+appender.search_request_slowlog_json_appender.type = RollingFile
+appender.search_request_slowlog_json_appender.name = search_request_slowlog_json_appender
+appender.search_request_slowlog_json_appender.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs\
+  .cluster_name}_index_search_slowlog.json
+appender.search_request_slowlog_json_appender.filePermissions = rw-r-----
+appender.search_request_slowlog_json_appender.layout.type = OpenSearchJsonLayout
+appender.search_request_slowlog_json_appender.layout.type_name = search_request_slowlog
+appender.search_request_slowlog_json_appender.layout.opensearchmessagefields=message,took,took_millis,phase_took,total_hits,search_type,shards,source,id
+
+appender.search_request_slowlog_json_appender.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs\
+  .cluster_name}_index_search_slowlog-%i.json.gz
+appender.search_request_slowlog_json_appender.policies.type = Policies
+appender.search_request_slowlog_json_appender.policies.size.type = SizeBasedTriggeringPolicy
+appender.search_request_slowlog_json_appender.policies.size.size = 1GB
+appender.search_request_slowlog_json_appender.strategy.type = DefaultRolloverStrategy
+appender.search_request_slowlog_json_appender.strategy.max = 4
+#################################################
+######## Search Request Slowlog Log File -  old style pattern ####
+appender.search_request_slowlog_log_appender.type = RollingFile
+appender.search_request_slowlog_log_appender.name = search_request_slowlog_log_appender
+appender.search_request_slowlog_log_appender.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}\
+  _index_search_slowlog.log
+appender.search_request_slowlog_log_appender.filePermissions = rw-r-----
+appender.search_request_slowlog_log_appender.layout.type = PatternLayout
+appender.search_request_slowlog_log_appender.layout.pattern = [%d{ISO8601}][%-5p][%c{1.}] [%node_name]%marker %m%n
+
+appender.search_request_slowlog_log_appender.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}\
+  _index_search_slowlog-%i.log.gz
+appender.search_request_slowlog_log_appender.policies.type = Policies
+appender.search_request_slowlog_log_appender.policies.size.type = SizeBasedTriggeringPolicy
+appender.search_request_slowlog_log_appender.policies.size.size = 1GB
+appender.search_request_slowlog_log_appender.strategy.type = DefaultRolloverStrategy
+appender.search_request_slowlog_log_appender.strategy.max = 4
+#################################################
+logger.search_request_slowlog_logger.name = cluster.search.request.slowlog
+logger.search_request_slowlog_logger.level = trace
+logger.search_request_slowlog_logger.appenderRef.search_request_slowlog_json_appender.ref = search_request_slowlog_json_appender
+logger.search_request_slowlog_logger.appenderRef.search_request_slowlog_log_appender.ref = search_request_slowlog_log_appender
+logger.search_request_slowlog_logger.additivity = false
+
 ######## Search slowlog JSON ####################
 appender.index_search_slowlog_rolling.type = RollingFile
 appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling

distribution/tools/upgrade-cli/licenses/jackson-annotations-2.16.0.jar.sha1

@@ -0,0 +1 @@
+dc30995f7428c0a405eba9b8c619b20d2b3b9905

distribution/tools/upgrade-cli/licenses/jackson-databind-2.16.0.jar.sha1

@@ -0,0 +1 @@
+3a6b7f8ff7b30d518bbd65678e9c30cd881f19a7