Merge branch 'main' into remove_feature_flag_9

docs/changelog/116015.yaml

@@ -0,0 +1,6 @@
+pr: 116015
+summary: Empty percentile results no longer throw no_such_element_exception in Anomaly Detection jobs
+area: Machine Learning
+type: bug
+issues:
+ - 116013

docs/changelog/116211.yaml

@@ -0,0 +1,5 @@
+pr: 116211
+summary: Use underlying `ByteBuf` `refCount` for `ReleasableBytesReference`
+area: Network
+type: bug
+issues: []

docs/changelog/116212.yaml

@@ -0,0 +1,6 @@
+pr: 116212
+summary: Handle status code 0 in S3 CMU response
+area: Snapshot/Restore
+type: bug
+issues:
+ - 102294

docs/changelog/116219.yaml

@@ -0,0 +1,6 @@
+pr: 116219
+summary: "[apm-data] Apply lazy rollover on index template creation"
+area: Data streams
+type: bug
+issues:
+ - 116230

docs/reference/connector/docs/_connectors-create-client.asciidoc

@@ -3,7 +3,7 @@
 
 To create a new {service-name} connector:
 
-. Navigate to the *Search -> Connectors* page in the Kibana UI.
+. In the Kibana UI, navigate to the *Search -> Content -> Connectors* page from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field].
 . Follow the instructions to create a new  *{service-name}* self-managed connector.
 
 [discrete#es-connectors-{service-name-stub}-client-create-use-the-api]

docs/reference/connector/docs/_connectors-create-native.asciidoc

@@ -3,7 +3,7 @@
 
 To create a new {service-name} connector:
 
-. Navigate to the *Search -> Connectors* page in the Kibana UI.
+. In the Kibana UI, navigate to the *Search -> Content -> Connectors* page from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field].
 . Follow the instructions to create a new native  *{service-name}* connector.
 
 For additional operations, see <<es-connectors-usage>>.

docs/reference/connector/docs/connectors-hosted-tutorial-mongo.asciidoc

@@ -123,7 +123,7 @@ Once you're deployment is created, navigate to *Search*.
 The Elastic connector will sync your MongoDB data into a search-optimized Elasticsearch index.
 The first step is to create your index in the Kibana UI.
 
-In the main menu navigate to *Search > Content > Indices*.
+In the main menu, navigate to *Search > Content > Indices*, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field].
 
 Follow these steps to create your index:
 
@@ -178,7 +178,7 @@ If all the configuration details are correct, the sync will begin and documents
 
 As soon as your first documents are synced, you can view the documents and inspect the mapping for the index:
 
-* In Kibana, navigate to *Search* > *Content* > *Indices*.
+* In Kibana, navigate to *Search* > *Content* > *Indices* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field].
 * Select your index, for example `search-mongo-sample`.
 * Choose the *Documents* tab to view the synced documents.
 Expand a document to view its fields.

docs/reference/connector/docs/connectors-managed-service.asciidoc

@@ -80,7 +80,7 @@ Create a new index to be managed by the connector.
 
 Continue from above, or navigate to the following location within the {kib} UI:
 
-*Search > Content > Elasticsearch indices*
+*Search > Content > Elasticsearch indices* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field].
 
 Choose the index to configure, and then choose the *Configuration* tab.
 

docs/reference/connector/docs/connectors-usage.asciidoc

@@ -3,7 +3,7 @@
 
 This document describes operations available to <<es-native-connectors,managed connectors>> and <<es-build-connector,self-managed connectors>>, using the UI.
 
-In the Kibana UI, go to *Search > Content > Connectors* to view a summary of all your connectors and sync jobs, and to create new connectors.
+In the Kibana UI, navigate to *Search > Content > Connectors* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. Here, you can view a summary of all your connectors and sync jobs, and to create new connectors.
 
 [TIP]
 ====
@@ -24,7 +24,7 @@ Once you've chosen the data source type you'd like to sync, you'll be prompted t
 
 View and manage all Elasticsearch indices managed by connectors.
 
-In the {kib} UI, navigate to *Search > Content > Connectors* to view a list of connector indices and their attributes, including connector type health and ingestion status.
+In the {kib} UI, navigate to *Search > Content > Connectors* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. Here, you can view a list of connector indices and their attributes, including connector type health and ingestion status.
 
 Within this interface, you can choose to view the details for each existing index or delete an index.
 Or, you can <<es-connectors-usage-index-create,create a new connector index>>.
@@ -82,7 +82,7 @@ The workflow for these updates is as follows:
 
 After creating an index to be managed by a connector, you can configure automatic, recurring syncs.
 
-In the {kib} UI, navigate to *Search > Content > Connectors*.
+In the {kib} UI, navigate to *Search > Content > Connectors* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field].
 
 Choose the index to configure, and then choose the *Scheduling* tab.
 
@@ -107,7 +107,7 @@ You may want to <<es-connectors-usage-index-view,view the index details>> to see
 
 After creating the index to be managed by a connector, you can request a single sync at any time.
 
-In the {kib} UI, navigate to *Search > Content > Elasticsearch indices*.
+In the {kib} UI, navigate to *Search > Content > Elasticsearch indices* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field].
 
 Then choose the index to sync.
 
@@ -128,7 +128,7 @@ This operation requires access to Kibana and the `write` {ref}/security-privileg
 
 After a sync has started, you can cancel the sync before it completes.
 
-In the {kib} UI, navigate to *Search > Content > Elasticsearch indices*.
+In the {kib} UI, navigate to *Search > Content > Elasticsearch indices* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field].
 
 Then choose the index with the running sync.
 
@@ -144,7 +144,7 @@ This operation requires access to Kibana and the `write` {ref}/security-privileg
 
 View the index details to see a variety of information that communicate the status of the index and connector.
 
-In the {kib} UI, navigate to *Search > Content > Elasticsearch indices*.
+In the {kib} UI, navigate to *Search > Content > Elasticsearch indices* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field].
 
 Then choose the index to view.
 
@@ -192,7 +192,7 @@ This operation requires access to Kibana and the `read` {ref}/security-privilege
 View the documents the connector has synced from the data.
 Additionally view the index mappings to determine the current document schema.
 
-In the {kib} UI, navigate to *Search > Content > Elasticsearch indices*.
+In the {kib} UI, navigate to *Search > Content > Elasticsearch indices* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field].
 
 Then choose the index to view.
 
@@ -211,7 +211,7 @@ See <<es-connectors-security>> for security details.
 
 Use <<es-sync-rules,sync rules>> to limit which documents are fetched from the data source, or limit which fetched documents are stored in Elastic.
 
-In the {kib} UI, navigate to *Search > Content > Elasticsearch indices*.
+In the {kib} UI, navigate to *Search > Content > Elasticsearch indices* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field].
 
 Then choose the index to manage and choose the *Sync rules* tab.
 
@@ -220,6 +220,6 @@ Then choose the index to manage and choose the *Sync rules* tab.
 
 Use {ref}/ingest-pipeline-search.html[ingest pipelines] to transform fetched data before it is stored in Elastic.
 
-In the {kib} UI, navigate to *Search > Content > Elasticsearch indices*.
+In the {kib} UI, navigate to *Search > Content > Elasticsearch indices* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field].
 
 Then choose the index to manage and choose the *Pipelines* tab.

docs/reference/connector/docs/dls-e2e-guide.asciidoc

@@ -54,7 +54,7 @@ To build our search experience for our SharePoint Online data, we need to create
 
 Follow these steps to create a Search Application in the Kibana UI:
 
-. Navigate to *Search > Search Applications*.
+. Navigate to *Search > Search Applications* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field].
 . Select *Create*.
 . *Name* the Search Application.
 . Select the *index* used by the SharePoint Online connector.

docs/reference/connector/docs/postgresql-connector-client-tutorial.asciidoc

@@ -70,7 +70,7 @@ To complete this tutorial, you'll need to complete the following steps:
 Elastic connectors enable you to create searchable, read-only replicas of your data sources in Elasticsearch.
 The first step in setting up your self-managed connector is to create an index.
 
-In the {kibana-ref}[Kibana^] UI go to *Search > Content > Elasticsearch indices*.
+In the {kibana-ref}[Kibana^] UI, navigate to *Search > Content > Elasticsearch indices* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field].
 
 Create a new connector index:
 

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java

@@ -897,8 +897,13 @@ public void compareAndExchangeRegister(
         final var clientReference = blobStore.clientReference();
         ActionListener.run(ActionListener.releaseAfter(listener.delegateResponse((delegate, e) -> {
             logger.trace(() -> Strings.format("[%s]: compareAndExchangeRegister failed", key), e);
-            if (e instanceof AmazonS3Exception amazonS3Exception && amazonS3Exception.getStatusCode() == 404) {
-                // an uncaught 404 means that our multipart upload was aborted by a concurrent operation before we could complete it
+            if (e instanceof AmazonS3Exception amazonS3Exception
+                && (amazonS3Exception.getStatusCode() == 404
+                    || amazonS3Exception.getStatusCode() == 0 && "NoSuchUpload".equals(amazonS3Exception.getErrorCode()))) {
+                // An uncaught 404 means that our multipart upload was aborted by a concurrent operation before we could complete it.
+                // Also (rarely) S3 can start processing the request during a concurrent abort and this can result in a 200 OK with an
+                // <Error><Code>NoSuchUpload</Code>... in the response, which the SDK translates to status code 0. Either way, this means
+                // that our write encountered contention:
                 delegate.onResponse(OptionalBytesReference.MISSING);
             } else {
                 delegate.onFailure(e);

modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4MessageInboundHandler.java

@@ -14,10 +14,8 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.bytes.ReleasableBytesReference;
 import org.elasticsearch.common.network.ThreadWatchdog;
-import org.elasticsearch.core.RefCounted;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.transport.InboundPipeline;
 import org.elasticsearch.transport.Transports;
@@ -52,9 +50,8 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception
 
         final ByteBuf buffer = (ByteBuf) msg;
         Netty4TcpChannel channel = ctx.channel().attr(Netty4Transport.CHANNEL_KEY).get();
-        final BytesReference wrapped = Netty4Utils.toBytesReference(buffer);
         activityTracker.startActivity();
-        try (ReleasableBytesReference reference = new ReleasableBytesReference(wrapped, new ByteBufRefCounted(buffer))) {
+        try (ReleasableBytesReference reference = Netty4Utils.toReleasableBytesReference(buffer)) {
             pipeline.handleBytes(channel, reference);
         } finally {
             activityTracker.stopActivity();
@@ -81,35 +78,4 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception {
         super.channelInactive(ctx);
     }
 
-    private record ByteBufRefCounted(ByteBuf buffer) implements RefCounted {
-
-        @Override
-        public void incRef() {
-            buffer.retain();
-        }
-
-        @Override
-        public boolean tryIncRef() {
-            if (hasReferences() == false) {
-                return false;
-            }
-            try {
-                buffer.retain();
-            } catch (RuntimeException e) {
-                assert hasReferences() == false;
-                return false;
-            }
-            return true;
-        }
-
-        @Override
-        public boolean decRef() {
-            return buffer.release();
-        }
-
-        @Override
-        public boolean hasReferences() {
-            return buffer.refCnt() > 0;
-        }
-    }
 }

modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java

@@ -32,6 +32,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Booleans;
+import org.elasticsearch.core.RefCounted;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.http.HttpBody;
 import org.elasticsearch.transport.TransportException;
@@ -130,8 +131,51 @@ public static BytesReference toBytesReference(final ByteBuf buffer) {
         }
     }
 
+    /**
+     * Wrap Netty's {@link ByteBuf} into {@link ReleasableBytesReference} and delegating reference count to ByteBuf.
+     */
     public static ReleasableBytesReference toReleasableBytesReference(final ByteBuf buffer) {
-        return new ReleasableBytesReference(toBytesReference(buffer), buffer::release);
+        return new ReleasableBytesReference(toBytesReference(buffer), toRefCounted(buffer));
+    }
+
+    static ByteBufRefCounted toRefCounted(final ByteBuf buf) {
+        return new ByteBufRefCounted(buf);
+    }
+
+    record ByteBufRefCounted(ByteBuf buffer) implements RefCounted {
+
+        public int refCnt() {
+            return buffer.refCnt();
+        }
+
+        @Override
+        public void incRef() {
+            buffer.retain();
+        }
+
+        @Override
+        public boolean tryIncRef() {
+            if (hasReferences() == false) {
+                return false;
+            }
+            try {
+                buffer.retain();
+            } catch (RuntimeException e) {
+                assert hasReferences() == false;
+                return false;
+            }
+            return true;
+        }
+
+        @Override
+        public boolean decRef() {
+            return buffer.release();
+        }
+
+        @Override
+        public boolean hasReferences() {
+            return buffer.refCnt() > 0;
+        }
     }
 
     public static HttpBody.Full fullHttpBodyFrom(final ByteBuf buf) {

modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4UtilsTests.java

@@ -11,6 +11,7 @@
 
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.CompositeByteBuf;
+import io.netty.buffer.PooledByteBufAllocator;
 import io.netty.buffer.Unpooled;
 
 import org.apache.lucene.util.BytesRef;
@@ -68,6 +69,43 @@ public void testToChannelBuffer() throws IOException {
         assertArrayEquals(BytesReference.toBytes(ref), BytesReference.toBytes(bytesReference));
     }
 
+    /**
+     * Test that wrapped reference counted object from netty reflects correct counts in ES RefCounted
+     */
+    public void testToRefCounted() {
+        var buf = PooledByteBufAllocator.DEFAULT.buffer(1);
+        assertEquals(1, buf.refCnt());
+
+        var refCounted = Netty4Utils.toRefCounted(buf);
+        assertEquals(1, refCounted.refCnt());
+
+        buf.retain();
+        assertEquals(2, refCounted.refCnt());
+
+        refCounted.incRef();
+        assertEquals(3, refCounted.refCnt());
+        assertEquals(buf.refCnt(), refCounted.refCnt());
+
+        refCounted.decRef();
+        assertEquals(2, refCounted.refCnt());
+        assertEquals(buf.refCnt(), refCounted.refCnt());
+        assertTrue(refCounted.hasReferences());
+
+        refCounted.decRef();
+        refCounted.decRef();
+        assertFalse(refCounted.hasReferences());
+    }
+
+    /**
+     * Ensures that released ByteBuf cannot be accessed from ReleasableBytesReference
+     */
+    public void testToReleasableBytesReferenceThrowOnByteBufRelease() {
+        var buf = PooledByteBufAllocator.DEFAULT.buffer(1);
+        var relBytes = Netty4Utils.toReleasableBytesReference(buf);
+        buf.release();
+        assertThrows(AssertionError.class, () -> relBytes.get(0));
+    }
+
     private BytesReference getRandomizedBytesReference(int length) throws IOException {
         // we know bytes stream output always creates a paged bytes reference, we use it to create randomized content
         ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(length, bigarrays);