Merge branch 'main' into tweak-bwc-caching

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/BuildPluginFuncTest.groovy

@@ -119,7 +119,7 @@ class BuildPluginFuncTest extends AbstractGradleFuncTest {
             noticeFile.set(file("NOTICE"))
             """
         when:
-        def result = gradleRunner("assemble", "-x", "generateHistoricalFeaturesMetadata").build()
+        def result = gradleRunner("assemble", "-x", "generateClusterFeaturesMetadata").build()
         then:
         result.task(":assemble").outcome == TaskOutcome.SUCCESS
         file("build/distributions/hello-world.jar").exists()

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy

@@ -303,7 +303,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
         """
 
         when:
-        def result = gradleRunner('assemble', '--stacktrace', '-x', 'generateHistoricalFeaturesMetadata').build()
+        def result = gradleRunner('assemble', '--stacktrace', '-x', 'generateClusterFeaturesMetadata').build()
 
         then:
         result.task(":generatePom").outcome == TaskOutcome.SUCCESS

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java

@@ -14,7 +14,7 @@
 import org.elasticsearch.gradle.internal.conventions.util.Util;
 import org.elasticsearch.gradle.internal.info.BuildParameterExtension;
 import org.elasticsearch.gradle.internal.precommit.JarHellPrecommitPlugin;
-import org.elasticsearch.gradle.internal.test.HistoricalFeaturesMetadataPlugin;
+import org.elasticsearch.gradle.internal.test.ClusterFeaturesMetadataPlugin;
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin;
 import org.elasticsearch.gradle.plugin.PluginPropertiesExtension;
 import org.elasticsearch.gradle.testclusters.ElasticsearchCluster;
@@ -38,7 +38,7 @@ public void apply(Project project) {
         project.getPluginManager().apply(PluginBuildPlugin.class);
         project.getPluginManager().apply(JarHellPrecommitPlugin.class);
         project.getPluginManager().apply(ElasticsearchJavaPlugin.class);
-        project.getPluginManager().apply(HistoricalFeaturesMetadataPlugin.class);
+        project.getPluginManager().apply(ClusterFeaturesMetadataPlugin.class);
         boolean isCi = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class).isCi();
         // Clear default dependencies added by public PluginBuildPlugin as we add our
         // own project dependencies for internal builds

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BuildPlugin.java

@@ -12,7 +12,7 @@
 import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin;
 import org.elasticsearch.gradle.internal.precommit.InternalPrecommitTasks;
 import org.elasticsearch.gradle.internal.snyk.SnykDependencyMonitoringGradlePlugin;
-import org.elasticsearch.gradle.internal.test.HistoricalFeaturesMetadataPlugin;
+import org.elasticsearch.gradle.internal.test.ClusterFeaturesMetadataPlugin;
 import org.gradle.api.InvalidUserDataException;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
@@ -63,7 +63,7 @@ public void apply(final Project project) {
         project.getPluginManager().apply(ElasticsearchJavadocPlugin.class);
         project.getPluginManager().apply(DependenciesInfoPlugin.class);
         project.getPluginManager().apply(SnykDependencyMonitoringGradlePlugin.class);
-        project.getPluginManager().apply(HistoricalFeaturesMetadataPlugin.class);
+        project.getPluginManager().apply(ClusterFeaturesMetadataPlugin.class);
         InternalPrecommitTasks.create(project, true);
         configureLicenseAndNotice(project);
     }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/HistoricalFeaturesMetadataPlugin.java → build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/ClusterFeaturesMetadataPlugin.java

@@ -21,10 +21,10 @@
 import java.util.Map;
 
 /**
- * Extracts historical feature metadata into a machine-readable format for use in backward compatibility testing.
+ * Extracts cluster feature metadata into a machine-readable format for use in backward compatibility testing.
  */
-public class HistoricalFeaturesMetadataPlugin implements Plugin<Project> {
-    public static final String HISTORICAL_FEATURES_JSON = "historical-features.json";
+public class ClusterFeaturesMetadataPlugin implements Plugin<Project> {
+    public static final String CLUSTER_FEATURES_JSON = "cluster-features.json";
     public static final String FEATURES_METADATA_TYPE = "features-metadata-json";
     public static final String FEATURES_METADATA_CONFIGURATION = "featuresMetadata";
 
@@ -40,13 +40,13 @@ public void apply(Project project) {
         SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
         SourceSet mainSourceSet = sourceSets.getByName(SourceSet.MAIN_SOURCE_SET_NAME);
 
-        TaskProvider<HistoricalFeaturesMetadataTask> generateTask = project.getTasks()
-            .register("generateHistoricalFeaturesMetadata", HistoricalFeaturesMetadataTask.class, task -> {
+        TaskProvider<ClusterFeaturesMetadataTask> generateTask = project.getTasks()
+            .register("generateClusterFeaturesMetadata", ClusterFeaturesMetadataTask.class, task -> {
                 task.setClasspath(
                     featureMetadataExtractorConfig.plus(mainSourceSet.getRuntimeClasspath())
                         .plus(project.getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME))
                 );
-                task.getOutputFile().convention(project.getLayout().getBuildDirectory().file(HISTORICAL_FEATURES_JSON));
+                task.getOutputFile().convention(project.getLayout().getBuildDirectory().file(CLUSTER_FEATURES_JSON));
             });
 
         Configuration featuresMetadataArtifactConfig = project.getConfigurations().create(FEATURES_METADATA_CONFIGURATION, c -> {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/HistoricalFeaturesMetadataTask.java → build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/ClusterFeaturesMetadataTask.java

@@ -26,7 +26,7 @@
 import javax.inject.Inject;
 
 @CacheableTask
-public abstract class HistoricalFeaturesMetadataTask extends DefaultTask {
+public abstract class ClusterFeaturesMetadataTask extends DefaultTask {
     private FileCollection classpath;
 
     @OutputFile
@@ -46,30 +46,30 @@ public void setClasspath(FileCollection classpath) {
 
     @TaskAction
     public void execute() {
-        getWorkerExecutor().noIsolation().submit(HistoricalFeaturesMetadataWorkAction.class, params -> {
+        getWorkerExecutor().noIsolation().submit(ClusterFeaturesMetadataWorkAction.class, params -> {
             params.getClasspath().setFrom(getClasspath());
             params.getOutputFile().set(getOutputFile());
         });
     }
 
-    public interface HistoricalFeaturesWorkParameters extends WorkParameters {
+    public interface ClusterFeaturesWorkParameters extends WorkParameters {
         ConfigurableFileCollection getClasspath();
 
         RegularFileProperty getOutputFile();
     }
 
-    public abstract static class HistoricalFeaturesMetadataWorkAction implements WorkAction<HistoricalFeaturesWorkParameters> {
+    public abstract static class ClusterFeaturesMetadataWorkAction implements WorkAction<ClusterFeaturesWorkParameters> {
         private final ExecOperations execOperations;
 
         @Inject
-        public HistoricalFeaturesMetadataWorkAction(ExecOperations execOperations) {
+        public ClusterFeaturesMetadataWorkAction(ExecOperations execOperations) {
             this.execOperations = execOperations;
         }
 
         @Override
         public void execute() {
             LoggedExec.javaexec(execOperations, spec -> {
-                spec.getMainClass().set("org.elasticsearch.extractor.features.HistoricalFeaturesMetadataExtractor");
+                spec.getMainClass().set("org.elasticsearch.extractor.features.ClusterFeaturesMetadataExtractor");
                 spec.classpath(getParameters().getClasspath());
                 spec.args(getParameters().getOutputFile().get().getAsFile().getAbsolutePath());
             });

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java

@@ -20,8 +20,8 @@
 import org.elasticsearch.gradle.distribution.ElasticsearchDistributionTypes;
 import org.elasticsearch.gradle.internal.ElasticsearchJavaBasePlugin;
 import org.elasticsearch.gradle.internal.InternalDistributionDownloadPlugin;
+import org.elasticsearch.gradle.internal.test.ClusterFeaturesMetadataPlugin;
 import org.elasticsearch.gradle.internal.test.ErrorReportingTestListener;
-import org.elasticsearch.gradle.internal.test.HistoricalFeaturesMetadataPlugin;
 import org.elasticsearch.gradle.plugin.BasePluginBuildPlugin;
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin;
 import org.elasticsearch.gradle.plugin.PluginPropertiesExtension;
@@ -116,12 +116,12 @@ public void apply(Project project) {
         extractedPluginsConfiguration.extendsFrom(pluginsConfiguration);
         configureArtifactTransforms(project);
 
-        // Create configuration for aggregating historical feature metadata
+        // Create configuration for aggregating feature metadata
         FileCollection featureMetadataConfig = project.getConfigurations().create(FEATURES_METADATA_CONFIGURATION, c -> {
             c.setCanBeConsumed(false);
             c.setCanBeResolved(true);
             c.attributes(
-                a -> a.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, HistoricalFeaturesMetadataPlugin.FEATURES_METADATA_TYPE)
+                a -> a.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ClusterFeaturesMetadataPlugin.FEATURES_METADATA_TYPE)
             );
             c.defaultDependencies(d -> d.add(project.getDependencies().project(Map.of("path", ":server"))));
             c.withDependencies(dependencies -> {
@@ -136,10 +136,7 @@ public void apply(Project project) {
                 c.setCanBeConsumed(false);
                 c.setCanBeResolved(true);
                 c.attributes(
-                    a -> a.attribute(
-                        ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE,
-                        HistoricalFeaturesMetadataPlugin.FEATURES_METADATA_TYPE
-                    )
+                    a -> a.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ClusterFeaturesMetadataPlugin.FEATURES_METADATA_TYPE)
                 );
                 c.defaultDependencies(
                     d -> d.add(project.getDependencies().project(Map.of("path", ":distribution", "configuration", "featuresMetadata")))

distribution/build.gradle

@@ -14,7 +14,7 @@ import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.internal.ConcatFilesTask
 import org.elasticsearch.gradle.internal.DependenciesInfoPlugin
 import org.elasticsearch.gradle.internal.NoticeTask
-import org.elasticsearch.gradle.internal.test.HistoricalFeaturesMetadataPlugin
+import org.elasticsearch.gradle.internal.test.ClusterFeaturesMetadataPlugin
 
 import java.nio.file.Files
 import java.nio.file.Path
@@ -33,7 +33,7 @@ configurations {
   }
   featuresMetadata {
     attributes {
-      attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, HistoricalFeaturesMetadataPlugin.FEATURES_METADATA_TYPE)
+      attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ClusterFeaturesMetadataPlugin.FEATURES_METADATA_TYPE)
     }
   }
 }

docs/changelog/117312.yaml

@@ -0,0 +1,5 @@
+pr: 117312
+summary: Add missing `async_search` query parameters to rest-api-spec
+area: Search
+type: bug
+issues: []

modules/repository-s3/build.gradle

@@ -45,6 +45,7 @@ dependencies {
   testImplementation project(':test:fixtures:s3-fixture')
   yamlRestTestImplementation project(":test:framework")
   yamlRestTestImplementation project(':test:fixtures:s3-fixture')
+  yamlRestTestImplementation project(':test:fixtures:ec2-imds-fixture')
   yamlRestTestImplementation project(':test:fixtures:minio-fixture')
   internalClusterTestImplementation project(':test:fixtures:minio-fixture')
 

modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java

@@ -9,47 +9,71 @@
 
 package org.elasticsearch.repositories.s3;
 
+import fixture.aws.imds.Ec2ImdsHttpFixture;
 import fixture.s3.S3HttpFixture;
-import fixture.s3.S3HttpFixtureWithEC2;
 import fixture.s3.S3HttpFixtureWithSessionToken;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
 
+import org.elasticsearch.cluster.routing.Murmur3HashFunction;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.junit.ClassRule;
 import org.junit.rules.RuleChain;
 import org.junit.rules.TestRule;
 
+import java.util.Set;
+
 @ThreadLeakFilters(filters = { TestContainersThreadFilter.class })
 @ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482
 public class RepositoryS3ClientYamlTestSuiteIT extends AbstractRepositoryS3ClientYamlTestSuiteIT {
 
-    public static final S3HttpFixture s3Fixture = new S3HttpFixture();
-    public static final S3HttpFixtureWithSessionToken s3HttpFixtureWithSessionToken = new S3HttpFixtureWithSessionToken();
-    public static final S3HttpFixtureWithEC2 s3Ec2 = new S3HttpFixtureWithEC2();
+    private static final String HASHED_SEED = Integer.toString(Murmur3HashFunction.hash(System.getProperty("tests.seed")));
+    private static final String TEMPORARY_SESSION_TOKEN = "session_token-" + HASHED_SEED;
+    private static final String IMDS_ACCESS_KEY = "imds-access-key-" + HASHED_SEED;
+    private static final String IMDS_SESSION_TOKEN = "imds-session-token-" + HASHED_SEED;
+
+    private static final S3HttpFixture s3Fixture = new S3HttpFixture();
+
+    private static final S3HttpFixtureWithSessionToken s3HttpFixtureWithSessionToken = new S3HttpFixtureWithSessionToken(
+        "session_token_bucket",
+        "session_token_base_path_integration_tests",
+        System.getProperty("s3TemporaryAccessKey"),
+        TEMPORARY_SESSION_TOKEN
+    );
+
+    private static final S3HttpFixtureWithSessionToken s3HttpFixtureWithImdsSessionToken = new S3HttpFixtureWithSessionToken(
+        "ec2_bucket",
+        "ec2_base_path",
+        IMDS_ACCESS_KEY,
+        IMDS_SESSION_TOKEN
+    );
 
-    private static final String s3TemporarySessionToken = "session_token";
+    private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(IMDS_ACCESS_KEY, IMDS_SESSION_TOKEN, Set.of());
 
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-s3")
         .keystore("s3.client.integration_test_permanent.access_key", System.getProperty("s3PermanentAccessKey"))
         .keystore("s3.client.integration_test_permanent.secret_key", System.getProperty("s3PermanentSecretKey"))
         .keystore("s3.client.integration_test_temporary.access_key", System.getProperty("s3TemporaryAccessKey"))
         .keystore("s3.client.integration_test_temporary.secret_key", System.getProperty("s3TemporarySecretKey"))
-        .keystore("s3.client.integration_test_temporary.session_token", s3TemporarySessionToken)
+        .keystore("s3.client.integration_test_temporary.session_token", TEMPORARY_SESSION_TOKEN)
         .setting("s3.client.integration_test_permanent.endpoint", s3Fixture::getAddress)
         .setting("s3.client.integration_test_temporary.endpoint", s3HttpFixtureWithSessionToken::getAddress)
-        .setting("s3.client.integration_test_ec2.endpoint", s3Ec2::getAddress)
-        .systemProperty("com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", s3Ec2::getAddress)
+        .setting("s3.client.integration_test_ec2.endpoint", s3HttpFixtureWithImdsSessionToken::getAddress)
+        .systemProperty("com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", ec2ImdsHttpFixture::getAddress)
         .build();
 
     @ClassRule
-    public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(s3Ec2).around(s3HttpFixtureWithSessionToken).around(cluster);
+    public static TestRule ruleChain = RuleChain.outerRule(s3Fixture)
+        .around(s3HttpFixtureWithSessionToken)
+        .around(s3HttpFixtureWithImdsSessionToken)
+        .around(ec2ImdsHttpFixture)
+        .around(cluster);
 
     @ParametersFactory
     public static Iterable<Object[]> parameters() throws Exception {

modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsClientYamlTestSuiteIT.java

@@ -9,28 +9,48 @@
 
 package org.elasticsearch.repositories.s3;
 
-import fixture.s3.S3HttpFixtureWithECS;
+import fixture.aws.imds.Ec2ImdsHttpFixture;
+import fixture.s3.S3HttpFixtureWithSessionToken;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
+import org.elasticsearch.cluster.routing.Murmur3HashFunction;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.junit.ClassRule;
 import org.junit.rules.RuleChain;
 import org.junit.rules.TestRule;
 
+import java.util.Set;
+
 public class RepositoryS3EcsClientYamlTestSuiteIT extends AbstractRepositoryS3ClientYamlTestSuiteIT {
-    private static final S3HttpFixtureWithECS s3Ecs = new S3HttpFixtureWithECS();
+
+    private static final String HASHED_SEED = Integer.toString(Murmur3HashFunction.hash(System.getProperty("tests.seed")));
+    private static final String ECS_ACCESS_KEY = "ecs-access-key-" + HASHED_SEED;
+    private static final String ECS_SESSION_TOKEN = "ecs-session-token-" + HASHED_SEED;
+
+    private static final S3HttpFixtureWithSessionToken s3Fixture = new S3HttpFixtureWithSessionToken(
+        "ecs_bucket",
+        "ecs_base_path",
+        ECS_ACCESS_KEY,
+        ECS_SESSION_TOKEN
+    );
+
+    private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(
+        ECS_ACCESS_KEY,
+        ECS_SESSION_TOKEN,
+        Set.of("/ecs_credentials_endpoint")
+    );
 
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-s3")
-        .setting("s3.client.integration_test_ecs.endpoint", s3Ecs::getAddress)
-        .environment("AWS_CONTAINER_CREDENTIALS_FULL_URI", () -> (s3Ecs.getAddress() + "/ecs_credentials_endpoint"))
+        .setting("s3.client.integration_test_ecs.endpoint", s3Fixture::getAddress)
+        .environment("AWS_CONTAINER_CREDENTIALS_FULL_URI", () -> ec2ImdsHttpFixture.getAddress() + "/ecs_credentials_endpoint")
         .build();
 
     @ClassRule
-    public static TestRule ruleChain = RuleChain.outerRule(s3Ecs).around(cluster);
+    public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(ec2ImdsHttpFixture).around(cluster);
 
     @ParametersFactory
     public static Iterable<Object[]> parameters() throws Exception {

muted-tests.yml

@@ -118,9 +118,6 @@ tests:
 - class: org.elasticsearch.search.SearchServiceTests
   method: testParseSourceValidation
   issue: https://github.com/elastic/elasticsearch/issues/115936
-- class: org.elasticsearch.search.query.SearchQueryIT
-  method: testAllDocsQueryString
-  issue: https://github.com/elastic/elasticsearch/issues/115728
 - class: org.elasticsearch.xpack.application.connector.ConnectorIndexServiceTests
   issue: https://github.com/elastic/elasticsearch/issues/116087
 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
@@ -232,9 +229,6 @@ tests:
 - class: org.elasticsearch.xpack.inference.DefaultEndPointsIT
   method: testInferDeploysDefaultElser
   issue: https://github.com/elastic/elasticsearch/issues/114913
-- class: org.elasticsearch.xpack.esql.action.EsqlActionTaskIT
-  method: testCancelRequestWhenFailingFetchingPages
-  issue: https://github.com/elastic/elasticsearch/issues/117397
 - class: org.elasticsearch.xpack.security.operator.OperatorPrivilegesIT
   method: testEveryActionIsEitherOperatorOnlyOrNonOperator
   issue: https://github.com/elastic/elasticsearch/issues/102992