From 5f3b3801347c4df66d319f9d45ef9beb3c5d1383 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sun, 24 Nov 2024 20:37:45 +0100 Subject: [PATCH 1/5] Unmute 115728 (#117431) This is long fixed by #116264 Fixes #115728 --- muted-tests.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index edc13f3c47b78..f33ca972b7d36 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -118,9 +118,6 @@ tests: - class: org.elasticsearch.search.SearchServiceTests method: testParseSourceValidation issue: https://github.com/elastic/elasticsearch/issues/115936 -- class: org.elasticsearch.search.query.SearchQueryIT - method: testAllDocsQueryString - issue: https://github.com/elastic/elasticsearch/issues/115728 - class: org.elasticsearch.xpack.application.connector.ConnectorIndexServiceTests issue: https://github.com/elastic/elasticsearch/issues/116087 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT From e701697eb509029cc23184e430e30378d19ad714 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Sun, 24 Nov 2024 19:39:09 +0000 Subject: [PATCH 2/5] Remove historical features infrastructure (#117043) v9 can only talk to 8.18, and historical features are a maximum of 8.12, so we can remove all historical features and infrastructure. --- .../internal/BuildPluginFuncTest.groovy | 2 +- .../internal/PublishPluginFuncTest.groovy | 2 +- .../BaseInternalPluginBuildPlugin.java | 4 +- .../gradle/internal/BuildPlugin.java | 4 +- ...ava => ClusterFeaturesMetadataPlugin.java} | 12 +- ....java => ClusterFeaturesMetadataTask.java} | 12 +- .../test/rest/RestTestBasePlugin.java | 11 +- distribution/build.gradle | 4 +- .../cluster/ClusterFeatures.java | 2 +- .../elasticsearch/features/FeatureData.java | 69 +---------- .../features/FeatureService.java | 16 +-- .../features/FeatureSpecification.java | 11 -- .../features/FeatureServiceTests.java | 117 ++---------------- .../test/rest/ESRestTestCase.java | 21 +--- .../test/rest/ESRestTestFeatureService.java | 54 ++------ ... => ClusterFeaturesMetadataExtractor.java} | 30 ++--- ...lusterFeaturesMetadataExtractorTests.java} | 19 +-- .../xpack/esql/qa/rest/EsqlSpecTestCase.java | 6 +- .../xpack/esql/action/EsqlCapabilities.java | 3 - 19 files changed, 62 insertions(+), 337 deletions(-) rename build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/{HistoricalFeaturesMetadataPlugin.java => ClusterFeaturesMetadataPlugin.java} (83%) rename build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/{HistoricalFeaturesMetadataTask.java => ClusterFeaturesMetadataTask.java} (81%) rename test/metadata-extractor/src/main/java/org/elasticsearch/extractor/features/{HistoricalFeaturesMetadataExtractor.java => ClusterFeaturesMetadataExtractor.java} (69%) rename test/metadata-extractor/src/test/java/org/elasticsearch/extractor/features/{HistoricalFeaturesMetadataExtractorTests.java => ClusterFeaturesMetadataExtractorTests.java} (67%) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/BuildPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/BuildPluginFuncTest.groovy index 03b044583add0..63bb732d8a11d 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/BuildPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/BuildPluginFuncTest.groovy @@ -119,7 +119,7 @@ class BuildPluginFuncTest extends AbstractGradleFuncTest { noticeFile.set(file("NOTICE")) """ when: - def result = gradleRunner("assemble", "-x", "generateHistoricalFeaturesMetadata").build() + def result = gradleRunner("assemble", "-x", "generateClusterFeaturesMetadata").build() then: result.task(":assemble").outcome == TaskOutcome.SUCCESS file("build/distributions/hello-world.jar").exists() diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy index c7e11ba96c7dd..a199ff9d3eac5 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy @@ -303,7 +303,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { """ when: - def result = gradleRunner('assemble', '--stacktrace', '-x', 'generateHistoricalFeaturesMetadata').build() + def result = gradleRunner('assemble', '--stacktrace', '-x', 'generateClusterFeaturesMetadata').build() then: result.task(":generatePom").outcome == TaskOutcome.SUCCESS diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java index 49887dac5b6fd..2b79bc2b9173e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java @@ -14,7 +14,7 @@ import org.elasticsearch.gradle.internal.conventions.util.Util; import org.elasticsearch.gradle.internal.info.BuildParameterExtension; import org.elasticsearch.gradle.internal.precommit.JarHellPrecommitPlugin; -import org.elasticsearch.gradle.internal.test.HistoricalFeaturesMetadataPlugin; +import org.elasticsearch.gradle.internal.test.ClusterFeaturesMetadataPlugin; import org.elasticsearch.gradle.plugin.PluginBuildPlugin; import org.elasticsearch.gradle.plugin.PluginPropertiesExtension; import org.elasticsearch.gradle.testclusters.ElasticsearchCluster; @@ -38,7 +38,7 @@ public void apply(Project project) { project.getPluginManager().apply(PluginBuildPlugin.class); project.getPluginManager().apply(JarHellPrecommitPlugin.class); project.getPluginManager().apply(ElasticsearchJavaPlugin.class); - project.getPluginManager().apply(HistoricalFeaturesMetadataPlugin.class); + project.getPluginManager().apply(ClusterFeaturesMetadataPlugin.class); boolean isCi = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class).isCi(); // Clear default dependencies added by public PluginBuildPlugin as we add our // own project dependencies for internal builds diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BuildPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BuildPlugin.java index 75984e1bc6998..fb8a9858e24d5 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BuildPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BuildPlugin.java @@ -12,7 +12,7 @@ import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.elasticsearch.gradle.internal.precommit.InternalPrecommitTasks; import org.elasticsearch.gradle.internal.snyk.SnykDependencyMonitoringGradlePlugin; -import org.elasticsearch.gradle.internal.test.HistoricalFeaturesMetadataPlugin; +import org.elasticsearch.gradle.internal.test.ClusterFeaturesMetadataPlugin; import org.gradle.api.InvalidUserDataException; import org.gradle.api.Plugin; import org.gradle.api.Project; @@ -63,7 +63,7 @@ public void apply(final Project project) { project.getPluginManager().apply(ElasticsearchJavadocPlugin.class); project.getPluginManager().apply(DependenciesInfoPlugin.class); project.getPluginManager().apply(SnykDependencyMonitoringGradlePlugin.class); - project.getPluginManager().apply(HistoricalFeaturesMetadataPlugin.class); + project.getPluginManager().apply(ClusterFeaturesMetadataPlugin.class); InternalPrecommitTasks.create(project, true); configureLicenseAndNotice(project); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/HistoricalFeaturesMetadataPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/ClusterFeaturesMetadataPlugin.java similarity index 83% rename from build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/HistoricalFeaturesMetadataPlugin.java rename to build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/ClusterFeaturesMetadataPlugin.java index be972f11d4586..0c8a99fa82398 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/HistoricalFeaturesMetadataPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/ClusterFeaturesMetadataPlugin.java @@ -21,10 +21,10 @@ import java.util.Map; /** - * Extracts historical feature metadata into a machine-readable format for use in backward compatibility testing. + * Extracts cluster feature metadata into a machine-readable format for use in backward compatibility testing. */ -public class HistoricalFeaturesMetadataPlugin implements Plugin { - public static final String HISTORICAL_FEATURES_JSON = "historical-features.json"; +public class ClusterFeaturesMetadataPlugin implements Plugin { + public static final String CLUSTER_FEATURES_JSON = "cluster-features.json"; public static final String FEATURES_METADATA_TYPE = "features-metadata-json"; public static final String FEATURES_METADATA_CONFIGURATION = "featuresMetadata"; @@ -40,13 +40,13 @@ public void apply(Project project) { SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); SourceSet mainSourceSet = sourceSets.getByName(SourceSet.MAIN_SOURCE_SET_NAME); - TaskProvider generateTask = project.getTasks() - .register("generateHistoricalFeaturesMetadata", HistoricalFeaturesMetadataTask.class, task -> { + TaskProvider generateTask = project.getTasks() + .register("generateClusterFeaturesMetadata", ClusterFeaturesMetadataTask.class, task -> { task.setClasspath( featureMetadataExtractorConfig.plus(mainSourceSet.getRuntimeClasspath()) .plus(project.getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME)) ); - task.getOutputFile().convention(project.getLayout().getBuildDirectory().file(HISTORICAL_FEATURES_JSON)); + task.getOutputFile().convention(project.getLayout().getBuildDirectory().file(CLUSTER_FEATURES_JSON)); }); Configuration featuresMetadataArtifactConfig = project.getConfigurations().create(FEATURES_METADATA_CONFIGURATION, c -> { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/HistoricalFeaturesMetadataTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/ClusterFeaturesMetadataTask.java similarity index 81% rename from build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/HistoricalFeaturesMetadataTask.java rename to build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/ClusterFeaturesMetadataTask.java index a2ea7af210dfd..aa4f90e4d2367 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/HistoricalFeaturesMetadataTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/ClusterFeaturesMetadataTask.java @@ -26,7 +26,7 @@ import javax.inject.Inject; @CacheableTask -public abstract class HistoricalFeaturesMetadataTask extends DefaultTask { +public abstract class ClusterFeaturesMetadataTask extends DefaultTask { private FileCollection classpath; @OutputFile @@ -46,30 +46,30 @@ public void setClasspath(FileCollection classpath) { @TaskAction public void execute() { - getWorkerExecutor().noIsolation().submit(HistoricalFeaturesMetadataWorkAction.class, params -> { + getWorkerExecutor().noIsolation().submit(ClusterFeaturesMetadataWorkAction.class, params -> { params.getClasspath().setFrom(getClasspath()); params.getOutputFile().set(getOutputFile()); }); } - public interface HistoricalFeaturesWorkParameters extends WorkParameters { + public interface ClusterFeaturesWorkParameters extends WorkParameters { ConfigurableFileCollection getClasspath(); RegularFileProperty getOutputFile(); } - public abstract static class HistoricalFeaturesMetadataWorkAction implements WorkAction { + public abstract static class ClusterFeaturesMetadataWorkAction implements WorkAction { private final ExecOperations execOperations; @Inject - public HistoricalFeaturesMetadataWorkAction(ExecOperations execOperations) { + public ClusterFeaturesMetadataWorkAction(ExecOperations execOperations) { this.execOperations = execOperations; } @Override public void execute() { LoggedExec.javaexec(execOperations, spec -> { - spec.getMainClass().set("org.elasticsearch.extractor.features.HistoricalFeaturesMetadataExtractor"); + spec.getMainClass().set("org.elasticsearch.extractor.features.ClusterFeaturesMetadataExtractor"); spec.classpath(getParameters().getClasspath()); spec.args(getParameters().getOutputFile().get().getAsFile().getAbsolutePath()); }); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java index b44cfdac69ba7..559c0f60abc08 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java @@ -20,8 +20,8 @@ import org.elasticsearch.gradle.distribution.ElasticsearchDistributionTypes; import org.elasticsearch.gradle.internal.ElasticsearchJavaBasePlugin; import org.elasticsearch.gradle.internal.InternalDistributionDownloadPlugin; +import org.elasticsearch.gradle.internal.test.ClusterFeaturesMetadataPlugin; import org.elasticsearch.gradle.internal.test.ErrorReportingTestListener; -import org.elasticsearch.gradle.internal.test.HistoricalFeaturesMetadataPlugin; import org.elasticsearch.gradle.plugin.BasePluginBuildPlugin; import org.elasticsearch.gradle.plugin.PluginBuildPlugin; import org.elasticsearch.gradle.plugin.PluginPropertiesExtension; @@ -116,12 +116,12 @@ public void apply(Project project) { extractedPluginsConfiguration.extendsFrom(pluginsConfiguration); configureArtifactTransforms(project); - // Create configuration for aggregating historical feature metadata + // Create configuration for aggregating feature metadata FileCollection featureMetadataConfig = project.getConfigurations().create(FEATURES_METADATA_CONFIGURATION, c -> { c.setCanBeConsumed(false); c.setCanBeResolved(true); c.attributes( - a -> a.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, HistoricalFeaturesMetadataPlugin.FEATURES_METADATA_TYPE) + a -> a.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ClusterFeaturesMetadataPlugin.FEATURES_METADATA_TYPE) ); c.defaultDependencies(d -> d.add(project.getDependencies().project(Map.of("path", ":server")))); c.withDependencies(dependencies -> { @@ -136,10 +136,7 @@ public void apply(Project project) { c.setCanBeConsumed(false); c.setCanBeResolved(true); c.attributes( - a -> a.attribute( - ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, - HistoricalFeaturesMetadataPlugin.FEATURES_METADATA_TYPE - ) + a -> a.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ClusterFeaturesMetadataPlugin.FEATURES_METADATA_TYPE) ); c.defaultDependencies( d -> d.add(project.getDependencies().project(Map.of("path", ":distribution", "configuration", "featuresMetadata"))) diff --git a/distribution/build.gradle b/distribution/build.gradle index e3481706ef230..bfbf10ac85e2f 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -14,7 +14,7 @@ import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.internal.ConcatFilesTask import org.elasticsearch.gradle.internal.DependenciesInfoPlugin import org.elasticsearch.gradle.internal.NoticeTask -import org.elasticsearch.gradle.internal.test.HistoricalFeaturesMetadataPlugin +import org.elasticsearch.gradle.internal.test.ClusterFeaturesMetadataPlugin import java.nio.file.Files import java.nio.file.Path @@ -33,7 +33,7 @@ configurations { } featuresMetadata { attributes { - attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, HistoricalFeaturesMetadataPlugin.FEATURES_METADATA_TYPE) + attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ClusterFeaturesMetadataPlugin.FEATURES_METADATA_TYPE) } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterFeatures.java b/server/src/main/java/org/elasticsearch/cluster/ClusterFeatures.java index 57b90454c7e8b..ad285cbd391cd 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterFeatures.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterFeatures.java @@ -95,7 +95,7 @@ public Set allNodeFeatures() { /** * {@code true} if {@code feature} is present on all nodes in the cluster. *

- * NOTE: This should not be used directly, as it does not read historical features. + * NOTE: This should not be used directly. * Please use {@link org.elasticsearch.features.FeatureService#clusterHasFeature} instead. */ @SuppressForbidden(reason = "directly reading cluster features") diff --git a/server/src/main/java/org/elasticsearch/features/FeatureData.java b/server/src/main/java/org/elasticsearch/features/FeatureData.java index 991bb4d82be3d..65b95eae27e06 100644 --- a/server/src/main/java/org/elasticsearch/features/FeatureData.java +++ b/server/src/main/java/org/elasticsearch/features/FeatureData.java @@ -9,25 +9,19 @@ package org.elasticsearch.features; -import org.elasticsearch.Version; import org.elasticsearch.common.Strings; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.NavigableMap; import java.util.Set; -import java.util.TreeMap; - -import static org.elasticsearch.features.FeatureService.CLUSTER_FEATURES_ADDED_VERSION; /** - * Reads and consolidate features exposed by a list {@link FeatureSpecification}, grouping them into historical features and node - * features for the consumption of {@link FeatureService} + * Reads and consolidate features exposed by a list {@link FeatureSpecification}, + * grouping them together for the consumption of {@link FeatureService} */ public class FeatureData { @@ -40,19 +34,14 @@ public class FeatureData { } } - private final NavigableMap> historicalFeatures; private final Map nodeFeatures; - private FeatureData(NavigableMap> historicalFeatures, Map nodeFeatures) { - this.historicalFeatures = historicalFeatures; + private FeatureData(Map nodeFeatures) { this.nodeFeatures = nodeFeatures; } public static FeatureData createFromSpecifications(List specs) { Map allFeatures = new HashMap<>(); - - // Initialize historicalFeatures with empty version to guarantee there's a floor entry for every version - NavigableMap> historicalFeatures = new TreeMap<>(Map.of(Version.V_EMPTY, Set.of())); Map nodeFeatures = new HashMap<>(); for (FeatureSpecification spec : specs) { Set specFeatures = spec.getFeatures(); @@ -61,39 +50,6 @@ public static FeatureData createFromSpecifications(List new HashSet<>()).add(hfe.getKey().id()); - } - for (NodeFeature f : specFeatures) { FeatureSpecification existing = allFeatures.putIfAbsent(f.id(), spec); if (existing != null && existing.getClass() != spec.getClass()) { @@ -106,24 +62,7 @@ public static FeatureData createFromSpecifications(List> consolidateHistoricalFeatures( - NavigableMap> declaredHistoricalFeatures - ) { - // update each version by adding in all features from previous versions - Set featureAggregator = new HashSet<>(); - for (Map.Entry> versions : declaredHistoricalFeatures.entrySet()) { - featureAggregator.addAll(versions.getValue()); - versions.setValue(Set.copyOf(featureAggregator)); - } - - return Collections.unmodifiableNavigableMap(declaredHistoricalFeatures); - } - - public NavigableMap> getHistoricalFeatures() { - return historicalFeatures; + return new FeatureData(Map.copyOf(nodeFeatures)); } public Map getNodeFeatures() { diff --git a/server/src/main/java/org/elasticsearch/features/FeatureService.java b/server/src/main/java/org/elasticsearch/features/FeatureService.java index 1d911a75a4838..9a0ac7cafc183 100644 --- a/server/src/main/java/org/elasticsearch/features/FeatureService.java +++ b/server/src/main/java/org/elasticsearch/features/FeatureService.java @@ -9,7 +9,6 @@ package org.elasticsearch.features; -import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.logging.LogManager; @@ -17,8 +16,6 @@ import java.util.List; import java.util.Map; -import java.util.NavigableMap; -import java.util.Set; /** * Manages information on the features supported by nodes in the cluster. @@ -34,9 +31,6 @@ public class FeatureService { private static final Logger logger = LogManager.getLogger(FeatureService.class); - public static final Version CLUSTER_FEATURES_ADDED_VERSION = Version.V_8_12_0; - - private final NavigableMap> historicalFeatures; private final Map nodeFeatures; /** @@ -47,13 +41,12 @@ public FeatureService(List specs) { var featureData = FeatureData.createFromSpecifications(specs); nodeFeatures = featureData.getNodeFeatures(); - historicalFeatures = featureData.getHistoricalFeatures(); logger.info("Registered local node features {}", nodeFeatures.keySet().stream().sorted().toList()); } /** - * The non-historical features supported by this node. + * The features supported by this node. * @return Map of {@code feature-id} to its declaring {@code NodeFeature} object. */ public Map getNodeFeatures() { @@ -65,11 +58,6 @@ public Map getNodeFeatures() { */ @SuppressForbidden(reason = "We need basic feature information from cluster state") public boolean clusterHasFeature(ClusterState state, NodeFeature feature) { - if (state.clusterFeatures().clusterHasFeature(feature)) { - return true; - } - - var features = historicalFeatures.floorEntry(state.getNodes().getMinNodeVersion()); - return features != null && features.getValue().contains(feature.id()); + return state.clusterFeatures().clusterHasFeature(feature); } } diff --git a/server/src/main/java/org/elasticsearch/features/FeatureSpecification.java b/server/src/main/java/org/elasticsearch/features/FeatureSpecification.java index 03f0dd89f172e..c37bc4488f109 100644 --- a/server/src/main/java/org/elasticsearch/features/FeatureSpecification.java +++ b/server/src/main/java/org/elasticsearch/features/FeatureSpecification.java @@ -9,9 +9,6 @@ package org.elasticsearch.features; -import org.elasticsearch.Version; - -import java.util.Map; import java.util.Set; /** @@ -49,12 +46,4 @@ default Set getFeatures() { default Set getTestFeatures() { return Set.of(); } - - /** - * Returns information on historical features that should be deemed to be present on all nodes - * on or above the {@link Version} specified. - */ - default Map getHistoricalFeatures() { - return Map.of(); - } } diff --git a/server/src/test/java/org/elasticsearch/features/FeatureServiceTests.java b/server/src/test/java/org/elasticsearch/features/FeatureServiceTests.java index e103704c89649..874a6a96313e4 100644 --- a/server/src/test/java/org/elasticsearch/features/FeatureServiceTests.java +++ b/server/src/test/java/org/elasticsearch/features/FeatureServiceTests.java @@ -9,15 +9,9 @@ package org.elasticsearch.features; -import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.node.DiscoveryNodeUtils; -import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.IndexVersions; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.VersionUtils; import java.util.List; import java.util.Map; @@ -30,79 +24,36 @@ public class FeatureServiceTests extends ESTestCase { private static class TestFeatureSpecification implements FeatureSpecification { private final Set features; - private final Map historicalFeatures; - private TestFeatureSpecification(Set features, Map historicalFeatures) { + private TestFeatureSpecification(Set features) { this.features = features; - this.historicalFeatures = historicalFeatures; } @Override public Set getFeatures() { return features; } - - @Override - public Map getHistoricalFeatures() { - return historicalFeatures; - } } public void testFailsDuplicateFeatures() { // these all need to be separate classes to trigger the exception - FeatureSpecification fs1 = new TestFeatureSpecification(Set.of(new NodeFeature("f1")), Map.of()) { - }; - FeatureSpecification fs2 = new TestFeatureSpecification(Set.of(new NodeFeature("f1")), Map.of()) { - }; - FeatureSpecification hfs1 = new TestFeatureSpecification(Set.of(), Map.of(new NodeFeature("f1"), Version.V_8_11_0)) { + FeatureSpecification fs1 = new TestFeatureSpecification(Set.of(new NodeFeature("f1"))) { }; - FeatureSpecification hfs2 = new TestFeatureSpecification(Set.of(), Map.of(new NodeFeature("f1"), Version.V_8_11_0)) { + FeatureSpecification fs2 = new TestFeatureSpecification(Set.of(new NodeFeature("f1"))) { }; assertThat( expectThrows(IllegalArgumentException.class, () -> new FeatureService(List.of(fs1, fs2))).getMessage(), containsString("Duplicate feature") ); - assertThat( - expectThrows(IllegalArgumentException.class, () -> new FeatureService(List.of(hfs1, hfs2))).getMessage(), - containsString("Duplicate feature") - ); - assertThat( - expectThrows(IllegalArgumentException.class, () -> new FeatureService(List.of(fs1, hfs1))).getMessage(), - containsString("Duplicate feature") - ); - } - - public void testFailsNonHistoricalVersion() { - FeatureSpecification fs = new TestFeatureSpecification( - Set.of(), - Map.of(new NodeFeature("f1"), Version.fromId(FeatureService.CLUSTER_FEATURES_ADDED_VERSION.id + 1)) - ); - - assertThat( - expectThrows(IllegalArgumentException.class, () -> new FeatureService(List.of(fs))).getMessage(), - containsString("not a historical version") - ); - } - - public void testFailsSameRegularAndHistoricalFeature() { - FeatureSpecification fs = new TestFeatureSpecification( - Set.of(new NodeFeature("f1")), - Map.of(new NodeFeature("f1"), Version.V_8_12_0) - ); - - assertThat( - expectThrows(IllegalArgumentException.class, () -> new FeatureService(List.of(fs))).getMessage(), - containsString("cannot be declared as both a regular and historical feature") - ); } public void testGetNodeFeaturesCombinesAllSpecs() { List specs = List.of( - new TestFeatureSpecification(Set.of(new NodeFeature("f1"), new NodeFeature("f2")), Map.of()), - new TestFeatureSpecification(Set.of(new NodeFeature("f3")), Map.of()), - new TestFeatureSpecification(Set.of(new NodeFeature("f4"), new NodeFeature("f5")), Map.of()), - new TestFeatureSpecification(Set.of(), Map.of()) + new TestFeatureSpecification(Set.of(new NodeFeature("f1"), new NodeFeature("f2"))), + new TestFeatureSpecification(Set.of(new NodeFeature("f3"))), + new TestFeatureSpecification(Set.of(new NodeFeature("f4"), new NodeFeature("f5"))), + new TestFeatureSpecification(Set.of()) ); FeatureService service = new FeatureService(specs); @@ -111,10 +62,10 @@ public void testGetNodeFeaturesCombinesAllSpecs() { public void testStateHasFeatures() { List specs = List.of( - new TestFeatureSpecification(Set.of(new NodeFeature("f1"), new NodeFeature("f2")), Map.of()), - new TestFeatureSpecification(Set.of(new NodeFeature("f3")), Map.of()), - new TestFeatureSpecification(Set.of(new NodeFeature("f4"), new NodeFeature("f5")), Map.of()), - new TestFeatureSpecification(Set.of(), Map.of()) + new TestFeatureSpecification(Set.of(new NodeFeature("f1"), new NodeFeature("f2"))), + new TestFeatureSpecification(Set.of(new NodeFeature("f3"))), + new TestFeatureSpecification(Set.of(new NodeFeature("f4"), new NodeFeature("f5"))), + new TestFeatureSpecification(Set.of()) ); ClusterState state = ClusterState.builder(ClusterName.DEFAULT) @@ -130,50 +81,4 @@ public void testStateHasFeatures() { assertFalse(service.clusterHasFeature(state, new NodeFeature("nf2"))); assertFalse(service.clusterHasFeature(state, new NodeFeature("nf3"))); } - - private static ClusterState stateWithMinVersion(Version version) { - DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(); - nodes.add(DiscoveryNodeUtils.builder("node").version(version, IndexVersions.ZERO, IndexVersion.current()).build()); - for (int n = randomInt(5); n >= 0; n--) { - nodes.add( - DiscoveryNodeUtils.builder("node" + n) - .version( - VersionUtils.randomVersionBetween(random(), version, Version.CURRENT), - IndexVersions.ZERO, - IndexVersion.current() - ) - .build() - ); - } - - return ClusterState.builder(ClusterName.DEFAULT).nodes(nodes).build(); - } - - public void testStateHasHistoricalFeatures() { - NodeFeature v8_11_0 = new NodeFeature("hf_8.11.0"); - NodeFeature v8_10_0 = new NodeFeature("hf_8.10.0"); - NodeFeature v7_17_0 = new NodeFeature("hf_7.17.0"); - List specs = List.of( - new TestFeatureSpecification(Set.of(), Map.of(v8_11_0, Version.V_8_11_0)), - new TestFeatureSpecification(Set.of(), Map.of(v8_10_0, Version.V_8_10_0)), - new TestFeatureSpecification(Set.of(), Map.of(v7_17_0, Version.V_7_17_0)) - ); - - FeatureService service = new FeatureService(specs); - assertTrue(service.clusterHasFeature(stateWithMinVersion(Version.V_8_11_0), v8_11_0)); - assertTrue(service.clusterHasFeature(stateWithMinVersion(Version.V_8_11_0), v8_10_0)); - assertTrue(service.clusterHasFeature(stateWithMinVersion(Version.V_8_11_0), v7_17_0)); - - assertFalse(service.clusterHasFeature(stateWithMinVersion(Version.V_8_10_0), v8_11_0)); - assertTrue(service.clusterHasFeature(stateWithMinVersion(Version.V_8_10_0), v8_10_0)); - assertTrue(service.clusterHasFeature(stateWithMinVersion(Version.V_8_10_0), v7_17_0)); - - assertFalse(service.clusterHasFeature(stateWithMinVersion(Version.V_7_17_0), v8_11_0)); - assertFalse(service.clusterHasFeature(stateWithMinVersion(Version.V_7_17_0), v8_10_0)); - assertTrue(service.clusterHasFeature(stateWithMinVersion(Version.V_7_17_0), v7_17_0)); - - assertFalse(service.clusterHasFeature(stateWithMinVersion(Version.V_7_16_0), v8_11_0)); - assertFalse(service.clusterHasFeature(stateWithMinVersion(Version.V_7_16_0), v8_10_0)); - assertFalse(service.clusterHasFeature(stateWithMinVersion(Version.V_7_16_0), v7_17_0)); - } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index a4195a07e7621..8ca9c0709b359 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -63,7 +63,6 @@ import org.elasticsearch.core.PathUtils; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.UpdateForV9; -import org.elasticsearch.features.FeatureSpecification; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.health.node.selection.HealthNode; import org.elasticsearch.index.IndexSettings; @@ -398,29 +397,11 @@ public void initClient() throws IOException { assert nodesVersions != null; } - /** - * Override to provide additional test-only historical features. - * - * Note: This extension point cannot be used to add cluster features. The provided {@link FeatureSpecification}s - * must contain only historical features, otherwise an assertion error is thrown. - */ - protected List additionalTestOnlyHistoricalFeatures() { - return List.of(); - } - protected final TestFeatureService createTestFeatureService( Map> clusterStateFeatures, Set semanticNodeVersions ) { - // Historical features information is unavailable when using legacy test plugins - if (ESRestTestFeatureService.hasFeatureMetadata() == false) { - logger.warn( - "This test is running on the legacy test framework; historical features from production code will not be available. " - + "You need to port the test to the new test plugins in order to use historical features from production code. " - + "If this is a legacy feature used only in tests, you can add it to a test-only FeatureSpecification." - ); - } - return new ESRestTestFeatureService(additionalTestOnlyHistoricalFeatures(), semanticNodeVersions, clusterStateFeatures.values()); + return new ESRestTestFeatureService(semanticNodeVersions, clusterStateFeatures.values()); } protected static boolean has(ProductFeature feature) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestFeatureService.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestFeatureService.java index cd3406e7ddac5..9054dc6f94182 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestFeatureService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestFeatureService.java @@ -13,9 +13,6 @@ import org.elasticsearch.core.PathUtils; import org.elasticsearch.core.Strings; import org.elasticsearch.core.SuppressForbidden; -import org.elasticsearch.features.FeatureData; -import org.elasticsearch.features.FeatureSpecification; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.json.JsonXContent; @@ -25,13 +22,9 @@ import java.io.InputStream; import java.io.UncheckedIOException; import java.nio.file.Files; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; -import java.util.List; -import java.util.Map; import java.util.Set; import java.util.function.BiConsumer; import java.util.function.Predicate; @@ -48,33 +41,12 @@ class ESRestTestFeatureService implements TestFeatureService { */ private static final Pattern VERSION_FEATURE_PATTERN = Pattern.compile("gte_v(\\d+\\.\\d+\\.\\d+)"); - private final Set knownHistoricalFeatureNames; private final Collection nodeVersions; private final Collection> nodeFeatures; - private final Collection> nodeHistoricalFeatures; - ESRestTestFeatureService(List featureSpecs, Set nodeVersions, Collection> nodeFeatures) { - List specs = new ArrayList<>(featureSpecs); - if (MetadataHolder.HISTORICAL_FEATURES != null) { - specs.add(MetadataHolder.HISTORICAL_FEATURES); - } - FeatureData featureData = FeatureData.createFromSpecifications(specs); - assert featureData.getNodeFeatures().isEmpty() - : Strings.format( - "Only historical features can be injected via ESRestTestCase#additionalTestOnlyHistoricalFeatures(), rejecting %s", - featureData.getNodeFeatures().keySet() - ); - this.knownHistoricalFeatureNames = featureData.getHistoricalFeatures().lastEntry().getValue(); + ESRestTestFeatureService(Set nodeVersions, Collection> nodeFeatures) { this.nodeVersions = nodeVersions; this.nodeFeatures = nodeFeatures; - this.nodeHistoricalFeatures = nodeVersions.stream() - .map(featureData.getHistoricalFeatures()::floorEntry) - .map(Map.Entry::getValue) - .toList(); - } - - public static boolean hasFeatureMetadata() { - return MetadataHolder.HISTORICAL_FEATURES != null; } private static boolean checkCollection(Collection coll, Predicate pred, boolean any) { @@ -83,11 +55,10 @@ private static boolean checkCollection(Collection coll, Predicate pred @Override public boolean clusterHasFeature(String featureId, boolean any) { - if (checkCollection(nodeFeatures, s -> s.contains(featureId), any) - || checkCollection(nodeHistoricalFeatures, s -> s.contains(featureId), any)) { + if (checkCollection(nodeFeatures, s -> s.contains(featureId), any)) { return true; } - if (MetadataHolder.FEATURE_NAMES.contains(featureId) || knownHistoricalFeatureNames.contains(featureId)) { + if (MetadataHolder.FEATURE_NAMES.contains(featureId)) { return false; // feature known but not present } @@ -131,24 +102,20 @@ public boolean clusterHasFeature(String featureId, boolean any) { return false; } + public static boolean hasFeatureMetadata() { + return MetadataHolder.FEATURE_NAMES.isEmpty() == false; + } + private static class MetadataHolder { - private static final FeatureSpecification HISTORICAL_FEATURES; private static final Set FEATURE_NAMES; static { String metadataPath = System.getProperty("tests.features.metadata.path"); if (metadataPath == null) { FEATURE_NAMES = emptySet(); - HISTORICAL_FEATURES = null; } else { Set featureNames = new HashSet<>(); - Map historicalFeatures = new HashMap<>(); loadFeatureMetadata(metadataPath, (key, value) -> { - if (key.equals("historical_features") && value instanceof Map map) { - for (var entry : map.entrySet()) { - historicalFeatures.put(new NodeFeature((String) entry.getKey()), Version.fromString((String) entry.getValue())); - } - } if (key.equals("feature_names") && value instanceof Collection collection) { for (var entry : collection) { featureNames.add((String) entry); @@ -156,13 +123,6 @@ private static class MetadataHolder { } }); FEATURE_NAMES = Collections.unmodifiableSet(featureNames); - Map unmodifiableHistoricalFeatures = Collections.unmodifiableMap(historicalFeatures); - HISTORICAL_FEATURES = new FeatureSpecification() { - @Override - public Map getHistoricalFeatures() { - return unmodifiableHistoricalFeatures; - } - }; } } diff --git a/test/metadata-extractor/src/main/java/org/elasticsearch/extractor/features/HistoricalFeaturesMetadataExtractor.java b/test/metadata-extractor/src/main/java/org/elasticsearch/extractor/features/ClusterFeaturesMetadataExtractor.java similarity index 69% rename from test/metadata-extractor/src/main/java/org/elasticsearch/extractor/features/HistoricalFeaturesMetadataExtractor.java rename to test/metadata-extractor/src/main/java/org/elasticsearch/extractor/features/ClusterFeaturesMetadataExtractor.java index 3ffa27126fac8..3a090a1b3fadc 100644 --- a/test/metadata-extractor/src/main/java/org/elasticsearch/extractor/features/HistoricalFeaturesMetadataExtractor.java +++ b/test/metadata-extractor/src/main/java/org/elasticsearch/extractor/features/ClusterFeaturesMetadataExtractor.java @@ -9,9 +9,8 @@ package org.elasticsearch.extractor.features; -import org.elasticsearch.Version; -import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.logging.LogConfigurator; +import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.features.FeatureSpecification; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.xcontent.XContentGenerator; @@ -25,14 +24,12 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; -import java.util.HashMap; import java.util.HashSet; -import java.util.Map; import java.util.ServiceLoader; import java.util.Set; import java.util.stream.Stream; -public class HistoricalFeaturesMetadataExtractor { +public class ClusterFeaturesMetadataExtractor { private final ClassLoader classLoader; static { @@ -40,7 +37,7 @@ public class HistoricalFeaturesMetadataExtractor { LogConfigurator.configureESLogging(); } - public HistoricalFeaturesMetadataExtractor(ClassLoader classLoader) { + public ClusterFeaturesMetadataExtractor(ClassLoader classLoader) { this.classLoader = classLoader; } @@ -56,9 +53,7 @@ public static void main(String[] args) { printUsageAndExit(); } - new HistoricalFeaturesMetadataExtractor(HistoricalFeaturesMetadataExtractor.class.getClassLoader()).generateMetadataFile( - outputFile - ); + new ClusterFeaturesMetadataExtractor(ClusterFeaturesMetadataExtractor.class.getClassLoader()).generateMetadataFile(outputFile); } public void generateMetadataFile(Path outputFile) { @@ -67,13 +62,7 @@ public void generateMetadataFile(Path outputFile) { XContentGenerator generator = JsonXContent.jsonXContent.createGenerator(os) ) { generator.writeStartObject(); - extractHistoricalFeatureMetadata((historical, names) -> { - generator.writeFieldName("historical_features"); - generator.writeStartObject(); - for (Map.Entry entry : historical.entrySet()) { - generator.writeStringField(entry.getKey().id(), entry.getValue().toString()); - } - generator.writeEndObject(); + extractClusterFeaturesMetadata(names -> { generator.writeFieldName("feature_names"); generator.writeStartArray(); for (var entry : names) { @@ -87,22 +76,19 @@ public void generateMetadataFile(Path outputFile) { } } - void extractHistoricalFeatureMetadata(CheckedBiConsumer, Set, IOException> metadataConsumer) - throws IOException { - Map historicalFeatures = new HashMap<>(); + void extractClusterFeaturesMetadata(CheckedConsumer, IOException> metadataConsumer) throws IOException { Set featureNames = new HashSet<>(); ServiceLoader featureSpecLoader = ServiceLoader.load(FeatureSpecification.class, classLoader); for (FeatureSpecification featureSpecification : featureSpecLoader) { - historicalFeatures.putAll(featureSpecification.getHistoricalFeatures()); Stream.concat(featureSpecification.getFeatures().stream(), featureSpecification.getTestFeatures().stream()) .map(NodeFeature::id) .forEach(featureNames::add); } - metadataConsumer.accept(historicalFeatures, featureNames); + metadataConsumer.accept(featureNames); } private static void printUsageAndExit() { - System.err.println("Usage: HistoricalFeaturesMetadataExtractor "); + System.err.println("Usage: ClusterFeaturesMetadataExtractor "); System.exit(1); } } diff --git a/test/metadata-extractor/src/test/java/org/elasticsearch/extractor/features/HistoricalFeaturesMetadataExtractorTests.java b/test/metadata-extractor/src/test/java/org/elasticsearch/extractor/features/ClusterFeaturesMetadataExtractorTests.java similarity index 67% rename from test/metadata-extractor/src/test/java/org/elasticsearch/extractor/features/HistoricalFeaturesMetadataExtractorTests.java rename to test/metadata-extractor/src/test/java/org/elasticsearch/extractor/features/ClusterFeaturesMetadataExtractorTests.java index d810f17ae552e..af69aaff86cc5 100644 --- a/test/metadata-extractor/src/test/java/org/elasticsearch/extractor/features/HistoricalFeaturesMetadataExtractorTests.java +++ b/test/metadata-extractor/src/test/java/org/elasticsearch/extractor/features/ClusterFeaturesMetadataExtractorTests.java @@ -9,8 +9,6 @@ package org.elasticsearch.extractor.features; -import org.elasticsearch.Version; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.json.JsonXContent; @@ -21,7 +19,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Collection; -import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -29,25 +26,19 @@ import static org.elasticsearch.xcontent.XContentParserConfiguration.EMPTY; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.not; -public class HistoricalFeaturesMetadataExtractorTests extends ESTestCase { +public class ClusterFeaturesMetadataExtractorTests extends ESTestCase { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); public void testExtractHistoricalMetadata() throws IOException { - HistoricalFeaturesMetadataExtractor extractor = new HistoricalFeaturesMetadataExtractor(this.getClass().getClassLoader()); - Map nodeFeatureVersionMap = new HashMap<>(); + ClusterFeaturesMetadataExtractor extractor = new ClusterFeaturesMetadataExtractor(this.getClass().getClassLoader()); Set featureNamesSet = new HashSet<>(); - extractor.extractHistoricalFeatureMetadata((historical, names) -> { - nodeFeatureVersionMap.putAll(historical); - featureNamesSet.addAll(names); - }); - // assertThat(nodeFeatureVersionMap, not(anEmptyMap())); + extractor.extractClusterFeaturesMetadata(featureNamesSet::addAll); assertThat(featureNamesSet, not(empty())); assertThat(featureNamesSet, hasItem("test_features_enabled")); @@ -55,11 +46,7 @@ public void testExtractHistoricalMetadata() throws IOException { extractor.generateMetadataFile(outputFile); try (XContentParser parser = JsonXContent.jsonXContent.createParser(EMPTY, Files.newInputStream(outputFile))) { Map parsedMap = parser.map(); - assertThat(parsedMap, hasKey("historical_features")); assertThat(parsedMap, hasKey("feature_names")); - @SuppressWarnings("unchecked") - Map historicalFeaturesMap = (Map) (parsedMap.get("historical_features")); - nodeFeatureVersionMap.forEach((key, value) -> assertThat(historicalFeaturesMap, hasEntry(key.id(), value.toString()))); @SuppressWarnings("unchecked") Collection featureNamesList = (Collection) (parsedMap.get("feature_names")); diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java index 6ebf05755ef5e..265d9f7bd8cd5 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java @@ -51,7 +51,6 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.LongStream; -import java.util.stream.Stream; import static org.apache.lucene.geo.GeoEncodingUtils.decodeLatitude; import static org.apache.lucene.geo.GeoEncodingUtils.decodeLongitude; @@ -207,10 +206,7 @@ protected static void checkCapabilities(RestClient client, TestFeatureService te } } - var features = Stream.concat( - new EsqlFeatures().getFeatures().stream(), - new EsqlFeatures().getHistoricalFeatures().keySet().stream() - ).map(NodeFeature::id).collect(Collectors.toSet()); + var features = new EsqlFeatures().getFeatures().stream().map(NodeFeature::id).collect(Collectors.toSet()); for (String feature : testCase.requiredCapabilities) { var esqlFeature = "esql." + feature; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index a186b784e95fb..d675f772b5a3b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -567,9 +567,6 @@ public static Set capabilities(boolean all) { for (NodeFeature feature : new EsqlFeatures().getFeatures()) { caps.add(cap(feature)); } - for (NodeFeature feature : new EsqlFeatures().getHistoricalFeatures().keySet()) { - caps.add(cap(feature)); - } return Set.copyOf(caps); } From d22a946fa736f4b08b9e0ec655afff4c5446c4d4 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Sun, 24 Nov 2024 17:39:20 -0800 Subject: [PATCH 3/5] Fix testCancelRequestWhenFailingFetchingPages (#117437) Each data-node request involves two exchange sinks: an external one for fetching pages from the coordinator and an internal one for node-level reduction. Currently, the test selects one of these sinks randomly, leading to assertion failures. This update ensures the test consistently selects the external exchange sink. Closes #117397 --- muted-tests.yml | 3 --- .../xpack/esql/action/EsqlActionTaskIT.java | 24 +++++++++++++------ 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index f33ca972b7d36..0d2e6b991a5c3 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -229,9 +229,6 @@ tests: - class: org.elasticsearch.xpack.inference.DefaultEndPointsIT method: testInferDeploysDefaultElser issue: https://github.com/elastic/elasticsearch/issues/114913 -- class: org.elasticsearch.xpack.esql.action.EsqlActionTaskIT - method: testCancelRequestWhenFailingFetchingPages - issue: https://github.com/elastic/elasticsearch/issues/117397 - class: org.elasticsearch.xpack.security.operator.OperatorPrivilegesIT method: testEveryActionIsEitherOperatorOnlyOrNonOperator issue: https://github.com/elastic/elasticsearch/issues/102992 diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java index 460ab0f5b8b38..56453a291ea81 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java @@ -392,12 +392,13 @@ protected void doRun() throws Exception { .get(); ensureYellowAndNoInitializingShards("test"); request.query("FROM test | LIMIT 10"); - request.pragmas(randomPragmas()); + QueryPragmas pragmas = randomPragmas(); + request.pragmas(pragmas); PlainActionFuture future = new PlainActionFuture<>(); client.execute(EsqlQueryAction.INSTANCE, request, future); ExchangeService exchangeService = internalCluster().getInstance(ExchangeService.class, dataNode); - boolean waitedForPages; - final String sessionId; + final boolean waitedForPages; + final String exchangeId; try { List foundTasks = new ArrayList<>(); assertBusy(() -> { @@ -411,13 +412,22 @@ protected void doRun() throws Exception { assertThat(tasks, hasSize(1)); foundTasks.addAll(tasks); }); - sessionId = foundTasks.get(0).taskId().toString(); + final String sessionId = foundTasks.get(0).taskId().toString(); assertTrue(fetchingStarted.await(1, TimeUnit.MINUTES)); - String exchangeId = exchangeService.sinkKeys().stream().filter(s -> s.startsWith(sessionId)).findFirst().get(); + List sinkKeys = exchangeService.sinkKeys() + .stream() + .filter( + s -> s.startsWith(sessionId) + // exclude the node-level reduction sink + && s.endsWith("[n]") == false + ) + .toList(); + assertThat(sinkKeys.toString(), sinkKeys.size(), equalTo(1)); + exchangeId = sinkKeys.get(0); ExchangeSinkHandler exchangeSink = exchangeService.getSinkHandler(exchangeId); waitedForPages = randomBoolean(); if (waitedForPages) { - // do not fail exchange requests until we have some pages + // do not fail exchange requests until we have some pages. assertBusy(() -> assertThat(exchangeSink.bufferSize(), greaterThan(0))); } } finally { @@ -429,7 +439,7 @@ protected void doRun() throws Exception { // As a result, the exchange sinks on data-nodes won't be removed until the inactive_timeout elapses, which is // longer than the assertBusy timeout. if (waitedForPages == false) { - exchangeService.finishSinkHandler(sessionId, failure); + exchangeService.finishSinkHandler(exchangeId, failure); } } finally { transportService.clearAllRules(); From 2f8bb0b23ce6070335fb750d9e76265f558ea3a9 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 25 Nov 2024 11:43:36 +0400 Subject: [PATCH 4/5] Add missing async_search query parameters to rest-api-spec (#117312) --- docs/changelog/117312.yaml | 5 +++++ .../rest-api-spec/api/async_search.submit.json | 15 +++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 docs/changelog/117312.yaml diff --git a/docs/changelog/117312.yaml b/docs/changelog/117312.yaml new file mode 100644 index 0000000000000..302b91388ef2b --- /dev/null +++ b/docs/changelog/117312.yaml @@ -0,0 +1,5 @@ +pr: 117312 +summary: Add missing `async_search` query parameters to rest-api-spec +area: Search +type: bug +issues: [] diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/async_search.submit.json b/rest-api-spec/src/main/resources/rest-api-spec/api/async_search.submit.json index 5cd2b0e26459e..a7a7ebe838eab 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/async_search.submit.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/async_search.submit.json @@ -65,6 +65,11 @@ "type":"boolean", "description":"Specify whether wildcard and prefix queries should be analyzed (default: false)" }, + "ccs_minimize_roundtrips":{ + "type":"boolean", + "default":false, + "description":"When doing a cross-cluster search, setting it to true may improve overall search latency, particularly when searching clusters with a large number of shards. However, when set to true, the progress of searches on the remote clusters will not be received until the search finishes on all clusters." + }, "default_operator":{ "type":"enum", "options":[ @@ -126,6 +131,16 @@ "type":"string", "description":"Specify the node or shard the operation should be performed on (default: random)" }, + "pre_filter_shard_size":{ + "type":"number", + "default": 1, + "description":"Cannot be changed: this is to enforce the execution of a pre-filter roundtrip to retrieve statistics from each shard so that the ones that surely don’t hold any document matching the query get skipped." + }, + "rest_total_hits_as_int":{ + "type":"boolean", + "description":"Indicates whether hits.total should be rendered as an integer or an object in the rest search response", + "default":false + }, "q":{ "type":"string", "description":"Query in the Lucene query string syntax" From b0c49766f6a2301f8938629bfdadf76459329b8d Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 25 Nov 2024 08:01:21 +0000 Subject: [PATCH 5/5] Extract IMDS test fixture from S3 fixture (#117324) The S3 and IMDS services are separate things in practice, we shouldn't be conflating them as we do today. This commit introduces a new independent test fixture just for the IMDS endpoint and migrates the relevant tests to use it. Relates ES-9984 --- modules/repository-s3/build.gradle | 1 + .../s3/RepositoryS3ClientYamlTestSuiteIT.java | 42 +++- .../RepositoryS3EcsClientYamlTestSuiteIT.java | 30 ++- settings.gradle | 1 + test/fixtures/ec2-imds-fixture/build.gradle | 19 ++ .../fixture/aws/imds/Ec2ImdsHttpFixture.java | 66 ++++++ .../fixture/aws/imds/Ec2ImdsHttpHandler.java | 98 +++++++++ .../aws/imds/Ec2ImdsHttpHandlerTests.java | 188 ++++++++++++++++++ .../java/fixture/s3/S3HttpFixtureWithEC2.java | 84 -------- .../java/fixture/s3/S3HttpFixtureWithECS.java | 48 ----- .../s3/S3HttpFixtureWithSessionToken.java | 12 +- 11 files changed, 433 insertions(+), 156 deletions(-) create mode 100644 test/fixtures/ec2-imds-fixture/build.gradle create mode 100644 test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpFixture.java create mode 100644 test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpHandler.java create mode 100644 test/fixtures/ec2-imds-fixture/src/test/java/fixture/aws/imds/Ec2ImdsHttpHandlerTests.java delete mode 100644 test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithEC2.java delete mode 100644 test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithECS.java diff --git a/modules/repository-s3/build.gradle b/modules/repository-s3/build.gradle index 1301d17606d63..9a7f0a5994d73 100644 --- a/modules/repository-s3/build.gradle +++ b/modules/repository-s3/build.gradle @@ -45,6 +45,7 @@ dependencies { testImplementation project(':test:fixtures:s3-fixture') yamlRestTestImplementation project(":test:framework") yamlRestTestImplementation project(':test:fixtures:s3-fixture') + yamlRestTestImplementation project(':test:fixtures:ec2-imds-fixture') yamlRestTestImplementation project(':test:fixtures:minio-fixture') internalClusterTestImplementation project(':test:fixtures:minio-fixture') diff --git a/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java b/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java index 0ae8af0989fa6..64cb3c3fd3a69 100644 --- a/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java +++ b/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java @@ -9,8 +9,8 @@ package org.elasticsearch.repositories.s3; +import fixture.aws.imds.Ec2ImdsHttpFixture; import fixture.s3.S3HttpFixture; -import fixture.s3.S3HttpFixtureWithEC2; import fixture.s3.S3HttpFixtureWithSessionToken; import com.carrotsearch.randomizedtesting.annotations.Name; @@ -18,6 +18,7 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.elasticsearch.cluster.routing.Murmur3HashFunction; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; @@ -25,15 +26,34 @@ import org.junit.rules.RuleChain; import org.junit.rules.TestRule; +import java.util.Set; + @ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) @ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482 public class RepositoryS3ClientYamlTestSuiteIT extends AbstractRepositoryS3ClientYamlTestSuiteIT { - public static final S3HttpFixture s3Fixture = new S3HttpFixture(); - public static final S3HttpFixtureWithSessionToken s3HttpFixtureWithSessionToken = new S3HttpFixtureWithSessionToken(); - public static final S3HttpFixtureWithEC2 s3Ec2 = new S3HttpFixtureWithEC2(); + private static final String HASHED_SEED = Integer.toString(Murmur3HashFunction.hash(System.getProperty("tests.seed"))); + private static final String TEMPORARY_SESSION_TOKEN = "session_token-" + HASHED_SEED; + private static final String IMDS_ACCESS_KEY = "imds-access-key-" + HASHED_SEED; + private static final String IMDS_SESSION_TOKEN = "imds-session-token-" + HASHED_SEED; + + private static final S3HttpFixture s3Fixture = new S3HttpFixture(); + + private static final S3HttpFixtureWithSessionToken s3HttpFixtureWithSessionToken = new S3HttpFixtureWithSessionToken( + "session_token_bucket", + "session_token_base_path_integration_tests", + System.getProperty("s3TemporaryAccessKey"), + TEMPORARY_SESSION_TOKEN + ); + + private static final S3HttpFixtureWithSessionToken s3HttpFixtureWithImdsSessionToken = new S3HttpFixtureWithSessionToken( + "ec2_bucket", + "ec2_base_path", + IMDS_ACCESS_KEY, + IMDS_SESSION_TOKEN + ); - private static final String s3TemporarySessionToken = "session_token"; + private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(IMDS_ACCESS_KEY, IMDS_SESSION_TOKEN, Set.of()); public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .module("repository-s3") @@ -41,15 +61,19 @@ public class RepositoryS3ClientYamlTestSuiteIT extends AbstractRepositoryS3Clien .keystore("s3.client.integration_test_permanent.secret_key", System.getProperty("s3PermanentSecretKey")) .keystore("s3.client.integration_test_temporary.access_key", System.getProperty("s3TemporaryAccessKey")) .keystore("s3.client.integration_test_temporary.secret_key", System.getProperty("s3TemporarySecretKey")) - .keystore("s3.client.integration_test_temporary.session_token", s3TemporarySessionToken) + .keystore("s3.client.integration_test_temporary.session_token", TEMPORARY_SESSION_TOKEN) .setting("s3.client.integration_test_permanent.endpoint", s3Fixture::getAddress) .setting("s3.client.integration_test_temporary.endpoint", s3HttpFixtureWithSessionToken::getAddress) - .setting("s3.client.integration_test_ec2.endpoint", s3Ec2::getAddress) - .systemProperty("com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", s3Ec2::getAddress) + .setting("s3.client.integration_test_ec2.endpoint", s3HttpFixtureWithImdsSessionToken::getAddress) + .systemProperty("com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", ec2ImdsHttpFixture::getAddress) .build(); @ClassRule - public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(s3Ec2).around(s3HttpFixtureWithSessionToken).around(cluster); + public static TestRule ruleChain = RuleChain.outerRule(s3Fixture) + .around(s3HttpFixtureWithSessionToken) + .around(s3HttpFixtureWithImdsSessionToken) + .around(ec2ImdsHttpFixture) + .around(cluster); @ParametersFactory public static Iterable parameters() throws Exception { diff --git a/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsClientYamlTestSuiteIT.java b/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsClientYamlTestSuiteIT.java index fa21797540c17..a522c9b17145b 100644 --- a/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsClientYamlTestSuiteIT.java +++ b/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsClientYamlTestSuiteIT.java @@ -9,28 +9,48 @@ package org.elasticsearch.repositories.s3; -import fixture.s3.S3HttpFixtureWithECS; +import fixture.aws.imds.Ec2ImdsHttpFixture; +import fixture.s3.S3HttpFixtureWithSessionToken; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.cluster.routing.Murmur3HashFunction; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.junit.ClassRule; import org.junit.rules.RuleChain; import org.junit.rules.TestRule; +import java.util.Set; + public class RepositoryS3EcsClientYamlTestSuiteIT extends AbstractRepositoryS3ClientYamlTestSuiteIT { - private static final S3HttpFixtureWithECS s3Ecs = new S3HttpFixtureWithECS(); + + private static final String HASHED_SEED = Integer.toString(Murmur3HashFunction.hash(System.getProperty("tests.seed"))); + private static final String ECS_ACCESS_KEY = "ecs-access-key-" + HASHED_SEED; + private static final String ECS_SESSION_TOKEN = "ecs-session-token-" + HASHED_SEED; + + private static final S3HttpFixtureWithSessionToken s3Fixture = new S3HttpFixtureWithSessionToken( + "ecs_bucket", + "ecs_base_path", + ECS_ACCESS_KEY, + ECS_SESSION_TOKEN + ); + + private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture( + ECS_ACCESS_KEY, + ECS_SESSION_TOKEN, + Set.of("/ecs_credentials_endpoint") + ); public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .module("repository-s3") - .setting("s3.client.integration_test_ecs.endpoint", s3Ecs::getAddress) - .environment("AWS_CONTAINER_CREDENTIALS_FULL_URI", () -> (s3Ecs.getAddress() + "/ecs_credentials_endpoint")) + .setting("s3.client.integration_test_ecs.endpoint", s3Fixture::getAddress) + .environment("AWS_CONTAINER_CREDENTIALS_FULL_URI", () -> ec2ImdsHttpFixture.getAddress() + "/ecs_credentials_endpoint") .build(); @ClassRule - public static TestRule ruleChain = RuleChain.outerRule(s3Ecs).around(cluster); + public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(ec2ImdsHttpFixture).around(cluster); @ParametersFactory public static Iterable parameters() throws Exception { diff --git a/settings.gradle b/settings.gradle index 333f8272447c2..7bf03263031f1 100644 --- a/settings.gradle +++ b/settings.gradle @@ -87,6 +87,7 @@ List projects = [ 'server', 'test:framework', 'test:fixtures:azure-fixture', + 'test:fixtures:ec2-imds-fixture', 'test:fixtures:gcs-fixture', 'test:fixtures:hdfs-fixture', 'test:fixtures:krb5kdc-fixture', diff --git a/test/fixtures/ec2-imds-fixture/build.gradle b/test/fixtures/ec2-imds-fixture/build.gradle new file mode 100644 index 0000000000000..7ad194acbb8fd --- /dev/null +++ b/test/fixtures/ec2-imds-fixture/build.gradle @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +apply plugin: 'elasticsearch.java' + +description = 'Fixture for emulating the Instance Metadata Service (IMDS) running in AWS EC2' + +dependencies { + api project(':server') + api("junit:junit:${versions.junit}") { + transitive = false + } + api project(':test:framework') +} diff --git a/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpFixture.java b/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpFixture.java new file mode 100644 index 0000000000000..68f46d778018c --- /dev/null +++ b/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpFixture.java @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package fixture.aws.imds; + +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; + +import org.junit.rules.ExternalResource; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.util.Objects; +import java.util.Set; + +public class Ec2ImdsHttpFixture extends ExternalResource { + + private HttpServer server; + + private final String accessKey; + private final String sessionToken; + private final Set alternativeCredentialsEndpoints; + + public Ec2ImdsHttpFixture(String accessKey, String sessionToken, Set alternativeCredentialsEndpoints) { + this.accessKey = accessKey; + this.sessionToken = sessionToken; + this.alternativeCredentialsEndpoints = alternativeCredentialsEndpoints; + } + + protected HttpHandler createHandler() { + return new Ec2ImdsHttpHandler(accessKey, sessionToken, alternativeCredentialsEndpoints); + } + + public String getAddress() { + return "http://" + server.getAddress().getHostString() + ":" + server.getAddress().getPort(); + } + + public void stop(int delay) { + server.stop(delay); + } + + protected void before() throws Throwable { + server = HttpServer.create(resolveAddress(), 0); + server.createContext("/", Objects.requireNonNull(createHandler())); + server.start(); + } + + @Override + protected void after() { + stop(0); + } + + private static InetSocketAddress resolveAddress() { + try { + return new InetSocketAddress(InetAddress.getByName("localhost"), 0); + } catch (UnknownHostException e) { + throw new RuntimeException(e); + } + } +} diff --git a/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpHandler.java b/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpHandler.java new file mode 100644 index 0000000000000..04e5e83bddfa9 --- /dev/null +++ b/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpHandler.java @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package fixture.aws.imds; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; + +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.rest.RestStatus; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Clock; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Collection; +import java.util.Objects; +import java.util.Set; + +import static org.elasticsearch.test.ESTestCase.randomIdentifier; + +/** + * Minimal HTTP handler that emulates the EC2 IMDS server + */ +@SuppressForbidden(reason = "this test uses a HttpServer to emulate the EC2 IMDS endpoint") +public class Ec2ImdsHttpHandler implements HttpHandler { + + private static final String IMDS_SECURITY_CREDENTIALS_PATH = "/latest/meta-data/iam/security-credentials/"; + + private final String accessKey; + private final String sessionToken; + private final Set validCredentialsEndpoints = ConcurrentCollections.newConcurrentSet(); + + public Ec2ImdsHttpHandler(String accessKey, String sessionToken, Collection alternativeCredentialsEndpoints) { + this.accessKey = Objects.requireNonNull(accessKey); + this.sessionToken = Objects.requireNonNull(sessionToken); + this.validCredentialsEndpoints.addAll(alternativeCredentialsEndpoints); + } + + @Override + public void handle(final HttpExchange exchange) throws IOException { + // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html + + try (exchange) { + final var path = exchange.getRequestURI().getPath(); + final var requestMethod = exchange.getRequestMethod(); + + if ("PUT".equals(requestMethod) && "/latest/api/token".equals(path)) { + // Reject IMDSv2 probe + exchange.sendResponseHeaders(RestStatus.METHOD_NOT_ALLOWED.getStatus(), -1); + return; + } + + if ("GET".equals(requestMethod)) { + if (path.equals(IMDS_SECURITY_CREDENTIALS_PATH)) { + final var profileName = randomIdentifier(); + validCredentialsEndpoints.add(IMDS_SECURITY_CREDENTIALS_PATH + profileName); + final byte[] response = profileName.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "text/plain"); + exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); + exchange.getResponseBody().write(response); + return; + } else if (validCredentialsEndpoints.contains(path)) { + final byte[] response = Strings.format( + """ + { + "AccessKeyId": "%s", + "Expiration": "%s", + "RoleArn": "%s", + "SecretAccessKey": "%s", + "Token": "%s" + }""", + accessKey, + ZonedDateTime.now(Clock.systemUTC()).plusDays(1L).format(DateTimeFormatter.ISO_DATE_TIME), + randomIdentifier(), + randomIdentifier(), + sessionToken + ).getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); + exchange.getResponseBody().write(response); + return; + } + } + + ExceptionsHelper.maybeDieOnAnotherThread(new AssertionError("not supported: " + requestMethod + " " + path)); + } + } +} diff --git a/test/fixtures/ec2-imds-fixture/src/test/java/fixture/aws/imds/Ec2ImdsHttpHandlerTests.java b/test/fixtures/ec2-imds-fixture/src/test/java/fixture/aws/imds/Ec2ImdsHttpHandlerTests.java new file mode 100644 index 0000000000000..5d5cbfae3fa60 --- /dev/null +++ b/test/fixtures/ec2-imds-fixture/src/test/java/fixture/aws/imds/Ec2ImdsHttpHandlerTests.java @@ -0,0 +1,188 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package fixture.aws.imds; + +import com.sun.net.httpserver.Headers; +import com.sun.net.httpserver.HttpContext; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpPrincipal; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.URI; +import java.util.Set; + +public class Ec2ImdsHttpHandlerTests extends ESTestCase { + + public void testImdsV1() throws IOException { + final var accessKey = randomIdentifier(); + final var sessionToken = randomIdentifier(); + + final var handler = new Ec2ImdsHttpHandler(accessKey, sessionToken, Set.of()); + + final var roleResponse = handleRequest(handler, "GET", "/latest/meta-data/iam/security-credentials/"); + assertEquals(RestStatus.OK, roleResponse.status()); + final var profileName = roleResponse.body().utf8ToString(); + assertTrue(Strings.hasText(profileName)); + + final var credentialsResponse = handleRequest(handler, "GET", "/latest/meta-data/iam/security-credentials/" + profileName); + assertEquals(RestStatus.OK, credentialsResponse.status()); + + final var responseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), credentialsResponse.body().streamInput(), false); + assertEquals(Set.of("AccessKeyId", "Expiration", "RoleArn", "SecretAccessKey", "Token"), responseMap.keySet()); + assertEquals(accessKey, responseMap.get("AccessKeyId")); + assertEquals(sessionToken, responseMap.get("Token")); + } + + public void testImdsV2Disabled() { + assertEquals( + RestStatus.METHOD_NOT_ALLOWED, + handleRequest(new Ec2ImdsHttpHandler(randomIdentifier(), randomIdentifier(), Set.of()), "PUT", "/latest/api/token").status() + ); + } + + private record TestHttpResponse(RestStatus status, BytesReference body) {} + + private static TestHttpResponse handleRequest(Ec2ImdsHttpHandler handler, String method, String uri) { + final var httpExchange = new TestHttpExchange(method, uri, BytesArray.EMPTY, TestHttpExchange.EMPTY_HEADERS); + try { + handler.handle(httpExchange); + } catch (IOException e) { + fail(e); + } + assertNotEquals(0, httpExchange.getResponseCode()); + return new TestHttpResponse(RestStatus.fromCode(httpExchange.getResponseCode()), httpExchange.getResponseBodyContents()); + } + + private static class TestHttpExchange extends HttpExchange { + + private static final Headers EMPTY_HEADERS = new Headers(); + + private final String method; + private final URI uri; + private final BytesReference requestBody; + private final Headers requestHeaders; + + private final Headers responseHeaders = new Headers(); + private final BytesStreamOutput responseBody = new BytesStreamOutput(); + private int responseCode; + + TestHttpExchange(String method, String uri, BytesReference requestBody, Headers requestHeaders) { + this.method = method; + this.uri = URI.create(uri); + this.requestBody = requestBody; + this.requestHeaders = requestHeaders; + } + + @Override + public Headers getRequestHeaders() { + return requestHeaders; + } + + @Override + public Headers getResponseHeaders() { + return responseHeaders; + } + + @Override + public URI getRequestURI() { + return uri; + } + + @Override + public String getRequestMethod() { + return method; + } + + @Override + public HttpContext getHttpContext() { + return null; + } + + @Override + public void close() {} + + @Override + public InputStream getRequestBody() { + try { + return requestBody.streamInput(); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + @Override + public OutputStream getResponseBody() { + return responseBody; + } + + @Override + public void sendResponseHeaders(int rCode, long responseLength) { + this.responseCode = rCode; + } + + @Override + public InetSocketAddress getRemoteAddress() { + return null; + } + + @Override + public int getResponseCode() { + return responseCode; + } + + public BytesReference getResponseBodyContents() { + return responseBody.bytes(); + } + + @Override + public InetSocketAddress getLocalAddress() { + return null; + } + + @Override + public String getProtocol() { + return "HTTP/1.1"; + } + + @Override + public Object getAttribute(String name) { + return null; + } + + @Override + public void setAttribute(String name, Object value) { + fail("setAttribute not implemented"); + } + + @Override + public void setStreams(InputStream i, OutputStream o) { + fail("setStreams not implemented"); + } + + @Override + public HttpPrincipal getPrincipal() { + fail("getPrincipal not implemented"); + throw new UnsupportedOperationException("getPrincipal not implemented"); + } + } + +} diff --git a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithEC2.java b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithEC2.java deleted file mode 100644 index d7048cbea6b8a..0000000000000 --- a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithEC2.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ -package fixture.s3; - -import com.sun.net.httpserver.HttpHandler; - -import org.elasticsearch.rest.RestStatus; - -import java.nio.charset.StandardCharsets; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.util.Locale; - -public class S3HttpFixtureWithEC2 extends S3HttpFixtureWithSessionToken { - - private static final String EC2_PATH = "/latest/meta-data/iam/security-credentials/"; - private static final String EC2_PROFILE = "ec2Profile"; - - public S3HttpFixtureWithEC2() { - this(true); - } - - public S3HttpFixtureWithEC2(boolean enabled) { - this(enabled, "ec2_bucket", "ec2_base_path", "ec2_access_key", "ec2_session_token"); - } - - public S3HttpFixtureWithEC2(boolean enabled, String bucket, String basePath, String accessKey, String sessionToken) { - super(enabled, bucket, basePath, accessKey, sessionToken); - } - - @Override - protected HttpHandler createHandler() { - final HttpHandler delegate = super.createHandler(); - - return exchange -> { - final String path = exchange.getRequestURI().getPath(); - // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html - if ("GET".equals(exchange.getRequestMethod()) && path.startsWith(EC2_PATH)) { - if (path.equals(EC2_PATH)) { - final byte[] response = EC2_PROFILE.getBytes(StandardCharsets.UTF_8); - exchange.getResponseHeaders().add("Content-Type", "text/plain"); - exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); - exchange.getResponseBody().write(response); - exchange.close(); - return; - - } else if (path.equals(EC2_PATH + EC2_PROFILE)) { - final byte[] response = buildCredentialResponse(accessKey, sessionToken).getBytes(StandardCharsets.UTF_8); - exchange.getResponseHeaders().add("Content-Type", "application/json"); - exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); - exchange.getResponseBody().write(response); - exchange.close(); - return; - } - - final byte[] response = "unknown profile".getBytes(StandardCharsets.UTF_8); - exchange.getResponseHeaders().add("Content-Type", "text/plain"); - exchange.sendResponseHeaders(RestStatus.NOT_FOUND.getStatus(), response.length); - exchange.getResponseBody().write(response); - exchange.close(); - return; - - } - delegate.handle(exchange); - }; - } - - protected static String buildCredentialResponse(final String ec2AccessKey, final String ec2SessionToken) { - return String.format(Locale.ROOT, """ - { - "AccessKeyId": "%s", - "Expiration": "%s", - "RoleArn": "arn", - "SecretAccessKey": "secret_access_key", - "Token": "%s" - }""", ec2AccessKey, ZonedDateTime.now().plusDays(1L).format(DateTimeFormatter.ISO_DATE_TIME), ec2SessionToken); - } -} diff --git a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithECS.java b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithECS.java deleted file mode 100644 index d6266ea75dd3a..0000000000000 --- a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithECS.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ -package fixture.s3; - -import com.sun.net.httpserver.HttpHandler; - -import org.elasticsearch.rest.RestStatus; - -import java.nio.charset.StandardCharsets; - -public class S3HttpFixtureWithECS extends S3HttpFixtureWithEC2 { - - public S3HttpFixtureWithECS() { - this(true); - } - - public S3HttpFixtureWithECS(boolean enabled) { - this(enabled, "ecs_bucket", "ecs_base_path", "ecs_access_key", "ecs_session_token"); - } - - public S3HttpFixtureWithECS(boolean enabled, String bucket, String basePath, String accessKey, String sessionToken) { - super(enabled, bucket, basePath, accessKey, sessionToken); - } - - @Override - protected HttpHandler createHandler() { - final HttpHandler delegate = super.createHandler(); - - return exchange -> { - // https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html - if ("GET".equals(exchange.getRequestMethod()) && exchange.getRequestURI().getPath().equals("/ecs_credentials_endpoint")) { - final byte[] response = buildCredentialResponse(accessKey, sessionToken).getBytes(StandardCharsets.UTF_8); - exchange.getResponseHeaders().add("Content-Type", "application/json"); - exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); - exchange.getResponseBody().write(response); - exchange.close(); - return; - } - delegate.handle(exchange); - }; - } -} diff --git a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithSessionToken.java b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithSessionToken.java index 1a1cbba651e06..001cc34d9b20d 100644 --- a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithSessionToken.java +++ b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithSessionToken.java @@ -18,16 +18,8 @@ public class S3HttpFixtureWithSessionToken extends S3HttpFixture { protected final String sessionToken; - public S3HttpFixtureWithSessionToken() { - this(true); - } - - public S3HttpFixtureWithSessionToken(boolean enabled) { - this(enabled, "session_token_bucket", "session_token_base_path_integration_tests", "session_token_access_key", "session_token"); - } - - public S3HttpFixtureWithSessionToken(boolean enabled, String bucket, String basePath, String accessKey, String sessionToken) { - super(enabled, bucket, basePath, accessKey); + public S3HttpFixtureWithSessionToken(String bucket, String basePath, String accessKey, String sessionToken) { + super(true, bucket, basePath, accessKey); this.sessionToken = sessionToken; }