Fix running cluster lookup

- after loaded from configuration cache
- in multiproject running parallel scenario

build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy

@@ -15,16 +15,12 @@ import org.elasticsearch.gradle.internal.test.AntFixture
 import org.gradle.api.file.FileSystemOperations
 import org.gradle.api.file.ProjectLayout
 import org.gradle.api.provider.ProviderFactory
-import org.gradle.api.tasks.Internal
 import org.gradle.process.ExecOperations
 
 import javax.inject.Inject
 
 abstract class AntFixtureStop extends LoggedExec implements FixtureStop {
 
-    @Internal
-    AntFixture fixture
-
     @Inject
     AntFixtureStop(ProjectLayout projectLayout,
                    ExecOperations execOperations,
@@ -35,11 +31,12 @@ abstract class AntFixtureStop extends LoggedExec implements FixtureStop {
 
     void setFixture(AntFixture fixture) {
         assert this.fixture == null
-        this.fixture = fixture;
-        final Object pid = "${-> this.fixture.pid}"
-        onlyIf("pidFile exists") { fixture.pidFile.exists() }
+        def pidFile = fixture.pidFile
+        def fixtureName = fixture.name
+        final Object pid = "${-> Integer.parseInt(pidFile.getText('UTF-8').trim())}"
+        onlyIf("pidFile exists") { pidFile.exists() }
         doFirst {
-            logger.info("Shutting down ${fixture.name} with pid ${pid}")
+            logger.info("Shutting down ${fixtureName} with pid ${pid}")
         }
 
         if (OS.current() == OS.WINDOWS) {
@@ -51,9 +48,8 @@ abstract class AntFixtureStop extends LoggedExec implements FixtureStop {
         }
         doLast {
             fileSystemOperations.delete {
-                it.delete(fixture.pidFile)
+                it.delete(pidFile)
             }
         }
-        this.fixture = fixture
     }
 }

build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntTask.groovy

@@ -29,11 +29,6 @@ import java.nio.charset.Charset
  */
 public abstract class AntTask extends DefaultTask {
 
-    /**
-     * A buffer that will contain the output of the ant code run,
-     * if the output was not already written directly to stdout.
-     */
-    public final ByteArrayOutputStream outputBuffer = new ByteArrayOutputStream()
 
     @Inject
     protected FileSystemOperations getFileSystemOperations() {
@@ -57,6 +52,11 @@ public abstract class AntTask extends DefaultTask {
 
         // otherwise groovy replaces System.out, and you have no chance to debug
         // ant.saveStreams = false
+        /**
+         * A buffer that will contain the output of the ant code run,
+         * if the output was not already written directly to stdout.
+         */
+        ByteArrayOutputStream outputBuffer = new ByteArrayOutputStream()
 
         final int outputLevel = logger.isDebugEnabled() ? Project.MSG_DEBUG : Project.MSG_INFO
         final PrintStream stream = useStdout() ? System.out : new PrintStream(outputBuffer, true, Charset.defaultCharset().name())

build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/test/AntFixture.groovy

@@ -10,22 +10,30 @@
 package org.elasticsearch.gradle.internal.test
 
 import org.elasticsearch.gradle.OS
+
 import org.elasticsearch.gradle.internal.AntFixtureStop
 import org.elasticsearch.gradle.internal.AntTask
 import org.gradle.api.GradleException
+import org.gradle.api.file.ProjectLayout
+import org.gradle.api.provider.Provider
+import org.gradle.api.provider.ProviderFactory
 import org.gradle.api.tasks.Internal
 import org.gradle.api.tasks.TaskProvider
 
+import javax.inject.Inject
+
 /**
  * A fixture for integration tests which runs in a separate process launched by Ant.
  */
-class AntFixture extends AntTask implements Fixture {
+class AntFixture extends AntTask {
 
     /** The path to the executable that starts the fixture. */
     @Internal
     String executable
 
     private final List<Object> arguments = new ArrayList<>()
+    private ProjectLayout projectLayout
+    private final ProviderFactory providerFactory
 
     void args(Object... args) {
         arguments.addAll(args)
@@ -69,19 +77,14 @@ class AntFixture extends AntTask implements Fixture {
         return tmpFile.exists()
     }
 
-    private final TaskProvider<AntFixtureStop> stopTask
-
-    AntFixture() {
-        stopTask = createStopTask()
+    @Inject
+    AntFixture(ProjectLayout projectLayout, ProviderFactory providerFactory) {
+        this.providerFactory = providerFactory
+        this.projectLayout = projectLayout;
+        TaskProvider<AntFixtureStop> stopTask = createStopTask()
         finalizedBy(stopTask)
     }
 
-    @Override
-    @Internal
-    TaskProvider<AntFixtureStop> getStopTask() {
-        return stopTask
-    }
-
     @Override
     protected void runAnt(AntBuilder ant) {
         // reset everything
@@ -231,7 +234,7 @@ class AntFixture extends AntTask implements Fixture {
      */
     @Internal
     protected File getBaseDir() {
-        return new File(project.buildDir, "fixtures/${name}")
+        return new File(projectLayout.getBuildDirectory().getAsFile().get(), "fixtures/${name}")
     }
 
     /** Returns the working directory for the process. Defaults to "cwd" inside baseDir. */
@@ -242,7 +245,7 @@ class AntFixture extends AntTask implements Fixture {
 
     /** Returns the file the process writes its pid to. Defaults to "pid" inside baseDir. */
     @Internal
-    protected File getPidFile() {
+    File getPidFile() {
         return new File(baseDir, 'pid')
     }
 
@@ -264,6 +267,12 @@ class AntFixture extends AntTask implements Fixture {
         return portsFile.readLines("UTF-8").get(0)
     }
 
+    @Internal
+    Provider<String> getAddressAndPortProvider() {
+        File thePortFile = portsFile
+        return providerFactory.provider(() -> thePortFile.readLines("UTF-8").get(0))
+    }
+
     /** Returns a file that wraps around the actual command when {@code spawn == true}. */
     @Internal
     protected File getWrapperScript() {

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterInfo.java

@@ -6,19 +6,20 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
-
 package org.elasticsearch.gradle.testclusters;
 
+import java.io.File;
 import java.util.List;
 
 public class TestClusterInfo {
     private final List<String> allHttpSocketURI;
     private final List<String> allTransportPortURI;
+    private final List<File> auditLogs;
 
-    public TestClusterInfo(List<String> allHttpSocketURI, List<String> allTransportPortURI) {
-
+    public TestClusterInfo(List<String> allHttpSocketURI, List<String> allTransportPortURI, List<File> auditLogs) {
         this.allHttpSocketURI = allHttpSocketURI;
         this.allTransportPortURI = allTransportPortURI;
+        this.auditLogs = auditLogs;
     }
 
     public List<String> getAllHttpSocketURI() {
@@ -28,4 +29,8 @@ public List<String> getAllHttpSocketURI() {
     public List<String> getAllTransportPortURI() {
         return allTransportPortURI;
     }
+
+    public List<File> getAuditLogs() {
+        return auditLogs;
+    }
 }

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterValueSource.java

@@ -20,12 +20,15 @@ public abstract class TestClusterValueSource implements ValueSource<TestClusterI
     @Override
     public TestClusterInfo obtain() {
         String clusterName = getParameters().getClusterName().get();
-        return getParameters().getService().get().getClusterDetails(clusterName);
+        String path = getParameters().getPath().get();
+        return getParameters().getService().get().getClusterDetails(path, clusterName);
     }
 
     interface Parameters extends ValueSourceParameters {
         Property<String> getClusterName();
 
+        Property<String> getPath();
+
         Property<TestClustersRegistry> getService();
     }
 }

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersRegistry.java

@@ -19,6 +19,7 @@
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import javax.inject.Inject;
 
@@ -85,19 +86,32 @@ public void stopCluster(ElasticsearchCluster cluster, boolean taskFailed) {
             claimsInventory.put(cluster, currentClaims);
             cluster.setClaims(currentClaims);
             if (currentClaims <= 0 && runningClusters.contains(cluster)) {
+                System.out.println("TestClustersRegistry.stopCluster " + cluster.getName() + " stopping...");
                 cluster.stop(false);
                 runningClusters.remove(cluster);
             }
         }
     }
 
-    public TestClusterInfo getClusterDetails(String clusterName) {
-        ElasticsearchCluster cluster = runningClusters.stream().filter(c -> c.getName().equals(clusterName)).findFirst().orElseThrow();
-        return new TestClusterInfo(cluster.getAllHttpSocketURI(), cluster.getAllTransportPortURI());
+    public TestClusterInfo getClusterDetails(String path, String clusterName) {
+        ElasticsearchCluster cluster = runningClusters.stream()
+            .filter(c -> c.getPath().equals(path))
+            .filter(c -> c.getName().equals(clusterName))
+            .findFirst()
+            .orElseThrow();
+        return new TestClusterInfo(
+            cluster.getAllHttpSocketURI(),
+            cluster.getAllTransportPortURI(),
+            cluster.getNodes().stream().map(n -> n.getAuditLog()).collect(Collectors.toList())
+        );
     }
 
-    public void restart(String clusterName) {
-        ElasticsearchCluster cluster = runningClusters.stream().filter(c -> c.getName().equals(clusterName)).findFirst().orElseThrow();
+    public void restart(String path, String clusterName) {
+        ElasticsearchCluster cluster = runningClusters.stream()
+            .filter(c -> c.getPath().equals(path))
+            .filter(c -> c.getName().equals(clusterName))
+            .findFirst()
+            .orElseThrow();
         cluster.restart();
     }
 

plugins/discovery-ec2/qa/amazon-ec2/build.gradle

@@ -8,7 +8,6 @@
  */
 
 import org.apache.tools.ant.filters.ReplaceTokens
-import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.internal.test.AntFixture
 import org.elasticsearch.gradle.internal.test.RestIntegTestTask
 import org.elasticsearch.gradle.internal.test.rest.LegacyYamlRestTestPlugin
@@ -55,8 +54,9 @@ tasks.named("yamlRestTest").configure { enabled = false }
 ['KeyStore', 'EnvVariables', 'SystemProperties', 'ContainerCredentials', 'InstanceProfile'].forEach { action ->
   TaskProvider<AntFixture> fixture = tasks.register("ec2Fixture${action}", AntFixture) {
     dependsOn project.sourceSets.yamlRestTest.runtimeClasspath
-    env 'CLASSPATH', "${-> project.sourceSets.yamlRestTest.runtimeClasspath.asPath}"
-    executable = "${buildParams.runtimeJavaHome.get()}/bin/java"
+    FileCollection cp = project.sourceSets.yamlRestTest.runtimeClasspath
+    env 'CLASSPATH', "${-> cp.asPath}"
+    executable = "${buildParams.runtimeJavaHome}/bin/java"
     args 'org.elasticsearch.discovery.ec2.AmazonEC2Fixture', baseDir, "${buildDir}/testclusters/yamlRestTest${action}-1/config/unicast_hosts.txt"
   }
 
@@ -72,15 +72,17 @@ tasks.named("yamlRestTest").configure { enabled = false }
     dependsOn(yamlRestTestTask)
   }
 
+  Provider<String> addressAndPort = fixture.get().addressAndPortProvider
+
   testClusters.matching { it.name == yamlRestTestTask.name}.configureEach {
     numberOfNodes = ec2NumberOfNodes
     plugin ':plugins:discovery-ec2'
 
     setting 'discovery.seed_providers', 'ec2'
     setting 'network.host', '_ec2_'
-    setting 'discovery.ec2.endpoint', { "http://${-> fixture.get().addressAndPort}" }, IGNORE_VALUE
+    setting 'discovery.ec2.endpoint', { "http://${-> addressAndPort.get()}" }, IGNORE_VALUE
 
-    systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "http://${-> fixture.get().addressAndPort}" }, IGNORE_VALUE
+    systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "http://${-> addressAndPort.get() }" }, IGNORE_VALUE
   }
 }
 
@@ -107,9 +109,10 @@ tasks.named("ec2FixtureContainerCredentials").configure {
   env 'ACTIVATE_CONTAINER_CREDENTIALS', true
 }
 
+def someProvider = tasks.findByName("ec2FixtureContainerCredentials").addressAndPortProvider
 testClusters.matching { it.name == "yamlRestTestContainerCredentials" }.configureEach {
   environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI',
-    { "http://${-> tasks.findByName("ec2FixtureContainerCredentials").addressAndPort}/ecs_credentials_endpoint" }, IGNORE_VALUE
+    { "http://${-> someProvider.get()}/ecs_credentials_endpoint" }, IGNORE_VALUE
 }
 
 // Extra config for InstanceProfile

x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle

@@ -15,6 +15,8 @@ dependencies {
   testImplementation project(':x-pack:plugin:ccr:qa')
 }
 
+def clusterPath = getPath()
+
 def leaderCluster = testClusters.register("leader-cluster") {
   testDistribution = 'DEFAULT'
   setting 'xpack.license.self_generated.type', 'trial'
@@ -33,6 +35,7 @@ def followCluster = testClusters.register("follow-cluster") {
     TestClustersPlugin.REGISTRY_SERVICE_NAME
   )
   def leaderInfo = project.getProviders().of(TestClusterValueSource.class) {
+    it.parameters.path.set(clusterPath)
     it.parameters.clusterName.set("leader-cluster")
     it.parameters.service = serviceProvider
   }
@@ -73,10 +76,12 @@ tasks.register("follow-cluster", RestIntegTestTask) {
     TestClustersPlugin.REGISTRY_SERVICE_NAME
   )
   def leaderInfo = project.getProviders().of(TestClusterValueSource.class) {
+    it.parameters.path.set(clusterPath)
     it.parameters.clusterName.set("leader-cluster")
     it.parameters.service = serviceProvider
   }
   def followInfo = project.getProviders().of(TestClusterValueSource.class) {
+    it.parameters.path.set(clusterPath)
     it.parameters.clusterName.set("follow-cluster")
     it.parameters.service = serviceProvider
   }