Splitup x-pack testing in more build steps

We want to keep pull request and intake build time below 40min

.buildkite/pipelines/intake.template.yml

@@ -32,6 +32,14 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: n1-standard-32
       buildDirectory: /dev/shm/bk
+  - label: part4
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: n1-standard-32
+      buildDirectory: /dev/shm/bk
   - group: bwc-snapshots
     steps:
       - label: "{{matrix.BWC_VERSION}} / bwc-snapshots"

.buildkite/pipelines/intake.yml

@@ -33,6 +33,14 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: n1-standard-32
       buildDirectory: /dev/shm/bk
+  - label: part4
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: n1-standard-32
+      buildDirectory: /dev/shm/bk
   - group: bwc-snapshots
     steps:
       - label: "{{matrix.BWC_VERSION}} / bwc-snapshots"

.buildkite/pipelines/pull-request/part-4-fips.yml

@@ -0,0 +1,11 @@
+config:
+  allow-labels: "Team:Security"
+steps:
+  - label: part-4-fips
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true checkPart4
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/pull-request/part-4-windows.yml

@@ -0,0 +1,14 @@
+config:
+  allow-labels: "test-windows"
+steps:
+  - label: part-4-windows
+    command: .\.buildkite\scripts\run-script.ps1 bash .buildkite/scripts/windows-run-gradle.sh
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-windows-2022
+      machineType: custom-32-98304
+      diskType: pd-ssd
+      diskSizeGb: 350
+    env:
+      GRADLE_TASK: checkPart4

.buildkite/pipelines/pull-request/part-4.yml

@@ -0,0 +1,11 @@
+config:
+  skip-target-branches: "7.17"
+steps:
+  - label: part-4
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart4
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

build.gradle

@@ -161,8 +161,10 @@ tasks.register("verifyVersions") {
     String versionMapping = backportConfig.get("branchLabelMapping").fields().find { it.value.textValue() == 'main' }.key
     String expectedMapping = "^v${versions.elasticsearch.replaceAll('-SNAPSHOT', '')}\$"
     if (versionMapping != expectedMapping) {
-      throw new GradleException("Backport label mapping for branch 'main' is '${versionMapping}' but should be " +
-        "'${expectedMapping}'. Update .backportrc.json.")
+      throw new GradleException(
+        "Backport label mapping for branch 'main' is '${versionMapping}' but should be " +
+          "'${expectedMapping}'. Update .backportrc.json."
+      )
     }
   }
 }
@@ -211,9 +213,9 @@ allprojects {
   project.ext {
     // for ide hacks...
     isEclipse = providers.systemProperty("eclipse.launcher").isPresent() ||   // Detects gradle launched from Eclipse's IDE
-            providers.systemProperty("eclipse.application").isPresent() ||    // Detects gradle launched from the Eclipse compiler server
-            gradle.startParameter.taskNames.contains('eclipse') ||  // Detects gradle launched from the command line to do eclipse stuff
-            gradle.startParameter.taskNames.contains('cleanEclipse')
+      providers.systemProperty("eclipse.application").isPresent() ||    // Detects gradle launched from the Eclipse compiler server
+      gradle.startParameter.taskNames.contains('eclipse') ||  // Detects gradle launched from the command line to do eclipse stuff
+      gradle.startParameter.taskNames.contains('cleanEclipse')
   }
 
   ext.bwc_tests_enabled = bwc_tests_enabled
@@ -229,10 +231,10 @@ allprojects {
     eclipse.classpath.file.whenMerged { classpath ->
       if (false == forbiddenApisTest.bundledSignatures.contains('jdk-non-portable')) {
         classpath.entries
-                .findAll { it.kind == "con" && it.toString().contains("org.eclipse.jdt.launching.JRE_CONTAINER") }
-                .each {
-                  it.accessRules.add(new AccessRule("accessible", "com/sun/net/httpserver/*"))
-                }
+          .findAll { it.kind == "con" && it.toString().contains("org.eclipse.jdt.launching.JRE_CONTAINER") }
+          .each {
+            it.accessRules.add(new AccessRule("accessible", "com/sun/net/httpserver/*"))
+          }
       }
     }
   }
@@ -248,6 +250,8 @@ allprojects {
   plugins.withId('lifecycle-base') {
     if (project.path.startsWith(":x-pack:")) {
       if (project.path.contains("security") || project.path.contains(":ml")) {
+        tasks.register('checkPart4') { dependsOn 'check' }
+      } else if (project.path == ":x-pack:plugin" || project.path.contains("ql") ||  project.path.contains("smoke-test")) {
         tasks.register('checkPart3') { dependsOn 'check' }
       } else {
         tasks.register('checkPart2') { dependsOn 'check' }
@@ -256,7 +260,7 @@ allprojects {
       tasks.register('checkPart1') { dependsOn 'check' }
     }
 
-    tasks.register('functionalTests') { dependsOn 'check'}
+    tasks.register('functionalTests') { dependsOn 'check' }
   }
 
   /*
@@ -281,7 +285,7 @@ allprojects {
       // :test:framework:test cannot run before and after :server:test
       return
     }
-    tasks.matching { it.name.equals('integTest')}.configureEach {integTestTask ->
+    tasks.matching { it.name.equals('integTest') }.configureEach { integTestTask ->
       integTestTask.mustRunAfter tasks.matching { it.name.equals("test") }
     }
 
@@ -290,7 +294,7 @@ allprojects {
         Project upstreamProject = dep.dependencyProject
         if (project.path != upstreamProject?.path) {
           for (String taskName : ['test', 'integTest']) {
-            project.tasks.matching { it.name == taskName }.configureEach {task ->
+            project.tasks.matching { it.name == taskName }.configureEach { task ->
               task.shouldRunAfter(upstreamProject.tasks.matching { upStreamTask -> upStreamTask.name == taskName })
             }
           }