Replace cloud-ess docker image with wolfi-ess (elastic#114413)

* Replace cloud-ess docker image with wolfi-ess
   We just replaced the existing implementation of cloud-ess with what was wolfi-ess which is a wolfi based ess image.
   The cloud image itself will be removed in a future commit it was not used anywhere

* Switch to test cloud docker image instead of default docker in packaging pr tests.
  This adds way more coverage than the default docker image which is also barely touched

(cherry picked from commit f1f5ee0)

# Conflicts:
#	build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

.buildkite/pipelines/pull-request/packaging-tests-unix.yml

@@ -5,7 +5,7 @@ steps:
     steps:
       - label: "{{matrix.image}} / docker / packaging-tests-unix"
         key: "packaging-tests-unix-docker"
-        command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.docker
+        command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.docker-cloud-ess
         timeout_in_minutes: 300
         matrix:
           setup:

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -24,10 +24,6 @@ public enum DockerBase {
     // Base image with extras for Cloud
     CLOUD("ubuntu:20.04", "-cloud", "apt-get"),
 
-    // Based on CLOUD above, with more extras. We don't set a base image because
-    // we programmatically extend from the Cloud image.
-    CLOUD_ESS(null, "-cloud-ess", "apt-get"),
-
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
@@ -36,10 +32,10 @@ public enum DockerBase {
         "-wolfi",
         "apk"
     ),
-
+    // spotless:on
     // Based on WOLFI above, with more extras. We don't set a base image because
-    // we programmatically extend from the Wolfi image.
-    WOLFI_ESS(null, "-wolfi-ess", "apk");
+    // we programmatically extend from the wolfi image.
+    CLOUD_ESS(null, "-cloud-ess", "apk");
 
     private final String image;
     private final String suffix;

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java

@@ -181,9 +181,6 @@ private static String distributionProjectName(ElasticsearchDistribution distribu
         if (distribution.getType() == InternalElasticsearchDistributionTypes.DOCKER_WOLFI) {
             return projectName + "wolfi-docker" + archString + "-export";
         }
-        if (distribution.getType() == InternalElasticsearchDistributionTypes.DOCKER_WOLFI_ESS) {
-            return projectName + "wolfi-ess-docker" + archString + "-export";
-        }
         return projectName + distribution.getType().getName();
     }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java

@@ -22,7 +22,6 @@ public class InternalElasticsearchDistributionTypes {
     public static ElasticsearchDistributionType DOCKER_CLOUD = new DockerCloudElasticsearchDistributionType();
     public static ElasticsearchDistributionType DOCKER_CLOUD_ESS = new DockerCloudEssElasticsearchDistributionType();
     public static ElasticsearchDistributionType DOCKER_WOLFI = new DockerWolfiElasticsearchDistributionType();
-    public static ElasticsearchDistributionType DOCKER_WOLFI_ESS = new DockerWolfiEssElasticsearchDistributionType();
 
     public static List<ElasticsearchDistributionType> ALL_INTERNAL = List.of(
         DEB,
@@ -32,7 +31,6 @@ public class InternalElasticsearchDistributionTypes {
         DOCKER_IRONBANK,
         DOCKER_CLOUD,
         DOCKER_CLOUD_ESS,
-        DOCKER_WOLFI,
-        DOCKER_WOLFI_ESS
+        DOCKER_WOLFI
     );
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java

@@ -54,7 +54,6 @@
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_IRONBANK;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_UBI;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_WOLFI;
-import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_WOLFI_ESS;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.RPM;
 
 /**
@@ -153,7 +152,6 @@ private static Map<ElasticsearchDistributionType, TaskProvider<?>> lifecycleTask
         lifecyleTasks.put(DOCKER_CLOUD, project.getTasks().register(taskPrefix + ".docker-cloud"));
         lifecyleTasks.put(DOCKER_CLOUD_ESS, project.getTasks().register(taskPrefix + ".docker-cloud-ess"));
         lifecyleTasks.put(DOCKER_WOLFI, project.getTasks().register(taskPrefix + ".docker-wolfi"));
-        lifecyleTasks.put(DOCKER_WOLFI_ESS, project.getTasks().register(taskPrefix + ".docker-wolfi-ess"));
         lifecyleTasks.put(ARCHIVE, project.getTasks().register(taskPrefix + ".archives"));
         lifecyleTasks.put(DEB, project.getTasks().register(taskPrefix + ".packages"));
         lifecyleTasks.put(RPM, lifecyleTasks.get(DEB));

distribution/docker/README.md

@@ -7,7 +7,7 @@ the [DockerBase] enum.
    * UBI - the same as the default image, but based upon [RedHat's UBI
      images][ubi], specifically their minimal flavour.
    * Wolfi - the same as the default image, but based upon [Wolfi](https://github.com/wolfi-dev)
-   * Wolfi ESS - this directly extends the Wolfi image, and adds all ES plugins
+   * Cloud ESS - this directly extends the Wolfi image, and adds all ES plugins
      that the ES build generates in an archive directory. It also sets an
      environment variable that points at this directory. This allows plugins to
      be installed from the archive instead of the internet, speeding up
@@ -23,20 +23,13 @@ the [DockerBase] enum.
      software (FOSS) and Commercial off-the-shelf (COTS). In practice, this is
      another UBI build, this time on the regular UBI image, with extra
      hardening. See below for more details.
-
    * Cloud - this is mostly the same as the default image, with some notable differences:
       * `filebeat` and `metricbeat` are included
       * `wget` is included
       * The `ENTRYPOINT` is just `/bin/tini`, and the `CMD` is
         `/app/elasticsearch.sh`. In normal use this file would be bind-mounted
         in, but the image ships a stub version of this file so that the image
         can still be tested.
-   * Cloud ESS - this directly extends the Cloud image, and adds all ES plugins
-     that the ES build generates in an archive directory. It also sets an
-     environment variable that points at this directory. This allows plugins to
-     be installed from the archive instead of the internet, speeding up
-     deployment times.
-
 The long-term goal is for both Cloud images to be retired in favour of the
 default image.
 

distribution/docker/build.gradle

@@ -381,7 +381,7 @@ private static List<String> generateTags(DockerBase base, Architecture architect
   String image = "elasticsearch${base.suffix}"
 
   String namespace = 'elasticsearch'
-  if (base == DockerBase.CLOUD || base == DockerBase.CLOUD_ESS || base == DockerBase.WOLFI_ESS) {
+  if (base == DockerBase.CLOUD || base == DockerBase.CLOUD_ESS) {
     namespace += '-ci'
   }
 
@@ -446,7 +446,8 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) {
   }
 }
 
-void addBuildEssDockerImageTask(Architecture architecture, DockerBase dockerBase) {
+void addBuildEssDockerImageTask(Architecture architecture) {
+  DockerBase dockerBase = DockerBase.CLOUD_ESS
   String arch = architecture == Architecture.AARCH64 ? '-aarch64' : ''
   String contextDir = "${project.buildDir}/docker-context/elasticsearch${dockerBase.suffix}-${VersionProperties.elasticsearch}-docker-build-context${arch}"
 
@@ -460,22 +461,20 @@ void addBuildEssDockerImageTask(Architecture architecture, DockerBase dockerBase
         from configurations.allPlugins
       }
 
-      if (dockerBase == DockerBase.WOLFI_ESS) {
-        // If we're performing a release build, but `build.id` hasn't been set, we can
-        // infer that we're not at the Docker building stage of the build, and therefore
-        // we should skip the beats part of the build.
-        String buildId = providers.systemProperty('build.id').getOrNull()
-        boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null || useDra
+      // If we're performing a release build, but `build.id` hasn't been set, we can
+      // infer that we're not at the Docker building stage of the build, and therefore
+      // we should skip the beats part of the build.
+      String buildId = providers.systemProperty('build.id').getOrNull()
+      boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null || useDra
 
-        if (includeBeats) {
-          from configurations.getByName("filebeat_${architecture.classifier}")
-          from configurations.getByName("metricbeat_${architecture.classifier}")
-        }
-        // For some reason, the artifact name can differ depending on what repository we used.
-        rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz"
+      if (includeBeats) {
+        from configurations.getByName("filebeat_${architecture.classifier}")
+        from configurations.getByName("metricbeat_${architecture.classifier}")
       }
+      // For some reason, the artifact name can differ depending on what repository we used.
+      rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz"
 
-      String baseSuffix = dockerBase == DockerBase.CLOUD_ESS ? DockerBase.CLOUD.suffix : DockerBase.WOLFI.suffix
+      String baseSuffix = DockerBase.WOLFI.suffix
       from(projectDir.resolve("src/docker/Dockerfile.ess")) {
         expand(
           [
@@ -493,7 +492,7 @@ void addBuildEssDockerImageTask(Architecture architecture, DockerBase dockerBase
   final TaskProvider<DockerBuildTask> buildDockerImageTask =
     tasks.register(taskName("build", architecture, dockerBase, "DockerImage"), DockerBuildTask) {
 
-      DockerBase base = dockerBase == DockerBase.CLOUD_ESS ? DockerBase.CLOUD : DockerBase.WOLFI
+      DockerBase base = DockerBase.WOLFI
 
       TaskProvider<DockerBuildTask> buildBaseTask = tasks.named(taskName("build", architecture, base, "DockerImage"))
       inputs.files(buildBaseTask)
@@ -519,16 +518,15 @@ void addBuildEssDockerImageTask(Architecture architecture, DockerBase dockerBase
 
 for (final Architecture architecture : Architecture.values()) {
   for (final DockerBase base : DockerBase.values()) {
-    if (base == DockerBase.CLOUD_ESS || base == DockerBase.WOLFI_ESS) {
+    if (base == DockerBase.CLOUD_ESS) {
       continue
     }
     addBuildDockerContextTask(architecture, base)
     addTransformDockerContextTask(architecture, base)
     addBuildDockerImageTask(architecture, base)
   }
 
-  addBuildEssDockerImageTask(architecture, DockerBase.CLOUD_ESS)
-  addBuildEssDockerImageTask(architecture, DockerBase.WOLFI_ESS)
+  addBuildEssDockerImageTask(architecture)
 }
 
 def exportDockerImages = tasks.register("exportDockerImages")
@@ -564,8 +562,7 @@ subprojects { Project subProject ->
         (base == DockerBase.CLOUD ? 'cloud.tar' :
           (base == DockerBase.CLOUD_ESS ? 'cloud-ess.tar' :
             (base == DockerBase.WOLFI ? 'wolfi.tar' :
-              (base == DockerBase.WOLFI_ESS ? 'wolfi-ess.tar' :
-                'docker.tar')))))
+                'docker.tar'))))
     final String artifactName = "elasticsearch${arch}${base.suffix}_test"
 
     final String exportTaskName = taskName("export", architecture, base, 'DockerImage')

distribution/docker/src/docker/Dockerfile.ess

@@ -2,34 +2,31 @@ FROM ${base_image} AS builder
 
 USER root
 
-<% if (docker_base == "wolfi_ess") { %>
-    # Add plugins infrastructure
-    RUN mkdir -p /opt/plugins/archive
-    RUN chmod -R 0555 /opt/plugins
-
-    COPY filebeat-${version}.tar.gz metricbeat-${version}.tar.gz /tmp/
-    RUN set -eux ; \\
-        for beat in filebeat metricbeat ; do \\
-          if [ ! -s /tmp/\$beat-${version}.tar.gz ]; then \\
-            echo "/tmp/\$beat-${version}.tar.gz is empty - cannot uncompress" 2>&1 ; \\
-            exit 1 ; \\
-          fi ; \\
-          if ! tar tf /tmp/\$beat-${version}.tar.gz >/dev/null; then \\
-            echo "/tmp/\$beat-${version}.tar.gz is corrupt - cannot uncompress" 2>&1 ; \\
-            exit 1 ; \\
-          fi ; \\
-          mkdir -p /opt/\$beat ; \\
-          tar xf /tmp/\$beat-${version}.tar.gz -C /opt/\$beat --strip-components=1 ; \\
-        done
-<% } %>
+# Add plugins infrastructure
+RUN mkdir -p /opt/plugins/archive
+RUN chmod -R 0555 /opt/plugins
+
+COPY filebeat-${version}.tar.gz metricbeat-${version}.tar.gz /tmp/
+RUN set -eux ; \\
+    for beat in filebeat metricbeat ; do \\
+      if [ ! -s /tmp/\$beat-${version}.tar.gz ]; then \\
+        echo "/tmp/\$beat-${version}.tar.gz is empty - cannot uncompress" 2>&1 ; \\
+        exit 1 ; \\
+      fi ; \\
+      if ! tar tf /tmp/\$beat-${version}.tar.gz >/dev/null; then \\
+        echo "/tmp/\$beat-${version}.tar.gz is corrupt - cannot uncompress" 2>&1 ; \\
+        exit 1 ; \\
+      fi ; \\
+      mkdir -p /opt/\$beat ; \\
+      tar xf /tmp/\$beat-${version}.tar.gz -C /opt/\$beat --strip-components=1 ; \\
+    done
 
 COPY plugins/*.zip /opt/plugins/archive/
 
 RUN chown 1000:1000 /opt/plugins/archive/*
 RUN chmod 0444 /opt/plugins/archive/*
 
 FROM ${base_image}
-<% if (docker_base == "wolfi_ess") { %>
 USER root
 
 RUN <%= retry.loop("apk", "export DEBIAN_FRONTEND=noninteractive && apk update && apk update && apk add --no-cache wget") %>
@@ -44,8 +41,4 @@ RUN mkdir /app && \\
 
 COPY --from=builder --chown=0:0 /opt /opt
 USER 1000:0
-<% } else { %>
-COPY --from=builder /opt/plugins /opt/plugins
-<% } %>
-
 ENV ES_PLUGIN_ARCHIVE_DIR /opt/plugins/archive

qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java

@@ -99,6 +99,7 @@
  *     <li>The default image with a custom, small base image</li>
  *     <li>A UBI-based image</li>
  *     <li>Another UBI image for Iron Bank</li>
+ *     <li>A WOLFI-based image</li>
  *     <li>Images for Cloud</li>
  * </ul>
  */
@@ -170,9 +171,7 @@ public void test012SecurityCanBeDisabled() throws Exception {
     public void test020PluginsListWithNoPlugins() {
         assumeTrue(
             "Only applies to non-Cloud images",
-            distribution.packaging != Packaging.DOCKER_CLOUD
-                && distribution().packaging != Packaging.DOCKER_CLOUD_ESS
-                && distribution().packaging != Packaging.DOCKER_WOLFI_ESS
+            distribution.packaging != Packaging.DOCKER_CLOUD && distribution().packaging != Packaging.DOCKER_CLOUD_ESS
         );
 
         final Installation.Executables bin = installation.executables();
@@ -203,15 +202,14 @@ public void test021InstallPlugin() {
      * Checks that ESS images can install plugins from the local archive.
      */
     public void test022InstallPluginsFromLocalArchive() {
-        assumeTrue(
-            "Only ESS images have a local archive",
-            distribution().packaging == Packaging.DOCKER_CLOUD_ESS || distribution().packaging == Packaging.DOCKER_WOLFI_ESS
-        );
+        assumeTrue("Only ESS images have a local archive", distribution().packaging == Packaging.DOCKER_CLOUD_ESS);
 
         final String plugin = "analysis-icu";
         final Installation.Executables bin = installation.executables();
 
+        listPluginArchive().forEach(System.out::println);
         assertThat("Expected " + plugin + " to not be installed", listPlugins(), not(hasItems(plugin)));
+        assertThat("Expected " + plugin + " available in archive", listPluginArchive(), hasSize(16));
 
         // Stuff the proxy settings with garbage, so any attempt to go out to the internet would fail
         sh.getEnv()
@@ -259,10 +257,7 @@ public void test023InstallPluginUsingConfigFile() {
      * Checks that ESS images can manage plugins from the local archive by deploying a plugins config file.
      */
     public void test024InstallPluginFromArchiveUsingConfigFile() {
-        assumeTrue(
-            "Only ESS image has a plugin archive",
-            distribution().packaging == Packaging.DOCKER_CLOUD_ESS || distribution().packaging == Packaging.DOCKER_WOLFI_ESS
-        );
+        assumeTrue("Only ESS image has a plugin archive", distribution().packaging == Packaging.DOCKER_CLOUD_ESS);
 
         final String filename = "elasticsearch-plugins.yml";
         append(tempDir.resolve(filename), """
@@ -394,7 +389,7 @@ public void test040JavaUsesTheOsProvidedKeystore() {
         if (distribution.packaging == Packaging.DOCKER_UBI || distribution.packaging == Packaging.DOCKER_IRON_BANK) {
             // In these images, the `cacerts` file ought to be a symlink here
             assertThat(path, equalTo("/etc/pki/ca-trust/extracted/java/cacerts"));
-        } else if (distribution.packaging == Packaging.DOCKER_WOLFI || distribution.packaging == Packaging.DOCKER_WOLFI_ESS) {
+        } else if (distribution.packaging == Packaging.DOCKER_WOLFI || distribution.packaging == Packaging.DOCKER_CLOUD_ESS) {
             // In these images, the `cacerts` file ought to be a symlink here
             assertThat(path, equalTo("/etc/ssl/certs/java/cacerts"));
         } else {
@@ -1121,10 +1116,8 @@ public void test170DefaultShellIsBash() {
      */
     public void test171AdditionalCliOptionsAreForwarded() throws Exception {
         assumeTrue(
-            "Does not apply to Cloud and wolfi ess images, because they don't use the default entrypoint",
-            distribution.packaging != Packaging.DOCKER_CLOUD
-                && distribution().packaging != Packaging.DOCKER_CLOUD_ESS
-                && distribution().packaging != Packaging.DOCKER_WOLFI_ESS
+            "Does not apply to Cloud and Cloud ESS images, because they don't use the default entrypoint",
+            distribution.packaging != Packaging.DOCKER_CLOUD && distribution().packaging != Packaging.DOCKER_CLOUD_ESS
         );
 
         runContainer(distribution(), builder().runArgs("bin/elasticsearch", "-Ecluster.name=kimchy").envVar("ELASTIC_PASSWORD", PASSWORD));
@@ -1211,11 +1204,7 @@ public void test310IronBankImageHasNoAdditionalLabels() throws Exception {
      * Check that the Cloud image contains the required Beats
      */
     public void test400CloudImageBundlesBeats() {
-        assumeTrue(
-            distribution.packaging == Packaging.DOCKER_CLOUD
-                || distribution.packaging == Packaging.DOCKER_CLOUD_ESS
-                || distribution.packaging == Packaging.DOCKER_WOLFI_ESS
-        );
+        assumeTrue(distribution.packaging == Packaging.DOCKER_CLOUD || distribution.packaging == Packaging.DOCKER_CLOUD_ESS);
 
         final List<String> contents = listContents("/opt");
         assertThat("Expected beats in /opt", contents, hasItems("filebeat", "metricbeat"));
@@ -1233,6 +1222,10 @@ private List<String> listPlugins() {
         return sh.run(bin.pluginTool + " list").stdout().lines().collect(Collectors.toList());
     }
 
+    private List<String> listPluginArchive() {
+        return sh.run("ls -lh /opt/plugins/archive").stdout().lines().collect(Collectors.toList());
+    }
+
     /**
      * Check that readiness listener works
      */

qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java

@@ -436,7 +436,7 @@ private void verifyKeystorePermissions() {
         switch (distribution.packaging) {
             case TAR, ZIP -> assertThat(keystore, file(File, ARCHIVE_OWNER, ARCHIVE_OWNER, p660));
             case DEB, RPM -> assertThat(keystore, file(File, "root", "elasticsearch", p660));
-            case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI, DOCKER_WOLFI_ESS -> assertThat(
+            case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> assertThat(
                 keystore,
                 DockerFileMatcher.file(p660)
             );