From 5462e000e364eb54f7601a841cf7a356c0252c31 Mon Sep 17 00:00:00 2001 From: Mariusz Jozala Date: Thu, 5 Dec 2024 12:34:29 +0100 Subject: [PATCH 1/5] [ci] Add debian-12 to matrix in packaging and platform jobs --- .buildkite/pipelines/periodic-packaging.template.yml | 1 + .buildkite/pipelines/periodic-packaging.yml | 1 + .buildkite/pipelines/periodic-platform-support.yml | 1 + .buildkite/pipelines/pull-request/packaging-tests-unix.yml | 1 + 4 files changed, 4 insertions(+) diff --git a/.buildkite/pipelines/periodic-packaging.template.yml b/.buildkite/pipelines/periodic-packaging.template.yml index aff0add62a2b6..b00ef51ec8f07 100644 --- a/.buildkite/pipelines/periodic-packaging.template.yml +++ b/.buildkite/pipelines/periodic-packaging.template.yml @@ -18,6 +18,7 @@ steps: - rhel-8 - rhel-9 - almalinux-8 + - almalinux-9 agents: provider: gcp image: family/elasticsearch-{{matrix.image}} diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 9bcd61ac1273c..c58201258fbbf 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -19,6 +19,7 @@ steps: - rhel-8 - rhel-9 - almalinux-8 + - almalinux-9 agents: provider: gcp image: family/elasticsearch-{{matrix.image}} diff --git a/.buildkite/pipelines/periodic-platform-support.yml b/.buildkite/pipelines/periodic-platform-support.yml index 8bee3a78f8316..33e9a040b22cf 100644 --- a/.buildkite/pipelines/periodic-platform-support.yml +++ b/.buildkite/pipelines/periodic-platform-support.yml @@ -18,6 +18,7 @@ steps: - rhel-8 - rhel-9 - almalinux-8 + - almalinux-9 agents: provider: gcp image: family/elasticsearch-{{matrix.image}} diff --git a/.buildkite/pipelines/pull-request/packaging-tests-unix.yml b/.buildkite/pipelines/pull-request/packaging-tests-unix.yml index 6c7dadfd454ed..3b7973d05a338 100644 --- a/.buildkite/pipelines/pull-request/packaging-tests-unix.yml +++ b/.buildkite/pipelines/pull-request/packaging-tests-unix.yml @@ -21,6 +21,7 @@ steps: - rhel-8 - rhel-9 - almalinux-8 + - almalinux-9 PACKAGING_TASK: - docker - docker-cloud-ess From 0e4ce2098b6cd5f8327c4396b34be0f42a9ef9b7 Mon Sep 17 00:00:00 2001 From: Mariusz Jozala Date: Tue, 10 Dec 2024 13:30:11 +0100 Subject: [PATCH 2/5] Upgrade SmbTestContainer base image Upgraded SmbTestContainer base image from Ubuntu 16.04 to 24.04 to avoid hanging Python module compilation when installing samba package. Installing SMB had to be moved from container building to the starting because SYS_ADMIN capability is required. --- x-pack/test/smb-fixture/build.gradle | 2 ++ .../test/fixtures/smb/SmbTestContainer.java | 21 +++++++++++++------ .../resources/smb/provision/installsmb.sh | 2 +- 3 files changed, 18 insertions(+), 7 deletions(-) mode change 100644 => 100755 x-pack/test/smb-fixture/src/main/resources/smb/provision/installsmb.sh diff --git a/x-pack/test/smb-fixture/build.gradle b/x-pack/test/smb-fixture/build.gradle index aeb5626ce9508..a982259edb2dd 100644 --- a/x-pack/test/smb-fixture/build.gradle +++ b/x-pack/test/smb-fixture/build.gradle @@ -2,6 +2,8 @@ apply plugin: 'elasticsearch.java' apply plugin: 'elasticsearch.cache-test-fixtures' dependencies { + implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" + api project(':test:fixtures:testcontainer-utils') api "junit:junit:${versions.junit}" api "org.testcontainers:testcontainers:${versions.testcontainer}" diff --git a/x-pack/test/smb-fixture/src/main/java/org/elasticsearch/test/fixtures/smb/SmbTestContainer.java b/x-pack/test/smb-fixture/src/main/java/org/elasticsearch/test/fixtures/smb/SmbTestContainer.java index 10f589e4e1df3..5242f470af7a3 100644 --- a/x-pack/test/smb-fixture/src/main/java/org/elasticsearch/test/fixtures/smb/SmbTestContainer.java +++ b/x-pack/test/smb-fixture/src/main/java/org/elasticsearch/test/fixtures/smb/SmbTestContainer.java @@ -7,12 +7,15 @@ package org.elasticsearch.test.fixtures.smb; +import com.github.dockerjava.api.model.Capability; + import org.elasticsearch.test.fixtures.testcontainers.DockerEnvironmentAwareTestContainer; +import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.images.builder.ImageFromDockerfile; public final class SmbTestContainer extends DockerEnvironmentAwareTestContainer { - private static final String DOCKER_BASE_IMAGE = "ubuntu:16.04"; + private static final String DOCKER_BASE_IMAGE = "ubuntu:24.04"; public static final int AD_LDAP_PORT = 636; public static final int AD_LDAP_GC_PORT = 3269; @@ -20,15 +23,15 @@ public SmbTestContainer() { super( new ImageFromDockerfile("es-smb-fixture").withDockerfileFromBuilder( builder -> builder.from(DOCKER_BASE_IMAGE) - .run("apt-get update -qqy && apt-get install -qqy samba ldap-utils") + .env("TZ", "Etc/UTC") + .run("DEBIAN_FRONTEND=noninteractive apt-get update -qqy && apt-get install -qqy tzdata winbind samba ldap-utils") .copy("fixture/provision/installsmb.sh", "/fixture/provision/installsmb.sh") .copy("fixture/certs/ca.key", "/fixture/certs/ca.key") .copy("fixture/certs/ca.pem", "/fixture/certs/ca.pem") .copy("fixture/certs/cert.pem", "/fixture/certs/cert.pem") .copy("fixture/certs/key.pem", "/fixture/certs/key.pem") .run("chmod +x /fixture/provision/installsmb.sh") - .run("/fixture/provision/installsmb.sh") - .cmd("service samba-ad-dc restart && sleep infinity") + .cmd("/fixture/provision/installsmb.sh && service samba-ad-dc restart && echo Samba started && sleep infinity") .build() ) .withFileFromClasspath("fixture/provision/installsmb.sh", "/smb/provision/installsmb.sh") @@ -37,10 +40,16 @@ public SmbTestContainer() { .withFileFromClasspath("fixture/certs/cert.pem", "/smb/certs/cert.pem") .withFileFromClasspath("fixture/certs/key.pem", "/smb/certs/key.pem") ); - // addExposedPort(389); - // addExposedPort(3268); + addExposedPort(AD_LDAP_PORT); addExposedPort(AD_LDAP_GC_PORT); + + setWaitStrategy(Wait.forLogMessage(".*Samba started.*", 1)); + + getCreateContainerCmdModifiers().add(createContainerCmd -> { + createContainerCmd.getHostConfig().withCapAdd(Capability.SYS_ADMIN); + return createContainerCmd; + }); } public String getAdLdapUrl() { diff --git a/x-pack/test/smb-fixture/src/main/resources/smb/provision/installsmb.sh b/x-pack/test/smb-fixture/src/main/resources/smb/provision/installsmb.sh old mode 100644 new mode 100755 index 463238b9f50c2..fe939431bb435 --- a/x-pack/test/smb-fixture/src/main/resources/smb/provision/installsmb.sh +++ b/x-pack/test/smb-fixture/src/main/resources/smb/provision/installsmb.sh @@ -21,7 +21,7 @@ cat $SSL_DIR/ca.pem >> /etc/ssl/certs/ca-certificates.crt mv /etc/samba/smb.conf /etc/samba/smb.conf.orig -samba-tool domain provision --server-role=dc --use-rfc2307 --dns-backend=SAMBA_INTERNAL --realm=AD.TEST.ELASTICSEARCH.COM --domain=ADES --adminpass=Passw0rd --use-ntvfs +samba-tool domain provision --server-role=dc --use-rfc2307 --dns-backend=SAMBA_INTERNAL --realm=AD.TEST.ELASTICSEARCH.COM --domain=ADES --adminpass=Passw0rd cp /var/lib/samba/private/krb5.conf /etc/krb5.conf From fef99c79dddde319e26e1a3662d887deccff4378 Mon Sep 17 00:00:00 2001 From: Mariusz Jozala Date: Tue, 10 Dec 2024 18:13:11 +0100 Subject: [PATCH 3/5] Improve waiting strategy for SMB test container --- .../elasticsearch/test/fixtures/smb/SmbTestContainer.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/x-pack/test/smb-fixture/src/main/java/org/elasticsearch/test/fixtures/smb/SmbTestContainer.java b/x-pack/test/smb-fixture/src/main/java/org/elasticsearch/test/fixtures/smb/SmbTestContainer.java index 5242f470af7a3..de7fb2f41e3f8 100644 --- a/x-pack/test/smb-fixture/src/main/java/org/elasticsearch/test/fixtures/smb/SmbTestContainer.java +++ b/x-pack/test/smb-fixture/src/main/java/org/elasticsearch/test/fixtures/smb/SmbTestContainer.java @@ -11,6 +11,7 @@ import org.elasticsearch.test.fixtures.testcontainers.DockerEnvironmentAwareTestContainer; import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.containers.wait.strategy.WaitAllStrategy; import org.testcontainers.images.builder.ImageFromDockerfile; public final class SmbTestContainer extends DockerEnvironmentAwareTestContainer { @@ -44,7 +45,11 @@ public SmbTestContainer() { addExposedPort(AD_LDAP_PORT); addExposedPort(AD_LDAP_GC_PORT); - setWaitStrategy(Wait.forLogMessage(".*Samba started.*", 1)); + setWaitStrategy( + new WaitAllStrategy() + .withStrategy(Wait.forLogMessage(".*Samba started.*", 1)) + .withStrategy(Wait.forListeningPort()) + ); getCreateContainerCmdModifiers().add(createContainerCmd -> { createContainerCmd.getHostConfig().withCapAdd(Capability.SYS_ADMIN); From 6821d99531bc1540eca78a2eae4b17988203830a Mon Sep 17 00:00:00 2001 From: Mariusz Jozala Date: Tue, 10 Dec 2024 18:37:07 +0100 Subject: [PATCH 4/5] Spotless fix --- .../org/elasticsearch/test/fixtures/smb/SmbTestContainer.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/test/smb-fixture/src/main/java/org/elasticsearch/test/fixtures/smb/SmbTestContainer.java b/x-pack/test/smb-fixture/src/main/java/org/elasticsearch/test/fixtures/smb/SmbTestContainer.java index de7fb2f41e3f8..b562aab140c11 100644 --- a/x-pack/test/smb-fixture/src/main/java/org/elasticsearch/test/fixtures/smb/SmbTestContainer.java +++ b/x-pack/test/smb-fixture/src/main/java/org/elasticsearch/test/fixtures/smb/SmbTestContainer.java @@ -46,9 +46,7 @@ public SmbTestContainer() { addExposedPort(AD_LDAP_GC_PORT); setWaitStrategy( - new WaitAllStrategy() - .withStrategy(Wait.forLogMessage(".*Samba started.*", 1)) - .withStrategy(Wait.forListeningPort()) + new WaitAllStrategy().withStrategy(Wait.forLogMessage(".*Samba started.*", 1)).withStrategy(Wait.forListeningPort()) ); getCreateContainerCmdModifiers().add(createContainerCmd -> { From 2035fb0fd95b928541b7770cdca502322bff108f Mon Sep 17 00:00:00 2001 From: Mariusz Jozala Date: Wed, 11 Dec 2024 12:07:58 +0100 Subject: [PATCH 5/5] Longer timeout for starting container to avoid flakiness --- .../elasticsearch/test/fixtures/smb/SmbTestContainer.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/test/smb-fixture/src/main/java/org/elasticsearch/test/fixtures/smb/SmbTestContainer.java b/x-pack/test/smb-fixture/src/main/java/org/elasticsearch/test/fixtures/smb/SmbTestContainer.java index b562aab140c11..27d8257f4be10 100644 --- a/x-pack/test/smb-fixture/src/main/java/org/elasticsearch/test/fixtures/smb/SmbTestContainer.java +++ b/x-pack/test/smb-fixture/src/main/java/org/elasticsearch/test/fixtures/smb/SmbTestContainer.java @@ -14,6 +14,8 @@ import org.testcontainers.containers.wait.strategy.WaitAllStrategy; import org.testcontainers.images.builder.ImageFromDockerfile; +import java.time.Duration; + public final class SmbTestContainer extends DockerEnvironmentAwareTestContainer { private static final String DOCKER_BASE_IMAGE = "ubuntu:24.04"; @@ -46,7 +48,9 @@ public SmbTestContainer() { addExposedPort(AD_LDAP_GC_PORT); setWaitStrategy( - new WaitAllStrategy().withStrategy(Wait.forLogMessage(".*Samba started.*", 1)).withStrategy(Wait.forListeningPort()) + new WaitAllStrategy().withStartupTimeout(Duration.ofSeconds(120)) + .withStrategy(Wait.forLogMessage(".*Samba started.*", 1)) + .withStrategy(Wait.forListeningPort()) ); getCreateContainerCmdModifiers().add(createContainerCmd -> {