diff --git a/clients/hadoopfs/src/test/java/io/lakefs/FSTestBase.java b/clients/hadoopfs/src/test/java/io/lakefs/FSTestBase.java index 6b41dd1da4b..de4a9fd3c66 100644 --- a/clients/hadoopfs/src/test/java/io/lakefs/FSTestBase.java +++ b/clients/hadoopfs/src/test/java/io/lakefs/FSTestBase.java @@ -121,20 +121,12 @@ protected String getS3Key(StagingLocation stagingLocation) { return removeStart(stagingLocation.getPhysicalAddress(), s3Base); } - /** - * Override to add to Hadoop configuration. - */ - protected void addHadoopConfiguration(Configuration conf) { - } - @Before public void hadoopSetup() throws IOException, URISyntaxException { s3Base = "s3a://UNUSED/"; // Overridden if S3 will be used! conf = new Configuration(false); - addHadoopConfiguration(conf); - conf.set("fs.lakefs.impl", "io.lakefs.LakeFSFileSystem"); conf.set("fs.lakefs.access.key", "unused-but-checked"); diff --git a/clients/hadoopfs/src/test/java/io/lakefs/LakeFSFileSystemServerS3Test.java b/clients/hadoopfs/src/test/java/io/lakefs/LakeFSFileSystemServerS3Test.java index 2fae94fed72..3e2c50f2eb1 100644 --- a/clients/hadoopfs/src/test/java/io/lakefs/LakeFSFileSystemServerS3Test.java +++ b/clients/hadoopfs/src/test/java/io/lakefs/LakeFSFileSystemServerS3Test.java @@ -15,6 +15,7 @@ import com.amazonaws.services.s3.model.*; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -45,12 +46,13 @@ public static Iterable data() { public String unusedAddressCreatorType; @Parameter(0) - public PhysicalAddressCreator pac; + public PhysicalAddressCreator pacParam; @Override - protected void moreHadoopSetup() { - super.moreHadoopSetup(); - pac.initConfiguration(conf); + // pac is defined in a superclass but parametrized here. + protected void setPac() { + pac = pacParam; + LOG.info("Set pac!"); } @Test diff --git a/clients/hadoopfs/src/test/java/io/lakefs/S3FSTestBase.java b/clients/hadoopfs/src/test/java/io/lakefs/S3FSTestBase.java index 46a404d766b..1a97bb39f08 100644 --- a/clients/hadoopfs/src/test/java/io/lakefs/S3FSTestBase.java +++ b/clients/hadoopfs/src/test/java/io/lakefs/S3FSTestBase.java @@ -49,10 +49,10 @@ public abstract class S3FSTestBase extends FSTestBase { protected String s3Endpoint; protected AmazonS3 s3Client; - private static final DockerImageName MINIO = DockerImageName.parse("minio/minio:RELEASE.2021-06-07T21-40-51Z"); + private static final DockerImageName MINIO = DockerImageName.parse("minio/minio:RELEASE.2024-11-07T00-52-20Z"); @Rule - public final GenericContainer s3 = new GenericContainer(MINIO.toString()). + public final GenericContainer s3 = new GenericContainer(MINIO). withCommand("minio", "server", "/data"). withEnv("MINIO_ROOT_USER", S3_ACCESS_KEY_ID). withEnv("MINIO_ROOT_PASSWORD", S3_SECRET_ACCESS_KEY). @@ -74,7 +74,7 @@ public void s3ClientSetup() { ClientConfiguration clientConfiguration = new ClientConfiguration() .withSignerOverride("AWSS3V4SignerType"); - s3Endpoint = String.format("http://s3.local.lakefs.io:%d", s3.getMappedPort(9000)); + s3Endpoint = String.format("http://s3.local.lakefs.io:%d/", s3.getMappedPort(9000)); s3Client = new AmazonS3Client(creds, clientConfiguration); @@ -85,7 +85,8 @@ public void s3ClientSetup() { s3Bucket = makeS3BucketName(); s3Base = String.format("s3://%s/", s3Bucket); - LOG.info("S3: bucket {} => base URL {}", s3Bucket, s3Base); + LOG.info("S3 [endpoint {}]: bucket {} => base URL {}", + s3Endpoint, s3Bucket, s3Base); CreateBucketRequest cbr = new CreateBucketRequest(s3Bucket); s3Client.createBucket(cbr); @@ -126,7 +127,10 @@ protected void assertS3Object(StagingLocation stagingLocation, String contents) } } + protected void setPac() {} + protected void moreHadoopSetup() { + setPac(); s3ClientSetup(); conf.set("fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem"); @@ -134,6 +138,9 @@ protected void moreHadoopSetup() { conf.set(org.apache.hadoop.fs.s3a.Constants.SECRET_KEY, S3_SECRET_ACCESS_KEY); conf.set(org.apache.hadoop.fs.s3a.Constants.ENDPOINT, s3Endpoint); conf.set(org.apache.hadoop.fs.s3a.Constants.BUFFER_DIR, "/tmp/s3a"); + pac.initConfiguration(conf); + + LOG.info("Setup done!"); } public static interface PhysicalAddressCreator {