HADOOP-19072 S3A: Override fs.s3a.performance.flags for tests (ADDEND…

…UM 2) (#6993) Second followup to #6543; all hadoop-aws integration tests complete correctly even when fs.s3a.performance.flags = * Contributed by Viraj Jasani
apache · Aug 14, 2024 · fa83c9a · fa83c9a
1 parent 74ff007
commit fa83c9a
Show file tree

Hide file tree

Showing 13 changed files with 69 additions and 74 deletions.
diff --git a/...ls/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java b/...ls/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java
@@ -29,9 +29,7 @@
 import org.apache.hadoop.fs.contract.AbstractFSContract;
 import org.apache.hadoop.fs.s3a.S3ATestUtils;
 
-import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
-import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
 
 /**
  * S3A contract tests creating files.
@@ -70,11 +68,9 @@ protected AbstractFSContract createContract(Configuration conf) {
 
   @Override
   protected Configuration createConfiguration() {
-    final Configuration conf = super.createConfiguration();
-    removeBaseAndBucketOverrides(conf,
-        FS_S3A_CREATE_PERFORMANCE,
-        FS_S3A_PERFORMANCE_FLAGS);
-    conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, createPerformance);
+    final Configuration conf = setPerformanceFlags(
+        super.createConfiguration(),
+        createPerformance ? "create" : "");
     S3ATestUtils.disableFilesystemCaching(conf);
     return conf;
   }

diff --git a/...ols/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java b/...ols/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java
@@ -22,9 +22,7 @@
 import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
 import org.apache.hadoop.fs.contract.AbstractFSContract;
 
-import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
-import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
 
 /**
  * Test dir operations on S3A.
@@ -33,13 +31,9 @@ public class ITestS3AContractMkdir extends AbstractContractMkdirTest {
 
   @Override
   protected Configuration createConfiguration() {
-    Configuration conf = super.createConfiguration();
-    removeBaseAndBucketOverrides(
-        conf,
-        FS_S3A_CREATE_PERFORMANCE,
-        FS_S3A_PERFORMANCE_FLAGS);
-    conf.set(FS_S3A_PERFORMANCE_FLAGS, "");
-    return conf;
+    return setPerformanceFlags(
+        super.createConfiguration(),
+        "");
   }
 
   @Override

diff --git a/.../src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdirWithCreatePerf.java b/.../src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdirWithCreatePerf.java
@@ -29,9 +29,7 @@
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
-import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
-import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
 
 /**
  * Test mkdir operations on S3A with create performance mode.
@@ -40,14 +38,9 @@ public class ITestS3AContractMkdirWithCreatePerf extends AbstractContractMkdirTe
 
   @Override
   protected Configuration createConfiguration() {
-    Configuration conf = super.createConfiguration();
-    removeBaseAndBucketOverrides(
-        conf,
-        FS_S3A_CREATE_PERFORMANCE,
-        FS_S3A_PERFORMANCE_FLAGS);
-    conf.setStrings(FS_S3A_PERFORMANCE_FLAGS,
+    return setPerformanceFlags(
+        super.createConfiguration(),
         "create,mkdir");
-    return conf;
   }
 
   @Override

diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFSMainOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFSMainOperations.java
@@ -31,6 +31,7 @@
 
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.createTestPath;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.isCreatePerformanceEnabled;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
 
 /**
  * S3A Test suite for the FSMainOperationsBaseTest tests.
@@ -46,7 +47,10 @@ public ITestS3AFSMainOperations() {
 
   @Override
   protected FileSystem createFileSystem() throws Exception {
-    contract = new S3AContract(new Configuration());
+    Configuration conf = setPerformanceFlags(
+        new Configuration(),
+        "");
+    contract = new S3AContract(conf);
     contract.init();
     return contract.getTestFileSystem();
   }

diff --git a/...op-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java b/...op-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
@@ -40,9 +40,7 @@
 
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
-import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
-import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
 import static org.apache.hadoop.fs.s3a.Statistic.*;
 import static org.apache.hadoop.fs.s3a.performance.OperationCost.*;
 import static org.apache.hadoop.test.GenericTestUtils.getTestDir;
@@ -80,12 +78,9 @@ public ITestS3AFileOperationCost(
 
   @Override
   public Configuration createConfiguration() {
-    final Configuration conf = super.createConfiguration();
-    removeBaseAndBucketOverrides(conf,
-        FS_S3A_CREATE_PERFORMANCE,
-        FS_S3A_PERFORMANCE_FLAGS);
-    conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, isKeepingMarkers());
-    return conf;
+    return setPerformanceFlags(
+        super.createConfiguration(),
+        isKeepingMarkers() ? "create" : "");
   }
 
   /**

diff --git a/...p-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java b/...p-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
@@ -35,6 +35,7 @@
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.isCreatePerformanceEnabled;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 import static org.junit.Assume.*;
 import static org.junit.Assert.*;
@@ -65,7 +66,9 @@ protected int getGlobalTimeout() {
   @Before
   public void setUp() throws Exception {
     nameThread();
-    Configuration conf = new Configuration();
+    Configuration conf = setPerformanceFlags(
+        new Configuration(),
+        "");
 
     fs = S3ATestUtils.createTestFileSystem(conf);
     assumeNotNull(fs);

diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
@@ -1127,6 +1127,25 @@ public static void assumeStoreAwsHosted(final FileSystem fs) {
             .getTrimmed(ENDPOINT, DEFAULT_ENDPOINT)));
   }
 
+  /**
+   * Modify the config by setting the performance flags and return the modified config.
+   *
+   * @param conf The configuration object.
+   * @param flagStr The performance flag string.
+   * @return The modified configuration object.
+   */
+  public static Configuration setPerformanceFlags(final Configuration conf,
+      final String flagStr) {
+    removeBaseAndBucketOverrides(
+        conf,
+        FS_S3A_CREATE_PERFORMANCE,
+        FS_S3A_PERFORMANCE_FLAGS);
+    if (flagStr != null) {
+      conf.set(FS_S3A_PERFORMANCE_FLAGS, flagStr);
+    }
+    return conf;
+  }
+
   /**
    * Helper class to do diffs of metrics.
    */

diff --git a/...ws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdir.java b/...ws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdir.java
@@ -18,9 +18,7 @@
 import org.apache.hadoop.fs.s3a.S3ATestUtils;
 import org.junit.Before;
 
-import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
-import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
 
 /**
  * Extends FileContextCreateMkdirBaseTest for a S3a FileContext.
@@ -30,11 +28,9 @@ public class ITestS3AFileContextCreateMkdir
 
   @Before
   public void setUp() throws Exception {
-    Configuration conf = new Configuration();
-    removeBaseAndBucketOverrides(
-        conf,
-        FS_S3A_CREATE_PERFORMANCE,
-        FS_S3A_PERFORMANCE_FLAGS);
+    Configuration conf = setPerformanceFlags(
+        new Configuration(),
+        null);
     fc = S3ATestUtils.createTestFileContext(conf);
     super.setUp();
   }

diff --git a/...t/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdirCreatePerf.java b/...t/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdirCreatePerf.java
@@ -20,9 +20,7 @@
 import org.apache.hadoop.fs.FileContextCreateMkdirBaseTest;
 import org.apache.hadoop.fs.s3a.S3ATestUtils;
 
-import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
-import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 
 /**
@@ -34,12 +32,8 @@ public class ITestS3AFileContextCreateMkdirCreatePerf
 
   @Before
   public void setUp() throws Exception {
-    Configuration conf = new Configuration();
-    removeBaseAndBucketOverrides(
-        conf,
-        FS_S3A_CREATE_PERFORMANCE,
-        FS_S3A_PERFORMANCE_FLAGS);
-    conf.setStrings(FS_S3A_PERFORMANCE_FLAGS,
+    Configuration conf = setPerformanceFlags(
+        new Configuration(),
         "mkdir");
     fc = S3ATestUtils.createTestFileContext(conf);
     super.setUp();

diff --git a/...src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextMainOperations.java b/...src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextMainOperations.java
@@ -27,6 +27,8 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.s3a.S3ATestUtils;
 
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
+
 /**
  * S3A implementation of FileContextMainOperationsBaseTest.
  */
@@ -36,7 +38,10 @@ public class ITestS3AFileContextMainOperations
 
   @Before
   public void setUp() throws IOException, Exception {
-    Configuration conf = new Configuration();
+    Configuration conf = setPerformanceFlags(
+        new Configuration(),
+        "");
+
     fc = S3ATestUtils.createTestFileContext(conf);
     super.setUp();
   }

diff --git a/...hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java b/...hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java
@@ -21,6 +21,8 @@
 import org.junit.Ignore;
 import org.junit.Test;
 
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
+
 /**
  * S3a implementation of FileContextURIBase.
  */
@@ -30,7 +32,9 @@ public class ITestS3AFileContextURI extends FileContextURIBase {
 
   @Before
   public void setUp() throws IOException, Exception {
-    conf = new Configuration();
+    conf = setPerformanceFlags(
+        new Configuration(),
+        "");
 
     fc1 = S3ATestUtils.createTestFileContext(conf);
     fc2 = S3ATestUtils.createTestFileContext(conf); //different object, same FS

diff --git a/...ls/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestCreateFileCost.java b/...ls/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestCreateFileCost.java
@@ -42,9 +42,8 @@
 import static org.apache.hadoop.fs.contract.ContractTestUtils.toChar;
 import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_HEADER;
 import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
-import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
 import static org.apache.hadoop.fs.s3a.Constants.XA_HEADER_PREFIX;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
 import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_BULK_DELETE_REQUEST;
 import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_DELETE_REQUEST;
 import static org.apache.hadoop.fs.s3a.performance.OperationCost.CREATE_FILE_NO_OVERWRITE;
@@ -105,11 +104,9 @@ private OperationCost expected(OperationCost source) {
 
   @Override
   public Configuration createConfiguration() {
-    final Configuration conf = super.createConfiguration();
-    removeBaseAndBucketOverrides(conf,
-        FS_S3A_CREATE_PERFORMANCE,
-        FS_S3A_PERFORMANCE_FLAGS);
-    conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, createPerformance);
+    final Configuration conf = setPerformanceFlags(
+        super.createConfiguration(),
+        createPerformance ? "create" : "");
     S3ATestUtils.disableFilesystemCaching(conf);
     return conf;
   }

diff --git a/...ols/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java b/...ols/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java
@@ -39,9 +39,7 @@
 import org.apache.hadoop.fs.s3a.Tristate;
 import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum;
 
-import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
-import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
 import static org.apache.hadoop.fs.s3a.Statistic.*;
 import static org.apache.hadoop.fs.s3a.performance.OperationCost.*;
 import static org.apache.hadoop.fs.s3a.performance.OperationCostValidator.probe;
@@ -80,12 +78,9 @@ public ITestS3ADeleteCost(final String name,
 
   @Override
   public Configuration createConfiguration() {
-    Configuration conf = super.createConfiguration();
-    removeBaseAndBucketOverrides(conf,
-        FS_S3A_CREATE_PERFORMANCE,
-        FS_S3A_PERFORMANCE_FLAGS);
-    conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, false);
-    return conf;
+    return setPerformanceFlags(
+        super.createConfiguration(),
+        "");
   }
 
   @Override