diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java index a1067ddc0ecfe..a6590e99e6caf 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java @@ -29,9 +29,7 @@ import org.apache.hadoop.fs.contract.AbstractFSContract; import org.apache.hadoop.fs.s3a.S3ATestUtils; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; /** * S3A contract tests creating files. @@ -70,11 +68,9 @@ protected AbstractFSContract createContract(Configuration conf) { @Override protected Configuration createConfiguration() { - final Configuration conf = super.createConfiguration(); - removeBaseAndBucketOverrides(conf, - FS_S3A_CREATE_PERFORMANCE, - FS_S3A_PERFORMANCE_FLAGS); - conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, createPerformance); + final Configuration conf = setPerformanceFlags( + super.createConfiguration(), + createPerformance ? "create" : ""); S3ATestUtils.disableFilesystemCaching(conf); return conf; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java index bce67ed67f31d..847f6980b5619 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java @@ -22,9 +22,7 @@ import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; import org.apache.hadoop.fs.contract.AbstractFSContract; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; /** * Test dir operations on S3A. @@ -33,13 +31,9 @@ public class ITestS3AContractMkdir extends AbstractContractMkdirTest { @Override protected Configuration createConfiguration() { - Configuration conf = super.createConfiguration(); - removeBaseAndBucketOverrides( - conf, - FS_S3A_CREATE_PERFORMANCE, - FS_S3A_PERFORMANCE_FLAGS); - conf.set(FS_S3A_PERFORMANCE_FLAGS, ""); - return conf; + return setPerformanceFlags( + super.createConfiguration(), + ""); } @Override diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdirWithCreatePerf.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdirWithCreatePerf.java index cacd6945d2fa0..4b2468de97bb8 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdirWithCreatePerf.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdirWithCreatePerf.java @@ -29,9 +29,7 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; /** * Test mkdir operations on S3A with create performance mode. @@ -40,14 +38,9 @@ public class ITestS3AContractMkdirWithCreatePerf extends AbstractContractMkdirTe @Override protected Configuration createConfiguration() { - Configuration conf = super.createConfiguration(); - removeBaseAndBucketOverrides( - conf, - FS_S3A_CREATE_PERFORMANCE, - FS_S3A_PERFORMANCE_FLAGS); - conf.setStrings(FS_S3A_PERFORMANCE_FLAGS, + return setPerformanceFlags( + super.createConfiguration(), "create,mkdir"); - return conf; } @Override diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFSMainOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFSMainOperations.java index 013ec901d0a77..0281c57f5cbce 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFSMainOperations.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFSMainOperations.java @@ -31,6 +31,7 @@ import static org.apache.hadoop.fs.s3a.S3ATestUtils.createTestPath; import static org.apache.hadoop.fs.s3a.S3ATestUtils.isCreatePerformanceEnabled; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; /** * S3A Test suite for the FSMainOperationsBaseTest tests. @@ -46,7 +47,10 @@ public ITestS3AFSMainOperations() { @Override protected FileSystem createFileSystem() throws Exception { - contract = new S3AContract(new Configuration()); + Configuration conf = setPerformanceFlags( + new Configuration(), + ""); + contract = new S3AContract(conf); contract.init(); return contract.getTestFileSystem(); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java index aa46557e9104b..585317c8daf3a 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java @@ -40,9 +40,7 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.*; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; import static org.apache.hadoop.fs.s3a.Statistic.*; import static org.apache.hadoop.fs.s3a.performance.OperationCost.*; import static org.apache.hadoop.test.GenericTestUtils.getTestDir; @@ -80,12 +78,9 @@ public ITestS3AFileOperationCost( @Override public Configuration createConfiguration() { - final Configuration conf = super.createConfiguration(); - removeBaseAndBucketOverrides(conf, - FS_S3A_CREATE_PERFORMANCE, - FS_S3A_PERFORMANCE_FLAGS); - conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, isKeepingMarkers()); - return conf; + return setPerformanceFlags( + super.createConfiguration(), + isKeepingMarkers() ? "create" : ""); } /** diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java index 56827043c9b82..4808145765822 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java @@ -35,6 +35,7 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.skip; import static org.apache.hadoop.fs.s3a.S3ATestUtils.isCreatePerformanceEnabled; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.junit.Assume.*; import static org.junit.Assert.*; @@ -65,7 +66,9 @@ protected int getGlobalTimeout() { @Before public void setUp() throws Exception { nameThread(); - Configuration conf = new Configuration(); + Configuration conf = setPerformanceFlags( + new Configuration(), + ""); fs = S3ATestUtils.createTestFileSystem(conf); assumeNotNull(fs); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index d8bb5898aa076..3a3f875f5f0d5 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -1127,6 +1127,25 @@ public static void assumeStoreAwsHosted(final FileSystem fs) { .getTrimmed(ENDPOINT, DEFAULT_ENDPOINT))); } + /** + * Modify the config by setting the performance flags and return the modified config. + * + * @param conf The configuration object. + * @param flagStr The performance flag string. + * @return The modified configuration object. + */ + public static Configuration setPerformanceFlags(final Configuration conf, + final String flagStr) { + removeBaseAndBucketOverrides( + conf, + FS_S3A_CREATE_PERFORMANCE, + FS_S3A_PERFORMANCE_FLAGS); + if (flagStr != null) { + conf.set(FS_S3A_PERFORMANCE_FLAGS, flagStr); + } + return conf; + } + /** * Helper class to do diffs of metrics. */ diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdir.java index ce7e9d4a43d80..095d2239eed70 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdir.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdir.java @@ -18,9 +18,7 @@ import org.apache.hadoop.fs.s3a.S3ATestUtils; import org.junit.Before; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; /** * Extends FileContextCreateMkdirBaseTest for a S3a FileContext. @@ -30,11 +28,9 @@ public class ITestS3AFileContextCreateMkdir @Before public void setUp() throws Exception { - Configuration conf = new Configuration(); - removeBaseAndBucketOverrides( - conf, - FS_S3A_CREATE_PERFORMANCE, - FS_S3A_PERFORMANCE_FLAGS); + Configuration conf = setPerformanceFlags( + new Configuration(), + null); fc = S3ATestUtils.createTestFileContext(conf); super.setUp(); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdirCreatePerf.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdirCreatePerf.java index 64039e4c5206c..68dde70bfeb50 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdirCreatePerf.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdirCreatePerf.java @@ -20,9 +20,7 @@ import org.apache.hadoop.fs.FileContextCreateMkdirBaseTest; import org.apache.hadoop.fs.s3a.S3ATestUtils; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; import static org.apache.hadoop.test.LambdaTestUtils.intercept; /** @@ -34,12 +32,8 @@ public class ITestS3AFileContextCreateMkdirCreatePerf @Before public void setUp() throws Exception { - Configuration conf = new Configuration(); - removeBaseAndBucketOverrides( - conf, - FS_S3A_CREATE_PERFORMANCE, - FS_S3A_PERFORMANCE_FLAGS); - conf.setStrings(FS_S3A_PERFORMANCE_FLAGS, + Configuration conf = setPerformanceFlags( + new Configuration(), "mkdir"); fc = S3ATestUtils.createTestFileContext(conf); super.setUp(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextMainOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextMainOperations.java index b28f88e43b42e..cc630484a131c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextMainOperations.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextMainOperations.java @@ -27,6 +27,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3a.S3ATestUtils; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; + /** * S3A implementation of FileContextMainOperationsBaseTest. */ @@ -36,7 +38,10 @@ public class ITestS3AFileContextMainOperations @Before public void setUp() throws IOException, Exception { - Configuration conf = new Configuration(); + Configuration conf = setPerformanceFlags( + new Configuration(), + ""); + fc = S3ATestUtils.createTestFileContext(conf); super.setUp(); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java index bef359cca73c0..54161d10128e5 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java @@ -21,6 +21,8 @@ import org.junit.Ignore; import org.junit.Test; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; + /** * S3a implementation of FileContextURIBase. */ @@ -30,7 +32,9 @@ public class ITestS3AFileContextURI extends FileContextURIBase { @Before public void setUp() throws IOException, Exception { - conf = new Configuration(); + conf = setPerformanceFlags( + new Configuration(), + ""); fc1 = S3ATestUtils.createTestFileContext(conf); fc2 = S3ATestUtils.createTestFileContext(conf); //different object, same FS diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestCreateFileCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestCreateFileCost.java index 5bd4bf412ffa5..65786bf6d6919 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestCreateFileCost.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestCreateFileCost.java @@ -42,9 +42,8 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.toChar; import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_HEADER; import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS; import static org.apache.hadoop.fs.s3a.Constants.XA_HEADER_PREFIX; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_BULK_DELETE_REQUEST; import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_DELETE_REQUEST; import static org.apache.hadoop.fs.s3a.performance.OperationCost.CREATE_FILE_NO_OVERWRITE; @@ -105,11 +104,9 @@ private OperationCost expected(OperationCost source) { @Override public Configuration createConfiguration() { - final Configuration conf = super.createConfiguration(); - removeBaseAndBucketOverrides(conf, - FS_S3A_CREATE_PERFORMANCE, - FS_S3A_PERFORMANCE_FLAGS); - conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, createPerformance); + final Configuration conf = setPerformanceFlags( + super.createConfiguration(), + createPerformance ? "create" : ""); S3ATestUtils.disableFilesystemCaching(conf); return conf; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java index 9979b72e7110d..6bd4114f07cc3 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java @@ -39,9 +39,7 @@ import org.apache.hadoop.fs.s3a.Tristate; import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; import static org.apache.hadoop.fs.s3a.Statistic.*; import static org.apache.hadoop.fs.s3a.performance.OperationCost.*; import static org.apache.hadoop.fs.s3a.performance.OperationCostValidator.probe; @@ -80,12 +78,9 @@ public ITestS3ADeleteCost(final String name, @Override public Configuration createConfiguration() { - Configuration conf = super.createConfiguration(); - removeBaseAndBucketOverrides(conf, - FS_S3A_CREATE_PERFORMANCE, - FS_S3A_PERFORMANCE_FLAGS); - conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, false); - return conf; + return setPerformanceFlags( + super.createConfiguration(), + ""); } @Override