diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md index b8cb3ff732b36..1cee1bc937011 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md @@ -295,6 +295,7 @@ understands the risks. * If an option is to be tuned which may relax semantics, a new option MUST be defined. * Unknown flags are ignored; this is to avoid compatibility. * The option `*` means "turn everything on". This is implicitly unstable across releases. +* Other stores may retain stricter semantics. | *Option* | *Meaning* | Since | |----------|--------------------|:------| diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md index 7222eee98baeb..16384f275e924 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md @@ -628,6 +628,10 @@ on third party stores. test.fs.s3a.create.create.acl.enabled false + + test.fs.s3a.perf.enabled + false + ``` See [Third Party Stores](third_party_stores.html) for more on this topic. @@ -767,6 +771,18 @@ Tests in `ITestS3AContentEncoding` may need disabling false ``` + +### Disabling tests running in performance mode + +Some tests running in performance mode turn off the safety checks. They expect breaking posix semantics. +For stores with stricter semantics, these test cases must be disabled. +```xml + + test.fs.s3a.perf.enabled + false + +``` + ### Tests which may fail (and which you can ignore) * `ITestS3AContractMultipartUploader` tests `testMultipartUploadAbort` and `testSingleUpload` raising `FileNotFoundException` diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java index 000caf328837e..033c2d94c7bf8 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java @@ -29,9 +29,11 @@ import org.apache.hadoop.fs.contract.AbstractFSContract; import org.apache.hadoop.fs.s3a.S3ATestUtils; +import static org.apache.hadoop.fs.s3a.S3ATestConstants.KEY_PERFORMANCE_TESTS_ENABLED; import static org.apache.hadoop.fs.s3a.Constants.CONNECTION_EXPECT_CONTINUE; import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfNotEnabled; /** * S3A contract tests creating files. @@ -84,6 +86,9 @@ protected Configuration createConfiguration() { conf, CONNECTION_EXPECT_CONTINUE); conf.setBoolean(CONNECTION_EXPECT_CONTINUE, expectContinue); + if (createPerformance) { + skipIfNotEnabled(conf, KEY_PERFORMANCE_TESTS_ENABLED, "Skipping tests running in performance mode"); + } S3ATestUtils.disableFilesystemCaching(conf); return conf; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdirWithCreatePerf.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdirWithCreatePerf.java index 4b2468de97bb8..45dfc391b001d 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdirWithCreatePerf.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdirWithCreatePerf.java @@ -29,7 +29,9 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; +import static org.apache.hadoop.fs.s3a.S3ATestConstants.KEY_PERFORMANCE_TESTS_ENABLED; import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfNotEnabled; /** * Test mkdir operations on S3A with create performance mode. @@ -50,6 +52,8 @@ protected AbstractFSContract createContract(Configuration conf) { @Test public void testMkdirOverParentFile() throws Throwable { + skipIfNotEnabled(getContract().getConf(), KEY_PERFORMANCE_TESTS_ENABLED, + "Skipping tests running in performance mode"); describe("try to mkdir where a parent is a file, should pass"); FileSystem fs = getFileSystem(); Path path = methodPath(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java index 9ab1768b2aba1..71cf7be18f52f 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java @@ -63,6 +63,11 @@ public interface S3ATestConstants { */ String KEY_ACL_TESTS_ENABLED = TEST_FS_S3A + "create.acl.enabled"; + /** + * A property set to true if tests running in performance mode are enabled: {@value } + */ + String KEY_PERFORMANCE_TESTS_ENABLED = TEST_FS_S3A + "perf.enabled"; + /** * A property set to true if V1 tests are enabled: {@value}. */