From ba380695a895b705c27b0e1659d6656bd25405c9 Mon Sep 17 00:00:00 2001 From: chungen0126 Date: Mon, 9 Dec 2024 19:08:42 +0800 Subject: [PATCH 1/2] HADOOP-19351. S3A: Add config option to skip test with performance mode --- .../markdown/tools/hadoop-aws/performance.md | 1 + .../site/markdown/tools/hadoop-aws/testing.md | 16 ++++++++++++++++ .../fs/contract/s3a/ITestS3AContractCreate.java | 5 +++++ .../s3a/ITestS3AContractMkdirWithCreatePerf.java | 4 ++++ .../apache/hadoop/fs/s3a/S3ATestConstants.java | 5 +++++ 5 files changed, 31 insertions(+) diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md index b8cb3ff732b36..940df2ad6fa0c 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md @@ -295,6 +295,7 @@ understands the risks. * If an option is to be tuned which may relax semantics, a new option MUST be defined. * Unknown flags are ignored; this is to avoid compatibility. * The option `*` means "turn everything on". This is implicitly unstable across releases. +* Other stores may retain stricter semantics. | *Option* | *Meaning* | Since | |----------|--------------------|:------| diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md index 7222eee98baeb..16384f275e924 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md @@ -628,6 +628,10 @@ on third party stores. test.fs.s3a.create.create.acl.enabled false + + test.fs.s3a.perf.enabled + false + ``` See [Third Party Stores](third_party_stores.html) for more on this topic. @@ -767,6 +771,18 @@ Tests in `ITestS3AContentEncoding` may need disabling false ``` + +### Disabling tests running in performance mode + +Some tests running in performance mode turn off the safety checks. They expect breaking posix semantics. +For stores with stricter semantics, these test cases must be disabled. +```xml + + test.fs.s3a.perf.enabled + false + +``` + ### Tests which may fail (and which you can ignore) * `ITestS3AContractMultipartUploader` tests `testMultipartUploadAbort` and `testSingleUpload` raising `FileNotFoundException` diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java index 000caf328837e..033c2d94c7bf8 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java @@ -29,9 +29,11 @@ import org.apache.hadoop.fs.contract.AbstractFSContract; import org.apache.hadoop.fs.s3a.S3ATestUtils; +import static org.apache.hadoop.fs.s3a.S3ATestConstants.KEY_PERFORMANCE_TESTS_ENABLED; import static org.apache.hadoop.fs.s3a.Constants.CONNECTION_EXPECT_CONTINUE; import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfNotEnabled; /** * S3A contract tests creating files. @@ -84,6 +86,9 @@ protected Configuration createConfiguration() { conf, CONNECTION_EXPECT_CONTINUE); conf.setBoolean(CONNECTION_EXPECT_CONTINUE, expectContinue); + if (createPerformance) { + skipIfNotEnabled(conf, KEY_PERFORMANCE_TESTS_ENABLED, "Skipping tests running in performance mode"); + } S3ATestUtils.disableFilesystemCaching(conf); return conf; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdirWithCreatePerf.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdirWithCreatePerf.java index 4b2468de97bb8..45dfc391b001d 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdirWithCreatePerf.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdirWithCreatePerf.java @@ -29,7 +29,9 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; +import static org.apache.hadoop.fs.s3a.S3ATestConstants.KEY_PERFORMANCE_TESTS_ENABLED; import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfNotEnabled; /** * Test mkdir operations on S3A with create performance mode. @@ -50,6 +52,8 @@ protected AbstractFSContract createContract(Configuration conf) { @Test public void testMkdirOverParentFile() throws Throwable { + skipIfNotEnabled(getContract().getConf(), KEY_PERFORMANCE_TESTS_ENABLED, + "Skipping tests running in performance mode"); describe("try to mkdir where a parent is a file, should pass"); FileSystem fs = getFileSystem(); Path path = methodPath(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java index 9ab1768b2aba1..71cf7be18f52f 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java @@ -63,6 +63,11 @@ public interface S3ATestConstants { */ String KEY_ACL_TESTS_ENABLED = TEST_FS_S3A + "create.acl.enabled"; + /** + * A property set to true if tests running in performance mode are enabled: {@value } + */ + String KEY_PERFORMANCE_TESTS_ENABLED = TEST_FS_S3A + "perf.enabled"; + /** * A property set to true if V1 tests are enabled: {@value}. */ From 1426453c012f8f0d9394baf9e33651a1d5aa19d5 Mon Sep 17 00:00:00 2001 From: chungen0126 Date: Fri, 13 Dec 2024 01:54:41 +0800 Subject: [PATCH 2/2] remove blanks --- .../src/site/markdown/tools/hadoop-aws/performance.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md index 940df2ad6fa0c..1cee1bc937011 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md @@ -295,7 +295,7 @@ understands the risks. * If an option is to be tuned which may relax semantics, a new option MUST be defined. * Unknown flags are ignored; this is to avoid compatibility. * The option `*` means "turn everything on". This is implicitly unstable across releases. -* Other stores may retain stricter semantics. +* Other stores may retain stricter semantics. | *Option* | *Meaning* | Since | |----------|--------------------|:------|