From fc42da70cea536e67997636d8fcb7833932a5d98 Mon Sep 17 00:00:00 2001 From: manika137 <80244229+manika137@users.noreply.github.com> Date: Tue, 17 Dec 2024 02:02:00 -0800 Subject: [PATCH] HADOOP-19357: [ABFS] Optimizations for Retry Handling and Client Side Throttling (#7216) Default for following configs are changed: Client-side throttling (CST): Off Client Backoff - 500ms (reduced from 3sec) Max Backoff - 25s (reduced from 30sec) Min Backoff - 500ms (reduced from 3sec) Contributed by Manika Joshi (@manika137) --- .../constants/FileSystemConfigurations.java | 8 ++++---- .../services/ITestExponentialRetryPolicy.java | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index aeb3196966636..6ada0e8bcca74 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -39,11 +39,11 @@ public final class FileSystemConfigurations { private static final int SIXTY_SECONDS = 60_000; // Retry parameter defaults. - public static final int DEFAULT_MIN_BACKOFF_INTERVAL = 3_000; // 3s - public static final int DEFAULT_MAX_BACKOFF_INTERVAL = 30_000; // 30s + public static final int DEFAULT_MIN_BACKOFF_INTERVAL = 500; // 500ms + public static final int DEFAULT_MAX_BACKOFF_INTERVAL = 25_000; // 25s public static final boolean DEFAULT_STATIC_RETRY_FOR_CONNECTION_TIMEOUT_ENABLED = true; public static final int DEFAULT_STATIC_RETRY_INTERVAL = 1_000; // 1s - public static final int DEFAULT_BACKOFF_INTERVAL = 3_000; // 3s + public static final int DEFAULT_BACKOFF_INTERVAL = 500; // 500ms public static final int DEFAULT_MAX_RETRY_ATTEMPTS = 30; public static final int DEFAULT_CUSTOM_TOKEN_FETCH_RETRY_COUNT = 3; @@ -108,7 +108,7 @@ public final class FileSystemConfigurations { public static final boolean DEFAULT_ENABLE_FLUSH = true; public static final boolean DEFAULT_DISABLE_OUTPUTSTREAM_FLUSH = true; - public static final boolean DEFAULT_ENABLE_AUTOTHROTTLING = true; + public static final boolean DEFAULT_ENABLE_AUTOTHROTTLING = false; public static final int DEFAULT_METRIC_IDLE_TIMEOUT_MS = 60_000; public static final int DEFAULT_METRIC_ANALYSIS_TIMEOUT_MS = 60_000; public static final boolean DEFAULT_FS_AZURE_ACCOUNT_LEVEL_THROTTLING_ENABLED = true; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java index 9ead6d196382e..1d289eabfa9bd 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java @@ -96,6 +96,24 @@ public void testDefaultMaxIORetryCount() throws Exception { testMaxIOConfig(abfsConfig); } + @Test + public void testClientSideThrottlingConfigs() throws Exception { + final Configuration configuration = new Configuration(); + configuration.setBoolean(FS_AZURE_ENABLE_AUTOTHROTTLING, true); + AbfsConfiguration abfsConfiguration = new AbfsConfiguration(configuration, + DUMMY_ACCOUNT_NAME); + Assertions.assertThat(abfsConfiguration.isAutoThrottlingEnabled()) + .describedAs("Client-side throttling enabled by configuration key") + .isTrue(); + + configuration.unset(FS_AZURE_ENABLE_AUTOTHROTTLING); + AbfsConfiguration abfsConfiguration2 = new AbfsConfiguration(configuration, + DUMMY_ACCOUNT_NAME); + Assertions.assertThat(abfsConfiguration2.isAutoThrottlingEnabled()) + .describedAs("Client-side throttling should be disabled by default") + .isFalse(); + } + @Test public void testThrottlingIntercept() throws Exception { AzureBlobFileSystem fs = getFileSystem();