HADOOP-19357: [ABFS] Optimizations for Retry Handling and Client Side…

… Throttling (#7216) Default for following configs are changed: Client-side throttling (CST): Off Client Backoff - 500ms (reduced from 3sec) Max Backoff - 25s (reduced from 30sec) Min Backoff - 500ms (reduced from 3sec) Contributed by Manika Joshi (@manika137)
apache · Dec 17, 2024 · fc42da7 · fc42da7
1 parent efb83ec
commit fc42da7
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 4 deletions.
diff --git a/...azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/...azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
@@ -39,11 +39,11 @@ public final class FileSystemConfigurations {
   private static final int SIXTY_SECONDS = 60_000;
 
   // Retry parameter defaults.
-  public static final int DEFAULT_MIN_BACKOFF_INTERVAL = 3_000;  // 3s
-  public static final int DEFAULT_MAX_BACKOFF_INTERVAL = 30_000;  // 30s
+  public static final int DEFAULT_MIN_BACKOFF_INTERVAL = 500;  // 500ms
+  public static final int DEFAULT_MAX_BACKOFF_INTERVAL = 25_000;  // 25s
   public static final boolean DEFAULT_STATIC_RETRY_FOR_CONNECTION_TIMEOUT_ENABLED = true;
   public static final int DEFAULT_STATIC_RETRY_INTERVAL = 1_000; // 1s
-  public static final int DEFAULT_BACKOFF_INTERVAL = 3_000;  // 3s
+  public static final int DEFAULT_BACKOFF_INTERVAL = 500;  // 500ms
   public static final int DEFAULT_MAX_RETRY_ATTEMPTS = 30;
   public static final int DEFAULT_CUSTOM_TOKEN_FETCH_RETRY_COUNT = 3;
 
@@ -108,7 +108,7 @@ public final class FileSystemConfigurations {
 
   public static final boolean DEFAULT_ENABLE_FLUSH = true;
   public static final boolean DEFAULT_DISABLE_OUTPUTSTREAM_FLUSH = true;
-  public static final boolean DEFAULT_ENABLE_AUTOTHROTTLING = true;
+  public static final boolean DEFAULT_ENABLE_AUTOTHROTTLING = false;
   public static final int DEFAULT_METRIC_IDLE_TIMEOUT_MS = 60_000;
   public static final int DEFAULT_METRIC_ANALYSIS_TIMEOUT_MS = 60_000;
   public static final boolean DEFAULT_FS_AZURE_ACCOUNT_LEVEL_THROTTLING_ENABLED = true;

diff --git a/...ure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java b/...ure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java
@@ -96,6 +96,24 @@ public void testDefaultMaxIORetryCount() throws Exception {
     testMaxIOConfig(abfsConfig);
   }
 
+  @Test
+  public void testClientSideThrottlingConfigs() throws Exception {
+    final Configuration configuration = new Configuration();
+    configuration.setBoolean(FS_AZURE_ENABLE_AUTOTHROTTLING, true);
+    AbfsConfiguration abfsConfiguration = new AbfsConfiguration(configuration,
+            DUMMY_ACCOUNT_NAME);
+   Assertions.assertThat(abfsConfiguration.isAutoThrottlingEnabled())
+            .describedAs("Client-side throttling enabled by configuration key")
+            .isTrue();
+
+    configuration.unset(FS_AZURE_ENABLE_AUTOTHROTTLING);
+    AbfsConfiguration abfsConfiguration2 = new AbfsConfiguration(configuration,
+            DUMMY_ACCOUNT_NAME);
+    Assertions.assertThat(abfsConfiguration2.isAutoThrottlingEnabled())
+            .describedAs("Client-side throttling should be disabled by default")
+            .isFalse();
+  }
+
   @Test
   public void testThrottlingIntercept() throws Exception {
     AzureBlobFileSystem fs = getFileSystem();