Update recipe.py

leap-stc · Jun 5, 2024 · c364cc1 · c364cc1
1 parent 54cbb34
commit c364cc1
Showing 1 changed file with 3 additions and 4 deletions.
diff --git a/feedstock/recipe.py b/feedstock/recipe.py
@@ -209,7 +209,7 @@ def expand(self, pcoll: beam.PCollection) -> beam.PCollection:
     | CheckpointFileTransfer(
         transfer_target=cache_target,
         max_executors=10,
-        concurrency_per_executor=5,
+        concurrency_per_executor=2,
         initial_backoff=3.0,
         fsspec_sync_patch=False,# works but is slow. Testing with fsspec and new backoff retry
         )
@@ -240,9 +240,8 @@ def expand(self, pcoll: beam.PCollection) -> beam.PCollection:
     beam.Create(lowres_mlo_pattern.items())
     | CheckpointFileTransfer(
         transfer_target=cache_target,
-        max_executors=10,
-        concurrency_per_executor=5,
-        initial_backoff=3.0,
+        max_executors=5,
+        concurrency_per_executor=3,
         fsspec_sync_patch=False,# works but is slow. Testing with fsspec and new backoff retry
         )
     | OpenURLWithFSSpec(cache=None, fsspec_sync_patch=True)