adding local_directory option to dask cluster in executor

CMCC-Foundation · Oct 10, 2024 · 4e4a9b2 · 4e4a9b2
1 parent b0be622
commit 4e4a9b2
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/executor/app/main.py b/executor/app/main.py
@@ -1,4 +1,5 @@
 import os
+import tempfile
 import time
 import datetime
 import pika
@@ -455,6 +456,7 @@ def get_size(self, location_path):
     dask_cluster_opts["n_workers"] = int(os.getenv("DASK_N_WORKERS", 1))
     dask_cluster_opts["memory_limit"] = os.getenv("DASK_MEMORY_LIMIT", "auto")
     dask_cluster_opts['thread_per_worker'] = int(os.getenv("DASK_THREADS_PER_WORKER", 8))
+    dask_cluster_opts['local_directory'] = os.getenv("LOCAL_DIR", tempfile.mkdtemp(f'{os.uname()[1]}'))
 
 
     executor = Executor(broker=broker, store_path=store_path, dask_cluster_opts=dask_cluster_opts)