From 01fe78cc3b84051608bcd2d530d12a97a381554f Mon Sep 17 00:00:00 2001 From: Floris Calkoen Date: Thu, 1 Aug 2024 09:31:24 +0200 Subject: [PATCH] . --- scripts/python/make_gcts.py | 6 +++--- src/coastpy/utils/dask.py | 11 ++++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/scripts/python/make_gcts.py b/scripts/python/make_gcts.py index 3d89679..88c684f 100644 --- a/scripts/python/make_gcts.py +++ b/scripts/python/make_gcts.py @@ -458,9 +458,9 @@ def generate_filtered_transects( logging.info(f"Transects written to {TMP_BASE_URI}") - logging.info("Restarting client...") - client.restart() - logging.info(f"Client dashboard link: {client.dashboard_link}") + # logging.info("Restarting client...") + # client.restart() + # logging.info(f"Client dashboard link: {client.dashboard_link}") transects = dask_geopandas.read_parquet( TMP_BASE_URI, storage_options=storage_options diff --git a/src/coastpy/utils/dask.py b/src/coastpy/utils/dask.py index 8c9f14b..5523c27 100644 --- a/src/coastpy/utils/dask.py +++ b/src/coastpy/utils/dask.py @@ -86,12 +86,13 @@ def _create_slurm_client(self, *args: Any, **kwargs: Any) -> Client: """ from dask_jobqueue import SLURMCluster - # Define default values specific to SLURM + slurm_configs = { - # "cores": 1, # Cores per worker - # "processes": True, # Processes per worker + "cores": 5, # Cores per worker + "processes": 5, # Processes per worker # "n_workers": 5, - # "memory": "12GB", # Memory per worker + "memory": "60GB", # Memory per worker + # job_script_prologue=["module load devel/cuda/12.1"], # "local_directory": "/scratch/frcalkoen/tmp", # "walltime": "1:00:00", } @@ -100,7 +101,7 @@ def _create_slurm_client(self, *args: Any, **kwargs: Any) -> Client: # Create the SLURM cluster cluster = SLURMCluster(*args, **slurm_configs) - cluster.scale(jobs=5) + cluster.scale(jobs=1) logging.info(f"{cluster.job_script()}")