From 2eead1771bb19ffcd0b810be0d997ceec5fd2c74 Mon Sep 17 00:00:00 2001 From: Tom White Date: Tue, 29 Oct 2024 20:52:41 +0000 Subject: [PATCH] Increase `allowed_mem` default to 2GB (#599) --- cubed/__init__.py | 4 ++-- cubed/tests/test_core.py | 8 ++++---- docs/configuration.md | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cubed/__init__.py b/cubed/__init__.py index 4c6fc9cc..ada6c2f2 100644 --- a/cubed/__init__.py +++ b/cubed/__init__.py @@ -9,8 +9,8 @@ config = Config( "cubed", - # default spec is local temp dir and a modest amount of memory (200MB, of which 100MB is reserved) - defaults=[{"spec": {"allowed_mem": 200_000_000, "reserved_mem": 100_000_000}}], + # default spec is local temp dir and a reasonable amount of memory (2GB, of which 100MB is reserved) + defaults=[{"spec": {"allowed_mem": "2GB", "reserved_mem": "100MB"}}], ) from .core.array import compute, measure_reserved_mem, visualize diff --git a/cubed/tests/test_core.py b/cubed/tests/test_core.py index 2892843b..ba916517 100644 --- a/cubed/tests/test_core.py +++ b/cubed/tests/test_core.py @@ -335,7 +335,7 @@ def test_default_spec(executor): def test_default_spec_allowed_mem_exceeded(): # default spec fails for large computations - a = xp.ones((20000, 1000), chunks=(10000, 1000)) + a = xp.ones((20000, 10000), chunks=(10000, 10000)) with pytest.raises(ValueError): xp.negative(a) @@ -344,10 +344,10 @@ def test_default_spec_config_override(): # override default spec to increase allowed_mem from cubed import config - with config.set({"spec.allowed_mem": "500MB"}): - a = xp.ones((20000, 1000), chunks=(10000, 1000)) + with config.set({"spec.allowed_mem": "4GB"}): + a = xp.ones((20000, 10000), chunks=(10000, 10000)) b = xp.negative(a) - assert_array_equal(b.compute(), -np.ones((20000, 1000))) + assert_array_equal(b.compute(), -np.ones((20000, 10000))) @pytest.mark.parametrize( diff --git a/docs/configuration.md b/docs/configuration.md index e88831e6..6241fdf7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -95,7 +95,7 @@ These properties can be passed directly to the {py:class}`Spec ` con | Property | Default | Description | |--------------------|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------| | `work_dir` | `None` | The directory path (specified as an fsspec URL) used for storing intermediate data. If not set, the user's temporary directory is used. | -| `allowed_mem` | `200MB` | The total memory available to a worker for running a task. This includes any `reserved_mem` that has been set. | +| `allowed_mem` | `2GB` | The total memory available to a worker for running a task. This includes any `reserved_mem` that has been set. | | `reserved_mem` | `100MB` | The memory reserved on a worker for non-data use when running a task | | `executor_name` | `single-threaded` | The executor for running computations. One of `single-threaded`, `threads`, `processes`, `beam`, `coiled`, `dask`, `lithops`, `modal`. | | `executor_options` | `None` | Options to pass to the executor on construction. See below for possible options for each executor. |