diff --git a/.doctrees/array-api.doctree b/.doctrees/array-api.doctree index 917dfdb0..0ca7defc 100644 Binary files a/.doctrees/array-api.doctree and b/.doctrees/array-api.doctree differ diff --git a/.doctrees/design.doctree b/.doctrees/design.doctree index 16f3d22c..18491c82 100644 Binary files a/.doctrees/design.doctree and b/.doctrees/design.doctree differ diff --git a/.doctrees/environment.pickle b/.doctrees/environment.pickle index 4cf1791a..6a7c1d3d 100644 Binary files a/.doctrees/environment.pickle and b/.doctrees/environment.pickle differ diff --git a/.doctrees/user-guide/optimization.doctree b/.doctrees/user-guide/optimization.doctree index 7a681a9d..5cc0dddf 100644 Binary files a/.doctrees/user-guide/optimization.doctree and b/.doctrees/user-guide/optimization.doctree differ diff --git a/_sources/user-guide/optimization.md b/_sources/user-guide/optimization.md index b8e3a458..ec556546 100644 --- a/_sources/user-guide/optimization.md +++ b/_sources/user-guide/optimization.md @@ -112,9 +112,9 @@ e.visualize(optimize_function=opt_fn) The `max_total_num_input_blocks` argument to `multiple_inputs_optimize_dag` specifies the maximum number of input blocks (chunks) that are allowed in the fused operation. -Again, this is to limit the number of reads that an individual task must perform. The default is `None`, which means that operations are fused only if they have the same number of tasks. If set to an integer, then this limitation is removed, and tasks with a different number of tasks will be fused - as long as the total number of input blocks does not exceed the maximum. This setting is useful for reductions, and can be set using `functools.partial`: +Again, this is to limit the number of reads that an individual task must perform. If set to `None`, operations are fused only if they have the same number of tasks. If set to an integer (the default is 10), then tasks with a different number of tasks will be fused - as long as the total number of input blocks does not exceed the maximum. This setting is useful for reductions, and can be changed using `functools.partial`: ```python -opt_fn = partial(multiple_inputs_optimize_dag, max_total_num_input_blocks=10) +opt_fn = partial(multiple_inputs_optimize_dag, max_total_num_input_blocks=20) e.visualize(optimize_function=opt_fn) ``` diff --git a/_static/documentation_options.js b/_static/documentation_options.js index 69267185..64cb74cf 100644 --- a/_static/documentation_options.js +++ b/_static/documentation_options.js @@ -1,5 +1,5 @@ const DOCUMENTATION_OPTIONS = { - VERSION: 'b3396a9', + VERSION: '4b18053', LANGUAGE: 'en', COLLAPSE_INDEX: false, BUILDER: 'html', diff --git a/api.html b/api.html index 6be4ce22..97eb47bb 100644 --- a/api.html +++ b/api.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/array-api.html b/array-api.html index db2b8ad0..b819dcd7 100644 --- a/array-api.html +++ b/array-api.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/computation.html b/computation.html index 7d7d3982..342d0bee 100644 --- a/computation.html +++ b/computation.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/configuration.html b/configuration.html index 5afccba8..bb5f0544 100644 --- a/configuration.html +++ b/configuration.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/contributing.html b/contributing.html index 0699976d..c1259c76 100644 --- a/contributing.html +++ b/contributing.html @@ -39,7 +39,7 @@ - + @@ -49,7 +49,7 @@ - diff --git a/design.html b/design.html index c8a2a656..9eb5c90d 100644 --- a/design.html +++ b/design.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/generated/cubed.Array.compute.html b/generated/cubed.Array.compute.html index 9e89b40e..4ff53081 100644 --- a/generated/cubed.Array.compute.html +++ b/generated/cubed.Array.compute.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/generated/cubed.Array.html b/generated/cubed.Array.html index 2520b47c..f0bfd8c3 100644 --- a/generated/cubed.Array.html +++ b/generated/cubed.Array.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/generated/cubed.Array.rechunk.html b/generated/cubed.Array.rechunk.html index da2603d9..7e115206 100644 --- a/generated/cubed.Array.rechunk.html +++ b/generated/cubed.Array.rechunk.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/generated/cubed.Array.visualize.html b/generated/cubed.Array.visualize.html index ac74cd78..f1724595 100644 --- a/generated/cubed.Array.visualize.html +++ b/generated/cubed.Array.visualize.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/generated/cubed.Callback.html b/generated/cubed.Callback.html index 7b0b5916..1ade6b17 100644 --- a/generated/cubed.Callback.html +++ b/generated/cubed.Callback.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/generated/cubed.Spec.html b/generated/cubed.Spec.html index 38b827f4..59041ec1 100644 --- a/generated/cubed.Spec.html +++ b/generated/cubed.Spec.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/generated/cubed.TaskEndEvent.html b/generated/cubed.TaskEndEvent.html index dc66f951..0968ddda 100644 --- a/generated/cubed.TaskEndEvent.html +++ b/generated/cubed.TaskEndEvent.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/generated/cubed.apply_gufunc.html b/generated/cubed.apply_gufunc.html index f3975b12..931ef208 100644 --- a/generated/cubed.apply_gufunc.html +++ b/generated/cubed.apply_gufunc.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/generated/cubed.compute.html b/generated/cubed.compute.html index beb408dd..39bd85a8 100644 --- a/generated/cubed.compute.html +++ b/generated/cubed.compute.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/generated/cubed.from_array.html b/generated/cubed.from_array.html index dc589994..6e835216 100644 --- a/generated/cubed.from_array.html +++ b/generated/cubed.from_array.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/generated/cubed.from_zarr.html b/generated/cubed.from_zarr.html index d79e6eba..0964c893 100644 --- a/generated/cubed.from_zarr.html +++ b/generated/cubed.from_zarr.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/generated/cubed.map_blocks.html b/generated/cubed.map_blocks.html index 3e8101a3..e8cfc7f9 100644 --- a/generated/cubed.map_blocks.html +++ b/generated/cubed.map_blocks.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/generated/cubed.map_overlap.html b/generated/cubed.map_overlap.html index 24d6551e..f2bf0287 100644 --- a/generated/cubed.map_overlap.html +++ b/generated/cubed.map_overlap.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/generated/cubed.measure_reserved_mem.html b/generated/cubed.measure_reserved_mem.html index 7afa9287..c6a71e3f 100644 --- a/generated/cubed.measure_reserved_mem.html +++ b/generated/cubed.measure_reserved_mem.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/generated/cubed.nanmean.html b/generated/cubed.nanmean.html index f9f04144..a58df305 100644 --- a/generated/cubed.nanmean.html +++ b/generated/cubed.nanmean.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/generated/cubed.nansum.html b/generated/cubed.nansum.html index 81ed147a..e51fe583 100644 --- a/generated/cubed.nansum.html +++ b/generated/cubed.nansum.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/generated/cubed.pad.html b/generated/cubed.pad.html index b970ef1e..c4d167b4 100644 --- a/generated/cubed.pad.html +++ b/generated/cubed.pad.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/generated/cubed.random.random.html b/generated/cubed.random.random.html index e3233e97..e61236fb 100644 --- a/generated/cubed.random.random.html +++ b/generated/cubed.random.random.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/generated/cubed.rechunk.html b/generated/cubed.rechunk.html index 1e022b57..eea61565 100644 --- a/generated/cubed.rechunk.html +++ b/generated/cubed.rechunk.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/generated/cubed.store.html b/generated/cubed.store.html index cdf708ca..2e53a03a 100644 --- a/generated/cubed.store.html +++ b/generated/cubed.store.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/generated/cubed.to_zarr.html b/generated/cubed.to_zarr.html index 29c21ec4..9f3fd016 100644 --- a/generated/cubed.to_zarr.html +++ b/generated/cubed.to_zarr.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/generated/cubed.visualize.html b/generated/cubed.visualize.html index b00160e8..f55e38f9 100644 --- a/generated/cubed.visualize.html +++ b/generated/cubed.visualize.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/genindex.html b/genindex.html index 11d82295..3769510b 100644 --- a/genindex.html +++ b/genindex.html @@ -38,7 +38,7 @@ - + @@ -47,7 +47,7 @@ - diff --git a/getting-started/demo.html b/getting-started/demo.html index 506c90cd..672ef7b7 100644 --- a/getting-started/demo.html +++ b/getting-started/demo.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/getting-started/index.html b/getting-started/index.html index d9528f9e..47457858 100644 --- a/getting-started/index.html +++ b/getting-started/index.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/getting-started/installation.html b/getting-started/installation.html index e3c02d5e..de007eaa 100644 --- a/getting-started/installation.html +++ b/getting-started/installation.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/index.html b/index.html index b082edfd..0faba3c0 100644 --- a/index.html +++ b/index.html @@ -39,7 +39,7 @@ - + @@ -49,7 +49,7 @@ - diff --git a/objects.inv b/objects.inv index 55ca9200..8b971134 100644 Binary files a/objects.inv and b/objects.inv differ diff --git a/operations.html b/operations.html index fda47e25..9eba9541 100644 --- a/operations.html +++ b/operations.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/related-projects.html b/related-projects.html index 6809c20b..c7f5d0df 100644 --- a/related-projects.html +++ b/related-projects.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/search.html b/search.html index cf9a7865..2c63a928 100644 --- a/search.html +++ b/search.html @@ -37,7 +37,7 @@ - + @@ -49,7 +49,7 @@ - diff --git a/searchindex.js b/searchindex.js index 430643b2..551ac621 100644 --- a/searchindex.js +++ b/searchindex.js @@ -1 +1 @@ -Search.setIndex({"alltitles": {"API Reference": [[0, null]], "Advanced settings": [[38, "advanced-settings"]], "Allowed memory": [[37, "allowed-memory"]], "Array": [[0, "array"]], "Array API": [[5, "array-api"]], "Articles": [[31, "articles"]], "Bounded-memory serverless distributed N-dimensional array processing": [[31, "bounded-memory-serverless-distributed-n-dimensional-array-processing"]], "Callbacks": [[34, "callbacks"]], "Chunk sizes": [[37, "chunk-sizes"]], "Chunk-specific functions": [[0, "chunk-specific-functions"]], "Cloud storage": [[41, "cloud-storage"]], "Computation": [[2, null]], "Conda": [[30, "conda"]], "Configuration": [[3, null]], "Contributing": [[4, null]], "Core operations": [[5, "core-operations"]], "Cubed": [[31, null]], "Dask": [[33, "dask"]], "Debugging optimization": [[38, "debugging-optimization"]], "Deleting intermediate data": [[41, "deleting-intermediate-data"]], "Demo": [[28, null]], "Dependency Tree": [[32, "dependency-tree"]], "Design": [[5, null]], "Development": [[4, "development"]], "Diagnosing Performance": [[40, "diagnosing-performance"]], "Diagnostics": [[34, null]], "Differences between Cubed and the standard": [[1, "differences-between-cubed-and-the-standard"]], "Different Cloud Providers": [[40, "different-cloud-providers"]], "Different Executors": [[40, "different-executors"]], "Documentation": [[31, "documentation"]], "Environment variables": [[3, "environment-variables"]], "Examples in use": [[34, "examples-in-use"]], "Execution": [[2, "execution"]], "Executor options": [[3, "executor-options"]], "Executors": [[35, null]], "For developers": [[31, null]], "For users": [[31, null]], "Getting Started": [[29, null]], "History": [[34, "history"]], "IO": [[0, "io"]], "Installation": [[30, null]], "Local single-machine executors": [[35, "local-single-machine-executors"]], "Map fusion": [[38, "map-fusion"]], "Memory": [[2, "memory"], [37, null]], "Memray": [[34, "memray"]], "Multi-pipeline Calculation": [[40, "multi-pipeline-calculation"]], "Multi-step Calculation": [[40, "multi-step-calculation"]], "Multiple-input fusion": [[38, "multiple-input-fusion"]], "Non-standardised functions": [[0, "non-standardised-functions"]], "Operations": [[32, null]], "Optimization": [[38, null]], "Optional dependencies": [[30, "optional-dependencies"]], "Other Performance Considerations": [[40, "other-performance-considerations"]], "Pip": [[30, "pip"]], "Plan": [[2, "plan"]], "Preface: Types of Scaling": [[40, "preface-types-of-scaling"]], "Previous work": [[33, "previous-work"]], "Primitive operations": [[5, "primitive-operations"]], "Progress bar": [[34, "progress-bar"]], "Projected memory": [[37, "projected-memory"]], "Python Array API": [[1, null]], "Random number generation": [[0, "random-number-generation"]], "Reference": [[3, "reference"]], "Related Projects": [[33, null]], "Reliability": [[39, null]], "Reserved memory": [[37, "reserved-memory"]], "Retries": [[39, "retries"]], "Runtime": [[0, "runtime"], [5, "runtime"]], "Scaling": [[40, null]], "Single-step Calculation": [[40, "single-step-calculation"]], "Spec object": [[3, "spec-object"]], "Spec options": [[3, "spec-options"]], "Specification": [[3, "specification"]], "Specifying an executor": [[35, "specifying-an-executor"]], "Storage": [[5, "storage"], [41, null]], "Stragglers": [[39, "stragglers"]], "Strong consistency": [[39, "strong-consistency"]], "Theoretical vs Practical Scaling of Cubed": [[40, "theoretical-vs-practical-scaling-of-cubed"]], "Timeline": [[34, "timeline"]], "Timeouts": [[39, "timeouts"]], "Tips": [[40, "tips"]], "Total number of input blocks": [[38, "total-number-of-input-blocks"]], "Total number of source arrays": [[38, "total-number-of-source-arrays"]], "User Guide": [[36, null]], "Visualize the computation plan": [[34, "visualize-the-computation-plan"]], "Which cloud service executor should I use?": [[35, "which-cloud-service-executor-should-i-use"]], "Why Cubed?": [[42, null]], "Xarray": [[33, "xarray"]], "YAML file": [[3, "yaml-file"]], "beam": [[3, "beam"]], "blockwise": [[32, "blockwise"]], "coiled": [[3, "coiled"]], "cubed.Array": [[6, null]], "cubed.Array.compute": [[7, null]], "cubed.Array.rechunk": [[8, null]], "cubed.Array.visualize": [[9, null]], "cubed.Callback": [[10, null]], "cubed.Spec": [[11, null]], "cubed.TaskEndEvent": [[12, null]], "cubed.apply_gufunc": [[13, null]], "cubed.compute": [[14, null]], "cubed.from_array": [[15, null]], "cubed.from_zarr": [[16, null]], "cubed.map_blocks": [[17, null]], "cubed.map_overlap": [[18, null]], "cubed.measure_reserved_mem": [[19, null]], "cubed.nanmean": [[20, null]], "cubed.nansum": [[21, null]], "cubed.pad": [[22, null]], "cubed.random.random": [[23, null]], "cubed.rechunk": [[24, null]], "cubed.store": [[25, null]], "cubed.to_zarr": [[26, null]], "cubed.visualize": [[27, null]], "dask": [[3, "dask"]], "elemwise": [[32, "elemwise"]], "lithops": [[3, "lithops"]], "map_blocks": [[32, "map-blocks"]], "map_selection": [[32, "map-selection"]], "modal": [[3, "modal"]], "processes": [[3, "processes"]], "rechunk": [[32, "rechunk"]], "reduction and arg_reduction": [[32, "reduction-and-arg-reduction"]], "single-threaded": [[3, "single-threaded"]], "threads": [[3, "threads"]]}, "docnames": ["api", "array-api", "computation", "configuration", "contributing", "design", "generated/cubed.Array", "generated/cubed.Array.compute", "generated/cubed.Array.rechunk", "generated/cubed.Array.visualize", "generated/cubed.Callback", "generated/cubed.Spec", "generated/cubed.TaskEndEvent", "generated/cubed.apply_gufunc", "generated/cubed.compute", "generated/cubed.from_array", "generated/cubed.from_zarr", "generated/cubed.map_blocks", "generated/cubed.map_overlap", "generated/cubed.measure_reserved_mem", "generated/cubed.nanmean", "generated/cubed.nansum", "generated/cubed.pad", "generated/cubed.random.random", "generated/cubed.rechunk", "generated/cubed.store", "generated/cubed.to_zarr", "generated/cubed.visualize", "getting-started/demo", "getting-started/index", "getting-started/installation", "index", "operations", "related-projects", "user-guide/diagnostics", "user-guide/executors", "user-guide/index", "user-guide/memory", "user-guide/optimization", "user-guide/reliability", "user-guide/scaling", "user-guide/storage", "why-cubed"], "envversion": {"sphinx": 64, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["api.rst", "array-api.md", "computation.md", "configuration.md", "contributing.md", "design.md", "generated/cubed.Array.rst", "generated/cubed.Array.compute.rst", "generated/cubed.Array.rechunk.rst", "generated/cubed.Array.visualize.rst", "generated/cubed.Callback.rst", "generated/cubed.Spec.rst", "generated/cubed.TaskEndEvent.rst", "generated/cubed.apply_gufunc.rst", "generated/cubed.compute.rst", "generated/cubed.from_array.rst", "generated/cubed.from_zarr.rst", "generated/cubed.map_blocks.rst", "generated/cubed.map_overlap.rst", "generated/cubed.measure_reserved_mem.rst", "generated/cubed.nanmean.rst", "generated/cubed.nansum.rst", "generated/cubed.pad.rst", "generated/cubed.random.random.rst", "generated/cubed.rechunk.rst", "generated/cubed.store.rst", "generated/cubed.to_zarr.rst", "generated/cubed.visualize.rst", "getting-started/demo.md", "getting-started/index.md", "getting-started/installation.md", "index.md", "operations.md", "related-projects.md", "user-guide/diagnostics.md", "user-guide/executors.md", "user-guide/index.md", "user-guide/memory.md", "user-guide/optimization.md", "user-guide/reliability.md", "user-guide/scaling.md", "user-guide/storage.md", "why-cubed.md"], "indexentries": {"__init__() (cubed.array method)": [[6, "cubed.Array.__init__", false]], "__init__() (cubed.callback method)": [[10, "cubed.Callback.__init__", false]], "__init__() (cubed.spec method)": [[11, "cubed.Spec.__init__", false]], "__init__() (cubed.taskendevent method)": [[12, "cubed.TaskEndEvent.__init__", false]], "apply_gufunc() (in module cubed)": [[13, "cubed.apply_gufunc", false]], "arange() (in module cubed.array_api)": [[1, "cubed.array_api.arange", false]], "array (class in cubed)": [[6, "cubed.Array", false]], "asarray() (in module cubed.array_api)": [[1, "cubed.array_api.asarray", false]], "broadcast_to() (in module cubed.array_api)": [[1, "cubed.array_api.broadcast_to", false]], "callback (class in cubed)": [[10, "cubed.Callback", false]], "compute() (cubed.array method)": [[7, "cubed.Array.compute", false]], "compute() (in module cubed)": [[14, "cubed.compute", false]], "concat() (in module cubed.array_api)": [[1, "cubed.array_api.concat", false]], "empty() (in module cubed.array_api)": [[1, "cubed.array_api.empty", false]], "empty_like() (in module cubed.array_api)": [[1, "cubed.array_api.empty_like", false]], "eye() (in module cubed.array_api)": [[1, "cubed.array_api.eye", false]], "from_array() (in module cubed)": [[15, "cubed.from_array", false]], "from_zarr() (in module cubed)": [[16, "cubed.from_zarr", false]], "full() (in module cubed.array_api)": [[1, "cubed.array_api.full", false]], "full_like() (in module cubed.array_api)": [[1, "cubed.array_api.full_like", false]], "linspace() (in module cubed.array_api)": [[1, "cubed.array_api.linspace", false]], "map_blocks() (in module cubed)": [[17, "cubed.map_blocks", false]], "map_overlap() (in module cubed)": [[18, "cubed.map_overlap", false]], "measure_reserved_mem() (in module cubed)": [[19, "cubed.measure_reserved_mem", false]], "nanmean() (in module cubed)": [[20, "cubed.nanmean", false]], "nansum() (in module cubed)": [[21, "cubed.nansum", false]], "ones() (in module cubed.array_api)": [[1, "cubed.array_api.ones", false]], "ones_like() (in module cubed.array_api)": [[1, "cubed.array_api.ones_like", false]], "pad() (in module cubed)": [[22, "cubed.pad", false]], "random() (in module cubed.random)": [[23, "cubed.random.random", false]], "rechunk() (cubed.array method)": [[8, "cubed.Array.rechunk", false]], "rechunk() (in module cubed)": [[24, "cubed.rechunk", false]], "spec (class in cubed)": [[11, "cubed.Spec", false]], "store() (in module cubed)": [[25, "cubed.store", false]], "taskendevent (class in cubed)": [[12, "cubed.TaskEndEvent", false]], "to_zarr() (in module cubed)": [[26, "cubed.to_zarr", false]], "visualize() (cubed.array method)": [[9, "cubed.Array.visualize", false]], "visualize() (in module cubed)": [[27, "cubed.visualize", false]], "zeros() (in module cubed.array_api)": [[1, "cubed.array_api.zeros", false]], "zeros_like() (in module cubed.array_api)": [[1, "cubed.array_api.zeros_like", false]]}, "objects": {"cubed": [[6, 0, 1, "", "Array"], [10, 0, 1, "", "Callback"], [11, 0, 1, "", "Spec"], [12, 0, 1, "", "TaskEndEvent"], [13, 2, 1, "", "apply_gufunc"], [14, 2, 1, "", "compute"], [15, 2, 1, "", "from_array"], [16, 2, 1, "", "from_zarr"], [17, 2, 1, "", "map_blocks"], [18, 2, 1, "", "map_overlap"], [19, 2, 1, "", "measure_reserved_mem"], [20, 2, 1, "", "nanmean"], [21, 2, 1, "", "nansum"], [22, 2, 1, "", "pad"], [24, 2, 1, "", "rechunk"], [25, 2, 1, "", "store"], [26, 2, 1, "", "to_zarr"], [27, 2, 1, "", "visualize"]], "cubed.Array": [[6, 1, 1, "", "__init__"], [7, 1, 1, "", "compute"], [8, 1, 1, "", "rechunk"], [9, 1, 1, "", "visualize"]], "cubed.Callback": [[10, 1, 1, "", "__init__"]], "cubed.Spec": [[11, 1, 1, "", "__init__"]], "cubed.TaskEndEvent": [[12, 1, 1, "", "__init__"]], "cubed.array_api": [[1, 2, 1, "", "arange"], [1, 2, 1, "", "asarray"], [1, 2, 1, "", "broadcast_to"], [1, 2, 1, "", "concat"], [1, 2, 1, "", "empty"], [1, 2, 1, "", "empty_like"], [1, 2, 1, "", "eye"], [1, 2, 1, "", "full"], [1, 2, 1, "", "full_like"], [1, 2, 1, "", "linspace"], [1, 2, 1, "", "ones"], [1, 2, 1, "", "ones_like"], [1, 2, 1, "", "zeros"], [1, 2, 1, "", "zeros_like"]], "cubed.random": [[23, 2, 1, "", "random"]]}, "objnames": {"0": ["py", "class", "Python class"], "1": ["py", "method", "Python method"], "2": ["py", "function", "Python function"]}, "objtypes": {"0": "py:class", "1": "py:method", "2": "py:function"}, "terms": {"": [3, 5, 7, 31, 32, 33, 34, 35, 37, 38, 40, 41, 42], "0": [1, 11, 23, 30, 34], "00": 34, "001": [34, 38], "002": [34, 38], "003": [34, 38], "004": 38, "005": 38, "1": [1, 11, 12, 23, 28, 34, 35, 38, 41], "10": [4, 28, 38], "100": 34, "1000": 35, "10000": 34, "100kb": 28, "100mb": [3, 37, 40], "11": [3, 34], "12": 1, "180": 3, "2": [3, 11, 28, 30, 34, 35, 38], "20": [35, 37], "2000": 3, "200mb": 34, "2022": 1, "2023": 1, "2gb": [3, 35, 37, 40], "3": [3, 4, 28, 34, 38], "300": 3, "4": [28, 34, 38], "400mb": 34, "438": 1, "5": [28, 34, 38], "5000": [3, 34], "50000": 3, "6": [28, 34, 38], "7": [28, 30, 34, 38], "8": [28, 34, 38], "9": [28, 34, 38], "A": [0, 2, 3, 35, 37, 39, 40, 42], "And": [33, 38], "As": [2, 37], "At": 35, "By": [34, 38, 41], "For": [2, 32, 37, 39, 40, 41], "If": [3, 7, 9, 11, 14, 27, 34, 35, 37, 38, 39, 41], "In": [25, 32, 33, 34, 37, 38, 40, 41], "It": [3, 19, 32, 33, 34, 35], "No": 32, "Not": 32, "On": [38, 41], "One": [2, 3, 40], "Or": [2, 3], "The": [1, 2, 3, 5, 7, 8, 9, 11, 14, 16, 18, 19, 24, 25, 26, 27, 32, 33, 34, 35, 37, 38, 40, 41], "Then": 34, "There": [3, 5, 34, 37, 38, 40], "These": [0, 3, 5, 38], "To": [3, 30, 34, 38, 39, 40, 41], "With": 40, "__init__": [6, 10, 11, 12], "abil": 37, "about": [12, 13, 32, 33, 34], "abov": [34, 35, 38], "abstract": [33, 42], "accept": 1, "account": 35, "accur": 37, "achiev": 40, "act": 39, "activ": 4, "actual": [3, 34, 37], "acycl": [2, 34], "ad": [2, 34, 40], "adapt": 34, "add": [3, 28, 32, 34, 38, 40, 41], "addit": [34, 38], "adjust": 40, "adopt": 42, "advanc": 36, "advantag": [2, 33, 40], "affect": [34, 40], "after": [8, 18, 24, 34, 39, 41], "ag": 41, "again": [38, 39], "aggregr": 32, "ahead": [2, 37], "aim": [29, 37, 40], "algorithm": [32, 37], "all": [3, 5, 32, 34, 35, 40, 42], "alloc": [13, 34, 42], "allow": [32, 36, 38, 39], "allow_rechunk": 13, "allowed_mem": [3, 11, 28, 35, 37, 40], "alon": 34, "along": [20, 32], "alreadi": [7, 14], "also": [2, 3, 28, 30, 35, 39, 40], "although": [32, 35], "alwai": [34, 42], "amazon": [35, 39], "amount": [13, 19, 34, 35, 37, 38, 39, 40, 42], "an": [2, 3, 4, 8, 9, 11, 15, 16, 19, 22, 24, 26, 27, 29, 32, 34, 36, 37, 38, 39, 40, 42], "analys": 37, "ani": [3, 7, 11, 19, 34, 35, 38, 40, 41], "annot": 34, "anoth": [2, 37, 39, 40, 42], "anyth": 3, "apach": [5, 30, 31, 42], "api": [6, 28, 31, 32, 33], "appli": [13, 17, 18, 32, 40], "applic": 5, "apply_gufunc": 31, "approach": [41, 42], "appropri": [3, 41], "ar": [0, 1, 2, 3, 4, 5, 9, 13, 27, 30, 32, 33, 34, 35, 37, 38, 39, 40, 41, 42], "arang": 1, "arbitrari": 11, "arg": [13, 17, 18], "arg_reduct": 31, "argument": [1, 3, 18, 19, 34, 38], "arithmet": 20, "around": [37, 40], "arrai": [2, 3, 10, 13, 14, 15, 16, 17, 18, 21, 22, 24, 25, 26, 27, 28, 32, 33, 34, 35, 37, 40, 41, 42], "arrang": 32, "array_api": [1, 3, 28, 34, 38], "arrow": 32, "asarrai": [1, 15, 28, 34, 38], "assembl": 32, "assum": 13, "astyp": 38, "atom": 39, "attempt": [3, 39, 42], "attribut": [5, 6, 11, 12, 32], "auto": [1, 15], "automat": [3, 34, 35, 37, 38, 40, 41], "avail": [3, 11, 28, 37, 39], "averag": 40, "avoid": [32, 42], "aw": [3, 30, 35, 40, 41], "awai": [38, 42], "ax": [13, 32], "axi": [1, 13, 20, 21, 32], "b": [3, 28, 34, 38], "back": [5, 6, 34, 35], "backup": [3, 39, 40], "bag": 33, "base": 40, "baselin": 37, "basi": 37, "basic": [3, 37], "basicconfig": 38, "batch": 3, "batch_siz": 3, "beam": [5, 30, 31, 33, 40, 42], "becaus": [35, 37], "been": [0, 3, 7, 11, 14, 28, 33, 34, 35, 38, 39, 40, 42], "befor": [2, 3, 7, 9, 14, 27, 34, 35, 38, 40, 41], "being": 38, "below": [3, 35, 37, 38], "better": 33, "between": [31, 32, 42], "big": 40, "bin": 34, "block": [5, 17, 18, 32, 34, 40], "blockwis": [2, 31, 33, 38], "blosc": 11, "blue": 5, "bool": [7, 9, 14, 18, 27], "both": [2, 33, 38, 39, 40], "bottom": [5, 32], "bound": [33, 37, 40, 42], "boundari": 18, "box": 34, "branch": 40, "break": 42, "bring": 39, "broadcast": 32, "broadcast_to": 1, "bucket": 41, "budget": 37, "buffer": 34, "build": [2, 35, 37], "built": [2, 5], "bulk": 42, "byte": [11, 13, 19], "c": [3, 28, 30, 34, 38], "calcul": 37, "call": [2, 18, 32, 33, 34, 35, 38, 39], "callabl": [7, 9, 14, 18, 27], "callback": [7, 12, 14, 36], "can": [0, 2, 3, 9, 11, 19, 27, 30, 33, 34, 35, 37, 38, 39, 40, 41, 42], "cancel": 39, "cannot": [32, 40], "carefulli": 42, "carri": 40, "case": [34, 37, 38, 40, 41, 42], "cd": 34, "central": 42, "certain": [39, 41], "challeng": 42, "chang": [3, 8, 24, 32, 33, 34, 38, 40], "characterist": 40, "check": 37, "child": 3, "choic": 34, "choos": 2, "chosen": 5, "chunk": [1, 2, 3, 5, 6, 8, 15, 17, 18, 22, 23, 24, 28, 31, 32, 33, 34, 35, 36, 38, 39, 40, 42], "circumst": 39, "class": [6, 10, 11, 12], "clean": 41, "clear": 41, "clevel": 11, "click": 41, "client": [3, 30, 39], "cloud": [3, 5, 19, 28, 33, 36, 37, 39, 42], "cluster": [35, 40], "clutter": 32, "cname": 11, "code": 3, "coil": [30, 35], "coiled_kwarg": 3, "collect": [25, 34], "colour": 34, "come": [35, 40], "command": 41, "common": [37, 42], "commonli": 38, "compar": 35, "complet": [12, 30, 34, 39, 40], "complex": [37, 40], "compon": 33, "compos": [5, 32], "compress": [11, 34, 37], "compressor": 11, "comput": [3, 5, 9, 10, 11, 16, 19, 20, 25, 26, 27, 28, 31, 33, 35, 36, 37, 38, 39, 40, 41, 42], "compute_arrays_in_parallel": [3, 40], "compute_kwarg": 3, "concat": [1, 38], "concept": 36, "concern": 33, "concurr": [3, 40], "conda": [4, 29], "confid": 37, "configur": [11, 31, 35, 37, 39, 40, 42], "conflict": 30, "conform": 6, "conserv": 37, "consid": [39, 40], "consider": 36, "consist": 36, "consol": 41, "constant": [18, 34], "constraint": 40, "construct": 3, "constructor": 3, "consult": 32, "contain": 37, "continu": 33, "contribut": 31, "control": 42, "conveni": 41, "convent": 28, "convert": [2, 32, 33, 34, 40], "copi": 1, "core": [3, 28, 31, 32, 33, 35, 38], "corearrai": [8, 14, 27], "corner": 34, "correspond": [17, 18, 32, 33], "cost": 41, "could": 40, "coupl": 3, "cov": 30, "cover": [3, 39], "coverag": 1, "cpu": 3, "creat": [0, 4, 15, 19, 33, 34, 38, 41], "creation": [0, 1, 3], "csv": 34, "cube": [0, 2, 3, 4, 5, 28, 29, 30, 32, 33, 34, 35, 36, 37, 38, 39, 41], "cubed_config": 3, "cubed_spec__allowed_mem": 3, "cubed_spec__executor_nam": 3, "cubed_spec__executor_options__runtim": 3, "cubed_spec__executor_options__runtime_memori": 3, "cubed_spec__executor_options__use_backup": 3, "cubed_spec__work_dir": 3, "cumul": 40, "current": [3, 13, 18, 25, 34, 35, 39, 40], "cutdown": 13, "d": [34, 38], "dag": [2, 34], "dai": 41, "dask": [5, 13, 28, 30, 31, 40, 42], "data": [3, 8, 11, 19, 24, 34, 35, 36, 37, 38, 42], "dataflow": [3, 33, 35, 42], "datafram": 33, "dataset": [35, 40], "daunt": 33, "debug": [34, 36], "decompos": 33, "decompress": 37, "dedic": 41, "deeper": 40, "default": [3, 7, 9, 11, 14, 25, 26, 27, 28, 34, 35, 38, 39, 40, 41], "defin": [5, 33, 40], "delai": [33, 40], "deleg": 5, "delet": 36, "deliber": [3, 42], "demo": [29, 31], "depend": [2, 7, 29, 31, 34, 39, 40], "deploi": 3, "depth": [18, 32], "describ": [28, 35, 39], "descript": 3, "design": [3, 31, 40, 42], "desir": [8, 24], "detail": [2, 3, 32, 35, 38], "determin": [34, 39], "develop": 34, "devic": 1, "diagaram": 37, "diagnos": 36, "diagnost": [30, 31, 36, 40], "diagram": 32, "dict": [3, 11, 18], "dictionari": 11, "differ": [3, 30, 31, 32, 38, 39], "difficult": 38, "dill": 30, "dimens": 32, "dimension": 42, "dip": 34, "direct": [2, 34], "directli": [3, 32], "directori": [3, 11, 19, 34, 41], "disabl": 11, "disadvantag": [2, 40], "discuss": [2, 35, 37], "disk": [2, 5, 9, 27, 35], "displai": [9, 27, 34, 40], "disproportion": [39, 40, 42], "distribut": [2, 3, 30, 33, 35, 39, 40, 42], "do": [34, 39, 40, 42], "doc": 3, "document": 3, "doe": [32, 35, 38, 41], "doesn": [3, 9, 27, 35, 37], "don": [2, 3, 34, 35, 41], "done": [34, 40], "donfig": 3, "dot": [9, 27], "doubl": 3, "down": 39, "drop": 32, "drop_axi": 17, "dtype": [1, 2, 5, 17, 18, 21, 32, 37], "due": 30, "duplic": 39, "dure": [10, 37], "e": [4, 38, 40], "each": [2, 3, 18, 32, 34, 37, 39, 40], "eager": [25, 26], "easi": 35, "easier": 33, "echo": 34, "edg": 2, "effect": 38, "effici": [2, 37], "effort": 42, "either": 38, "element": [18, 21, 32, 40], "elementwis": 40, "elemwis": 31, "embarrassingli": 42, "empti": 1, "empty_lik": 1, "enabl": [34, 38, 39, 40], "encapsul": 3, "endpoint": 1, "engin": [2, 42], "enhanc": 40, "enough": 40, "ensur": [34, 37, 41], "enter": 41, "environ": [2, 4, 35, 40], "equal": 40, "equival": [3, 13], "error": 39, "eschew": 42, "especi": [30, 40], "essenti": 28, "estim": 37, "etc": [11, 31], "evalu": 28, "even": [39, 40, 42], "event": [7, 10, 14, 34], "everi": [2, 5, 18, 35, 40], "everyth": 42, "exampl": [2, 3, 5, 11, 28, 31, 32, 35, 37, 38, 39, 40], "exce": [37, 38], "except": [1, 3, 37, 39], "excess": [35, 38], "exclud": 19, "execut": [31, 37, 40], "executor": [5, 7, 11, 14, 19, 25, 26, 28, 30, 31, 34, 36, 37, 39, 41], "executor_nam": [3, 11, 35], "executor_opt": [3, 11], "exist": 40, "expand": 3, "expect": [39, 40], "explain": [34, 38], "explicitli": 2, "export": 3, "expos": 3, "express": 2, "extens": [5, 9, 27], "extent": 42, "extern": 5, "extra": [1, 18], "extra_projected_mem": 13, "ey": 1, "f": 34, "factor": [37, 40], "fail": [3, 39], "fals": [3, 7, 9, 14, 18, 20, 21, 27, 34, 38, 39, 40], "fan": 38, "far": [35, 40], "fashion": 39, "faster": 40, "fault": 39, "featur": [3, 34, 39, 40], "fee": 35, "feed": 40, "few": [1, 5, 39, 40, 42], "fewer": 40, "file": [9, 27, 34, 35, 39, 41], "filenam": [9, 27, 34], "filesystem": 41, "fill_valu": 1, "final": 32, "fine": 33, "first": [32, 34, 37, 38, 40], "firstli": 40, "fit": 35, "five": 5, "fix": 40, "flame": 34, "flamegraph": 34, "flexibl": 33, "float": 23, "float32": 38, "follow": [1, 3, 11, 31, 32, 34, 37, 38, 41], "forg": 30, "form": [11, 32, 41], "format": [9, 27, 34], "formerli": 42, "four": [3, 37], "fourier": 1, "framework": 42, "free": 35, "from": [3, 5, 7, 14, 15, 16, 17, 18, 32, 34, 37, 38, 39, 40, 42], "from_arrai": 0, "from_zarr": 0, "fsspec": [3, 11, 19, 41], "fuctool": 38, "full": [1, 30, 40], "full_lik": 1, "func": [13, 17, 18], "function": [1, 2, 3, 5, 7, 9, 13, 14, 17, 18, 19, 27, 30, 31, 32, 34, 35, 37, 38, 40], "function_end_tstamp": 12, "function_start_tstamp": 12, "functionexecutor": 3, "functool": 38, "fundament": [32, 33], "further": [37, 42], "fuse": [32, 34, 38, 40], "fusion": [2, 7, 9, 14, 27, 36], "futur": 3, "g": 40, "gain": 42, "gb": 11, "gc": 35, "gcf": 40, "gcp": [3, 30], "gcsf": 30, "gener": [5, 13, 31, 32, 34, 37, 40, 42], "general_blockwis": 32, "get": [4, 31, 35, 36, 38, 40, 41], "github": 4, "given": [3, 19, 21, 37, 40], "global": 39, "go": [5, 34, 42], "good": [3, 11, 37, 40], "googl": [3, 33, 35, 39, 40, 41, 42], "grain": 33, "graph": [2, 7, 9, 14, 27, 33, 34, 38], "graphic": 34, "graphviz": 30, "greater": 37, "green": 32, "group": [16, 26], "guarante": [31, 35, 40, 42], "guid": [19, 29, 31], "ha": [2, 3, 11, 28, 30, 33, 34, 35, 37, 38, 39, 40, 42], "had": 35, "hadoop": 42, "half": 23, "hand": 38, "handi": 3, "handl": [18, 35, 40], "hard": 3, "have": [0, 2, 3, 7, 14, 29, 32, 33, 34, 35, 37, 38, 39, 40], "head": 4, "heavili": [5, 33], "help": [36, 37, 38], "henc": [39, 40], "here": [3, 32, 34, 38, 40, 42], "hidden": [9, 27], "high": [2, 29, 33, 37, 42], "higher": [33, 38], "highest": [35, 37], "hist": 34, "historycallback": 34, "hold": 40, "horizont": [34, 40], "how": [2, 3, 18, 32, 34, 37, 38, 40, 41, 42], "howev": [2, 13, 42], "html": 34, "i": [1, 2, 3, 5, 7, 9, 11, 13, 14, 18, 19, 25, 26, 27, 28, 31, 32, 33, 34, 36, 37, 38, 39, 40, 41, 42], "id": [11, 34], "ideal": [34, 40], "idempot": 39, "ident": 39, "ignor": 20, "illustr": 37, "imag": [3, 9, 27, 34], "immedi": [25, 26, 39], "implement": [1, 5, 19, 25, 28, 31, 32, 33, 42], "implicitli": 2, "import": [3, 9, 27, 28, 34, 35, 37, 38, 40, 41], "importantli": 39, "impos": 38, "improv": [34, 37, 40, 42], "includ": [0, 3, 5, 9, 11, 27, 33], "incur": 41, "independ": 40, "index": 32, "indic": 3, "individu": [3, 38], "infinit": 40, "influenc": 33, "inform": [12, 13, 32, 34, 37, 38], "inherit": 5, "input": [3, 16, 17, 18, 32, 36, 37, 40], "instal": [4, 19, 29, 31, 34], "instanc": [3, 40], "instanti": 3, "instead": 42, "instruct": 41, "int": [11, 18, 19], "integ": 38, "integr": [31, 33], "intend": 35, "interact": 28, "interest": 42, "intermedi": [2, 3, 7, 11, 14, 19, 34, 36, 38, 40], "intern": 34, "interv": 23, "interven": 33, "intro": 31, "introduc": 36, "introduct": 29, "invok": [2, 3], "involv": [2, 4, 34, 42], "io": [5, 31, 38, 39], "ipython": [9, 27], "issu": [1, 34], "iter": 40, "its": [8, 18, 24, 32, 34, 37, 38, 40], "itself": 37, "jinja2": 30, "job": 41, "jpeg": [9, 27], "jpg": [9, 27], "jupyt": 34, "just": [28, 32, 34, 38], "k": 1, "kb": 11, "keepdim": [13, 20, 21], "kei": [3, 32, 39], "keyword": [1, 3, 18, 19], "kill": 3, "kind": 38, "known": [2, 37, 40, 42], "kwarg": [7, 13, 14, 17, 18, 19, 25, 26], "l": 34, "label": 34, "lambda": [35, 40], "larg": [35, 37, 39, 40, 42], "larger": [35, 40], "last": 42, "later": 34, "launch": 39, "layer": [5, 33], "lazi": [2, 28], "lead": [40, 42], "least": 37, "less": 33, "lesser": 42, "let": 5, "level": [2, 29, 33, 38, 42], "librari": [31, 33], "lifecycl": 41, "like": [3, 5, 15, 25, 28, 32, 33, 35, 37, 38, 41, 42], "lilac": 34, "limit": [13, 38, 40], "line": 34, "linear": 40, "linspac": 1, "list": [1, 3, 7, 14, 18, 30], "lithop": [5, 19, 30, 31, 34, 35, 42], "live": 3, "ll": 28, "load": [16, 34], "local": [5, 11, 28, 34, 36, 37, 41], "locat": 3, "log": 38, "logic": 40, "long": [34, 38, 40], "longer": [3, 34, 38, 39, 40], "look": [3, 29, 32, 38], "lot": [41, 42], "lz4": 11, "m": [30, 34], "machin": [28, 36, 37, 40], "made": [3, 42], "magic": 40, "mai": [2, 5, 18, 32, 35, 38, 40, 41], "main": [2, 3, 32], "mainli": 3, "maintain": 40, "major": 42, "make": [2, 33, 37, 40], "manag": [3, 37, 42], "mani": [30, 32, 38], "manipul": 1, "manual": 41, "map": [2, 7, 9, 14, 18, 27, 32, 36, 40], "map_block": [5, 31], "map_direct": 32, "map_overlap": 31, "map_select": 31, "mapreduc": 42, "mark": [9, 27], "match": [32, 38, 40], "materi": [2, 34], "matplotlib": 30, "matur": [33, 35], "max_tasks_per_child": 3, "max_total_num_input_block": 38, "max_total_source_arrai": 38, "max_work": [3, 40], "maximum": [3, 31, 37, 38, 40], "mb": 11, "mean": [3, 5, 20, 34, 37, 38, 39, 40], "measur": [19, 37, 40], "measure_reserved_mem": [37, 40], "memori": [3, 11, 13, 19, 33, 34, 35, 36, 40, 42], "memrai": 36, "messag": 39, "method": [6, 10, 11, 12, 34, 38], "middl": 32, "might": 40, "minim": 30, "minimum": 40, "minut": 35, "mitig": [3, 39], "mock": 30, "modal": [5, 19, 30, 31, 35, 40], "mode": 22, "model": [2, 42], "monitor": 34, "more": [13, 28, 32, 33, 34, 35, 37, 38, 39, 40], "most": [3, 35, 36, 42], "much": [34, 35, 37, 40], "multi": 28, "multipl": [2, 13, 14, 17, 18, 27, 31, 32, 34, 36], "multiple_inputs_optimize_dag": 38, "must": [3, 19, 25, 32, 38, 39], "n": [3, 42], "n_col": 1, "n_row": 1, "name": [4, 6, 9, 12, 27, 33, 41], "nan": [20, 21], "natur": [2, 33, 37], "nbyte": 38, "necessari": 40, "need": [2, 5, 13, 32, 34, 37, 39, 40, 41], "neg": 38, "neighbor": 18, "neither": 3, "nest": 3, "never": 37, "new": [5, 13, 33, 34, 37, 40], "new_axi": 17, "newer": 33, "next": 40, "node": [2, 34, 42], "non": [3, 11, 31, 37, 40], "none": [1, 3, 7, 9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 38], "notat": 32, "note": [3, 18, 25, 26, 32, 34, 35, 38, 39, 41], "notebook": [9, 27, 34], "noth": [28, 40], "notic": [3, 28, 38], "now": [32, 38], "np": 18, "num": [1, 38], "num_task": 12, "number": [3, 5, 18, 31, 32, 34, 35, 39, 40], "numblock": 32, "numcodec": 11, "numpi": [2, 5, 28, 32, 33], "numpy_groupi": 30, "o": 34, "obj": 1, "object": [2, 10, 15, 19, 25, 28, 41], "observ": 40, "off": [34, 38, 39, 40], "offer": 40, "often": 42, "old": 41, "onc": [14, 34, 39], "one": [0, 2, 3, 18, 32, 34, 38, 39, 40, 41, 42], "ones": [1, 33, 38], "ones_lik": 1, "onli": [2, 3, 7, 9, 14, 18, 27, 32, 33, 34, 35, 38, 39, 40, 42], "op": [34, 38], "open": 23, "oper": [2, 9, 19, 25, 26, 27, 31, 33, 34, 37, 38, 39, 40, 41, 42], "opportun": 2, "opt_fn": 38, "optim": [2, 7, 9, 14, 27, 31, 32, 34, 36, 40, 42], "optimize_funct": [7, 9, 14, 27, 38], "optimize_graph": [7, 9, 14, 27, 34, 38], "option": [7, 9, 11, 14, 16, 19, 25, 26, 27, 29, 34], "optional_depend": 30, "orang": [32, 34], "order": [32, 37, 40], "origin": 39, "other": [3, 5, 36, 38], "otherwis": [7, 9, 14, 27, 38], "out": [34, 35, 36, 40, 41], "outer": 32, "output": [2, 9, 13, 18, 26, 27, 32, 34, 37, 38, 39, 40], "output_dtyp": 13, "output_s": 13, "outsid": 3, "over": [4, 18, 21, 32, 41], "overal": [34, 38, 39, 40], "overhead": 35, "overlap": 18, "overview": 29, "own": 40, "packag": [3, 19, 30, 33], "pad_width": 22, "page": [1, 3, 32, 40], "pair": 2, "panda": 30, "paragraph": 39, "parallel": [2, 3, 33, 40, 42], "paramet": [7, 8, 9, 11, 13, 14, 16, 18, 19, 24, 25, 26, 27, 40], "part": [32, 35, 40], "partial": 38, "partial_reduc": 32, "particular": [3, 35, 37, 38], "pass": [3, 11, 18, 19, 34, 40, 42], "path": [3, 11, 16, 19, 26], "pdf": [9, 27], "peak": 19, "peak_measured_mem": 37, "peak_measured_mem_end": 12, "peak_measured_mem_start": 12, "per": [13, 40], "perfect": 34, "perform": [7, 9, 14, 27, 32, 34, 36, 37, 38, 42], "period": 41, "persist": [40, 41, 42], "phase": 37, "pick": 38, "pink": 32, "pip": [4, 29, 34], "plan": [6, 31, 33, 36, 37, 38, 40], "pleas": 4, "plenti": 37, "plot": 34, "png": [9, 27], "possibl": [3, 29, 37, 38, 40, 42], "potenti": 40, "power": 38, "practic": 36, "pre": 39, "preced": 38, "precis": 37, "predecessor": 38, "prefac": 36, "preserv": 32, "previou": [31, 38, 39], "previous": 32, "primit": [2, 31, 32, 33, 38], "print": 28, "problem": [40, 42], "process": [2, 5, 19, 25, 26, 33, 34, 35, 37, 39, 40, 42], "processor": 40, "processpoolexecutor": 3, "produc": [9, 18, 27, 34, 37], "profil": 34, "program": [3, 42], "project": [5, 30, 31, 34, 36], "projected_mem": 37, "properli": 37, "properti": 3, "proportion": 40, "provid": [5, 9, 27, 32, 34, 35, 37, 39, 42], "provis": 40, "public": 5, "purpos": 42, "put": [37, 42], "py": 34, "pydot": 30, "pyproject": 30, "pytest": 30, "python": [0, 3, 4, 5, 6, 13, 19, 25, 26, 28, 30, 31, 33, 34, 35, 37], "pywren": 42, "quickli": [29, 35, 39], "r": 4, "rais": 37, "ran": 37, "random": [3, 5, 31, 34], "rather": [32, 41], "re": 40, "read": [32, 33, 34, 37, 38, 39, 40, 42], "readm": 28, "reason": [3, 33, 37, 38, 40], "receiv": 10, "rechunk": [2, 31, 33, 40, 42], "recommend": 35, "recomput": [7, 14], "red": 5, "reduc": [32, 38, 40], "reduct": [31, 37, 38, 40], "refer": [13, 31, 32, 35, 40], "region": [35, 41], "regular": 41, "rel": 35, "relat": 31, "reli": 39, "reliabl": [31, 35, 36], "remot": 35, "remov": [38, 41], "render": [9, 27, 34], "report": [19, 37], "repres": [32, 34], "represent": [2, 40], "req": 30, "requir": [2, 4, 19, 34, 35, 37, 40, 42], "reserv": [3, 11, 19, 36], "reserved_mem": [3, 11, 19, 37], "resourc": [11, 28, 40], "restrict": 40, "result": [28, 32, 34, 38], "resum": [7, 14], "retain": 34, "retri": [3, 36], "retriev": 32, "return": [8, 9, 16, 19, 21, 23, 24, 27, 32], "reus": 3, "rich": [30, 34], "richprogressbar": 34, "right": 34, "rm": 41, "room": 37, "round": [32, 34], "rule": [37, 41], "run": [2, 3, 7, 11, 14, 19, 25, 26, 28, 30, 33, 34, 35, 37, 38, 39, 40, 41, 42], "runner": 33, "runtim": [2, 3, 19, 25, 26, 31, 33, 35, 37, 41, 42], "runtime_memori": 3, "s3": [3, 35, 39, 41], "s3f": 30, "same": [3, 38, 40, 41], "save": [25, 26, 34], "scale": [2, 31, 34, 35, 36, 39, 42], "scenario": 40, "schema": 34, "seaborn": 30, "second": [3, 32, 35], "section": 39, "see": [2, 3, 28, 30, 34, 38, 40], "select": [32, 41], "send": [7, 14], "sent": 32, "separ": [30, 34, 40, 41], "serverless": [2, 33, 40, 42], "servic": [36, 37, 40, 41, 42], "set": [2, 3, 7, 11, 13, 14, 19, 30, 35, 36, 37, 39, 40, 41], "sever": 33, "shape": [1, 5, 8, 18, 24, 32], "share": [3, 18], "short": 41, "should": [11, 18, 36, 37, 38, 40, 41], "show": [9, 27, 32, 34, 35], "show_hidden": [9, 27], "shown": [32, 34, 38], "shuffl": [2, 11, 40, 42], "sign": [35, 40], "signatur": 13, "signifi": 34, "signific": 40, "similar": [13, 33], "similarli": [32, 34], "simpl": [3, 28, 34, 35], "simplest": [32, 38, 40], "simpli": 34, "simultan": 40, "sinc": [2, 3, 32, 33, 34, 35, 37, 38, 39, 42], "singl": [28, 32, 33, 34, 36, 39], "size": [2, 23, 32, 34, 36, 40, 42], "slice": 32, "slide": 31, "slow": [2, 38, 39], "slowli": 40, "small": [34, 35, 38], "smaller": 37, "so": [3, 28, 33, 34, 35, 37, 38, 39, 40, 41], "solut": 40, "some": [18, 33, 37, 38, 39, 40], "sometim": [35, 38], "sourc": 25, "space": 41, "spark": 42, "spec": [1, 6, 7, 14, 15, 16, 17, 19, 23, 28, 34, 35, 37, 41], "specif": [11, 31, 40], "specifi": [3, 11, 18, 19, 20, 28, 32, 36, 38, 41], "specul": 39, "speed": [38, 40], "spent": [34, 42], "split_everi": [20, 21], "squar": 34, "squeez": 32, "stack": 38, "stage": 38, "standard": [0, 6, 28, 31, 32, 33], "standardis": 31, "start": [1, 28, 31, 35, 38, 40, 42], "startup": [34, 35, 40], "stat": 34, "stateless": [5, 42], "statist": 34, "statu": 1, "step": [1, 34], "stick": 40, "stop": 1, "storag": [2, 6, 11, 16, 26, 31, 36, 39, 40, 42], "storage_opt": 11, "store": [3, 5, 11, 16, 19, 26, 34, 40, 41], "str": [9, 11, 19, 27], "straggler": [3, 34, 36, 40], "straightforward": [2, 35], "string": [16, 26], "strong": [36, 40], "style": 3, "submit": 3, "subset": [5, 28, 32], "success": 42, "suffici": 40, "suggest": 40, "suit": 34, "sum": [21, 37], "support": [1, 3, 13, 18, 32, 33, 38], "sure": [40, 41], "svg": [9, 27, 34], "system": [5, 19, 39, 40, 41, 42], "t": [2, 3, 7, 9, 14, 27, 33, 34, 35, 37, 38, 41], "tab": 41, "tail": 34, "take": [3, 34, 39, 40], "taken": 39, "target": [25, 42], "target_stor": 24, "task": [2, 3, 5, 11, 12, 13, 19, 33, 34, 35, 37, 38, 39, 40], "task_create_tstamp": 12, "task_result_tstamp": 12, "tb": 11, "tell": 13, "temp": [3, 35, 41], "tempor": 34, "temporari": [3, 41], "ten": 37, "terabyt": 40, "test": [3, 30, 34, 35], "test_mem_util": 34, "than": [3, 13, 32, 33, 37, 38, 39, 40, 41], "thei": [3, 32, 33, 34, 37, 38, 40], "them": [2, 32, 34, 39, 41], "theme": 42, "theoret": 36, "therebi": [40, 42], "therefor": [32, 35], "thi": [2, 3, 5, 7, 8, 9, 11, 13, 18, 19, 25, 26, 28, 29, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42], "thing": 3, "those": 40, "though": [40, 42], "thread": [35, 40], "threadpoolexecutor": 3, "three": [3, 32, 33, 34, 38, 39], "through": [3, 5], "throughput": 40, "thumb": 37, "tightli": [37, 42], "time": [2, 3, 32, 34, 35, 37, 38, 39, 40, 41, 42], "timeline_viz": 34, "timelinevisualizationcallback": 34, "timeout": [3, 36], "tini": [19, 34], "tip": 36, "tmp": 28, "tmpdir": 41, "to_zarr": 34, "togeth": [2, 38], "toler": 39, "toml": 30, "tomwhit": [3, 35, 41], "too": [32, 38], "took": 34, "tool": 34, "top": [5, 32], "total": [3, 11, 39, 40], "tqdm": [30, 34], "tqdmprogressbar": 34, "trace": 34, "track": [1, 32, 34], "traction": 42, "transfer": 35, "transform": 1, "travers": 2, "treat": 21, "tree": 31, "tree_reduc": 32, "trigger": 2, "trim": 18, "trivial": [19, 40], "true": [1, 3, 7, 9, 14, 27, 38, 40], "tune": [40, 42], "tupl": [8, 18, 24], "turn": [3, 34, 38, 39, 40], "twice": 34, "two": [5, 32, 34, 40], "txt": 4, "type": [5, 8, 9, 16, 18, 19, 24, 25, 26, 27, 34, 36], "typic": 41, "ufunc": 13, "under": [3, 34], "underli": 5, "underscor": 3, "understand": [33, 34, 38, 40, 42], "undoubtedli": 42, "unit": [5, 11], "unlik": 32, "unnecessari": 41, "unoptim": 38, "unpredict": 42, "until": 32, "up": [2, 3, 35, 37, 38, 39, 40, 41], "updat": 39, "upgrad": 40, "upper": 37, "url": [3, 11, 19, 41], "us": [2, 3, 5, 7, 9, 11, 14, 16, 19, 25, 26, 27, 28, 29, 30, 31, 32, 33, 36, 37, 38, 40, 41, 42], "usag": [13, 31, 37, 40, 42], "use_backup": [3, 39, 40], "user": [3, 33, 35, 42], "usual": 34, "util": [33, 37], "v": 36, "valu": [3, 11, 18, 32, 37, 38], "vari": [19, 40], "varieti": [3, 33, 34, 35], "variou": 37, "vector": 13, "veri": [2, 4, 34, 35, 40, 42], "versa": 40, "version": [1, 13, 19], "versu": 40, "vertic": [34, 40], "via": [31, 42], "vice": 40, "view": 34, "violat": 40, "visual": [33, 36, 38], "wa": [5, 33, 34], "wai": [3, 34, 39, 40], "wait": 40, "want": [34, 38], "wasn": 33, "we": [3, 28, 32, 34, 35, 37, 38, 40], "weak": 40, "welcom": 4, "well": [5, 33, 34, 37], "were": 32, "what": [33, 34, 38], "whatev": 37, "when": [2, 3, 11, 19, 33, 34, 35, 37, 38, 39, 40, 41], "where": [2, 11, 33, 38, 40], "wherea": [33, 40], "whether": [3, 18], "which": [2, 3, 5, 9, 11, 27, 28, 30, 32, 33, 34, 36, 37, 38, 40, 41, 42], "while": [2, 7, 14, 32, 34, 37, 40], "white": 32, "who": 40, "whole": 39, "why": [31, 38], "widespread": 42, "within": [3, 37], "without": [8, 24, 32, 40], "won": [7, 14], "word": 40, "work": [19, 31, 32, 34, 35, 37, 39, 41, 42], "work_dir": [3, 11, 19, 28, 35, 41], "worker": [3, 11, 34, 35, 37, 40, 42], "would": [34, 37, 38, 40, 41], "write": [2, 9, 25, 27, 34, 35, 37, 39, 40, 41, 42], "written": [2, 32, 34, 38, 39], "x": [1, 15, 20, 21, 22, 24, 25, 26], "xarrai": 31, "xp": [3, 28, 34, 38], "yaml": 35, "year": [33, 42], "yet": [0, 28], "you": [3, 13, 29, 30, 33, 34, 35, 36, 37, 38, 39, 40, 41], "your": [3, 34, 35, 37, 40], "zappi": 33, "zarr": [2, 5, 6, 11, 16, 25, 26, 31, 34, 35, 37, 39, 40, 41, 42], "zarr_compressor": 11, "zarrai": 6, "zero": [1, 21], "zeros_lik": 1}, "titles": ["API Reference", "Python Array API", "Computation", "Configuration", "Contributing", "Design", "cubed.Array", "cubed.Array.compute", "cubed.Array.rechunk", "cubed.Array.visualize", "cubed.Callback", "cubed.Spec", "cubed.TaskEndEvent", "cubed.apply_gufunc", "cubed.compute", "cubed.from_array", "cubed.from_zarr", "cubed.map_blocks", "cubed.map_overlap", "cubed.measure_reserved_mem", "cubed.nanmean", "cubed.nansum", "cubed.pad", "cubed.random.random", "cubed.rechunk", "cubed.store", "cubed.to_zarr", "cubed.visualize", "Demo", "Getting Started", "Installation", "Cubed", "Operations", "Related Projects", "Diagnostics", "Executors", "User Guide", "Memory", "Optimization", "Reliability", "Scaling", "Storage", "Why Cubed?"], "titleterms": {"For": 31, "advanc": 38, "allow": 37, "an": 35, "api": [0, 1, 5], "apply_gufunc": 13, "arg_reduct": 32, "arrai": [0, 1, 5, 6, 7, 8, 9, 31, 38], "articl": 31, "bar": 34, "beam": 3, "between": 1, "block": 38, "blockwis": 32, "bound": 31, "calcul": 40, "callback": [10, 34], "chunk": [0, 37], "cloud": [35, 40, 41], "coil": 3, "comput": [2, 7, 14, 34], "conda": 30, "configur": 3, "consider": 40, "consist": 39, "contribut": 4, "core": 5, "cube": [1, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 31, 40, 42], "dask": [3, 33], "data": 41, "debug": 38, "delet": 41, "demo": 28, "depend": [30, 32], "design": 5, "develop": [4, 31], "diagnos": 40, "diagnost": 34, "differ": [1, 40], "dimension": 31, "distribut": 31, "document": 31, "elemwis": 32, "environ": 3, "exampl": 34, "execut": 2, "executor": [3, 35, 40], "file": 3, "from_arrai": 15, "from_zarr": 16, "function": 0, "fusion": 38, "gener": 0, "get": 29, "guid": 36, "histori": 34, "i": 35, "input": 38, "instal": 30, "intermedi": 41, "io": 0, "lithop": 3, "local": 35, "machin": 35, "map": 38, "map_block": [17, 32], "map_overlap": 18, "map_select": 32, "measure_reserved_mem": 19, "memori": [2, 31, 37], "memrai": 34, "modal": 3, "multi": 40, "multipl": 38, "n": 31, "nanmean": 20, "nansum": 21, "non": 0, "number": [0, 38], "object": 3, "oper": [5, 32], "optim": 38, "option": [3, 30], "other": 40, "pad": 22, "perform": 40, "pip": 30, "pipelin": 40, "plan": [2, 34], "practic": 40, "prefac": 40, "previou": 33, "primit": 5, "process": [3, 31], "progress": 34, "project": [33, 37], "provid": 40, "python": 1, "random": [0, 23], "rechunk": [8, 24, 32], "reduct": 32, "refer": [0, 3], "relat": 33, "reliabl": 39, "reserv": 37, "retri": 39, "runtim": [0, 5], "scale": 40, "serverless": 31, "servic": 35, "set": 38, "should": 35, "singl": [3, 35, 40], "size": 37, "sourc": 38, "spec": [3, 11], "specif": [0, 3], "specifi": 35, "standard": 1, "standardis": 0, "start": 29, "step": 40, "storag": [5, 41], "store": 25, "straggler": 39, "strong": 39, "taskendev": 12, "theoret": 40, "thread": 3, "timelin": 34, "timeout": 39, "tip": 40, "to_zarr": 26, "total": 38, "tree": 32, "type": 40, "us": [34, 35], "user": [31, 36], "v": 40, "variabl": 3, "visual": [9, 27, 34], "which": 35, "why": 42, "work": 33, "xarrai": 33, "yaml": 3}}) \ No newline at end of file +Search.setIndex({"alltitles": {"API Reference": [[0, null]], "Advanced settings": [[38, "advanced-settings"]], "Allowed memory": [[37, "allowed-memory"]], "Array": [[0, "array"]], "Array API": [[5, "array-api"]], "Articles": [[31, "articles"]], "Bounded-memory serverless distributed N-dimensional array processing": [[31, "bounded-memory-serverless-distributed-n-dimensional-array-processing"]], "Callbacks": [[34, "callbacks"]], "Chunk sizes": [[37, "chunk-sizes"]], "Chunk-specific functions": [[0, "chunk-specific-functions"]], "Cloud storage": [[41, "cloud-storage"]], "Computation": [[2, null]], "Conda": [[30, "conda"]], "Configuration": [[3, null]], "Contributing": [[4, null]], "Core operations": [[5, "core-operations"]], "Cubed": [[31, null]], "Dask": [[33, "dask"]], "Debugging optimization": [[38, "debugging-optimization"]], "Deleting intermediate data": [[41, "deleting-intermediate-data"]], "Demo": [[28, null]], "Dependency Tree": [[32, "dependency-tree"]], "Design": [[5, null]], "Development": [[4, "development"]], "Diagnosing Performance": [[40, "diagnosing-performance"]], "Diagnostics": [[34, null]], "Differences between Cubed and the standard": [[1, "differences-between-cubed-and-the-standard"]], "Different Cloud Providers": [[40, "different-cloud-providers"]], "Different Executors": [[40, "different-executors"]], "Documentation": [[31, "documentation"]], "Environment variables": [[3, "environment-variables"]], "Examples in use": [[34, "examples-in-use"]], "Execution": [[2, "execution"]], "Executor options": [[3, "executor-options"]], "Executors": [[35, null]], "For developers": [[31, null]], "For users": [[31, null]], "Getting Started": [[29, null]], "History": [[34, "history"]], "IO": [[0, "io"]], "Installation": [[30, null]], "Local single-machine executors": [[35, "local-single-machine-executors"]], "Map fusion": [[38, "map-fusion"]], "Memory": [[2, "memory"], [37, null]], "Memray": [[34, "memray"]], "Multi-pipeline Calculation": [[40, "multi-pipeline-calculation"]], "Multi-step Calculation": [[40, "multi-step-calculation"]], "Multiple-input fusion": [[38, "multiple-input-fusion"]], "Non-standardised functions": [[0, "non-standardised-functions"]], "Operations": [[32, null]], "Optimization": [[38, null]], "Optional dependencies": [[30, "optional-dependencies"]], "Other Performance Considerations": [[40, "other-performance-considerations"]], "Pip": [[30, "pip"]], "Plan": [[2, "plan"]], "Preface: Types of Scaling": [[40, "preface-types-of-scaling"]], "Previous work": [[33, "previous-work"]], "Primitive operations": [[5, "primitive-operations"]], "Progress bar": [[34, "progress-bar"]], "Projected memory": [[37, "projected-memory"]], "Python Array API": [[1, null]], "Random number generation": [[0, "random-number-generation"]], "Reference": [[3, "reference"]], "Related Projects": [[33, null]], "Reliability": [[39, null]], "Reserved memory": [[37, "reserved-memory"]], "Retries": [[39, "retries"]], "Runtime": [[0, "runtime"], [5, "runtime"]], "Scaling": [[40, null]], "Single-step Calculation": [[40, "single-step-calculation"]], "Spec object": [[3, "spec-object"]], "Spec options": [[3, "spec-options"]], "Specification": [[3, "specification"]], "Specifying an executor": [[35, "specifying-an-executor"]], "Storage": [[5, "storage"], [41, null]], "Stragglers": [[39, "stragglers"]], "Strong consistency": [[39, "strong-consistency"]], "Theoretical vs Practical Scaling of Cubed": [[40, "theoretical-vs-practical-scaling-of-cubed"]], "Timeline": [[34, "timeline"]], "Timeouts": [[39, "timeouts"]], "Tips": [[40, "tips"]], "Total number of input blocks": [[38, "total-number-of-input-blocks"]], "Total number of source arrays": [[38, "total-number-of-source-arrays"]], "User Guide": [[36, null]], "Visualize the computation plan": [[34, "visualize-the-computation-plan"]], "Which cloud service executor should I use?": [[35, "which-cloud-service-executor-should-i-use"]], "Why Cubed?": [[42, null]], "Xarray": [[33, "xarray"]], "YAML file": [[3, "yaml-file"]], "beam": [[3, "beam"]], "blockwise": [[32, "blockwise"]], "coiled": [[3, "coiled"]], "cubed.Array": [[6, null]], "cubed.Array.compute": [[7, null]], "cubed.Array.rechunk": [[8, null]], "cubed.Array.visualize": [[9, null]], "cubed.Callback": [[10, null]], "cubed.Spec": [[11, null]], "cubed.TaskEndEvent": [[12, null]], "cubed.apply_gufunc": [[13, null]], "cubed.compute": [[14, null]], "cubed.from_array": [[15, null]], "cubed.from_zarr": [[16, null]], "cubed.map_blocks": [[17, null]], "cubed.map_overlap": [[18, null]], "cubed.measure_reserved_mem": [[19, null]], "cubed.nanmean": [[20, null]], "cubed.nansum": [[21, null]], "cubed.pad": [[22, null]], "cubed.random.random": [[23, null]], "cubed.rechunk": [[24, null]], "cubed.store": [[25, null]], "cubed.to_zarr": [[26, null]], "cubed.visualize": [[27, null]], "dask": [[3, "dask"]], "elemwise": [[32, "elemwise"]], "lithops": [[3, "lithops"]], "map_blocks": [[32, "map-blocks"]], "map_selection": [[32, "map-selection"]], "modal": [[3, "modal"]], "processes": [[3, "processes"]], "rechunk": [[32, "rechunk"]], "reduction and arg_reduction": [[32, "reduction-and-arg-reduction"]], "single-threaded": [[3, "single-threaded"]], "threads": [[3, "threads"]]}, "docnames": ["api", "array-api", "computation", "configuration", "contributing", "design", "generated/cubed.Array", "generated/cubed.Array.compute", "generated/cubed.Array.rechunk", "generated/cubed.Array.visualize", "generated/cubed.Callback", "generated/cubed.Spec", "generated/cubed.TaskEndEvent", "generated/cubed.apply_gufunc", "generated/cubed.compute", "generated/cubed.from_array", "generated/cubed.from_zarr", "generated/cubed.map_blocks", "generated/cubed.map_overlap", "generated/cubed.measure_reserved_mem", "generated/cubed.nanmean", "generated/cubed.nansum", "generated/cubed.pad", "generated/cubed.random.random", "generated/cubed.rechunk", "generated/cubed.store", "generated/cubed.to_zarr", "generated/cubed.visualize", "getting-started/demo", "getting-started/index", "getting-started/installation", "index", "operations", "related-projects", "user-guide/diagnostics", "user-guide/executors", "user-guide/index", "user-guide/memory", "user-guide/optimization", "user-guide/reliability", "user-guide/scaling", "user-guide/storage", "why-cubed"], "envversion": {"sphinx": 64, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["api.rst", "array-api.md", "computation.md", "configuration.md", "contributing.md", "design.md", "generated/cubed.Array.rst", "generated/cubed.Array.compute.rst", "generated/cubed.Array.rechunk.rst", "generated/cubed.Array.visualize.rst", "generated/cubed.Callback.rst", "generated/cubed.Spec.rst", "generated/cubed.TaskEndEvent.rst", "generated/cubed.apply_gufunc.rst", "generated/cubed.compute.rst", "generated/cubed.from_array.rst", "generated/cubed.from_zarr.rst", "generated/cubed.map_blocks.rst", "generated/cubed.map_overlap.rst", "generated/cubed.measure_reserved_mem.rst", "generated/cubed.nanmean.rst", "generated/cubed.nansum.rst", "generated/cubed.pad.rst", "generated/cubed.random.random.rst", "generated/cubed.rechunk.rst", "generated/cubed.store.rst", "generated/cubed.to_zarr.rst", "generated/cubed.visualize.rst", "getting-started/demo.md", "getting-started/index.md", "getting-started/installation.md", "index.md", "operations.md", "related-projects.md", "user-guide/diagnostics.md", "user-guide/executors.md", "user-guide/index.md", "user-guide/memory.md", "user-guide/optimization.md", "user-guide/reliability.md", "user-guide/scaling.md", "user-guide/storage.md", "why-cubed.md"], "indexentries": {"__init__() (cubed.array method)": [[6, "cubed.Array.__init__", false]], "__init__() (cubed.callback method)": [[10, "cubed.Callback.__init__", false]], "__init__() (cubed.spec method)": [[11, "cubed.Spec.__init__", false]], "__init__() (cubed.taskendevent method)": [[12, "cubed.TaskEndEvent.__init__", false]], "apply_gufunc() (in module cubed)": [[13, "cubed.apply_gufunc", false]], "arange() (in module cubed.array_api)": [[1, "cubed.array_api.arange", false]], "array (class in cubed)": [[6, "cubed.Array", false]], "asarray() (in module cubed.array_api)": [[1, "cubed.array_api.asarray", false]], "broadcast_to() (in module cubed.array_api)": [[1, "cubed.array_api.broadcast_to", false]], "callback (class in cubed)": [[10, "cubed.Callback", false]], "compute() (cubed.array method)": [[7, "cubed.Array.compute", false]], "compute() (in module cubed)": [[14, "cubed.compute", false]], "concat() (in module cubed.array_api)": [[1, "cubed.array_api.concat", false]], "empty() (in module cubed.array_api)": [[1, "cubed.array_api.empty", false]], "empty_like() (in module cubed.array_api)": [[1, "cubed.array_api.empty_like", false]], "eye() (in module cubed.array_api)": [[1, "cubed.array_api.eye", false]], "from_array() (in module cubed)": [[15, "cubed.from_array", false]], "from_zarr() (in module cubed)": [[16, "cubed.from_zarr", false]], "full() (in module cubed.array_api)": [[1, "cubed.array_api.full", false]], "full_like() (in module cubed.array_api)": [[1, "cubed.array_api.full_like", false]], "linspace() (in module cubed.array_api)": [[1, "cubed.array_api.linspace", false]], "map_blocks() (in module cubed)": [[17, "cubed.map_blocks", false]], "map_overlap() (in module cubed)": [[18, "cubed.map_overlap", false]], "measure_reserved_mem() (in module cubed)": [[19, "cubed.measure_reserved_mem", false]], "nanmean() (in module cubed)": [[20, "cubed.nanmean", false]], "nansum() (in module cubed)": [[21, "cubed.nansum", false]], "ones() (in module cubed.array_api)": [[1, "cubed.array_api.ones", false]], "ones_like() (in module cubed.array_api)": [[1, "cubed.array_api.ones_like", false]], "pad() (in module cubed)": [[22, "cubed.pad", false]], "random() (in module cubed.random)": [[23, "cubed.random.random", false]], "rechunk() (cubed.array method)": [[8, "cubed.Array.rechunk", false]], "rechunk() (in module cubed)": [[24, "cubed.rechunk", false]], "spec (class in cubed)": [[11, "cubed.Spec", false]], "store() (in module cubed)": [[25, "cubed.store", false]], "taskendevent (class in cubed)": [[12, "cubed.TaskEndEvent", false]], "to_zarr() (in module cubed)": [[26, "cubed.to_zarr", false]], "visualize() (cubed.array method)": [[9, "cubed.Array.visualize", false]], "visualize() (in module cubed)": [[27, "cubed.visualize", false]], "zeros() (in module cubed.array_api)": [[1, "cubed.array_api.zeros", false]], "zeros_like() (in module cubed.array_api)": [[1, "cubed.array_api.zeros_like", false]]}, "objects": {"cubed": [[6, 0, 1, "", "Array"], [10, 0, 1, "", "Callback"], [11, 0, 1, "", "Spec"], [12, 0, 1, "", "TaskEndEvent"], [13, 2, 1, "", "apply_gufunc"], [14, 2, 1, "", "compute"], [15, 2, 1, "", "from_array"], [16, 2, 1, "", "from_zarr"], [17, 2, 1, "", "map_blocks"], [18, 2, 1, "", "map_overlap"], [19, 2, 1, "", "measure_reserved_mem"], [20, 2, 1, "", "nanmean"], [21, 2, 1, "", "nansum"], [22, 2, 1, "", "pad"], [24, 2, 1, "", "rechunk"], [25, 2, 1, "", "store"], [26, 2, 1, "", "to_zarr"], [27, 2, 1, "", "visualize"]], "cubed.Array": [[6, 1, 1, "", "__init__"], [7, 1, 1, "", "compute"], [8, 1, 1, "", "rechunk"], [9, 1, 1, "", "visualize"]], "cubed.Callback": [[10, 1, 1, "", "__init__"]], "cubed.Spec": [[11, 1, 1, "", "__init__"]], "cubed.TaskEndEvent": [[12, 1, 1, "", "__init__"]], "cubed.array_api": [[1, 2, 1, "", "arange"], [1, 2, 1, "", "asarray"], [1, 2, 1, "", "broadcast_to"], [1, 2, 1, "", "concat"], [1, 2, 1, "", "empty"], [1, 2, 1, "", "empty_like"], [1, 2, 1, "", "eye"], [1, 2, 1, "", "full"], [1, 2, 1, "", "full_like"], [1, 2, 1, "", "linspace"], [1, 2, 1, "", "ones"], [1, 2, 1, "", "ones_like"], [1, 2, 1, "", "zeros"], [1, 2, 1, "", "zeros_like"]], "cubed.random": [[23, 2, 1, "", "random"]]}, "objnames": {"0": ["py", "class", "Python class"], "1": ["py", "method", "Python method"], "2": ["py", "function", "Python function"]}, "objtypes": {"0": "py:class", "1": "py:method", "2": "py:function"}, "terms": {"": [3, 5, 7, 31, 32, 33, 34, 35, 37, 38, 40, 41, 42], "0": [1, 11, 23, 30, 34], "00": 34, "001": [34, 38], "002": [34, 38], "003": [34, 38], "004": 38, "005": 38, "1": [1, 11, 12, 23, 28, 34, 35, 38, 41], "10": [4, 28, 38], "100": 34, "1000": 35, "10000": 34, "100kb": 28, "100mb": [3, 37, 40], "11": [3, 34], "12": 1, "180": 3, "2": [3, 11, 28, 30, 34, 35, 38], "20": [35, 37, 38], "2000": 3, "200mb": 34, "2022": 1, "2023": 1, "2gb": [3, 35, 37, 40], "3": [3, 4, 28, 34, 38], "300": 3, "4": [28, 34, 38], "400mb": 34, "438": 1, "5": [28, 34, 38], "5000": [3, 34], "50000": 3, "6": [28, 34, 38], "7": [28, 30, 34, 38], "8": [28, 34, 38], "9": [28, 34, 38], "A": [0, 2, 3, 35, 37, 39, 40, 42], "And": [33, 38], "As": [2, 37], "At": 35, "By": [34, 38, 41], "For": [2, 32, 37, 39, 40, 41], "If": [3, 7, 9, 11, 14, 27, 34, 35, 37, 38, 39, 41], "In": [25, 32, 33, 34, 37, 38, 40, 41], "It": [3, 19, 32, 33, 34, 35], "No": 32, "Not": 32, "On": [38, 41], "One": [2, 3, 40], "Or": [2, 3], "The": [1, 2, 3, 5, 7, 8, 9, 11, 14, 16, 18, 19, 24, 25, 26, 27, 32, 33, 34, 35, 37, 38, 40, 41], "Then": 34, "There": [3, 5, 34, 37, 38, 40], "These": [0, 3, 5, 38], "To": [3, 30, 34, 38, 39, 40, 41], "With": 40, "__init__": [6, 10, 11, 12], "abil": 37, "about": [12, 13, 32, 33, 34], "abov": [34, 35, 38], "abstract": [33, 42], "accept": 1, "account": 35, "accur": 37, "achiev": 40, "act": 39, "activ": 4, "actual": [3, 34, 37], "acycl": [2, 34], "ad": [2, 34, 40], "adapt": 34, "add": [3, 28, 32, 34, 38, 40, 41], "addit": [34, 38], "adjust": 40, "adopt": 42, "advanc": 36, "advantag": [2, 33, 40], "affect": [34, 40], "after": [8, 18, 24, 34, 39, 41], "ag": 41, "again": [38, 39], "aggregr": 32, "ahead": [2, 37], "aim": [29, 37, 40], "algorithm": [32, 37], "all": [3, 5, 32, 34, 35, 40, 42], "alloc": [13, 34, 42], "allow": [32, 36, 38, 39], "allow_rechunk": 13, "allowed_mem": [3, 11, 28, 35, 37, 40], "alon": 34, "along": [20, 32], "alreadi": [7, 14], "also": [2, 3, 28, 30, 35, 39, 40], "although": [32, 35], "alwai": [34, 42], "amazon": [35, 39], "amount": [13, 19, 34, 35, 37, 38, 39, 40, 42], "an": [2, 3, 4, 8, 9, 11, 15, 16, 19, 22, 24, 26, 27, 29, 32, 34, 36, 37, 38, 39, 40, 42], "analys": 37, "ani": [3, 7, 11, 19, 34, 35, 38, 40, 41], "annot": 34, "anoth": [2, 37, 39, 40, 42], "anyth": 3, "apach": [5, 30, 31, 42], "api": [6, 28, 31, 32, 33], "appli": [13, 17, 18, 32, 40], "applic": 5, "apply_gufunc": 31, "approach": [41, 42], "appropri": [3, 41], "ar": [0, 1, 2, 3, 4, 5, 9, 13, 27, 30, 32, 33, 34, 35, 37, 38, 39, 40, 41, 42], "arang": 1, "arbitrari": 11, "arg": [13, 17, 18], "arg_reduct": 31, "argument": [1, 3, 18, 19, 34, 38], "arithmet": 20, "around": [37, 40], "arrai": [2, 3, 10, 13, 14, 15, 16, 17, 18, 21, 22, 24, 25, 26, 27, 28, 32, 33, 34, 35, 37, 40, 41, 42], "arrang": 32, "array_api": [1, 3, 28, 34, 38], "arrow": 32, "asarrai": [1, 15, 28, 34, 38], "assembl": 32, "assum": 13, "astyp": 38, "atom": 39, "attempt": [3, 39, 42], "attribut": [5, 6, 11, 12, 32], "auto": [1, 15], "automat": [3, 34, 35, 37, 38, 40, 41], "avail": [3, 11, 28, 37, 39], "averag": 40, "avoid": [32, 42], "aw": [3, 30, 35, 40, 41], "awai": [38, 42], "ax": [13, 32], "axi": [1, 13, 20, 21, 32], "b": [3, 28, 34, 38], "back": [5, 6, 34, 35], "backup": [3, 39, 40], "bag": 33, "base": 40, "baselin": 37, "basi": 37, "basic": [3, 37], "basicconfig": 38, "batch": 3, "batch_siz": 3, "beam": [5, 30, 31, 33, 40, 42], "becaus": [35, 37], "been": [0, 3, 7, 11, 14, 28, 33, 34, 35, 38, 39, 40, 42], "befor": [2, 3, 7, 9, 14, 27, 34, 35, 38, 40, 41], "being": 38, "below": [3, 35, 37, 38], "better": 33, "between": [31, 32, 42], "big": 40, "bin": 34, "block": [5, 17, 18, 32, 34, 40], "blockwis": [2, 31, 33, 38], "blosc": 11, "blue": 5, "bool": [7, 9, 14, 18, 27], "both": [2, 33, 38, 39, 40], "bottom": [5, 32], "bound": [33, 37, 40, 42], "boundari": 18, "box": 34, "branch": 40, "break": 42, "bring": 39, "broadcast": 32, "broadcast_to": 1, "bucket": 41, "budget": 37, "buffer": 34, "build": [2, 35, 37], "built": [2, 5], "bulk": 42, "byte": [11, 13, 19], "c": [3, 28, 30, 34, 38], "calcul": 37, "call": [2, 18, 32, 33, 34, 35, 38, 39], "callabl": [7, 9, 14, 18, 27], "callback": [7, 12, 14, 36], "can": [0, 2, 3, 9, 11, 19, 27, 30, 33, 34, 35, 37, 38, 39, 40, 41, 42], "cancel": 39, "cannot": [32, 40], "carefulli": 42, "carri": 40, "case": [34, 37, 38, 40, 41, 42], "cd": 34, "central": 42, "certain": [39, 41], "challeng": 42, "chang": [3, 8, 24, 32, 33, 34, 38, 40], "characterist": 40, "check": 37, "child": 3, "choic": 34, "choos": 2, "chosen": 5, "chunk": [1, 2, 3, 5, 6, 8, 15, 17, 18, 22, 23, 24, 28, 31, 32, 33, 34, 35, 36, 38, 39, 40, 42], "circumst": 39, "class": [6, 10, 11, 12], "clean": 41, "clear": 41, "clevel": 11, "click": 41, "client": [3, 30, 39], "cloud": [3, 5, 19, 28, 33, 36, 37, 39, 42], "cluster": [35, 40], "clutter": 32, "cname": 11, "code": 3, "coil": [30, 35], "coiled_kwarg": 3, "collect": [25, 34], "colour": 34, "come": [35, 40], "command": 41, "common": [37, 42], "commonli": 38, "compar": 35, "complet": [12, 30, 34, 39, 40], "complex": [37, 40], "compon": 33, "compos": [5, 32], "compress": [11, 34, 37], "compressor": 11, "comput": [3, 5, 9, 10, 11, 16, 19, 20, 25, 26, 27, 28, 31, 33, 35, 36, 37, 38, 39, 40, 41, 42], "compute_arrays_in_parallel": [3, 40], "compute_kwarg": 3, "concat": [1, 38], "concept": 36, "concern": 33, "concurr": [3, 40], "conda": [4, 29], "confid": 37, "configur": [11, 31, 35, 37, 39, 40, 42], "conflict": 30, "conform": 6, "conserv": 37, "consid": [39, 40], "consider": 36, "consist": 36, "consol": 41, "constant": [18, 34], "constraint": 40, "construct": 3, "constructor": 3, "consult": 32, "contain": 37, "continu": 33, "contribut": 31, "control": 42, "conveni": 41, "convent": 28, "convert": [2, 32, 33, 34, 40], "copi": 1, "core": [3, 28, 31, 32, 33, 35, 38], "corearrai": [8, 14, 27], "corner": 34, "correspond": [17, 18, 32, 33], "cost": 41, "could": 40, "coupl": 3, "cov": 30, "cover": [3, 39], "coverag": 1, "cpu": 3, "creat": [0, 4, 15, 19, 33, 34, 38, 41], "creation": [0, 1, 3], "csv": 34, "cube": [0, 2, 3, 4, 5, 28, 29, 30, 32, 33, 34, 35, 36, 37, 38, 39, 41], "cubed_config": 3, "cubed_spec__allowed_mem": 3, "cubed_spec__executor_nam": 3, "cubed_spec__executor_options__runtim": 3, "cubed_spec__executor_options__runtime_memori": 3, "cubed_spec__executor_options__use_backup": 3, "cubed_spec__work_dir": 3, "cumul": 40, "current": [3, 13, 18, 25, 34, 35, 39, 40], "cutdown": 13, "d": [34, 38], "dag": [2, 34], "dai": 41, "dask": [5, 13, 28, 30, 31, 40, 42], "data": [3, 8, 11, 19, 24, 34, 35, 36, 37, 38, 42], "dataflow": [3, 33, 35, 42], "datafram": 33, "dataset": [35, 40], "daunt": 33, "debug": [34, 36], "decompos": 33, "decompress": 37, "dedic": 41, "deeper": 40, "default": [3, 7, 9, 11, 14, 25, 26, 27, 28, 34, 35, 38, 39, 40, 41], "defin": [5, 33, 40], "delai": [33, 40], "deleg": 5, "delet": 36, "deliber": [3, 42], "demo": [29, 31], "depend": [2, 7, 29, 31, 34, 39, 40], "deploi": 3, "depth": [18, 32], "describ": [28, 35, 39], "descript": 3, "design": [3, 31, 40, 42], "desir": [8, 24], "detail": [2, 3, 32, 35, 38], "determin": [34, 39], "develop": 34, "devic": 1, "diagaram": 37, "diagnos": 36, "diagnost": [30, 31, 36, 40], "diagram": 32, "dict": [3, 11, 18], "dictionari": 11, "differ": [3, 30, 31, 32, 38, 39], "difficult": 38, "dill": 30, "dimens": 32, "dimension": 42, "dip": 34, "direct": [2, 34], "directli": [3, 32], "directori": [3, 11, 19, 34, 41], "disabl": 11, "disadvantag": [2, 40], "discuss": [2, 35, 37], "disk": [2, 5, 9, 27, 35], "displai": [9, 27, 34, 40], "disproportion": [39, 40, 42], "distribut": [2, 3, 30, 33, 35, 39, 40, 42], "do": [34, 39, 40, 42], "doc": 3, "document": 3, "doe": [32, 35, 38, 41], "doesn": [3, 9, 27, 35, 37], "don": [2, 3, 34, 35, 41], "done": [34, 40], "donfig": 3, "dot": [9, 27], "doubl": 3, "down": 39, "drop": 32, "drop_axi": 17, "dtype": [1, 2, 5, 17, 18, 21, 32, 37], "due": 30, "duplic": 39, "dure": [10, 37], "e": [4, 38, 40], "each": [2, 3, 18, 32, 34, 37, 39, 40], "eager": [25, 26], "easi": 35, "easier": 33, "echo": 34, "edg": 2, "effect": 38, "effici": [2, 37], "effort": 42, "either": 38, "element": [18, 21, 32, 40], "elementwis": 40, "elemwis": 31, "embarrassingli": 42, "empti": 1, "empty_lik": 1, "enabl": [34, 38, 39, 40], "encapsul": 3, "endpoint": 1, "engin": [2, 42], "enhanc": 40, "enough": 40, "ensur": [34, 37, 41], "enter": 41, "environ": [2, 4, 35, 40], "equal": 40, "equival": [3, 13], "error": 39, "eschew": 42, "especi": [30, 40], "essenti": 28, "estim": 37, "etc": [11, 31], "evalu": 28, "even": [39, 40, 42], "event": [7, 10, 14, 34], "everi": [2, 5, 18, 35, 40], "everyth": 42, "exampl": [2, 3, 5, 11, 28, 31, 32, 35, 37, 38, 39, 40], "exce": [37, 38], "except": [1, 3, 37, 39], "excess": [35, 38], "exclud": 19, "execut": [31, 37, 40], "executor": [5, 7, 11, 14, 19, 25, 26, 28, 30, 31, 34, 36, 37, 39, 41], "executor_nam": [3, 11, 35], "executor_opt": [3, 11], "exist": 40, "expand": 3, "expect": [39, 40], "explain": [34, 38], "explicitli": 2, "export": 3, "expos": 3, "express": 2, "extens": [5, 9, 27], "extent": 42, "extern": 5, "extra": [1, 18], "extra_projected_mem": 13, "ey": 1, "f": 34, "factor": [37, 40], "fail": [3, 39], "fals": [3, 7, 9, 14, 18, 20, 21, 27, 34, 38, 39, 40], "fan": 38, "far": [35, 40], "fashion": 39, "faster": 40, "fault": 39, "featur": [3, 34, 39, 40], "fee": 35, "feed": 40, "few": [1, 5, 39, 40, 42], "fewer": 40, "file": [9, 27, 34, 35, 39, 41], "filenam": [9, 27, 34], "filesystem": 41, "fill_valu": 1, "final": 32, "fine": 33, "first": [32, 34, 37, 38, 40], "firstli": 40, "fit": 35, "five": 5, "fix": 40, "flame": 34, "flamegraph": 34, "flexibl": 33, "float": 23, "float32": 38, "follow": [1, 3, 11, 31, 32, 34, 37, 38, 41], "forg": 30, "form": [11, 32, 41], "format": [9, 27, 34], "formerli": 42, "four": [3, 37], "fourier": 1, "framework": 42, "free": 35, "from": [3, 5, 7, 14, 15, 16, 17, 18, 32, 34, 37, 38, 39, 40, 42], "from_arrai": 0, "from_zarr": 0, "fsspec": [3, 11, 19, 41], "fuctool": 38, "full": [1, 30, 40], "full_lik": 1, "func": [13, 17, 18], "function": [1, 2, 3, 5, 7, 9, 13, 14, 17, 18, 19, 27, 30, 31, 32, 34, 35, 37, 38, 40], "function_end_tstamp": 12, "function_start_tstamp": 12, "functionexecutor": 3, "functool": 38, "fundament": [32, 33], "further": [37, 42], "fuse": [32, 34, 38, 40], "fusion": [2, 7, 9, 14, 27, 36], "futur": 3, "g": 40, "gain": 42, "gb": 11, "gc": 35, "gcf": 40, "gcp": [3, 30], "gcsf": 30, "gener": [5, 13, 31, 32, 34, 37, 40, 42], "general_blockwis": 32, "get": [4, 31, 35, 36, 38, 40, 41], "github": 4, "given": [3, 19, 21, 37, 40], "global": 39, "go": [5, 34, 42], "good": [3, 11, 37, 40], "googl": [3, 33, 35, 39, 40, 41, 42], "grain": 33, "graph": [2, 7, 9, 14, 27, 33, 34, 38], "graphic": 34, "graphviz": 30, "greater": 37, "green": 32, "group": [16, 26], "guarante": [31, 35, 40, 42], "guid": [19, 29, 31], "ha": [2, 3, 11, 28, 30, 33, 34, 35, 37, 38, 39, 40, 42], "had": 35, "hadoop": 42, "half": 23, "hand": 38, "handi": 3, "handl": [18, 35, 40], "hard": 3, "have": [0, 2, 3, 7, 14, 29, 32, 33, 34, 35, 37, 38, 39, 40], "head": 4, "heavili": [5, 33], "help": [36, 37, 38], "henc": [39, 40], "here": [3, 32, 34, 38, 40, 42], "hidden": [9, 27], "high": [2, 29, 33, 37, 42], "higher": [33, 38], "highest": [35, 37], "hist": 34, "historycallback": 34, "hold": 40, "horizont": [34, 40], "how": [2, 3, 18, 32, 34, 37, 38, 40, 41, 42], "howev": [2, 13, 42], "html": 34, "i": [1, 2, 3, 5, 7, 9, 11, 13, 14, 18, 19, 25, 26, 27, 28, 31, 32, 33, 34, 36, 37, 38, 39, 40, 41, 42], "id": [11, 34], "ideal": [34, 40], "idempot": 39, "ident": 39, "ignor": 20, "illustr": 37, "imag": [3, 9, 27, 34], "immedi": [25, 26, 39], "implement": [1, 5, 19, 25, 28, 31, 32, 33, 42], "implicitli": 2, "import": [3, 9, 27, 28, 34, 35, 37, 38, 40, 41], "importantli": 39, "impos": 38, "improv": [34, 37, 40, 42], "includ": [0, 3, 5, 9, 11, 27, 33], "incur": 41, "independ": 40, "index": 32, "indic": 3, "individu": [3, 38], "infinit": 40, "influenc": 33, "inform": [12, 13, 32, 34, 37, 38], "inherit": 5, "input": [3, 16, 17, 18, 32, 36, 37, 40], "instal": [4, 19, 29, 31, 34], "instanc": [3, 40], "instanti": 3, "instead": 42, "instruct": 41, "int": [11, 18, 19], "integ": 38, "integr": [31, 33], "intend": 35, "interact": 28, "interest": 42, "intermedi": [2, 3, 7, 11, 14, 19, 34, 36, 38, 40], "intern": 34, "interv": 23, "interven": 33, "intro": 31, "introduc": 36, "introduct": 29, "invok": [2, 3], "involv": [2, 4, 34, 42], "io": [5, 31, 38, 39], "ipython": [9, 27], "issu": [1, 34], "iter": 40, "its": [8, 18, 24, 32, 34, 37, 38, 40], "itself": 37, "jinja2": 30, "job": 41, "jpeg": [9, 27], "jpg": [9, 27], "jupyt": 34, "just": [28, 32, 34, 38], "k": 1, "kb": 11, "keepdim": [13, 20, 21], "kei": [3, 32, 39], "keyword": [1, 3, 18, 19], "kill": 3, "kind": 38, "known": [2, 37, 40, 42], "kwarg": [7, 13, 14, 17, 18, 19, 25, 26], "l": 34, "label": 34, "lambda": [35, 40], "larg": [35, 37, 39, 40, 42], "larger": [35, 40], "last": 42, "later": 34, "launch": 39, "layer": [5, 33], "lazi": [2, 28], "lead": [40, 42], "least": 37, "less": 33, "lesser": 42, "let": 5, "level": [2, 29, 33, 38, 42], "librari": [31, 33], "lifecycl": 41, "like": [3, 5, 15, 25, 28, 32, 33, 35, 37, 38, 41, 42], "lilac": 34, "limit": [13, 38, 40], "line": 34, "linear": 40, "linspac": 1, "list": [1, 3, 7, 14, 18, 30], "lithop": [5, 19, 30, 31, 34, 35, 42], "live": 3, "ll": 28, "load": [16, 34], "local": [5, 11, 28, 34, 36, 37, 41], "locat": 3, "log": 38, "logic": 40, "long": [34, 38, 40], "longer": [3, 34, 38, 39, 40], "look": [3, 29, 32, 38], "lot": [41, 42], "lz4": 11, "m": [30, 34], "machin": [28, 36, 37, 40], "made": [3, 42], "magic": 40, "mai": [2, 5, 18, 32, 35, 38, 40, 41], "main": [2, 3, 32], "mainli": 3, "maintain": 40, "major": 42, "make": [2, 33, 37, 40], "manag": [3, 37, 42], "mani": [30, 32, 38], "manipul": 1, "manual": 41, "map": [2, 7, 9, 14, 18, 27, 32, 36, 40], "map_block": [5, 31], "map_direct": 32, "map_overlap": 31, "map_select": 31, "mapreduc": 42, "mark": [9, 27], "match": [32, 38, 40], "materi": [2, 34], "matplotlib": 30, "matur": [33, 35], "max_tasks_per_child": 3, "max_total_num_input_block": 38, "max_total_source_arrai": 38, "max_work": [3, 40], "maximum": [3, 31, 37, 38, 40], "mb": 11, "mean": [3, 5, 20, 34, 37, 39, 40], "measur": [19, 37, 40], "measure_reserved_mem": [37, 40], "memori": [3, 11, 13, 19, 33, 34, 35, 36, 40, 42], "memrai": 36, "messag": 39, "method": [6, 10, 11, 12, 34, 38], "middl": 32, "might": 40, "minim": 30, "minimum": 40, "minut": 35, "mitig": [3, 39], "mock": 30, "modal": [5, 19, 30, 31, 35, 40], "mode": 22, "model": [2, 42], "monitor": 34, "more": [13, 28, 32, 33, 34, 35, 37, 38, 39, 40], "most": [3, 35, 36, 42], "much": [34, 35, 37, 40], "multi": 28, "multipl": [2, 13, 14, 17, 18, 27, 31, 32, 34, 36], "multiple_inputs_optimize_dag": 38, "must": [3, 19, 25, 32, 38, 39], "n": [3, 42], "n_col": 1, "n_row": 1, "name": [4, 6, 9, 12, 27, 33, 41], "nan": [20, 21], "natur": [2, 33, 37], "nbyte": 38, "necessari": 40, "need": [2, 5, 13, 32, 34, 37, 39, 40, 41], "neg": 38, "neighbor": 18, "neither": 3, "nest": 3, "never": 37, "new": [5, 13, 33, 34, 37, 40], "new_axi": 17, "newer": 33, "next": 40, "node": [2, 34, 42], "non": [3, 11, 31, 37, 40], "none": [1, 3, 7, 9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 38], "notat": 32, "note": [3, 18, 25, 26, 32, 34, 35, 38, 39, 41], "notebook": [9, 27, 34], "noth": [28, 40], "notic": [3, 28, 38], "now": [32, 38], "np": 18, "num": [1, 38], "num_task": 12, "number": [3, 5, 18, 31, 32, 34, 35, 39, 40], "numblock": 32, "numcodec": 11, "numpi": [2, 5, 28, 32, 33], "numpy_groupi": 30, "o": 34, "obj": 1, "object": [2, 10, 15, 19, 25, 28, 41], "observ": 40, "off": [34, 38, 39, 40], "offer": 40, "often": 42, "old": 41, "onc": [14, 34, 39], "one": [0, 2, 3, 18, 32, 34, 38, 39, 40, 41, 42], "ones": [1, 33, 38], "ones_lik": 1, "onli": [2, 3, 7, 9, 14, 18, 27, 32, 33, 34, 35, 38, 39, 40, 42], "op": [34, 38], "open": 23, "oper": [2, 9, 19, 25, 26, 27, 31, 33, 34, 37, 38, 39, 40, 41, 42], "opportun": 2, "opt_fn": 38, "optim": [2, 7, 9, 14, 27, 31, 32, 34, 36, 40, 42], "optimize_funct": [7, 9, 14, 27, 38], "optimize_graph": [7, 9, 14, 27, 34, 38], "option": [7, 9, 11, 14, 16, 19, 25, 26, 27, 29, 34], "optional_depend": 30, "orang": [32, 34], "order": [32, 37, 40], "origin": 39, "other": [3, 5, 36, 38], "otherwis": [7, 9, 14, 27, 38], "out": [34, 35, 36, 40, 41], "outer": 32, "output": [2, 9, 13, 18, 26, 27, 32, 34, 37, 38, 39, 40], "output_dtyp": 13, "output_s": 13, "outsid": 3, "over": [4, 18, 21, 32, 41], "overal": [34, 38, 39, 40], "overhead": 35, "overlap": 18, "overview": 29, "own": 40, "packag": [3, 19, 30, 33], "pad_width": 22, "page": [1, 3, 32, 40], "pair": 2, "panda": 30, "paragraph": 39, "parallel": [2, 3, 33, 40, 42], "paramet": [7, 8, 9, 11, 13, 14, 16, 18, 19, 24, 25, 26, 27, 40], "part": [32, 35, 40], "partial": 38, "partial_reduc": 32, "particular": [3, 35, 37, 38], "pass": [3, 11, 18, 19, 34, 40, 42], "path": [3, 11, 16, 19, 26], "pdf": [9, 27], "peak": 19, "peak_measured_mem": 37, "peak_measured_mem_end": 12, "peak_measured_mem_start": 12, "per": [13, 40], "perfect": 34, "perform": [7, 9, 14, 27, 32, 34, 36, 37, 38, 42], "period": 41, "persist": [40, 41, 42], "phase": 37, "pick": 38, "pink": 32, "pip": [4, 29, 34], "plan": [6, 31, 33, 36, 37, 38, 40], "pleas": 4, "plenti": 37, "plot": 34, "png": [9, 27], "possibl": [3, 29, 37, 38, 40, 42], "potenti": 40, "power": 38, "practic": 36, "pre": 39, "preced": 38, "precis": 37, "predecessor": 38, "prefac": 36, "preserv": 32, "previou": [31, 38, 39], "previous": 32, "primit": [2, 31, 32, 33, 38], "print": 28, "problem": [40, 42], "process": [2, 5, 19, 25, 26, 33, 34, 35, 37, 39, 40, 42], "processor": 40, "processpoolexecutor": 3, "produc": [9, 18, 27, 34, 37], "profil": 34, "program": [3, 42], "project": [5, 30, 31, 34, 36], "projected_mem": 37, "properli": 37, "properti": 3, "proportion": 40, "provid": [5, 9, 27, 32, 34, 35, 37, 39, 42], "provis": 40, "public": 5, "purpos": 42, "put": [37, 42], "py": 34, "pydot": 30, "pyproject": 30, "pytest": 30, "python": [0, 3, 4, 5, 6, 13, 19, 25, 26, 28, 30, 31, 33, 34, 35, 37], "pywren": 42, "quickli": [29, 35, 39], "r": 4, "rais": 37, "ran": 37, "random": [3, 5, 31, 34], "rather": [32, 41], "re": 40, "read": [32, 33, 34, 37, 38, 39, 40, 42], "readm": 28, "reason": [3, 33, 37, 38, 40], "receiv": 10, "rechunk": [2, 31, 33, 40, 42], "recommend": 35, "recomput": [7, 14], "red": 5, "reduc": [32, 38, 40], "reduct": [31, 37, 38, 40], "refer": [13, 31, 32, 35, 40], "region": [35, 41], "regular": 41, "rel": 35, "relat": 31, "reli": 39, "reliabl": [31, 35, 36], "remot": 35, "remov": 41, "render": [9, 27, 34], "report": [19, 37], "repres": [32, 34], "represent": [2, 40], "req": 30, "requir": [2, 4, 19, 34, 35, 37, 40, 42], "reserv": [3, 11, 19, 36], "reserved_mem": [3, 11, 19, 37], "resourc": [11, 28, 40], "restrict": 40, "result": [28, 32, 34, 38], "resum": [7, 14], "retain": 34, "retri": [3, 36], "retriev": 32, "return": [8, 9, 16, 19, 21, 23, 24, 27, 32], "reus": 3, "rich": [30, 34], "richprogressbar": 34, "right": 34, "rm": 41, "room": 37, "round": [32, 34], "rule": [37, 41], "run": [2, 3, 7, 11, 14, 19, 25, 26, 28, 30, 33, 34, 35, 37, 38, 39, 40, 41, 42], "runner": 33, "runtim": [2, 3, 19, 25, 26, 31, 33, 35, 37, 41, 42], "runtime_memori": 3, "s3": [3, 35, 39, 41], "s3f": 30, "same": [3, 38, 40, 41], "save": [25, 26, 34], "scale": [2, 31, 34, 35, 36, 39, 42], "scenario": 40, "schema": 34, "seaborn": 30, "second": [3, 32, 35], "section": 39, "see": [2, 3, 28, 30, 34, 38, 40], "select": [32, 41], "send": [7, 14], "sent": 32, "separ": [30, 34, 40, 41], "serverless": [2, 33, 40, 42], "servic": [36, 37, 40, 41, 42], "set": [2, 3, 7, 11, 13, 14, 19, 30, 35, 36, 37, 39, 40, 41], "sever": 33, "shape": [1, 5, 8, 18, 24, 32], "share": [3, 18], "short": 41, "should": [11, 18, 36, 37, 38, 40, 41], "show": [9, 27, 32, 34, 35], "show_hidden": [9, 27], "shown": [32, 34, 38], "shuffl": [2, 11, 40, 42], "sign": [35, 40], "signatur": 13, "signifi": 34, "signific": 40, "similar": [13, 33], "similarli": [32, 34], "simpl": [3, 28, 34, 35], "simplest": [32, 38, 40], "simpli": 34, "simultan": 40, "sinc": [2, 3, 32, 33, 34, 35, 37, 38, 39, 42], "singl": [28, 32, 33, 34, 36, 39], "size": [2, 23, 32, 34, 36, 40, 42], "slice": 32, "slide": 31, "slow": [2, 38, 39], "slowli": 40, "small": [34, 35, 38], "smaller": 37, "so": [3, 28, 33, 34, 35, 37, 38, 39, 40, 41], "solut": 40, "some": [18, 33, 37, 38, 39, 40], "sometim": [35, 38], "sourc": 25, "space": 41, "spark": 42, "spec": [1, 6, 7, 14, 15, 16, 17, 19, 23, 28, 34, 35, 37, 41], "specif": [11, 31, 40], "specifi": [3, 11, 18, 19, 20, 28, 32, 36, 38, 41], "specul": 39, "speed": [38, 40], "spent": [34, 42], "split_everi": [20, 21], "squar": 34, "squeez": 32, "stack": 38, "stage": 38, "standard": [0, 6, 28, 31, 32, 33], "standardis": 31, "start": [1, 28, 31, 35, 38, 40, 42], "startup": [34, 35, 40], "stat": 34, "stateless": [5, 42], "statist": 34, "statu": 1, "step": [1, 34], "stick": 40, "stop": 1, "storag": [2, 6, 11, 16, 26, 31, 36, 39, 40, 42], "storage_opt": 11, "store": [3, 5, 11, 16, 19, 26, 34, 40, 41], "str": [9, 11, 19, 27], "straggler": [3, 34, 36, 40], "straightforward": [2, 35], "string": [16, 26], "strong": [36, 40], "style": 3, "submit": 3, "subset": [5, 28, 32], "success": 42, "suffici": 40, "suggest": 40, "suit": 34, "sum": [21, 37], "support": [1, 3, 13, 18, 32, 33, 38], "sure": [40, 41], "svg": [9, 27, 34], "system": [5, 19, 39, 40, 41, 42], "t": [2, 3, 7, 9, 14, 27, 33, 34, 35, 37, 38, 41], "tab": 41, "tail": 34, "take": [3, 34, 39, 40], "taken": 39, "target": [25, 42], "target_stor": 24, "task": [2, 3, 5, 11, 12, 13, 19, 33, 34, 35, 37, 38, 39, 40], "task_create_tstamp": 12, "task_result_tstamp": 12, "tb": 11, "tell": 13, "temp": [3, 35, 41], "tempor": 34, "temporari": [3, 41], "ten": 37, "terabyt": 40, "test": [3, 30, 34, 35], "test_mem_util": 34, "than": [3, 13, 32, 33, 37, 38, 39, 40, 41], "thei": [3, 32, 33, 34, 37, 38, 40], "them": [2, 32, 34, 39, 41], "theme": 42, "theoret": 36, "therebi": [40, 42], "therefor": [32, 35], "thi": [2, 3, 5, 7, 8, 9, 11, 13, 18, 19, 25, 26, 28, 29, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42], "thing": 3, "those": 40, "though": [40, 42], "thread": [35, 40], "threadpoolexecutor": 3, "three": [3, 32, 33, 34, 38, 39], "through": [3, 5], "throughput": 40, "thumb": 37, "tightli": [37, 42], "time": [2, 3, 32, 34, 35, 37, 38, 39, 40, 41, 42], "timeline_viz": 34, "timelinevisualizationcallback": 34, "timeout": [3, 36], "tini": [19, 34], "tip": 36, "tmp": 28, "tmpdir": 41, "to_zarr": 34, "togeth": [2, 38], "toler": 39, "toml": 30, "tomwhit": [3, 35, 41], "too": [32, 38], "took": 34, "tool": 34, "top": [5, 32], "total": [3, 11, 39, 40], "tqdm": [30, 34], "tqdmprogressbar": 34, "trace": 34, "track": [1, 32, 34], "traction": 42, "transfer": 35, "transform": 1, "travers": 2, "treat": 21, "tree": 31, "tree_reduc": 32, "trigger": 2, "trim": 18, "trivial": [19, 40], "true": [1, 3, 7, 9, 14, 27, 38, 40], "tune": [40, 42], "tupl": [8, 18, 24], "turn": [3, 34, 38, 39, 40], "twice": 34, "two": [5, 32, 34, 40], "txt": 4, "type": [5, 8, 9, 16, 18, 19, 24, 25, 26, 27, 34, 36], "typic": 41, "ufunc": 13, "under": [3, 34], "underli": 5, "underscor": 3, "understand": [33, 34, 38, 40, 42], "undoubtedli": 42, "unit": [5, 11], "unlik": 32, "unnecessari": 41, "unoptim": 38, "unpredict": 42, "until": 32, "up": [2, 3, 35, 37, 38, 39, 40, 41], "updat": 39, "upgrad": 40, "upper": 37, "url": [3, 11, 19, 41], "us": [2, 3, 5, 7, 9, 11, 14, 16, 19, 25, 26, 27, 28, 29, 30, 31, 32, 33, 36, 37, 38, 40, 41, 42], "usag": [13, 31, 37, 40, 42], "use_backup": [3, 39, 40], "user": [3, 33, 35, 42], "usual": 34, "util": [33, 37], "v": 36, "valu": [3, 11, 18, 32, 37, 38], "vari": [19, 40], "varieti": [3, 33, 34, 35], "variou": 37, "vector": 13, "veri": [2, 4, 34, 35, 40, 42], "versa": 40, "version": [1, 13, 19], "versu": 40, "vertic": [34, 40], "via": [31, 42], "vice": 40, "view": 34, "violat": 40, "visual": [33, 36, 38], "wa": [5, 33, 34], "wai": [3, 34, 39, 40], "wait": 40, "want": [34, 38], "wasn": 33, "we": [3, 28, 32, 34, 35, 37, 38, 40], "weak": 40, "welcom": 4, "well": [5, 33, 34, 37], "were": 32, "what": [33, 34, 38], "whatev": 37, "when": [2, 3, 11, 19, 33, 34, 35, 37, 38, 39, 40, 41], "where": [2, 11, 33, 38, 40], "wherea": [33, 40], "whether": [3, 18], "which": [2, 3, 5, 9, 11, 27, 28, 30, 32, 33, 34, 36, 37, 38, 40, 41, 42], "while": [2, 7, 14, 32, 34, 37, 40], "white": 32, "who": 40, "whole": 39, "why": [31, 38], "widespread": 42, "within": [3, 37], "without": [8, 24, 32, 40], "won": [7, 14], "word": 40, "work": [19, 31, 32, 34, 35, 37, 39, 41, 42], "work_dir": [3, 11, 19, 28, 35, 41], "worker": [3, 11, 34, 35, 37, 40, 42], "would": [34, 37, 38, 40, 41], "write": [2, 9, 25, 27, 34, 35, 37, 39, 40, 41, 42], "written": [2, 32, 34, 38, 39], "x": [1, 15, 20, 21, 22, 24, 25, 26], "xarrai": 31, "xp": [3, 28, 34, 38], "yaml": 35, "year": [33, 42], "yet": [0, 28], "you": [3, 13, 29, 30, 33, 34, 35, 36, 37, 38, 39, 40, 41], "your": [3, 34, 35, 37, 40], "zappi": 33, "zarr": [2, 5, 6, 11, 16, 25, 26, 31, 34, 35, 37, 39, 40, 41, 42], "zarr_compressor": 11, "zarrai": 6, "zero": [1, 21], "zeros_lik": 1}, "titles": ["API Reference", "Python Array API", "Computation", "Configuration", "Contributing", "Design", "cubed.Array", "cubed.Array.compute", "cubed.Array.rechunk", "cubed.Array.visualize", "cubed.Callback", "cubed.Spec", "cubed.TaskEndEvent", "cubed.apply_gufunc", "cubed.compute", "cubed.from_array", "cubed.from_zarr", "cubed.map_blocks", "cubed.map_overlap", "cubed.measure_reserved_mem", "cubed.nanmean", "cubed.nansum", "cubed.pad", "cubed.random.random", "cubed.rechunk", "cubed.store", "cubed.to_zarr", "cubed.visualize", "Demo", "Getting Started", "Installation", "Cubed", "Operations", "Related Projects", "Diagnostics", "Executors", "User Guide", "Memory", "Optimization", "Reliability", "Scaling", "Storage", "Why Cubed?"], "titleterms": {"For": 31, "advanc": 38, "allow": 37, "an": 35, "api": [0, 1, 5], "apply_gufunc": 13, "arg_reduct": 32, "arrai": [0, 1, 5, 6, 7, 8, 9, 31, 38], "articl": 31, "bar": 34, "beam": 3, "between": 1, "block": 38, "blockwis": 32, "bound": 31, "calcul": 40, "callback": [10, 34], "chunk": [0, 37], "cloud": [35, 40, 41], "coil": 3, "comput": [2, 7, 14, 34], "conda": 30, "configur": 3, "consider": 40, "consist": 39, "contribut": 4, "core": 5, "cube": [1, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 31, 40, 42], "dask": [3, 33], "data": 41, "debug": 38, "delet": 41, "demo": 28, "depend": [30, 32], "design": 5, "develop": [4, 31], "diagnos": 40, "diagnost": 34, "differ": [1, 40], "dimension": 31, "distribut": 31, "document": 31, "elemwis": 32, "environ": 3, "exampl": 34, "execut": 2, "executor": [3, 35, 40], "file": 3, "from_arrai": 15, "from_zarr": 16, "function": 0, "fusion": 38, "gener": 0, "get": 29, "guid": 36, "histori": 34, "i": 35, "input": 38, "instal": 30, "intermedi": 41, "io": 0, "lithop": 3, "local": 35, "machin": 35, "map": 38, "map_block": [17, 32], "map_overlap": 18, "map_select": 32, "measure_reserved_mem": 19, "memori": [2, 31, 37], "memrai": 34, "modal": 3, "multi": 40, "multipl": 38, "n": 31, "nanmean": 20, "nansum": 21, "non": 0, "number": [0, 38], "object": 3, "oper": [5, 32], "optim": 38, "option": [3, 30], "other": 40, "pad": 22, "perform": 40, "pip": 30, "pipelin": 40, "plan": [2, 34], "practic": 40, "prefac": 40, "previou": 33, "primit": 5, "process": [3, 31], "progress": 34, "project": [33, 37], "provid": 40, "python": 1, "random": [0, 23], "rechunk": [8, 24, 32], "reduct": 32, "refer": [0, 3], "relat": 33, "reliabl": 39, "reserv": 37, "retri": 39, "runtim": [0, 5], "scale": 40, "serverless": 31, "servic": 35, "set": 38, "should": 35, "singl": [3, 35, 40], "size": 37, "sourc": 38, "spec": [3, 11], "specif": [0, 3], "specifi": 35, "standard": 1, "standardis": 0, "start": 29, "step": 40, "storag": [5, 41], "store": 25, "straggler": 39, "strong": 39, "taskendev": 12, "theoret": 40, "thread": 3, "timelin": 34, "timeout": 39, "tip": 40, "to_zarr": 26, "total": 38, "tree": 32, "type": 40, "us": [34, 35], "user": [31, 36], "v": 40, "variabl": 3, "visual": [9, 27, 34], "which": 35, "why": 42, "work": 33, "xarrai": 33, "yaml": 3}}) \ No newline at end of file diff --git a/user-guide/diagnostics.html b/user-guide/diagnostics.html index 6a3704b8..9008de97 100644 --- a/user-guide/diagnostics.html +++ b/user-guide/diagnostics.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/user-guide/executors.html b/user-guide/executors.html index 638d54e3..f1b3d3ba 100644 --- a/user-guide/executors.html +++ b/user-guide/executors.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/user-guide/index.html b/user-guide/index.html index 16aeae40..c1658d3b 100644 --- a/user-guide/index.html +++ b/user-guide/index.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/user-guide/memory.html b/user-guide/memory.html index ca433e07..06364867 100644 --- a/user-guide/memory.html +++ b/user-guide/memory.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - diff --git a/user-guide/optimization.html b/user-guide/optimization.html index 72855f2e..e22f7d64 100644 --- a/user-guide/optimization.html +++ b/user-guide/optimization.html @@ -39,7 +39,7 @@ - + @@ -50,7 +50,7 @@ - @@ -441,8 +441,8 @@
The max_total_num_input_blocks
argument to multiple_inputs_optimize_dag
specifies the maximum number of input blocks (chunks) that are allowed in the fused operation.
Again, this is to limit the number of reads that an individual task must perform. The default is None
, which means that operations are fused only if they have the same number of tasks. If set to an integer, then this limitation is removed, and tasks with a different number of tasks will be fused - as long as the total number of input blocks does not exceed the maximum. This setting is useful for reductions, and can be set using functools.partial
:
opt_fn = partial(multiple_inputs_optimize_dag, max_total_num_input_blocks=10)
+Again, this is to limit the number of reads that an individual task must perform. If set to None
, operations are fused only if they have the same number of tasks. If set to an integer (the default is 10), then tasks with a different number of tasks will be fused - as long as the total number of input blocks does not exceed the maximum. This setting is useful for reductions, and can be changed using functools.partial
:
+opt_fn = partial(multiple_inputs_optimize_dag, max_total_num_input_blocks=20)
e.visualize(optimize_function=opt_fn)
diff --git a/user-guide/reliability.html b/user-guide/reliability.html
index 228758fd..d73e5590 100644
--- a/user-guide/reliability.html
+++ b/user-guide/reliability.html
@@ -39,7 +39,7 @@
-
+
@@ -50,7 +50,7 @@
-
diff --git a/user-guide/scaling.html b/user-guide/scaling.html
index b5db85e1..702cca32 100644
--- a/user-guide/scaling.html
+++ b/user-guide/scaling.html
@@ -39,7 +39,7 @@
-
+
@@ -50,7 +50,7 @@
-
diff --git a/user-guide/storage.html b/user-guide/storage.html
index 8f9fb44d..76f80c19 100644
--- a/user-guide/storage.html
+++ b/user-guide/storage.html
@@ -39,7 +39,7 @@
-
+
@@ -50,7 +50,7 @@
-
diff --git a/why-cubed.html b/why-cubed.html
index e0798408..47f75573 100644
--- a/why-cubed.html
+++ b/why-cubed.html
@@ -39,7 +39,7 @@
-
+
@@ -50,7 +50,7 @@
-