Changed default download chunk size to 10MB (#528)

CHANGELOG.md

@@ -10,8 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 ### Changed
+- Changed default `chunk_size` of various `download` functions from None to 10MB. This improves the handling of large downloads and reduces memory usage. ([#528](https://github.com/Open-EO/openeo-python-client/issues/528))
 - `Connection.execute()` and `DataCube.execute()` now have a `auto_decode` argument. If set to True (default) the response will be decoded as a JSON and throw an exception if this fails, if set to False the raw `requests.Response` object will be returned. ([#499](https://github.com/Open-EO/openeo-python-client/issues/499))
-
 ### Removed
 
 ### Fixed

openeo/rest/__init__.py

@@ -2,6 +2,9 @@
 
 from openeo import BaseOpenEoException
 
+# TODO: get from config file
+DEFAULT_DOWNLOAD_CHUNK_SIZE = 10_000_000  # 10MB
+
 
 class OpenEoClientException(BaseOpenEoException):
     """Base class for OpenEO client exceptions"""

openeo/rest/connection.py

@@ -51,6 +51,7 @@
     OidcRefreshTokenAuthenticator,
     OidcResourceOwnerPasswordAuthenticator,
 )
+from openeo.rest import DEFAULT_DOWNLOAD_CHUNK_SIZE
 from openeo.rest.datacube import DataCube, InputDate
 from openeo.rest.graph_building import CollectionProperty
 from openeo.rest.job import BatchJob, RESTJob
@@ -1539,8 +1540,10 @@ def download(
         self,
         graph: Union[dict, FlatGraphableMixin, str, Path],
         outputfile: Union[Path, str, None] = None,
+        *,
         timeout: Optional[int] = None,
         validate: Optional[bool] = None,
+        chunk_size: int = DEFAULT_DOWNLOAD_CHUNK_SIZE,
     ) -> Union[None, bytes]:
         """
         Downloads the result of a process graph synchronously,
@@ -1553,6 +1556,7 @@ def download(
         :param timeout: timeout to wait for response
         :param validate: Optional toggle to enable/prevent validation of the process graphs before execution
             (overruling the connection's ``auto_validate`` setting).
+        :param chunk_size: chunk size for streaming response.
         """
         pg_with_metadata = self._build_request_with_process_graph(process_graph=graph)
         self._preflight_validation(pg_with_metadata=pg_with_metadata, validate=validate)
@@ -1566,7 +1570,7 @@ def download(
 
         if outputfile is not None:
             with Path(outputfile).open(mode="wb") as f:
-                for chunk in response.iter_content(chunk_size=None):
+                for chunk in response.iter_content(chunk_size=chunk_size):
                     f.write(chunk)
         else:
             return response.content

openeo/rest/job.py

@@ -19,7 +19,13 @@
     render_error,
 )
 from openeo.internal.warnings import deprecated, legacy_alias
-from openeo.rest import JobFailedException, OpenEoApiError, OpenEoClientException, OpenEoApiPlainError
+from openeo.rest import (
+    JobFailedException,
+    OpenEoApiError,
+    OpenEoClientException,
+    OpenEoApiPlainError,
+    DEFAULT_DOWNLOAD_CHUNK_SIZE,
+)
 from openeo.util import ensure_dir
 
 if typing.TYPE_CHECKING:
@@ -351,13 +357,16 @@ def __repr__(self):
             n=self.name, t=self.metadata.get("type", "unknown"), h=self.href
         )
 
-    def download(self, target: Optional[Union[Path, str]] = None, chunk_size=None) -> Path:
+    def download(
+        self, target: Optional[Union[Path, str]] = None, *, chunk_size: int = DEFAULT_DOWNLOAD_CHUNK_SIZE
+    ) -> Path:
         """
         Download asset to given location
 
         :param target: download target path. Can be an existing folder
             (in which case the filename advertised by backend will be used)
             or full file name. By default, the working directory will be used.
+        :param chunk_size: chunk size for streaming response.
         """
         target = Path(target or Path.cwd())
         if target.is_dir():

openeo/rest/userfile.py

@@ -5,6 +5,7 @@
 from typing import Any, Dict, Optional, Union
 
 from openeo.util import ensure_dir
+from openeo.rest import DEFAULT_DOWNLOAD_CHUNK_SIZE
 
 if typing.TYPE_CHECKING:
     # Imports for type checking only (circular import issue at runtime).
@@ -66,7 +67,7 @@ def download(self, target: Union[Path, str] = None) -> Path:
         ensure_dir(target.parent)
 
         with target.open(mode="wb") as f:
-            for chunk in response.iter_content(chunk_size=None):
+            for chunk in response.iter_content(chunk_size=DEFAULT_DOWNLOAD_CHUNK_SIZE):
                 f.write(chunk)
 
         return target