From 59e3eca2a80641032cdb7eb76cdb6a07ffa2b7ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20H=C3=B6chenberger?= Date: Sat, 13 Apr 2024 13:10:50 +0200 Subject: [PATCH 01/10] Use Python logging system for emitting log messages when called from the Python API Fixes #141 --- src/openneuro/__init__.py | 5 +++++ src/openneuro/_cli.py | 3 +++ src/openneuro/_config.py | 5 +++-- src/openneuro/_download.py | 21 +++++++++++---------- src/openneuro/_logging.py | 14 ++++++++++++++ 5 files changed, 36 insertions(+), 12 deletions(-) create mode 100644 src/openneuro/_logging.py diff --git a/src/openneuro/__init__.py b/src/openneuro/__init__.py index b722c63..f68fa57 100644 --- a/src/openneuro/__init__.py +++ b/src/openneuro/__init__.py @@ -14,3 +14,8 @@ from openneuro._download import download as download from openneuro._download import login as login + +# Assume we're not running from the CLI by default. +# _cli.download()` or `_cli.login()` will change this. +# Only used for logging. +_RUNNING_FROM_CLI = False diff --git a/src/openneuro/_cli.py b/src/openneuro/_cli.py index 9160b9b..cc714db 100644 --- a/src/openneuro/_cli.py +++ b/src/openneuro/_cli.py @@ -61,6 +61,8 @@ def download_cli( ] = 5, ) -> None: """Download datasets from OpenNeuro.""" + openneuro._RUNNING_FROM_CLI = True + download( dataset=dataset, tag=tag, @@ -77,6 +79,7 @@ def download_cli( @app.command(name="login") def login_cli() -> None: """Login to OpenNeuro and store an access token.""" + openneuro._RUNNING_FROM_CLI = True login() diff --git a/src/openneuro/_config.py b/src/openneuro/_config.py index 7d6ef4b..c23ba19 100644 --- a/src/openneuro/_config.py +++ b/src/openneuro/_config.py @@ -6,7 +6,8 @@ from typing import TypedDict import platformdirs -from tqdm.auto import tqdm + +from openneuro._logging import log CONFIG_DIR = Path( platformdirs.user_config_dir(appname="openneuro-py", appauthor=False, roaming=True) @@ -25,7 +26,7 @@ class Config(TypedDict): def init_config() -> None: """Initialize a new OpenNeuro configuration file.""" - tqdm.write( + log( "🙏 Please login to your OpenNeuro account and go to: " "My Account → Obtain an API Key" ) diff --git a/src/openneuro/_download.py b/src/openneuro/_download.py index 3e421b2..68e8832 100644 --- a/src/openneuro/_download.py +++ b/src/openneuro/_download.py @@ -17,6 +17,7 @@ from openneuro import __version__ from openneuro._config import BASE_URL, get_token, init_config +from openneuro._logging import log if hasattr(sys.stdout, "encoding") and sys.stdout.encoding.lower() == "utf-8": stdout_unicode = True @@ -111,7 +112,7 @@ def _safe_query(query, *, timeout=None) -> tuple[dict[str, Any] | None, bool]: session.cookies.set_cookie( requests.cookies.create_cookie("accessToken", token) ) - tqdm.write("🍪 Using API token to log in") + log("🍪 Using API token to log in") except ValueError: pass # No login gql_endpoint = RequestsEndpoint(url=gql_url, session=session, timeout=timeout) @@ -131,7 +132,7 @@ def _check_snapshot_exists( response_json, request_timed_out = _safe_query(query) if request_timed_out and max_retries > 0: - tqdm.write("Request timed out while fetching list of snapshots, retrying …") + log("Request timed out while fetching list of snapshots, retrying …") asyncio.sleep(retry_backoff) # pyright: ignore[reportUnusedCoroutine] max_retries -= 1 retry_backoff *= 2 @@ -193,7 +194,7 @@ def _get_download_metadata( request_timed_out = True if request_timed_out and max_retries > 0: - tqdm.write(_unicode("Request timed out while fetching metadata, retrying")) + log(_unicode("Request timed out while fetching metadata, retrying")) asyncio.sleep(retry_backoff) # pyright: ignore[reportUnusedCoroutine] max_retries -= 1 retry_backoff *= 2 @@ -429,7 +430,7 @@ async def _retry_download( retry_backoff: float, semaphore: asyncio.Semaphore, ) -> None: - tqdm.write( + log( _unicode( f"Request timed out while downloading {outfile}, retrying in " f"{retry_backoff} sec", @@ -772,8 +773,8 @@ def download( f" {msg_please} report {msg_problems} and {msg_bugs} at\n" f" https://github.com/hoechenberger/openneuro-py/issues\n" ) - tqdm.write(msg) - tqdm.write(_unicode(f"Preparing to download {dataset}", emoji="🌍")) + log(msg) + log(_unicode(f"Preparing to download {dataset}", emoji="🌍")) if target_dir is None: target_dir = Path(dataset) @@ -803,7 +804,7 @@ def download( local_tag = _get_local_tag(dataset_id=dataset, dataset_dir=target_dir) if local_tag is None: - tqdm.write( + log( "Cannot determine local revision of the dataset, " "and the target directory is not empty. If the " "download fails, you may want to try again with a " @@ -882,7 +883,7 @@ def download( f"Retrieving up to {len(files)} files " f"({max_concurrent_downloads} concurrent downloads)." ) - tqdm.write(_unicode(msg, emoji="📥", end="")) + log(_unicode(msg, emoji="📥", end="")) coroutine = _download_files( target_dir=target_dir, @@ -902,5 +903,5 @@ def download( except RuntimeError: asyncio.run(coroutine) - tqdm.write(_unicode(f"Finished downloading {dataset}.\n", emoji="✅", end="")) - tqdm.write(_unicode("Please enjoy your brains.\n", emoji="🧠", end="")) + log(_unicode(f"Finished downloading {dataset}.\n", emoji="✅", end="")) + log(_unicode("Please enjoy your brains.\n", emoji="🧠", end="")) diff --git a/src/openneuro/_logging.py b/src/openneuro/_logging.py new file mode 100644 index 0000000..7d861b7 --- /dev/null +++ b/src/openneuro/_logging.py @@ -0,0 +1,14 @@ +import logging + +from tqdm.auto import tqdm + +from openneuro import _RUNNING_FROM_CLI + +logger = logging.getLogger("openneuro-py") + + +def log(message: str) -> None: + if _RUNNING_FROM_CLI: + logger.log(level=logging.INFO, msg=message) + else: + tqdm.write(message) From 3bfd5ee5723b3fed8fec4740d8f1e2c3ac5c4e1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20H=C3=B6chenberger?= Date: Sat, 13 Apr 2024 13:43:36 +0200 Subject: [PATCH 02/10] Refactor --- src/openneuro/_config.py | 10 ++++++--- src/openneuro/_download.py | 46 +++++++++++--------------------------- src/openneuro/_logging.py | 23 +++++++++++++++++-- 3 files changed, 41 insertions(+), 38 deletions(-) diff --git a/src/openneuro/_config.py b/src/openneuro/_config.py index c23ba19..4a7f1e7 100644 --- a/src/openneuro/_config.py +++ b/src/openneuro/_config.py @@ -7,7 +7,7 @@ import platformdirs -from openneuro._logging import log +from openneuro._logging import _unicode, log CONFIG_DIR = Path( platformdirs.user_config_dir(appname="openneuro-py", appauthor=False, roaming=True) @@ -27,8 +27,12 @@ class Config(TypedDict): def init_config() -> None: """Initialize a new OpenNeuro configuration file.""" log( - "🙏 Please login to your OpenNeuro account and go to: " - "My Account → Obtain an API Key" + _unicode( + "Please login to your OpenNeuro account and go to: " + "My Account → Obtain an API Key", + emoji="🙏", + end="", + ) ) api_key = getpass.getpass("OpenNeuro API key (input hidden): ") diff --git a/src/openneuro/_download.py b/src/openneuro/_download.py index 68e8832..4213260 100644 --- a/src/openneuro/_download.py +++ b/src/openneuro/_download.py @@ -3,7 +3,6 @@ import hashlib import json import string -import sys from collections.abc import Generator, Iterable from difflib import get_close_matches from pathlib import Path, PurePosixPath @@ -17,15 +16,7 @@ from openneuro import __version__ from openneuro._config import BASE_URL, get_token, init_config -from openneuro._logging import log - -if hasattr(sys.stdout, "encoding") and sys.stdout.encoding.lower() == "utf-8": - stdout_unicode = True -elif hasattr(sys.stdout, "reconfigure"): - sys.stdout.reconfigure(encoding="utf-8") - stdout_unicode = True -else: - stdout_unicode = False +from openneuro._logging import _unicode, log def login() -> None: @@ -112,7 +103,7 @@ def _safe_query(query, *, timeout=None) -> tuple[dict[str, Any] | None, bool]: session.cookies.set_cookie( requests.cookies.create_cookie("accessToken", token) ) - log("🍪 Using API token to log in") + log(_unicode("Using API token to log in", emoji="🍪")) except ValueError: pass # No login gql_endpoint = RequestsEndpoint(url=gql_url, session=session, timeout=timeout) @@ -612,14 +603,6 @@ def _get_local_tag(*, dataset_id: str, dataset_dir: Path) -> str | None: return local_version -def _unicode(msg: str, *, emoji: str = " ", end: str = "…") -> str: - if stdout_unicode: - msg = f"{emoji} {msg} {end}" - elif end == "…": - msg = f"{msg} ..." - return msg - - def _iterate_filenames( files: Iterable[dict], *, @@ -759,21 +742,18 @@ def download( The maximum number of downloads to run in parallel. """ - msg_problems = "problems 🤯" if stdout_unicode else "problems" - msg_bugs = "bugs 🪲" if stdout_unicode else "bugs" - msg_hello = "👋 Hello!" if stdout_unicode else "Hello!" - msg_great_to_see_you = "Great to see you!" - if stdout_unicode: - msg_great_to_see_you += " 🤗" - msg_please = "👉 Please" if stdout_unicode else " Please" - - msg = ( - f"\n{msg_hello} This is openneuro-py {__version__}. " - f"{msg_great_to_see_you}\n\n" - f" {msg_please} report {msg_problems} and {msg_bugs} at\n" - f" https://github.com/hoechenberger/openneuro-py/issues\n" + print("yo!!") + log("hi!!!") + log(_unicode("Hello! This is openneuro-py {__version__}. ", emoji="👋", end="")) + log(_unicode("Great to see you!", emoji="🤗", end="\n")) + log( + _unicode( + "Please report problems and bugs at\n" + " https://github.com/hoechenberger/openneuro-py/issues\n", + emoji="👉", + end="\n", + ) ) - log(msg) log(_unicode(f"Preparing to download {dataset}", emoji="🌍")) if target_dir is None: diff --git a/src/openneuro/_logging.py b/src/openneuro/_logging.py index 7d861b7..604cfe6 100644 --- a/src/openneuro/_logging.py +++ b/src/openneuro/_logging.py @@ -1,14 +1,33 @@ import logging +import sys from tqdm.auto import tqdm -from openneuro import _RUNNING_FROM_CLI +# logger = logging.getLogger("openneuro-py") +logger = logging.getLogger() -logger = logging.getLogger("openneuro-py") + +if hasattr(sys.stdout, "encoding") and sys.stdout.encoding.lower() == "utf-8": + stdout_unicode = True +elif hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8") + stdout_unicode = True +else: + stdout_unicode = False def log(message: str) -> None: + from openneuro import _RUNNING_FROM_CLI # avoid circular import + if _RUNNING_FROM_CLI: logger.log(level=logging.INFO, msg=message) else: tqdm.write(message) + + +def _unicode(msg: str, *, emoji: str = " ", end: str = "…") -> str: + if stdout_unicode: + msg = f"{emoji} {msg} {end}" + elif end == "…": + msg = f"{msg} ..." + return msg From e75849850738286c784c4cfd0489d42badb8ac33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20H=C3=B6chenberger?= Date: Sat, 13 Apr 2024 20:11:55 +0200 Subject: [PATCH 03/10] Fix --- src/openneuro/_download.py | 10 ++++------ src/openneuro/_logging.py | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/openneuro/_download.py b/src/openneuro/_download.py index 4213260..ec845cc 100644 --- a/src/openneuro/_download.py +++ b/src/openneuro/_download.py @@ -742,14 +742,12 @@ def download( The maximum number of downloads to run in parallel. """ - print("yo!!") - log("hi!!!") - log(_unicode("Hello! This is openneuro-py {__version__}. ", emoji="👋", end="")) - log(_unicode("Great to see you!", emoji="🤗", end="\n")) + log(_unicode(f"Hello! This is openneuro-py {__version__}. ", emoji="👋", end="")) + log(_unicode("Great to see you!", emoji="🤗", end="")) log( _unicode( - "Please report problems and bugs at\n" - " https://github.com/hoechenberger/openneuro-py/issues\n", + "Please report problems and bugs at " + "https://github.com/hoechenberger/openneuro-py/issues", emoji="👉", end="\n", ) diff --git a/src/openneuro/_logging.py b/src/openneuro/_logging.py index 604cfe6..dad74d0 100644 --- a/src/openneuro/_logging.py +++ b/src/openneuro/_logging.py @@ -3,8 +3,8 @@ from tqdm.auto import tqdm -# logger = logging.getLogger("openneuro-py") -logger = logging.getLogger() +logging.basicConfig(format="%(message)s", level=logging.INFO) +logger = logging.getLogger("openneuro-py") if hasattr(sys.stdout, "encoding") and sys.stdout.encoding.lower() == "utf-8": From 0a1f9d1cd44652845cd29546d7314bfa2544cdd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20H=C3=B6chenberger?= Date: Sat, 13 Apr 2024 20:13:35 +0200 Subject: [PATCH 04/10] Forgot one --- src/openneuro/_download.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/openneuro/_download.py b/src/openneuro/_download.py index ec845cc..7fa1156 100644 --- a/src/openneuro/_download.py +++ b/src/openneuro/_download.py @@ -123,7 +123,9 @@ def _check_snapshot_exists( response_json, request_timed_out = _safe_query(query) if request_timed_out and max_retries > 0: - log("Request timed out while fetching list of snapshots, retrying …") + log( + _unicode(msg="Request timed out while fetching list of snapshots, retrying") + ) asyncio.sleep(retry_backoff) # pyright: ignore[reportUnusedCoroutine] max_retries -= 1 retry_backoff *= 2 From 73bd58e532403deadba00c77d044887539bbf0c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20H=C3=B6chenberger?= Date: Sat, 13 Apr 2024 20:14:57 +0200 Subject: [PATCH 05/10] Better --- src/openneuro/_download.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/openneuro/_download.py b/src/openneuro/_download.py index 7fa1156..5890f11 100644 --- a/src/openneuro/_download.py +++ b/src/openneuro/_download.py @@ -883,5 +883,5 @@ def download( except RuntimeError: asyncio.run(coroutine) - log(_unicode(f"Finished downloading {dataset}.\n", emoji="✅", end="")) - log(_unicode("Please enjoy your brains.\n", emoji="🧠", end="")) + log(_unicode(f"Finished downloading {dataset}.", emoji="✅", end="\n")) + log(_unicode("Please enjoy your brains.", emoji="🧠", end="\n")) From c700a04881cb1340655a129224196c261fb3b36c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20H=C3=B6chenberger?= Date: Sun, 14 Apr 2024 08:34:44 +0200 Subject: [PATCH 06/10] Add `cli_only` kwarg --- src/openneuro/_download.py | 10 +++++++--- src/openneuro/_logging.py | 16 +++++++++++++++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/openneuro/_download.py b/src/openneuro/_download.py index 5890f11..ada6b74 100644 --- a/src/openneuro/_download.py +++ b/src/openneuro/_download.py @@ -744,7 +744,10 @@ def download( The maximum number of downloads to run in parallel. """ - log(_unicode(f"Hello! This is openneuro-py {__version__}. ", emoji="👋", end="")) + log( + _unicode(f"Hello! This is openneuro-py {__version__}. ", emoji="👋", end=""), + cli_only=True, + ) log(_unicode("Great to see you!", emoji="🤗", end="")) log( _unicode( @@ -752,7 +755,8 @@ def download( "https://github.com/hoechenberger/openneuro-py/issues", emoji="👉", end="\n", - ) + ), + cli_only=True, ) log(_unicode(f"Preparing to download {dataset}", emoji="🌍")) @@ -884,4 +888,4 @@ def download( asyncio.run(coroutine) log(_unicode(f"Finished downloading {dataset}.", emoji="✅", end="\n")) - log(_unicode("Please enjoy your brains.", emoji="🧠", end="\n")) + log(_unicode("Please enjoy your brains.", emoji="🧠", end="\n"), cli_only=True) diff --git a/src/openneuro/_logging.py b/src/openneuro/_logging.py index dad74d0..dce3f29 100644 --- a/src/openneuro/_logging.py +++ b/src/openneuro/_logging.py @@ -16,9 +16,23 @@ stdout_unicode = False -def log(message: str) -> None: +def log(message: str, cli_only: bool = False) -> None: + """Emit a log message. + + Parameters + ---------- + message + The message to emit. + cli_only + Whether to emit the message only when running from the CLI. If `False`, the + message will shop up when running from the CLI and the Python API. + + """ from openneuro import _RUNNING_FROM_CLI # avoid circular import + if cli_only and not _RUNNING_FROM_CLI: + return + if _RUNNING_FROM_CLI: logger.log(level=logging.INFO, msg=message) else: From 3b16bbc5c131692b9de4266fd80886444c293f0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20H=C3=B6chenberger?= Date: Sun, 14 Apr 2024 14:24:15 +0200 Subject: [PATCH 07/10] Make pytest print logging output --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 84f7d30..3a7143a 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -57,4 +57,4 @@ jobs: openneuro-py download --help openneuro-py login --help - name: Test with pytest - run: pytest + run: pytest -s From a7e94366d2ffea772f3278a06dbf4b48362b7bee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20H=C3=B6chenberger?= Date: Sun, 14 Apr 2024 14:35:35 +0200 Subject: [PATCH 08/10] Simplify logging --- src/openneuro/_config.py | 12 +++++------- src/openneuro/_download.py | 40 +++++++++++++++++--------------------- src/openneuro/_logging.py | 23 +++++++++++++++++++--- 3 files changed, 43 insertions(+), 32 deletions(-) diff --git a/src/openneuro/_config.py b/src/openneuro/_config.py index 4a7f1e7..ad3e636 100644 --- a/src/openneuro/_config.py +++ b/src/openneuro/_config.py @@ -7,7 +7,7 @@ import platformdirs -from openneuro._logging import _unicode, log +from openneuro._logging import log CONFIG_DIR = Path( platformdirs.user_config_dir(appname="openneuro-py", appauthor=False, roaming=True) @@ -27,12 +27,10 @@ class Config(TypedDict): def init_config() -> None: """Initialize a new OpenNeuro configuration file.""" log( - _unicode( - "Please login to your OpenNeuro account and go to: " - "My Account → Obtain an API Key", - emoji="🙏", - end="", - ) + "Please login to your OpenNeuro account and go to: " + "My Account → Obtain an API Key", + emoji="🙏", + end="", ) api_key = getpass.getpass("OpenNeuro API key (input hidden): ") diff --git a/src/openneuro/_download.py b/src/openneuro/_download.py index ada6b74..8758d2c 100644 --- a/src/openneuro/_download.py +++ b/src/openneuro/_download.py @@ -103,7 +103,7 @@ def _safe_query(query, *, timeout=None) -> tuple[dict[str, Any] | None, bool]: session.cookies.set_cookie( requests.cookies.create_cookie("accessToken", token) ) - log(_unicode("Using API token to log in", emoji="🍪")) + log("Using API token to log in", emoji="🍪") except ValueError: pass # No login gql_endpoint = RequestsEndpoint(url=gql_url, session=session, timeout=timeout) @@ -123,9 +123,7 @@ def _check_snapshot_exists( response_json, request_timed_out = _safe_query(query) if request_timed_out and max_retries > 0: - log( - _unicode(msg="Request timed out while fetching list of snapshots, retrying") - ) + log("Request timed out while fetching list of snapshots, retrying") asyncio.sleep(retry_backoff) # pyright: ignore[reportUnusedCoroutine] max_retries -= 1 retry_backoff *= 2 @@ -187,7 +185,7 @@ def _get_download_metadata( request_timed_out = True if request_timed_out and max_retries > 0: - log(_unicode("Request timed out while fetching metadata, retrying")) + log("Request timed out while fetching metadata, retrying") asyncio.sleep(retry_backoff) # pyright: ignore[reportUnusedCoroutine] max_retries -= 1 retry_backoff *= 2 @@ -424,11 +422,9 @@ async def _retry_download( semaphore: asyncio.Semaphore, ) -> None: log( - _unicode( - f"Request timed out while downloading {outfile}, retrying in " - f"{retry_backoff} sec", - emoji="🔄", - ) + f"Request timed out while downloading {outfile}, retrying in " + f"{retry_backoff} sec", + emoji="🔄", ) await asyncio.sleep(retry_backoff) max_retries -= 1 @@ -745,20 +741,20 @@ def download( """ log( - _unicode(f"Hello! This is openneuro-py {__version__}. ", emoji="👋", end=""), + f"Hello! This is openneuro-py {__version__}. ", + emoji="👋", + end="", cli_only=True, ) - log(_unicode("Great to see you!", emoji="🤗", end="")) + log("Great to see you!", emoji="🤗", end="") log( - _unicode( - "Please report problems and bugs at " - "https://github.com/hoechenberger/openneuro-py/issues", - emoji="👉", - end="\n", - ), + "Please report problems and bugs at " + "https://github.com/hoechenberger/openneuro-py/issues", + emoji="👉", + end="\n", cli_only=True, ) - log(_unicode(f"Preparing to download {dataset}", emoji="🌍")) + log(f"Preparing to download {dataset}", emoji="🌍") if target_dir is None: target_dir = Path(dataset) @@ -867,7 +863,7 @@ def download( f"Retrieving up to {len(files)} files " f"({max_concurrent_downloads} concurrent downloads)." ) - log(_unicode(msg, emoji="📥", end="")) + log(msg, emoji="📥", end="") coroutine = _download_files( target_dir=target_dir, @@ -887,5 +883,5 @@ def download( except RuntimeError: asyncio.run(coroutine) - log(_unicode(f"Finished downloading {dataset}.", emoji="✅", end="\n")) - log(_unicode("Please enjoy your brains.", emoji="🧠", end="\n"), cli_only=True) + log(f"Finished downloading {dataset}.", emoji="✅", end="\n") + log("Please enjoy your brains.", emoji="🧠", end="\n", cli_only=True) diff --git a/src/openneuro/_logging.py b/src/openneuro/_logging.py index dce3f29..dc752f6 100644 --- a/src/openneuro/_logging.py +++ b/src/openneuro/_logging.py @@ -16,13 +16,22 @@ stdout_unicode = False -def log(message: str, cli_only: bool = False) -> None: +def log( + message: str, + emoji: str | None = None, + end: str | None = None, + cli_only: bool = False, +) -> None: """Emit a log message. Parameters ---------- message The message to emit. + emoji + Unicode eomji to prepend. + end + String to append. By default, `"…"`. cli_only Whether to emit the message only when running from the CLI. If `False`, the message will shop up when running from the CLI and the Python API. @@ -33,10 +42,18 @@ def log(message: str, cli_only: bool = False) -> None: if cli_only and not _RUNNING_FROM_CLI: return + if emoji is None: + emoji = " " + if end is None: + end = "…" + + message_unicode = _unicode(message, emoji=emoji, end=end) + del message + if _RUNNING_FROM_CLI: - logger.log(level=logging.INFO, msg=message) + logger.log(level=logging.INFO, msg=message_unicode) else: - tqdm.write(message) + tqdm.write(message_unicode) def _unicode(msg: str, *, emoji: str = " ", end: str = "…") -> str: From d9b9e57217a0b686cede10e5f37b4726f4825f42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20H=C3=B6chenberger?= Date: Sun, 14 Apr 2024 14:36:52 +0200 Subject: [PATCH 09/10] Add retry emoji --- src/openneuro/_download.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/openneuro/_download.py b/src/openneuro/_download.py index 8758d2c..f4368d5 100644 --- a/src/openneuro/_download.py +++ b/src/openneuro/_download.py @@ -123,7 +123,7 @@ def _check_snapshot_exists( response_json, request_timed_out = _safe_query(query) if request_timed_out and max_retries > 0: - log("Request timed out while fetching list of snapshots, retrying") + log("Request timed out while fetching list of snapshots, retrying", emoji="🔄") asyncio.sleep(retry_backoff) # pyright: ignore[reportUnusedCoroutine] max_retries -= 1 retry_backoff *= 2 @@ -185,7 +185,7 @@ def _get_download_metadata( request_timed_out = True if request_timed_out and max_retries > 0: - log("Request timed out while fetching metadata, retrying") + log("Request timed out while fetching metadata, retrying", emoji="🔄") asyncio.sleep(retry_backoff) # pyright: ignore[reportUnusedCoroutine] max_retries -= 1 retry_backoff *= 2 From ae9d108c80a7c91386500f4d92e2aff7e82d2462 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20H=C3=B6chenberger?= Date: Sun, 14 Apr 2024 14:48:07 +0200 Subject: [PATCH 10/10] Add CLI download test --- .github/workflows/run-tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 3a7143a..16b9b6d 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -58,3 +58,8 @@ jobs: openneuro-py login --help - name: Test with pytest run: pytest -s + - name: Test download from CLI + run: | + openneuro-py download --dataset=ds000248 --include=participants.tsv --target-dir=/tmp/ds000248 + ls -lah /tmp/ds000248 +