From 663f5bf3aef4d0ff1285e28e9406934113017b4a Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Thu, 25 Jul 2024 17:52:16 +0000 Subject: [PATCH 01/33] feat: add separate cli app for database admin --- .gitignore | 2 -- .vscode/settings.json | 3 +++ build-linux.sh | 2 +- build-windows.sh | 2 +- config.yml | 2 +- pyproject.toml | 8 +++++--- src/imap_db/main.py | 24 ++++++++++++++++++++++++ src/{ => imap_mag}/__init__.py | 0 src/{ => imap_mag}/appConfig.py | 18 ++++++++++-------- src/{ => imap_mag}/appLogging.py | 0 src/{ => imap_mag}/appUtils.py | 0 src/{ => imap_mag}/imapProcessing.py | 6 +++++- src/{ => imap_mag}/main.py | 2 +- src/{ => imap_mag}/xtce/tlm_20240724.xml | 0 tests/config/hk.yaml | 2 +- tests/test_main.py | 3 +-- 16 files changed, 53 insertions(+), 21 deletions(-) create mode 100644 src/imap_db/main.py rename src/{ => imap_mag}/__init__.py (100%) rename src/{ => imap_mag}/appConfig.py (57%) rename src/{ => imap_mag}/appLogging.py (100%) rename src/{ => imap_mag}/appUtils.py (100%) rename src/{ => imap_mag}/imapProcessing.py (93%) rename src/{ => imap_mag}/main.py (98%) rename src/{ => imap_mag}/xtce/tlm_20240724.xml (100%) diff --git a/.gitignore b/.gitignore index 6fbe803..00081c1 100644 --- a/.gitignore +++ b/.gitignore @@ -131,8 +131,6 @@ dmypy.json # mkdocs build dir site/ -src/main.py .work /output -deploy/test-env/ssh_test_keys/ diff --git a/.vscode/settings.json b/.vscode/settings.json index 8f7355b..86da224 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -14,5 +14,8 @@ "cSpell.words": [ "xtce", "xtcedef" + ], + "python.analysis.extraPaths": [ + "./src" ] } diff --git a/build-linux.sh b/build-linux.sh index eedf31b..9a1d817 100755 --- a/build-linux.sh +++ b/build-linux.sh @@ -7,7 +7,7 @@ mkdir -p dist docker run \ - --volume "$(pwd):/src/" \ + --volume "$(pwd):/src/imap_mag" \ batonogov/pyinstaller-linux:latest \ "rm -rf dist/pyinstaller && \ python3 -m pip install poetry && \ diff --git a/build-windows.sh b/build-windows.sh index 1cc42a2..982470d 100755 --- a/build-windows.sh +++ b/build-windows.sh @@ -7,7 +7,7 @@ mkdir -p dist docker run \ - --volume "$(pwd):/src/" \ + --volume "$(pwd):/src/imap_mag" \ batonogov/pyinstaller-windows:latest \ "rm -rf dist/pyinstaller && \ python -m pip install poetry && \ diff --git a/config.yml b/config.yml index 8d9f1a8..604dbad 100644 --- a/config.yml +++ b/config.yml @@ -8,4 +8,4 @@ destination: filename: result.cdf packet-definition: - hk: src/xtce/tlm_20240724.xml + hk: src/imap_mag/xtce/tlm_20240724.xml diff --git a/pyproject.toml b/pyproject.toml index fd7120b..6c2cf10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,8 @@ description = "Process IMAP data" authors = ["alastairtree"] readme = "README.md" packages = [ - { include = "src" }, + { include = "imap_mag", from = "src", to="imap_mag" }, + { include = "imap_db", from = "src", to="imap_db" }, ] [tool.poetry.dependencies] @@ -27,14 +28,15 @@ ruff = "^0.5.4" [tool.poetry.scripts] # can execute via poetry, e.g. `poetry run imap-mag hello world` -imap-mag = 'src.main:app' +imap-mag = 'imap_mag.main:app' +imap-db = 'imap_db.main:app' [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.poetry-pyinstaller-plugin.scripts] -imap-mag = { source = "src/main.py", type = "onefile", bundle = false } +imap-mag = { source = "src/imap_mag/main.py", type = "onefile", bundle = false } [tool.ruff] target-version = "py312" diff --git a/src/imap_db/main.py b/src/imap_db/main.py new file mode 100644 index 0000000..5bd3792 --- /dev/null +++ b/src/imap_db/main.py @@ -0,0 +1,24 @@ +"""Main module.""" + +from typing import Annotated + +# cli +import typer + +app = typer.Typer() +globalState = {"verbose": False} + + +@app.command() +def hello(name: str): + print(f"Hello {name}") + + +@app.callback() +def main(verbose: Annotated[bool, typer.Option("--verbose", "-v")] = False): + if verbose: + globalState["verbose"] = True + + +if __name__ == "__main__": + app() # pragma: no cover diff --git a/src/__init__.py b/src/imap_mag/__init__.py similarity index 100% rename from src/__init__.py rename to src/imap_mag/__init__.py diff --git a/src/appConfig.py b/src/imap_mag/appConfig.py similarity index 57% rename from src/appConfig.py rename to src/imap_mag/appConfig.py index 0555d89..10b4e1a 100644 --- a/src/appConfig.py +++ b/src/imap_mag/appConfig.py @@ -1,18 +1,17 @@ """App configuration module.""" -from enum import Enum from pathlib import Path from pydantic import BaseModel +from pydantic.config import ConfigDict -class Source(BaseModel): - folder: Path +def hyphenize(field: str): + return field.replace("_", "-") -class DestinationType(Enum): - LOCAL = "local" - SFTP = "sftp" +class Source(BaseModel): + folder: Path class Destination(BaseModel): @@ -21,11 +20,14 @@ class Destination(BaseModel): class PacketDefinition(BaseModel): - hk: Path = Path("src/xtce/tlm_20240724.xml") + hk: Path class AppConfig(BaseModel): source: Source work_folder: Path = Path(".work") destination: Destination - packet_definition: PacketDefinition = PacketDefinition() + packet_definition: PacketDefinition + + # pydantic configuration to allow hyphenated fields + model_config = ConfigDict(alias_generator=hyphenize) diff --git a/src/appLogging.py b/src/imap_mag/appLogging.py similarity index 100% rename from src/appLogging.py rename to src/imap_mag/appLogging.py diff --git a/src/appUtils.py b/src/imap_mag/appUtils.py similarity index 100% rename from src/appUtils.py rename to src/imap_mag/appUtils.py diff --git a/src/imapProcessing.py b/src/imap_mag/imapProcessing.py similarity index 93% rename from src/imapProcessing.py rename to src/imap_mag/imapProcessing.py index e430633..9a93e50 100644 --- a/src/imapProcessing.py +++ b/src/imap_mag/imapProcessing.py @@ -7,7 +7,7 @@ import xarray as xr from space_packet_parser import parser, xtcedef -from src import appConfig, appUtils +from . import appConfig, appUtils class FileProcessor(abc.ABC): @@ -35,6 +35,10 @@ class HKProcessor(FileProcessor): def initialize(self, config: appConfig.AppConfig) -> None: self.xtcePacketDefinition = config.packet_definition.hk + if not self.xtcePacketDefinition.exists(): + raise FileNotFoundError( + f"XTCE packet definition file not found: {self.xtcePacketDefinition}" + ) def process(self, file: Path) -> Path: """Process HK with XTCE tools and create CSV file.""" diff --git a/src/main.py b/src/imap_mag/main.py similarity index 98% rename from src/main.py rename to src/imap_mag/main.py index 4024089..0181c69 100644 --- a/src/main.py +++ b/src/imap_mag/main.py @@ -14,7 +14,7 @@ import yaml # app code -from src import appConfig, appLogging, imapProcessing +from . import appConfig, appLogging, imapProcessing app = typer.Typer() globalState = {"verbose": False} diff --git a/src/xtce/tlm_20240724.xml b/src/imap_mag/xtce/tlm_20240724.xml similarity index 100% rename from src/xtce/tlm_20240724.xml rename to src/imap_mag/xtce/tlm_20240724.xml diff --git a/tests/config/hk.yaml b/tests/config/hk.yaml index c9caaf2..f3d4af1 100644 --- a/tests/config/hk.yaml +++ b/tests/config/hk.yaml @@ -8,4 +8,4 @@ destination: filename: result.csv packet-definition: - hk: src/xtce/tlm_20240724.xml + hk: src/imap_mag/xtce/tlm_20240724.xml diff --git a/tests/test_main.py b/tests/test_main.py index d1ac94e..f4b639b 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -6,10 +6,9 @@ from pathlib import Path import pytest +from imap_mag.main import app from typer.testing import CliRunner -from src.main import app - runner = CliRunner() From ed41d3b0000706189c90d96329b79d2d3b882a95 Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Thu, 25 Jul 2024 19:33:56 +0000 Subject: [PATCH 02/33] feat: Add database admin CLI app and migrations --- .devcontainer/Dockerfile | 3 + .vscode/settings.json | 12 + alembic.ini | 115 +++++++ deploy/entrypoint.sh | 3 + poetry.lock | 317 +++++++++++++++++- pyproject.toml | 4 + src/imap_db/README.md | 72 ++++ src/imap_db/__init__.py | 4 + src/imap_db/main.py | 77 ++++- src/imap_db/migrations/env.py | 71 ++++ src/imap_db/migrations/script.py.mako | 24 ++ ...4_07_25-d0457f3e98c8_create_files_table.py | 34 ++ src/imap_db/model.py | 16 + src/imap_mag/__init__.py | 1 - 14 files changed, 742 insertions(+), 11 deletions(-) create mode 100644 alembic.ini create mode 100644 src/imap_db/README.md create mode 100644 src/imap_db/__init__.py create mode 100644 src/imap_db/migrations/env.py create mode 100644 src/imap_db/migrations/script.py.mako create mode 100644 src/imap_db/migrations/versions/2024_07_25-d0457f3e98c8_create_files_table.py create mode 100644 src/imap_db/model.py diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index dc66208..1f0c685 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -13,3 +13,6 @@ ENV PATH="$PYENV_ROOT/bin:$PATH" RUN curl https://pyenv.run | bash RUN $PYENV_ROOT/bin/pyenv install -v ${PYTHON_VERSIONS} +USER root +RUN apt-get update && apt-get install -y --no-install-recommends \ + postgresql-client diff --git a/.vscode/settings.json b/.vscode/settings.json index 86da224..bbadb39 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -17,5 +17,17 @@ ], "python.analysis.extraPaths": [ "./src" + ], + "sqltools.connections": [ + { + "previewLimit": 50, + "server": "host.docker.internal", + "port": 5432, + "driver": "PostgreSQL", + "name": "postgres", + "password": "", + "database": "imap", + "username": "postgres" + } ] } diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..e98fde2 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,115 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = src/imap_db/migrations + +#sqlalchemy.url = postgresql://postgres:postgres@db/imap +sqlalchemy.url = postgresql://postgres:postgres@host.docker.internal:5432/imap + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +file_template = %%(year)d_%%(month).2d_%%(day).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python-dateutil library that can be +# installed by adding `alembic[tz]` to the pip requirements +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to migrations/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:migrations/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +; hooks = black +; black.type = console_scripts +; black.entrypoint = black +; black.options = -l 79 REVISION_SCRIPT_FILENAME + +# format using "ruff" - use the exec runner, execute a binary +ruff_format.type = exec +ruff_format.executable = %(here)s/.venv/bin/ruff +ruff_format.options = format REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/deploy/entrypoint.sh b/deploy/entrypoint.sh index 77f6096..fff3668 100644 --- a/deploy/entrypoint.sh +++ b/deploy/entrypoint.sh @@ -2,6 +2,9 @@ set -e +imap-db create-db +imap-db upgrade-db + while : do imap-mag hello world diff --git a/poetry.lock b/poetry.lock index e0898ca..7deb489 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,24 @@ # This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +[[package]] +name = "alembic" +version = "1.13.2" +description = "A database migration tool for SQLAlchemy." +optional = false +python-versions = ">=3.8" +files = [ + {file = "alembic-1.13.2-py3-none-any.whl", hash = "sha256:6b8733129a6224a9a711e17c99b08462dbf7cc9670ba8f2e2ae9af860ceb1953"}, + {file = "alembic-1.13.2.tar.gz", hash = "sha256:1ff0ae32975f4fd96028c39ed9bb3c867fe3af956bd7bb37343b54c9fe7445ef"}, +] + +[package.dependencies] +Mako = "*" +SQLAlchemy = ">=1.3.0" +typing-extensions = ">=4" + +[package.extras] +tz = ["backports.zoneinfo"] + [[package]] name = "altgraph" version = "0.17.4" @@ -311,6 +330,77 @@ docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1 testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8.0.1)", "pytest (>=7.4.3)", "pytest-asyncio (>=0.21)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)", "virtualenv (>=20.26.2)"] typing = ["typing-extensions (>=4.8)"] +[[package]] +name = "greenlet" +version = "3.0.3" +description = "Lightweight in-process concurrent programming" +optional = false +python-versions = ">=3.7" +files = [ + {file = "greenlet-3.0.3-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:9da2bd29ed9e4f15955dd1595ad7bc9320308a3b766ef7f837e23ad4b4aac31a"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d353cadd6083fdb056bb46ed07e4340b0869c305c8ca54ef9da3421acbdf6881"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dca1e2f3ca00b84a396bc1bce13dd21f680f035314d2379c4160c98153b2059b"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3ed7fb269f15dc662787f4119ec300ad0702fa1b19d2135a37c2c4de6fadfd4a"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd4f49ae60e10adbc94b45c0b5e6a179acc1736cf7a90160b404076ee283cf83"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:73a411ef564e0e097dbe7e866bb2dda0f027e072b04da387282b02c308807405"}, + {file = "greenlet-3.0.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7f362975f2d179f9e26928c5b517524e89dd48530a0202570d55ad6ca5d8a56f"}, + {file = "greenlet-3.0.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:649dde7de1a5eceb258f9cb00bdf50e978c9db1b996964cd80703614c86495eb"}, + {file = "greenlet-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:68834da854554926fbedd38c76e60c4a2e3198c6fbed520b106a8986445caaf9"}, + {file = "greenlet-3.0.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:b1b5667cced97081bf57b8fa1d6bfca67814b0afd38208d52538316e9422fc61"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52f59dd9c96ad2fc0d5724107444f76eb20aaccb675bf825df6435acb7703559"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:afaff6cf5200befd5cec055b07d1c0a5a06c040fe5ad148abcd11ba6ab9b114e"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe754d231288e1e64323cfad462fcee8f0288654c10bdf4f603a39ed923bef33"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2797aa5aedac23af156bbb5a6aa2cd3427ada2972c828244eb7d1b9255846379"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7f009caad047246ed379e1c4dbcb8b020f0a390667ea74d2387be2998f58a22"}, + {file = "greenlet-3.0.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c5e1536de2aad7bf62e27baf79225d0d64360d4168cf2e6becb91baf1ed074f3"}, + {file = "greenlet-3.0.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:894393ce10ceac937e56ec00bb71c4c2f8209ad516e96033e4b3b1de270e200d"}, + {file = "greenlet-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:1ea188d4f49089fc6fb283845ab18a2518d279c7cd9da1065d7a84e991748728"}, + {file = "greenlet-3.0.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:70fb482fdf2c707765ab5f0b6655e9cfcf3780d8d87355a063547b41177599be"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4d1ac74f5c0c0524e4a24335350edad7e5f03b9532da7ea4d3c54d527784f2e"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:149e94a2dd82d19838fe4b2259f1b6b9957d5ba1b25640d2380bea9c5df37676"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:15d79dd26056573940fcb8c7413d84118086f2ec1a8acdfa854631084393efcc"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881b7db1ebff4ba09aaaeae6aa491daeb226c8150fc20e836ad00041bcb11230"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fcd2469d6a2cf298f198f0487e0a5b1a47a42ca0fa4dfd1b6862c999f018ebbf"}, + {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1f672519db1796ca0d8753f9e78ec02355e862d0998193038c7073045899f305"}, + {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2516a9957eed41dd8f1ec0c604f1cdc86758b587d964668b5b196a9db5bfcde6"}, + {file = "greenlet-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:bba5387a6975598857d86de9eac14210a49d554a77eb8261cc68b7d082f78ce2"}, + {file = "greenlet-3.0.3-cp37-cp37m-macosx_11_0_universal2.whl", hash = "sha256:5b51e85cb5ceda94e79d019ed36b35386e8c37d22f07d6a751cb659b180d5274"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:daf3cb43b7cf2ba96d614252ce1684c1bccee6b2183a01328c98d36fcd7d5cb0"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99bf650dc5d69546e076f413a87481ee1d2d09aaaaaca058c9251b6d8c14783f"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dd6e660effd852586b6a8478a1d244b8dc90ab5b1321751d2ea15deb49ed414"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3391d1e16e2a5a1507d83e4a8b100f4ee626e8eca43cf2cadb543de69827c4c"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1f145462f1fa6e4a4ae3c0f782e580ce44d57c8f2c7aae1b6fa88c0b2efdb41"}, + {file = "greenlet-3.0.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1a7191e42732df52cb5f39d3527217e7ab73cae2cb3694d241e18f53d84ea9a7"}, + {file = "greenlet-3.0.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0448abc479fab28b00cb472d278828b3ccca164531daab4e970a0458786055d6"}, + {file = "greenlet-3.0.3-cp37-cp37m-win32.whl", hash = "sha256:b542be2440edc2d48547b5923c408cbe0fc94afb9f18741faa6ae970dbcb9b6d"}, + {file = "greenlet-3.0.3-cp37-cp37m-win_amd64.whl", hash = "sha256:01bc7ea167cf943b4c802068e178bbf70ae2e8c080467070d01bfa02f337ee67"}, + {file = "greenlet-3.0.3-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:1996cb9306c8595335bb157d133daf5cf9f693ef413e7673cb07e3e5871379ca"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ddc0f794e6ad661e321caa8d2f0a55ce01213c74722587256fb6566049a8b04"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c9db1c18f0eaad2f804728c67d6c610778456e3e1cc4ab4bbd5eeb8e6053c6fc"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7170375bcc99f1a2fbd9c306f5be8764eaf3ac6b5cb968862cad4c7057756506"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b66c9c1e7ccabad3a7d037b2bcb740122a7b17a53734b7d72a344ce39882a1b"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:098d86f528c855ead3479afe84b49242e174ed262456c342d70fc7f972bc13c4"}, + {file = "greenlet-3.0.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:81bb9c6d52e8321f09c3d165b2a78c680506d9af285bfccbad9fb7ad5a5da3e5"}, + {file = "greenlet-3.0.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fd096eb7ffef17c456cfa587523c5f92321ae02427ff955bebe9e3c63bc9f0da"}, + {file = "greenlet-3.0.3-cp38-cp38-win32.whl", hash = "sha256:d46677c85c5ba00a9cb6f7a00b2bfa6f812192d2c9f7d9c4f6a55b60216712f3"}, + {file = "greenlet-3.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:419b386f84949bf0e7c73e6032e3457b82a787c1ab4a0e43732898a761cc9dbf"}, + {file = "greenlet-3.0.3-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:da70d4d51c8b306bb7a031d5cff6cc25ad253affe89b70352af5f1cb68e74b53"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:086152f8fbc5955df88382e8a75984e2bb1c892ad2e3c80a2508954e52295257"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d73a9fe764d77f87f8ec26a0c85144d6a951a6c438dfe50487df5595c6373eac"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7dcbe92cc99f08c8dd11f930de4d99ef756c3591a5377d1d9cd7dd5e896da71"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1551a8195c0d4a68fac7a4325efac0d541b48def35feb49d803674ac32582f61"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64d7675ad83578e3fc149b617a444fab8efdafc9385471f868eb5ff83e446b8b"}, + {file = "greenlet-3.0.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b37eef18ea55f2ffd8f00ff8fe7c8d3818abd3e25fb73fae2ca3b672e333a7a6"}, + {file = "greenlet-3.0.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:77457465d89b8263bca14759d7c1684df840b6811b2499838cc5b040a8b5b113"}, + {file = "greenlet-3.0.3-cp39-cp39-win32.whl", hash = "sha256:57e8974f23e47dac22b83436bdcf23080ade568ce77df33159e019d161ce1d1e"}, + {file = "greenlet-3.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:c5ee858cfe08f34712f548c3c363e807e7186f03ad7a5039ebadb29e8c6be067"}, + {file = "greenlet-3.0.3.tar.gz", hash = "sha256:43374442353259554ce33599da8b692d5aa96f8976d567d4badf263371fbe491"}, +] + +[package.extras] +docs = ["Sphinx", "furo"] +test = ["objgraph", "psutil"] + [[package]] name = "identify" version = "2.6.0" @@ -369,6 +459,25 @@ files = [ [package.dependencies] altgraph = ">=0.17" +[[package]] +name = "mako" +version = "1.3.5" +description = "A super-fast templating language that borrows the best ideas from the existing templating languages." +optional = false +python-versions = ">=3.8" +files = [ + {file = "Mako-1.3.5-py3-none-any.whl", hash = "sha256:260f1dbc3a519453a9c856dedfe4beb4e50bd5a26d96386cb6c80856556bb91a"}, + {file = "Mako-1.3.5.tar.gz", hash = "sha256:48dbc20568c1d276a2698b36d968fa76161bf127194907ea6fc594fa81f943bc"}, +] + +[package.dependencies] +MarkupSafe = ">=0.9.2" + +[package.extras] +babel = ["Babel"] +lingua = ["lingua"] +testing = ["pytest"] + [[package]] name = "markdown-it-py" version = "3.0.0" @@ -393,6 +502,75 @@ profiling = ["gprof2dot"] rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + [[package]] name = "mdurl" version = "0.1.2" @@ -613,6 +791,28 @@ nodeenv = ">=0.11.1" pyyaml = ">=5.1" virtualenv = ">=20.10.0" +[[package]] +name = "psycopg2" +version = "2.9.9" +description = "psycopg2 - Python-PostgreSQL Database Adapter" +optional = false +python-versions = ">=3.7" +files = [ + {file = "psycopg2-2.9.9-cp310-cp310-win32.whl", hash = "sha256:38a8dcc6856f569068b47de286b472b7c473ac7977243593a288ebce0dc89516"}, + {file = "psycopg2-2.9.9-cp310-cp310-win_amd64.whl", hash = "sha256:426f9f29bde126913a20a96ff8ce7d73fd8a216cfb323b1f04da402d452853c3"}, + {file = "psycopg2-2.9.9-cp311-cp311-win32.whl", hash = "sha256:ade01303ccf7ae12c356a5e10911c9e1c51136003a9a1d92f7aa9d010fb98372"}, + {file = "psycopg2-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:121081ea2e76729acfb0673ff33755e8703d45e926e416cb59bae3a86c6a4981"}, + {file = "psycopg2-2.9.9-cp312-cp312-win32.whl", hash = "sha256:d735786acc7dd25815e89cc4ad529a43af779db2e25aa7c626de864127e5a024"}, + {file = "psycopg2-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:a7653d00b732afb6fc597e29c50ad28087dcb4fbfb28e86092277a559ae4e693"}, + {file = "psycopg2-2.9.9-cp37-cp37m-win32.whl", hash = "sha256:5e0d98cade4f0e0304d7d6f25bbfbc5bd186e07b38eac65379309c4ca3193efa"}, + {file = "psycopg2-2.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:7e2dacf8b009a1c1e843b5213a87f7c544b2b042476ed7755be813eaf4e8347a"}, + {file = "psycopg2-2.9.9-cp38-cp38-win32.whl", hash = "sha256:ff432630e510709564c01dafdbe996cb552e0b9f3f065eb89bdce5bd31fabf4c"}, + {file = "psycopg2-2.9.9-cp38-cp38-win_amd64.whl", hash = "sha256:bac58c024c9922c23550af2a581998624d6e02350f4ae9c5f0bc642c633a2d5e"}, + {file = "psycopg2-2.9.9-cp39-cp39-win32.whl", hash = "sha256:c92811b2d4c9b6ea0285942b2e7cac98a59e166d59c588fe5cfe1eda58e72d59"}, + {file = "psycopg2-2.9.9-cp39-cp39-win_amd64.whl", hash = "sha256:de80739447af31525feddeb8effd640782cf5998e1a4e9192ebdf829717e3913"}, + {file = "psycopg2-2.9.9.tar.gz", hash = "sha256:d1454bde93fb1e224166811694d600e746430c006fbb031ea06ecc2ea41bf156"}, +] + [[package]] name = "pydantic" version = "2.8.2" @@ -1031,6 +1231,121 @@ files = [ [package.dependencies] bitstring = ">=4.0.1" +[[package]] +name = "sqlalchemy" +version = "2.0.31" +description = "Database Abstraction Library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "SQLAlchemy-2.0.31-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f2a213c1b699d3f5768a7272de720387ae0122f1becf0901ed6eaa1abd1baf6c"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9fea3d0884e82d1e33226935dac990b967bef21315cbcc894605db3441347443"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3ad7f221d8a69d32d197e5968d798217a4feebe30144986af71ada8c548e9fa"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f2bee229715b6366f86a95d497c347c22ddffa2c7c96143b59a2aa5cc9eebbc"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cd5b94d4819c0c89280b7c6109c7b788a576084bf0a480ae17c227b0bc41e109"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:750900a471d39a7eeba57580b11983030517a1f512c2cb287d5ad0fcf3aebd58"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-win32.whl", hash = "sha256:7bd112be780928c7f493c1a192cd8c5fc2a2a7b52b790bc5a84203fb4381c6be"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-win_amd64.whl", hash = "sha256:5a48ac4d359f058474fadc2115f78a5cdac9988d4f99eae44917f36aa1476327"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f68470edd70c3ac3b6cd5c2a22a8daf18415203ca1b036aaeb9b0fb6f54e8298"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2e2c38c2a4c5c634fe6c3c58a789712719fa1bf9b9d6ff5ebfce9a9e5b89c1ca"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd15026f77420eb2b324dcb93551ad9c5f22fab2c150c286ef1dc1160f110203"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2196208432deebdfe3b22185d46b08f00ac9d7b01284e168c212919891289396"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:352b2770097f41bff6029b280c0e03b217c2dcaddc40726f8f53ed58d8a85da4"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:56d51ae825d20d604583f82c9527d285e9e6d14f9a5516463d9705dab20c3740"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-win32.whl", hash = "sha256:6e2622844551945db81c26a02f27d94145b561f9d4b0c39ce7bfd2fda5776dac"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-win_amd64.whl", hash = "sha256:ccaf1b0c90435b6e430f5dd30a5aede4764942a695552eb3a4ab74ed63c5b8d3"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3b74570d99126992d4b0f91fb87c586a574a5872651185de8297c6f90055ae42"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f77c4f042ad493cb8595e2f503c7a4fe44cd7bd59c7582fd6d78d7e7b8ec52c"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd1591329333daf94467e699e11015d9c944f44c94d2091f4ac493ced0119449"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:74afabeeff415e35525bf7a4ecdab015f00e06456166a2eba7590e49f8db940e"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b9c01990d9015df2c6f818aa8f4297d42ee71c9502026bb074e713d496e26b67"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:66f63278db425838b3c2b1c596654b31939427016ba030e951b292e32b99553e"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-win32.whl", hash = "sha256:0b0f658414ee4e4b8cbcd4a9bb0fd743c5eeb81fc858ca517217a8013d282c96"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-win_amd64.whl", hash = "sha256:fa4b1af3e619b5b0b435e333f3967612db06351217c58bfb50cee5f003db2a5a"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f43e93057cf52a227eda401251c72b6fbe4756f35fa6bfebb5d73b86881e59b0"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d337bf94052856d1b330d5fcad44582a30c532a2463776e1651bd3294ee7e58b"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c06fb43a51ccdff3b4006aafee9fcf15f63f23c580675f7734245ceb6b6a9e05"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:b6e22630e89f0e8c12332b2b4c282cb01cf4da0d26795b7eae16702a608e7ca1"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:79a40771363c5e9f3a77f0e28b3302801db08040928146e6808b5b7a40749c88"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-win32.whl", hash = "sha256:501ff052229cb79dd4c49c402f6cb03b5a40ae4771efc8bb2bfac9f6c3d3508f"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-win_amd64.whl", hash = "sha256:597fec37c382a5442ffd471f66ce12d07d91b281fd474289356b1a0041bdf31d"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:dc6d69f8829712a4fd799d2ac8d79bdeff651c2301b081fd5d3fe697bd5b4ab9"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:23b9fbb2f5dd9e630db70fbe47d963c7779e9c81830869bd7d137c2dc1ad05fb"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a21c97efcbb9f255d5c12a96ae14da873233597dfd00a3a0c4ce5b3e5e79704"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26a6a9837589c42b16693cf7bf836f5d42218f44d198f9343dd71d3164ceeeac"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:dc251477eae03c20fae8db9c1c23ea2ebc47331bcd73927cdcaecd02af98d3c3"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:2fd17e3bb8058359fa61248c52c7b09a97cf3c820e54207a50af529876451808"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-win32.whl", hash = "sha256:c76c81c52e1e08f12f4b6a07af2b96b9b15ea67ccdd40ae17019f1c373faa227"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-win_amd64.whl", hash = "sha256:4b600e9a212ed59355813becbcf282cfda5c93678e15c25a0ef896b354423238"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b6cf796d9fcc9b37011d3f9936189b3c8074a02a4ed0c0fbbc126772c31a6d4"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:78fe11dbe37d92667c2c6e74379f75746dc947ee505555a0197cfba9a6d4f1a4"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fc47dc6185a83c8100b37acda27658fe4dbd33b7d5e7324111f6521008ab4fe"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a41514c1a779e2aa9a19f67aaadeb5cbddf0b2b508843fcd7bafdf4c6864005"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:afb6dde6c11ea4525318e279cd93c8734b795ac8bb5dda0eedd9ebaca7fa23f1"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:3f9faef422cfbb8fd53716cd14ba95e2ef655400235c3dfad1b5f467ba179c8c"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-win32.whl", hash = "sha256:fc6b14e8602f59c6ba893980bea96571dd0ed83d8ebb9c4479d9ed5425d562e9"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-win_amd64.whl", hash = "sha256:3cb8a66b167b033ec72c3812ffc8441d4e9f5f78f5e31e54dcd4c90a4ca5bebc"}, + {file = "SQLAlchemy-2.0.31-py3-none-any.whl", hash = "sha256:69f3e3c08867a8e4856e92d7afb618b95cdee18e0bc1647b77599722c9a28911"}, + {file = "SQLAlchemy-2.0.31.tar.gz", hash = "sha256:b607489dd4a54de56984a0c7656247504bd5523d9d0ba799aef59d4add009484"}, +] + +[package.dependencies] +greenlet = {version = "!=0.4.17", markers = "python_version < \"3.13\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} +typing-extensions = ">=4.6.0" + +[package.extras] +aiomysql = ["aiomysql (>=0.2.0)", "greenlet (!=0.4.17)"] +aioodbc = ["aioodbc", "greenlet (!=0.4.17)"] +aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"] +asyncio = ["greenlet (!=0.4.17)"] +asyncmy = ["asyncmy (>=0.2.3,!=0.2.4,!=0.2.6)", "greenlet (!=0.4.17)"] +mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2,!=1.1.5)"] +mssql = ["pyodbc"] +mssql-pymssql = ["pymssql"] +mssql-pyodbc = ["pyodbc"] +mypy = ["mypy (>=0.910)"] +mysql = ["mysqlclient (>=1.4.0)"] +mysql-connector = ["mysql-connector-python"] +oracle = ["cx_oracle (>=8)"] +oracle-oracledb = ["oracledb (>=1.0.1)"] +postgresql = ["psycopg2 (>=2.7)"] +postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] +postgresql-pg8000 = ["pg8000 (>=1.29.1)"] +postgresql-psycopg = ["psycopg (>=3.0.7)"] +postgresql-psycopg2binary = ["psycopg2-binary"] +postgresql-psycopg2cffi = ["psycopg2cffi"] +postgresql-psycopgbinary = ["psycopg[binary] (>=3.0.7)"] +pymysql = ["pymysql"] +sqlcipher = ["sqlcipher3_binary"] + +[[package]] +name = "sqlalchemy-utils" +version = "0.41.2" +description = "Various utility functions for SQLAlchemy." +optional = false +python-versions = ">=3.7" +files = [ + {file = "SQLAlchemy-Utils-0.41.2.tar.gz", hash = "sha256:bc599c8c3b3319e53ce6c5c3c471120bd325d0071fb6f38a10e924e3d07b9990"}, + {file = "SQLAlchemy_Utils-0.41.2-py3-none-any.whl", hash = "sha256:85cf3842da2bf060760f955f8467b87983fb2e30f1764fd0e24a48307dc8ec6e"}, +] + +[package.dependencies] +SQLAlchemy = ">=1.3" + +[package.extras] +arrow = ["arrow (>=0.3.4)"] +babel = ["Babel (>=1.3)"] +color = ["colour (>=0.0.4)"] +encrypted = ["cryptography (>=0.6)"] +intervals = ["intervals (>=0.7.1)"] +password = ["passlib (>=1.6,<2.0)"] +pendulum = ["pendulum (>=2.0.5)"] +phone = ["phonenumbers (>=5.9.2)"] +test = ["Jinja2 (>=2.3)", "Pygments (>=1.2)", "backports.zoneinfo", "docutils (>=0.10)", "flake8 (>=2.4.0)", "flexmock (>=0.9.7)", "isort (>=4.2.2)", "pg8000 (>=1.12.4)", "psycopg (>=3.1.8)", "psycopg2 (>=2.5.1)", "psycopg2cffi (>=2.8.1)", "pymysql", "pyodbc", "pytest (==7.4.4)", "python-dateutil (>=2.6)", "pytz (>=2014.2)"] +test-all = ["Babel (>=1.3)", "Jinja2 (>=2.3)", "Pygments (>=1.2)", "arrow (>=0.3.4)", "backports.zoneinfo", "colour (>=0.0.4)", "cryptography (>=0.6)", "docutils (>=0.10)", "flake8 (>=2.4.0)", "flexmock (>=0.9.7)", "furl (>=0.4.1)", "intervals (>=0.7.1)", "isort (>=4.2.2)", "passlib (>=1.6,<2.0)", "pendulum (>=2.0.5)", "pg8000 (>=1.12.4)", "phonenumbers (>=5.9.2)", "psycopg (>=3.1.8)", "psycopg2 (>=2.5.1)", "psycopg2cffi (>=2.8.1)", "pymysql", "pyodbc", "pytest (==7.4.4)", "python-dateutil", "python-dateutil (>=2.6)", "pytz (>=2014.2)"] +timezone = ["python-dateutil"] +url = ["furl (>=0.4.1)"] + [[package]] name = "tomli" version = "2.0.1" @@ -1143,4 +1458,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "227eab6dfbfac71e0c187a7fbbe43a18f858e995f972a76dfe8cb92cc7aea57e" +content-hash = "305f2da4b140953bf1663b4e567a40f3f355e07b5a45afae6840f8970764ddc7" diff --git a/pyproject.toml b/pyproject.toml index 6c2cf10..72e82da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,10 @@ space-packet-parser = "^4.2.0" xarray = "^2024.6.0" numpy = "^2.0.1" typer = "^0.12.3" +sqlalchemy = "^2.0.31" +psycopg2 = "^2.9.9" +alembic = "^1.13.2" +sqlalchemy-utils = "^0.41.2" [tool.poetry.group.dev.dependencies] pytest = "^8.3.1" diff --git a/src/imap_db/README.md b/src/imap_db/README.md new file mode 100644 index 0000000..fa7eef5 --- /dev/null +++ b/src/imap_db/README.md @@ -0,0 +1,72 @@ +# imap-db: admin util for DB managment and schema updates + +This is a utility for managing the database schema and data for the IMAP data pipeline. + +## Database commands + +CLI Commands to demonstrate database use, and to manage the database have been added. Typer has been used to make a user friendly CLI app. + +```bash +# connect to postgress, create the database if required and then create 2 tables based on the ORM. Thhis comand shows how to use the ORM to create the database schema and populate some rows of data +imap-db create-db + +# upgrade the database schema to the latest version. This command uses alembic to apply the migrations in the migrations folder +imap-db upgrade-db + +# Use sql alchemy to query the database. +imap-db query-db + +# clean up once done and drop the database +imap-db drop-db +``` + +The database connection details including password are loaded from a config file alembic.ini but can easily be overridden from the SQLALCHEMY_URL environment variable. + +## Database migrations + +To enable you to manage a production database over time you can use alembic to migrate the data schema. Migrations are the .py files in the `/migrations` folder. The `alembic.ini` file configures the database connection string and the location of the migrations folder. Use the `alembic` command line tool to add and run the migrations. + +```bash +export DB_HOST=host.docker.internal +# create database using the postgres client cli tools - apt-get install -y postgresql-client +createdb imap -h $DB_HOST -U postgres --port 5432 + +# upgrade an empty database to the latest version +alembic upgrade head + +# or generate a SQL script so you can apply the migration manually +alembic upgrade head --sql > upgrade.sql + +# create a new migration by detetcing changes in the ORM. Will create a new file in the migrations folder +alembic revision --autogenerate -m "Added some table or column" + +# Remove the db (postgresql-client) +dropdb imap -h $DB_HOST -U postgres --port 5432 +``` + +## IDE, Docker, Python + +The app uses VS code with docker the devcontainers feature to setup a python environment with all tools preinstalled. All you need is vscode and docker to be able develop. + +You can connect to the database externally using Azure data studio or some other database tool on 127.0.0.1:5432 as well as using sqltools from within VSCode. + +## Command line database access + +It is also possible to connect to the database from the command line using psql which is pre-installed in the dev container. The database is exposed on port 5432 on localhost and host "db". The password is in the alembic.ini file. + +```bash +$ psql -U postgres -p 5432 -h db -d imap +Password for user postgres: +psql (15.3 (Debian 15.3-0+deb12u1)) +Type "help" for help. + ^ +imap=# SELECT * FROM file; + id | name | fullname +----+-----------+----------------------- + 1 | file1.txt | /path/to/file1.txt + 2 | file2.txt | /path/to/file2.txt + 3 | file3.txt | /path/to/file3.txt +(3 rows) + +imap=# exit +``` diff --git a/src/imap_db/__init__.py b/src/imap_db/__init__.py new file mode 100644 index 0000000..cc2c489 --- /dev/null +++ b/src/imap_db/__init__.py @@ -0,0 +1,4 @@ +import os +import sys + +sys.path.append(os.path.dirname(os.path.realpath(__file__))) diff --git a/src/imap_db/main.py b/src/imap_db/main.py index 5bd3792..22acaa9 100644 --- a/src/imap_db/main.py +++ b/src/imap_db/main.py @@ -1,23 +1,82 @@ """Main module.""" -from typing import Annotated +import os -# cli +import sqlalchemy import typer +from alembic import command, config +from sqlalchemy import create_engine, select +from sqlalchemy.orm import Session +from sqlalchemy_utils import create_database, database_exists, drop_database + +from .model import Base, File + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = config.Config("alembic.ini") app = typer.Typer() -globalState = {"verbose": False} + +# enable overriding the database url from the OS env +env_override_url = os.getenv("SQLALCHEMY_URL") +if env_override_url is not None and len(env_override_url) > 0: + url = config.set_main_option("sqlalchemy.url", env_override_url) + +url = config.get_main_option("sqlalchemy.url") +engine = create_engine(url, echo=True) # echo for dev to outputing SQL + + +@app.command() +def create_db(with_schema: bool = False, with_data: bool = False): + print("sql sqlalchemy version: " + sqlalchemy.__version__) + + if not database_exists(engine.url): + print("Creating db") + create_database(engine.url) + else: + print("Db already exists") + + if with_schema: + print("Creating all tables") + Base.metadata.create_all(engine) + + if with_data: + print("Loading some data") + with Session(engine) as session: + f1 = File(name="file1.txt", path="/path/to/file1.txt") + session.add_all([f1]) + session.commit() + + print("Database create complete") + + +@app.command() +def drop_db(): + if database_exists(engine.url): + print("Dropping db") + drop_database(engine.url) + else: + print("Skipped - Db does not exist") @app.command() -def hello(name: str): - print(f"Hello {name}") +def query_db(): + session = Session(engine) + stmt = select(File).where(File.name.in_(["file1.txt"])) + + for user in session.scalars(stmt): + print(user) + + +@app.command() +def upgrade_db(): + script_location = "src/imap_db/migrations" + print("Running DB migrations in %r on %r", script_location, url) -@app.callback() -def main(verbose: Annotated[bool, typer.Option("--verbose", "-v")] = False): - if verbose: - globalState["verbose"] = True + config.set_main_option("script_location", script_location) + config.set_main_option("sqlalchemy.url", url) + command.upgrade(config, "head") if __name__ == "__main__": diff --git a/src/imap_db/migrations/env.py b/src/imap_db/migrations/env.py new file mode 100644 index 0000000..b5add4c --- /dev/null +++ b/src/imap_db/migrations/env.py @@ -0,0 +1,71 @@ +from logging.config import fileConfig + +from alembic import context +from imap_db.model import Base +from sqlalchemy import engine_from_config, pool + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/src/imap_db/migrations/script.py.mako b/src/imap_db/migrations/script.py.mako new file mode 100644 index 0000000..55df286 --- /dev/null +++ b/src/imap_db/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/src/imap_db/migrations/versions/2024_07_25-d0457f3e98c8_create_files_table.py b/src/imap_db/migrations/versions/2024_07_25-d0457f3e98c8_create_files_table.py new file mode 100644 index 0000000..32c67b0 --- /dev/null +++ b/src/imap_db/migrations/versions/2024_07_25-d0457f3e98c8_create_files_table.py @@ -0,0 +1,34 @@ +"""Create files table. + +Revision ID: d0457f3e98c8 +Revises: +Create Date: 2024-07-25 19:00:38.921571 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "d0457f3e98c8" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "files", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(length=128), nullable=False), + sa.Column("path", sa.String(length=256), nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("files") + # ### end Alembic commands ### diff --git a/src/imap_db/model.py b/src/imap_db/model.py new file mode 100644 index 0000000..92ea7ce --- /dev/null +++ b/src/imap_db/model.py @@ -0,0 +1,16 @@ +from sqlalchemy import String +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column + + +class Base(DeclarativeBase): + pass + + +class File(Base): + __tablename__ = "files" + id: Mapped[int] = mapped_column(primary_key=True) + name: Mapped[str] = mapped_column(String(128)) + path: Mapped[str] = mapped_column(String(256)) + + def __repr__(self) -> str: # noqa: D105 + return f"" diff --git a/src/imap_mag/__init__.py b/src/imap_mag/__init__.py index 81a000c..e69de29 100644 --- a/src/imap_mag/__init__.py +++ b/src/imap_mag/__init__.py @@ -1 +0,0 @@ -"""Top-level package for cli app.""" From f9f6046806240fd8b417d91d0bc167ba8de8467b Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Thu, 25 Jul 2024 20:11:05 +0000 Subject: [PATCH 03/33] bug: fix docker build deps and cli app paths on install --- deploy/Dockerfile | 3 +++ poetry.lock | 38 +++++++++++++++++++------------------- pyproject.toml | 8 ++++---- 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 9982e81..06184f3 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -20,6 +20,9 @@ RUN adduser -u 5678 --disabled-password --gecos "" appuser && \ chown -R appuser /app && \ chmod +x /app/entrypoint.sh +# Install the postgres client and any other dependencies needed to install our app +RUN apt-get update && apt-get install -y libpq-dev gcc + WORKDIR /app USER appuser diff --git a/poetry.lock b/poetry.lock index 7deb489..4734876 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1154,29 +1154,29 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] [[package]] name = "ruff" -version = "0.5.4" +version = "0.5.5" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.5.4-py3-none-linux_armv6l.whl", hash = "sha256:82acef724fc639699b4d3177ed5cc14c2a5aacd92edd578a9e846d5b5ec18ddf"}, - {file = "ruff-0.5.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:da62e87637c8838b325e65beee485f71eb36202ce8e3cdbc24b9fcb8b99a37be"}, - {file = "ruff-0.5.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e98ad088edfe2f3b85a925ee96da652028f093d6b9b56b76fc242d8abb8e2059"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c55efbecc3152d614cfe6c2247a3054cfe358cefbf794f8c79c8575456efe19"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f9b85eaa1f653abd0a70603b8b7008d9e00c9fa1bbd0bf40dad3f0c0bdd06793"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0cf497a47751be8c883059c4613ba2f50dd06ec672692de2811f039432875278"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:09c14ed6a72af9ccc8d2e313d7acf7037f0faff43cde4b507e66f14e812e37f7"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:628f6b8f97b8bad2490240aa84f3e68f390e13fabc9af5c0d3b96b485921cd60"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3520a00c0563d7a7a7c324ad7e2cde2355733dafa9592c671fb2e9e3cd8194c1"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93789f14ca2244fb91ed481456f6d0bb8af1f75a330e133b67d08f06ad85b516"}, - {file = "ruff-0.5.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:029454e2824eafa25b9df46882f7f7844d36fd8ce51c1b7f6d97e2615a57bbcc"}, - {file = "ruff-0.5.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:9492320eed573a13a0bc09a2957f17aa733fff9ce5bf00e66e6d4a88ec33813f"}, - {file = "ruff-0.5.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a6e1f62a92c645e2919b65c02e79d1f61e78a58eddaebca6c23659e7c7cb4ac7"}, - {file = "ruff-0.5.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:768fa9208df2bec4b2ce61dbc7c2ddd6b1be9fb48f1f8d3b78b3332c7d71c1ff"}, - {file = "ruff-0.5.4-py3-none-win32.whl", hash = "sha256:e1e7393e9c56128e870b233c82ceb42164966f25b30f68acbb24ed69ce9c3a4e"}, - {file = "ruff-0.5.4-py3-none-win_amd64.whl", hash = "sha256:58b54459221fd3f661a7329f177f091eb35cf7a603f01d9eb3eb11cc348d38c4"}, - {file = "ruff-0.5.4-py3-none-win_arm64.whl", hash = "sha256:bd53da65f1085fb5b307c38fd3c0829e76acf7b2a912d8d79cadcdb4875c1eb7"}, - {file = "ruff-0.5.4.tar.gz", hash = "sha256:2795726d5f71c4f4e70653273d1c23a8182f07dd8e48c12de5d867bfb7557eed"}, + {file = "ruff-0.5.5-py3-none-linux_armv6l.whl", hash = "sha256:605d589ec35d1da9213a9d4d7e7a9c761d90bba78fc8790d1c5e65026c1b9eaf"}, + {file = "ruff-0.5.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:00817603822a3e42b80f7c3298c8269e09f889ee94640cd1fc7f9329788d7bf8"}, + {file = "ruff-0.5.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:187a60f555e9f865a2ff2c6984b9afeffa7158ba6e1eab56cb830404c942b0f3"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe26fc46fa8c6e0ae3f47ddccfbb136253c831c3289bba044befe68f467bfb16"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4ad25dd9c5faac95c8e9efb13e15803cd8bbf7f4600645a60ffe17c73f60779b"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f70737c157d7edf749bcb952d13854e8f745cec695a01bdc6e29c29c288fc36e"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:cfd7de17cef6ab559e9f5ab859f0d3296393bc78f69030967ca4d87a541b97a0"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a09b43e02f76ac0145f86a08e045e2ea452066f7ba064fd6b0cdccb486f7c3e7"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d0b856cb19c60cd40198be5d8d4b556228e3dcd545b4f423d1ad812bfdca5884"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3687d002f911e8a5faf977e619a034d159a8373514a587249cc00f211c67a091"}, + {file = "ruff-0.5.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:ac9dc814e510436e30d0ba535f435a7f3dc97f895f844f5b3f347ec8c228a523"}, + {file = "ruff-0.5.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:af9bdf6c389b5add40d89b201425b531e0a5cceb3cfdcc69f04d3d531c6be74f"}, + {file = "ruff-0.5.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d40a8533ed545390ef8315b8e25c4bb85739b90bd0f3fe1280a29ae364cc55d8"}, + {file = "ruff-0.5.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:cab904683bf9e2ecbbe9ff235bfe056f0eba754d0168ad5407832928d579e7ab"}, + {file = "ruff-0.5.5-py3-none-win32.whl", hash = "sha256:696f18463b47a94575db635ebb4c178188645636f05e934fdf361b74edf1bb2d"}, + {file = "ruff-0.5.5-py3-none-win_amd64.whl", hash = "sha256:50f36d77f52d4c9c2f1361ccbfbd09099a1b2ea5d2b2222c586ab08885cf3445"}, + {file = "ruff-0.5.5-py3-none-win_arm64.whl", hash = "sha256:3191317d967af701f1b73a31ed5788795936e423b7acce82a2b63e26eb3e89d6"}, + {file = "ruff-0.5.5.tar.gz", hash = "sha256:cc5516bdb4858d972fbc31d246bdb390eab8df1a26e2353be2dbc0c2d7f5421a"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index 72e82da..c44161c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,8 +5,8 @@ description = "Process IMAP data" authors = ["alastairtree"] readme = "README.md" packages = [ - { include = "imap_mag", from = "src", to="imap_mag" }, - { include = "imap_db", from = "src", to="imap_db" }, + { include = "src/imap_mag" }, + { include = "src/imap_db" }, ] [tool.poetry.dependencies] @@ -32,8 +32,8 @@ ruff = "^0.5.4" [tool.poetry.scripts] # can execute via poetry, e.g. `poetry run imap-mag hello world` -imap-mag = 'imap_mag.main:app' -imap-db = 'imap_db.main:app' +imap-mag = 'src.imap_mag.main:app' +imap-db = 'src.imap_db.main:app' [build-system] requires = ["poetry-core"] From 7cda6061a239fc71de0ff8abeb78cd91c537fe16 Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Thu, 25 Jul 2024 20:27:07 +0000 Subject: [PATCH 04/33] bug: fix app paths for tests --- pyproject.toml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c44161c..086ab69 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,8 +5,8 @@ description = "Process IMAP data" authors = ["alastairtree"] readme = "README.md" packages = [ - { include = "src/imap_mag" }, - { include = "src/imap_db" }, + { include = "src/imap_mag", to = "imap_mag" }, + { include = "src/imap_db", to = "imap_db" }, ] [tool.poetry.dependencies] @@ -35,6 +35,11 @@ ruff = "^0.5.4" imap-mag = 'src.imap_mag.main:app' imap-db = 'src.imap_db.main:app' +[tool.pytest.ini_options] +pythonpath = [ + ".", "src" +] + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" From 7114791d0a8c823c08ac4bda0549adf7924ebc72 Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Thu, 25 Jul 2024 20:32:03 +0000 Subject: [PATCH 05/33] bug: fix installed cli app paths again --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 086ab69..f1c9eb5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,8 +5,8 @@ description = "Process IMAP data" authors = ["alastairtree"] readme = "README.md" packages = [ - { include = "src/imap_mag", to = "imap_mag" }, - { include = "src/imap_db", to = "imap_db" }, + { include = "src/imap_mag" }, + { include = "src/imap_db" }, ] [tool.poetry.dependencies] From bbfcb83f575cb8c75d848cb9f18185e604c51e22 Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Thu, 25 Jul 2024 21:13:10 +0000 Subject: [PATCH 06/33] bug: fix db migrations --- deploy/entrypoint.sh | 3 ++- src/imap_db/main.py | 6 ++++++ src/imap_db/migrations/env.py | 3 ++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/deploy/entrypoint.sh b/deploy/entrypoint.sh index fff3668..ab97f4a 100644 --- a/deploy/entrypoint.sh +++ b/deploy/entrypoint.sh @@ -1,9 +1,10 @@ #!/bin/bash set -e - +echo "start DB admin" imap-db create-db imap-db upgrade-db +echo "DB admin complete" while : do diff --git a/src/imap_db/main.py b/src/imap_db/main.py index 22acaa9..c5c4b6a 100644 --- a/src/imap_db/main.py +++ b/src/imap_db/main.py @@ -1,6 +1,7 @@ """Main module.""" import os +import pathlib import sqlalchemy import typer @@ -71,7 +72,12 @@ def query_db(): @app.command() def upgrade_db(): + folder = pathlib.Path(__file__).parent.resolve() script_location = "src/imap_db/migrations" + + # combine them in OS agnostic way + script_location = os.path.join(folder, script_location) + print("Running DB migrations in %r on %r", script_location, url) config.set_main_option("script_location", script_location) diff --git a/src/imap_db/migrations/env.py b/src/imap_db/migrations/env.py index b5add4c..8dcfe34 100644 --- a/src/imap_db/migrations/env.py +++ b/src/imap_db/migrations/env.py @@ -1,9 +1,10 @@ from logging.config import fileConfig from alembic import context -from imap_db.model import Base from sqlalchemy import engine_from_config, pool +from src.imap_db.model import Base + # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config From fe50dd2d2ea7d2027106a4a3f377edc2772ab1ef Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Thu, 25 Jul 2024 21:20:31 +0000 Subject: [PATCH 07/33] feat: Update script location for db migrations --- src/imap_db/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/imap_db/main.py b/src/imap_db/main.py index c5c4b6a..aeffefd 100644 --- a/src/imap_db/main.py +++ b/src/imap_db/main.py @@ -73,7 +73,7 @@ def query_db(): @app.command() def upgrade_db(): folder = pathlib.Path(__file__).parent.resolve() - script_location = "src/imap_db/migrations" + script_location = "migrations" # combine them in OS agnostic way script_location = os.path.join(folder, script_location) From 2adb9856b88794b7f3278b679fd5690d60d389b1 Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Thu, 25 Jul 2024 21:27:03 +0000 Subject: [PATCH 08/33] feat: Include alembic.ini in distribution packages --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index f1c9eb5..2aecb04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,9 @@ packages = [ { include = "src/imap_mag" }, { include = "src/imap_db" }, ] +include = [ + { path = "alembic.ini", format = ["sdist", "wheel"] } +] [tool.poetry.dependencies] python = ">=3.9,<3.13" From a466aa3a09e2587b18fe953b9485e659b4b98bb7 Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Thu, 25 Jul 2024 21:35:24 +0000 Subject: [PATCH 09/33] bug: fix path to ini file --- alembic.ini => src/imap_db/alembic.ini | 0 src/imap_db/main.py | 4 +++- 2 files changed, 3 insertions(+), 1 deletion(-) rename alembic.ini => src/imap_db/alembic.ini (100%) diff --git a/alembic.ini b/src/imap_db/alembic.ini similarity index 100% rename from alembic.ini rename to src/imap_db/alembic.ini diff --git a/src/imap_db/main.py b/src/imap_db/main.py index aeffefd..2c29b75 100644 --- a/src/imap_db/main.py +++ b/src/imap_db/main.py @@ -14,7 +14,9 @@ # this is the Alembic Config object, which provides # access to the values within the .ini file in use. -config = config.Config("alembic.ini") +folder = pathlib.Path(__file__).parent.resolve() +alembic_ini = os.path.join(folder, "alembic.ini") +config = config.Config(alembic_ini) app = typer.Typer() From b6c9606c73c38465c8a6e530652a803951d85df4 Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Thu, 25 Jul 2024 21:40:25 +0000 Subject: [PATCH 10/33] feat: tag latests on all images --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 24e0235..eb241cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,7 +106,7 @@ jobs: # use custom value instead of git tag type=semver,pattern={{version}},value=${{ env.PACKAGE_VERSION }} # set latest tag for default branch - type=raw,value=latest,enable={{is_default_branch}} + type=raw,value=latest # branch event type=ref,event=branch # pull request event From f11b1c5083a62b75c34b7e8008f8038dc6d6f558 Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Thu, 25 Jul 2024 21:41:49 +0000 Subject: [PATCH 11/33] Include alembic.ini in distribution packages try 2 --- pyproject.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2aecb04..f1c9eb5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,9 +8,6 @@ packages = [ { include = "src/imap_mag" }, { include = "src/imap_db" }, ] -include = [ - { path = "alembic.ini", format = ["sdist", "wheel"] } -] [tool.poetry.dependencies] python = ">=3.9,<3.13" From f18ac7d306886f8975b406ec50e6b1c0d2d30687 Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Sun, 28 Jul 2024 20:34:22 +0000 Subject: [PATCH 12/33] feat: quick hack to download SDC data --- .devcontainer/devcontainer.json | 4 +- poetry.lock | 104 ++++++++++++++++++- pyproject.toml | 2 + src/SDC.py | 138 +++++++++++++++++++++++++ src/main.py | 49 ++++++++- src/sdcApiClient.py | 176 ++++++++++++++++++++++++++++++++ src/time.py | 34 ++++++ 7 files changed, 504 insertions(+), 3 deletions(-) create mode 100644 src/SDC.py create mode 100644 src/sdcApiClient.py create mode 100644 src/time.py diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 3274e36..8bbffaf 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -31,7 +31,9 @@ "remoteEnv": { // add the pyenv bin and poetry to the PATH for the dev user vscode "PATH": "/home/vscode/.pyenv/bin:/home/vscode/.local/bin:${containerEnv:PATH}", - "WEBPODA_AUTH_CODE": "${localEnv:WEBPODA_AUTH_CODE}" + "WEBPODA_AUTH_CODE": "${localEnv:WEBPODA_AUTH_CODE}", + "SDC_AUTH_CODE": "${localEnv:SDC_AUTH_CODE}", + "IMAP_DATA_ACCESS_URL": "${localEnv:IMAP_DATA_ACCESS_URL}", }, "postCreateCommand": "pip install --user -U pre-commit && pre-commit install-hooks && pre-commit autoupdate", // install poetry in first startup in vscode diff --git a/poetry.lock b/poetry.lock index 7c0f46b..0e5c035 100644 --- a/poetry.lock +++ b/poetry.lock @@ -22,6 +22,67 @@ files = [ {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, ] +[[package]] +name = "astropy" +version = "6.1.2" +description = "Astronomy and astrophysics core library" +optional = false +python-versions = ">=3.10" +files = [ + {file = "astropy-6.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a64eb948c8c1c87545592ff5e3ba366f3a71615dea6532a96891874b03bd9a5d"}, + {file = "astropy-6.1.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c50fa9dcd4fbafd54c5da15092f8d9200b2c82711f8971dd23c139920c6c780c"}, + {file = "astropy-6.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f80865e18ffbe2f9901e59e6f750218b823b5c591f687c2bca3adf0f2a6af4e"}, + {file = "astropy-6.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:305433b7571d3dbcbc264dbf96ec334a89836ddd78d0d15f77821b90eef3f7b4"}, + {file = "astropy-6.1.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b2521be1a1e76c92444905da84cee541e052408632d7fc1fb853e57ef5190963"}, + {file = "astropy-6.1.2-cp310-cp310-win32.whl", hash = "sha256:8f846339fdd093b261dc33a85a78eafa04598b4d8f1807a18ceb0f6eb9a097ef"}, + {file = "astropy-6.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:18747bae9a1eee0e5a408907b82219ddc356198de0948a80bb7d27143e780b7d"}, + {file = "astropy-6.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4563a6d5643c321acb508792ccbec5f1c62302e3271109229ab023d69902a712"}, + {file = "astropy-6.1.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5f8cbd0e3d4b17715e508de2ef0f84057a810b3724b6219181f49d726c1d6436"}, + {file = "astropy-6.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:04eead3eb28021a5853edb620ed6f50311bd5d272ccad06ed82fee293441a834"}, + {file = "astropy-6.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5ee7e334a0601858fcd4b72490b0626174ac97fd591fc3408b496d20167f186"}, + {file = "astropy-6.1.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:99b1d4cb739ff5c20a76e4c42ed38478a8fbd8482fada504796e0d55d39cb5bd"}, + {file = "astropy-6.1.2-cp311-cp311-win32.whl", hash = "sha256:2e25057dd6b5fd8f543f2d08f46fcf6a3691135231f1c016da477df22a25e13b"}, + {file = "astropy-6.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:8bd518b0c94c48a74e95d8b949bd50bf6f72cf1dd56ed925c19c689a39aaaab4"}, + {file = "astropy-6.1.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:4aaa06dc984ff3e409019a51935ac9c31875baa538de04c1634ab02f727dd52b"}, + {file = "astropy-6.1.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:12d292909a86f00316c9d3007ae8991906c23461400dba1cb6de63ad55449a32"}, + {file = "astropy-6.1.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d959819a695083f0653e0b28c661f4388fdb0c812ccc3f5c343626ec5a1708e5"}, + {file = "astropy-6.1.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:075c54dd9cd8eab52b2de4eddddec0543dfaf7879c231a811b9ba872514f87f6"}, + {file = "astropy-6.1.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8a32996e01553ba5469c0cebf9d7f6587ed11d691f88a0d0879b4ab0609e8f7f"}, + {file = "astropy-6.1.2-cp312-cp312-win32.whl", hash = "sha256:c39fcd493753e4f3628ee775171611fc1c0cc419bc61f7fe69b84ec02b117a54"}, + {file = "astropy-6.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:2d35bf528e8bc6b0f87db9d7ade428964bab51b7bbcf0f11ad3790fa60fcb279"}, + {file = "astropy-6.1.2.tar.gz", hash = "sha256:a2103d4e24e90389a820cfcdaaf4ca2d1ab22e5fd72978d147ff5cace54f1d3a"}, +] + +[package.dependencies] +astropy-iers-data = ">=0.2024.7.1.0.34.3" +numpy = ">=1.23" +packaging = ">=19.0" +pyerfa = ">=2.0.1.1" +PyYAML = ">=3.13" + +[package.extras] +all = ["asdf-astropy (>=0.3)", "astropy[recommended]", "astropy[typing]", "beautifulsoup4", "bleach", "bottleneck", "certifi", "dask[array]", "fsspec[http] (>=2023.4.0)", "h5py", "html5lib", "ipython (>=4.2)", "jplephem", "mpmath", "pandas", "pre-commit", "pyarrow (>=5.0.0)", "pytest (>=7.0)", "pytz", "s3fs (>=2023.4.0)", "sortedcontainers"] +docs = ["Jinja2 (>=3.1.3)", "astropy[recommended]", "matplotlib (>=3.9.1)", "numpy (<2.0)", "pytest (>=7.0)", "sphinx", "sphinx-astropy[confv2] (>=1.9.1)", "sphinx-changelog (>=1.2.0)", "sphinx-design", "sphinxcontrib-globalsubs (>=0.1.1)", "tomli"] +recommended = ["matplotlib (>=3.3,!=3.4.0,!=3.5.2)", "scipy (>=1.8)"] +test = ["pytest (>=7.0)", "pytest-astropy (>=0.10)", "pytest-astropy-header (>=0.2.1)", "pytest-doctestplus (>=0.12)", "pytest-xdist", "threadpoolctl"] +test-all = ["array-api-strict", "astropy[test]", "coverage[toml]", "ipython (>=4.2)", "objgraph", "sgp4 (>=2.3)", "skyfield (>=1.20)"] +typing = ["typing-extensions (>=4.0.0)"] + +[[package]] +name = "astropy-iers-data" +version = "0.2024.7.22.0.34.13" +description = "IERS Earth Rotation and Leap Second tables for the astropy core package" +optional = false +python-versions = ">=3.8" +files = [ + {file = "astropy_iers_data-0.2024.7.22.0.34.13-py3-none-any.whl", hash = "sha256:567a6cb261dd62f60862ee8d38e70fb2c88dfad03e962bc8138397a22e33003d"}, + {file = "astropy_iers_data-0.2024.7.22.0.34.13.tar.gz", hash = "sha256:9bbb4bfc28bc8e834a6b3946a312ce3490c285abeab8fd9b1e98b11fdee6f92c"}, +] + +[package.extras] +docs = ["pytest"] +test = ["hypothesis", "pytest", "pytest-remotedata"] + [[package]] name = "bitarray" version = "2.9.2" @@ -446,6 +507,20 @@ files = [ {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, ] +[[package]] +name = "imap-data-access" +version = "0.7.0" +description = "IMAP SDC Data Access" +optional = false +python-versions = "*" +files = [ + {file = "imap_data_access-0.7.0.tar.gz", hash = "sha256:f0db935949d048394fc554b308b1e4a1572a18acd41636462d37c309c7cb4c9d"}, +] + +[package.extras] +dev = ["imap_data_access[test]", "pre-commit", "ruff"] +test = ["pytest", "pytest-cov"] + [[package]] name = "iniconfig" version = "2.0.0" @@ -835,6 +910,33 @@ files = [ [package.dependencies] typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" +[[package]] +name = "pyerfa" +version = "2.0.1.4" +description = "Python bindings for ERFA" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pyerfa-2.0.1.4-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ff112353944bf705342741f2fe41674f97154a302b0295eaef7381af92ad2b3a"}, + {file = "pyerfa-2.0.1.4-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:900b266a3862baa9560d6b1b184dcc14e0e76d550ff70d32336d3989b2ed18ca"}, + {file = "pyerfa-2.0.1.4-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:610d2bc314e140d876b93b1287c7c81685434873c8700cc3e1596193f77d1071"}, + {file = "pyerfa-2.0.1.4-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e4508dd7ffd7b27b7f67168643764454887e990ca9e4584824f0e3ab5884c0f"}, + {file = "pyerfa-2.0.1.4-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:83a44ba84ebfc3244412ecbf1065c087c382da84f1c3eee1f2a0638d9046ac96"}, + {file = "pyerfa-2.0.1.4-cp39-abi3-win32.whl", hash = "sha256:46d3bed0ac666f08d8364b34a00b8c6595358d6c4f4532da8d13fac0e5227baa"}, + {file = "pyerfa-2.0.1.4-cp39-abi3-win_amd64.whl", hash = "sha256:bc3cf45967ac1af77a777deb050fb08bbc75256dd97ca6005e4d385358b7af40"}, + {file = "pyerfa-2.0.1.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:88a8d0f3608a66871615bd168fcddf674dce9f7568c239a03cf8d9936161d032"}, + {file = "pyerfa-2.0.1.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9045e9f786c76cb55da86ada3405c378c32b88f6e3c6296cb288496ab374b068"}, + {file = "pyerfa-2.0.1.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:39cf838c9a21e40d4e3183bead65b3ce6af763c4a727f87d84909c9be7d3a33c"}, + {file = "pyerfa-2.0.1.4.tar.gz", hash = "sha256:acb8a6713232ea35c04bc6e40ac4e461dfcc817d395ef2a3c8051c1a33249dd3"}, +] + +[package.dependencies] +numpy = ">=1.19" + +[package.extras] +docs = ["sphinx-astropy (>=1.3)"] +test = ["pytest", "pytest-doctestplus (>=0.7)"] + [[package]] name = "pygments" version = "2.18.0" @@ -1266,4 +1368,4 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "012ad9c3f09624d49607a97b5daef6101bd7d8587b4f8e37fac748c5da008a22" +content-hash = "9d1e7c57a9c2a972fe2db1d75b71c2fec76fef6c7a6e6abeda6767efb51003e9" diff --git a/pyproject.toml b/pyproject.toml index b72836a..812c39a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,8 @@ numpy = "^2.0.1" typer = "^0.12.3" requests = "^2.32.3" pandas = "^2.2.2" +imap-data-access = "^0.7.0" +astropy = "^6.1.2" [tool.poetry.group.dev.dependencies] pytest = "^8.3.1" diff --git a/src/SDC.py b/src/SDC.py new file mode 100644 index 0000000..c275116 --- /dev/null +++ b/src/SDC.py @@ -0,0 +1,138 @@ +"""Program to retrieve and process MAG CDF files.""" + +import logging +import typing +from datetime import datetime + +import pandas as pd + +from src.sdcApiClient import ISDCApiClient +from src.time import Time + + +class SDCOptions(typing.TypedDict): + """Options for SOC interactions.""" + + level: str + start_date: str + end_date: str + output_dir: str + force: bool + + +class SDC: + """Manage SOC data.""" + + __PACKET_DETAILS = ( + { + "packet": "MAG_SCI_NORM", + "raw_descriptor": "raw", + "descriptor": "norm", + "variations": ["-magi", "-mago"], + }, + { + "packet": "MAG_SCI_BURST", + "raw_descriptor": "raw", + "descriptor": "burst", + "variations": ["-magi", "-mago"], + }, + ) + + __data_access: ISDCApiClient + + def __init__(self, data_access: ISDCApiClient) -> None: + """Initialize SDC interface.""" + + self.__data_access = data_access + + def QueryAndDownload(self, **options: typing.Unpack[SDCOptions]) -> None: + """Retrieve SDC data.""" + for details in self.__PACKET_DETAILS: + (start, end) = self.__extract_time_range( + options["start_date"], + options["end_date"], + ) + + date_range: pd.DatetimeIndex = pd.date_range( + start=start.date, end=end.date, freq="D", normalize=True + ) + + for date in date_range.to_pydatetime(): + download_files = self.__check_download_needed(details, date, **options) + + (new_version, previous_version) = self.__data_access.unique_version( + level=options["level"], + start_date=date, + ) + + if download_files: + version = new_version + else: + assert previous_version is not None + version = previous_version + + for var in details["variations"]: + files = self.__data_access.get_filename( + level=options["level"], + descriptor=str(details["descriptor"]) + str(var), + start_date=date, + end_date=None, + version=version, + extension="cdf", + ) + + if files is not None: + for file in files: + self.__data_access.download(file["file_path"]) + + def __extract_time_range(self, start_date: str, end_date: str) -> tuple[Time, Time]: + """Extract time range as S/C and ERT Time object.""" + + start = self.__convert_to_datetime(start_date, "start date") + end = self.__convert_to_datetime(end_date, "end date") + + return (Time(start, 0), Time(end, 0)) + + def __convert_to_datetime(self, string: str, name: str) -> datetime: + """Convert string to datetime.""" + + try: + return pd.to_datetime(string) + except Exception as e: + logging.error(f"Error parsing {name}: {e}") + raise e + + def __check_download_needed( + self, + details: dict, + date: datetime, + **options: typing.Unpack[SDCOptions], + ) -> bool: + """Check if files need to be downloaded.""" + + if options["force"]: + logging.debug("Forcing download.") + return True + else: + for var in details["variations"]: + files: list[dict[str, str]] = self.__data_access.query( + level=options["level"], + descriptor=details["descriptor"] + str(var), + start_date=date, + end_date=None, + version=None, + extension="cdf", + ) + + if not files: + logging.debug( + "Downloading new files. " + f"No existing files found for {details} on {date}. " + ) + return True + + logging.debug( + "Not downloading new files. " + f"Files found for {details} on {date}:\n{files}" + ) + return False diff --git a/src/main.py b/src/main.py index 2a727f0..05ae10f 100644 --- a/src/main.py +++ b/src/main.py @@ -4,6 +4,7 @@ import os import shutil from datetime import datetime +from enum import Enum from pathlib import Path from typing import Annotated @@ -14,7 +15,8 @@ import yaml # app code -from src import appConfig, appLogging, appUtils, imapProcessing, webPODA +from src import SDC, appConfig, appLogging, appUtils, imapProcessing, webPODA +from src.sdcApiClient import SDCApiClient app = typer.Typer() globalState = {"verbose": False} @@ -169,6 +171,51 @@ def fetch_binary( appUtils.copyFileToDestination(result, configFile.destination) +class LevelEnum(str, Enum): + level_1 = "l1" + level_2 = "l2" + level_3 = "l3" + + +# E.g., imap-mag fetch-science --start-date 2025-05-02 --end-date 2025-05-03 +@app.command() +def fetch_science( + auth_code: Annotated[ + str, + typer.Option( + envvar="SDC_AUTH_CODE", + help="IMAP Science Data Centre API Key", + ), + ], + start_date: Annotated[str, typer.Option(help="Start date for the download")], + end_date: Annotated[str, typer.Option(help="End date for the download")], + level: Annotated[ + LevelEnum, typer.Option(help="Level to download") + ] = LevelEnum.level_2, + force: Annotated[ + bool, + typer.Option("--force", "-f", help="Force download even if the file exists"), + ] = False, + config: Annotated[Path, typer.Option()] = Path("config.yml"), +): + configFile: appConfig.AppConfig = commandInit(config) + + if not auth_code: + logging.critical("No SDC_AUTH_CODE API key provided") + raise typer.Abort() + + logging.info(f"Downloading {level} science from {start_date} to {end_date}.") + + data_access = SDCApiClient(data_dir=str(configFile.work_folder)) + + sdc = SDC.SDC(data_access) + + # TODO: any better way than passing a dictionary? Strongly typed? + sdc.QueryAndDownload( + level=level, start_date=start_date, end_date=end_date, force=force + ) + + @app.callback() def main(verbose: Annotated[bool, typer.Option("--verbose", "-v")] = False): if verbose: diff --git a/src/sdcApiClient.py b/src/sdcApiClient.py new file mode 100644 index 0000000..1a4117e --- /dev/null +++ b/src/sdcApiClient.py @@ -0,0 +1,176 @@ +"""Interact with SDC APIs to get MAG data via imap-data-access.""" + +import abc +import logging +import pathlib +import typing +from datetime import datetime + +import imap_data_access + + +class FileOptions(typing.TypedDict): + """Options for generating file name.""" + + level: str | None + descriptor: str | None + start_date: datetime | None + version: str | None + + +class VersionOptions(typing.TypedDict): + """Options for determining unique version.""" + + level: str | None + start_date: datetime | None + + +class QueryOptions(typing.TypedDict): + """Options for query.""" + + level: str | None + descriptor: str | None + start_date: datetime | None + end_date: datetime | None + version: str | None + extension: str | None + + +class ISDCApiClient(abc.ABC): + """Interface for interacting with imap-data-access.""" + + @staticmethod + @abc.abstractmethod + def get_file_path(**options: typing.Unpack[FileOptions]) -> tuple[str, str]: + """Get file path for data from imap-data-access.""" + pass + + @abc.abstractmethod + def upload(self, file_name: str) -> None: + """Upload data to imap-data-access.""" + pass + + @abc.abstractmethod + def unique_version( + self, **options: typing.Unpack[VersionOptions] + ) -> tuple[str, str | None]: + """Determine a unique version for the data by querying imap_data_access.""" + pass + + @abc.abstractmethod + def query(self, **options: typing.Unpack[QueryOptions]) -> list[dict[str, str]]: + """Download data from imap-data-access.""" + pass + + @abc.abstractmethod + def get_filename( + self, **options: typing.Unpack[FileOptions] + ) -> list[dict[str, str]] | None: + """Wait for file to be available in imap-data-access.""" + pass + + @abc.abstractmethod + def download(self, file_name: str) -> pathlib.Path: + """Download data from imap-data-access.""" + pass + + +class SDCApiClient(ISDCApiClient): + """Class for uploading and downloading MAG data via imap-data-access.""" + + def __init__(self, data_dir: str) -> None: + imap_data_access.config["DATA_DIR"] = pathlib.Path(data_dir) + + @staticmethod + def get_file_path(**options: typing.Unpack[FileOptions]) -> tuple[str, str]: + """Get file path for data from imap-data-access.""" + + science_file = imap_data_access.ScienceFilePath.generate_from_inputs( + instrument="mag", + data_level=options["level"], + descriptor=options["descriptor"], + start_time=options["start_date"].strftime("%Y%m%d"), + version=options["version"], + ) + + return (science_file.filename, science_file.construct_path()) + + def upload(self, file_name: str) -> None: + """Upload data to imap-data-access.""" + + logging.debug(f"Uploading {file_name} to imap-data-access.") + + try: + imap_data_access.upload(file_name) + except imap_data_access.io.IMAPDataAccessError as e: + logging.warn(f"Upload failed: {e}") + + def unique_version( + self, **options: typing.Unpack[VersionOptions] + ) -> tuple[str, str | None]: + """Determine a unique version for the data by querying imap_data_access.""" + + files: list[dict[str, str]] = self.query( + **options, + descriptor=None, + end_date=options["start_date"], + version=None, + extension=None, + ) + + if not files: + logging.debug(f"No existing files found for {options}.") + return ("v000", None) + + max_version: str = max(files, key=lambda x: x["version"])["version"] + unique_version: str = f"v{int(max_version[1:]) + 1:03d}" + + logging.debug( + f"Existing files found, using: unique={unique_version}, " + f"previous={max_version}." + ) + + return (unique_version, max_version) + + def query(self, **options: typing.Unpack[QueryOptions]) -> list[dict[str, str]]: + """Download data from imap-data-access.""" + + return imap_data_access.query( + instrument="mag", + data_level=options["level"], + descriptor=options["descriptor"], + start_date=( + options["start_date"].strftime("%Y%m%d") + if options["start_date"] + else None + ), + end_date=( + options["end_date"].strftime("%Y%m%d") if options["end_date"] else None + ), + version=options["version"], + extension=options["extension"], + ) + + def get_filename( + self, **options: typing.Unpack[FileOptions] + ) -> list[dict[str, str]] | None: + science_file = imap_data_access.ScienceFilePath.generate_from_inputs( + instrument="mag", + data_level=options["level"], + descriptor=options["descriptor"], + start_time=options["start_date"].strftime("%Y%m%d"), + version=options["version"], + ) + + file_name: list[dict[str, str]] = self.query(**options) + + logging.info(f"File {science_file.filename} generated.") + + return file_name + + def download(self, file_name: str) -> pathlib.Path: + """Download data from imap-data-access.""" + + logging.debug(f"Downloading {file_name} from imap-data-access.") + + return pathlib.Path(imap_data_access.download(file_name)) diff --git a/src/time.py b/src/time.py new file mode 100644 index 0000000..f33d9ce --- /dev/null +++ b/src/time.py @@ -0,0 +1,34 @@ +"""Definitions of supported time formats.""" + +from datetime import datetime + +import astropy.time + + +class Time: + """Define time and conversions.""" + + date: datetime + gps: int + + def __init__(self, date: datetime, gsp: int) -> None: + """Initialize Time object.""" + + self.date = date + self.gps = gsp + + @staticmethod + def from_gps(gps: int): + """Create Time object from GSP time.""" + + date = astropy.time.Time(gps / 1e6, format="gps").datetime + + return Time(date, gps) + + @staticmethod + def from_datetime(date: datetime): + """Create Time object from datetime object.""" + + gps = astropy.time.Time(date).gps * 1e6 + + return Time(date, gps) From dd10dd26705f959ce18294972f9d2099ba18bb83 Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Sun, 28 Jul 2024 20:45:04 +0000 Subject: [PATCH 13/33] feat: copy files after download --- deploy/entrypoint.sh | 5 ++++- src/SDC.py | 12 ++++++++++-- src/main.py | 7 ++++++- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/deploy/entrypoint.sh b/deploy/entrypoint.sh index a77fd86..cbd71cc 100644 --- a/deploy/entrypoint.sh +++ b/deploy/entrypoint.sh @@ -4,11 +4,14 @@ set -e while : do - imap-mag hello world + ls -l /data imap-mag fetch-binary --config tests/config/hk_download.yaml --apid 1063 --start-date 2025-05-02 --end-date 2025-05-03 + imap-mag process --config tests/config/hk_process.yaml MAG_HSK_PW.pkts + imap-mag fetch-science --start-date 2025-05-02 --end-date 2025-05-03 + ls -l /data sleep 3600 # 1 Hour diff --git a/src/SDC.py b/src/SDC.py index c275116..088e30a 100644 --- a/src/SDC.py +++ b/src/SDC.py @@ -3,6 +3,7 @@ import logging import typing from datetime import datetime +from pathlib import Path import pandas as pd @@ -45,8 +46,11 @@ def __init__(self, data_access: ISDCApiClient) -> None: self.__data_access = data_access - def QueryAndDownload(self, **options: typing.Unpack[SDCOptions]) -> None: + def QueryAndDownload(self, **options: typing.Unpack[SDCOptions]) -> list[Path]: """Retrieve SDC data.""" + + downloaded = [] + for details in self.__PACKET_DETAILS: (start, end) = self.__extract_time_range( options["start_date"], @@ -83,7 +87,11 @@ def QueryAndDownload(self, **options: typing.Unpack[SDCOptions]) -> None: if files is not None: for file in files: - self.__data_access.download(file["file_path"]) + downloaded += [ + self.__data_access.download(file["file_path"]) + ] + + return downloaded def __extract_time_range(self, start_date: str, end_date: str) -> tuple[Time, Time]: """Extract time range as S/C and ERT Time object.""" diff --git a/src/main.py b/src/main.py index 05ae10f..51ff36b 100644 --- a/src/main.py +++ b/src/main.py @@ -211,10 +211,15 @@ def fetch_science( sdc = SDC.SDC(data_access) # TODO: any better way than passing a dictionary? Strongly typed? - sdc.QueryAndDownload( + files = sdc.QueryAndDownload( level=level, start_date=start_date, end_date=end_date, force=force ) + # TODO: save the files to the database + + for file in files: + appUtils.copyFileToDestination(file, configFile.destination) + @app.callback() def main(verbose: Annotated[bool, typer.Option("--verbose", "-v")] = False): From 0243b41e8e03022f1337f419d74042cefda8e77f Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Sun, 28 Jul 2024 21:57:33 +0000 Subject: [PATCH 14/33] feat: save new files to db after downoad --- config-hk-download.yaml | 9 + config-hk-process.yaml | 11 + config-sci.yml | 4 + deploy/entrypoint.sh | 15 +- src/imap_db/main.py | 3 +- src/imap_mag/DB.py | 38 +++ src/imap_mag/main.py | 471 +++++++++++++++++----------------- tests/config/hk_download.yaml | 2 +- 8 files changed, 315 insertions(+), 238 deletions(-) create mode 100644 config-hk-download.yaml create mode 100644 config-hk-process.yaml create mode 100644 config-sci.yml create mode 100644 src/imap_mag/DB.py diff --git a/config-hk-download.yaml b/config-hk-download.yaml new file mode 100644 index 0000000..dc7ef88 --- /dev/null +++ b/config-hk-download.yaml @@ -0,0 +1,9 @@ + +work-folder: /data/.work + +destination: + folder: /data/hk_l0/ + filename: power.pkts + +packet-definition: + hk: src/imap_mag/xtce/tlm_20240724.xml diff --git a/config-hk-process.yaml b/config-hk-process.yaml new file mode 100644 index 0000000..9f4fc4b --- /dev/null +++ b/config-hk-process.yaml @@ -0,0 +1,11 @@ +source: + folder: /data/hk_l0/ + +work-folder: /data/.work + +destination: + folder: /data/hk_l1/ + filename: result.csv + +packet-definition: + hk: src/imap_mag/xtce/tlm_20240724.xml diff --git a/config-sci.yml b/config-sci.yml new file mode 100644 index 0000000..fe58105 --- /dev/null +++ b/config-sci.yml @@ -0,0 +1,4 @@ + +work-folder: /data/science + + diff --git a/deploy/entrypoint.sh b/deploy/entrypoint.sh index fe72fd0..b25e911 100644 --- a/deploy/entrypoint.sh +++ b/deploy/entrypoint.sh @@ -1,20 +1,25 @@ #!/bin/bash set -e -echo "start DB admin" + imap-db create-db + imap-db upgrade-db + echo "DB admin complete" while : do - ls -l /data + # delete all data + rm -rf /data/* + + imap-mag fetch-binary --config config-hk-download.yaml --apid 1063 --start-date 2025-05-02 --end-date 2025-05-03 - imap-mag fetch-binary --config tests/config/hk_download.yaml --apid 1063 --start-date 2025-05-02 --end-date 2025-05-03 + imap-mag process --config config-hk-process.yaml MAG_HSK_PW.pkts - imap-mag process --config tests/config/hk_process.yaml MAG_HSK_PW.pkts + imap-mag fetch-science --start-date 2025-05-02 --end-date 2025-05-03 --config config-sci.yaml - imap-mag fetch-science --start-date 2025-05-02 --end-date 2025-05-03 + imap-db query-db ls -l /data diff --git a/src/imap_db/main.py b/src/imap_db/main.py index 2c29b75..45bd792 100644 --- a/src/imap_db/main.py +++ b/src/imap_db/main.py @@ -66,7 +66,8 @@ def drop_db(): def query_db(): session = Session(engine) - stmt = select(File).where(File.name.in_(["file1.txt"])) + # stmt = select(File).where(File.name.in_(["file1.txt"])) + stmt = select(File).where(File.name is not None) for user in session.scalars(stmt): print(user) diff --git a/src/imap_mag/DB.py b/src/imap_mag/DB.py new file mode 100644 index 0000000..9dff8f3 --- /dev/null +++ b/src/imap_mag/DB.py @@ -0,0 +1,38 @@ +import os + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from src.imap_db.model import File + + +class DB: + def __init__(self, db_url=None): + env_url = os.getenv("SQLALCHEMY_URL") + if db_url is None and env_url is not None: + db_url = env_url + + self.engine = create_engine(db_url) + self.Session = sessionmaker(bind=self.engine) + + def insert_files(self, files: list[File]): + session = self.Session() + try: + for file in files: + # check file does not already exist + existing_file = ( + session.query(File) + .filter_by(name=file.name, path=file.path) + .first() + ) + if existing_file is not None: + continue + + session.add(file) + + session.commit() + except Exception as e: + session.rollback() + raise e + finally: + session.close() diff --git a/src/imap_mag/main.py b/src/imap_mag/main.py index ec2d3c1..b0c2f29 100644 --- a/src/imap_mag/main.py +++ b/src/imap_mag/main.py @@ -1,231 +1,240 @@ -"""Main module.""" - -import logging -import os -import shutil -from datetime import datetime -from enum import Enum -from pathlib import Path -from typing import Annotated - -# cli -import typer - -# config -import yaml - -# app code -from . import SDC, appConfig, appLogging, appUtils, imapProcessing, webPODA -from .sdcApiClient import SDCApiClient - -app = typer.Typer() -globalState = {"verbose": False} - - -def commandInit(config: Path) -> appConfig.AppConfig: - # load and verify the config file - if config is None: - logging.critical("No config file") - raise typer.Abort() - if config.is_file(): - configFileDict = yaml.safe_load(open(config)) - logging.debug(f"Config file contents: {configFileDict}") - elif config.is_dir(): - logging.critical("Config is a directory, need a yml file") - raise typer.Abort() - elif not config.exists(): - logging.critical("The config doesn't exist") - raise typer.Abort() - else: - pass - - configFile = appConfig.AppConfig(**configFileDict) - - # set up the work folder - if not configFile.work_folder: - configFile.work_folder = Path(".work") - - if not os.path.exists(configFile.work_folder): - logging.debug(f"Creating work folder {configFile.work_folder}") - os.makedirs(configFile.work_folder) - - # initialise all logging into the workfile - level = "debug" if globalState["verbose"] else "info" - - # TODO: the log file loation should be configurable so we can keep the logs on RDS - # Or maybe just ship them there after the fact? Or log to both? - logFile = Path( - configFile.work_folder, - f"{datetime.now().strftime('%Y_%m_%d-%I_%M_%S_%p')}.log", - ) - if not appLogging.set_up_logging( - console_log_output="stdout", - console_log_level=level, - console_log_color=True, - logfile_file=logFile, - logfile_log_level="debug", - logfile_log_color=False, - log_line_template="%(color_on)s[%(asctime)s] [%(levelname)-8s] %(message)s%(color_off)s", - console_log_line_template="%(color_on)s%(message)s%(color_off)s", - ): - print("Failed to set up logging, aborting.") - raise typer.Abort() - - return configFile - - -@app.command() -def hello(name: str): - print(f"Hello {name}") - - -# E.g imap-mag process --config config.yml solo_L2_mag-rtn-ll-internal_20240210_V00.cdf -@app.command() -def process( - file: Annotated[ - str, typer.Argument(help="The file name or pattern to match for the input file") - ], - config: Annotated[Path, typer.Option()] = Path("config.yml"), -): - """Sample processing job.""" - # TODO: semantic logging - # TODO: handle file system/cloud files - abstraction layer needed for files - # TODO: move shared logic to a library - - configFile: appConfig.AppConfig = commandInit(config) - - logging.debug(f"Grabbing file matching {file} in {configFile.source.folder}") - - # get all files in \\RDS.IMPERIAL.AC.UK\rds\project\solarorbitermagnetometer\live\SO-MAG-Web\quicklooks_py\ - files = [] - folder = configFile.source.folder - - # if pattern contains a % - if "%" in file: - updatedFile = datetime.now().strftime(file) - logging.info(f"Pattern contains a %, replacing '{file} with {updatedFile}") - file = updatedFile - - # list all files in the share - for matchedFile in folder.iterdir(): - if matchedFile.is_file(): - if matchedFile.match(file): - files.append(matchedFile) - - # get the most recently modified matching file - files.sort(key=lambda f: f.stat().st_mtime, reverse=True) - - if len(files) == 0: - logging.critical(f"No files matching {file} found in {folder}") - raise typer.Abort() - - logging.info( - f"Found {len(files)} matching files. Select the most recent one:" - f"{files[0].absolute().as_posix()}" - ) - - # copy the file to configFile.work_folder - workFile = Path(configFile.work_folder, files[0].name) - logging.debug(f"Copying {files[0]} to {workFile}") - workFile = Path(shutil.copy2(files[0], configFile.work_folder)) - - # TODO: do something with the data! - fileProcessor = imapProcessing.dispatchFile(workFile) - fileProcessor.initialize(configFile) - result = fileProcessor.process(workFile) - - appUtils.copyFileToDestination(result, configFile.destination) - - -# E.g., imap-mag fetch-binary --apid 1063 --start-date 2025-05-02 --end-date 2025-05-03 -@app.command() -def fetch_binary( - auth_code: Annotated[ - str, - typer.Option( - envvar="WEBPODA_AUTH_CODE", - help="WebPODA authentication code", - ), - ], - apid: Annotated[int, typer.Option(help="ApID to download")], - start_date: Annotated[str, typer.Option(help="Start date for the download")], - end_date: Annotated[str, typer.Option(help="End date for the download")], - config: Annotated[Path, typer.Option()] = Path("config.yml"), -): - configFile: appConfig.AppConfig = commandInit(config) - - if not auth_code: - logging.critical("No WebPODA authorization code provided") - raise typer.Abort() - - packet: str = appUtils.getPacketFromApID(apid) - logging.info(f"Downloading raw packet {packet} from {start_date} to {end_date}.") - - poda = webPODA.WebPODA(auth_code, configFile.work_folder) - result: str = poda.download( - packet=packet, - start_date=appUtils.convertToDatetime(start_date), - end_date=appUtils.convertToDatetime(end_date), - ) - - appUtils.copyFileToDestination(result, configFile.destination) - - -class LevelEnum(str, Enum): - level_1 = "l1" - level_2 = "l2" - level_3 = "l3" - - -# E.g., imap-mag fetch-science --start-date 2025-05-02 --end-date 2025-05-03 -@app.command() -def fetch_science( - auth_code: Annotated[ - str, - typer.Option( - envvar="SDC_AUTH_CODE", - help="IMAP Science Data Centre API Key", - ), - ], - start_date: Annotated[str, typer.Option(help="Start date for the download")], - end_date: Annotated[str, typer.Option(help="End date for the download")], - level: Annotated[ - LevelEnum, typer.Option(help="Level to download") - ] = LevelEnum.level_2, - force: Annotated[ - bool, - typer.Option("--force", "-f", help="Force download even if the file exists"), - ] = False, - config: Annotated[Path, typer.Option()] = Path("config.yml"), -): - configFile: appConfig.AppConfig = commandInit(config) - - if not auth_code: - logging.critical("No SDC_AUTH_CODE API key provided") - raise typer.Abort() - - logging.info(f"Downloading {level} science from {start_date} to {end_date}.") - - data_access = SDCApiClient(data_dir=str(configFile.work_folder)) - - sdc = SDC.SDC(data_access) - - # TODO: any better way than passing a dictionary? Strongly typed? - files = sdc.QueryAndDownload( - level=level, start_date=start_date, end_date=end_date, force=force - ) - - # TODO: save the files to the database - - for file in files: - appUtils.copyFileToDestination(file, configFile.destination) - - -@app.callback() -def main(verbose: Annotated[bool, typer.Option("--verbose", "-v")] = False): - if verbose: - globalState["verbose"] = True - - -if __name__ == "__main__": - app() # pragma: no cover +"""Main module.""" + +import logging +import os +import shutil +from datetime import datetime +from enum import Enum +from pathlib import Path +from typing import Annotated + +# cli +import typer + +# config +import yaml + +from src.imap_db.model import File + +# app code +from . import DB, SDC, appConfig, appLogging, appUtils, imapProcessing, webPODA +from .sdcApiClient import SDCApiClient + +app = typer.Typer() +globalState = {"verbose": False} + + +def commandInit(config: Path) -> appConfig.AppConfig: + # load and verify the config file + if config is None: + logging.critical("No config file") + raise typer.Abort() + if config.is_file(): + configFileDict = yaml.safe_load(open(config)) + logging.debug(f"Config file contents: {configFileDict}") + elif config.is_dir(): + logging.critical("Config is a directory, need a yml file") + raise typer.Abort() + elif not config.exists(): + logging.critical("The config doesn't exist") + raise typer.Abort() + else: + pass + + configFile = appConfig.AppConfig(**configFileDict) + + # set up the work folder + if not configFile.work_folder: + configFile.work_folder = Path(".work") + + if not os.path.exists(configFile.work_folder): + logging.debug(f"Creating work folder {configFile.work_folder}") + os.makedirs(configFile.work_folder) + + # initialise all logging into the workfile + level = "debug" if globalState["verbose"] else "info" + + # TODO: the log file loation should be configurable so we can keep the logs on RDS + # Or maybe just ship them there after the fact? Or log to both? + logFile = Path( + configFile.work_folder, + f"{datetime.now().strftime('%Y_%m_%d-%I_%M_%S_%p')}.log", + ) + if not appLogging.set_up_logging( + console_log_output="stdout", + console_log_level=level, + console_log_color=True, + logfile_file=logFile, + logfile_log_level="debug", + logfile_log_color=False, + log_line_template="%(color_on)s[%(asctime)s] [%(levelname)-8s] %(message)s%(color_off)s", + console_log_line_template="%(color_on)s%(message)s%(color_off)s", + ): + print("Failed to set up logging, aborting.") + raise typer.Abort() + + return configFile + + +@app.command() +def hello(name: str): + print(f"Hello {name}") + + +# E.g imap-mag process --config config.yml solo_L2_mag-rtn-ll-internal_20240210_V00.cdf +@app.command() +def process( + file: Annotated[ + str, typer.Argument(help="The file name or pattern to match for the input file") + ], + config: Annotated[Path, typer.Option()] = Path("config.yml"), +): + """Sample processing job.""" + # TODO: semantic logging + # TODO: handle file system/cloud files - abstraction layer needed for files + # TODO: move shared logic to a library + + configFile: appConfig.AppConfig = commandInit(config) + + logging.debug(f"Grabbing file matching {file} in {configFile.source.folder}") + + # get all files in \\RDS.IMPERIAL.AC.UK\rds\project\solarorbitermagnetometer\live\SO-MAG-Web\quicklooks_py\ + files = [] + folder = configFile.source.folder + + # if pattern contains a % + if "%" in file: + updatedFile = datetime.now().strftime(file) + logging.info(f"Pattern contains a %, replacing '{file} with {updatedFile}") + file = updatedFile + + # list all files in the share + for matchedFile in folder.iterdir(): + if matchedFile.is_file(): + if matchedFile.match(file): + files.append(matchedFile) + + # get the most recently modified matching file + files.sort(key=lambda f: f.stat().st_mtime, reverse=True) + + if len(files) == 0: + logging.critical(f"No files matching {file} found in {folder}") + raise typer.Abort() + + logging.info( + f"Found {len(files)} matching files. Select the most recent one:" + f"{files[0].absolute().as_posix()}" + ) + + # copy the file to configFile.work_folder + workFile = Path(configFile.work_folder, files[0].name) + logging.debug(f"Copying {files[0]} to {workFile}") + workFile = Path(shutil.copy2(files[0], configFile.work_folder)) + + # TODO: do something with the data! + fileProcessor = imapProcessing.dispatchFile(workFile) + fileProcessor.initialize(configFile) + result = fileProcessor.process(workFile) + + appUtils.copyFileToDestination(result, configFile.destination) + + +# E.g., imap-mag fetch-binary --apid 1063 --start-date 2025-05-02 --end-date 2025-05-03 +@app.command() +def fetch_binary( + auth_code: Annotated[ + str, + typer.Option( + envvar="WEBPODA_AUTH_CODE", + help="WebPODA authentication code", + ), + ], + apid: Annotated[int, typer.Option(help="ApID to download")], + start_date: Annotated[str, typer.Option(help="Start date for the download")], + end_date: Annotated[str, typer.Option(help="End date for the download")], + config: Annotated[Path, typer.Option()] = Path("config.yml"), +): + configFile: appConfig.AppConfig = commandInit(config) + + if not auth_code: + logging.critical("No WebPODA authorization code provided") + raise typer.Abort() + + packet: str = appUtils.getPacketFromApID(apid) + logging.info(f"Downloading raw packet {packet} from {start_date} to {end_date}.") + + poda = webPODA.WebPODA(auth_code, configFile.work_folder) + result: str = poda.download( + packet=packet, + start_date=appUtils.convertToDatetime(start_date), + end_date=appUtils.convertToDatetime(end_date), + ) + + appUtils.copyFileToDestination(result, configFile.destination) + + +class LevelEnum(str, Enum): + level_1 = "l1" + level_2 = "l2" + level_3 = "l3" + + +# E.g., imap-mag fetch-science --start-date 2025-05-02 --end-date 2025-05-03 +@app.command() +def fetch_science( + auth_code: Annotated[ + str, + typer.Option( + envvar="SDC_AUTH_CODE", + help="IMAP Science Data Centre API Key", + ), + ], + start_date: Annotated[str, typer.Option(help="Start date for the download")], + end_date: Annotated[str, typer.Option(help="End date for the download")], + level: Annotated[ + LevelEnum, typer.Option(help="Level to download") + ] = LevelEnum.level_2, + force: Annotated[ + bool, + typer.Option("--force", "-f", help="Force download even if the file exists"), + ] = False, + config: Annotated[Path, typer.Option()] = Path("config-sci.yml"), +): + configFile: appConfig.AppConfig = commandInit(config) + + if not auth_code: + logging.critical("No SDC_AUTH_CODE API key provided") + raise typer.Abort() + + logging.info(f"Downloading {level} science from {start_date} to {end_date}.") + + data_access = SDCApiClient(data_dir=str(configFile.work_folder)) + + sdc = SDC.SDC(data_access) + + # TODO: any better way than passing a dictionary? Strongly typed? + files = sdc.QueryAndDownload( + level=level, start_date=start_date, end_date=end_date, force=force + ) + + records = [] + for file in files: + records.append(File(name=file.name, path=file.absolute().as_posix())) + + db = DB() + db.insert_files(records) + + logging.info(f"Downloaded {len(files)} files and saved to database") + + # for file in files: + # appUtils.copyFileToDestination(file, configFile.destination) + + +@app.callback() +def main(verbose: Annotated[bool, typer.Option("--verbose", "-v")] = False): + if verbose: + globalState["verbose"] = True + + +if __name__ == "__main__": + app() # pragma: no cover diff --git a/tests/config/hk_download.yaml b/tests/config/hk_download.yaml index 43411a1..f155116 100644 --- a/tests/config/hk_download.yaml +++ b/tests/config/hk_download.yaml @@ -8,4 +8,4 @@ destination: filename: power.pkts packet-definition: - hk: src/xtce/tlm_20240724.xml + hk: src/imap_mag/xtce/tlm_20240724.xml From 95da0eddfe7624d6725b22838a6631015622a1de Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Sun, 28 Jul 2024 23:14:13 +0100 Subject: [PATCH 15/33] bug: fix typing.Unpack in py3.10 --- src/imap_mag/SDC.py | 5 +++-- src/imap_mag/sdcApiClient.py | 17 +++++++++-------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/imap_mag/SDC.py b/src/imap_mag/SDC.py index e655470..00c2796 100644 --- a/src/imap_mag/SDC.py +++ b/src/imap_mag/SDC.py @@ -6,6 +6,7 @@ from pathlib import Path import pandas as pd +from typing_extensions import Unpack from .sdcApiClient import ISDCApiClient from .time import Time @@ -46,7 +47,7 @@ def __init__(self, data_access: ISDCApiClient) -> None: self.__data_access = data_access - def QueryAndDownload(self, **options: typing.Unpack[SDCOptions]) -> list[Path]: + def QueryAndDownload(self, **options: Unpack[SDCOptions]) -> list[Path]: """Retrieve SDC data.""" downloaded = [] @@ -114,7 +115,7 @@ def __check_download_needed( self, details: dict, date: datetime, - **options: typing.Unpack[SDCOptions], + **options: Unpack[SDCOptions], ) -> bool: """Check if files need to be downloaded.""" diff --git a/src/imap_mag/sdcApiClient.py b/src/imap_mag/sdcApiClient.py index 1a4117e..1653ad3 100644 --- a/src/imap_mag/sdcApiClient.py +++ b/src/imap_mag/sdcApiClient.py @@ -7,6 +7,7 @@ from datetime import datetime import imap_data_access +from typing_extensions import Unpack class FileOptions(typing.TypedDict): @@ -41,7 +42,7 @@ class ISDCApiClient(abc.ABC): @staticmethod @abc.abstractmethod - def get_file_path(**options: typing.Unpack[FileOptions]) -> tuple[str, str]: + def get_file_path(**options: Unpack[FileOptions]) -> tuple[str, str]: """Get file path for data from imap-data-access.""" pass @@ -52,19 +53,19 @@ def upload(self, file_name: str) -> None: @abc.abstractmethod def unique_version( - self, **options: typing.Unpack[VersionOptions] + self, **options: Unpack[VersionOptions] ) -> tuple[str, str | None]: """Determine a unique version for the data by querying imap_data_access.""" pass @abc.abstractmethod - def query(self, **options: typing.Unpack[QueryOptions]) -> list[dict[str, str]]: + def query(self, **options: Unpack[QueryOptions]) -> list[dict[str, str]]: """Download data from imap-data-access.""" pass @abc.abstractmethod def get_filename( - self, **options: typing.Unpack[FileOptions] + self, **options: Unpack[FileOptions] ) -> list[dict[str, str]] | None: """Wait for file to be available in imap-data-access.""" pass @@ -82,7 +83,7 @@ def __init__(self, data_dir: str) -> None: imap_data_access.config["DATA_DIR"] = pathlib.Path(data_dir) @staticmethod - def get_file_path(**options: typing.Unpack[FileOptions]) -> tuple[str, str]: + def get_file_path(**options: Unpack[FileOptions]) -> tuple[str, str]: """Get file path for data from imap-data-access.""" science_file = imap_data_access.ScienceFilePath.generate_from_inputs( @@ -106,7 +107,7 @@ def upload(self, file_name: str) -> None: logging.warn(f"Upload failed: {e}") def unique_version( - self, **options: typing.Unpack[VersionOptions] + self, **options: Unpack[VersionOptions] ) -> tuple[str, str | None]: """Determine a unique version for the data by querying imap_data_access.""" @@ -132,7 +133,7 @@ def unique_version( return (unique_version, max_version) - def query(self, **options: typing.Unpack[QueryOptions]) -> list[dict[str, str]]: + def query(self, **options: Unpack[QueryOptions]) -> list[dict[str, str]]: """Download data from imap-data-access.""" return imap_data_access.query( @@ -152,7 +153,7 @@ def query(self, **options: typing.Unpack[QueryOptions]) -> list[dict[str, str]]: ) def get_filename( - self, **options: typing.Unpack[FileOptions] + self, **options: Unpack[FileOptions] ) -> list[dict[str, str]] | None: science_file = imap_data_access.ScienceFilePath.generate_from_inputs( instrument="mag", From 6af8fa4a6d7a08130482109153dc11c21d282bd8 Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Sun, 28 Jul 2024 22:17:41 +0000 Subject: [PATCH 16/33] task: whitespace tidy --- pyproject.toml | 148 ++++++++++++++++++++++++------------------------- 1 file changed, 74 insertions(+), 74 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 457c372..a3e34da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,74 +1,74 @@ -[project] -requires-python = ">=3.10" -name = "imap-mag" - -[tool.poetry] -name = "imap-mag" -version = "0.1.0" -description = "Process IMAP data" -authors = ["alastairtree"] -readme = "README.md" -packages = [ - { include = "src/imap_mag" }, - { include = "src/imap_db" }, -] - -[tool.poetry.dependencies] -python = ">=3.10,<3.13" -pyyaml = "^6.0.1" -typing-extensions = "^4.9.0" -pydantic = "^2.6.1" -space-packet-parser = "^4.2.0" -xarray = "^2024.6.0" -numpy = "^2.0.1" -typer = "^0.12.3" -sqlalchemy = "^2.0.31" -psycopg2 = "^2.9.9" -alembic = "^1.13.2" -sqlalchemy-utils = "^0.41.2" -requests = "^2.32.3" -pandas = "^2.2.2" -imap-data-access = "^0.7.0" -astropy = "^6.1.2" - -[tool.poetry.group.dev.dependencies] -pytest = "^8.3.1" -pytest-cov = "^5.0.0" -pyinstaller = "^6.5.0" -pre-commit = "^3.7.1" -ruff = "^0.5.4" - -[tool.poetry.scripts] -# can execute via poetry, e.g. `poetry run imap-mag hello world` -imap-mag = 'src.imap_mag.main:app' -imap-db = 'src.imap_db.main:app' - -[tool.pytest.ini_options] -pythonpath = [ - ".", "src" -] - -[build-system] -requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" - -[tool.poetry-pyinstaller-plugin.scripts] -imap-mag = { source = "src/imap_mag/main.py", type = "onefile", bundle = false } - -[tool.ruff.lint] -select = [ - "D", # pydocstyle - "E", # pycodestyle - "F", # Pyflakes - "I", # isort - "UP", # pyupgrade - "RUF" # ruff -] -pydocstyle.convention = "google" - -[tool.ruff.lint.per-file-ignores] -# Ignore: -# * Missing docstring in public module, class, method, function, package and __init__ -# * Line too long -# * Optional replaced as X | None -"*" = ["D100", "D101", "D102", "D103", "D104", "D107", "D202", "E501", "UP007"] +[project] +requires-python = ">=3.10" +name = "imap-mag" + +[tool.poetry] +name = "imap-mag" +version = "0.1.0" +description = "Process IMAP data" +authors = ["alastairtree"] +readme = "README.md" +packages = [ + { include = "src/imap_mag" }, + { include = "src/imap_db" }, +] + +[tool.poetry.dependencies] +python = ">=3.10,<3.13" +pyyaml = "^6.0.1" +typing-extensions = "^4.9.0" +pydantic = "^2.6.1" +space-packet-parser = "^4.2.0" +xarray = "^2024.6.0" +numpy = "^2.0.1" +typer = "^0.12.3" +sqlalchemy = "^2.0.31" +psycopg2 = "^2.9.9" +alembic = "^1.13.2" +sqlalchemy-utils = "^0.41.2" +requests = "^2.32.3" +pandas = "^2.2.2" +imap-data-access = "^0.7.0" +astropy = "^6.1.2" + +[tool.poetry.group.dev.dependencies] +pytest = "^8.3.1" +pytest-cov = "^5.0.0" +pyinstaller = "^6.5.0" +pre-commit = "^3.7.1" +ruff = "^0.5.4" + +[tool.poetry.scripts] +# can execute via poetry, e.g. `poetry run imap-mag hello world` +imap-mag = 'src.imap_mag.main:app' +imap-db = 'src.imap_db.main:app' + +[tool.pytest.ini_options] +pythonpath = [ + ".", "src" +] + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry-pyinstaller-plugin.scripts] +imap-mag = { source = "src/imap_mag/main.py", type = "onefile", bundle = false } + +[tool.ruff.lint] +select = [ + "D", # pydocstyle + "E", # pycodestyle + "F", # Pyflakes + "I", # isort + "UP", # pyupgrade + "RUF" # ruff +] +pydocstyle.convention = "google" + +[tool.ruff.lint.per-file-ignores] +# Ignore: +# * Missing docstring in public module, class, method, function, package and __init__ +# * Line too long +# * Optional replaced as X | None +"*" = ["D100", "D101", "D102", "D103", "D104", "D107", "D202", "E501", "UP007"] From 780590cc8d7b4321e86d5a44a031b3ab46f1b527 Mon Sep 17 00:00:00 2001 From: mhairifin Date: Mon, 29 Jul 2024 09:36:23 +0100 Subject: [PATCH 17/33] carry over toolkit library code for use by pipeline --- src/mag_toolkit/CDFLoader.py | 16 ++++ src/mag_toolkit/__init__.py | 0 .../calibration/CalibrationApplicator.py | 46 +++++++++++ .../calibration/CalibrationExceptions.py | 2 + src/mag_toolkit/calibration/Calibrator.py | 78 +++++++++++++++++++ src/mag_toolkit/calibration/MatlabWrapper.py | 16 ++++ .../calibration/calibrationFormat.py | 48 ++++++++++++ .../calibration/calibrationFormatProcessor.py | 51 ++++++++++++ 8 files changed, 257 insertions(+) create mode 100644 src/mag_toolkit/CDFLoader.py create mode 100644 src/mag_toolkit/__init__.py create mode 100644 src/mag_toolkit/calibration/CalibrationApplicator.py create mode 100644 src/mag_toolkit/calibration/CalibrationExceptions.py create mode 100644 src/mag_toolkit/calibration/Calibrator.py create mode 100644 src/mag_toolkit/calibration/MatlabWrapper.py create mode 100644 src/mag_toolkit/calibration/calibrationFormat.py create mode 100644 src/mag_toolkit/calibration/calibrationFormatProcessor.py diff --git a/src/mag_toolkit/CDFLoader.py b/src/mag_toolkit/CDFLoader.py new file mode 100644 index 0000000..d572bed --- /dev/null +++ b/src/mag_toolkit/CDFLoader.py @@ -0,0 +1,16 @@ +from pathlib import Path + +from cdflib import xarray + + +def load_cdf(inputPath: Path): + """Wraps cdlibs xarray reader.""" + if inputPath.is_file(): + return xarray.cdf_to_xarray(inputPath) + else: + raise FileExistsError() + + +def write_cdf(dataset, outputPath: Path): + """Wraps cdflib xarray writer.""" + xarray.xarray_to_cdf(dataset, outputPath) diff --git a/src/mag_toolkit/__init__.py b/src/mag_toolkit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/mag_toolkit/calibration/CalibrationApplicator.py b/src/mag_toolkit/calibration/CalibrationApplicator.py new file mode 100644 index 0000000..6c5c7d2 --- /dev/null +++ b/src/mag_toolkit/calibration/CalibrationApplicator.py @@ -0,0 +1,46 @@ +import logging +from pathlib import Path + +import numpy as np + +from src.mag_toolkit.calibration.CalibrationExceptions import CalibrationValidityError +from src.mag_toolkit.calibration.calibrationFormat import CalibrationFormat +from src.mag_toolkit.calibration.calibrationFormatProcessor import ( + CalibrationFormatProcessor, +) +from src.mag_toolkit.CDFLoader import load_cdf, write_cdf + + +class CalibrationApplicator: + def apply(self, calibrationFile, dataFile, outputFile) -> Path: + """Currently operating on unprocessed data.""" + data = load_cdf(dataFile) + calibrationCollection: CalibrationFormat = ( + CalibrationFormatProcessor.loadFromPath(calibrationFile) + ) + + logging.info("Loaded calibration file and data file") + + try: + self.checkValidity(data, calibrationCollection) + except CalibrationValidityError as e: + logging.info(f"{e} -> continuing application of calibration regardless") + + logging.info("Dataset and calibration file deemed compatible") + + for eachCal in calibrationCollection.calibrations: + data.vectors[0] = data.vectors[0] + eachCal.offsets.X + data.vectors[1] = data.vectors[1] + eachCal.offsets.Y + data.vectors[2] = data.vectors[2] + eachCal.offsets.Z + + write_cdf(data, outputFile) + + return outputFile + + def checkValidity(self, data, calibrationCollection): + # check for time validity + if data.epoch[0] < np.datetime64( + calibrationCollection.valid_start + ) or data.epoch[1] > np.datetime64(calibrationCollection.valid_end): + logging.debug("Data outside of calibration validity range") + raise CalibrationValidityError("Data outside of calibration validity range") diff --git a/src/mag_toolkit/calibration/CalibrationExceptions.py b/src/mag_toolkit/calibration/CalibrationExceptions.py new file mode 100644 index 0000000..6b2db9b --- /dev/null +++ b/src/mag_toolkit/calibration/CalibrationExceptions.py @@ -0,0 +1,2 @@ +class CalibrationValidityError(Exception): + pass diff --git a/src/mag_toolkit/calibration/Calibrator.py b/src/mag_toolkit/calibration/Calibrator.py new file mode 100644 index 0000000..194b9ec --- /dev/null +++ b/src/mag_toolkit/calibration/Calibrator.py @@ -0,0 +1,78 @@ +from abc import ABC, abstractmethod +from datetime import datetime +from enum import Enum + +import numpy as np + +from src.mag_toolkit.calibration import MatlabWrapper +from src.mag_toolkit.calibration.calibrationFormat import ( + CalibrationFormat, + OffsetCollection, + SingleCalibration, + Unit, +) + + +class CalibratorType(str, Enum): + SPINAXIS = ("SpinAxisCalibrator",) + SPINPLANE = "SpinPlaneCalibrator" + + +class Calibrator(ABC): + def generateOffsets(self, data) -> OffsetCollection: + """Generates a set of offsets.""" + (timestamps, x_offsets, y_offsets, z_offsets) = self.runCalibration(data) + + offsetCollection = OffsetCollection(X=x_offsets, Y=y_offsets, Z=z_offsets) + + sensor_name = "MAGO" + + singleCalibration = SingleCalibration( + timestamps=timestamps, + offsets=offsetCollection, + units=Unit.NT, + instrument=sensor_name, + creation_timestamp=datetime.now(), + method=str(self.name), + ) + return singleCalibration + + def generateCalibration(self, data) -> CalibrationFormat: + singleCalibration = self.generateOffsets(data) + calibration = CalibrationFormat( + valid_start=singleCalibration.timestamps[0], + valid_end=singleCalibration.timestamps[-1], + calibrations=[singleCalibration], + ) + return calibration + + @abstractmethod + def runCalibration(self, data): + """Calibration that generates offsets and timestamps.""" + + +class SpinAxisCalibrator(Calibrator): + def __init__(self): + self.name = CalibratorType.SPINAXIS + + def runCalibration(self, data): + (timestamps, z_offsets) = MatlabWrapper.simulateSpinAxisCalibration(data) + + return ( + timestamps, + np.zeros(len(z_offsets)), + np.zeros(len(z_offsets)), + z_offsets, + ) + + +class SpinPlaneCalibrator(Calibrator): + def __init__(self): + self.name = CalibratorType.SPINPLANE + + def runCalibration(self, data): + (timestamps, x_offsets, y_offsets) = ( + MatlabWrapper.simulateSpinPlaneCalibration() + ) + + return (timestamps, x_offsets, y_offsets, np.zeros(len(x_offsets))) diff --git a/src/mag_toolkit/calibration/MatlabWrapper.py b/src/mag_toolkit/calibration/MatlabWrapper.py new file mode 100644 index 0000000..33402a5 --- /dev/null +++ b/src/mag_toolkit/calibration/MatlabWrapper.py @@ -0,0 +1,16 @@ +from datetime import datetime + + +def simulateSpinAxisCalibration(xarray): + timestamps = [datetime(2022, 3, 3)] + offsets = [3.256] + + return (timestamps, offsets) + + +def simulateSpinPlaneCalibration(xarray): + timestamps = [datetime(2022, 3, 3)] + offsets_x = [3.256] + offsets_y = [2.76] + + return (timestamps, offsets_x, offsets_y) diff --git a/src/mag_toolkit/calibration/calibrationFormat.py b/src/mag_toolkit/calibration/calibrationFormat.py new file mode 100644 index 0000000..a07494e --- /dev/null +++ b/src/mag_toolkit/calibration/calibrationFormat.py @@ -0,0 +1,48 @@ +from datetime import datetime +from enum import Enum +from typing import Optional, Self + +from pydantic import BaseModel, model_validator + + +class Unit(Enum): + NT = "nT" + UT = "uT" + T = "T" + + +class Instrument(Enum): + MAGO = "MAGO" + MAGI = "MAGI" + + +class OffsetCollection(BaseModel): + X: list[float] + Y: list[float] + Z: list[float] + + @model_validator(mode="after") + def check_lengths_match(self) -> Self: + if ( + len(self.X) != len(self.Y) + or len(self.Y) != len(self.Z) + or len(self.X) != len(self.Z) + ): + raise ValueError("Length of offset lists do not match") + return self + + +class SingleCalibration(BaseModel): + timestamps: list[datetime] + offsets: OffsetCollection + units: Unit + instrument: Instrument + creation_timestamp: datetime + method: str + comment: Optional[str] = None + + +class CalibrationFormat(BaseModel): + valid_start: datetime + valid_end: datetime + calibrations: list[SingleCalibration] diff --git a/src/mag_toolkit/calibration/calibrationFormatProcessor.py b/src/mag_toolkit/calibration/calibrationFormatProcessor.py new file mode 100644 index 0000000..fa53d1a --- /dev/null +++ b/src/mag_toolkit/calibration/calibrationFormatProcessor.py @@ -0,0 +1,51 @@ +import os +from pathlib import Path + +import yaml +from pydantic import ValidationError + +from src.mag_toolkit.calibration.calibrationFormat import CalibrationFormat + + +class CalibrationFormatProcessor: + def loadFromPath(calibrationPath: Path) -> CalibrationFormat: + try: + as_dict = yaml.safe_load(open(calibrationPath)) + model = CalibrationFormat(**as_dict) + return model + except ValidationError as e: + print(e) + return None + except FileNotFoundError as e: + print(e) + return None + + def loadFromDict(calibrationDict: dict) -> CalibrationFormat: + try: + model = CalibrationFormat(**calibrationDict) + return model + except ValidationError as e: + print(e) + return None + + def getWriteable(CalibrationFormat: CalibrationFormat): + json = CalibrationFormat.model_dump_json() + + return json + + def writeToFile( + CalibrationFormat: CalibrationFormat, filepath: Path, createDirectory=False + ): + json = CalibrationFormat.model_dump_json() + + if createDirectory: + os.makedirs(os.path.dirname(filepath), exist_ok=True) + + try: + with open(filepath, "w+") as f: + f.write(json) + except Exception as e: + print(e) + print(f"Failed to write calibration to {filepath}") + + return filepath From 38709366528b6b6b8b7a18d57a3b30b0eb9c8e2d Mon Sep 17 00:00:00 2001 From: mhairifin Date: Mon, 29 Jul 2024 09:38:28 +0100 Subject: [PATCH 18/33] add calibrate and apply commands to command line tool --- src/main.py | 108 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 93 insertions(+), 15 deletions(-) diff --git a/src/main.py b/src/main.py index 4024089..8f0a6c6 100644 --- a/src/main.py +++ b/src/main.py @@ -15,6 +15,17 @@ # app code from src import appConfig, appLogging, imapProcessing +from src.mag_toolkit import CDFLoader +from src.mag_toolkit.calibration.CalibrationApplicator import CalibrationApplicator +from src.mag_toolkit.calibration.calibrationFormatProcessor import ( + CalibrationFormatProcessor, +) +from src.mag_toolkit.calibration.Calibrator import ( + Calibrator, + CalibratorType, + SpinAxisCalibrator, + SpinPlaneCalibrator, +) app = typer.Typer() globalState = {"verbose": False} @@ -77,21 +88,7 @@ def hello(name: str): print(f"Hello {name}") -# E.g imap-mag process --config config.yml solo_L2_mag-rtn-ll-internal_20240210_V00.cdf -@app.command() -def process( - config: Annotated[Path, typer.Option()] = Path("config.yml"), - file: str = typer.Argument( - help="The file name or pattern to match for the input file" - ), -): - """Sample processing job.""" - # TODO: semantic logging - # TODO: handle file system/cloud files - abstraction layer needed for files - # TODO: move shared logic to a library - - configFile: appConfig.AppConfig = commandInit(config) - +def prepareWorkFile(file, configFile): logging.debug(f"Grabbing file matching {file} in {configFile.source.folder}") # get all files in \\RDS.IMPERIAL.AC.UK\rds\project\solarorbitermagnetometer\live\SO-MAG-Web\quicklooks_py\ @@ -127,11 +124,35 @@ def process( logging.debug(f"Copying {files[0]} to {workFile}") workFile = Path(shutil.copy2(files[0], configFile.work_folder)) + return workFile + + +# E.g imap-mag process --config config.yml solo_L2_mag-rtn-ll-internal_20240210_V00.cdf +@app.command() +def process( + config: Annotated[Path, typer.Option()] = Path("config.yml"), + file: str = typer.Argument( + help="The file name or pattern to match for the input file" + ), +): + """Sample processing job.""" + # TODO: semantic logging + # TODO: handle file system/cloud files - abstraction layer needed for files + # TODO: move shared logic to a library + + configFile: appConfig.AppConfig = commandInit(config) + + workFile = prepareWorkFile(file, configFile) + # TODO: do something with the data! fileProcessor = imapProcessing.dispatchFile(workFile) fileProcessor.initialize(configFile) result = fileProcessor.process(workFile) + copyFromWorkArea(result, configFile) + + +def copyFromWorkArea(result, configFile): # copy the result to the destination destinationFile = Path(configFile.destination.folder) @@ -148,6 +169,63 @@ def process( logging.info(f"Copy complete: {completed}") +# imap-mag calibrate --config calibration_config.yml --method SpinAxisCalibrator imap_mag_l1b_norm-mago_20250502_v000.cdf +@app.command() +def calibrate( + config: Annotated[Path, typer.Option()] = Path("calibration_config.yml"), + method: Annotated[CalibratorType, typer.Option()] = "SpinAxisCalibrator", + input: str = typer.Argument( + help="The file name or pattern to match for the input file" + ), +): + # TODO: Define specific calibration configuration + # Using AppConfig for now to piggyback off of configuration + # verification and work area setup + configFile: appConfig.AppConfig = commandInit(config) + + workFile = prepareWorkFile(input, configFile) + calibrator: Calibrator + + match method: + case CalibratorType.SPINAXIS: + calibrator = SpinAxisCalibrator() + case CalibratorType.SPINPLANE: + calibrator = SpinPlaneCalibrator() + + inputData = CDFLoader.load_cdf(workFile) + calibration = calibrator.generateCalibration(inputData) + + tempOutputFile = os.path.join(configFile.work_folder, "calibration.json") + + result = CalibrationFormatProcessor.writeToFile(calibration, tempOutputFile) + + copyFromWorkArea(result, configFile) + + +# imap-mag apply --config calibration_application_config.yml --calibration calibration.json imap_mag_l1a_norm-mago_20250502_v000.cdf +@app.command() +def apply( + config: Annotated[Path, typer.Option()] = Path( + "calibration_application_config.yml" + ), + calibration: Annotated[str, typer.Option()] = "calibration.json", + input: str = typer.Argument( + help="The file name or pattern to match for the input file" + ), +): + configFile: appConfig.AppConfig = commandInit(config) + + workDataFile = prepareWorkFile(input, configFile) + workCalibrationFile = prepareWorkFile(calibration, configFile) + workOutputFile = os.path.join(configFile.work_folder, "l2_data.cdf") + + applier = CalibrationApplicator() + + L2_file = applier.apply(workCalibrationFile, workDataFile, workOutputFile) + + copyFromWorkArea(L2_file, configFile) + + @app.callback() def main(verbose: Annotated[bool, typer.Option("--verbose", "-v")] = False): if verbose: From 11ece9e049c20315e00e2419d2445ad0587b04b3 Mon Sep 17 00:00:00 2001 From: mhairifin Date: Mon, 29 Jul 2024 09:39:39 +0100 Subject: [PATCH 19/33] add tests and test data --- calibration_application_config.yml | 8 +++++++ calibration_config.yml | 8 +++++++ poetry.lock | 21 ++++++++++++++++- pyproject.toml | 1 + tests/data/2025/calibration.json | 27 ++++++++++++++++++++++ tests/test_main.py | 36 ++++++++++++++++++++++++++++++ 6 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 calibration_application_config.yml create mode 100644 calibration_config.yml create mode 100644 tests/data/2025/calibration.json diff --git a/calibration_application_config.yml b/calibration_application_config.yml new file mode 100644 index 0000000..b748acb --- /dev/null +++ b/calibration_application_config.yml @@ -0,0 +1,8 @@ +source: + folder: tests/data/2025 + +work-folder: .work + +destination: + folder: output/ + filename: L2.cdf \ No newline at end of file diff --git a/calibration_config.yml b/calibration_config.yml new file mode 100644 index 0000000..5ed5230 --- /dev/null +++ b/calibration_config.yml @@ -0,0 +1,8 @@ +source: + folder: tests/data/2025 + +work-folder: .work + +destination: + folder: output/ + filename: calibration.json \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index e0898ca..b9e899e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -167,6 +167,25 @@ files = [ [package.dependencies] bitarray = ">=2.9.0,<3.0.0" +[[package]] +name = "cdflib" +version = "1.3.1" +description = "A python CDF reader toolkit" +optional = false +python-versions = ">=3.8" +files = [ + {file = "cdflib-1.3.1-py3-none-any.whl", hash = "sha256:115494bfffd23b92c41629d5fcfdefab77112a4d9d1ff6023f87c5444d2e9aeb"}, + {file = "cdflib-1.3.1.tar.gz", hash = "sha256:7bb296e02ac7c47536c43bcc20d24796bfd9d4ec39490ec74953972203f26913"}, +] + +[package.dependencies] +numpy = ">=1.21" + +[package.extras] +dev = ["ipython", "pre-commit"] +docs = ["astropy", "netcdf4", "sphinx", "sphinx-automodapi", "sphinx-copybutton", "sphinx-rtd-theme", "xarray"] +tests = ["astropy", "h5netcdf", "hypothesis", "netcdf4", "pytest (>=3.9)", "pytest-cov", "pytest-remotedata", "xarray"] + [[package]] name = "cfgv" version = "3.4.0" @@ -1143,4 +1162,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "227eab6dfbfac71e0c187a7fbbe43a18f858e995f972a76dfe8cb92cc7aea57e" +content-hash = "84457c9f3b3266abe03dcf3e643897d5edacc5400d9d35fb59673636bd83104d" diff --git a/pyproject.toml b/pyproject.toml index fd7120b..88752e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ space-packet-parser = "^4.2.0" xarray = "^2024.6.0" numpy = "^2.0.1" typer = "^0.12.3" +cdflib = "^1.3.1" [tool.poetry.group.dev.dependencies] pytest = "^8.3.1" diff --git a/tests/data/2025/calibration.json b/tests/data/2025/calibration.json new file mode 100644 index 0000000..9127c29 --- /dev/null +++ b/tests/data/2025/calibration.json @@ -0,0 +1,27 @@ +{ + "calibrations": [ + { + "comment": null, + "creation_timestamp": "2024-07-26T16:51:58.212401", + "instrument": "MAGO", + "method": "CalibratorType.SPINAXIS", + "offsets": { + "X": [ + 0.0 + ], + "Y": [ + 0.0 + ], + "Z": [ + 3.256 + ] + }, + "timestamps": [ + "2022-03-03T00:00:00" + ], + "units": "nT" + } + ], + "valid_end": "2022-03-03T00:00:00", + "valid_start": "2022-03-03T00:00:00" +} diff --git a/tests/test_main.py b/tests/test_main.py index d1ac94e..69b4b5f 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -74,3 +74,39 @@ def test_process_with_binary_hk_converts_to_csv(tidyDataFolders): assert expectedFirstLine == lines[1] assert expectedLastLine == lines[-1] assert expectedNumRows == len(lines) + + +def test_calibration_creates_calibration_file(): + result = runner.invoke( + app, + [ + "calibrate", + "--config", + "calibration_config.yml", + "--method", + "SpinAxisCalibrator", + "imap_mag_l1a_norm-mago_20250502_v000.cdf", + ], + ) + + print("\n" + str(result.stdout)) + assert result.exit_code == 0 + assert Path("output/calibration.json").exists() + + +def test_application_creates_L2_file(): + result = runner.invoke( + app, + [ + "apply", + "--config", + "calibration_application_config.yml", + "--calibration", + "calibration.json", + "imap_mag_l1a_norm-mago_20250502_v000.cdf", + ], + ) + + print("\n" + str(result.stdout)) + assert result.exit_code == 0 + assert Path("output/L2.cdf").exists() From 4dc9b0cfa54a24d1c75f102ad70386b0f1eeafe9 Mon Sep 17 00:00:00 2001 From: mhairifin Date: Mon, 29 Jul 2024 11:18:16 +0100 Subject: [PATCH 20/33] resolve typing issues --- src/mag_toolkit/calibration/Calibrator.py | 29 ++++++++----------- src/mag_toolkit/calibration/MatlabWrapper.py | 28 ++++++++++++++++-- .../calibration/calibrationFormat.py | 4 +-- 3 files changed, 39 insertions(+), 22 deletions(-) diff --git a/src/mag_toolkit/calibration/Calibrator.py b/src/mag_toolkit/calibration/Calibrator.py index 194b9ec..75b857b 100644 --- a/src/mag_toolkit/calibration/Calibrator.py +++ b/src/mag_toolkit/calibration/Calibrator.py @@ -2,8 +2,6 @@ from datetime import datetime from enum import Enum -import numpy as np - from src.mag_toolkit.calibration import MatlabWrapper from src.mag_toolkit.calibration.calibrationFormat import ( CalibrationFormat, @@ -14,21 +12,25 @@ class CalibratorType(str, Enum): - SPINAXIS = ("SpinAxisCalibrator",) + SPINAXIS = "SpinAxisCalibrator" SPINPLANE = "SpinPlaneCalibrator" class Calibrator(ABC): def generateOffsets(self, data) -> OffsetCollection: """Generates a set of offsets.""" - (timestamps, x_offsets, y_offsets, z_offsets) = self.runCalibration(data) + basicCalibration = self.runCalibration(data) - offsetCollection = OffsetCollection(X=x_offsets, Y=y_offsets, Z=z_offsets) + offsetCollection = OffsetCollection( + X=basicCalibration.x_offsets, + Y=basicCalibration.y_offsets, + Z=basicCalibration.z_offsets, + ) sensor_name = "MAGO" singleCalibration = SingleCalibration( - timestamps=timestamps, + timestamps=basicCalibration.timestamps, offsets=offsetCollection, units=Unit.NT, instrument=sensor_name, @@ -56,14 +58,9 @@ def __init__(self): self.name = CalibratorType.SPINAXIS def runCalibration(self, data): - (timestamps, z_offsets) = MatlabWrapper.simulateSpinAxisCalibration(data) + calibration = MatlabWrapper.simulateSpinAxisCalibration(data) - return ( - timestamps, - np.zeros(len(z_offsets)), - np.zeros(len(z_offsets)), - z_offsets, - ) + return calibration class SpinPlaneCalibrator(Calibrator): @@ -71,8 +68,6 @@ def __init__(self): self.name = CalibratorType.SPINPLANE def runCalibration(self, data): - (timestamps, x_offsets, y_offsets) = ( - MatlabWrapper.simulateSpinPlaneCalibration() - ) + calibration = MatlabWrapper.simulateSpinPlaneCalibration() - return (timestamps, x_offsets, y_offsets, np.zeros(len(x_offsets))) + return calibration diff --git a/src/mag_toolkit/calibration/MatlabWrapper.py b/src/mag_toolkit/calibration/MatlabWrapper.py index 33402a5..1e70b3e 100644 --- a/src/mag_toolkit/calibration/MatlabWrapper.py +++ b/src/mag_toolkit/calibration/MatlabWrapper.py @@ -1,16 +1,38 @@ from datetime import datetime +import numpy as np +from pydantic import BaseModel -def simulateSpinAxisCalibration(xarray): + +class BasicCalibration(BaseModel): + timestamps: list[datetime] + x_offsets: list[float] + y_offsets: list[float] + z_offsets: list[float] + + +def simulateSpinAxisCalibration(xarray) -> BasicCalibration: + # TODO: Transfer to MATLAB to get spin axis offsets timestamps = [datetime(2022, 3, 3)] offsets = [3.256] - return (timestamps, offsets) + return BasicCalibration( + timestamps=timestamps, + x_offsets=np.zeros(len(offsets)), + y_offsets=np.zeros(len(offsets)), + z_offsets=offsets, + ) def simulateSpinPlaneCalibration(xarray): + # TODO: Transfer to MATLAB to get spin plane offsets timestamps = [datetime(2022, 3, 3)] offsets_x = [3.256] offsets_y = [2.76] - return (timestamps, offsets_x, offsets_y) + return BasicCalibration( + timestamps=timestamps, + x_offsets=offsets_x, + y_offsets=offsets_y, + z_offsets=np.zeros(len(offsets_x)), + ) diff --git a/src/mag_toolkit/calibration/calibrationFormat.py b/src/mag_toolkit/calibration/calibrationFormat.py index a07494e..eb117d5 100644 --- a/src/mag_toolkit/calibration/calibrationFormat.py +++ b/src/mag_toolkit/calibration/calibrationFormat.py @@ -1,6 +1,6 @@ from datetime import datetime from enum import Enum -from typing import Optional, Self +from typing import Optional from pydantic import BaseModel, model_validator @@ -22,7 +22,7 @@ class OffsetCollection(BaseModel): Z: list[float] @model_validator(mode="after") - def check_lengths_match(self) -> Self: + def check_lengths_match(self): if ( len(self.X) != len(self.Y) or len(self.Y) != len(self.Z) From 56542b7974506264f3da04f2fbff1fdd7955f67c Mon Sep 17 00:00:00 2001 From: mhairifin Date: Mon, 29 Jul 2024 09:36:23 +0100 Subject: [PATCH 21/33] carry over toolkit library code for use by pipeline --- src/mag_toolkit/CDFLoader.py | 16 ++++ src/mag_toolkit/__init__.py | 0 .../calibration/CalibrationApplicator.py | 46 +++++++++++ .../calibration/CalibrationExceptions.py | 2 + src/mag_toolkit/calibration/Calibrator.py | 78 +++++++++++++++++++ src/mag_toolkit/calibration/MatlabWrapper.py | 16 ++++ .../calibration/calibrationFormat.py | 48 ++++++++++++ .../calibration/calibrationFormatProcessor.py | 51 ++++++++++++ 8 files changed, 257 insertions(+) create mode 100644 src/mag_toolkit/CDFLoader.py create mode 100644 src/mag_toolkit/__init__.py create mode 100644 src/mag_toolkit/calibration/CalibrationApplicator.py create mode 100644 src/mag_toolkit/calibration/CalibrationExceptions.py create mode 100644 src/mag_toolkit/calibration/Calibrator.py create mode 100644 src/mag_toolkit/calibration/MatlabWrapper.py create mode 100644 src/mag_toolkit/calibration/calibrationFormat.py create mode 100644 src/mag_toolkit/calibration/calibrationFormatProcessor.py diff --git a/src/mag_toolkit/CDFLoader.py b/src/mag_toolkit/CDFLoader.py new file mode 100644 index 0000000..d572bed --- /dev/null +++ b/src/mag_toolkit/CDFLoader.py @@ -0,0 +1,16 @@ +from pathlib import Path + +from cdflib import xarray + + +def load_cdf(inputPath: Path): + """Wraps cdlibs xarray reader.""" + if inputPath.is_file(): + return xarray.cdf_to_xarray(inputPath) + else: + raise FileExistsError() + + +def write_cdf(dataset, outputPath: Path): + """Wraps cdflib xarray writer.""" + xarray.xarray_to_cdf(dataset, outputPath) diff --git a/src/mag_toolkit/__init__.py b/src/mag_toolkit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/mag_toolkit/calibration/CalibrationApplicator.py b/src/mag_toolkit/calibration/CalibrationApplicator.py new file mode 100644 index 0000000..6c5c7d2 --- /dev/null +++ b/src/mag_toolkit/calibration/CalibrationApplicator.py @@ -0,0 +1,46 @@ +import logging +from pathlib import Path + +import numpy as np + +from src.mag_toolkit.calibration.CalibrationExceptions import CalibrationValidityError +from src.mag_toolkit.calibration.calibrationFormat import CalibrationFormat +from src.mag_toolkit.calibration.calibrationFormatProcessor import ( + CalibrationFormatProcessor, +) +from src.mag_toolkit.CDFLoader import load_cdf, write_cdf + + +class CalibrationApplicator: + def apply(self, calibrationFile, dataFile, outputFile) -> Path: + """Currently operating on unprocessed data.""" + data = load_cdf(dataFile) + calibrationCollection: CalibrationFormat = ( + CalibrationFormatProcessor.loadFromPath(calibrationFile) + ) + + logging.info("Loaded calibration file and data file") + + try: + self.checkValidity(data, calibrationCollection) + except CalibrationValidityError as e: + logging.info(f"{e} -> continuing application of calibration regardless") + + logging.info("Dataset and calibration file deemed compatible") + + for eachCal in calibrationCollection.calibrations: + data.vectors[0] = data.vectors[0] + eachCal.offsets.X + data.vectors[1] = data.vectors[1] + eachCal.offsets.Y + data.vectors[2] = data.vectors[2] + eachCal.offsets.Z + + write_cdf(data, outputFile) + + return outputFile + + def checkValidity(self, data, calibrationCollection): + # check for time validity + if data.epoch[0] < np.datetime64( + calibrationCollection.valid_start + ) or data.epoch[1] > np.datetime64(calibrationCollection.valid_end): + logging.debug("Data outside of calibration validity range") + raise CalibrationValidityError("Data outside of calibration validity range") diff --git a/src/mag_toolkit/calibration/CalibrationExceptions.py b/src/mag_toolkit/calibration/CalibrationExceptions.py new file mode 100644 index 0000000..6b2db9b --- /dev/null +++ b/src/mag_toolkit/calibration/CalibrationExceptions.py @@ -0,0 +1,2 @@ +class CalibrationValidityError(Exception): + pass diff --git a/src/mag_toolkit/calibration/Calibrator.py b/src/mag_toolkit/calibration/Calibrator.py new file mode 100644 index 0000000..194b9ec --- /dev/null +++ b/src/mag_toolkit/calibration/Calibrator.py @@ -0,0 +1,78 @@ +from abc import ABC, abstractmethod +from datetime import datetime +from enum import Enum + +import numpy as np + +from src.mag_toolkit.calibration import MatlabWrapper +from src.mag_toolkit.calibration.calibrationFormat import ( + CalibrationFormat, + OffsetCollection, + SingleCalibration, + Unit, +) + + +class CalibratorType(str, Enum): + SPINAXIS = ("SpinAxisCalibrator",) + SPINPLANE = "SpinPlaneCalibrator" + + +class Calibrator(ABC): + def generateOffsets(self, data) -> OffsetCollection: + """Generates a set of offsets.""" + (timestamps, x_offsets, y_offsets, z_offsets) = self.runCalibration(data) + + offsetCollection = OffsetCollection(X=x_offsets, Y=y_offsets, Z=z_offsets) + + sensor_name = "MAGO" + + singleCalibration = SingleCalibration( + timestamps=timestamps, + offsets=offsetCollection, + units=Unit.NT, + instrument=sensor_name, + creation_timestamp=datetime.now(), + method=str(self.name), + ) + return singleCalibration + + def generateCalibration(self, data) -> CalibrationFormat: + singleCalibration = self.generateOffsets(data) + calibration = CalibrationFormat( + valid_start=singleCalibration.timestamps[0], + valid_end=singleCalibration.timestamps[-1], + calibrations=[singleCalibration], + ) + return calibration + + @abstractmethod + def runCalibration(self, data): + """Calibration that generates offsets and timestamps.""" + + +class SpinAxisCalibrator(Calibrator): + def __init__(self): + self.name = CalibratorType.SPINAXIS + + def runCalibration(self, data): + (timestamps, z_offsets) = MatlabWrapper.simulateSpinAxisCalibration(data) + + return ( + timestamps, + np.zeros(len(z_offsets)), + np.zeros(len(z_offsets)), + z_offsets, + ) + + +class SpinPlaneCalibrator(Calibrator): + def __init__(self): + self.name = CalibratorType.SPINPLANE + + def runCalibration(self, data): + (timestamps, x_offsets, y_offsets) = ( + MatlabWrapper.simulateSpinPlaneCalibration() + ) + + return (timestamps, x_offsets, y_offsets, np.zeros(len(x_offsets))) diff --git a/src/mag_toolkit/calibration/MatlabWrapper.py b/src/mag_toolkit/calibration/MatlabWrapper.py new file mode 100644 index 0000000..33402a5 --- /dev/null +++ b/src/mag_toolkit/calibration/MatlabWrapper.py @@ -0,0 +1,16 @@ +from datetime import datetime + + +def simulateSpinAxisCalibration(xarray): + timestamps = [datetime(2022, 3, 3)] + offsets = [3.256] + + return (timestamps, offsets) + + +def simulateSpinPlaneCalibration(xarray): + timestamps = [datetime(2022, 3, 3)] + offsets_x = [3.256] + offsets_y = [2.76] + + return (timestamps, offsets_x, offsets_y) diff --git a/src/mag_toolkit/calibration/calibrationFormat.py b/src/mag_toolkit/calibration/calibrationFormat.py new file mode 100644 index 0000000..a07494e --- /dev/null +++ b/src/mag_toolkit/calibration/calibrationFormat.py @@ -0,0 +1,48 @@ +from datetime import datetime +from enum import Enum +from typing import Optional, Self + +from pydantic import BaseModel, model_validator + + +class Unit(Enum): + NT = "nT" + UT = "uT" + T = "T" + + +class Instrument(Enum): + MAGO = "MAGO" + MAGI = "MAGI" + + +class OffsetCollection(BaseModel): + X: list[float] + Y: list[float] + Z: list[float] + + @model_validator(mode="after") + def check_lengths_match(self) -> Self: + if ( + len(self.X) != len(self.Y) + or len(self.Y) != len(self.Z) + or len(self.X) != len(self.Z) + ): + raise ValueError("Length of offset lists do not match") + return self + + +class SingleCalibration(BaseModel): + timestamps: list[datetime] + offsets: OffsetCollection + units: Unit + instrument: Instrument + creation_timestamp: datetime + method: str + comment: Optional[str] = None + + +class CalibrationFormat(BaseModel): + valid_start: datetime + valid_end: datetime + calibrations: list[SingleCalibration] diff --git a/src/mag_toolkit/calibration/calibrationFormatProcessor.py b/src/mag_toolkit/calibration/calibrationFormatProcessor.py new file mode 100644 index 0000000..fa53d1a --- /dev/null +++ b/src/mag_toolkit/calibration/calibrationFormatProcessor.py @@ -0,0 +1,51 @@ +import os +from pathlib import Path + +import yaml +from pydantic import ValidationError + +from src.mag_toolkit.calibration.calibrationFormat import CalibrationFormat + + +class CalibrationFormatProcessor: + def loadFromPath(calibrationPath: Path) -> CalibrationFormat: + try: + as_dict = yaml.safe_load(open(calibrationPath)) + model = CalibrationFormat(**as_dict) + return model + except ValidationError as e: + print(e) + return None + except FileNotFoundError as e: + print(e) + return None + + def loadFromDict(calibrationDict: dict) -> CalibrationFormat: + try: + model = CalibrationFormat(**calibrationDict) + return model + except ValidationError as e: + print(e) + return None + + def getWriteable(CalibrationFormat: CalibrationFormat): + json = CalibrationFormat.model_dump_json() + + return json + + def writeToFile( + CalibrationFormat: CalibrationFormat, filepath: Path, createDirectory=False + ): + json = CalibrationFormat.model_dump_json() + + if createDirectory: + os.makedirs(os.path.dirname(filepath), exist_ok=True) + + try: + with open(filepath, "w+") as f: + f.write(json) + except Exception as e: + print(e) + print(f"Failed to write calibration to {filepath}") + + return filepath From 19cf0ba16415609138b25e5066e7a453f4345624 Mon Sep 17 00:00:00 2001 From: mhairifin Date: Mon, 29 Jul 2024 09:38:28 +0100 Subject: [PATCH 22/33] add calibrate and apply commands to command line tool --- src/imap_mag/main.py | 107 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 92 insertions(+), 15 deletions(-) diff --git a/src/imap_mag/main.py b/src/imap_mag/main.py index b0c2f29..ae9ffc7 100644 --- a/src/imap_mag/main.py +++ b/src/imap_mag/main.py @@ -16,9 +16,23 @@ from src.imap_db.model import File +from src.mag_toolkit import CDFLoader +from src.mag_toolkit.calibration.CalibrationApplicator import CalibrationApplicator +from src.mag_toolkit.calibration.calibrationFormatProcessor import ( + CalibrationFormatProcessor, +) +from src.mag_toolkit.calibration.Calibrator import ( + Calibrator, + CalibratorType, + SpinAxisCalibrator, + SpinPlaneCalibrator, +) + # app code from . import DB, SDC, appConfig, appLogging, appUtils, imapProcessing, webPODA from .sdcApiClient import SDCApiClient +from src import appConfig, appLogging, imapProcessing + app = typer.Typer() globalState = {"verbose": False} @@ -81,21 +95,7 @@ def hello(name: str): print(f"Hello {name}") -# E.g imap-mag process --config config.yml solo_L2_mag-rtn-ll-internal_20240210_V00.cdf -@app.command() -def process( - file: Annotated[ - str, typer.Argument(help="The file name or pattern to match for the input file") - ], - config: Annotated[Path, typer.Option()] = Path("config.yml"), -): - """Sample processing job.""" - # TODO: semantic logging - # TODO: handle file system/cloud files - abstraction layer needed for files - # TODO: move shared logic to a library - - configFile: appConfig.AppConfig = commandInit(config) - +def prepareWorkFile(file, configFile): logging.debug(f"Grabbing file matching {file} in {configFile.source.folder}") # get all files in \\RDS.IMPERIAL.AC.UK\rds\project\solarorbitermagnetometer\live\SO-MAG-Web\quicklooks_py\ @@ -131,6 +131,26 @@ def process( logging.debug(f"Copying {files[0]} to {workFile}") workFile = Path(shutil.copy2(files[0], configFile.work_folder)) + return workFile + + +# E.g imap-mag process --config config.yml solo_L2_mag-rtn-ll-internal_20240210_V00.cdf +@app.command() +def process( + config: Annotated[Path, typer.Option()] = Path("config.yml"), + file: str = typer.Argument( + help="The file name or pattern to match for the input file" + ), +): + """Sample processing job.""" + # TODO: semantic logging + # TODO: handle file system/cloud files - abstraction layer needed for files + # TODO: move shared logic to a library + + configFile: appConfig.AppConfig = commandInit(config) + + workFile = prepareWorkFile(file, configFile) + # TODO: do something with the data! fileProcessor = imapProcessing.dispatchFile(workFile) fileProcessor.initialize(configFile) @@ -230,6 +250,63 @@ def fetch_science( # appUtils.copyFileToDestination(file, configFile.destination) +# imap-mag calibrate --config calibration_config.yml --method SpinAxisCalibrator imap_mag_l1b_norm-mago_20250502_v000.cdf +@app.command() +def calibrate( + config: Annotated[Path, typer.Option()] = Path("calibration_config.yml"), + method: Annotated[CalibratorType, typer.Option()] = "SpinAxisCalibrator", + input: str = typer.Argument( + help="The file name or pattern to match for the input file" + ), +): + # TODO: Define specific calibration configuration + # Using AppConfig for now to piggyback off of configuration + # verification and work area setup + configFile: appConfig.AppConfig = commandInit(config) + + workFile = prepareWorkFile(input, configFile) + calibrator: Calibrator + + match method: + case CalibratorType.SPINAXIS: + calibrator = SpinAxisCalibrator() + case CalibratorType.SPINPLANE: + calibrator = SpinPlaneCalibrator() + + inputData = CDFLoader.load_cdf(workFile) + calibration = calibrator.generateCalibration(inputData) + + tempOutputFile = os.path.join(configFile.work_folder, "calibration.json") + + result = CalibrationFormatProcessor.writeToFile(calibration, tempOutputFile) + + appUtils.copyFileToDestination(result, configFile.destination) + + +# imap-mag apply --config calibration_application_config.yml --calibration calibration.json imap_mag_l1a_norm-mago_20250502_v000.cdf +@app.command() +def apply( + config: Annotated[Path, typer.Option()] = Path( + "calibration_application_config.yml" + ), + calibration: Annotated[str, typer.Option()] = "calibration.json", + input: str = typer.Argument( + help="The file name or pattern to match for the input file" + ), +): + configFile: appConfig.AppConfig = commandInit(config) + + workDataFile = prepareWorkFile(input, configFile) + workCalibrationFile = prepareWorkFile(calibration, configFile) + workOutputFile = os.path.join(configFile.work_folder, "l2_data.cdf") + + applier = CalibrationApplicator() + + L2_file = applier.apply(workCalibrationFile, workDataFile, workOutputFile) + + appUtils.copyFileToDestination(L2_file, configFile.destination) + + @app.callback() def main(verbose: Annotated[bool, typer.Option("--verbose", "-v")] = False): if verbose: From 47e6a4430d3a5675df25a535d5a6fa5eb6b0964b Mon Sep 17 00:00:00 2001 From: mhairifin Date: Mon, 29 Jul 2024 11:30:38 +0100 Subject: [PATCH 23/33] resolve poetry conflicts and testing conflicts --- calibration_application_config.yml | 8 +++++++ calibration_config.yml | 8 +++++++ poetry.lock | 33 +++++++++++++++++++++------ pyproject.toml | 1 + tests/data/2025/calibration.json | 27 ++++++++++++++++++++++ tests/test_main.py | 36 +++++++++++++++++++++++++++++- 6 files changed, 105 insertions(+), 8 deletions(-) create mode 100644 calibration_application_config.yml create mode 100644 calibration_config.yml create mode 100644 tests/data/2025/calibration.json diff --git a/calibration_application_config.yml b/calibration_application_config.yml new file mode 100644 index 0000000..b748acb --- /dev/null +++ b/calibration_application_config.yml @@ -0,0 +1,8 @@ +source: + folder: tests/data/2025 + +work-folder: .work + +destination: + folder: output/ + filename: L2.cdf \ No newline at end of file diff --git a/calibration_config.yml b/calibration_config.yml new file mode 100644 index 0000000..5ed5230 --- /dev/null +++ b/calibration_config.yml @@ -0,0 +1,8 @@ +source: + folder: tests/data/2025 + +work-folder: .work + +destination: + folder: output/ + filename: calibration.json \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index bae74a6..8cdad03 100644 --- a/poetry.lock +++ b/poetry.lock @@ -89,13 +89,13 @@ typing = ["typing-extensions (>=4.0.0)"] [[package]] name = "astropy-iers-data" -version = "0.2024.7.22.0.34.13" +version = "0.2024.7.29.0.32.7" description = "IERS Earth Rotation and Leap Second tables for the astropy core package" optional = false python-versions = ">=3.8" files = [ - {file = "astropy_iers_data-0.2024.7.22.0.34.13-py3-none-any.whl", hash = "sha256:567a6cb261dd62f60862ee8d38e70fb2c88dfad03e962bc8138397a22e33003d"}, - {file = "astropy_iers_data-0.2024.7.22.0.34.13.tar.gz", hash = "sha256:9bbb4bfc28bc8e834a6b3946a312ce3490c285abeab8fd9b1e98b11fdee6f92c"}, + {file = "astropy_iers_data-0.2024.7.29.0.32.7-py3-none-any.whl", hash = "sha256:7c8fb523731f6c2c039c112be089c998202020234d2a6155a1f3620b4cfbf24d"}, + {file = "astropy_iers_data-0.2024.7.29.0.32.7.tar.gz", hash = "sha256:32967816e4758603b571303ea3c8a836570aacb10f9bb258154f10d2913faf6d"}, ] [package.extras] @@ -247,6 +247,25 @@ files = [ [package.dependencies] bitarray = ">=2.9.0,<3.0.0" +[[package]] +name = "cdflib" +version = "1.3.1" +description = "A python CDF reader toolkit" +optional = false +python-versions = ">=3.8" +files = [ + {file = "cdflib-1.3.1-py3-none-any.whl", hash = "sha256:115494bfffd23b92c41629d5fcfdefab77112a4d9d1ff6023f87c5444d2e9aeb"}, + {file = "cdflib-1.3.1.tar.gz", hash = "sha256:7bb296e02ac7c47536c43bcc20d24796bfd9d4ec39490ec74953972203f26913"}, +] + +[package.dependencies] +numpy = ">=1.21" + +[package.extras] +dev = ["ipython", "pre-commit"] +docs = ["astropy", "netcdf4", "sphinx", "sphinx-automodapi", "sphinx-copybutton", "sphinx-rtd-theme", "xarray"] +tests = ["astropy", "h5netcdf", "hypothesis", "netcdf4", "pytest (>=3.9)", "pytest-cov", "pytest-remotedata", "xarray"] + [[package]] name = "certifi" version = "2024.7.4" @@ -1404,13 +1423,13 @@ files = [ [[package]] name = "setuptools" -version = "71.1.0" +version = "72.0.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-71.1.0-py3-none-any.whl", hash = "sha256:33874fdc59b3188304b2e7c80d9029097ea31627180896fb549c578ceb8a0855"}, - {file = "setuptools-71.1.0.tar.gz", hash = "sha256:032d42ee9fb536e33087fb66cac5f840eb9391ed05637b3f2a76a7c8fb477936"}, + {file = "setuptools-72.0.0-py3-none-any.whl", hash = "sha256:98b4d786a12fadd34eabf69e8d014b84e5fc655981e4ff419994700434ace132"}, + {file = "setuptools-72.0.0.tar.gz", hash = "sha256:5a0d9c6a2f332881a0153f629d8000118efd33255cfa802757924c53312c76da"}, ] [package.extras] @@ -1683,4 +1702,4 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "b88a488487eb736be738b1d142b111f0aced830ba5c887368835a063c47ec1e4" +content-hash = "3f8ff12d4fd300e1ffae7733880c456882114cf2decc722196d993e5e0aa285a" diff --git a/pyproject.toml b/pyproject.toml index a3e34da..1327b05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ requests = "^2.32.3" pandas = "^2.2.2" imap-data-access = "^0.7.0" astropy = "^6.1.2" +cdflib = "^1.3.1" [tool.poetry.group.dev.dependencies] pytest = "^8.3.1" diff --git a/tests/data/2025/calibration.json b/tests/data/2025/calibration.json new file mode 100644 index 0000000..9127c29 --- /dev/null +++ b/tests/data/2025/calibration.json @@ -0,0 +1,27 @@ +{ + "calibrations": [ + { + "comment": null, + "creation_timestamp": "2024-07-26T16:51:58.212401", + "instrument": "MAGO", + "method": "CalibratorType.SPINAXIS", + "offsets": { + "X": [ + 0.0 + ], + "Y": [ + 0.0 + ], + "Z": [ + 3.256 + ] + }, + "timestamps": [ + "2022-03-03T00:00:00" + ], + "units": "nT" + } + ], + "valid_end": "2022-03-03T00:00:00", + "valid_start": "2022-03-03T00:00:00" +} diff --git a/tests/test_main.py b/tests/test_main.py index 9482ebd..44ac199 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -96,4 +96,38 @@ def test_fetch_binary_downloads_hk_from_webpoda(tidyDataFolders): # Verify. assert result.exit_code == 0 - assert Path("output/power.pkts").exists() + assert Path("output/power.pkts").exists() # + + +def test_calibration_creates_calibration_file(): + result = runner.invoke( + app, + [ + "calibrate", + "--config", + "calibration_config.yml", + "--method", + "SpinAxisCalibrator", + "imap_mag_l1a_norm-mago_20250502_v000.cdf", + ], + ) + assert result.exit_code == 0 + assert Path("output/calibration.json").exists() + + +def test_application_creates_L2_file(): + result = runner.invoke( + app, + [ + "apply", + "--config", + "calibration_application_config.yml", + "--calibration", + "calibration.json", + "imap_mag_l1a_norm-mago_20250502_v000.cdf", + ], + ) + + print("\n" + str(result.stdout)) + assert result.exit_code == 0 + assert Path("output/L2.cdf").exists() From 474fdc656279933d1971dfbe6810c82516c41b69 Mon Sep 17 00:00:00 2001 From: mhairifin Date: Mon, 29 Jul 2024 11:18:16 +0100 Subject: [PATCH 24/33] resolve typing issues --- src/mag_toolkit/calibration/Calibrator.py | 29 ++++++++----------- src/mag_toolkit/calibration/MatlabWrapper.py | 28 ++++++++++++++++-- .../calibration/calibrationFormat.py | 4 +-- 3 files changed, 39 insertions(+), 22 deletions(-) diff --git a/src/mag_toolkit/calibration/Calibrator.py b/src/mag_toolkit/calibration/Calibrator.py index 194b9ec..75b857b 100644 --- a/src/mag_toolkit/calibration/Calibrator.py +++ b/src/mag_toolkit/calibration/Calibrator.py @@ -2,8 +2,6 @@ from datetime import datetime from enum import Enum -import numpy as np - from src.mag_toolkit.calibration import MatlabWrapper from src.mag_toolkit.calibration.calibrationFormat import ( CalibrationFormat, @@ -14,21 +12,25 @@ class CalibratorType(str, Enum): - SPINAXIS = ("SpinAxisCalibrator",) + SPINAXIS = "SpinAxisCalibrator" SPINPLANE = "SpinPlaneCalibrator" class Calibrator(ABC): def generateOffsets(self, data) -> OffsetCollection: """Generates a set of offsets.""" - (timestamps, x_offsets, y_offsets, z_offsets) = self.runCalibration(data) + basicCalibration = self.runCalibration(data) - offsetCollection = OffsetCollection(X=x_offsets, Y=y_offsets, Z=z_offsets) + offsetCollection = OffsetCollection( + X=basicCalibration.x_offsets, + Y=basicCalibration.y_offsets, + Z=basicCalibration.z_offsets, + ) sensor_name = "MAGO" singleCalibration = SingleCalibration( - timestamps=timestamps, + timestamps=basicCalibration.timestamps, offsets=offsetCollection, units=Unit.NT, instrument=sensor_name, @@ -56,14 +58,9 @@ def __init__(self): self.name = CalibratorType.SPINAXIS def runCalibration(self, data): - (timestamps, z_offsets) = MatlabWrapper.simulateSpinAxisCalibration(data) + calibration = MatlabWrapper.simulateSpinAxisCalibration(data) - return ( - timestamps, - np.zeros(len(z_offsets)), - np.zeros(len(z_offsets)), - z_offsets, - ) + return calibration class SpinPlaneCalibrator(Calibrator): @@ -71,8 +68,6 @@ def __init__(self): self.name = CalibratorType.SPINPLANE def runCalibration(self, data): - (timestamps, x_offsets, y_offsets) = ( - MatlabWrapper.simulateSpinPlaneCalibration() - ) + calibration = MatlabWrapper.simulateSpinPlaneCalibration() - return (timestamps, x_offsets, y_offsets, np.zeros(len(x_offsets))) + return calibration diff --git a/src/mag_toolkit/calibration/MatlabWrapper.py b/src/mag_toolkit/calibration/MatlabWrapper.py index 33402a5..1e70b3e 100644 --- a/src/mag_toolkit/calibration/MatlabWrapper.py +++ b/src/mag_toolkit/calibration/MatlabWrapper.py @@ -1,16 +1,38 @@ from datetime import datetime +import numpy as np +from pydantic import BaseModel -def simulateSpinAxisCalibration(xarray): + +class BasicCalibration(BaseModel): + timestamps: list[datetime] + x_offsets: list[float] + y_offsets: list[float] + z_offsets: list[float] + + +def simulateSpinAxisCalibration(xarray) -> BasicCalibration: + # TODO: Transfer to MATLAB to get spin axis offsets timestamps = [datetime(2022, 3, 3)] offsets = [3.256] - return (timestamps, offsets) + return BasicCalibration( + timestamps=timestamps, + x_offsets=np.zeros(len(offsets)), + y_offsets=np.zeros(len(offsets)), + z_offsets=offsets, + ) def simulateSpinPlaneCalibration(xarray): + # TODO: Transfer to MATLAB to get spin plane offsets timestamps = [datetime(2022, 3, 3)] offsets_x = [3.256] offsets_y = [2.76] - return (timestamps, offsets_x, offsets_y) + return BasicCalibration( + timestamps=timestamps, + x_offsets=offsets_x, + y_offsets=offsets_y, + z_offsets=np.zeros(len(offsets_x)), + ) diff --git a/src/mag_toolkit/calibration/calibrationFormat.py b/src/mag_toolkit/calibration/calibrationFormat.py index a07494e..eb117d5 100644 --- a/src/mag_toolkit/calibration/calibrationFormat.py +++ b/src/mag_toolkit/calibration/calibrationFormat.py @@ -1,6 +1,6 @@ from datetime import datetime from enum import Enum -from typing import Optional, Self +from typing import Optional from pydantic import BaseModel, model_validator @@ -22,7 +22,7 @@ class OffsetCollection(BaseModel): Z: list[float] @model_validator(mode="after") - def check_lengths_match(self) -> Self: + def check_lengths_match(self): if ( len(self.X) != len(self.Y) or len(self.Y) != len(self.Z) From 0f800a9d518c01de729029284876bebb7ac78925 Mon Sep 17 00:00:00 2001 From: mhairifin Date: Mon, 29 Jul 2024 11:36:32 +0100 Subject: [PATCH 25/33] dont require hk packet for appConfig, for calibration --- src/imap_mag/appConfig.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/imap_mag/appConfig.py b/src/imap_mag/appConfig.py index 10b4e1a..cef0dc3 100644 --- a/src/imap_mag/appConfig.py +++ b/src/imap_mag/appConfig.py @@ -1,6 +1,7 @@ """App configuration module.""" from pathlib import Path +from typing import Optional from pydantic import BaseModel from pydantic.config import ConfigDict @@ -27,7 +28,7 @@ class AppConfig(BaseModel): source: Source work_folder: Path = Path(".work") destination: Destination - packet_definition: PacketDefinition + packet_definition: Optional[PacketDefinition] = None # pydantic configuration to allow hyphenated fields model_config = ConfigDict(alias_generator=hyphenize) From 0e8a0787074ae978a2bc43872c072c23169b20c1 Mon Sep 17 00:00:00 2001 From: mhairifin Date: Mon, 29 Jul 2024 11:40:52 +0100 Subject: [PATCH 26/33] import resolve and tidying test data --- src/imap_mag/main.py | 4 ---- tests/test_main.py | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/imap_mag/main.py b/src/imap_mag/main.py index ae9ffc7..3ae487a 100644 --- a/src/imap_mag/main.py +++ b/src/imap_mag/main.py @@ -15,7 +15,6 @@ import yaml from src.imap_db.model import File - from src.mag_toolkit import CDFLoader from src.mag_toolkit.calibration.CalibrationApplicator import CalibrationApplicator from src.mag_toolkit.calibration.calibrationFormatProcessor import ( @@ -28,11 +27,8 @@ SpinPlaneCalibrator, ) -# app code from . import DB, SDC, appConfig, appLogging, appUtils, imapProcessing, webPODA from .sdcApiClient import SDCApiClient -from src import appConfig, appLogging, imapProcessing - app = typer.Typer() globalState = {"verbose": False} diff --git a/tests/test_main.py b/tests/test_main.py index 44ac199..7033ecb 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -99,7 +99,7 @@ def test_fetch_binary_downloads_hk_from_webpoda(tidyDataFolders): assert Path("output/power.pkts").exists() # -def test_calibration_creates_calibration_file(): +def test_calibration_creates_calibration_file(tidyDataFolders): result = runner.invoke( app, [ @@ -115,7 +115,7 @@ def test_calibration_creates_calibration_file(): assert Path("output/calibration.json").exists() -def test_application_creates_L2_file(): +def test_application_creates_L2_file(tidyDataFolders): result = runner.invoke( app, [ From b915d93654de477fceafdff4c4a0c0200361f872 Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Mon, 29 Jul 2024 11:50:03 +0100 Subject: [PATCH 27/33] feat: working demo --- config-hk-download.yaml | 2 ++ config-sci.yml | 7 +++++++ deploy/entrypoint.sh | 9 ++++++--- src/imap_mag/SDC.py | 9 +-------- src/imap_mag/main.py | 9 +++++---- 5 files changed, 21 insertions(+), 15 deletions(-) diff --git a/config-hk-download.yaml b/config-hk-download.yaml index dc7ef88..8e3c264 100644 --- a/config-hk-download.yaml +++ b/config-hk-download.yaml @@ -1,3 +1,5 @@ +source: + folder: /data/hk_l0/ work-folder: /data/.work diff --git a/config-sci.yml b/config-sci.yml index fe58105..dd281ed 100644 --- a/config-sci.yml +++ b/config-sci.yml @@ -1,4 +1,11 @@ +source: + folder: tests/data/2025 work-folder: /data/science +destination: + folder: output/ + filename: result.cdf +packet-definition: + hk: src/imap_mag/xtce/tlm_20240724.xml diff --git a/deploy/entrypoint.sh b/deploy/entrypoint.sh index b25e911..e0dbe4d 100644 --- a/deploy/entrypoint.sh +++ b/deploy/entrypoint.sh @@ -13,11 +13,14 @@ do # delete all data rm -rf /data/* - imap-mag fetch-binary --config config-hk-download.yaml --apid 1063 --start-date 2025-05-02 --end-date 2025-05-03 + START_DATE='2025-05-02' + END_DATE='2025-05-03' - imap-mag process --config config-hk-process.yaml MAG_HSK_PW.pkts + imap-mag fetch-binary --config config-hk-download.yaml --apid 1063 --start-date $START_DATE --end-date $END_DATE - imap-mag fetch-science --start-date 2025-05-02 --end-date 2025-05-03 --config config-sci.yaml + imap-mag process --config config-hk-process.yaml power.pkts + + imap-mag fetch-science --level l1b --start-date $START_DATE --end-date $END_DATE --config config-sci.yml imap-db query-db diff --git a/src/imap_mag/SDC.py b/src/imap_mag/SDC.py index 00c2796..659c423 100644 --- a/src/imap_mag/SDC.py +++ b/src/imap_mag/SDC.py @@ -63,19 +63,12 @@ def QueryAndDownload(self, **options: Unpack[SDCOptions]) -> list[Path]: ) for date in date_range.to_pydatetime(): - download_files = self.__check_download_needed(details, date, **options) - (new_version, previous_version) = self.__data_access.unique_version( + (version, previous_version) = self.__data_access.unique_version( level=options["level"], start_date=date, ) - if download_files: - version = new_version - else: - assert previous_version is not None - version = previous_version - for var in details["variations"]: files = self.__data_access.get_filename( level=options["level"], diff --git a/src/imap_mag/main.py b/src/imap_mag/main.py index b0c2f29..c806934 100644 --- a/src/imap_mag/main.py +++ b/src/imap_mag/main.py @@ -174,9 +174,10 @@ def fetch_binary( class LevelEnum(str, Enum): - level_1 = "l1" + level_1a = "l1a" + level_1b = "l1b" + level_1c = "l1c" level_2 = "l2" - level_3 = "l3" # E.g., imap-mag fetch-science --start-date 2025-05-02 --end-date 2025-05-03 @@ -214,14 +215,14 @@ def fetch_science( # TODO: any better way than passing a dictionary? Strongly typed? files = sdc.QueryAndDownload( - level=level, start_date=start_date, end_date=end_date, force=force + level=level.value, start_date=start_date, end_date=end_date, force=force ) records = [] for file in files: records.append(File(name=file.name, path=file.absolute().as_posix())) - db = DB() + db = DB.DB() db.insert_files(records) logging.info(f"Downloaded {len(files)} files and saved to database") From 142960c49e9ccb60e7e3850bac72690d2495b7f6 Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Mon, 29 Jul 2024 12:07:13 +0100 Subject: [PATCH 28/33] feat: add calibration to entrypoint --- calibration_application_config.yml | 6 +++--- calibration_config.yml | 6 +++--- deploy/entrypoint.sh | 4 ++++ 3 files changed, 10 insertions(+), 6 deletions(-) mode change 100644 => 100755 deploy/entrypoint.sh diff --git a/calibration_application_config.yml b/calibration_application_config.yml index b748acb..e105b77 100644 --- a/calibration_application_config.yml +++ b/calibration_application_config.yml @@ -1,8 +1,8 @@ source: - folder: tests/data/2025 + folder: /data/science/imap/mag/l1b/2025/05 work-folder: .work destination: - folder: output/ - filename: L2.cdf \ No newline at end of file + folder: /data/science/imap/mag/l2/2025/05 + filename: imap_mag_l2_norm-mago_20250511_v000.cdf diff --git a/calibration_config.yml b/calibration_config.yml index 5ed5230..9c89ae8 100644 --- a/calibration_config.yml +++ b/calibration_config.yml @@ -1,8 +1,8 @@ source: - folder: tests/data/2025 + folder: /data/science/imap/mag/l1b/2025/05 work-folder: .work destination: - folder: output/ - filename: calibration.json \ No newline at end of file + folder: /data/science/imap/mag/l1b/2025/05 + filename: calibration.json diff --git a/deploy/entrypoint.sh b/deploy/entrypoint.sh old mode 100644 new mode 100755 index e0dbe4d..164cc43 --- a/deploy/entrypoint.sh +++ b/deploy/entrypoint.sh @@ -24,6 +24,10 @@ do imap-db query-db + imap-mag calibrate --config calibration_config.yml --method SpinAxisCalibrator imap_mag_l1b_norm-mago_20250511_v000.cdf + + imap-mag apply --config calibration_application_config.yml --calibration calibration.json imap_mag_l1b_norm-mago_20250511_v000.cdf + ls -l /data sleep 3600 # 1 Hour From c86183fb4f301dbaa0002471bad2b1fef5264f0a Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Mon, 29 Jul 2024 12:17:17 +0100 Subject: [PATCH 29/33] test: fix bad path in test config files --- src/imap_mag/SDC.py | 1 - tests/config/calibration_application_config.yml | 8 ++++++++ tests/config/calibration_config.yml | 8 ++++++++ tests/test_main.py | 4 ++-- 4 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 tests/config/calibration_application_config.yml create mode 100644 tests/config/calibration_config.yml diff --git a/src/imap_mag/SDC.py b/src/imap_mag/SDC.py index 659c423..a8a6b61 100644 --- a/src/imap_mag/SDC.py +++ b/src/imap_mag/SDC.py @@ -63,7 +63,6 @@ def QueryAndDownload(self, **options: Unpack[SDCOptions]) -> list[Path]: ) for date in date_range.to_pydatetime(): - (version, previous_version) = self.__data_access.unique_version( level=options["level"], start_date=date, diff --git a/tests/config/calibration_application_config.yml b/tests/config/calibration_application_config.yml new file mode 100644 index 0000000..b748acb --- /dev/null +++ b/tests/config/calibration_application_config.yml @@ -0,0 +1,8 @@ +source: + folder: tests/data/2025 + +work-folder: .work + +destination: + folder: output/ + filename: L2.cdf \ No newline at end of file diff --git a/tests/config/calibration_config.yml b/tests/config/calibration_config.yml new file mode 100644 index 0000000..5ed5230 --- /dev/null +++ b/tests/config/calibration_config.yml @@ -0,0 +1,8 @@ +source: + folder: tests/data/2025 + +work-folder: .work + +destination: + folder: output/ + filename: calibration.json \ No newline at end of file diff --git a/tests/test_main.py b/tests/test_main.py index 7033ecb..7a77234 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -105,7 +105,7 @@ def test_calibration_creates_calibration_file(tidyDataFolders): [ "calibrate", "--config", - "calibration_config.yml", + "tests/config/calibration_config.yml", "--method", "SpinAxisCalibrator", "imap_mag_l1a_norm-mago_20250502_v000.cdf", @@ -121,7 +121,7 @@ def test_application_creates_L2_file(tidyDataFolders): [ "apply", "--config", - "calibration_application_config.yml", + "tests/config/calibration_application_config.yml", "--calibration", "calibration.json", "imap_mag_l1a_norm-mago_20250502_v000.cdf", From f608f9d731f37fd47fb4fb329ce595a0d7ba2ae7 Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Mon, 29 Jul 2024 13:02:56 +0000 Subject: [PATCH 30/33] bug: fix file paths and missing config from dockerfile --- .vscode/launch.json | 2 +- ...yml => calibration_application_config.yaml | 0 ...tion_config.yml => calibration_config.yaml | 0 config-sci.yml => config-sci.yaml | 0 config.yml => config.yaml | 0 deploy/Dockerfile | 1 + deploy/entrypoint.sh | 6 ++--- pyproject.toml | 1 + src/imap_db/main.py | 2 +- src/imap_mag/main.py | 24 ++++++++++--------- ...ml => calibration_application_config.yaml} | 0 ...ion_config.yml => calibration_config.yaml} | 0 tests/test_main.py | 6 ++--- 13 files changed, 23 insertions(+), 19 deletions(-) rename calibration_application_config.yml => calibration_application_config.yaml (100%) rename calibration_config.yml => calibration_config.yaml (100%) rename config-sci.yml => config-sci.yaml (100%) rename config.yml => config.yaml (100%) rename tests/config/{calibration_application_config.yml => calibration_application_config.yaml} (100%) rename tests/config/{calibration_config.yml => calibration_config.yaml} (100%) diff --git a/.vscode/launch.json b/.vscode/launch.json index 20cba3e..a595caa 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,7 +4,7 @@ "args": [ "process", "--config", - "config.yml" + "config.yaml" ], "console": "integratedTerminal", "justMyCode": true, diff --git a/calibration_application_config.yml b/calibration_application_config.yaml similarity index 100% rename from calibration_application_config.yml rename to calibration_application_config.yaml diff --git a/calibration_config.yml b/calibration_config.yaml similarity index 100% rename from calibration_config.yml rename to calibration_config.yaml diff --git a/config-sci.yml b/config-sci.yaml similarity index 100% rename from config-sci.yml rename to config-sci.yaml diff --git a/config.yml b/config.yaml similarity index 100% rename from config.yml rename to config.yaml diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 06184f3..49ddf57 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -11,6 +11,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 COPY deploy/entrypoint.sh /app/entrypoint.sh +COPY *.yaml /app/ COPY dist/python${PYTHON_VERSION}/${TOOL_PACKAGE} /app/python${PYTHON_VERSION}/ # Creates a non-root user with an explicit UID and adds permission to access the /app folder diff --git a/deploy/entrypoint.sh b/deploy/entrypoint.sh index 164cc43..577bc1e 100755 --- a/deploy/entrypoint.sh +++ b/deploy/entrypoint.sh @@ -20,13 +20,13 @@ do imap-mag process --config config-hk-process.yaml power.pkts - imap-mag fetch-science --level l1b --start-date $START_DATE --end-date $END_DATE --config config-sci.yml + imap-mag fetch-science --level l1b --start-date $START_DATE --end-date $END_DATE --config config-sci.yaml imap-db query-db - imap-mag calibrate --config calibration_config.yml --method SpinAxisCalibrator imap_mag_l1b_norm-mago_20250511_v000.cdf + imap-mag calibrate --config calibration_config.yaml --method SpinAxisCalibrator imap_mag_l1b_norm-mago_20250511_v000.cdf - imap-mag apply --config calibration_application_config.yml --calibration calibration.json imap_mag_l1b_norm-mago_20250511_v000.cdf + imap-mag apply --config calibration_application_config.yaml --calibration calibration.json imap_mag_l1b_norm-mago_20250511_v000.cdf ls -l /data diff --git a/pyproject.toml b/pyproject.toml index 1327b05..bf68710 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ readme = "README.md" packages = [ { include = "src/imap_mag" }, { include = "src/imap_db" }, + { include = "src/mag_toolkit" }, ] [tool.poetry.dependencies] diff --git a/src/imap_db/main.py b/src/imap_db/main.py index 45bd792..2d876bd 100644 --- a/src/imap_db/main.py +++ b/src/imap_db/main.py @@ -81,7 +81,7 @@ def upgrade_db(): # combine them in OS agnostic way script_location = os.path.join(folder, script_location) - print("Running DB migrations in %r on %r", script_location, url) + print("Running DB migrations in %r", script_location) config.set_main_option("script_location", script_location) config.set_main_option("sqlalchemy.url", url) diff --git a/src/imap_mag/main.py b/src/imap_mag/main.py index dc534a0..5818283 100644 --- a/src/imap_mag/main.py +++ b/src/imap_mag/main.py @@ -41,12 +41,14 @@ def commandInit(config: Path) -> appConfig.AppConfig: raise typer.Abort() if config.is_file(): configFileDict = yaml.safe_load(open(config)) - logging.debug(f"Config file contents: {configFileDict}") + logging.debug( + "Config file loaded from %s with content %s: ", config, configFileDict + ) elif config.is_dir(): - logging.critical("Config is a directory, need a yml file") + logging.critical("Config %s is a directory, need a yml file", config) raise typer.Abort() elif not config.exists(): - logging.critical("The config doesn't exist") + logging.critical("The config at $s does not exist", config) raise typer.Abort() else: pass @@ -130,10 +132,10 @@ def prepareWorkFile(file, configFile): return workFile -# E.g imap-mag process --config config.yml solo_L2_mag-rtn-ll-internal_20240210_V00.cdf +# E.g imap-mag process --config config.yaml solo_L2_mag-rtn-ll-internal_20240210_V00.cdf @app.command() def process( - config: Annotated[Path, typer.Option()] = Path("config.yml"), + config: Annotated[Path, typer.Option()] = Path("config.yaml"), file: str = typer.Argument( help="The file name or pattern to match for the input file" ), @@ -168,7 +170,7 @@ def fetch_binary( apid: Annotated[int, typer.Option(help="ApID to download")], start_date: Annotated[str, typer.Option(help="Start date for the download")], end_date: Annotated[str, typer.Option(help="End date for the download")], - config: Annotated[Path, typer.Option()] = Path("config.yml"), + config: Annotated[Path, typer.Option()] = Path("config.yaml"), ): configFile: appConfig.AppConfig = commandInit(config) @@ -215,7 +217,7 @@ def fetch_science( bool, typer.Option("--force", "-f", help="Force download even if the file exists"), ] = False, - config: Annotated[Path, typer.Option()] = Path("config-sci.yml"), + config: Annotated[Path, typer.Option()] = Path("config-sci.yaml"), ): configFile: appConfig.AppConfig = commandInit(config) @@ -247,10 +249,10 @@ def fetch_science( # appUtils.copyFileToDestination(file, configFile.destination) -# imap-mag calibrate --config calibration_config.yml --method SpinAxisCalibrator imap_mag_l1b_norm-mago_20250502_v000.cdf +# imap-mag calibrate --config calibration_config.yaml --method SpinAxisCalibrator imap_mag_l1b_norm-mago_20250502_v000.cdf @app.command() def calibrate( - config: Annotated[Path, typer.Option()] = Path("calibration_config.yml"), + config: Annotated[Path, typer.Option()] = Path("calibration_config.yaml"), method: Annotated[CalibratorType, typer.Option()] = "SpinAxisCalibrator", input: str = typer.Argument( help="The file name or pattern to match for the input file" @@ -280,11 +282,11 @@ def calibrate( appUtils.copyFileToDestination(result, configFile.destination) -# imap-mag apply --config calibration_application_config.yml --calibration calibration.json imap_mag_l1a_norm-mago_20250502_v000.cdf +# imap-mag apply --config calibration_application_config.yaml --calibration calibration.json imap_mag_l1a_norm-mago_20250502_v000.cdf @app.command() def apply( config: Annotated[Path, typer.Option()] = Path( - "calibration_application_config.yml" + "calibration_application_config.yaml" ), calibration: Annotated[str, typer.Option()] = "calibration.json", input: str = typer.Argument( diff --git a/tests/config/calibration_application_config.yml b/tests/config/calibration_application_config.yaml similarity index 100% rename from tests/config/calibration_application_config.yml rename to tests/config/calibration_application_config.yaml diff --git a/tests/config/calibration_config.yml b/tests/config/calibration_config.yaml similarity index 100% rename from tests/config/calibration_config.yml rename to tests/config/calibration_config.yaml diff --git a/tests/test_main.py b/tests/test_main.py index 7a77234..0b091a8 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -32,7 +32,7 @@ def test_process_with_valid_config_does_not_error(tidyDataFolders): [ "process", "--config", - "config.yml", + "config.yaml", "imap_mag_l1a_norm-mago_20250502_v000.cdf", ], ) @@ -105,7 +105,7 @@ def test_calibration_creates_calibration_file(tidyDataFolders): [ "calibrate", "--config", - "tests/config/calibration_config.yml", + "tests/config/calibration_config.yaml", "--method", "SpinAxisCalibrator", "imap_mag_l1a_norm-mago_20250502_v000.cdf", @@ -121,7 +121,7 @@ def test_application_creates_L2_file(tidyDataFolders): [ "apply", "--config", - "tests/config/calibration_application_config.yml", + "tests/config/calibration_application_config.yaml", "--calibration", "calibration.json", "imap_mag_l1a_norm-mago_20250502_v000.cdf", From 248c2d30bc50c669884697a956c9987368e1d427 Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Mon, 29 Jul 2024 13:20:45 +0000 Subject: [PATCH 31/33] task: add some docker scripts --- .env.template | 7 +++++++ .gitignore | 1 + run-docker.sh | 12 ++++++++++++ 3 files changed, 20 insertions(+) create mode 100644 .env.template create mode 100755 run-docker.sh diff --git a/.env.template b/.env.template new file mode 100644 index 0000000..b7e0965 --- /dev/null +++ b/.env.template @@ -0,0 +1,7 @@ +# DB Connection string +SQLALCHEMY_URL=postgresql://USER_HERE:PASSWORD_HERE@db:5432/imap + +# API credentials fpr LASP PODA API - See https://lasp.colorado.edu/ops/imap/poda/ +WEBPODA_AUTH_CODE= +# API credentials for SDC API - See https://github.com/IMAP-Science-Operations-Center/imap-data-access +SDC_AUTH_CODE= diff --git a/.gitignore b/.gitignore index 00081c1..7a6fd75 100644 --- a/.gitignore +++ b/.gitignore @@ -134,3 +134,4 @@ site/ .work /output +dev.env diff --git a/run-docker.sh b/run-docker.sh new file mode 100755 index 0000000..865faf7 --- /dev/null +++ b/run-docker.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e + +IMAGE_NAME="${IMAGE_NAME:-imap-pipeline-core/imap-mag}" + + +docker run --rm -it \ + --entrypoint /bin/sh \ + --env-file dev.env \ + -v /data:/data \ + $IMAGE_NAME + From 01e568f13a1e4b8a4503f140946a50072180be45 Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Mon, 29 Jul 2024 14:09:38 +0000 Subject: [PATCH 32/33] bug: fix docker permissions and missing files --- README.md | 10 ++++++++++ deploy/Dockerfile | 6 ++++++ run-docker.sh | 2 +- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e5678b5..4557258 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,16 @@ To use WebPODA APIs, an access token needs to be defined in the environment as ` pre-commit install ``` +5. To use the docker /data mount you need a folder on your WSL and a user with a given UID + +```bash +# in WSL on your HOST +mkdir -p /mnt/imap-data +sudo adduser -u 5678 --disabled-password --gecos "" appuser +# you have created the user with the same UID as in the container. now grant the folder to the user +chown -R appuser:appuser /mnt/imap-data +``` + ### Build, pack and test ```bash diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 49ddf57..f4fcb97 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -12,6 +12,10 @@ ENV PYTHONUNBUFFERED=1 COPY deploy/entrypoint.sh /app/entrypoint.sh COPY *.yaml /app/ + +#TODO: this is a hack mounting the src folder to the container +COPY src/imap_mag/xtce/*.xml /app/src/imap_mag/xtce/ + COPY dist/python${PYTHON_VERSION}/${TOOL_PACKAGE} /app/python${PYTHON_VERSION}/ # Creates a non-root user with an explicit UID and adds permission to access the /app folder @@ -19,6 +23,8 @@ COPY dist/python${PYTHON_VERSION}/${TOOL_PACKAGE} /app/python${PYTHON_VERSION}/ RUN adduser -u 5678 --disabled-password --gecos "" appuser && \ chown -R appuser /app && \ + mkdir -p /data && \ + chown -R appuser /data && \ chmod +x /app/entrypoint.sh # Install the postgres client and any other dependencies needed to install our app diff --git a/run-docker.sh b/run-docker.sh index 865faf7..c7122b6 100755 --- a/run-docker.sh +++ b/run-docker.sh @@ -7,6 +7,6 @@ IMAGE_NAME="${IMAGE_NAME:-imap-pipeline-core/imap-mag}" docker run --rm -it \ --entrypoint /bin/sh \ --env-file dev.env \ - -v /data:/data \ + -v /mnt/imap-data:/data \ $IMAGE_NAME From 5d916b77347742ec1c39c340adc11a56bb9a19df Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Mon, 29 Jul 2024 14:47:41 +0000 Subject: [PATCH 33/33] bug: delay boot for DB to start --- deploy/entrypoint.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy/entrypoint.sh b/deploy/entrypoint.sh index 577bc1e..c18e2a4 100755 --- a/deploy/entrypoint.sh +++ b/deploy/entrypoint.sh @@ -1,6 +1,8 @@ #!/bin/bash set -e +echo "Starting IMAP MAG pipeline..." +sleep 20 imap-db create-db