Merge pull request #16 from ImperialCollegeLondon/feat/db-app-plus-po…

…da-and-sdc

Feat/db app plus poda and sdc

.devcontainer/Dockerfile

@@ -13,3 +13,6 @@ ENV PATH="$PYENV_ROOT/bin:$PATH"
 RUN curl https://pyenv.run | bash
 RUN $PYENV_ROOT/bin/pyenv install -v ${PYTHON_VERSIONS}
 
+USER root
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    postgresql-client

.devcontainer/devcontainer.json

@@ -31,7 +31,9 @@
     "remoteEnv": {
         // add the pyenv bin and poetry to the PATH for the dev user vscode
         "PATH": "/home/vscode/.pyenv/bin:/home/vscode/.local/bin:${containerEnv:PATH}",
-        "WEBPODA_AUTH_CODE": "${localEnv:WEBPODA_AUTH_CODE}"
+        "WEBPODA_AUTH_CODE": "${localEnv:WEBPODA_AUTH_CODE}",
+        "SDC_AUTH_CODE": "${localEnv:SDC_AUTH_CODE}",
+        "IMAP_DATA_ACCESS_URL": "${localEnv:IMAP_DATA_ACCESS_URL}",
     },
     "postCreateCommand": "pip install --user -U pre-commit && pre-commit install-hooks && pre-commit autoupdate",
     // install poetry in first startup in vscode

.env.template

@@ -0,0 +1,7 @@
+# DB Connection string
+SQLALCHEMY_URL=postgresql://USER_HERE:PASSWORD_HERE@db:5432/imap
+
+# API credentials fpr LASP PODA API - See https://lasp.colorado.edu/ops/imap/poda/
+WEBPODA_AUTH_CODE=
+# API credentials for SDC API - See https://github.com/IMAP-Science-Operations-Center/imap-data-access
+SDC_AUTH_CODE=

.github/workflows/ci.yml

@@ -107,7 +107,7 @@ jobs:
             # use custom value instead of git tag
             type=semver,pattern={{version}},value=${{ env.PACKAGE_VERSION }}
             # set latest tag for default branch
-            type=raw,value=latest,enable={{is_default_branch}}
+            type=raw,value=latest
             # branch event
             type=ref,event=branch
             # pull request event

.gitignore

@@ -131,8 +131,7 @@ dmypy.json
 
 # mkdocs build dir
 site/
-src/main.py
 
 .work
 /output
-deploy/test-env/ssh_test_keys/
+dev.env

.vscode/launch.json

@@ -4,7 +4,7 @@
             "args": [
                 "process",
                 "--config",
-                "config.yml"
+                "config.yaml"
             ],
             "console": "integratedTerminal",
             "justMyCode": true,

.vscode/settings.json

@@ -14,5 +14,20 @@
     "cSpell.words": [
         "xtce",
         "xtcedef"
+    ],
+    "python.analysis.extraPaths": [
+        "./src"
+    ],
+    "sqltools.connections": [
+        {
+            "previewLimit": 50,
+            "server": "host.docker.internal",
+            "port": 5432,
+            "driver": "PostgreSQL",
+            "name": "postgres",
+            "password": "",
+            "database": "imap",
+            "username": "postgres"
+        }
     ]
 }

README.md

@@ -29,6 +29,16 @@ To use WebPODA APIs, an access token needs to be defined in the environment as `
     pre-commit install
     ```
 
+5. To use the docker /data mount you need a folder on your WSL and a user with a given UID
+
+```bash
+# in WSL on your HOST
+mkdir -p /mnt/imap-data
+sudo adduser -u 5678 --disabled-password --gecos "" appuser
+# you have created the user with the same UID as in the container. now grant the folder to the user
+chown -R appuser:appuser /mnt/imap-data
+```
+
 ### Build, pack and test
 
 ```bash

build-linux.sh

@@ -7,7 +7,7 @@
 mkdir -p dist
 
 docker run \
-  --volume "$(pwd):/src/" \
+  --volume "$(pwd):/src/imap_mag" \
    batonogov/pyinstaller-linux:latest \
   "rm -rf dist/pyinstaller && \
   python3 -m pip install poetry && \

build-windows.sh

@@ -7,7 +7,7 @@
 mkdir -p dist
 
 docker run \
-  --volume "$(pwd):/src/" \
+  --volume "$(pwd):/src/imap_mag" \
    batonogov/pyinstaller-windows:latest \
   "rm -rf dist/pyinstaller && \
   python -m pip install poetry && \

calibration_application_config.yaml

@@ -0,0 +1,8 @@
+source:
+  folder: /data/science/imap/mag/l1b/2025/05
+
+work-folder: .work
+
+destination:
+  folder: /data/science/imap/mag/l2/2025/05
+  filename: imap_mag_l2_norm-mago_20250511_v000.cdf

calibration_config.yaml

@@ -0,0 +1,8 @@
+source:
+  folder: /data/science/imap/mag/l1b/2025/05
+
+work-folder: .work
+
+destination:
+  folder: /data/science/imap/mag/l1b/2025/05
+  filename: calibration.json

config-hk-download.yaml

@@ -0,0 +1,11 @@
+source:
+  folder: /data/hk_l0/
+
+work-folder: /data/.work
+
+destination:
+  folder: /data/hk_l0/
+  filename: power.pkts
+
+packet-definition:
+  hk: src/imap_mag/xtce/tlm_20240724.xml

config-hk-process.yaml

@@ -0,0 +1,11 @@
+source:
+  folder: /data/hk_l0/
+
+work-folder: /data/.work
+
+destination:
+  folder: /data/hk_l1/
+  filename: result.csv
+
+packet-definition:
+  hk: src/imap_mag/xtce/tlm_20240724.xml

config-sci.yaml

@@ -0,0 +1,11 @@
+source:
+  folder: tests/data/2025
+
+work-folder: /data/science
+
+destination:
+  folder: output/
+  filename: result.cdf
+
+packet-definition:
+  hk: src/imap_mag/xtce/tlm_20240724.xml

config.yml → config.yaml

@@ -8,4 +8,4 @@ destination:
   filename: result.cdf
 
 packet-definition:
-  hk: src/xtce/tlm_20240724.xml
+  hk: src/imap_mag/xtce/tlm_20240724.xml

deploy/Dockerfile

@@ -11,15 +11,25 @@ ENV PYTHONDONTWRITEBYTECODE=1
 ENV PYTHONUNBUFFERED=1
 
 COPY deploy/entrypoint.sh /app/entrypoint.sh
+COPY *.yaml /app/
+
+#TODO: this is a hack mounting the src folder to the container
+COPY src/imap_mag/xtce/*.xml /app/src/imap_mag/xtce/
+
 COPY dist/python${PYTHON_VERSION}/${TOOL_PACKAGE} /app/python${PYTHON_VERSION}/
 
 # Creates a non-root user with an explicit UID and adds permission to access the /app folder
 # For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers
 
 RUN adduser -u 5678 --disabled-password --gecos "" appuser && \
     chown -R appuser /app && \
+    mkdir -p /data && \
+    chown -R appuser /data && \
     chmod +x /app/entrypoint.sh
 
+# Install the postgres client and any other dependencies needed to install our app
+RUN apt-get update && apt-get install -y libpq-dev gcc
+
 WORKDIR /app
 USER appuser
 

deploy/entrypoint.sh

@@ -1,13 +1,34 @@
 #!/bin/bash
 set -e
 
+echo "Starting IMAP MAG pipeline..."
+sleep 20
+
+imap-db create-db
+
+imap-db upgrade-db
+
+echo "DB admin complete"
 
 while :
 do
-    imap-mag hello world
+    # delete all data
+    rm -rf /data/*
+
+    START_DATE='2025-05-02'
+    END_DATE='2025-05-03'
+
+    imap-mag fetch-binary --config config-hk-download.yaml --apid 1063 --start-date $START_DATE --end-date $END_DATE
+
+    imap-mag process --config config-hk-process.yaml power.pkts
+
+    imap-mag fetch-science --level l1b  --start-date $START_DATE --end-date $END_DATE --config config-sci.yaml
+
+    imap-db query-db
+
+    imap-mag calibrate --config calibration_config.yaml --method SpinAxisCalibrator imap_mag_l1b_norm-mago_20250511_v000.cdf
 
-    imap-mag fetch-binary --config tests/config/hk_download.yaml --apid 1063 --start-date 2025-05-02 --end-date 2025-05-03
-    imap-mag process --config tests/config/hk_process.yaml MAG_HSK_PW.pkts
+    imap-mag apply --config calibration_application_config.yaml --calibration calibration.json imap_mag_l1b_norm-mago_20250511_v000.cdf
 
     ls -l /data