feat: save new files to db after downoad

config-hk-download.yaml

@@ -0,0 +1,9 @@
+
+work-folder: /data/.work
+
+destination:
+  folder: /data/hk_l0/
+  filename: power.pkts
+
+packet-definition:
+  hk: src/imap_mag/xtce/tlm_20240724.xml

config-hk-process.yaml

@@ -0,0 +1,11 @@
+source:
+  folder: /data/hk_l0/
+
+work-folder: /data/.work
+
+destination:
+  folder: /data/hk_l1/
+  filename: result.csv
+
+packet-definition:
+  hk: src/imap_mag/xtce/tlm_20240724.xml

config-sci.yml

@@ -0,0 +1,4 @@
+
+work-folder: /data/science
+
+

deploy/entrypoint.sh

@@ -1,20 +1,25 @@
 #!/bin/bash
 set -e
 
-echo "start DB admin"
+
 imap-db create-db
+
 imap-db upgrade-db
+
 echo "DB admin complete"
 
 while :
 do
-    ls -l /data
+    # delete all data
+    rm -rf /data/*
+
+    imap-mag fetch-binary --config config-hk-download.yaml --apid 1063 --start-date 2025-05-02 --end-date 2025-05-03
 
-    imap-mag fetch-binary --config tests/config/hk_download.yaml --apid 1063 --start-date 2025-05-02 --end-date 2025-05-03
+    imap-mag process --config config-hk-process.yaml MAG_HSK_PW.pkts
 
-    imap-mag process --config tests/config/hk_process.yaml MAG_HSK_PW.pkts
+    imap-mag fetch-science --start-date 2025-05-02 --end-date 2025-05-03 --config config-sci.yaml
 
-    imap-mag fetch-science --start-date 2025-05-02 --end-date 2025-05-03
+    imap-db query-db
 
     ls -l /data
 

src/imap_db/main.py

@@ -66,7 +66,8 @@ def drop_db():
 def query_db():
     session = Session(engine)
 
-    stmt = select(File).where(File.name.in_(["file1.txt"]))
+    # stmt = select(File).where(File.name.in_(["file1.txt"]))
+    stmt = select(File).where(File.name is not None)
 
     for user in session.scalars(stmt):
         print(user)

src/imap_mag/DB.py

@@ -0,0 +1,38 @@
+import os
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from src.imap_db.model import File
+
+
+class DB:
+    def __init__(self, db_url=None):
+        env_url = os.getenv("SQLALCHEMY_URL")
+        if db_url is None and env_url is not None:
+            db_url = env_url
+
+        self.engine = create_engine(db_url)
+        self.Session = sessionmaker(bind=self.engine)
+
+    def insert_files(self, files: list[File]):
+        session = self.Session()
+        try:
+            for file in files:
+                # check file does not already exist
+                existing_file = (
+                    session.query(File)
+                    .filter_by(name=file.name, path=file.path)
+                    .first()
+                )
+                if existing_file is not None:
+                    continue
+
+                session.add(file)
+
+            session.commit()
+        except Exception as e:
+            session.rollback()
+            raise e
+        finally:
+            session.close()