Merge pull request #1 from mlibrary/database-with-fastapi

DWI-23: Database with fastapi

.devcontainer/devcontainer.json

@@ -19,8 +19,8 @@
 	"workspaceFolder": "/app",
 	"features": {
 		"ghcr.io/devcontainers/features/git:1": {},
-		"ghcr.io/devcontainers-contrib/features/ruff:1": {},
-		"ghcr.io/devcontainers/features/sshd:1": {}
+		"ghcr.io/devcontainers/features/sshd:1": {},
+		"ghcr.io/devcontainers/features/python:1": {}
 	}
 
 	// Features to add to the dev container. More info: https://containers.dev/features.

.github/pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+addopts = --cov=aim --cov-report=term-missing:skip-covered --no-header

.github/workflows/tests.yaml

@@ -0,0 +1,29 @@
+name: Run Tests
+
+on: push
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Create .env file
+        run: cat env.* > .env
+      - name: Load .env file
+        uses: xom9ikk/[email protected]
+      - name: Install poetry
+        run: pipx install poetry
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'poetry'
+      - run: poetry install
+      - name: setup pytest.ini
+        run: mv .github/pytest.ini pytest.ini
+      - name: Linting
+        run: poetry run ruff check
+      - name: Run tests
+        env:
+          CI: "true"
+        run: poetry run pytest 

.gitignore

@@ -9,4 +9,7 @@ __pycache__
 .local
 .gitconfig
 .venv
-.ssh
+.ssh
+htmlcov/
+.coverage
+.gnupg

Dockerfile

@@ -22,6 +22,11 @@ ARG GID=1000
 RUN groupadd -g ${GID} -o app
 RUN useradd -m -d /app -u ${UID} -g ${GID} -o -s /bin/bash app
 
+RUN apt-get update -yqq && apt-get install -yqq --no-install-recommends \
+  python3-dev \ 
+  default-libmysqlclient-dev \ 
+  build-essential \ 
+  pkg-config
 
 # Set the working directory to /app
 WORKDIR /app
@@ -31,6 +36,13 @@ CMD ["tail", "-f", "/dev/null"]
 # Both build and development need poetry, so it is its own step.
 FROM base as poetry
 
+
+RUN apt-get update -yqq && apt-get install -yqq --no-install-recommends \
+  python3-dev \ 
+  default-libmysqlclient-dev \ 
+  build-essential \ 
+  pkg-config
+
 RUN pip install poetry==${POETRY_VERSION}
 
 # Use this page as a reference for python and poetry environment variables: https://docs.python.org/3/using/cmdline.html#envvar-PYTHONUNBUFFERED

README.md

@@ -1,75 +1,65 @@
-# python-docker-boilerplate
-
-Boilerplate code for starting a python project with docker and docker-compose
-
-## How to set up your python environment
-
-### Install python
-
-On mac,
-
-* You can read this blog to install python in a right way in
-      python: https://opensource.com/article/19/5/python-3-default-mac
-
-* **Recommendation**: Install python using brew and pyenv
-
-### Managing python dependencies
-
-* **Install poetry**
-
-* On Mac OS, Windows and Linux,
-  * Install poetry:
-       * ``curl -sSL https://install.python-poetry.org | python3 -``
-         * This way allows poetry and its dependencies to be isolated from your dependencies. I don't recommend to use 
-         * pip to install poetry because poetry and your application dependencies will be installed in the same environment.
-       * ```poetry init```: 
-         * Use this command to set up your local environment, repository details, and dependencies. 
-         * It will generate a pyproject.toml file with the information you provide.
-           * Package name [python-starter]:
-           * Version [0.1.0]:
-           * Description []:
-           * Author []:  n 
-           * License []:
-           * Compatible Python versions [^3.11]: 
-           * Would you like to define your main dependencies interactively? (yes/no) [yes]: no
-           * Would you like to define your development dependencies interactively? (yes/no) no
-       * ```poetry install```: 
-         * Use this command to automatically install the dependencies specified in the pyproject.toml file.
-         * It will generate a poetry.lock file with the dependencies and their versions.
-         * It will create a virtual environment in the home directory, e.g. /Users/user_name/Library/Caches/pypoetry/..
-       * ```poetry env use python```: 
-         * Use this command to find the virtual environment directory, created by poetry.
-       * ```source ~/Library/Caches/pypoetry/virtualenvs/python-starter-0xoBsgdA-py3.11/bin/activate```
-         * Use this command to activate the virtual environment.
-       * ```poetry shell```: 
-         * Use this command to activate the virtual environment.
-       * ```poetry add pytest```: 
-         * Use this command to add dependencies.
-       * ```poetry add --dev pytest```:
-         * Use this command to add development dependencies.
-       * `` poetry update ``: 
-         * Use this command if you change your .toml file and want to generate a new version the .lock file
-
-## Set up in a docker environment
+# AIM-py 
+
+AIM's python code repository
+
+## Setup
+
+1. Clone the repo
 
 ```
-./init.sh
+git clone https://github.com/mlibrary/aim-py.git
+cd aim-py
 ```
 
+2. In the terminal, run the `init.sh` 
+```
+./init.sh
+```
 This will:
 
-* copy the project folder
+* set up the initial environment variables file
 * build the docker image
-* install the dependencies
-* create a container with the application
+* install the python dependencies
+* Set up the database for digifeeds
+
+`./init.sh` can be run more than once. 
+
+3. Edit `.env` with actual environment variables
+
+4. If using VSCode for editing, the repository is set up for use with dev containers. You will have to rebuild the container in there. 
+
+5. In the app container, use `poetry shell` to enable the virtual environment. Otherwise use:
+```
+ docker compose run --rm app poetry run YOUR_COMMAND
+```
+
+## Projects
+
+### Digifeeds
 
-## How to run the application
+Digifeeds code is in the `aim/digifeeds` folder. The `database` folder has the code for the database and its web API. 
 
-``docker compose exec app python --version``
+#### Database
+To run database migrations:
+```
+cd aim/digifeeds/database
+alembic upgrade heads
+```
+The alembic migrations live in the `aim/digifeeds/database/migrations` folder.
 
+#### Web API for the Database
+To run the api:
+
+In the `/app` folder of the container run:
+```
+uvicorn aim.digifeeds.database.main:app --host 0.0.0.0 --reload
+```
+
+In the browser go to: http://localhost:8000/docs to work with the API.
 
 ## Tests
 
-## Background
-This repository goes with this documentation:
-https://mlit.atlassian.net/wiki/spaces/LD/pages/10092544004/Python+in+LIT
+To run tests:
+```
+docker compose run --rm app poetry run pytest
+```

aim/digifeeds/bin/load_statuses.py

@@ -0,0 +1,10 @@
+from aim.digifeeds.database.models import load_statuses
+from aim.digifeeds.database.main import SessionLocal
+import sys
+
+def main():
+    with SessionLocal() as db_session:
+      load_statuses(session=db_session)
+
+if __name__=='__main__':
+    sys.exit(main())

aim/digifeeds/database/alembic.ini

@@ -0,0 +1,117 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+# Use forward slashes (/) also on windows to provide an os agnostic path
+script_location = migrations
+
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+# Uncomment the line below if you want the files to be prepended with date and time
+# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
+# for all available tokens
+# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python>=3.9 or backports.zoneinfo library.
+# Any required deps can installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to ZoneInfo()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to migrations/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:migrations/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+
+# set to 'true' to search source files recursively
+# in each "version_locations" directory
+# new in Alembic version 1.10
+# recursive_version_locations = false
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+#in env.py
+#sqlalchemy.url = driver://user:pass@localhost/dbname
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# lint with attempts to fix using "ruff" - use the exec runner, execute a binary
+# hooks = ruff
+# ruff.type = exec
+# ruff.executable = %(here)s/.venv/bin/ruff
+# ruff.options = --fix REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

aim/digifeeds/database/crud.py

@@ -0,0 +1,52 @@
+from sqlalchemy.orm import Session
+from aim.digifeeds.database import schemas
+from aim.digifeeds.database import models
+
+
+def get_item(db: Session, barcode: str):
+    return db.query(models.Item).filter(models.Item.barcode == barcode).first()
+
+
+def get_items(db: Session, in_zephir: bool | None):
+    if in_zephir is True:
+        return (
+            db.query(models.Item)
+            .filter(
+                models.Item.statuses.any(models.ItemStatus.status_name == "in_zephir")
+            )
+            .all()
+        )
+    elif in_zephir is False:
+        return (
+            db.query(models.Item)
+            .filter(
+                ~models.Item.statuses.any(models.ItemStatus.status_name == "in_zephir")
+            )
+            .all()
+        )
+
+    return db.query(models.Item).all()
+
+
+def add_item(db: Session, item: schemas.ItemCreate):
+    db_item = models.Item(barcode=item.barcode)
+    db.add(db_item)
+    db.commit()
+    db.refresh(db_item)
+    return db_item
+
+
+def get_status(db: Session, name: str):
+    return db.query(models.Status).filter(models.Status.name == name).first()
+
+
+def get_statuses(db: Session):
+    return db.query(models.Status).all()
+
+
+def add_item_status(db: Session, item: models.Item, status: models.Status):
+    db_item_status = models.ItemStatus(item=item, status=status)
+    db.add(db_item_status)
+    db.commit()
+    db.refresh(item)
+    return item