Use uv to manage dependencies

app/Dockerfile

@@ -7,21 +7,30 @@ WORKDIR /usr/src/app
 # set environment variables
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1
+ENV PYTHONPYCACHEPREFIX=/root/.cache/pycache/
+ENV PIP_CACHE_DIR=/var/cache/buildkit/pip
+
+RUN mkdir -p $PIP_CACHE_DIR
+RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
 
 # install system dependencies
-RUN apt-get update \
-  && apt-get --no-install-recommends install -y \
+RUN \
+  --mount=type=cache,target=/var/cache/apt,sharing=locked \
+  --mount=type=cache,target=/var/lib/apt,sharing=locked \
+  apt-get update \
+  && apt-get install --no-install-recommends -yqq \
   netcat=1.10-46 \
   gcc=4:10.2.1-1 \
   postgresql=13+225+deb11u1 \
-  graphviz=2.42.2-5 \
-  && apt-get clean \
-  && rm -rf /var/lib/apt/lists/*
+  graphviz=2.42.2-5
 
 # install dependencies
-RUN pip install --no-cache-dir pip~=21.3.1
 COPY ./requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
+# hadolint ignore=DL3042
+RUN \
+  --mount=type=cache,target=/root/.cache \
+  pip install uv==0.1.15 \
+  && uv pip install --system -r requirements.txt
 
 # copy entrypoint.sh
 COPY ./entrypoint.sh .

app/requirements.in

@@ -0,0 +1,17 @@
+black
+django~=4.2.0
+django-extensions
+django-linear-migrations
+django-phonenumber-field[phonenumbers]
+django-timezone-field
+djangorestframework
+drf-jwt
+drf-spectacular
+flake8
+isort
+markdown
+psycopg2-binary
+pydot
+pytest-cov
+pytest-django
+tzdata

app/requirements.txt

@@ -1,18 +1,107 @@
-cryptography~=37.0.2
-Django~=4.0.1
-django-extensions~=3.1.5
-django-linear-migrations~=2.12.0
-django-phonenumber-field[phonenumbers]~=6.3.0
-django-timezone-field==5.0
-djangorestframework~=3.13.1
-drf-jwt~=1.19.2
-drf-spectacular==0.22.1
-markdown~=3.4.1
-psycopg2-binary~=2.9.3
-pydot~=1.4.2
-pyparsing~=3.0.9
-pytest~=6.2.5
-pytest-cov~=3.0.0
-pytest-django~=4.5.2
-pytz==2022.1
-tzdata==2022.1
+asgiref==3.7.2
+    # via django
+attrs==23.2.0
+    # via
+    #   jsonschema
+    #   referencing
+black==24.2.0
+cffi==1.16.0
+    # via cryptography
+click==8.1.7
+    # via black
+coverage==7.4.3
+    # via pytest-cov
+cryptography==42.0.5
+    # via pyjwt
+django==4.2.11
+    # via
+    #   django-extensions
+    #   django-linear-migrations
+    #   django-phonenumber-field
+    #   django-timezone-field
+    #   djangorestframework
+    #   drf-jwt
+    #   drf-spectacular
+django-extensions==3.2.3
+django-linear-migrations==2.12.0
+django-phonenumber-field==7.3.0
+django-timezone-field==6.1.0
+djangorestframework==3.14.0
+    # via
+    #   drf-jwt
+    #   drf-spectacular
+drf-jwt==1.19.2
+drf-spectacular==0.27.1
+exceptiongroup==1.2.0
+    # via pytest
+flake8==7.0.0
+inflection==0.5.1
+    # via drf-spectacular
+iniconfig==2.0.0
+    # via pytest
+isort==5.13.2
+jsonschema==4.21.1
+    # via drf-spectacular
+jsonschema-specifications==2023.12.1
+    # via jsonschema
+markdown==3.5.2
+mccabe==0.7.0
+    # via flake8
+mypy-extensions==1.0.0
+    # via black
+packaging==23.2
+    # via
+    #   black
+    #   pytest
+pathspec==0.12.1
+    # via black
+phonenumbers==8.13.31
+    # via django-phonenumber-field
+platformdirs==4.2.0
+    # via black
+pluggy==1.4.0
+    # via pytest
+psycopg2-binary==2.9.9
+pycodestyle==2.11.1
+    # via flake8
+pycparser==2.21
+    # via cffi
+pydot==2.0.0
+pyflakes==3.2.0
+    # via flake8
+pyjwt==2.8.0
+    # via drf-jwt
+pyparsing==3.1.2
+    # via pydot
+pytest==8.0.2
+    # via
+    #   pytest-cov
+    #   pytest-django
+pytest-cov==4.1.0
+pytest-django==4.8.0
+pytz==2024.1
+    # via djangorestframework
+pyyaml==6.0.1
+    # via drf-spectacular
+referencing==0.33.0
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
+rpds-py==0.18.0
+    # via
+    #   jsonschema
+    #   referencing
+sqlparse==0.4.4
+    # via django
+tomli==2.0.1
+    # via
+    #   black
+    #   coverage
+    #   pytest
+typing-extensions==4.10.0
+    # via
+    #   asgiref
+    #   black
+tzdata==2024.1
+uritemplate==4.1.1
+    # via drf-spectacular

docs/tools/docker.md

@@ -0,0 +1,25 @@
+# Docker
+
+## Cache mount
+
+This helps speed up subsequent docker builds by caching intermediate files and reusing them across builds. It's available with docker buildkit. The key here is to disable anything that could delete the cache, because we want to preserve it. The cache mount is not going to end up in the docker image being built, so there's no concern about disk space usage.
+
+Put this flag between `RUN` and the command
+
+```docker hl_lines="2"
+RUN \
+--mount=type=cache,target=/root/.cache
+  pip install -r requirements.txt
+```
+
+For pip, the files are by default stored in `/root/.cache/pip`.  [Pip caching docs](https://pip.pypa.io/en/stable/topics/caching/)
+
+For apk, the cache directory is `/var/cache/apk/`. [APK wiki on local cache](https://wiki.alpinelinux.org/wiki/Local_APK_cache)
+
+For apt, the cache directory is `/var/cache/apt/`.
+
+??? info "References"
+    - [buildkit mount the cache](https://vsupalov.com/buildkit-cache-mount-dockerfile/)
+    - [proper usage of mount cache](https://dev.doroshev.com/blog/docker-mount-type-cache/)
+    - [mount cache reference](https://docs.docker.com/engine/reference/builder/#run---mounttypecache)
+    - [buildkit dockerfile reference](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md)

docs/tools/scripts.md

@@ -26,3 +26,10 @@ These scripts assume you are using bash.
 1. **createsuperuser.sh** - creates a default superuser.
 
     1. This assumes that `DJANGO_SUPERUSER_USERNAME` and `DJANGO_SUPERUSER_PASSWORD` are set in `.env.dev`
+
+1. **erd.sh** - generate ER diagram
+
+    - The image is saved to `app/erd.png`
+    - This script is dependent on the `graphviz` package
+
+1. **update-dependencies.sh** - update python dependencies to the latest versions

docs/tools/uv.md

@@ -0,0 +1,69 @@
+# uv
+
+We're using `uv` as a faster replacement to `pip` and `pip-tools`. See the [official documentation on getting started](https://github.com/astral-sh/uv#getting-started).
+
+## How we use it
+
+We're using `uv` to compile and install python dependencies, which replaces the functionalities of `pip` and `pip-tools`. `uv` can also create and maintain a virtual environment but we're not using it for now. In fact, we're suppressing it with the `--system` option during `uv pip install`.
+
+`uv` is already part of the `docker` image, so there's no need to install it on the host. It does require prepending the docker-compose information to run, for example: `docker-compose exec web uv pip compile requirements.in -o requirements.txt`. We'll omit the `docker-compose exec web` portion from now on in this document.
+
+`requirements.in` is the requirements file and `uv pip compile` generates `requirement.txt`, with pinned versions, similar to lock files in other languages.
+
+## Usage
+
+### Upgrade depencencies
+
+We shouldn't run this on every build, but we should do this manually every month/quarter or so.
+
+```bash
+# docker-compose exec web
+uv pip compile requirements.in -o requirements.txt --no-header --upgrade
+```
+
+Or run the script
+
+```bash
+./scripts/update-dependencies.sh
+```
+
+#### pip compile options
+
+Disable header in the generated file
+: `--no-header` This solves the problem unnecessary code churn caused by changing headers
+
+Upgrade all dependencies
+: `--upgrade`
+
+Generate pip-8 style hashes
+: `--generate-hashes` Hashes improve security but are not verified by `uv` at the moment. It is planned. Switch back to `pip` for installation if we need to verify hashes.
+
+Disable annotation of where dependencies come from
+: `--no-annotate` This makes the generated file shorter but less informative
+
+See [pip-compile docs](https://pip-tools.readthedocs.io/en/stable/cli/pip-compile/) for more options and explanation
+
+### Install dependencies
+
+This is used in the `Dockerfile` to install python dependencies.
+
+```bash
+uv pip install --system -r requirements.txt
+```
+
+#### pip install options
+
+Install to global
+: `--system` bypass the virtual environment requirement
+
+See [pip install docs](https://pip.pypa.io/en/stable/cli/pip_install/) for more options and explanation
+
+## Explanations
+
+### Global install
+
+We're using the `--system` option in the `Dockerfile` to bypass the virtual environment requirement for `uv`. This is because the docker image is already a virtual environment separate from the host.
+
+### Version pinning
+
+We're leaving most dependencies unpinned in `requirements.in` so that `pip compile` will pin the newest compatible versions in `requirements.txt`. The only manually pinned dependency is `django~=4.2.0`. The `x.2.x` versions have long term support, and we're using `4`, since `4.2` is the latest LTS available.

mkdocs.yml

@@ -23,6 +23,7 @@ markdown_extensions:
   - abbr
   - admonition
   - attr_list
+  - def_list
   - md_in_html
   - pymdownx.betterem
   - pymdownx.blocks.details

scripts/update-dependencies.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+set -x
+
+# generate requirements.txt with the latest package versions
+docker-compose exec web uv pip compile -o requirements.txt requirements.in --no-header --upgrade