Use poetry for dependency management (#394)

* Replace various requirements files with `pyproject.toml` and `poetry.lock`

One consequence of this was to downgrade to `pytest 6.x` for unit tests, since `pytest 7.x` is not compatible with `pytest-selenium`

* Use poetry in Makefile, Dockerfiles, docs generation, and CircleCI

* Update README and CHANGELOG

.circleci/config.yml

@@ -8,11 +8,11 @@ version: 2.1
 aliases:
   - &restore_deps_cache
     name: Restoring Python dependency cache
-    key: v3-requirements-{{ checksum ".python.installed" }}-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}
+    key: v4-requirements-{{ checksum "poetry.lock" }}
 
   - &save_deps_cache
     name: Saving Python dependency cache
-    key: v3-requirements-{{ checksum ".python.installed" }}-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}
+    key: v4-requirements-{{ checksum "poetry.lock" }}
     paths:
       - .venv
 
@@ -233,17 +233,14 @@ jobs:
 
   unit_test:
     docker:
-      - image: python:3.10.5-bullseye@sha256:dac61c6d3e7ac6deb2926dd96d38090dcba0cb1cf9196ccc5740f25ebe449f50
+      - image: cimg/python:3.11.2
       - image: cimg/postgres:12.8
         environment:
           POSTGRES_DB: testdb
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
     steps:
       - checkout
-      - run:
-          name: Python version
-          command: python3 --version > .python.installed
       - restore_cache: *restore_deps_cache
       - run:
           name: Run kinto_remote_settings plugin unit tests
@@ -252,12 +249,9 @@ jobs:
 
   lint_format:
     docker:
-      - image: python:3.10.5-bullseye@sha256:dac61c6d3e7ac6deb2926dd96d38090dcba0cb1cf9196ccc5740f25ebe449f50
+      - image: cimg/python:3.11.2
     steps:
       - checkout
-      - run:
-          name: Python version
-          command: python3 --version > .python.installed
       - restore_cache: *restore_deps_cache
       - run:
           name: Check linting and formatting
@@ -266,17 +260,17 @@ jobs:
 
   docs_test:
     docker:
-      - image: python:3.10.5-bullseye@sha256:dac61c6d3e7ac6deb2926dd96d38090dcba0cb1cf9196ccc5740f25ebe449f50
+      - image: cimg/python:3.11.2
     steps:
       - checkout
       - restore_cache:
-          name: Restoring Python docs dependency cache
-          key: v1-requirements-{{ checksum "docs/requirements.txt" }}
+          name: Restoring docs dependency cache
+          key: v2-docs-requirements-{{ checksum "poetry.lock" }}
       - run:
           name: Check documentation build
           command: make docs
-      - save_cache:
-          name: Saving Python docs dependency cache
-          key: v1-requirements-{{ checksum "docs/requirements.txt" }}
+      - save_cache: 
+          name: Saving docs dependency cache
+          key: v2-docs-requirements-{{ checksum "poetry.lock" }}
           paths:
             - .venv

.readthedocs.yaml

@@ -1,6 +1,12 @@
 version: 2
 
-python:
-   install:
-   - requirements: docs/requirements.txt
-
+build:
+  os: "ubuntu-22.04"
+  tools:
+    python: "3.11"
+  jobs:
+    post_create_environment:
+      - pip install poetry
+      - poetry config virtualenvs.create false
+    post_install:
+      - poetry install --only docs

CHANGELOG.rst

@@ -13,6 +13,7 @@ the version control of each dependency.
 - Improve Docker volume management
 - Add Docker compose profile support for test services
 - Move ``kinto-remote-settings`` plugin build step to ``compile`` stage of Dockerfile
+- Replace ``pip-tools`` with ``poetry`` for dependency management
 
 31.0.1 (2023-02-14)
 ===================

IntegrationTests.Dockerfile

@@ -1,16 +1,32 @@
-FROM python:3.11.3-slim
+FROM python:3.11.3-slim as build
+
+ENV PIP_NO_CACHE_DIR=off \
+    PIP_DISABLE_PIP_VERSION_CHECK=on \
+    POETRY_HOME="/opt/poetry" \
+    POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=true \
+    VIRTUAL_ENV=/opt/.venv \
+    PATH="/opt/.venv/bin:$PATH"
+
+# Install Poetry
+RUN python -m venv $POETRY_HOME && \
+    $POETRY_HOME/bin/pip install poetry==1.4.1 && \
+    $POETRY_HOME/bin/poetry --version
 
-ENV VIRTUAL_ENV=/opt/venv
-RUN python3 -m venv $VIRTUAL_ENV
-ENV PATH="$VIRTUAL_ENV/bin:$PATH" \ 
+WORKDIR /opt
+COPY pyproject.toml poetry.lock ./
+RUN $POETRY_HOME/bin/poetry install --only integration-tests --no-root
+
+FROM python:3.11.3-slim
+ENV PATH="/opt/.venv/bin:$PATH" \
     PYTHONUNBUFFERED=1 \
+    VIRTUAL_ENV=/opt/.venv \
     PYTHONPATH="/app:$PYTHONPATH"
 
 COPY /bin/update_and_install_system_packages.sh /opt
 RUN /opt/update_and_install_system_packages.sh wget
 
-COPY tests/requirements.txt /opt
-RUN pip install --no-cache-dir -r /opt/requirements.txt
+COPY --from=build $VIRTUAL_ENV $VIRTUAL_ENV
 
 WORKDIR /app
 COPY tests/ pyproject.toml ./

Makefile

@@ -22,11 +22,10 @@ maintainer-clean: distclean
 $(VENV)/bin/python:
 	python3 -m venv $(VENV)
 
-$(INSTALL_STAMP): $(VENV)/bin/python requirements.txt requirements-dev.txt
-	$(VENV)/bin/python -m pip install --upgrade pip wheel setuptools
-	$(VENV)/bin/pip install -r requirements.txt
-	$(VENV)/bin/pip install -e kinto-remote-settings
-	$(VENV)/bin/pip install -r requirements-dev.txt
+install: $(INSTALL_STAMP)
+$(INSTALL_STAMP): poetry.lock
+	@if [ -z $(shell command -v poetry 2> /dev/null) ]; then echo "Poetry could not be found. See https://python-poetry.org/docs/"; exit 2; fi
+	POETRY_VIRTUALENVS_IN_PROJECT=1 poetry install --no-root
 	touch $(INSTALL_STAMP)
 
 format: $(INSTALL_STAMP)
@@ -79,8 +78,8 @@ down:
 	docker-compose down
 
 install-docs: $(DOC_STAMP)
-$(DOC_STAMP): $(VENV)/bin/python docs/requirements.txt
-	$(VENV)/bin/pip install -Ur docs/requirements.txt
+$(DOC_STAMP): poetry.lock
+	POETRY_VIRTUALENVS_IN_PROJECT=1 poetry install --only docs
 	touch $(DOC_STAMP)
 
 docs: install-docs

README.rst

@@ -13,23 +13,29 @@ https://remote-settings.readthedocs.io
 Content
 -------
 
-This *Remote Settings* repository contains the following:
+This *Remote Settings* repository contains the following files and directories of note:
 
 * ``bin/``: container entry point and script(s)
 * ``config/``: example configuration file(s)
 * ``docs/``: documentation source files
 * ``kinto-remote-settings/``: Kinto plugin specific to Remote Settings
 * ``tests/``: browser and integration tests
-* ``requirements.in``: Python packages for the service (source of truth for ``requirements.txt``)
-* ``requirements-dev.txt``: Python packages for local development and tests
+* ``pyproject.toml``: contains dependency information and (most) config settings
 * ``VERSION``: SemVer version number that serves as both the version of the service and the ``kinto-remote-settings`` plugin
 
+Setup
+-----
+
+You will need:
+
+- Docker
+- ``docker-compose`` with `buildkit <https://docs.docker.com/develop/develop-images/build_enhancements/>`_ enabled
+- `poetry <https://python-poetry.org/>`_
+- `Make <https://www.gnu.org/software/make/>`_
 
 Run
 ---
 
-You need Docker and ``docker-compose``. Ensure `buildkit <https://docs.docker.com/develop/develop-images/build_enhancements/>`_ is enabled on your Docker engine.
-
 .. code-block:: shell
 
     make start
@@ -191,23 +197,24 @@ Now, from that ``bash`` session you can reach the other services like:
 Upgrade Things
 --------------
 
-Most common use-case is that you want to upgrade one of the dependencies.
+Dependabot is enabled on this repository, so it should keep dependencies up to date. 
 
-Top level dependencies are listed in ``requirements.in``.
-
-We use `pip-tools's pip-compile <https://pypi.org/project/pip-tools/>`_ command to generate the exhaustive list of pinned dependencies with their hash.
-
-To upgrade a single package, run:
+To manually edit dependency versions, use `standard poetry commands <https://python-poetry.org/docs/master/managing-dependencies/>`_. Because our
+usecase is somewhat complex with multiple groups and some dependencies appearing
+in multiple groups, sometimes the easiest way to update packages is to edit
+``pyproject.toml`` to the specified package version, then run:
 
 .. code-block:: shell
 
-    pip-compile --upgrade-package kinto-attachment
+    poetry lock --no-update
+
+to update the lockfile.
 
 To test that this installs run:
 
 .. code-block:: shell
 
-    docker-compose build web
+    make install
 
 
 About versioning

RemoteSettings.Dockerfile

@@ -1,24 +1,39 @@
 FROM python:3.11.3 as compile
 
-ENV VIRTUAL_ENV=/opt/venv
-RUN python3 -m venv $VIRTUAL_ENV
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+ENV PIP_NO_CACHE_DIR=off \
+    PIP_DISABLE_PIP_VERSION_CHECK=on \
+    POETRY_HOME="/opt/poetry" \
+    POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=true \
+    VIRTUAL_ENV=/opt/.venv \
+    PATH="/opt/.venv/bin:$PATH"
+
+# Install Poetry
+RUN python -m venv $POETRY_HOME && \
+    $POETRY_HOME/bin/pip install poetry==1.4.1 && \
+    $POETRY_HOME/bin/poetry --version
 
 WORKDIR /opt
-COPY requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt && \
+COPY ./poetry.lock ./pyproject.toml ./
+RUN $POETRY_HOME/bin/poetry install --only main --no-root && \
     uwsgi --build-plugin https://github.com/Datadog/uwsgi-dogstatsd
+
+# though we have kinto-remote-settings specified as a dependency in
+# pyproject.toml, we have it configured to install in editable mode for local
+# development. For building the container, we only install the "main"
+# dependency group so that we can use pip to install the packages in
+# non-editable mode
 COPY ./kinto-remote-settings ./kinto-remote-settings
 COPY VERSION .
-RUN pip install --no-cache-dir ./kinto-remote-settings
+RUN pip install ./kinto-remote-settings
 
 FROM python:3.11.3-slim as production
 
 ENV KINTO_INI=config/local.ini \
-    PATH="/opt/venv/bin:$PATH" \
+    PATH="/opt/.venv/bin:$PATH" \
     PORT=8888 \
     PYTHONUNBUFFERED=1 \
-    VIRTUAL_ENV=/opt/venv
+    VIRTUAL_ENV=/opt/.venv
 
 COPY /bin/update_and_install_system_packages.sh /opt
 RUN /opt/update_and_install_system_packages.sh \