From cbd488536d0d396bdedb9e411014b8f507943694 Mon Sep 17 00:00:00 2001
From: flbraun <mail@chiron-online.de>
Date: Wed, 25 Oct 2023 13:42:25 +0200
Subject: [PATCH] unbork our coverage setup

there are a couple of problems with our coverage setup:
1. the tox envs generate and attempt to merge coverage files, leading to race conditions.
2. after tox has run, the `tox` make target and the `test` workflow both _also_ try to process the coverage data.
3. the `test` workflow uploads a report to Codecov for each permutation of the python-django-matrix. this is not only inefficient, but also distorts our coverage ratings. at Codecov we're currently scoring 91% coverage, although it should be 93%. this is caused by not properly merging the coverage results of the test matrix and instead uploading them individually, causing our coverage score to be that of the last permutation to finish.
4. for some unexplained reason, usage of the `codecov/codecov-action` was replaced with Codecov's bash importer in fbd0cb17. at that point in time the bash importer had already been deprecated for 18 months (!) and the related repository had already been archived for 8 months. users are officially encouraged to migrate to a newer importer (as implemented in the previously used action).

problems 1-3 are addressed by implementing separation of concerns: test runs (whether in workflows or from make targets) now only emit partial coverage data files. combining them and generating reports is done afterward.

to address problem 4 the previously used `codecov/codecov-action` action is reinstated and properly configured.

finally, re-enable parallel tox runs for the `tox` make target. this was temporarily disabled to circumvent problems 1 and 2 in 7e92f081.
---
 .github/workflows/pythonapp.yml | 70 ++++++++++++++++++++++++++++++---
 Makefile                        | 15 ++++---
 pyproject.toml                  |  5 +--
 3 files changed, 74 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml
index 76a2ea9..4a4952e 100644
--- a/.github/workflows/pythonapp.yml
+++ b/.github/workflows/pythonapp.yml
@@ -1,4 +1,4 @@
-name: Test
+name: CI
 
 on:
   push:
@@ -9,8 +9,9 @@ on:
     - cron: '0 8 * * *'
 
 jobs:
+
   test:
-    name: 'Python ${{ matrix.python-version }} Django ${{ matrix.django-version }}'
+    name: 'Test: Python ${{ matrix.python-version }} Django ${{ matrix.django-version }}'
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -23,6 +24,7 @@ jobs:
     env:
       PYTHONUNBUFFERED: 1
       PYTHONWARNINGS: always
+      COVERAGE_FILE: coverage  # note: not .coverage: https://github.com/actions/upload-artifact/issues/447
     steps:
     - name: Checkout
       run: |
@@ -32,7 +34,7 @@ jobs:
         git checkout $GITHUB_SHA || (git fetch && git checkout $GITHUB_SHA)
 
     - name: 'Set up Python ${{ matrix.python-version }}'
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       # https://github.com/marketplace/actions/setup-python
       with:
         python-version: '${{ matrix.python-version }}'
@@ -57,8 +59,64 @@ jobs:
     - name: 'Install Browsers for Playwright tests'
       run: make playwright-install
 
-    - name: 'Run tests with Python ${{ matrix.python-version }} Django ${{ matrix.django-version }}'
+    - name: 'Run tests with Python ${{ matrix.python-version }} + Django ${{ matrix.django-version }}'
       run: poetry run tox -e $(echo py${{ matrix.python-version }}-django${{ matrix.django-version }} | tr -d .)
 
-    - name: 'Upload coverage report'
-      run: bash <(curl -s https://codecov.io/bash)
+    - name: Store partial coverage data
+      uses: actions/upload-artifact@v4
+      with:
+        name: coverage-data-py${{ matrix.python-version }}-dj${{ matrix.django-version }}
+        path: "coverage.*"  # note: not .coverage: https://github.com/actions/upload-artifact/issues/447
+        if-no-files-found: error
+        # crash if no files for upload could be found.
+        # the most likely reasons for that are an error with the coverage tool itself
+        # or a disparity between configured tox envs and the action's matrix.
+
+  coverage:
+    name: Coverage report
+    runs-on: ubuntu-latest
+    needs: test
+    env:
+      COVERAGE_FILE: coverage  # note: not .coverage: https://github.com/actions/upload-artifact/issues/447
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install coverage
+        run: python -m pip install --upgrade coverage[toml]
+
+      - name: Download partial coverage data
+        uses: actions/download-artifact@v4
+        with:
+          pattern: coverage-data-*
+          merge-multiple: true
+
+      - name: Combine individual data files
+        run: python -m coverage combine
+
+      - name: Print report
+        run: python -m coverage report
+
+      - name: Generate HTML report
+        run: python -m coverage html
+
+      - name: Upload HTML report artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-html-report
+          path: "htmlcov/*"
+          if-no-files-found: error
+
+      - name: Upload report to Codecov
+        uses: codecov/codecov-action@v4
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        with:
+          files: ${{ env.COVERAGE_FILE }}
+          fail_ci_if_error: true
+          verbose: True
diff --git a/Makefile b/Makefile
index 3eb60fd..22186d1 100644
--- a/Makefile
+++ b/Makefile
@@ -33,6 +33,9 @@ update: check-poetry ## Update the dependencies as according to the pyproject.to
 	poetry update -v
 	poetry install
 
+clear-partial-coverage:
+	rm -f .coverage.*
+
 lint: ## Run code formatters and linter
 	poetry run darker --diff --check
 	poetry run isort --check-only .
@@ -45,18 +48,18 @@ fix-code-style: ## Fix code formatting
 tox-listenvs: check-poetry ## List all tox test environments
 	poetry run tox --listenvs
 
-tox: check-poetry ## Run unittests via tox with all environments
-	poetry run tox
-	poetry run coverage combine --append
+tox: check-poetry clear-partial-coverage ## Run unittests via tox with all environments
+	poetry run tox p
+	poetry run coverage combine
 	poetry run coverage html
 	poetry run coverage report
 
 test: check-poetry  ## Run unittests
 	RAISE_LOG_OUTPUT=1 ./manage.sh test --parallel --shuffle --buffer
 
-coverage_test: ## Run tests and generate coverage html report
-	poetry run coverage run --rcfile=pyproject.toml manage.py test --parallel --shuffle
-	poetry run coverage combine --append
+coverage_test: clear-partial-coverage ## Run tests and generate coverage html report
+	poetry run coverage run --parallel-mode manage.py test --parallel --shuffle
+	poetry run coverage combine
 	poetry run coverage html
 	poetry run coverage report
 
diff --git a/pyproject.toml b/pyproject.toml
index 04cb322..514ad43 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -140,10 +140,7 @@ deps =
 commands_pre =
     {envpython} -m playwright install chromium firefox
 commands =
-    {envpython} -m coverage run --context='{envname}'
-    {envpython} -m coverage combine --append
-    {envpython} -m coverage xml
-    {envpython} -m coverage report
+    {envpython} -m coverage run --parallel-mode --context='{envname}'
 """