Merge pull request #3 from coretl/ci-split

Ci split

.devcontainer/devcontainer.json

@@ -1,7 +1,10 @@
 // For format details, see https://containers.dev/implementors/json_reference/
 {
     "name": "Python 3 Developer Container",
-    "image": "python:3.11",
+    "build": {
+        "dockerfile": "../Dockerfile",
+        "target": "developer"
+    },
     "remoteEnv": {
          // Allow X11 apps to run inside the container
         "DISPLAY": "${localEnv:DISPLAY}",
@@ -25,7 +28,7 @@
         "ghcr.io/devcontainers/features/common-utils:2": {
             "upgradePackages": false
         }
-    },    
+    },
     "runArgs": [
         // Allow the container to access the host X11 display and EPICS CA
         "--net=host",

.github/actions/install_requirements/action.yml

@@ -1,60 +1,34 @@
 name: Install requirements
-description: Run pip install with requirements and upload resulting requirements
+description: Install a version of python then call pip install and report what was installed
 inputs:
-  requirements_file:
-    description: Name of requirements file to use and upload
-    required: true
-  install_options:
+  python-version:
+    description: Python version to install, default is from Dockerfile
+    default: "dev"
+  pip-install:
     description: Parameters to pass to pip install
-    required: true
-  artifact_name:
-    description: A user friendly name to give the produced artifacts
-    required: true
-  python_version:
-    description: Python version to install
-    default: "3.x"
+    default: "-c requirements/dev-constraints.txt -e .[dev]"
 
 runs:
   using: composite
-
   steps:
+    - name: Get version of python
+      run: |
+        PYTHON_VERSION="${{ inputs.python-version }}"
+        if [ $PYTHON_VERSION == "dev" ]; then
+          PYTHON_VERSION=$(sed -n "s/ARG PYTHON_VERSION=//p" Dockerfile)
+        fi
+        echo "PYTHON_VERSION=$PYTHON_VERSION" >> "$GITHUB_ENV"
+      shell: bash
+
     - name: Setup python
       uses: actions/setup-python@v5
       with:
-        python-version: ${{ inputs.python_version }}
+        python-version: ${{ env.PYTHON_VERSION }}
 
-    - name: Pip install
-      run: |
-        touch ${{ inputs.requirements_file }}
-        # -c uses requirements.txt as constraints, see 'Validate requirements file'
-        pip install -c ${{ inputs.requirements_file }} ${{ inputs.install_options }}
+    - name: Install packages
+      run: pip install ${{ inputs.pip-install }}
       shell: bash
 
-    - name: Create lockfile
-      run: |
-        mkdir -p lockfiles
-        pip freeze --exclude-editable > lockfiles/${{ inputs.requirements_file }}
-        # delete the self referencing line and make sure it isn't blank
-        sed -i'' -e '/file:/d' lockfiles/${{ inputs.requirements_file }}
-      shell: bash
-
-    - name: Upload lockfiles
-      uses: actions/[email protected]
-      with:
-        name: lockfiles-${{ inputs.python_version }}-${{ inputs.artifact_name }}-${{ github.sha }}
-        path: lockfiles
-
-    # This eliminates the class of problems where the requirements being given no
-    # longer match what the packages themselves dictate. E.g. In the rare instance
-    # where I install some-package which used to depend on vulnerable-dependency
-    # but now uses good-dependency (despite being nominally the same version)
-    # pip will install both if given a requirements file with -r
-    - name: If requirements file exists, check it matches pip installed packages
-      run: |
-        if [ -s ${{ inputs.requirements_file }} ]; then
-          if ! diff -u ${{ inputs.requirements_file }} lockfiles/${{ inputs.requirements_file }}; then
-            echo "Error: ${{ inputs.requirements_file }} need the above changes to be exhaustive"
-            exit 1
-          fi
-        fi
+    - name: Report what was installed
+      run: pip freeze
       shell: bash

.github/workflows/_container.yml

@@ -0,0 +1,56 @@
+on:
+  workflow_call:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # Need this to get version number from last tag
+          fetch-depth: 0
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GitHub Docker Registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and export to Docker local cache
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          # Need load and tags so we can test it below
+          load: true
+          tags: tag_for_testing
+
+      - name: Test cli works in cached runtime image
+        run: docker run --rm tag_for_testing --version
+
+      - name: Create tags for publishing image
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tags: |
+            type=ref,event=tag
+            type=raw,value=latest
+
+      - name: Push cached image to container registry
+        if: github.ref_type == 'tag'
+        uses: docker/build-push-action@v5
+        # This does not build the image again, it will find the image in the
+        # Docker cache and publish it
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/_dist.yml

@@ -0,0 +1,36 @@
+on:
+  workflow_call:
+
+jobs:
+  build:
+    runs-on: "ubuntu-latest"
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # Need this to get version number from last tag
+          fetch-depth: 0
+
+      - name: Build sdist and wheel
+        run: >
+          export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) &&
+          pipx run build
+
+      - name: Upload sdist and wheel as artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist
+
+      - name: Check for packaging errors
+        run: pipx run twine check --strict dist/*
+
+      - name: Install produced wheel
+        uses: ./.github/actions/install_requirements
+        with:          
+          pip-install: dist/*.whl
+
+      - name: Test module --version works using the installed wheel
+        # If more than one module in src/ replace with module name to test
+        run: python -m $(ls src | head -1) --version

.github/workflows/docs.yml → .github/workflows/_docs.yml

@@ -1,19 +1,11 @@
-name: Docs CI
-
 on:
-  push:
-  pull_request:
+  workflow_call:
 
 jobs:
-  docs:
-    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository
+  build:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Avoid git conflicts when tag and branch pushed at same time
-        if: startsWith(github.ref, 'refs/tags')
-        run: sleep 60
-
       - name: Checkout
         uses: actions/checkout@v4
         with:
@@ -26,18 +18,24 @@ jobs:
 
       - name: Install python packages
         uses: ./.github/actions/install_requirements
-        with:
-          requirements_file: requirements-dev-3.x.txt
-          install_options: -e .[dev]
-          artifact_name: docs
 
       - name: Build docs
         run: tox -e docs
 
+      - name: Upload built docs artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: docs
+          path: build
+
       - name: Move to versioned directory
         run: mv build/html .github/pages/main
 
-      - name: Upload artifact
+      - name: Add other releases
+        run: echo Not done yet
+
+      - name: Upload github-pages artifact
+        if: github.ref_name == 'main'
         uses: actions/upload-pages-artifact@v3
         with:
           path: .github/pages
@@ -49,12 +47,13 @@ jobs:
     permissions:
       contents: read
       pages: write
-      id-token: write  
+      id-token: write
     environment:
       name: github-pages
-      url: ${{steps.deployment.outputs.page_url}}
+      url: ${{ steps.deployment.outputs.page_url }}
     runs-on: ubuntu-latest
-    needs: docs
+    needs: build
+    if: github.ref_name == 'main'
     steps:
       - name: Deploy to GitHub Pages
         id: deployment

.github/workflows/_lint.yml

@@ -0,0 +1,16 @@
+on:
+  workflow_call:
+
+jobs:
+  run:
+    runs-on: "ubuntu-latest"
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install python packages
+        uses: ./.github/actions/install_requirements
+
+      - name: Lint
+        run: tox -e pre-commit,pyright

.github/workflows/_pypi.yml

@@ -0,0 +1,16 @@
+on:
+  workflow_call:
+
+jobs:
+  upload:
+    runs-on: ubuntu-latest
+    environment: release
+
+    steps:
+      - name: Download dist artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: dist
+
+      - name: Publish to PyPI using trusted publishing
+        uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/_release.yml

@@ -0,0 +1,21 @@
+on:
+  workflow_call:
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Download artifacts
+        uses: actions/download-artifact@v3
+
+      - name: Create GitHub Release
+        # We pin to the SHA, not the tag, for security reasons.
+        # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions
+        uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # v0.1.15
+        with:
+          prerelease: ${{ contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc') }}
+          files: "*"
+          generate_release_notes: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/_test.yml

@@ -0,0 +1,46 @@
+on:
+  workflow_call:
+    inputs:
+      python-version:
+        type: string
+        description: The version of python to install
+        required: true
+      runs-on:
+        type: string
+        description: The runner to run this job on
+        required: true
+
+env:
+  # https://github.com/pytest-dev/pytest/issues/2042
+  PY_IGNORE_IMPORTMISMATCH: "1"
+
+jobs:
+  run:
+    runs-on: ${{ inputs.runs-on }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - if: inputs.python-version == 'dev'
+        name: Install dev constrained python packages
+        uses: ./.github/actions/install_requirements
+
+      - if: inputs.python-version != 'dev'
+        name: Install unconstrained python packages
+        uses: ./.github/actions/install_requirements
+        with:
+          python-version: ${{ inputs.python-version }}
+          pip-install: ".[dev]"
+
+      - name: Report dependencies
+        run: pipdeptree
+
+      - name: Run tests
+        run: tox -e pytest-cov
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          name: ${{ inputs.python-version }}/${{ inputs.runs-on }}
+          files: cov.xml