Updates tests and CI to run tests in a container (#177)

* Update CI to run in test env container

* COnsolidate coverage steps

* Add exception handling for package metadata

* Use poetry environements in ci

* Fixes typo in poetry command

* Point poetry at installed python

* Drops extra argument to poetry install

* Run coverage via poetry

* Cleanup pass

* Add ci conditional to ignore releases

* Switch to getpass when fetching usernames

* Switch to getpass when fetching usernames

* Remove old reference to workflow inputs

* Enable publish workflow for debugging

* Drops reusable action

* Merge distribution build and publish jobs

* Fix workflow typo

* Update job dependencies

* Update job names

* cleanup pass

* Drops accidental needs clause

* Add read permissions for packages

* Drops push trigger from publish workflow

.github/workflows/DocumentationBuild.yml

@@ -5,9 +5,6 @@ on:
   workflow_call:
   pull_request:
 
-permissions:
-  contents: read
-
 jobs:
   build:
     name: Build Documentation
@@ -16,5 +13,26 @@ jobs:
       - name: Checkout project source
         uses: actions/checkout@v3
 
-      - name: Build Docs
-        uses: ./.github/actions/build-sphinx
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: false
+
+      - name: Install dependencies
+        shell: bash
+        run: poetry install --with docs
+
+      - name: Build docs
+        shell: bash
+        working-directory: docs
+        run: make html SPHINXOPTS="-W"
+
+      - name: Upload docs to artifact storage
+        uses: actions/upload-pages-artifact@v1
+        with:
+          path: docs/build/html

.github/workflows/DocumentationPublish.yml

@@ -9,8 +9,8 @@ permissions:
   id-token: write
 
 jobs:
-  build-docs:
-    needs: run-tests
+  build:
+    name: Build
     uses: ./.github/workflows/DocumentationBuild.yml
     secrets: inherit
 

.github/workflows/PackagePublish.yml

@@ -6,30 +6,36 @@ on:
 
 permissions:
   contents: read
+  packages: read
   pages: write
   id-token: write
 
 jobs:
   run-tests:
-    name: Run Tests
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: [ "3.8", "3.9", "3.10", "3.11" ]
-
-    steps:
-      - name: Checkout source
-        uses: actions/checkout@v3
+    name: Tests
+    uses: ./.github/workflows/PackageTest.yml
+    secrets: inherit
 
-      - name: Run Tests
-        uses: ./.github/actions/run-tests
-        with:
-          python-version: ${{ matrix.python-version }}
-
-  build-distribution:
-    name: Build Source Distribution
+  build-docs:
+    name: Docs
     needs: run-tests
+    uses: ./.github/workflows/DocumentationBuild.yml
+    secrets: inherit
+
+  publish-package:
+    name: Publish Distribution
+    needs: [ build-docs ]
     runs-on: ubuntu-latest
+    environment: ${{ matrix.environment }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - host: https://py00.crc.pitt.edu
+            environment: publish-h2p
+          - host: https://upload.pypi.org/legacy/
+            environment: publish-pypi
 
     steps:
       - name: Checkout source
@@ -62,39 +68,6 @@ jobs:
           path: dist
           retention-days: 7
 
-  build-docs:
-    name: Build Documentation
-    needs: run-tests
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout project source
-        uses: actions/checkout@v3
-
-      - name: Run tests
-        uses: ./.github/actions/build-sphinx
-
-  publish-package:
-    name: Publish Distribution
-    needs: [ build-distribution, build-docs ]
-    runs-on: ubuntu-latest
-    environment: ${{ matrix.environment }}
-
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - host: https://py00.crc.pitt.edu
-            environment: publish-h2p
-          - host: https://upload.pypi.org/legacy/
-            environment: publish-pypi
-
-    steps:
-      - name: Download distribution from artifact storage
-        uses: actions/download-artifact@v3
-        with:
-          name: package-build
-          path: dist
-
       - name: Publish package
         uses: pypa/gh-action-pypi-publish@release/v1
         with:

.github/workflows/PackageTest.yml

@@ -12,41 +12,68 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ "3.8", "3.9", "3.10", "3.11" ]
+        slurm_version:
+          - 20.02.5.1
+          - 20.11.9.1
+          - 22.05.2.1
+        python_version:
+          - 3.8
+          - 3.9
+
+    container:
+      image: ghcr.io/pitt-crc/test-env:${{ matrix.slurm_version }}
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
 
     steps:
+      - name: Setup environment
+        run: /usr/local/bin/entrypoint.sh
+
       - name: Checkout repository
         uses: actions/checkout@v3
 
-      - name: Run tests
-        uses: ./.github/actions/run-tests
-        with:
-          python-version: ${{ matrix.python-version }}
-          coverage-output: coverage.xml
+      - name: Install poetry
+        run: pip install poetry
+
+      - name: Install dependencies
+        run: |
+          poetry env use python${{ matrix.python_version }}
+          poetry install --with tests
+
+      - name: Run tests with coverage
+        run: |
+          poetry run coverage run -m unittest discover 
+          poetry run coverage report --omit="tests/*"
+          poetry run coverage xml --omit="tests/*" -o coverage.xml
 
       # Report partial coverage results to codacy for the current python version
       - name: Report partial coverage results
+        if: github.event_name != 'release'
+        shell: bash
         run: bash <(curl -Ls https://coverage.codacy.com/get.sh) report --partial -l Python -r coverage.xml
         env:
           CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }}
 
   # Tell codacy we are done reporting test coverage
   report-code-coverage:
-    name: Report Code Coverage
-    runs-on: ubuntu-latest
+    name: Report Coverage
+    if: github.event_name != 'release'
     needs: run-tests
+    runs-on: ubuntu-latest
     steps:
       - name: Finish reporting coverage
+        shell: bash
         run: bash <(curl -Ls https://coverage.codacy.com/get.sh) final
         env:
           CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }}
 
   # Use this job for branch protection rules
   report-test-status:
     name: Report Test Status
-    runs-on: ubuntu-latest
-    needs: run-tests
     if: always()
+    needs: run-tests
+    runs-on: ubuntu-latest
     steps:
       - name: Check build status
         if: ${{ contains(needs.*.result, 'failure') }}

apps/__init__.py

@@ -2,4 +2,8 @@
 
 import importlib.metadata
 
-__version__ = importlib.metadata.version('crc_wrappers')
+try:
+    __version__ = importlib.metadata.version('quota-notifier')
+
+except importlib.metadata.PackageNotFoundError:  # pragma: no cover
+    __version__ = '0.0.0'

apps/crc_scancel.py

@@ -5,8 +5,8 @@
 prompt to confirm users are canceling the correct job.
 """
 
+import getpass
 from argparse import Namespace
-from os import environ
 
 from ._base_parser import BaseParser
 from .utils.system_info import Shell, SlurmInfo
@@ -15,7 +15,7 @@
 class CrcScancel(BaseParser):
     """Cancel a Slurm job submitted by the current user."""
 
-    user = environ['USER']
+    user = getpass.getuser()
 
     def __init__(self) -> None:
         """Define arguments for the command line interface"""

apps/crc_squeue.py

@@ -1,7 +1,7 @@
 """A simple wrapper around the Slurm ``squeue`` command."""
 
+import getpass
 from argparse import Namespace
-from os import environ
 from time import sleep
 
 from ._base_parser import BaseParser
@@ -36,7 +36,7 @@ def build_slurm_command(self, args: Namespace) -> str:
         """
 
         # Variables for building shell commands
-        user = f"-u {environ['USER']}"
+        user = f"-u {getpass.getuser()}"
 
         # Build the base command
         command_options = ["squeue -M all"]

pyproject.toml

@@ -46,6 +46,12 @@ crc-usage = "apps.crc_usage:CrcUsage.execute"
 python = "^3.8.0"
 dataset = "1.6.0"
 
+[tool.poetry.group.tests]
+optional = true
+
+[tool.poetry.group.tests.dependencies]
+coverage="*"
+
 [tool.poetry.group.docs]
 optional = true
 

tests/test_crc_squeue.py

@@ -1,6 +1,6 @@
 """Tests for the ``crc-squeue`` application."""
 
-from os import environ
+import getpass
 from unittest import TestCase
 
 from apps.crc_squeue import CrcSqueue
@@ -27,7 +27,7 @@ def test_all_option(self) -> None:
         """Test the ``--all`` argument disables ``-u`` in the piped slurm command"""
 
         app = CrcSqueue()
-        slurm_user_argument = "-u {0}".format(environ['USER'])
+        slurm_user_argument = f'-u {getpass.getuser()}'
 
         # The application should default to showing information for the current user
         args, _ = app.parse_known_args([''])