Merge pull request #109 from jfennick/pypy_ci

Use pypy to run workflows in CI

.github/workflows/build_and_run_workflows.yml

@@ -96,10 +96,21 @@ jobs:
       run: rm -rf "/home/$(whoami)/wic/"
       # For self-hosted runners, make sure we use new global config settings
 
+    # Using pypy increases performance / decreases cwltool runtime by about a minute.
+    - name: Append pypy to conda environment files
+      if: runner.os != 'Windows'
+      run: cd wic/install/ && echo "  - pypy" >> system_deps.yml
+
     - name: Remove old mamba environment
       if: always()
       run: rm -rf "/home/$(whoami)/actions-runner/_work/workflow-inference-compiler/workflow-inference-compiler/3/envs/wic_github_actions/"
       # For self-hosted runners, make sure we install into a new mamba environment
+      # NOTE: Every time the github self-hosted runner executes, it sets "set -e" in ~/.bash_profile
+      # So if we rm -rf the old mamba environment without also removing the mamba init code in ~/.bash_profile
+      # (or removing the file altogether), then unless we immediately re-create the environment,
+      # (i.e. if we try to run any other commands between removing and re-creating the environment)
+      # we will get "EnvironmentNameNotFound: Could not find conda environment: wic_github_actions"
+      # and (again, due to "set -e") the workflow step will fail.
 
     - name: Setup mamba (linux, macos)
       if: always()
@@ -126,7 +137,10 @@ jobs:
       if: always()
       # Also run mm-workflows command to generate
       # mm-workflows/autogenerated/schemas/config_schemas.json
-      run: cd mm-workflows/ && pip install ".[all]" && mm-workflows
+      # NOTE: Use ".[test]" instead of ".[all]"
+      # We do not want or need to install the workflow_deps extra.
+      # (Many of the packages conflict with pypy.)
+      run: cd mm-workflows/ && pip install ".[test]" && mm-workflows
 
     - name: PyTest Run Example Workflows
       if: always()

.github/workflows/build_and_run_workflows_toil.yml

@@ -66,6 +66,11 @@ jobs:
       run: rm -rf "/home/$(whoami)/actions-runner/_work/workflow-inference-compiler/workflow-inference-compiler/3/envs/wic_github_actions/"
       # For self-hosted runners, make sure we install into a new mamba environment
 
+    # Toil is currently incompatible with pypy
+    # - name: Append pypy to conda environment files
+    #   if: runner.os != 'Windows'
+    #   run: cd wic/install/ && echo "  - pypy" >> system_deps.yml
+
     - name: Setup mamba (linux, macos)
       if: always()
       uses: conda-incubator/[email protected]

.github/workflows/build_and_test.yml

@@ -111,6 +111,15 @@ jobs:
           ref: ${{ inputs.mm-workflows_ref }}
           path: mm-workflows
 
+      # NOTE: pypy actually decreases performance for build_and_test.yml (by a factor of ~2)
+      # - name: Append pypy to conda environment files
+      #   if: runner.os != 'Windows'
+      #   run: cd wic/install/ && echo "  - pypy" >> system_deps.yml
+
+      # - name: Append pypy to conda environment files
+      #   if: runner.os == 'Windows'
+      #   run: cd wic/install/ && echo "  - pypy" >> system_deps_windows.yml
+
       - name: Setup mamba (linux, macos)
         if: runner.os != 'Windows'
         uses: conda-incubator/[email protected]
@@ -146,7 +155,10 @@ jobs:
         if: always()
         # Also run mm-workflows command to generate
         # mm-workflows/autogenerated/schemas/config_schemas.json
-        run: cd mm-workflows/ && pip install ".[all]" && mm-workflows
+        # NOTE: Use ".[test]" instead of ".[all]"
+        # We do not want or need to install the workflow_deps extra.
+        # (Many of the packages conflict with pypy.)
+        run: cd mm-workflows/ && pip install ".[test]" && mm-workflows
 
       - name: Build Documentation
         if: always()