ci: separate workflows

The dune cache inside 'setup-ocaml' is trimmed to 5GB/jobcount, which is currently about 10. But not all jobs here use the dune cache, so separate the workflows. (Also the dune cache will get compressed afterwards). The SDK build does use the dune cache, but eventually it might become smaller, e.g. if we change it to install just the dependencies needed for SDK gen, and not everything. It does have a lot of unrelated jobs though. The workflows need a unique name, otherwise they cancel each-other. The dune cache is ~1.4GB now compressed, which should speed up the setup-ocaml step on next run as more dependencies will be cached. Setup XenAPI environment now only takes 2m51s instead of 5m55s. Signed-off-by: Edwin Török <[email protected]>
edwintorok · Apr 25, 2024 · 3907e21 · 3907e21
1 parent 342e125
commit 3907e21
Show file tree

Hide file tree

Showing 2 changed files with 138 additions and 126 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -12,109 +12,6 @@ concurrency: # On new push, cancel old workflows from the same PR, branch or tag
   cancel-in-progress: true
 
 jobs:
-  python-test:
-    name: Python tests
-    runs-on: ubuntu-22.04
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["2.7", "3.11"]
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0 # To check which files changed: origin/master..HEAD
-      - uses: LizardByte/setup-python-action@master
-        with:
-          python-version: ${{matrix.python-version}}
-
-      - uses: actions/cache@v4
-        name: Setup cache for running pre-commit fast
-        with:
-          path: ~/.cache/pre-commit
-          key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
-
-      - run: echo "::add-matcher::.github/workflows/python-warning-matcher.json"
-        name: "Setup GitHub for reporting Python warnings as annotations in pull request code review"
-
-      - uses: pre-commit/[email protected]
-        name: Run pre-commit checks (no spaces at end of lines, etc)
-        if: ${{ matrix.python-version != '2.7' }}
-        with:
-          extra_args: --all-files --verbose --hook-stage commit
-        env:
-          SKIP: no-commit-to-branch
-
-      - name: Install dependencies only needed for python 2
-        if: ${{ matrix.python-version == '2.7' }}
-        run: pip install enum
-
-      - name: Install dependencies only needed for python 3
-        if: ${{ matrix.python-version != '2.7' }}
-        run: pip install opentelemetry-api opentelemetry-exporter-zipkin-json opentelemetry-sdk pandas pytype toml wrapt
-
-      - name: Install common dependencies for Python ${{matrix.python-version}}
-        run: pip install future mock pytest-coverage pytest-mock
-
-      - name: Run Pytest for python 2 and get code coverage for Codecov
-        if: ${{ matrix.python-version == '2.7' }}
-        run: >
-          pytest
-          --cov=scripts --cov=ocaml/xcp-rrdd
-          scripts/ ocaml/xcp-rrdd -vv -rA
-          --junitxml=.git/pytest${{matrix.python-version}}.xml
-          --cov-report term-missing
-          --cov-report xml:.git/coverage${{matrix.python-version}}.xml
-        env:
-          PYTHONDEVMODE: yes
-
-      - name: Run Pytest for python 3 and get code coverage for Codecov
-        if: ${{ matrix.python-version != '2.7' }}
-        run: >
-          pytest
-          --cov=scripts --cov=ocaml/xcp-rrdd --cov=python3/
-          scripts/ ocaml/xcp-rrdd python3/ -vv -rA
-          --junitxml=.git/pytest${{matrix.python-version}}.xml
-          --cov-report term-missing
-          --cov-report xml:.git/coverage${{matrix.python-version}}.xml
-        env:
-          PYTHONDEVMODE: yes
-
-      - name: Upload Python ${{matrix.python-version}} coverage report to Codecov
-        uses: codecov/codecov-action@v3
-        with:
-          directory: .git
-          files: coverage${{matrix.python-version}}.xml
-          env_vars: OS,PYTHON
-          fail_ci_if_error: false
-          flags: python${{matrix.python-version}}
-          name: coverage${{matrix.python-version}}
-          verbose: true
-
-      - uses: dciborow/[email protected]
-        if: ${{ matrix.python-version != '2.7' }}
-        with:
-          reporter: github-pr-review
-          level: warning
-          # To be customized to cover remaining Python scripts:
-          glob_pattern: "**/*.py"
-        continue-on-error: true
-
-      - name: Run pytype checks
-        if: ${{ matrix.python-version != '2.7' }}
-        run: ./pytype_reporter.py
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          PYTYPE_REPORTER_DEBUG: True
-
-      - name: pyflakes
-        uses: reviewdog/action-pyflakes@v1
-        with:
-          github_token: ${{ secrets.github_token }}
-        continue-on-error: true
-
-
   ocaml-tests:
     name: Run OCaml tests
     runs-on: ubuntu-22.04
@@ -152,26 +49,3 @@ jobs:
 
       - name: Check disk space
         run: df -h || true
-
-  deprecation-test:
-    name: Deprecation tests
-    runs-on: ubuntu-22.04
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Generate empty configuration for make to be happy
-        run: touch config.mk
-
-      - name: quality-gate
-        run: make quality-gate
-
-  test-sdk-builds:
-    name: Test SDK builds
-    uses: ./.github/workflows/generate-and-build-sdks.yml
-    with:
-      # Ensure you also update ocaml-tests
-      # when changing this value, to keep builds
-      # consistent
-      xapi_version: "v0.0.0"
diff --git a/.github/workflows/other.yml b/.github/workflows/other.yml
@@ -0,0 +1,138 @@
+name: Build and test (other)
+
+on:
+  push:
+  pull_request:
+  schedule:
+    # run daily, this refreshes the cache
+    - cron: "13 2 * * *"
+
+concurrency: # On new push, cancel old workflows from the same PR, branch or tag:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  python-test:
+    name: Python tests
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["2.7", "3.11"]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # To check which files changed: origin/master..HEAD
+      - uses: LizardByte/setup-python-action@master
+        with:
+          python-version: ${{matrix.python-version}}
+
+      - uses: actions/cache@v4
+        name: Setup cache for running pre-commit fast
+        with:
+          path: ~/.cache/pre-commit
+          key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
+
+      - run: echo "::add-matcher::.github/workflows/python-warning-matcher.json"
+        name: "Setup GitHub for reporting Python warnings as annotations in pull request code review"
+
+      - uses: pre-commit/[email protected]
+        name: Run pre-commit checks (no spaces at end of lines, etc)
+        if: ${{ matrix.python-version != '2.7' }}
+        with:
+          extra_args: --all-files --verbose --hook-stage commit
+        env:
+          SKIP: no-commit-to-branch
+
+      - name: Install dependencies only needed for python 2
+        if: ${{ matrix.python-version == '2.7' }}
+        run: pip install enum
+
+      - name: Install dependencies only needed for python 3
+        if: ${{ matrix.python-version != '2.7' }}
+        run: pip install opentelemetry-api opentelemetry-exporter-zipkin-json opentelemetry-sdk pandas pytype toml wrapt
+
+      - name: Install common dependencies for Python ${{matrix.python-version}}
+        run: pip install future mock pytest-coverage pytest-mock
+
+      - name: Run Pytest for python 2 and get code coverage for Codecov
+        if: ${{ matrix.python-version == '2.7' }}
+        run: >
+          pytest
+          --cov=scripts --cov=ocaml/xcp-rrdd
+          scripts/ ocaml/xcp-rrdd -vv -rA
+          --junitxml=.git/pytest${{matrix.python-version}}.xml
+          --cov-report term-missing
+          --cov-report xml:.git/coverage${{matrix.python-version}}.xml
+        env:
+          PYTHONDEVMODE: yes
+
+      - name: Run Pytest for python 3 and get code coverage for Codecov
+        if: ${{ matrix.python-version != '2.7' }}
+        run: >
+          pytest
+          --cov=scripts --cov=ocaml/xcp-rrdd --cov=python3/
+          scripts/ ocaml/xcp-rrdd python3/ -vv -rA
+          --junitxml=.git/pytest${{matrix.python-version}}.xml
+          --cov-report term-missing
+          --cov-report xml:.git/coverage${{matrix.python-version}}.xml
+        env:
+          PYTHONDEVMODE: yes
+
+      - name: Upload Python ${{matrix.python-version}} coverage report to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          directory: .git
+          files: coverage${{matrix.python-version}}.xml
+          env_vars: OS,PYTHON
+          fail_ci_if_error: false
+          flags: python${{matrix.python-version}}
+          name: coverage${{matrix.python-version}}
+          verbose: true
+
+      - uses: dciborow/[email protected]
+        if: ${{ matrix.python-version != '2.7' }}
+        with:
+          reporter: github-pr-review
+          level: warning
+          # To be customized to cover remaining Python scripts:
+          glob_pattern: "**/*.py"
+        continue-on-error: true
+
+      - name: Run pytype checks
+        if: ${{ matrix.python-version != '2.7' }}
+        run: ./pytype_reporter.py
+        env:
+          PR_NUMBER: ${{ github.event.number }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PYTYPE_REPORTER_DEBUG: True
+
+      - name: pyflakes
+        uses: reviewdog/action-pyflakes@v1
+        with:
+          github_token: ${{ secrets.github_token }}
+        continue-on-error: true
+
+  deprecation-test:
+    name: Deprecation tests
+    runs-on: ubuntu-22.04
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Generate empty configuration for make to be happy
+        run: touch config.mk
+
+      - name: quality-gate
+        run: make quality-gate
+
+  test-sdk-builds:
+    name: Test SDK builds
+    uses: ./.github/workflows/generate-and-build-sdks.yml
+    with:
+      # Ensure you also update ocaml-tests
+      # when changing this value, to keep builds
+      # consistent
+      xapi_version: "v0.0.0"