From cc55dd2d5afd275b84552e3e22b638219028a62c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edwin=20T=C3=B6r=C3=B6k?= Date: Thu, 25 Apr 2024 09:18:05 +0100 Subject: [PATCH 1/6] CI: update to Ubuntu 22.04 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We need to explicitly install python2, since it is no longer the default. Eventually we may want to drop this completely, but we're not there yet. Signed-off-by: Edwin Török --- .github/workflows/main.yml | 4 ++-- .github/workflows/setup-xapi-environment/action.yml | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 42c06764498..9bb4aee895d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -117,7 +117,7 @@ jobs: ocaml-tests: name: Run OCaml tests - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: # Ensure you also update test-sdk-builds # when changing this value, to keep builds @@ -158,7 +158,7 @@ jobs: deprecation-test: name: Deprecation tests - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Checkout code diff --git a/.github/workflows/setup-xapi-environment/action.yml b/.github/workflows/setup-xapi-environment/action.yml index e25e0e184fb..c3785924e76 100644 --- a/.github/workflows/setup-xapi-environment/action.yml +++ b/.github/workflows/setup-xapi-environment/action.yml @@ -31,6 +31,10 @@ runs: shell: bash run: sudo apt-get update + - name: Install python2 + shell: bash + run: sudo apt-get install python2 + - name: Use disk with more space for TMPDIR and XDG_CACHE_HOME shell: bash run: | From c5e07819129c59bf17e7a4f01aa464f48e33e4c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edwin=20T=C3=B6r=C3=B6k?= Date: Thu, 25 Apr 2024 09:46:08 +0100 Subject: [PATCH 2/6] ci(nopin): pinning is very slow and not necessary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We do not publish or use the opam packages from this repo, and therefore pinning is unnecessary. 'opam install --deps-only --with-test .' works without pinning on the version of 'opam' we have here. Pinning ~71 packages is very slow since it makes a full copy of the source+.git folder every time. Skip pinning, this also saves a few gigabytes of disk space. This also makes the cleanup-xapi-environment action a no-op, so remove it. (in fact it'd fail because the output of opam list with master pins is empty). For testing the correctness of opam packages: * we still have the xs-opam CI * we can introduce opam-dune-lint that statically checks opam and dune for missing dependencies (needs new release first) This reduces CI times: * Run OCaml tests: from 15m24s to 10m16s * Setup XenAPI environment: from 7m1s to 5m55s Signed-off-by: Edwin Török --- .../workflows/cleanup-xapi-environment/action.yml | 13 ------------- .github/workflows/generate-and-build-sdks.yml | 3 --- .github/workflows/main.yml | 3 --- .github/workflows/setup-xapi-environment/action.yml | 1 + 4 files changed, 1 insertion(+), 19 deletions(-) delete mode 100644 .github/workflows/cleanup-xapi-environment/action.yml diff --git a/.github/workflows/cleanup-xapi-environment/action.yml b/.github/workflows/cleanup-xapi-environment/action.yml deleted file mode 100644 index 96323007e4e..00000000000 --- a/.github/workflows/cleanup-xapi-environment/action.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: Cleanup XenAPI environment -description: Cleanup XenAPI environment created using the setup-xapi-environment composite action - -runs: - using: "composite" - steps: - - name: Uninstall unversioned packages and remove pins - shell: bash - # This should purge them from the cache, unversioned package have - # 'master' as its version - run: | - opam list | awk -F " " '$2 == "master" { print $1 }' | xargs opam uninstall - opam pin list | cut -f1 -d "." | xargs opam unpin diff --git a/.github/workflows/generate-and-build-sdks.yml b/.github/workflows/generate-and-build-sdks.yml index db28438062f..55ac3f4f09a 100644 --- a/.github/workflows/generate-and-build-sdks.yml +++ b/.github/workflows/generate-and-build-sdks.yml @@ -42,9 +42,6 @@ jobs: name: SDK_Source_PowerShell path: _build/install/default/xapi/sdk/powershell/* - - name: Cleanup XenAPI environment - uses: ./.github/workflows/cleanup-xapi-environment - build-c-sdk: name: Build C SDK runs-on: ubuntu-latest diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9bb4aee895d..ada260060ad 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -153,9 +153,6 @@ jobs: - name: Check disk space run: df -h || true - - name: Cleanup XenAPI environment - uses: ./.github/workflows/cleanup-xapi-environment - deprecation-test: name: Deprecation tests runs-on: ubuntu-22.04 diff --git a/.github/workflows/setup-xapi-environment/action.yml b/.github/workflows/setup-xapi-environment/action.yml index c3785924e76..505032095ae 100644 --- a/.github/workflows/setup-xapi-environment/action.yml +++ b/.github/workflows/setup-xapi-environment/action.yml @@ -52,6 +52,7 @@ runs: ocaml-compiler: ${{ steps.dotenv.outputs.ocaml_version_full }} opam-repositories: | xs-opam: ${{ steps.dotenv.outputs.repository }} + opam-pin: false dune-cache: true - name: Install dependencies From ce2f58870cf783cf24a5d356e84f22658b9ea600 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edwin=20T=C3=B6r=C3=B6k?= Date: Thu, 25 Apr 2024 10:26:15 +0100 Subject: [PATCH 3/6] ci(opam-dune-cache): cache dune builds from opam MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Setting DUNE_CACHE_STORAGE_MODE=copy is needed during 'opam install', otherwise dune builds inside opam packages won't be cached. The sandbox used by opam is another partition/device and hardlinking fails with EXDEV. Signed-off-by: Edwin Török --- .github/workflows/setup-xapi-environment/action.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/setup-xapi-environment/action.yml b/.github/workflows/setup-xapi-environment/action.yml index 505032095ae..b703d523340 100644 --- a/.github/workflows/setup-xapi-environment/action.yml +++ b/.github/workflows/setup-xapi-environment/action.yml @@ -46,6 +46,8 @@ runs: echo "TMPDIR=${TMPDIR}" >>"$GITHUB_ENV" echo "XDG_CACHE_HOME=${XDG_CACHE_HOME}" >>"$GITHUB_ENV" + # We set DUNE_CACHE_STORAGE_MODE, it is required for dune cache to work inside opam for now, + # otherwise it gets EXDEV and considers it a cache miss - name: Use ocaml uses: ocaml/setup-ocaml@v2 with: @@ -54,6 +56,8 @@ runs: xs-opam: ${{ steps.dotenv.outputs.repository }} opam-pin: false dune-cache: true + env: + DUNE_CACHE_STORAGE_MODE: copy - name: Install dependencies shell: bash From 342e125728fff5b4ab845dc50db02f996c97ab95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edwin=20T=C3=B6r=C3=B6k?= Date: Thu, 25 Apr 2024 10:29:50 +0100 Subject: [PATCH 4/6] ci(norm): we have enough space now MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that we no longer pin the local opam packages we have a few more GiB of disk space, and we don't need to rm pre-existing files. This speeds up this step a little. Signed-off-by: Edwin Török --- .github/workflows/setup-xapi-environment/action.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/setup-xapi-environment/action.yml b/.github/workflows/setup-xapi-environment/action.yml index b703d523340..d46ae3a5b96 100644 --- a/.github/workflows/setup-xapi-environment/action.yml +++ b/.github/workflows/setup-xapi-environment/action.yml @@ -8,10 +8,6 @@ inputs: runs: using: "composite" steps: - - name: Free space - shell: bash - run: sudo rm -rf /usr/local/lib/android - - name: Pull configuration from xs-opam shell: bash run: | From 3907e2188d4644fbe994ce970182240b87ba10ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edwin=20T=C3=B6r=C3=B6k?= Date: Thu, 25 Apr 2024 10:44:29 +0100 Subject: [PATCH 5/6] ci: separate workflows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dune cache inside 'setup-ocaml' is trimmed to 5GB/jobcount, which is currently about 10. But not all jobs here use the dune cache, so separate the workflows. (Also the dune cache will get compressed afterwards). The SDK build does use the dune cache, but eventually it might become smaller, e.g. if we change it to install just the dependencies needed for SDK gen, and not everything. It does have a lot of unrelated jobs though. The workflows need a unique name, otherwise they cancel each-other. The dune cache is ~1.4GB now compressed, which should speed up the setup-ocaml step on next run as more dependencies will be cached. Setup XenAPI environment now only takes 2m51s instead of 5m55s. Signed-off-by: Edwin Török --- .github/workflows/main.yml | 126 -------------------------------- .github/workflows/other.yml | 138 ++++++++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+), 126 deletions(-) create mode 100644 .github/workflows/other.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ada260060ad..5f2dd84a112 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,109 +12,6 @@ concurrency: # On new push, cancel old workflows from the same PR, branch or tag cancel-in-progress: true jobs: - python-test: - name: Python tests - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - python-version: ["2.7", "3.11"] - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # To check which files changed: origin/master..HEAD - - uses: LizardByte/setup-python-action@master - with: - python-version: ${{matrix.python-version}} - - - uses: actions/cache@v4 - name: Setup cache for running pre-commit fast - with: - path: ~/.cache/pre-commit - key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} - - - run: echo "::add-matcher::.github/workflows/python-warning-matcher.json" - name: "Setup GitHub for reporting Python warnings as annotations in pull request code review" - - - uses: pre-commit/action@v3.0.1 - name: Run pre-commit checks (no spaces at end of lines, etc) - if: ${{ matrix.python-version != '2.7' }} - with: - extra_args: --all-files --verbose --hook-stage commit - env: - SKIP: no-commit-to-branch - - - name: Install dependencies only needed for python 2 - if: ${{ matrix.python-version == '2.7' }} - run: pip install enum - - - name: Install dependencies only needed for python 3 - if: ${{ matrix.python-version != '2.7' }} - run: pip install opentelemetry-api opentelemetry-exporter-zipkin-json opentelemetry-sdk pandas pytype toml wrapt - - - name: Install common dependencies for Python ${{matrix.python-version}} - run: pip install future mock pytest-coverage pytest-mock - - - name: Run Pytest for python 2 and get code coverage for Codecov - if: ${{ matrix.python-version == '2.7' }} - run: > - pytest - --cov=scripts --cov=ocaml/xcp-rrdd - scripts/ ocaml/xcp-rrdd -vv -rA - --junitxml=.git/pytest${{matrix.python-version}}.xml - --cov-report term-missing - --cov-report xml:.git/coverage${{matrix.python-version}}.xml - env: - PYTHONDEVMODE: yes - - - name: Run Pytest for python 3 and get code coverage for Codecov - if: ${{ matrix.python-version != '2.7' }} - run: > - pytest - --cov=scripts --cov=ocaml/xcp-rrdd --cov=python3/ - scripts/ ocaml/xcp-rrdd python3/ -vv -rA - --junitxml=.git/pytest${{matrix.python-version}}.xml - --cov-report term-missing - --cov-report xml:.git/coverage${{matrix.python-version}}.xml - env: - PYTHONDEVMODE: yes - - - name: Upload Python ${{matrix.python-version}} coverage report to Codecov - uses: codecov/codecov-action@v3 - with: - directory: .git - files: coverage${{matrix.python-version}}.xml - env_vars: OS,PYTHON - fail_ci_if_error: false - flags: python${{matrix.python-version}} - name: coverage${{matrix.python-version}} - verbose: true - - - uses: dciborow/action-pylint@0.1.0 - if: ${{ matrix.python-version != '2.7' }} - with: - reporter: github-pr-review - level: warning - # To be customized to cover remaining Python scripts: - glob_pattern: "**/*.py" - continue-on-error: true - - - name: Run pytype checks - if: ${{ matrix.python-version != '2.7' }} - run: ./pytype_reporter.py - env: - PR_NUMBER: ${{ github.event.number }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PYTYPE_REPORTER_DEBUG: True - - - name: pyflakes - uses: reviewdog/action-pyflakes@v1 - with: - github_token: ${{ secrets.github_token }} - continue-on-error: true - - ocaml-tests: name: Run OCaml tests runs-on: ubuntu-22.04 @@ -152,26 +49,3 @@ jobs: - name: Check disk space run: df -h || true - - deprecation-test: - name: Deprecation tests - runs-on: ubuntu-22.04 - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Generate empty configuration for make to be happy - run: touch config.mk - - - name: quality-gate - run: make quality-gate - - test-sdk-builds: - name: Test SDK builds - uses: ./.github/workflows/generate-and-build-sdks.yml - with: - # Ensure you also update ocaml-tests - # when changing this value, to keep builds - # consistent - xapi_version: "v0.0.0" diff --git a/.github/workflows/other.yml b/.github/workflows/other.yml new file mode 100644 index 00000000000..d65b7abe575 --- /dev/null +++ b/.github/workflows/other.yml @@ -0,0 +1,138 @@ +name: Build and test (other) + +on: + push: + pull_request: + schedule: + # run daily, this refreshes the cache + - cron: "13 2 * * *" + +concurrency: # On new push, cancel old workflows from the same PR, branch or tag: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + python-test: + name: Python tests + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + python-version: ["2.7", "3.11"] + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 # To check which files changed: origin/master..HEAD + - uses: LizardByte/setup-python-action@master + with: + python-version: ${{matrix.python-version}} + + - uses: actions/cache@v4 + name: Setup cache for running pre-commit fast + with: + path: ~/.cache/pre-commit + key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} + + - run: echo "::add-matcher::.github/workflows/python-warning-matcher.json" + name: "Setup GitHub for reporting Python warnings as annotations in pull request code review" + + - uses: pre-commit/action@v3.0.1 + name: Run pre-commit checks (no spaces at end of lines, etc) + if: ${{ matrix.python-version != '2.7' }} + with: + extra_args: --all-files --verbose --hook-stage commit + env: + SKIP: no-commit-to-branch + + - name: Install dependencies only needed for python 2 + if: ${{ matrix.python-version == '2.7' }} + run: pip install enum + + - name: Install dependencies only needed for python 3 + if: ${{ matrix.python-version != '2.7' }} + run: pip install opentelemetry-api opentelemetry-exporter-zipkin-json opentelemetry-sdk pandas pytype toml wrapt + + - name: Install common dependencies for Python ${{matrix.python-version}} + run: pip install future mock pytest-coverage pytest-mock + + - name: Run Pytest for python 2 and get code coverage for Codecov + if: ${{ matrix.python-version == '2.7' }} + run: > + pytest + --cov=scripts --cov=ocaml/xcp-rrdd + scripts/ ocaml/xcp-rrdd -vv -rA + --junitxml=.git/pytest${{matrix.python-version}}.xml + --cov-report term-missing + --cov-report xml:.git/coverage${{matrix.python-version}}.xml + env: + PYTHONDEVMODE: yes + + - name: Run Pytest for python 3 and get code coverage for Codecov + if: ${{ matrix.python-version != '2.7' }} + run: > + pytest + --cov=scripts --cov=ocaml/xcp-rrdd --cov=python3/ + scripts/ ocaml/xcp-rrdd python3/ -vv -rA + --junitxml=.git/pytest${{matrix.python-version}}.xml + --cov-report term-missing + --cov-report xml:.git/coverage${{matrix.python-version}}.xml + env: + PYTHONDEVMODE: yes + + - name: Upload Python ${{matrix.python-version}} coverage report to Codecov + uses: codecov/codecov-action@v3 + with: + directory: .git + files: coverage${{matrix.python-version}}.xml + env_vars: OS,PYTHON + fail_ci_if_error: false + flags: python${{matrix.python-version}} + name: coverage${{matrix.python-version}} + verbose: true + + - uses: dciborow/action-pylint@0.1.0 + if: ${{ matrix.python-version != '2.7' }} + with: + reporter: github-pr-review + level: warning + # To be customized to cover remaining Python scripts: + glob_pattern: "**/*.py" + continue-on-error: true + + - name: Run pytype checks + if: ${{ matrix.python-version != '2.7' }} + run: ./pytype_reporter.py + env: + PR_NUMBER: ${{ github.event.number }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PYTYPE_REPORTER_DEBUG: True + + - name: pyflakes + uses: reviewdog/action-pyflakes@v1 + with: + github_token: ${{ secrets.github_token }} + continue-on-error: true + + deprecation-test: + name: Deprecation tests + runs-on: ubuntu-22.04 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Generate empty configuration for make to be happy + run: touch config.mk + + - name: quality-gate + run: make quality-gate + + test-sdk-builds: + name: Test SDK builds + uses: ./.github/workflows/generate-and-build-sdks.yml + with: + # Ensure you also update ocaml-tests + # when changing this value, to keep builds + # consistent + xapi_version: "v0.0.0" From 15391c9f1d86eda4fa5f5d7489ff0256d114627f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edwin=20T=C3=B6r=C3=B6k?= Date: Fri, 26 Apr 2024 09:02:45 +0100 Subject: [PATCH 6/6] ci: trim dune cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We should eventually separate the opam dependency install into its own job that uploads an artifact/cache that other jobs can reuse (and that should have a bigger dune cache), however for now just trim the size of the cache. The default is 5GiB, which results in ~1.4GiB compressed, which would hit the 10GiB cache size limit in GitHub way too often. We may need to tweak this value in the future. Signed-off-by: Edwin Török --- .github/workflows/generate-and-build-sdks.yml | 3 +++ .github/workflows/main.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/generate-and-build-sdks.yml b/.github/workflows/generate-and-build-sdks.yml index 55ac3f4f09a..ad434d7d9a5 100644 --- a/.github/workflows/generate-and-build-sdks.yml +++ b/.github/workflows/generate-and-build-sdks.yml @@ -42,6 +42,9 @@ jobs: name: SDK_Source_PowerShell path: _build/install/default/xapi/sdk/powershell/* + - name: Trim dune cache + run: opam exec -- dune cache trim --size=2GiB + build-c-sdk: name: Build C SDK runs-on: ubuntu-latest diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5f2dd84a112..d4bf28aaab2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -49,3 +49,6 @@ jobs: - name: Check disk space run: df -h || true + + - name: Trim dune cache + run: opam exec -- dune cache trim --size=2GiB