diff --git a/.github/workflows/cleanup-xapi-environment/action.yml b/.github/workflows/cleanup-xapi-environment/action.yml deleted file mode 100644 index 96323007e4e..00000000000 --- a/.github/workflows/cleanup-xapi-environment/action.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: Cleanup XenAPI environment -description: Cleanup XenAPI environment created using the setup-xapi-environment composite action - -runs: - using: "composite" - steps: - - name: Uninstall unversioned packages and remove pins - shell: bash - # This should purge them from the cache, unversioned package have - # 'master' as its version - run: | - opam list | awk -F " " '$2 == "master" { print $1 }' | xargs opam uninstall - opam pin list | cut -f1 -d "." | xargs opam unpin diff --git a/.github/workflows/generate-and-build-sdks.yml b/.github/workflows/generate-and-build-sdks.yml index db28438062f..ad434d7d9a5 100644 --- a/.github/workflows/generate-and-build-sdks.yml +++ b/.github/workflows/generate-and-build-sdks.yml @@ -42,8 +42,8 @@ jobs: name: SDK_Source_PowerShell path: _build/install/default/xapi/sdk/powershell/* - - name: Cleanup XenAPI environment - uses: ./.github/workflows/cleanup-xapi-environment + - name: Trim dune cache + run: opam exec -- dune cache trim --size=2GiB build-c-sdk: name: Build C SDK diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 42c06764498..d4bf28aaab2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,112 +12,9 @@ concurrency: # On new push, cancel old workflows from the same PR, branch or tag cancel-in-progress: true jobs: - python-test: - name: Python tests - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - python-version: ["2.7", "3.11"] - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # To check which files changed: origin/master..HEAD - - uses: LizardByte/setup-python-action@master - with: - python-version: ${{matrix.python-version}} - - - uses: actions/cache@v4 - name: Setup cache for running pre-commit fast - with: - path: ~/.cache/pre-commit - key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} - - - run: echo "::add-matcher::.github/workflows/python-warning-matcher.json" - name: "Setup GitHub for reporting Python warnings as annotations in pull request code review" - - - uses: pre-commit/action@v3.0.1 - name: Run pre-commit checks (no spaces at end of lines, etc) - if: ${{ matrix.python-version != '2.7' }} - with: - extra_args: --all-files --verbose --hook-stage commit - env: - SKIP: no-commit-to-branch - - - name: Install dependencies only needed for python 2 - if: ${{ matrix.python-version == '2.7' }} - run: pip install enum - - - name: Install dependencies only needed for python 3 - if: ${{ matrix.python-version != '2.7' }} - run: pip install opentelemetry-api opentelemetry-exporter-zipkin-json opentelemetry-sdk pandas pytype toml wrapt - - - name: Install common dependencies for Python ${{matrix.python-version}} - run: pip install future mock pytest-coverage pytest-mock - - - name: Run Pytest for python 2 and get code coverage for Codecov - if: ${{ matrix.python-version == '2.7' }} - run: > - pytest - --cov=scripts --cov=ocaml/xcp-rrdd - scripts/ ocaml/xcp-rrdd -vv -rA - --junitxml=.git/pytest${{matrix.python-version}}.xml - --cov-report term-missing - --cov-report xml:.git/coverage${{matrix.python-version}}.xml - env: - PYTHONDEVMODE: yes - - - name: Run Pytest for python 3 and get code coverage for Codecov - if: ${{ matrix.python-version != '2.7' }} - run: > - pytest - --cov=scripts --cov=ocaml/xcp-rrdd --cov=python3/ - scripts/ ocaml/xcp-rrdd python3/ -vv -rA - --junitxml=.git/pytest${{matrix.python-version}}.xml - --cov-report term-missing - --cov-report xml:.git/coverage${{matrix.python-version}}.xml - env: - PYTHONDEVMODE: yes - - - name: Upload Python ${{matrix.python-version}} coverage report to Codecov - uses: codecov/codecov-action@v3 - with: - directory: .git - files: coverage${{matrix.python-version}}.xml - env_vars: OS,PYTHON - fail_ci_if_error: false - flags: python${{matrix.python-version}} - name: coverage${{matrix.python-version}} - verbose: true - - - uses: dciborow/action-pylint@0.1.0 - if: ${{ matrix.python-version != '2.7' }} - with: - reporter: github-pr-review - level: warning - # To be customized to cover remaining Python scripts: - glob_pattern: "**/*.py" - continue-on-error: true - - - name: Run pytype checks - if: ${{ matrix.python-version != '2.7' }} - run: ./pytype_reporter.py - env: - PR_NUMBER: ${{ github.event.number }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PYTYPE_REPORTER_DEBUG: True - - - name: pyflakes - uses: reviewdog/action-pyflakes@v1 - with: - github_token: ${{ secrets.github_token }} - continue-on-error: true - - ocaml-tests: name: Run OCaml tests - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: # Ensure you also update test-sdk-builds # when changing this value, to keep builds @@ -153,28 +50,5 @@ jobs: - name: Check disk space run: df -h || true - - name: Cleanup XenAPI environment - uses: ./.github/workflows/cleanup-xapi-environment - - deprecation-test: - name: Deprecation tests - runs-on: ubuntu-20.04 - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Generate empty configuration for make to be happy - run: touch config.mk - - - name: quality-gate - run: make quality-gate - - test-sdk-builds: - name: Test SDK builds - uses: ./.github/workflows/generate-and-build-sdks.yml - with: - # Ensure you also update ocaml-tests - # when changing this value, to keep builds - # consistent - xapi_version: "v0.0.0" + - name: Trim dune cache + run: opam exec -- dune cache trim --size=2GiB diff --git a/.github/workflows/other.yml b/.github/workflows/other.yml new file mode 100644 index 00000000000..d65b7abe575 --- /dev/null +++ b/.github/workflows/other.yml @@ -0,0 +1,138 @@ +name: Build and test (other) + +on: + push: + pull_request: + schedule: + # run daily, this refreshes the cache + - cron: "13 2 * * *" + +concurrency: # On new push, cancel old workflows from the same PR, branch or tag: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + python-test: + name: Python tests + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + python-version: ["2.7", "3.11"] + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 # To check which files changed: origin/master..HEAD + - uses: LizardByte/setup-python-action@master + with: + python-version: ${{matrix.python-version}} + + - uses: actions/cache@v4 + name: Setup cache for running pre-commit fast + with: + path: ~/.cache/pre-commit + key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} + + - run: echo "::add-matcher::.github/workflows/python-warning-matcher.json" + name: "Setup GitHub for reporting Python warnings as annotations in pull request code review" + + - uses: pre-commit/action@v3.0.1 + name: Run pre-commit checks (no spaces at end of lines, etc) + if: ${{ matrix.python-version != '2.7' }} + with: + extra_args: --all-files --verbose --hook-stage commit + env: + SKIP: no-commit-to-branch + + - name: Install dependencies only needed for python 2 + if: ${{ matrix.python-version == '2.7' }} + run: pip install enum + + - name: Install dependencies only needed for python 3 + if: ${{ matrix.python-version != '2.7' }} + run: pip install opentelemetry-api opentelemetry-exporter-zipkin-json opentelemetry-sdk pandas pytype toml wrapt + + - name: Install common dependencies for Python ${{matrix.python-version}} + run: pip install future mock pytest-coverage pytest-mock + + - name: Run Pytest for python 2 and get code coverage for Codecov + if: ${{ matrix.python-version == '2.7' }} + run: > + pytest + --cov=scripts --cov=ocaml/xcp-rrdd + scripts/ ocaml/xcp-rrdd -vv -rA + --junitxml=.git/pytest${{matrix.python-version}}.xml + --cov-report term-missing + --cov-report xml:.git/coverage${{matrix.python-version}}.xml + env: + PYTHONDEVMODE: yes + + - name: Run Pytest for python 3 and get code coverage for Codecov + if: ${{ matrix.python-version != '2.7' }} + run: > + pytest + --cov=scripts --cov=ocaml/xcp-rrdd --cov=python3/ + scripts/ ocaml/xcp-rrdd python3/ -vv -rA + --junitxml=.git/pytest${{matrix.python-version}}.xml + --cov-report term-missing + --cov-report xml:.git/coverage${{matrix.python-version}}.xml + env: + PYTHONDEVMODE: yes + + - name: Upload Python ${{matrix.python-version}} coverage report to Codecov + uses: codecov/codecov-action@v3 + with: + directory: .git + files: coverage${{matrix.python-version}}.xml + env_vars: OS,PYTHON + fail_ci_if_error: false + flags: python${{matrix.python-version}} + name: coverage${{matrix.python-version}} + verbose: true + + - uses: dciborow/action-pylint@0.1.0 + if: ${{ matrix.python-version != '2.7' }} + with: + reporter: github-pr-review + level: warning + # To be customized to cover remaining Python scripts: + glob_pattern: "**/*.py" + continue-on-error: true + + - name: Run pytype checks + if: ${{ matrix.python-version != '2.7' }} + run: ./pytype_reporter.py + env: + PR_NUMBER: ${{ github.event.number }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PYTYPE_REPORTER_DEBUG: True + + - name: pyflakes + uses: reviewdog/action-pyflakes@v1 + with: + github_token: ${{ secrets.github_token }} + continue-on-error: true + + deprecation-test: + name: Deprecation tests + runs-on: ubuntu-22.04 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Generate empty configuration for make to be happy + run: touch config.mk + + - name: quality-gate + run: make quality-gate + + test-sdk-builds: + name: Test SDK builds + uses: ./.github/workflows/generate-and-build-sdks.yml + with: + # Ensure you also update ocaml-tests + # when changing this value, to keep builds + # consistent + xapi_version: "v0.0.0" diff --git a/.github/workflows/setup-xapi-environment/action.yml b/.github/workflows/setup-xapi-environment/action.yml index e25e0e184fb..d46ae3a5b96 100644 --- a/.github/workflows/setup-xapi-environment/action.yml +++ b/.github/workflows/setup-xapi-environment/action.yml @@ -8,10 +8,6 @@ inputs: runs: using: "composite" steps: - - name: Free space - shell: bash - run: sudo rm -rf /usr/local/lib/android - - name: Pull configuration from xs-opam shell: bash run: | @@ -31,6 +27,10 @@ runs: shell: bash run: sudo apt-get update + - name: Install python2 + shell: bash + run: sudo apt-get install python2 + - name: Use disk with more space for TMPDIR and XDG_CACHE_HOME shell: bash run: | @@ -42,13 +42,18 @@ runs: echo "TMPDIR=${TMPDIR}" >>"$GITHUB_ENV" echo "XDG_CACHE_HOME=${XDG_CACHE_HOME}" >>"$GITHUB_ENV" + # We set DUNE_CACHE_STORAGE_MODE, it is required for dune cache to work inside opam for now, + # otherwise it gets EXDEV and considers it a cache miss - name: Use ocaml uses: ocaml/setup-ocaml@v2 with: ocaml-compiler: ${{ steps.dotenv.outputs.ocaml_version_full }} opam-repositories: | xs-opam: ${{ steps.dotenv.outputs.repository }} + opam-pin: false dune-cache: true + env: + DUNE_CACHE_STORAGE_MODE: copy - name: Install dependencies shell: bash