Backports (stable-5.21) #13290
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Tests | |
on: | |
push: | |
branches: | |
- main | |
- stable-* | |
pull_request: | |
workflow_dispatch: | |
schedule: | |
- cron: '0 0 * * *' # Test TICS daily | |
env: | |
LXD_REQUIRED_TESTS: "storage_buckets" | |
LXD_SKIP_TESTS: "clustering_upgrade clustering_upgrade_large" | |
GOCOVERDIR: ${{ ( github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' ) && '/home/runner/work/lxd/lxd/coverage' || '' }} | |
permissions: | |
contents: read | |
concurrency: | |
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} | |
cancel-in-progress: true | |
defaults: | |
run: | |
# Make sure bash is always invoked with `-eo pipefail` | |
# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsshell | |
shell: bash | |
jobs: | |
code-tests: | |
env: | |
CGO_CFLAGS: "-I/home/runner/go/bin/dqlite/include/" | |
CGO_LDFLAGS: "-L/home/runner/go/bin/dqlite/libs/" | |
LD_LIBRARY_PATH: "/home/runner/go/bin/dqlite/libs/" | |
LD_RUN_PATH: "/home/runner/go/bin/dqlite/libs/" | |
CGO_LDFLAGS_ALLOW: "(-Wl,-wrap,pthread_create)|(-Wl,-z,now)" | |
name: Code | |
runs-on: ubuntu-22.04 | |
steps: | |
- name: Checkout | |
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
with: | |
# A non-shallow clone is needed for the Differential ShellCheck | |
fetch-depth: 0 | |
- name: Tune disk performance | |
uses: ./.github/actions/tune-disk-performance | |
- name: Dependency Review | |
uses: actions/dependency-review-action@3b139cfc5fae8b618d3eae3675e383bb1769c019 # v4.5.0 | |
if: github.event_name == 'pull_request' | |
# XXX: `make static-analysis` also run shellcheck but this one provides | |
# useful feedback in the PR through github-code-scanning bot | |
- id: ShellCheck | |
name: Differential ShellCheck | |
uses: redhat-plumbers-in-action/differential-shellcheck@cc6721c45a8800cc666de45493545a07a638d121 # v5.4.0 | |
with: | |
token: ${{ secrets.GITHUB_TOKEN }} | |
strict-check-on-push: true | |
if: github.event_name == 'pull_request' | |
- name: Upload artifact with ShellCheck defects in SARIF format | |
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
with: | |
name: Differential ShellCheck SARIF | |
path: ${{ steps.ShellCheck.outputs.sarif }} | |
if: github.event_name == 'pull_request' | |
- name: Install Go | |
uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0 | |
with: | |
go-version-file: 'go.mod' | |
- name: Install build dependencies | |
uses: ./.github/actions/install-lxd-builddeps | |
- name: Download go dependencies | |
run: | | |
set -eux | |
sudo chmod o+w {go.mod,go.sum} | |
go mod download | |
- name: Check compatibility with min Go version | |
run: | | |
set -eux | |
GOMIN="$(sed -n 's/^GOMIN=\([0-9.]\+\)$/\1/p' Makefile)" | |
go mod tidy -go="${GOMIN}" | |
DOC_GOMIN="$(sed -n 's/^LXD requires Go \([0-9.]\+\) .*/\1/p' doc/requirements.md)" | |
[ "${GOMIN}" = "${DOC_GOMIN}" ] | |
- name: Make LXD tarball and unpack it | |
env: | |
CUSTOM_VERSION: "test" | |
run: | | |
set -eux | |
make dist | |
tar -xzf lxd-test.tar.gz -C /home/runner/work/lxd/ | |
rm lxd-test.tar.gz | |
- name: Build LXD dependencies | |
run: | | |
set -eux | |
# Build from unpacked dist tarball. | |
cd /home/runner/work/lxd/lxd-test | |
make deps | |
# Include dqlite libs in dependencies for system tests. | |
mkdir /home/runner/go/bin/dqlite | |
mv /home/runner/work/lxd/lxd-test/vendor/dqlite/include /home/runner/go/bin/dqlite/include | |
mv /home/runner/work/lxd/lxd-test/vendor/dqlite/.libs /home/runner/go/bin/dqlite/libs | |
- name: Build binaries | |
run: | | |
set -eux | |
# Build from unpacked dist tarball. | |
cd /home/runner/work/lxd/lxd-test | |
make | |
- name: Check lxc/lxd-agent binary sizes | |
run: | | |
set -eux | |
# Build lxc/lxd-agent the same way as done in the snap | |
go build -trimpath -o "/tmp/bin/lxc" github.com/canonical/lxd/lxc | |
CGO_ENABLED=0 go build -trimpath -o "/tmp/bin/lxd-agent" -tags=agent,netgo github.com/canonical/lxd/lxd-agent | |
strip -s /tmp/bin/* | |
# bin/max (sizes are in MiB) | |
SIZES="lxc 15 | |
lxd-agent 13" | |
MIB="$((1024 * 1024))" | |
while read -r bin max; do | |
cur="$(stat --format=%s "/tmp/bin/${bin}")" | |
min=$((max - 1)) | |
min_mib="$((min * MIB))" | |
max_mib="$((max * MIB))" | |
rm -f "/tmp/bin/${bin}" | |
if [ "${cur}" -gt "${max_mib}" ]; then | |
echo "FAIL: ${bin} binary size exceeds ${max}MiB" | |
exit 1 | |
fi | |
# XXX: check for when we need to lower the min/max sizes | |
if [ "${cur}" -lt "${min_mib}" ]; then | |
echo "Congratulations: ${bin} binary size reduced below ${min}MiB" | |
echo "It is now time to edit the workflow job to use smaller min/max sizes for ${bin}" | |
exit 1 | |
fi | |
echo "OK: ${bin} is between ${min} and ${max}MiB" | |
done <<< ${SIZES} | |
- name: Make GOCOVERDIR | |
run: mkdir -p "${GOCOVERDIR}" | |
if: env.GOCOVERDIR != '' | |
- name: Run static analysis | |
env: | |
GITHUB_BEFORE: ${{ github.event.before }} | |
run: | | |
set -eux | |
sudo chmod o+w ./lxd/metadata/configuration.json | |
sudo chmod o+w ./doc/metadata.txt | |
sudo chmod o+w ./po/* | |
sudo chmod o+w ./lxd/auth/entitlements_generated.go | |
sudo chmod o+w ./lxd/auth/drivers/openfga_model.openfga | |
make static-analysis | |
- name: Unit tests (all) | |
run: | | |
set -eux | |
sudo --preserve-env=CGO_CFLAGS,CGO_LDFLAGS,CGO_LDFLAGS_ALLOW,GOCOVERDIR,LD_LIBRARY_PATH LD_LIBRARY_PATH=${LD_LIBRARY_PATH} env "PATH=${PATH}" make check-unit | |
- name: Upload coverage data | |
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
with: | |
name: coverage-unit | |
path: ${{env.GOCOVERDIR}} | |
if: env.GOCOVERDIR != '' | |
- name: Download minio/mc to add to system test dependencies | |
uses: ./.github/actions/download-minio | |
- name: Strip binaries/libs | |
run: | | |
set -eux | |
strip --strip-all -v /home/runner/go/bin/dqlite/libs/*.so* | |
strip --strip-all -v /home/runner/go/bin/lxc* | |
strip --strip-all -v /home/runner/go/bin/lxd* | |
- name: Upload system test dependencies | |
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
with: | |
name: system-test-deps | |
path: | | |
/home/runner/go/bin/lxc* | |
/home/runner/go/bin/lxd* | |
/home/runner/go/bin/mc | |
/home/runner/go/bin/minio | |
/home/runner/go/bin/dqlite | |
retention-days: 1 | |
system-tests: | |
env: | |
LXD_CEPH_CLUSTER: "ceph" | |
LXD_CEPH_CEPHFS: "cephfs" | |
LXD_CEPH_CEPHOBJECT_RADOSGW: "http://127.0.0.1" | |
LXD_CONCURRENT: "1" | |
LXD_VERBOSE: "1" | |
LXD_OFFLINE: "1" | |
LXD_TMPFS: "1" | |
name: System | |
runs-on: ubuntu-22.04 | |
needs: code-tests | |
strategy: | |
fail-fast: false | |
matrix: | |
suite: ["cluster", "standalone"] | |
backend: ["dir", "btrfs", "lvm", "zfs", "ceph", "random"] | |
steps: | |
- name: Checkout | |
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
- name: Tune disk performance | |
uses: ./.github/actions/tune-disk-performance | |
- name: Reclaim disk space | |
uses: ./.github/actions/reclaim-disk-space | |
- name: Remove docker | |
uses: ./.github/actions/disable-docker | |
- name: Install Go | |
uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0 | |
with: | |
go-version-file: 'go.mod' | |
- name: Install runtime dependencies | |
uses: ./.github/actions/install-lxd-runtimedeps | |
- name: Download system test dependencies | |
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
with: | |
name: system-test-deps | |
merge-multiple: true | |
path: /home/runner/go/bin | |
- name: Set exec perms on LXD binaries | |
run: | | |
set -eux | |
ls -lR /home/runner/go/bin/ | |
chmod uog+x /home/runner/go/bin/* | |
- name: Setup MicroCeph | |
if: ${{ matrix.backend == 'ceph' }} | |
uses: ./.github/actions/setup-microceph | |
with: | |
microceph-channel: "squid/candidate" | |
- name: Make GOCOVERDIR | |
run: | | |
set -eux | |
mkdir -p "${GOCOVERDIR}" | |
if: env.GOCOVERDIR != '' | |
- name: "Run system tests (${{ matrix.suite }}, ${{ matrix.backend }})" | |
run: | | |
set -eux | |
chmod +x ~ | |
echo "root:1000000:1000000000" | sudo tee /etc/subuid /etc/subgid | |
cd test | |
sudo --preserve-env=PATH,GOPATH,GOCOVERDIR,GITHUB_ACTIONS,LXD_VERBOSE,LXD_BACKEND,LXD_CEPH_CLUSTER,LXD_CEPH_CEPHFS,LXD_CEPH_CEPHOBJECT_RADOSGW,LXD_OFFLINE,LXD_SKIP_TESTS,LXD_REQUIRED_TESTS, LXD_BACKEND=${{ matrix.backend }} ./main.sh ${{ matrix.suite }} | |
- name: Upload coverage data | |
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
with: | |
name: coverage-${{ matrix.suite }}-${{ matrix.backend }} | |
path: ${{env.GOCOVERDIR}} | |
if: env.GOCOVERDIR != '' | |
tics: | |
name: Tiobe TICS | |
runs-on: ubuntu-22.04 | |
needs: system-tests | |
env: | |
CGO_CFLAGS: "-I/home/runner/go/bin/dqlite/include/" | |
CGO_LDFLAGS: "-L/home/runner/go/bin/dqlite/libs/" | |
LD_LIBRARY_PATH: "/home/runner/go/bin/dqlite/libs/" | |
CGO_LDFLAGS_ALLOW: "(-Wl,-wrap,pthread_create)|(-Wl,-z,now)" | |
if: ${{ ( github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' ) && github.ref_name == 'main' && github.repository == 'canonical/lxd' }} | |
steps: | |
- name: Checkout | |
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
with: | |
# A non-shallow clone is needed for the Differential ShellCheck | |
fetch-depth: 0 | |
- name: Tune disk performance | |
uses: ./.github/actions/tune-disk-performance | |
- name: Reclaim disk space | |
uses: ./.github/actions/reclaim-disk-space | |
- name: Remove docker | |
uses: ./.github/actions/disable-docker | |
- name: Install Go | |
uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0 | |
with: | |
go-version-file: 'go.mod' | |
- name: Download coverage data | |
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
with: | |
pattern: coverage-* | |
path: ${{env.GOCOVERDIR}} | |
merge-multiple: true | |
- name: Download system test dependencies | |
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
with: | |
name: system-test-deps | |
merge-multiple: true | |
path: /home/runner/go/bin | |
- name: Install build dependencies | |
uses: ./.github/actions/install-lxd-builddeps | |
- name: Install dependencies | |
run: | | |
go install github.com/axw/gocov/gocov@latest | |
go install github.com/AlekSi/gocov-xml@latest | |
go install honnef.co/go/tools/cmd/staticcheck@latest | |
- name: Convert coverage files | |
run: | | |
go tool covdata textfmt -i="${GOCOVERDIR}" -o "${GOCOVERDIR}"/coverage.out | |
gocov convert "${GOCOVERDIR}"/coverage.out > "${GOCOVERDIR}"/coverage.json | |
gocov-xml < "${GOCOVERDIR}"/coverage.json > "${GOCOVERDIR}"/coverage-go.xml | |
- name: Run TICS | |
uses: tiobe/tics-github-action@03294702eb0a8e13c06ff1949c7bb6643b4c60fc # v3.2.1 | |
with: | |
mode: qserver | |
project: LXD | |
viewerUrl: https://canonical.tiobe.com/tiobeweb/TICS/api/cfg?name=default | |
branchdir: ${{ github.workspace }} | |
ticsAuthToken: ${{ secrets.TICS_AUTH_TOKEN }} | |
installTics: true | |
calc: ALL | |
tmpdir: /tmp/tics | |
client: | |
name: Client | |
strategy: | |
fail-fast: false | |
matrix: | |
os: | |
- ubuntu-latest | |
- macos-latest | |
- windows-latest | |
runs-on: ${{ matrix.os }} | |
steps: | |
- name: Checkout code | |
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
- name: Install Go | |
uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0 | |
with: | |
go-version-file: 'go.mod' | |
- name: Create build directory | |
run: | | |
set -eux | |
mkdir bin | |
- name: Build static lxc (x86_64) | |
env: | |
CGO_ENABLED: 0 | |
GOARCH: amd64 | |
run: | | |
set -eux | |
go build -ldflags "-s -w" -o trimpath -o bin/lxc.x86_64 ./lxc | |
- name: Build static lxc (aarch64) | |
env: | |
CGO_ENABLED: 0 | |
GOARCH: arm64 | |
run: | | |
set -eux | |
go build -ldflags "-s -w" -o trimpath -o bin/lxc.aarch64 ./lxc | |
- name: Build static lxd-benchmark | |
if: runner.os == 'Linux' | |
env: | |
CGO_ENABLED: 0 | |
run: | | |
set -eux | |
GOARCH=amd64 go build -ldflags "-s -w" -o trimpath -o bin/lxd-benchmark.x86_64 ./lxd-benchmark | |
GOARCH=arm64 go build -ldflags "-s -w" -o trimpath -o bin/lxd-benchmark.aarch64 ./lxd-benchmark | |
- name: Build static lxd-migrate | |
if: runner.os == 'Linux' | |
env: | |
CGO_ENABLED: 0 | |
run: | | |
set -eux | |
GOARCH=amd64 go build -ldflags "-s -w" -o trimpath -o bin/lxd-migrate.x86_64 ./lxd-migrate | |
GOARCH=arm64 go build -ldflags "-s -w" -o trimpath -o bin/lxd-migrate.aarch64 ./lxd-migrate | |
- name: Unit tests (client) | |
env: | |
CGO_ENABLED: 0 | |
run: | | |
set -eux | |
go test -v ./client/... | |
- name: Unit tests (lxc) | |
env: | |
CGO_ENABLED: 0 | |
run: | | |
set -eux | |
go test -v ./lxc/... | |
- name: Unit tests (shared) | |
env: | |
CGO_ENABLED: 0 | |
run: | | |
set -eux | |
go test -v ./shared/... | |
- name: Upload lxc client artifacts | |
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
continue-on-error: true | |
with: | |
name: lxd-clients-${{ runner.os }} | |
path: bin/ | |
documentation: | |
name: Documentation | |
runs-on: ubuntu-latest | |
steps: | |
- name: Checkout | |
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
- name: Install Go | |
uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0 | |
with: | |
go-version-file: 'go.mod' | |
- name: Install dependencies | |
run: | | |
set -eux | |
sudo apt-get install aspell aspell-en | |
sudo snap install mdl | |
- name: Run markdown linter | |
run: | | |
set -eux | |
make doc-lint | |
- name: Build docs (Sphinx) | |
shell: 'script -q -e -c "export TERM=xterm-256color; bash {0}"' | |
run: | | |
set -eux | |
make doc | |
if [ -s doc/.sphinx/warnings.txt ]; then cat doc/.sphinx/warnings.txt; exit 1; fi | |
- name: Run spell checker | |
run: | | |
set -eux | |
make doc-spellcheck | |
- name: Run inclusive naming checker | |
uses: get-woke/woke-action@b2ec032c4a2c912142b38a6a453ad62017813ed0 # v0 | |
with: | |
fail-on-error: true | |
woke-args: "*.md **/*.md -c https://github.com/canonical/Inclusive-naming/raw/main/config.yml" | |
- name: Run link checker | |
# Run link checker during scheduled CI runs only | |
if: ${{ github.event_name == 'schedule' }} | |
shell: 'script -q -e -c "export TERM=xterm-256color; bash {0}"' | |
run: | | |
set -eux | |
make doc-linkcheck | |
- name: Upload documentation artifacts | |
if: always() | |
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
with: | |
name: documentation | |
path: doc/_build | |
snap: | |
name: Trigger snap edge build | |
runs-on: ubuntu-22.04 | |
needs: [code-tests, system-tests, client, documentation] | |
if: ${{ github.repository == 'canonical/lxd' && github.event_name == 'push' && github.actor != 'dependabot[bot]' }} | |
steps: | |
- name: Checkout code | |
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
- name: Setup Launchpad SSH access | |
env: | |
SSH_AUTH_SOCK: /tmp/ssh_agent.sock | |
LAUNCHPAD_LXD_BOT_KEY: ${{ secrets.LAUNCHPAD_LXD_BOT_KEY }} | |
run: | | |
set -eux | |
mkdir -m 0700 -p ~/.ssh/ | |
ssh-agent -a "${SSH_AUTH_SOCK}" > /dev/null | |
ssh-add - <<< "${{ secrets.LAUNCHPAD_LXD_BOT_KEY }}" | |
ssh-add -L > ~/.ssh/id_ed25519.pub | |
# In ephemeral environments like GitHub Action runners, relying on TOFU isn't providing any security | |
# so require the key obtained by `ssh-keyscan` to match the expected hash from https://help.launchpad.net/SSHFingerprints | |
ssh-keyscan git.launchpad.net >> ~/.ssh/known_hosts | |
ssh-keygen -qlF git.launchpad.net | grep -xF 'git.launchpad.net RSA SHA256:UNOzlP66WpDuEo34Wgs8mewypV0UzqHLsIFoqwe8dYo' | |
- name: Install Go | |
uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0 | |
with: | |
go-version-file: 'go.mod' | |
- name: Trigger Launchpad snap build | |
env: | |
SSH_AUTH_SOCK: /tmp/ssh_agent.sock | |
TARGET: >- | |
${{ fromJson('{ | |
"main": "latest-edge", | |
"stable-5.21": "5.21-edge", | |
"stable-5.0": "5.0-edge", | |
}')[github.ref_name] }} | |
run: | | |
set -eux | |
git config --global transfer.fsckobjects true | |
git config --global user.name "Canonical LXD Bot" | |
git config --global user.email "[email protected]" | |
git config --global commit.gpgsign true | |
git config --global gpg.format "ssh" | |
git config --global user.signingkey ~/.ssh/id_ed25519.pub | |
localRev="$(git rev-parse HEAD)" | |
go install github.com/canonical/lxd-ci/lxd-snapcraft@latest | |
git clone -b "${TARGET}" git+ssh://[email protected]/~canonical-lxd/lxd ~/lxd-pkg-snap-lp | |
cd ~/lxd-pkg-snap-lp | |
lxd-snapcraft -package lxd -set-version "git-${localRev:0:7}" -set-source-commit "${localRev}" | |
git add --all | |
git commit --all --quiet -s --allow-empty -m "Automatic upstream build (${TARGET})" -m "Upstream commit: ${localRev}" | |
git show | |
git push --quiet |