From 5c2bf9eee31a47735072cb3b51a8f1fc454bc4b7 Mon Sep 17 00:00:00 2001 From: Lokesh Mandvekar Date: Wed, 4 Dec 2024 16:57:30 +0530 Subject: [PATCH] [skip-ci] TMT: system tests This commit introduces TMT test jobs triggered via packit to run system tests on testing-farm infrastructure. Tests are run for all active Fedora and CentOS Stream environments for aarch64 and x86_64. The same test plan will be reused by Fedora for bodhi, zuul and fedora-ci gating tests. Packit will handle syncing of test plan and sources from upstream to downstream. Jobs will be selectively triggered for `main` v/s release branches based on the `release` PR label. The GHA for labeling PRs has also been updated in this commit. Test jobs on the main branch will include builds from the `rhcontainerbot/podman-next` copr while tests on release branches will only include official distro packages. Signed-off-by: Lokesh Mandvekar --- .github/labeler.yml | 2 + .packit.yaml | 73 ++++++++++++++++++++++++++++++++++-- Makefile | 19 ++++++---- plans/main.fmf | 32 ++++++++++++++++ plans/rootless-user-setup.sh | 10 +++++ plans/tmt-envvars | 21 +++++++++++ rpm/gating.yaml | 18 +++++++++ rpm/podman.spec | 2 + test/system/tmt/main.fmf | 36 ++++++++++++++++++ test/system/tmt/test.sh | 7 ++++ 10 files changed, 208 insertions(+), 12 deletions(-) create mode 100644 plans/main.fmf create mode 100644 plans/rootless-user-setup.sh create mode 100644 plans/tmt-envvars create mode 100644 rpm/gating.yaml create mode 100644 test/system/tmt/main.fmf create mode 100644 test/system/tmt/test.sh diff --git a/.github/labeler.yml b/.github/labeler.yml index ce5a69b115..6e0821681b 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -6,3 +6,5 @@ kind/api-change: machine: - changed-files: - any-glob-to-any-file: pkg/machine/** +release: + - base-branch: [^v?(0|[1-9]\d*).(0|[1-9]\d*)$] diff --git a/.packit.yaml b/.packit.yaml index 637e02041b..0880a349fe 100644 --- a/.packit.yaml +++ b/.packit.yaml @@ -5,6 +5,23 @@ downstream_package_name: podman upstream_tag_template: v{version} +# These files get synced from upstream to downstream (Fedora / CentOS Stream) on every +# propose-downstream job. This is done so tests maintained upstream can be run +# downstream in Zuul CI and Bodhi. +# Ref: https://packit.dev/docs/configuration#files_to_sync +files_to_sync: + - src: rpm/gating.yaml + dest: gating.yaml + delete: true + - src: plans/ + dest: plans/ + delete: true + mkpath: true + - src: .fmf/ + dest: .fmf/ + delete: true + - .packit.yaml + packages: podman-fedora: pkg_tool: fedpkg @@ -31,12 +48,12 @@ jobs: failure_comment: message: "Ephemeral COPR build failed. @containers/packit-build please check." enable_net: true - targets: + targets: &fedora_copr_targets - fedora-all-x86_64 - fedora-all-aarch64 - job: copr_build - trigger: pull_request + trigger: ignore packages: [podman-eln] notifications: *packit_build_failure_notification enable_net: true @@ -53,7 +70,7 @@ jobs: packages: [podman-centos] notifications: *packit_build_failure_notification enable_net: true - targets: + targets: ¢os_copr_targets - centos-stream-9-x86_64 - centos-stream-9-aarch64 - centos-stream-10-x86_64 @@ -72,9 +89,57 @@ jobs: enable_net: true - job: tests - identifier: cockpit-revdeps trigger: pull_request packages: [podman-fedora] + targets: *fedora_copr_targets + require: &dev_tests + label: + absent: + # Run only on PRs without `release` label + - release + tf_extra_params: + environments: + - artifacts: + # Test with latest builds from `rhcontainerbot/podman-next` copr + - type: repository-file + id: https://copr.fedorainfracloud.org/coprs/rhcontainerbot/podman-next/repo/fedora-$releasever/rhcontainerbot-podman-next-fedora-$releasever.repo + + - job: tests + trigger: pull_request + packages: [podman-fedora] + targets: *fedora_copr_targets + require: &release_tests + label: + present: + # Run only on PRs with `release` label + # Test with only the official Fedora packages + - release + # We want to preserve rpms built on release branches past the default 60 + # days. With `true` they'll never be deleted. + preserve_project: true + + - job: tests + trigger: pull_request + packages: [podman-centos] + targets: *centos_copr_targets + require: *dev_tests + tf_extra_params: + environments: + - artifacts: + - type: repository-file + id: https://copr.fedorainfracloud.org/coprs/rhcontainerbot/podman-next/repo/centos-stream-$releasever/rhcontainerbot-podman-next-centos-stream-$releasever.repo + + - job: tests + trigger: pull_request + packages: [podman-centos] + targets: *centos_copr_targets + require: *release_tests + preserve_project: true + + - job: tests + identifier: cockpit-revdeps + trigger: ignore + packages: [podman-fedora] notifications: failure_comment: message: "Cockpit tests failed for commit {commit_sha}. @martinpitt, @jelly, @mvollmer please check." diff --git a/Makefile b/Makefile index a17270fe78..20dbb35163 100644 --- a/Makefile +++ b/Makefile @@ -78,6 +78,9 @@ SED=sed GREP=grep MAN_L= man -l endif +PODMAN ?= $(CURDIR)/bin/podman +PODMAN_REMOTE ?= $(CURDIR)/bin/podman-remote +QUADLET ?= $(CURDIR)/bin/quadlet # This isn't what we actually build; it's a superset, used for target # dependencies. Basically: all *.go and *.c files, except *_test.go, @@ -706,8 +709,8 @@ localmachine: localsystem: # Wipe existing config, database, and cache: start with clean slate. $(RM) -rf ${HOME}/.local/share/containers ${HOME}/.config/containers - PODMAN=$(CURDIR)/bin/podman QUADLET=$(CURDIR)/bin/quadlet bats -T --filter-tags '!ci:parallel' test/system/ - PODMAN=$(CURDIR)/bin/podman QUADLET=$(CURDIR)/bin/quadlet bats -T --filter-tags ci:parallel -j $$(nproc) test/system/ + PODMAN=$(PODMAN) QUADLET=$(QUADLET) bats -T --filter-tags '!ci:parallel' test/system/ + PODMAN=$(PODMAN) QUADLET=$(QUADLET) bats -T --filter-tags ci:parallel -j $$(nproc) test/system/ .PHONY: remotesystem remotesystem: @@ -720,26 +723,26 @@ remotesystem: # . Stop server. rc=0;\ if timeout -v 1 true; then \ - if ./bin/podman-remote info; then \ + if $(PODMAN_REMOTE) info; then \ echo "Error: podman system service (not ours) is already running" >&2;\ exit 1;\ fi;\ - ./bin/podman system service --timeout=0 > $(if $(PODMAN_SERVER_LOG),$(PODMAN_SERVER_LOG),/dev/null) 2>&1 & \ + $(PODMAN) system service --timeout=0 > $(if $(PODMAN_SERVER_LOG),$(PODMAN_SERVER_LOG),/dev/null) 2>&1 & \ retry=5;\ while [ $$retry -ge 0 ]; do\ echo Waiting for server...;\ sleep 1;\ - ./bin/podman-remote info >/dev/null 2>&1 && break;\ + $(PODMAN_REMOTE) info >/dev/null 2>&1 && break;\ retry=$$(expr $$retry - 1);\ done;\ if [ $$retry -lt 0 ]; then\ - echo "Error: ./bin/podman system service did not come up" >&2;\ + echo "Error: $(PODMAN) system service did not come up" >&2;\ exit 1;\ fi;\ - env PODMAN="$(CURDIR)/bin/podman-remote" bats -T --filter-tags '!ci:parallel' test/system/ ;\ + env PODMAN="$(PODMAN_REMOTE)" bats -T --filter-tags '!ci:parallel' test/system/ ;\ rc=$$?; \ if [ $$rc -eq 0 ]; then \ - env PODMAN="$(CURDIR)/bin/podman-remote" bats -T --filter-tags ci:parallel -j $$(nproc) test/system/ ;\ + env PODMAN="$(PODMAN_REMOTE)" bats -T --filter-tags ci:parallel -j $$(nproc) test/system/ ;\ rc=$$?;\ fi; \ kill %1;\ diff --git a/plans/main.fmf b/plans/main.fmf new file mode 100644 index 0000000000..54913b501f --- /dev/null +++ b/plans/main.fmf @@ -0,0 +1,32 @@ +discover: + how: fmf +execute: + how: tmt +environment: + PODMAN: /usr/bin/podman + PODMAN_REMOTE: /usr/bin/podman-remote + PODMAN_TESTING: /usr/bin/podman-testing + QUADLET: /usr/libexec/podman/quadlet + ROOTLESS_USER: rootlessu +prepare: + how: shell + script: $TMT_TREE/plans/rootless-user-setup.sh + +/upstream: + summary: Run tests on upstream PRs + discover+: + filter: tag:upstream + adjust+: + enabled: false + when: initiator is not defined or initiator != packit + +/downstream: + summary: Run tests on bodhi / errata and dist-git PRs + discover+: + filter: tag:downstream + dist-git-install-builddeps: true + dist-git-source: true + dist-git-remove-fmf-root: true + adjust+: + enabled: false + when: initiator == packit diff --git a/plans/rootless-user-setup.sh b/plans/rootless-user-setup.sh new file mode 100644 index 0000000000..6e9bf91c31 --- /dev/null +++ b/plans/rootless-user-setup.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -exo pipefail + +# Set in TMT plan environment +ROOTLESS_USER=$ROOTLESS_USER + +useradd -Z user_u $ROOTLESS_USER +loginctl enable-linger $ROOTLESS_USER +chown -R $ROOTLESS_USER:$ROOTLESS_USER $TMT_TREE diff --git a/plans/tmt-envvars b/plans/tmt-envvars new file mode 100644 index 0000000000..d737a550c2 --- /dev/null +++ b/plans/tmt-envvars @@ -0,0 +1,21 @@ +PODMAN +PODMAN_REMOTE +PODMAN_TESTING +QUADLET +ROOTLESS_USER +TMT_TREE +TMT_PLAN_DATA +TMT_VERSION +TMT_TEST_NAME +TMT_TEST_DATA +TMT_TEST_SERIAL_NUMBER +TMT_TEST_ITERATION_ID +TMT_TEST_METADATA +TMT_SOURCE_DIR +TMT_REBOOT_COUNT +TMT_TEST_RESTART_COUNT +TMT_TOPOLOGY_BASH +TMT_TOPOLOGY_YAML +TMT_TEST_PIDFILE +TMT_TEST_PIDFILE_LOCK +TMT_TEST_PIDFILE_ROOT diff --git a/rpm/gating.yaml b/rpm/gating.yaml new file mode 100644 index 0000000000..ad0c790c44 --- /dev/null +++ b/rpm/gating.yaml @@ -0,0 +1,18 @@ +--- !Policy +product_versions: + - fedora-* +decision_contexts: + - bodhi_update_push_stable + - bodhi_update_push_testing +subject_type: koji_build +rules: + - !PassingTestCaseRule {test_case_name: fedora-ci.koji-build.tier0.functional} + +# recipients: jnovy, lsm5, santiago +--- !Policy +product_versions: + - rhel-10 + - rhel-9 +decision_context: osci_compose_gate +rules: + - !PassingTestCaseRule {test_case_name: osci.brew-build.tier0.functional} diff --git a/rpm/podman.spec b/rpm/podman.spec index dc523f16e6..bc06a7138b 100644 --- a/rpm/podman.spec +++ b/rpm/podman.spec @@ -24,6 +24,8 @@ %if %{defined fedora} %define build_with_btrfs 1 +# No machine stuff for rhel +%define machine 1 %endif %if %{defined copr_username} diff --git a/test/system/tmt/main.fmf b/test/system/tmt/main.fmf new file mode 100644 index 0000000000..982fd8dfa5 --- /dev/null +++ b/test/system/tmt/main.fmf @@ -0,0 +1,36 @@ +require: + - bats + - golang + - make + - podman + - podman-remote + - podman-tests + - slirp4netns +environment: + # `su -` resets all environment variables unless whitelisted + ROOTLESS_ENV_OPTS: "--whitelist-environment=$(cat $TMT_TREE/plans/tmt-envvars | tr '\n' ',') - $ROOTLESS_USER -c" + CLEANUP_CMD: $(rm -rf /home/$ROOTLESS_USER/.local/share/containers) + +/local-root: + tag: [ downstream, upstream ] + summary: local rootful test + test: ./test.sh localsystem + duration: 60m + +/local-rootless: + tag: [ downstream, upstream ] + summary: rootless test + test: $CLEANUP_CMD && su $ROOTLESS_ENV_OPTS './test.sh localsystem' + duration: 60m + +/remote-root: + tag: [ downstream, upstream ] + summary: remote rootful test + test: ./test.sh remotesystem + duration: 60m + +/remote-rootless: + tag: [ downstream, upstream ] + summary: remote rootless test + test: $CLEANUP_CMD && su $ROOTLESS_ENV_OPTS './test.sh remotesystem' + duration: 60m diff --git a/test/system/tmt/test.sh b/test/system/tmt/test.sh new file mode 100644 index 0000000000..be5733cc27 --- /dev/null +++ b/test/system/tmt/test.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -exo pipefail + +whoami + +make -C $TMT_TREE $1