From 20d5146e1e95a077b440ea1c242a5a7693247af5 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 26 Jul 2024 10:07:38 -0400 Subject: [PATCH] tests: Split up tmt tests into separate plans Because tmt right now doesn't support isolation automatically, and many of the tests we'll want to write mutate the system, we'll need to copy-paste the base plan. This changes the `make test-tmt` to have its own little tmt wrapper that dynamically scans the plans/ and sorts them by a priority, then runs them one by one currently. Closes: https://github.com/containers/bootc/issues/735 Signed-off-by: Colin Walters --- ...tegration-run.fmf => test-01-readonly.fmf} | 10 +++-- plans/test-20-local-upgrade.fmf | 12 ++++++ .../booted/{ => readonly}/001-test-status.nu | 0 tests/booted/{ => readonly}/basic.py | 0 tests/booted/readonly/tap.nu | 1 + ...rade.nu => test-image-pushpull-upgrade.nu} | 0 xtask/src/xtask.rs | 43 +++++++++++++++++-- 7 files changed, 59 insertions(+), 7 deletions(-) rename plans/{integration-run.fmf => test-01-readonly.fmf} (57%) create mode 100644 plans/test-20-local-upgrade.fmf rename tests/booted/{ => readonly}/001-test-status.nu (100%) rename tests/booted/{ => readonly}/basic.py (100%) create mode 120000 tests/booted/readonly/tap.nu rename tests/booted/{002-test-image-pushpull-upgrade.nu => test-image-pushpull-upgrade.nu} (100%) diff --git a/plans/integration-run.fmf b/plans/test-01-readonly.fmf similarity index 57% rename from plans/integration-run.fmf rename to plans/test-01-readonly.fmf index 1190fcd33..3611a96ec 100644 --- a/plans/integration-run.fmf +++ b/plans/test-01-readonly.fmf @@ -5,14 +5,18 @@ provision: # Generated by make test-tmt image: file://./target/testvm/disk.qcow2 disk: 20 -summary: Execute booted tests +summary: Execute booted readonly/nondestructive tests execute: how: tmt # There's currently two dynamic test frameworks; python and nushell. # python is well known and understood. nushell is less well known, but # is quite nice for running subprocesses and the like while making # it easy to parse JSON etc. + # All of these tests should generally be read-only - avoid any kind + # of persistent changes. + # If you need to do that, unfortunately right now that needs to be + # a separate plan. script: | set -xeu - pytest tests/booted/*.py - ls tests/booted/*-test-*.nu |sort -n | while read t; do nu $t; done + pytest tests/booted/readonly/*.py + ls tests/booted/readonly/*-test-*.nu |sort -n | while read t; do nu $t; done diff --git a/plans/test-20-local-upgrade.fmf b/plans/test-20-local-upgrade.fmf new file mode 100644 index 000000000..e095c60bf --- /dev/null +++ b/plans/test-20-local-upgrade.fmf @@ -0,0 +1,12 @@ +# +provision: + how: virtual + # Generated by make test-tmt + image: file://./target/testvm/disk.qcow2 + disk: 20 +summary: Execute local upgrade tests +execute: + how: tmt + # We avoid writing nontrivial shell script as a general rule, + # so this is written in nu. + script: exec nu tests/booted/test-image-pushpull-upgrade.nu diff --git a/tests/booted/001-test-status.nu b/tests/booted/readonly/001-test-status.nu similarity index 100% rename from tests/booted/001-test-status.nu rename to tests/booted/readonly/001-test-status.nu diff --git a/tests/booted/basic.py b/tests/booted/readonly/basic.py similarity index 100% rename from tests/booted/basic.py rename to tests/booted/readonly/basic.py diff --git a/tests/booted/readonly/tap.nu b/tests/booted/readonly/tap.nu new file mode 120000 index 000000000..56a69a5db --- /dev/null +++ b/tests/booted/readonly/tap.nu @@ -0,0 +1 @@ +../tap.nu \ No newline at end of file diff --git a/tests/booted/002-test-image-pushpull-upgrade.nu b/tests/booted/test-image-pushpull-upgrade.nu similarity index 100% rename from tests/booted/002-test-image-pushpull-upgrade.nu rename to tests/booted/test-image-pushpull-upgrade.nu diff --git a/xtask/src/xtask.rs b/xtask/src/xtask.rs index 5fb729ba8..ab4c58e0c 100644 --- a/xtask/src/xtask.rs +++ b/xtask/src/xtask.rs @@ -148,13 +148,48 @@ fn update_generated(sh: &Shell) -> Result<()> { #[context("test-integration")] fn test_tmt(sh: &Shell) -> Result<()> { + // We need to split most of our tests into separate plans because tmt doesn't + // support automatic isolation. (xref) + let mut all_plan_files = + sh.read_dir("plans")? + .into_iter() + .try_fold(Vec::new(), |mut acc, ent| -> Result<_> { + let path = Utf8PathBuf::try_from(ent)?; + let Some(ext) = path.extension() else { + return Ok(acc); + }; + if ext != "fmf" { + return Ok(acc); + } + let stem = path.file_stem().expect("file stem"); + let Some((prefix, suffix)) = stem.split_once('-') else { + return Ok(acc); + }; + if prefix != "test" { + return Ok(acc); + } + let Some((priority, _)) = suffix.split_once('-') else { + anyhow::bail!("Invalid test {path}"); + }; + let priority: u32 = priority + .parse() + .with_context(|| format!("Parsing {path}"))?; + acc.push((priority, stem.to_string())); + Ok(acc) + })?; + all_plan_files.sort_by_key(|v| v.0); + println!("Discovered plans: {all_plan_files:?}"); + cmd!(sh, "cargo run -p tests-integration run-vm prepare-tmt").run()?; // cc https://pagure.io/testcloud/pull-request/174 cmd!(sh, "rm -vf /var/tmp/tmt/testcloud/images/disk.qcow2").run()?; - if let Err(e) = cmd!(sh, "tmt run plans -n integration-run").run() { - // tmt annoyingly does not output errors by default - let _ = cmd!(sh, "tmt run -l report -vvv").run(); - return Err(e.into()); + + for (_prio, name) in all_plan_files { + if let Err(e) = cmd!(sh, "tmt run plans -n {name}").run() { + // tmt annoyingly does not output errors by default + let _ = cmd!(sh, "tmt run -l report -vvv").run(); + return Err(e.into()); + } } Ok(()) }