From 7fa0f487e727e57e0e5bf088366745e73ff2d39f Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Fri, 21 Jun 2024 11:46:20 +0200 Subject: [PATCH 1/2] Add libtest_mimic runner Signed-off-by: Ryan Levick --- Cargo.lock | 159 ++++++++++++++++++++++++++++ crates/conformance-tests/Cargo.toml | 3 +- crates/conformance-tests/src/lib.rs | 91 ++++++++++++++-- 3 files changed, 246 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 08f2956..c0d2c0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -38,6 +38,55 @@ dependencies = [ "memchr", ] +[[package]] +name = "anstream" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" + +[[package]] +name = "anstyle-parse" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" +dependencies = [ + "anstyle", + "windows-sys 0.52.0", +] + [[package]] name = "anyhow" version = "1.0.86" @@ -122,6 +171,52 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "clap" +version = "4.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5db83dced34638ad474f39f250d7fea9598bdd239eaced1bdf45d597da0f433f" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7e204572485eb3fbf28f871612191521df159bc3e15a9f5064c66dba3a8c05f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c780290ccf4fb26629baa7a1081e68ced113f1d3ec302fa5948f1c381ebf06c6" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" + +[[package]] +name = "colorchoice" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" + [[package]] name = "conformance-tests" version = "0.1.0" @@ -129,6 +224,7 @@ dependencies = [ "anyhow", "flate2", "json5", + "libtest-mimic", "reqwest", "serde", "tar", @@ -226,6 +322,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "escape8259" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5692dd7b5a1978a5aeb0ce83b7655c58ca8efdcb79d21036ea249da95afec2c6" + [[package]] name = "fastrand" version = "2.1.0" @@ -529,6 +631,12 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" + [[package]] name = "itoa" version = "1.0.11" @@ -581,6 +689,18 @@ version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +[[package]] +name = "libtest-mimic" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc0bda45ed5b3a2904262c1bb91e526127aa70e7ef3758aba2ef93cf896b9b58" +dependencies = [ + "clap", + "escape8259", + "termcolor", + "threadpool", +] + [[package]] name = "linux-raw-sys" version = "0.4.14" @@ -1075,6 +1195,12 @@ dependencies = [ "smallvec", ] +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "syn" version = "2.0.65" @@ -1142,6 +1268,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "test-environment" version = "0.1.0" @@ -1174,6 +1309,15 @@ dependencies = [ "syn", ] +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -1332,6 +1476,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "vcpkg" version = "0.2.15" @@ -1497,6 +1647,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/crates/conformance-tests/Cargo.toml b/crates/conformance-tests/Cargo.toml index 9777aeb..139b95e 100644 --- a/crates/conformance-tests/Cargo.toml +++ b/crates/conformance-tests/Cargo.toml @@ -6,7 +6,8 @@ edition = "2021" [dependencies] anyhow = "1.0" flate2 = "1.0" -json5 = "0.4.1" +json5 = "0.4" +libtest-mimic = "0.7" reqwest = { version = "0.12", features = ["blocking"] } serde = { version = "1.0", features = ["derive"] } tar = "0.4.40" diff --git a/crates/conformance-tests/src/lib.rs b/crates/conformance-tests/src/lib.rs index 03a55bf..97abfd5 100644 --- a/crates/conformance-tests/src/lib.rs +++ b/crates/conformance-tests/src/lib.rs @@ -3,6 +3,30 @@ pub mod config; use anyhow::Context as _; use std::path::{Path, PathBuf}; +/// Run the conformance tests +pub fn run_tests( + run: impl Fn(Test) -> anyhow::Result<()> + Send + Clone + 'static, +) -> anyhow::Result<()> { + let tests_dir = download_tests()?; + run_tests_from(tests_dir, run) +} + +/// Run the conformance tests located in the given directory +pub fn run_tests_from( + tests_dir: impl AsRef, + run: impl Fn(Test) -> anyhow::Result<()> + Send + Clone + 'static, +) -> anyhow::Result<()> { + let trials = tests_iter(tests_dir)? + .map(|test| { + let run = run.clone(); + libtest_mimic::Trial::test(&test.name.clone(), move || { + Ok(run(test).map_err(FullError::from)?) + }) + }) + .collect(); + libtest_mimic::run(&Default::default(), trials).exit(); +} + /// Download the conformance tests and return the path to the directory where they are written to pub fn download_tests() -> anyhow::Result { let response = reqwest::blocking::get( @@ -35,7 +59,7 @@ pub fn download_tests() -> anyhow::Result { /// Read the tests directory and get an iterator to each test's directory /// /// The test directory can be downloaded using the `download_tests` function. -pub fn tests(tests_dir: &Path) -> anyhow::Result> { +pub fn tests_iter(tests_dir: impl AsRef) -> anyhow::Result> { // Like `?` but returns error wrapped in `Some` for use in `filter_map` macro_rules! r#try { ($e:expr) => { @@ -84,6 +108,8 @@ pub struct Test { } pub mod assertions { + use crate::indent_lines; + use super::config::Response as ExpectedResponse; use test_environment::http::Response as ActualResponse; @@ -95,12 +121,15 @@ pub mod assertions { // We assert the status code first, because if it's wrong, the body and headers are likely wrong anyhow::ensure!( actual.status() == expected.status, - "actual status {} != expected status {}\nbody:\n{}", + "actual status {} != expected status {} - body: {}", actual.status(), expected.status, - actual - .text() - .unwrap_or_else(|_| String::from("")) + indent_lines( + &actual + .text() + .unwrap_or_else(|_| String::from("")), + 2 + ) ); // We assert the body next, because if it's wrong, it usually has more information as to why @@ -111,7 +140,9 @@ pub mod assertions { anyhow::ensure!( actual_body == expected_body, - "actual body != expected body\nactual:\n{actual_body}\nexpected:\n{expected_body}" + "actual body != expected body\nactual: {actual_body}\nexpected: {expected_body}", + actual_body = indent_lines(&actual_body, 2), + expected_body = indent_lines(&expected_body, 2) ); let mut actual_headers = actual @@ -148,3 +179,51 @@ pub mod assertions { Ok(()) } } + +/// A wrapper around `anyhow::Error` that prints the full chain of causes +struct FullError { + error: anyhow::Error, +} + +impl From for FullError { + fn from(error: anyhow::Error) -> Self { + Self { error } + } +} + +impl std::fmt::Display for FullError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", indent_lines(&self.error.to_string(), 2))?; + write_error_chain(f, &self.error)?; + Ok(()) + } +} + +fn write_error_chain(mut f: impl std::fmt::Write, err: &anyhow::Error) -> std::fmt::Result { + let Some(cause) = err.source() else { + return Ok(()); + }; + let is_multiple = cause.source().is_some(); + writeln!(f, "\nCaused by:")?; + for (i, err) in err.chain().skip(1).enumerate() { + let err = indent_lines(&err.to_string(), 6); + if is_multiple { + writeln!(f, "{i:>4}: {err}")?; + } else { + writeln!(f, " {err}")?; + } + } + Ok(()) +} + +/// Format string such that all lines after the first are indented +fn indent_lines(str: &str, indent: usize) -> String { + str.lines() + .enumerate() + .map(|(i, line)| { + let indent = if i == 0 { 0 } else { indent }; + format!("{}{}", " ".repeat(indent), line) + }) + .collect::>() + .join("\n") +} From 9c0ea29d31f48f1aa72776c26c40e1079a671254 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Fri, 21 Jun 2024 13:46:29 +0200 Subject: [PATCH 2/2] Change function signature after some feedback Signed-off-by: Ryan Levick --- components/key-value/src/lib.rs | 10 +++++----- crates/conformance-tests/src/lib.rs | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/components/key-value/src/lib.rs b/components/key-value/src/lib.rs index 0c75054..f4338af 100644 --- a/components/key-value/src/lib.rs +++ b/components/key-value/src/lib.rs @@ -22,19 +22,19 @@ impl Guest for Component { fn handle(request: IncomingRequest, response_out: ResponseOutparam) { let result = handle(request) .map(|r| match r { - Some(e) => response(500, format!("{e}").as_bytes()), - None => response(200, b""), + Err(e) => response(500, format!("{e}").as_bytes()), + Ok(()) => response(200, b""), }) .map_err(|e| ErrorCode::InternalError(Some(e.to_string()))); ResponseOutparam::set(response_out, result) } } -fn handle(_req: IncomingRequest) -> anyhow::Result> { +fn handle(_req: IncomingRequest) -> anyhow::Result> { anyhow::ensure!(matches!(Store::open("forbidden"), Err(Error::AccessDenied))); let store = match Store::open("default") { Ok(s) => s, - Err(e) => return Ok(Some(e)), + Err(e) => return Ok(Err(e)), }; // Ensure nothing set in `bar` key @@ -71,7 +71,7 @@ fn handle(_req: IncomingRequest) -> anyhow::Result> { anyhow::ensure!(matches!(store.get("qux"), Ok(None))); anyhow::ensure!(matches!(store.get_keys().as_deref(), Ok(&[]))); - Ok(None) + Ok(Ok(())) } fn response(status: u16, body: &[u8]) -> OutgoingResponse { diff --git a/crates/conformance-tests/src/lib.rs b/crates/conformance-tests/src/lib.rs index 97abfd5..15816f8 100644 --- a/crates/conformance-tests/src/lib.rs +++ b/crates/conformance-tests/src/lib.rs @@ -19,7 +19,7 @@ pub fn run_tests_from( let trials = tests_iter(tests_dir)? .map(|test| { let run = run.clone(); - libtest_mimic::Trial::test(&test.name.clone(), move || { + libtest_mimic::Trial::test(test.name.clone(), move || { Ok(run(test).map_err(FullError::from)?) }) }) @@ -142,7 +142,7 @@ pub mod assertions { actual_body == expected_body, "actual body != expected body\nactual: {actual_body}\nexpected: {expected_body}", actual_body = indent_lines(&actual_body, 2), - expected_body = indent_lines(&expected_body, 2) + expected_body = indent_lines(expected_body, 2) ); let mut actual_headers = actual