From a31c79af4c7d9d2ce3fad5bb6e113b258c492118 Mon Sep 17 00:00:00 2001 From: Patrick Roy Date: Tue, 15 Oct 2024 15:10:49 +0100 Subject: [PATCH] test: move "doing A/B-Test" print statement below ignore-check Move the print statement that states which A/B-Tests we're performing below the check that filters out ignored test cases. This way, we avoid claiming to do an A/B-Test for things that are later actually filtered out (meaning no A/B-Test actually happens). Signed-off-by: Patrick Roy --- tools/ab_test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/ab_test.py b/tools/ab_test.py index daf4c647de7..fa76f087cf8 100755 --- a/tools/ab_test.py +++ b/tools/ab_test.py @@ -211,9 +211,6 @@ def analyze_data( ), "A and B run produced incomparable data. This is a bug in the test!" for metric, (values_a, unit) in metrics_a.items(): - print( - f"Doing A/B-test for dimensions {dimension_set} and property {metric}" - ) result = check_regression( values_a, metrics_b[metric][0], n_resamples=n_resamples ) @@ -269,6 +266,8 @@ def analyze_data( if is_ignored(dict(dimension_set)): continue + print(f"Doing A/B-test for dimensions {dimension_set} and property {metric}") + values_a = processed_emf_a[dimension_set][metric][0] baseline_mean = statistics.mean(values_a)