From dec6e0168ef0893318cb20af0ffd3d554c0107aa Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 15 Mar 2022 15:26:40 -0700 Subject: [PATCH 1/2] Show thousandths place when reporting results --- pyperf/_compare.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyperf/_compare.py b/pyperf/_compare.py index a83ee2a1..41562470 100644 --- a/pyperf/_compare.py +++ b/pyperf/_compare.py @@ -39,9 +39,9 @@ def format_normalized_mean(norm_mean): if norm_mean == 1.0: return "no change" elif norm_mean < 1.0: - return "%.2fx faster" % (1.0 / norm_mean) + return "%.3fx faster" % (1.0 / norm_mean) else: - return "%.2fx slower" % norm_mean + return "%.3fx slower" % norm_mean def format_geometric_mean(norm_means): From 5f58b4fc96240ce022f95d4c02fea326f8f4070a Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 15 Mar 2022 15:26:47 -0700 Subject: [PATCH 2/2] Update tests --- pyperf/tests/test_perf_cli.py | 130 +++++++++++++++++----------------- pyperf/tests/test_timeit.py | 6 +- 2 files changed, 68 insertions(+), 68 deletions(-) diff --git a/pyperf/tests/test_perf_cli.py b/pyperf/tests/test_perf_cli.py index a559e5b7..ec047b89 100644 --- a/pyperf/tests/test_perf_cli.py +++ b/pyperf/tests/test_perf_cli.py @@ -129,7 +129,7 @@ def test_compare_to(self): stdout = self.compare('compare_to', ref_result, changed_result, '-v') expected = ('Mean +- std dev: [ref] 1.50 sec +- 0.50 sec ' - '-> [changed] 2.00 sec +- 0.50 sec: 1.33x slower\n' + '-> [changed] 2.00 sec +- 0.50 sec: 1.333x slower\n' 'Not significant!') self.assertEqual(stdout.rstrip(), expected) @@ -144,11 +144,11 @@ def test_compare_to_rest_table(self): stdout = self.compare('compare_to', ref_result, changed_result, '--table') expected = textwrap.dedent(''' - +-----------+----------+------------------------+ - | Benchmark | ref | changed | - +===========+==========+========================+ - | telco | 1.00 sec | 2.00 sec: 2.00x slower | - +-----------+----------+------------------------+ + +-----------+----------+-------------------------+ + | Benchmark | ref | changed | + +===========+==========+=========================+ + | telco | 1.00 sec | 2.00 sec: 2.000x slower | + +-----------+----------+-------------------------+ ''').strip() self.assertEqual(stdout.rstrip(), @@ -166,9 +166,9 @@ def test_compare_to_md_table(self): '--table-format', 'md') expected = textwrap.dedent(''' - | Benchmark | ref | changed | - |-----------|:--------:|:----------------------:| - | telco | 1.00 sec | 2.00 sec: 2.00x slower | + | Benchmark | ref | changed | + |-----------|:--------:|:-----------------------:| + | telco | 1.00 sec | 2.00 sec: 2.000x slower | ''').strip() self.assertEqual(stdout.rstrip(), @@ -207,7 +207,7 @@ def test_compare_to_not_significant_verbose(self): stdout = self.compare('compare_to', ref_result, changed_result, '-v') expected = ('Mean +- std dev: [ref] 1.50 sec +- 0.50 sec ' - '-> [changed] 2.00 sec +- 0.50 sec: 1.33x slower\n' + '-> [changed] 2.00 sec +- 0.50 sec: 1.333x slower\n' 'Not significant!') self.assertEqual(stdout.rstrip(), expected) @@ -236,36 +236,36 @@ def test_compare_to_cli(self): # 2 files expected = """ - [1]*1000: Mean +- std dev: [mult_list_py36] 2.13 us +- 0.06 us -> [mult_list_py37] 2.09 us +- 0.04 us: 1.02x faster - [1,2]*1000: Mean +- std dev: [mult_list_py36] 3.70 us +- 0.05 us -> [mult_list_py37] 5.28 us +- 0.09 us: 1.42x slower - [1,2,3]*1000: Mean +- std dev: [mult_list_py36] 4.61 us +- 0.13 us -> [mult_list_py37] 6.05 us +- 0.11 us: 1.31x slower + [1]*1000: Mean +- std dev: [mult_list_py36] 2.13 us +- 0.06 us -> [mult_list_py37] 2.09 us +- 0.04 us: 1.018x faster + [1,2]*1000: Mean +- std dev: [mult_list_py36] 3.70 us +- 0.05 us -> [mult_list_py37] 5.28 us +- 0.09 us: 1.425x slower + [1,2,3]*1000: Mean +- std dev: [mult_list_py36] 4.61 us +- 0.13 us -> [mult_list_py37] 6.05 us +- 0.11 us: 1.312x slower - Geometric mean: 1.22x slower + Geometric mean: 1.225x slower """ self.check_command(expected, 'compare_to', py36, py37) # 2 files grouped by speed expected = """ Slower (2): - - [1,2]*1000: 3.70 us +- 0.05 us -> 5.28 us +- 0.09 us: 1.42x slower - - [1,2,3]*1000: 4.61 us +- 0.13 us -> 6.05 us +- 0.11 us: 1.31x slower + - [1,2]*1000: 3.70 us +- 0.05 us -> 5.28 us +- 0.09 us: 1.425x slower + - [1,2,3]*1000: 4.61 us +- 0.13 us -> 6.05 us +- 0.11 us: 1.312x slower Faster (1): - - [1]*1000: 2.13 us +- 0.06 us -> 2.09 us +- 0.04 us: 1.02x faster + - [1]*1000: 2.13 us +- 0.06 us -> 2.09 us +- 0.04 us: 1.018x faster - Geometric mean: 1.22x slower + Geometric mean: 1.225x slower """ self.check_command(expected, 'compare_to', "--group-by-speed", py36, py37) # 2 files grouped by speed (with not significant) expected = """ Faster (2): - - [1,2]*1000: 3.70 us +- 0.05 us -> 3.18 us +- 0.08 us: 1.16x faster - - [1,2,3]*1000: 4.61 us +- 0.13 us -> 4.17 us +- 0.11 us: 1.11x faster + - [1,2]*1000: 3.70 us +- 0.05 us -> 3.18 us +- 0.08 us: 1.164x faster + - [1,2,3]*1000: 4.61 us +- 0.13 us -> 4.17 us +- 0.11 us: 1.106x faster Benchmark hidden because not significant (1): [1]*1000 - Geometric mean: 1.09x faster + Geometric mean: 1.087x faster """ self.check_command(expected, 'compare_to', "--group-by-speed", py36, py38) @@ -274,59 +274,59 @@ def test_compare_to_cli(self): [1]*1000 ======== - Mean +- std dev: [mult_list_py36] 2.13 us +- 0.06 us -> [mult_list_py37] 2.09 us +- 0.04 us: 1.02x faster - Mean +- std dev: [mult_list_py36] 2.13 us +- 0.06 us -> [mult_list_py38] 2.13 us +- 0.03 us: 1.00x slower + Mean +- std dev: [mult_list_py36] 2.13 us +- 0.06 us -> [mult_list_py37] 2.09 us +- 0.04 us: 1.018x faster + Mean +- std dev: [mult_list_py36] 2.13 us +- 0.06 us -> [mult_list_py38] 2.13 us +- 0.03 us: 1.001x slower Not significant! [1,2]*1000 ========== - Mean +- std dev: [mult_list_py36] 3.70 us +- 0.05 us -> [mult_list_py37] 5.28 us +- 0.09 us: 1.42x slower - Mean +- std dev: [mult_list_py36] 3.70 us +- 0.05 us -> [mult_list_py38] 3.18 us +- 0.08 us: 1.16x faster + Mean +- std dev: [mult_list_py36] 3.70 us +- 0.05 us -> [mult_list_py37] 5.28 us +- 0.09 us: 1.425x slower + Mean +- std dev: [mult_list_py36] 3.70 us +- 0.05 us -> [mult_list_py38] 3.18 us +- 0.08 us: 1.164x faster [1,2,3]*1000 ============ - Mean +- std dev: [mult_list_py36] 4.61 us +- 0.13 us -> [mult_list_py37] 6.05 us +- 0.11 us: 1.31x slower - Mean +- std dev: [mult_list_py36] 4.61 us +- 0.13 us -> [mult_list_py38] 4.17 us +- 0.11 us: 1.11x faster + Mean +- std dev: [mult_list_py36] 4.61 us +- 0.13 us -> [mult_list_py37] 6.05 us +- 0.11 us: 1.312x slower + Mean +- std dev: [mult_list_py36] 4.61 us +- 0.13 us -> [mult_list_py38] 4.17 us +- 0.11 us: 1.106x faster Geometric mean ============== - mult_list_py37: 1.22x slower - mult_list_py38: 1.09x faster + mult_list_py37: 1.225x slower + mult_list_py38: 1.087x faster """ self.check_command(expected, 'compare_to', py36, py37, py38) # 3 files as table expected = """ - +----------------+----------------+-----------------------+-----------------------+ - | Benchmark | mult_list_py36 | mult_list_py37 | mult_list_py38 | - +================+================+=======================+=======================+ - | [1]*1000 | 2.13 us | 2.09 us: 1.02x faster | not significant | - +----------------+----------------+-----------------------+-----------------------+ - | [1,2]*1000 | 3.70 us | 5.28 us: 1.42x slower | 3.18 us: 1.16x faster | - +----------------+----------------+-----------------------+-----------------------+ - | [1,2,3]*1000 | 4.61 us | 6.05 us: 1.31x slower | 4.17 us: 1.11x faster | - +----------------+----------------+-----------------------+-----------------------+ - | Geometric mean | (ref) | 1.22x slower | 1.09x faster | - +----------------+----------------+-----------------------+-----------------------+ + +----------------+----------------+------------------------+------------------------+ + | Benchmark | mult_list_py36 | mult_list_py37 | mult_list_py38 | + +================+================+========================+========================+ + | [1]*1000 | 2.13 us | 2.09 us: 1.018x faster | not significant | + +----------------+----------------+------------------------+------------------------+ + | [1,2]*1000 | 3.70 us | 5.28 us: 1.425x slower | 3.18 us: 1.164x faster | + +----------------+----------------+------------------------+------------------------+ + | [1,2,3]*1000 | 4.61 us | 6.05 us: 1.312x slower | 4.17 us: 1.106x faster | + +----------------+----------------+------------------------+------------------------+ + | Geometric mean | (ref) | 1.225x slower | 1.087x faster | + +----------------+----------------+------------------------+------------------------+ """ self.check_command(expected, 'compare_to', '--table', py36, py37, py38) # 3 files as table grouped by speed expected = """ - +----------------+----------------+-----------------------+ - | Benchmark | mult_list_py36 | mult_list_py37 | - +================+================+=======================+ - | [1]*1000 | 2.13 us | 2.09 us: 1.02x faster | - +----------------+----------------+-----------------------+ - | [1,2,3]*1000 | 4.61 us | 6.05 us: 1.31x slower | - +----------------+----------------+-----------------------+ - | [1,2]*1000 | 3.70 us | 5.28 us: 1.42x slower | - +----------------+----------------+-----------------------+ - | Geometric mean | (ref) | 1.22x slower | - +----------------+----------------+-----------------------+ + +----------------+----------------+------------------------+ + | Benchmark | mult_list_py36 | mult_list_py37 | + +================+================+========================+ + | [1]*1000 | 2.13 us | 2.09 us: 1.018x faster | + +----------------+----------------+------------------------+ + | [1,2,3]*1000 | 4.61 us | 6.05 us: 1.312x slower | + +----------------+----------------+------------------------+ + | [1,2]*1000 | 3.70 us | 5.28 us: 1.425x slower | + +----------------+----------------+------------------------+ + | Geometric mean | (ref) | 1.225x slower | + +----------------+----------------+------------------------+ """ self.check_command(expected, 'compare_to', '--table', "--group-by-speed", py36, py37) @@ -337,36 +337,36 @@ def test_compare_to_cli_min_speed(self): # 2 files, min-speed=10 expected = """ - [1,2]*1000: Mean +- std dev: [mult_list_py36] 3.70 us +- 0.05 us -> [mult_list_py37] 5.28 us +- 0.09 us: 1.42x slower - [1,2,3]*1000: Mean +- std dev: [mult_list_py36] 4.61 us +- 0.13 us -> [mult_list_py37] 6.05 us +- 0.11 us: 1.31x slower + [1,2]*1000: Mean +- std dev: [mult_list_py36] 3.70 us +- 0.05 us -> [mult_list_py37] 5.28 us +- 0.09 us: 1.425x slower + [1,2,3]*1000: Mean +- std dev: [mult_list_py36] 4.61 us +- 0.13 us -> [mult_list_py37] 6.05 us +- 0.11 us: 1.312x slower Benchmark hidden because not significant (1): [1]*1000 - Geometric mean: 1.22x slower + Geometric mean: 1.225x slower """ self.check_command(expected, 'compare_to', "--min-speed=10", py36, py37) # 2 files, min-speed=40 expected = """ - [1,2]*1000: Mean +- std dev: [mult_list_py36] 3.70 us +- 0.05 us -> [mult_list_py37] 5.28 us +- 0.09 us: 1.42x slower + [1,2]*1000: Mean +- std dev: [mult_list_py36] 3.70 us +- 0.05 us -> [mult_list_py37] 5.28 us +- 0.09 us: 1.425x slower Benchmark hidden because not significant (2): [1]*1000, [1,2,3]*1000 - Geometric mean: 1.22x slower + Geometric mean: 1.225x slower """ self.check_command(expected, 'compare_to', "--min-speed=40", py36, py37) # 3 files as table, min-speed=10 expected = """ - +----------------+----------------+-----------------------+-----------------------+ - | Benchmark | mult_list_py36 | mult_list_py37 | mult_list_py38 | - +================+================+=======================+=======================+ - | [1,2]*1000 | 3.70 us | 5.28 us: 1.42x slower | 3.18 us: 1.16x faster | - +----------------+----------------+-----------------------+-----------------------+ - | [1,2,3]*1000 | 4.61 us | 6.05 us: 1.31x slower | 4.17 us: 1.11x faster | - +----------------+----------------+-----------------------+-----------------------+ - | Geometric mean | (ref) | 1.22x slower | 1.09x faster | - +----------------+----------------+-----------------------+-----------------------+ + +----------------+----------------+------------------------+------------------------+ + | Benchmark | mult_list_py36 | mult_list_py37 | mult_list_py38 | + +================+================+========================+========================+ + | [1,2]*1000 | 3.70 us | 5.28 us: 1.425x slower | 3.18 us: 1.164x faster | + +----------------+----------------+------------------------+------------------------+ + | [1,2,3]*1000 | 4.61 us | 6.05 us: 1.312x slower | 4.17 us: 1.106x faster | + +----------------+----------------+------------------------+------------------------+ + | Geometric mean | (ref) | 1.225x slower | 1.087x faster | + +----------------+----------------+------------------------+------------------------+ Benchmark hidden because not significant (1): [1]*1000 """ diff --git a/pyperf/tests/test_timeit.py b/pyperf/tests/test_timeit.py index 18afadcb..8aded7c5 100644 --- a/pyperf/tests/test_timeit.py +++ b/pyperf/tests/test_timeit.py @@ -278,7 +278,7 @@ def test_compare_to(self): expected = textwrap.dedent(r''' ref: \. [0-9.]+ (?:ms|us) \+- [0-9.]+ (?:ms|us).* changed: \. [0-9.]+ (?:ms|us) \+- [0-9.]+ (?:ms|us).* - Mean \+- std dev: \[ref\] .* -> \[changed\] .*: (?:[0-9]+\.[0-9][0-9]x (?:faster|slower)|no change) + Mean \+- std dev: \[ref\] .* -> \[changed\] .*: (?:[0-9]+\.[0-9][0-9][0-9]x (?:faster|slower)|no change) ''').strip() expected = re.compile(expected, flags=re.DOTALL) self.assertRegex(cmd.stdout, expected) @@ -304,7 +304,7 @@ def test_compare_to_verbose(self): Compare ======= - Mean \+- std dev: .* -> .*: (?:[0-9]+\.[0-9][0-9]x (?:faster|slower)|no change) + Mean \+- std dev: .* -> .*: (?:[0-9]+\.[0-9][0-9][0-9]x (?:faster|slower)|no change) ''').strip() expected = re.compile(expected, flags=re.DOTALL) self.assertRegex(cmd.stdout, expected) @@ -314,7 +314,7 @@ def test_compare_to_quiet(self): args += COMPARE_BENCH cmd = tests.get_output(args) - expected = r'(?:Mean \+- std dev: .* -> .*: (?:[0-9]+\.[0-9][0-9]x (?:faster|slower)|no change)|Not significant!)' + expected = r'(?:Mean \+- std dev: .* -> .*: (?:[0-9]+\.[0-9][0-9][0-9]x (?:faster|slower)|no change)|Not significant!)' self.assertRegex(cmd.stdout, expected) def test_duplicate(self):