Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Output geometric mean to 0.1% precision #126

Closed
wants to merge 2 commits into from
Closed

Output geometric mean to 0.1% precision #126

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dec6e01
Show thousandths place when reporting results
brandtbucher Mar 15, 2022
5f58b4f
Update tests
brandtbucher Mar 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions pyperf/_compare.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,9 @@ def format_normalized_mean(norm_mean):
if norm_mean == 1.0:
return "no change"
elif norm_mean < 1.0:
return "%.2fx faster" % (1.0 / norm_mean)
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe leave this as it is, but take a precision argument (which can default to 2).

Then in format_geometric_mean() call format_normalized_mean(geo_mean, 3)

return "%.3fx faster" % (1.0 / norm_mean)
else:
return "%.2fx slower" % norm_mean
return "%.3fx slower" % norm_mean


def format_geometric_mean(norm_means):
Expand Down
130 changes: 65 additions & 65 deletions pyperf/tests/test_perf_cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ def test_compare_to(self):
stdout = self.compare('compare_to', ref_result, changed_result, '-v')

expected = ('Mean +- std dev: [ref] 1.50 sec +- 0.50 sec '
'-> [changed] 2.00 sec +- 0.50 sec: 1.33x slower\n'
'-> [changed] 2.00 sec +- 0.50 sec: 1.333x slower\n'
'Not significant!')
self.assertEqual(stdout.rstrip(),
expected)
Expand All @@ -144,11 +144,11 @@ def test_compare_to_rest_table(self):
stdout = self.compare('compare_to', ref_result, changed_result, '--table')

expected = textwrap.dedent('''
+-----------+----------+------------------------+
| Benchmark | ref | changed |
+===========+==========+========================+
| telco | 1.00 sec | 2.00 sec: 2.00x slower |
+-----------+----------+------------------------+
+-----------+----------+-------------------------+
| Benchmark | ref | changed |
+===========+==========+=========================+
| telco | 1.00 sec | 2.00 sec: 2.000x slower |
+-----------+----------+-------------------------+
''').strip()

self.assertEqual(stdout.rstrip(),
Expand All @@ -166,9 +166,9 @@ def test_compare_to_md_table(self):
'--table-format', 'md')

expected = textwrap.dedent('''
| Benchmark | ref | changed |
|-----------|:--------:|:----------------------:|
| telco | 1.00 sec | 2.00 sec: 2.00x slower |
| Benchmark | ref | changed |
|-----------|:--------:|:-----------------------:|
| telco | 1.00 sec | 2.00 sec: 2.000x slower |
''').strip()

self.assertEqual(stdout.rstrip(),
Expand Down Expand Up @@ -207,7 +207,7 @@ def test_compare_to_not_significant_verbose(self):
stdout = self.compare('compare_to', ref_result, changed_result, '-v')

expected = ('Mean +- std dev: [ref] 1.50 sec +- 0.50 sec '
'-> [changed] 2.00 sec +- 0.50 sec: 1.33x slower\n'
'-> [changed] 2.00 sec +- 0.50 sec: 1.333x slower\n'
'Not significant!')
self.assertEqual(stdout.rstrip(),
expected)
Expand Down Expand Up @@ -236,36 +236,36 @@ def test_compare_to_cli(self):

# 2 files
expected = """
[1]*1000: Mean +- std dev: [mult_list_py36] 2.13 us +- 0.06 us -> [mult_list_py37] 2.09 us +- 0.04 us: 1.02x faster
[1,2]*1000: Mean +- std dev: [mult_list_py36] 3.70 us +- 0.05 us -> [mult_list_py37] 5.28 us +- 0.09 us: 1.42x slower
[1,2,3]*1000: Mean +- std dev: [mult_list_py36] 4.61 us +- 0.13 us -> [mult_list_py37] 6.05 us +- 0.11 us: 1.31x slower
[1]*1000: Mean +- std dev: [mult_list_py36] 2.13 us +- 0.06 us -> [mult_list_py37] 2.09 us +- 0.04 us: 1.018x faster
[1,2]*1000: Mean +- std dev: [mult_list_py36] 3.70 us +- 0.05 us -> [mult_list_py37] 5.28 us +- 0.09 us: 1.425x slower
[1,2,3]*1000: Mean +- std dev: [mult_list_py36] 4.61 us +- 0.13 us -> [mult_list_py37] 6.05 us +- 0.11 us: 1.312x slower

Geometric mean: 1.22x slower
Geometric mean: 1.225x slower
"""
self.check_command(expected, 'compare_to', py36, py37)

# 2 files grouped by speed
expected = """
Slower (2):
- [1,2]*1000: 3.70 us +- 0.05 us -> 5.28 us +- 0.09 us: 1.42x slower
- [1,2,3]*1000: 4.61 us +- 0.13 us -> 6.05 us +- 0.11 us: 1.31x slower
- [1,2]*1000: 3.70 us +- 0.05 us -> 5.28 us +- 0.09 us: 1.425x slower
- [1,2,3]*1000: 4.61 us +- 0.13 us -> 6.05 us +- 0.11 us: 1.312x slower

Faster (1):
- [1]*1000: 2.13 us +- 0.06 us -> 2.09 us +- 0.04 us: 1.02x faster
- [1]*1000: 2.13 us +- 0.06 us -> 2.09 us +- 0.04 us: 1.018x faster

Geometric mean: 1.22x slower
Geometric mean: 1.225x slower
"""
self.check_command(expected, 'compare_to', "--group-by-speed", py36, py37)

# 2 files grouped by speed (with not significant)
expected = """
Faster (2):
- [1,2]*1000: 3.70 us +- 0.05 us -> 3.18 us +- 0.08 us: 1.16x faster
- [1,2,3]*1000: 4.61 us +- 0.13 us -> 4.17 us +- 0.11 us: 1.11x faster
- [1,2]*1000: 3.70 us +- 0.05 us -> 3.18 us +- 0.08 us: 1.164x faster
- [1,2,3]*1000: 4.61 us +- 0.13 us -> 4.17 us +- 0.11 us: 1.106x faster

Benchmark hidden because not significant (1): [1]*1000

Geometric mean: 1.09x faster
Geometric mean: 1.087x faster
"""
self.check_command(expected, 'compare_to', "--group-by-speed", py36, py38)

Expand All @@ -274,59 +274,59 @@ def test_compare_to_cli(self):
[1]*1000
========

Mean +- std dev: [mult_list_py36] 2.13 us +- 0.06 us -> [mult_list_py37] 2.09 us +- 0.04 us: 1.02x faster
Mean +- std dev: [mult_list_py36] 2.13 us +- 0.06 us -> [mult_list_py38] 2.13 us +- 0.03 us: 1.00x slower
Mean +- std dev: [mult_list_py36] 2.13 us +- 0.06 us -> [mult_list_py37] 2.09 us +- 0.04 us: 1.018x faster
Mean +- std dev: [mult_list_py36] 2.13 us +- 0.06 us -> [mult_list_py38] 2.13 us +- 0.03 us: 1.001x slower
Not significant!

[1,2]*1000
==========

Mean +- std dev: [mult_list_py36] 3.70 us +- 0.05 us -> [mult_list_py37] 5.28 us +- 0.09 us: 1.42x slower
Mean +- std dev: [mult_list_py36] 3.70 us +- 0.05 us -> [mult_list_py38] 3.18 us +- 0.08 us: 1.16x faster
Mean +- std dev: [mult_list_py36] 3.70 us +- 0.05 us -> [mult_list_py37] 5.28 us +- 0.09 us: 1.425x slower
Mean +- std dev: [mult_list_py36] 3.70 us +- 0.05 us -> [mult_list_py38] 3.18 us +- 0.08 us: 1.164x faster

[1,2,3]*1000
============

Mean +- std dev: [mult_list_py36] 4.61 us +- 0.13 us -> [mult_list_py37] 6.05 us +- 0.11 us: 1.31x slower
Mean +- std dev: [mult_list_py36] 4.61 us +- 0.13 us -> [mult_list_py38] 4.17 us +- 0.11 us: 1.11x faster
Mean +- std dev: [mult_list_py36] 4.61 us +- 0.13 us -> [mult_list_py37] 6.05 us +- 0.11 us: 1.312x slower
Mean +- std dev: [mult_list_py36] 4.61 us +- 0.13 us -> [mult_list_py38] 4.17 us +- 0.11 us: 1.106x faster

Geometric mean
==============

mult_list_py37: 1.22x slower
mult_list_py38: 1.09x faster
mult_list_py37: 1.225x slower
mult_list_py38: 1.087x faster
"""
self.check_command(expected, 'compare_to', py36, py37, py38)

# 3 files as table
expected = """
+----------------+----------------+-----------------------+-----------------------+
| Benchmark | mult_list_py36 | mult_list_py37 | mult_list_py38 |
+================+================+=======================+=======================+
| [1]*1000 | 2.13 us | 2.09 us: 1.02x faster | not significant |
+----------------+----------------+-----------------------+-----------------------+
| [1,2]*1000 | 3.70 us | 5.28 us: 1.42x slower | 3.18 us: 1.16x faster |
+----------------+----------------+-----------------------+-----------------------+
| [1,2,3]*1000 | 4.61 us | 6.05 us: 1.31x slower | 4.17 us: 1.11x faster |
+----------------+----------------+-----------------------+-----------------------+
| Geometric mean | (ref) | 1.22x slower | 1.09x faster |
+----------------+----------------+-----------------------+-----------------------+
+----------------+----------------+------------------------+------------------------+
| Benchmark | mult_list_py36 | mult_list_py37 | mult_list_py38 |
+================+================+========================+========================+
| [1]*1000 | 2.13 us | 2.09 us: 1.018x faster | not significant |
+----------------+----------------+------------------------+------------------------+
| [1,2]*1000 | 3.70 us | 5.28 us: 1.425x slower | 3.18 us: 1.164x faster |
+----------------+----------------+------------------------+------------------------+
| [1,2,3]*1000 | 4.61 us | 6.05 us: 1.312x slower | 4.17 us: 1.106x faster |
+----------------+----------------+------------------------+------------------------+
| Geometric mean | (ref) | 1.225x slower | 1.087x faster |
+----------------+----------------+------------------------+------------------------+
"""
self.check_command(expected, 'compare_to', '--table', py36, py37, py38)

# 3 files as table grouped by speed
expected = """
+----------------+----------------+-----------------------+
| Benchmark | mult_list_py36 | mult_list_py37 |
+================+================+=======================+
| [1]*1000 | 2.13 us | 2.09 us: 1.02x faster |
+----------------+----------------+-----------------------+
| [1,2,3]*1000 | 4.61 us | 6.05 us: 1.31x slower |
+----------------+----------------+-----------------------+
| [1,2]*1000 | 3.70 us | 5.28 us: 1.42x slower |
+----------------+----------------+-----------------------+
| Geometric mean | (ref) | 1.22x slower |
+----------------+----------------+-----------------------+
+----------------+----------------+------------------------+
| Benchmark | mult_list_py36 | mult_list_py37 |
+================+================+========================+
| [1]*1000 | 2.13 us | 2.09 us: 1.018x faster |
+----------------+----------------+------------------------+
| [1,2,3]*1000 | 4.61 us | 6.05 us: 1.312x slower |
+----------------+----------------+------------------------+
| [1,2]*1000 | 3.70 us | 5.28 us: 1.425x slower |
+----------------+----------------+------------------------+
| Geometric mean | (ref) | 1.225x slower |
+----------------+----------------+------------------------+
"""
self.check_command(expected, 'compare_to', '--table', "--group-by-speed", py36, py37)

Expand All @@ -337,36 +337,36 @@ def test_compare_to_cli_min_speed(self):

# 2 files, min-speed=10
expected = """
[1,2]*1000: Mean +- std dev: [mult_list_py36] 3.70 us +- 0.05 us -> [mult_list_py37] 5.28 us +- 0.09 us: 1.42x slower
[1,2,3]*1000: Mean +- std dev: [mult_list_py36] 4.61 us +- 0.13 us -> [mult_list_py37] 6.05 us +- 0.11 us: 1.31x slower
[1,2]*1000: Mean +- std dev: [mult_list_py36] 3.70 us +- 0.05 us -> [mult_list_py37] 5.28 us +- 0.09 us: 1.425x slower
[1,2,3]*1000: Mean +- std dev: [mult_list_py36] 4.61 us +- 0.13 us -> [mult_list_py37] 6.05 us +- 0.11 us: 1.312x slower

Benchmark hidden because not significant (1): [1]*1000

Geometric mean: 1.22x slower
Geometric mean: 1.225x slower
"""
self.check_command(expected, 'compare_to', "--min-speed=10", py36, py37)

# 2 files, min-speed=40
expected = """
[1,2]*1000: Mean +- std dev: [mult_list_py36] 3.70 us +- 0.05 us -> [mult_list_py37] 5.28 us +- 0.09 us: 1.42x slower
[1,2]*1000: Mean +- std dev: [mult_list_py36] 3.70 us +- 0.05 us -> [mult_list_py37] 5.28 us +- 0.09 us: 1.425x slower

Benchmark hidden because not significant (2): [1]*1000, [1,2,3]*1000

Geometric mean: 1.22x slower
Geometric mean: 1.225x slower
"""
self.check_command(expected, 'compare_to', "--min-speed=40", py36, py37)

# 3 files as table, min-speed=10
expected = """
+----------------+----------------+-----------------------+-----------------------+
| Benchmark | mult_list_py36 | mult_list_py37 | mult_list_py38 |
+================+================+=======================+=======================+
| [1,2]*1000 | 3.70 us | 5.28 us: 1.42x slower | 3.18 us: 1.16x faster |
+----------------+----------------+-----------------------+-----------------------+
| [1,2,3]*1000 | 4.61 us | 6.05 us: 1.31x slower | 4.17 us: 1.11x faster |
+----------------+----------------+-----------------------+-----------------------+
| Geometric mean | (ref) | 1.22x slower | 1.09x faster |
+----------------+----------------+-----------------------+-----------------------+
+----------------+----------------+------------------------+------------------------+
| Benchmark | mult_list_py36 | mult_list_py37 | mult_list_py38 |
+================+================+========================+========================+
| [1,2]*1000 | 3.70 us | 5.28 us: 1.425x slower | 3.18 us: 1.164x faster |
+----------------+----------------+------------------------+------------------------+
| [1,2,3]*1000 | 4.61 us | 6.05 us: 1.312x slower | 4.17 us: 1.106x faster |
+----------------+----------------+------------------------+------------------------+
| Geometric mean | (ref) | 1.225x slower | 1.087x faster |
+----------------+----------------+------------------------+------------------------+

Benchmark hidden because not significant (1): [1]*1000
"""
Expand Down
6 changes: 3 additions & 3 deletions pyperf/tests/test_timeit.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,7 @@ def test_compare_to(self):
expected = textwrap.dedent(r'''
ref: \. [0-9.]+ (?:ms|us) \+- [0-9.]+ (?:ms|us).*
changed: \. [0-9.]+ (?:ms|us) \+- [0-9.]+ (?:ms|us).*
Mean \+- std dev: \[ref\] .* -> \[changed\] .*: (?:[0-9]+\.[0-9][0-9]x (?:faster|slower)|no change)
Mean \+- std dev: \[ref\] .* -> \[changed\] .*: (?:[0-9]+\.[0-9][0-9][0-9]x (?:faster|slower)|no change)
''').strip()
expected = re.compile(expected, flags=re.DOTALL)
self.assertRegex(cmd.stdout, expected)
Expand All @@ -304,7 +304,7 @@ def test_compare_to_verbose(self):
Compare
=======

Mean \+- std dev: .* -> .*: (?:[0-9]+\.[0-9][0-9]x (?:faster|slower)|no change)
Mean \+- std dev: .* -> .*: (?:[0-9]+\.[0-9][0-9][0-9]x (?:faster|slower)|no change)
''').strip()
expected = re.compile(expected, flags=re.DOTALL)
self.assertRegex(cmd.stdout, expected)
Expand All @@ -314,7 +314,7 @@ def test_compare_to_quiet(self):
args += COMPARE_BENCH
cmd = tests.get_output(args)

expected = r'(?:Mean \+- std dev: .* -> .*: (?:[0-9]+\.[0-9][0-9]x (?:faster|slower)|no change)|Not significant!)'
expected = r'(?:Mean \+- std dev: .* -> .*: (?:[0-9]+\.[0-9][0-9][0-9]x (?:faster|slower)|no change)|Not significant!)'
self.assertRegex(cmd.stdout, expected)

def test_duplicate(self):
Expand Down