diff --git a/AUTHORS.rst b/AUTHORS.rst index ae3d941db4..33339da13c 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -19,6 +19,7 @@ their individual contributions. * `Alex Stapleton `_ * `Alex Willmer `_ (alex@moreati.org.uk) * `Andrea Pierré `_ +* `Andrea Reina `_ * `Ben Anhalt `_ * `Ben Peterson `_ (killthrush@hotmail.com) * `Benjamin Lee `_ (benjamindlee@me.com) diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..78dc7b0da1 --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,5 @@ +RELEASE_TYPE: patch + +This patch clarifies the reporting of time spent generating data. A +simple arithmetic mean of the percentage of time spent can be +misleading; reporting the actual time spent avoids misunderstandings. diff --git a/hypothesis-python/src/hypothesis/statistics.py b/hypothesis-python/src/hypothesis/statistics.py index 73377077cd..bdf56466e4 100644 --- a/hypothesis-python/src/hypothesis/statistics.py +++ b/hypothesis-python/src/hypothesis/statistics.py @@ -9,7 +9,6 @@ # obtain one at https://mozilla.org/MPL/2.0/. import math -import statistics from collections import Counter from hypothesis.utils.dynamicvariables import DynamicVariable @@ -41,6 +40,24 @@ def describe_targets(best_targets): return lines +def format_ms(times): + """Format `times` into a string representing approximate milliseconds. + + `times` is a collection of durations in seconds. + """ + ordered = sorted(times) + n = max(0, len(ordered) - 1) + lower = int(ordered[int(math.floor(n * 0.05))] * 1000) + upper = int(ordered[int(math.ceil(n * 0.95))] * 1000) + if upper == 0: + ms = "< 1ms" + elif lower == upper: + ms = f"~ {lower}ms" + else: + ms = f"~ {lower}-{upper} ms" + return ms + + def describe_statistics(stats_dict): """Return a multi-line string describing the passed run statistics. @@ -63,22 +80,11 @@ def describe_statistics(stats_dict): if not cases: continue statuses = Counter(t["status"] for t in cases) - runtimes = sorted(t["runtime"] for t in cases) - n = max(0, len(runtimes) - 1) - lower = int(runtimes[int(math.floor(n * 0.05))] * 1000) - upper = int(runtimes[int(math.ceil(n * 0.95))] * 1000) - if upper == 0: - ms = "< 1ms" - elif lower == upper: - ms = f"~ {lower}ms" - else: - ms = f"{lower}-{upper} ms" - drawtime_percent = 100 * statistics.mean( - t["drawtime"] / t["runtime"] if t["runtime"] > 0 else 0 for t in cases - ) + runtime_ms = format_ms(t["runtime"] for t in cases) + drawtime_ms = format_ms(t["drawtime"] for t in cases) lines.append( f" - during {phase} phase ({d['duration-seconds']:.2f} seconds):\n" - f" - Typical runtimes: {ms}, ~ {drawtime_percent:.0f}% in data generation\n" + f" - Typical runtimes: {runtime_ms}, of which {drawtime_ms} in data generation\n" f" - {statuses['valid']} passing examples, {statuses['interesting']} " f"failing examples, {statuses['invalid'] + statuses['overrun']} invalid examples" ) diff --git a/hypothesis-python/tests/cover/test_statistical_events.py b/hypothesis-python/tests/cover/test_statistical_events.py index 56b3733769..8e889aeaeb 100644 --- a/hypothesis-python/tests/cover/test_statistical_events.py +++ b/hypothesis-python/tests/cover/test_statistical_events.py @@ -8,6 +8,7 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. +import re import time import traceback @@ -145,7 +146,7 @@ def test(i): @pytest.mark.parametrize("draw_delay", [False, True]) @pytest.mark.parametrize("test_delay", [False, True]) -def test_draw_time_percentage(draw_delay, test_delay): +def test_draw_timing(draw_delay, test_delay): time.freeze() @st.composite @@ -161,11 +162,10 @@ def test(_): stats = describe_statistics(call_for_statistics(test)) if not draw_delay: - assert "~ 0%" in stats - elif test_delay: - assert "~ 50%" in stats + assert "< 1ms" in stats else: - assert "~ 100%" in stats + match = re.search(r"of which ~ (?P\d+)", stats) + assert 49 <= int(match.group("gentime")) <= 51 def test_has_lambdas_in_output():