From 9611b56e5072b6c3200a970a283800d463f6c45d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Fri, 3 Nov 2023 12:28:58 +0100 Subject: [PATCH 1/6] Make `-m test -T -j` use sys.monitoring --- Doc/library/trace.rst | 8 ++++- Lib/test/cov.py | 22 ++++++++++++++ Lib/test/libregrtest/cmdline.py | 14 +++++++-- Lib/test/libregrtest/main.py | 29 ++++++++++--------- Lib/test/libregrtest/result.py | 12 ++++++++ Lib/test/libregrtest/results.py | 15 ++++++++-- Lib/test/libregrtest/run_workers.py | 2 +- Lib/test/libregrtest/runtests.py | 1 + Lib/test/libregrtest/worker.py | 16 +++++++++- Lib/test/test_regrtest.py | 17 ++++++++--- Lib/trace.py | 10 +++++-- ...-11-03-18-59-13.gh-issue-110722.jvT1pb.rst | 2 ++ 12 files changed, 120 insertions(+), 28 deletions(-) create mode 100644 Lib/test/cov.py create mode 100644 Misc/NEWS.d/next/Tests/2023-11-03-18-59-13.gh-issue-110722.jvT1pb.rst diff --git a/Doc/library/trace.rst b/Doc/library/trace.rst index e9b59a6d186ba2..b547b99c28ccde 100644 --- a/Doc/library/trace.rst +++ b/Doc/library/trace.rst @@ -187,7 +187,8 @@ Programmatic Interface Merge in data from another :class:`CoverageResults` object. - .. method:: write_results(show_missing=True, summary=False, coverdir=None) + .. method:: write_results(show_missing=True, summary=False, coverdir=None,\ + ignore_missing_files=False) Write coverage results. Set *show_missing* to show lines that had no hits. Set *summary* to include in the output the coverage summary per @@ -195,6 +196,11 @@ Programmatic Interface result files will be output. If ``None``, the results for each source file are placed in its directory. + If *ignore_missing_files* is True, coverage counts for files that no + longer exist are silently ignored. Otherwise, a missing file will + raise a :exc:`FileNotFoundError`. + + A simple example demonstrating the use of the programmatic interface:: import sys diff --git a/Lib/test/cov.py b/Lib/test/cov.py new file mode 100644 index 00000000000000..29c6f84ce3a95e --- /dev/null +++ b/Lib/test/cov.py @@ -0,0 +1,22 @@ +"""A minimal hook for gathering line coverage of the standard library.""" + +import sys + +mon = sys.monitoring +mon.use_tool_id(mon.COVERAGE_ID, "regrtest coverage") + +FileName = str +LineNo = int +Location = tuple[FileName, LineNo] +COVERAGE: set[Location] = set() + +# `types` and `typing` not imported to avoid invalid coverage +def add_line( + code: "types.CodeType", + lineno: int, +) -> "typing.Literal[sys.monitoring.DISABLE]": + COVERAGE.add((code.co_filename, lineno)) + return mon.DISABLE + +mon.register_callback(mon.COVERAGE_ID, mon.events.LINE, add_line) +mon.set_events(mon.COVERAGE_ID, mon.events.LINE) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 1747511b57cc52..a5f02d6335f58f 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -2,7 +2,7 @@ import os.path import shlex import sys -from test.support import os_helper +from test.support import os_helper, Py_DEBUG from .utils import ALL_RESOURCES, RESOURCE_NAMES @@ -448,8 +448,16 @@ def _parse_args(args, **kwargs): if ns.single and ns.fromfile: parser.error("-s and -f don't go together!") - if ns.use_mp is not None and ns.trace: - parser.error("-T and -j don't go together!") + if ns.trace: + if ns.use_mp is not None: + if not Py_DEBUG: + parser.error("need --with-pydebug to use -T and -j together") + else: + print( + "Warning: collecting coverage without -j is imprecise. Configure" + " --with-pydebug and run -m test -T -j for best results.", + file=sys.stderr + ) if ns.python is not None: if ns.use_mp is None: parser.error("-p requires -j!") diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 9b86548c89fb2e..296e1b46317860 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -5,6 +5,7 @@ import sys import sysconfig import time +import trace from test import support from test.support import os_helper, MS_WINDOWS @@ -13,7 +14,7 @@ from .findtests import findtests, split_test_packages, list_cases from .logger import Logger from .pgo import setup_pgo_tests -from .result import State +from .result import State, TestResult from .results import TestResults, EXITCODE_INTERRUPTED from .runtests import RunTests, HuntRefleak from .setup import setup_process, setup_test_dir @@ -284,7 +285,9 @@ def display_result(self, runtests): self.results.display_result(runtests.tests, self.quiet, self.print_slowest) - def run_test(self, test_name: TestName, runtests: RunTests, tracer): + def run_test( + self, test_name: TestName, runtests: RunTests, tracer: trace.Trace | None + ) -> TestResult: if tracer is not None: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. @@ -299,9 +302,8 @@ def run_test(self, test_name: TestName, runtests: RunTests, tracer): return result - def run_tests_sequentially(self, runtests): + def run_tests_sequentially(self, runtests) -> trace.CoverageResults | None: if self.coverage: - import trace tracer = trace.Trace(trace=False, count=True) else: tracer = None @@ -349,7 +351,7 @@ def run_tests_sequentially(self, runtests): if previous_test: print(previous_test) - return tracer + return tracer.results() if tracer else None def get_state(self): state = self.results.get_state(self.fail_env_changed) @@ -361,7 +363,7 @@ def _run_tests_mp(self, runtests: RunTests, num_workers: int) -> None: from .run_workers import RunWorkers RunWorkers(num_workers, runtests, self.logger, self.results).run() - def finalize_tests(self, tracer): + def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: if self.next_single_filename: if self.next_single_test: with open(self.next_single_filename, 'w') as fp: @@ -369,10 +371,10 @@ def finalize_tests(self, tracer): else: os.unlink(self.next_single_filename) - if tracer is not None: - results = tracer.results() - results.write_results(show_missing=True, summary=True, - coverdir=self.coverage_dir) + if coverage is not None: + coverage.write_results(show_missing=True, summary=True, + coverdir=self.coverage_dir, + ignore_missing_files=True) if self.want_run_leaks: os.system("leaks %d" % os.getpid()) @@ -412,6 +414,7 @@ def create_run_tests(self, tests: TestTuple): hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, use_junit=(self.junit_filename is not None), + coverage=self.coverage, memory_limit=self.memory_limit, gc_threshold=self.gc_threshold, use_resources=self.use_resources, @@ -458,9 +461,9 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: try: if self.num_workers: self._run_tests_mp(runtests, self.num_workers) - tracer = None + coverage = self.results.get_coverage_results() else: - tracer = self.run_tests_sequentially(runtests) + coverage = self.run_tests_sequentially(runtests) self.display_result(runtests) @@ -471,7 +474,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: self.logger.stop_load_tracker() self.display_summary() - self.finalize_tests(tracer) + self.finalize_tests(coverage) return self.results.get_exitcode(self.fail_env_changed, self.fail_rerun) diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 8bfd3665ac93d5..74eae40440435d 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -78,6 +78,11 @@ def must_stop(state): } +FileName = str +LineNo = int +Location = tuple[FileName, LineNo] + + @dataclasses.dataclass(slots=True) class TestResult: test_name: TestName @@ -91,6 +96,9 @@ class TestResult: errors: list[tuple[str, str]] | None = None failures: list[tuple[str, str]] | None = None + # partial coverage in a worker run; not used by sequential in-process runs + covered_lines: list[Location] | None = None + def is_failed(self, fail_env_changed: bool) -> bool: if self.state == State.ENV_CHANGED: return fail_env_changed @@ -207,6 +215,10 @@ def _decode_test_result(data: dict[str, Any]) -> TestResult | dict[str, Any]: data.pop('__test_result__') if data['stats'] is not None: data['stats'] = TestStats(**data['stats']) + if data['covered_lines'] is not None: + data['covered_lines'] = [ + tuple(loc) for loc in data['covered_lines'] + ] return TestResult(**data) else: return data diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index 1feb43f8c074db..d215a243393d5a 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -1,13 +1,14 @@ import sys +import trace from .runtests import RunTests -from .result import State, TestResult, TestStats +from .result import State, TestResult, TestStats, Location from .utils import ( StrPath, TestName, TestTuple, TestList, FilterDict, printlist, count, format_duration) -# Python uses exit code 1 when an exception is not catched +# Python uses exit code 1 when an exception is not caught # argparse.ArgumentParser.error() uses exit code 2 EXITCODE_BAD_TEST = 2 EXITCODE_ENV_CHANGED = 3 @@ -34,6 +35,8 @@ def __init__(self): self.stats = TestStats() # used by --junit-xml self.testsuite_xml: list[str] = [] + # used by -T with -j + self.covered_lines: set[Location] = set() def is_all_good(self): return (not self.bad @@ -119,11 +122,17 @@ def accumulate_result(self, result: TestResult, runtests: RunTests): self.stats.accumulate(result.stats) if rerun: self.rerun.append(test_name) - + if result.covered_lines: + # we don't care about trace counts so we don't have to sum them up + self.covered_lines.update(result.covered_lines) xml_data = result.xml_data if xml_data: self.add_junit(xml_data) + def get_coverage_results(self) -> trace.CoverageResults: + counts = {loc: 1 for loc in frozenset(self. covered_lines)} + return trace.CoverageResults(counts=counts) + def need_rerun(self): return bool(self.rerun_results) diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index ab03cb54d6122e..99c2cf34d206d0 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -277,7 +277,7 @@ def run_tmp_files(self, worker_runtests: RunTests, # Python finalization: too late for libregrtest. if not support.is_wasi: # Don't check for leaked temporary files and directories if Python is - # run on WASI. WASI don't pass environment variables like TMPDIR to + # run on WASI. WASI doesn't pass environment variables like TMPDIR to # worker processes. tmp_dir = tempfile.mkdtemp(prefix="test_python_") tmp_dir = os.path.abspath(tmp_dir) diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index bfed1b4a2a5817..ac47c07f8d4341 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -85,6 +85,7 @@ class RunTests: hunt_refleak: HuntRefleak | None test_dir: StrPath | None use_junit: bool + coverage: bool memory_limit: str | None gc_threshold: int | None use_resources: tuple[str, ...] diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index 2eccfabc25223a..615e2211656dda 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -4,7 +4,7 @@ from typing import Any, NoReturn from test import support -from test.support import os_helper +from test.support import os_helper, Py_DEBUG from .setup import setup_process, setup_test_dir from .runtests import RunTests, JsonFile, JsonFileType @@ -30,6 +30,8 @@ def create_worker_process(runtests: RunTests, output_fd: int, python_opts = [opt for opt in python_opts if opt != "-E"] else: executable = (sys.executable,) + if runtests.coverage: + python_opts.append("-Xpresite=test.cov") cmd = [*executable, *python_opts, '-u', # Unbuffered stdout and stderr '-m', 'test.libregrtest.worker', @@ -87,6 +89,18 @@ def worker_process(worker_json: StrJSON) -> NoReturn: print(f"Re-running {test_name} in verbose mode", flush=True) result = run_single_test(test_name, runtests) + if runtests.coverage: + if "test.cov" in sys.modules: # imported by -Xpresite= + result.covered_lines = list(sys.modules["test.cov"].COVERAGE) + elif not Py_DEBUG: + print( + "Gathering coverage in worker processes requires --with-pydebug", + flush=True, + ) + else: + raise LookupError( + "`test.cov` not found in sys.modules but coverage wanted" + ) if json_file.file_type == JsonFileType.STDOUT: print() diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 6b03ea0dee1e23..a83d10d7769f64 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -306,14 +306,23 @@ def test_multiprocess(self): self.assertEqual(ns.use_mp, 2) self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid int value') - self.checkError([opt, '2', '-T'], "don't go together") - self.checkError([opt, '0', '-T'], "don't go together") - def test_coverage(self): + def test_coverage_sequential(self): for opt in '-T', '--coverage': with self.subTest(opt=opt): - ns = self.parse_args([opt]) + with support.captured_stderr() as stderr: + ns = self.parse_args([opt]) self.assertTrue(ns.trace) + self.assertIn( + "collecting coverage without -j is imprecise", + stderr.getvalue(), + ) + + def test_coverage_mp(self): + for opt in '-T', '--coverage': + with self.subTest(opt=opt): + ns = self.parse_args([opt, '-j1']) + self.assertTrue(ns.trace) def test_coverdir(self): for opt in '-D', '--coverdir': diff --git a/Lib/trace.py b/Lib/trace.py index fb9a423ea09fce..73e353e836da2c 100755 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -202,7 +202,8 @@ def update(self, other): for key in other_callers: callers[key] = 1 - def write_results(self, show_missing=True, summary=False, coverdir=None): + def write_results(self, show_missing=True, summary=False, coverdir=None, + ignore_missing_files=False): """ Write the coverage results. @@ -211,6 +212,9 @@ def write_results(self, show_missing=True, summary=False, coverdir=None): :param coverdir: If None, the results of each module are placed in its directory, otherwise it is included in the directory specified. + :param ignore_missing_files: If True, counts for files that no longer + exist are silently ignored. Otherwise, a missing file + will raise a FileNotFoundError. """ if self.calledfuncs: print() @@ -253,6 +257,9 @@ def write_results(self, show_missing=True, summary=False, coverdir=None): if filename.endswith(".pyc"): filename = filename[:-1] + if ignore_missing_files and not os.path.isfile(filename): + continue + if coverdir is None: dir = os.path.dirname(os.path.abspath(filename)) modulename = _modname(filename) @@ -278,7 +285,6 @@ def write_results(self, show_missing=True, summary=False, coverdir=None): percent = int(100 * n_hits / n_lines) sums[modulename] = n_lines, percent, modulename, filename - if summary and sums: print("lines cov% module (path)") for m in sorted(sums): diff --git a/Misc/NEWS.d/next/Tests/2023-11-03-18-59-13.gh-issue-110722.jvT1pb.rst b/Misc/NEWS.d/next/Tests/2023-11-03-18-59-13.gh-issue-110722.jvT1pb.rst new file mode 100644 index 00000000000000..ad1ac536a092ea --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-11-03-18-59-13.gh-issue-110722.jvT1pb.rst @@ -0,0 +1,2 @@ +Gathering line coverage of standard libraries within the regression test +suite is now precise, as well as much faster. Patch by Łukasz Langa. From bf65fbe5c4576bb2b161521e711f5109d3ab1b27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Fri, 3 Nov 2023 20:25:32 +0100 Subject: [PATCH 2/6] Only run -T -j test when Python is built --with-pydebug --- Lib/test/test_regrtest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index a83d10d7769f64..d7b9f801092498 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -318,11 +318,12 @@ def test_coverage_sequential(self): stderr.getvalue(), ) + @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') def test_coverage_mp(self): for opt in '-T', '--coverage': with self.subTest(opt=opt): ns = self.parse_args([opt, '-j1']) - self.assertTrue(ns.trace) + self.assertTrue(ns.trace) def test_coverdir(self): for opt in '-D', '--coverdir': From 1adbf57b16e589acdb43b8094bd550ba5598c79a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Mon, 6 Nov 2023 14:35:03 +0100 Subject: [PATCH 3/6] Simplify `get_coverage_results` --- Lib/test/libregrtest/results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index d215a243393d5a..71aaef3ae9ae61 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -130,7 +130,7 @@ def accumulate_result(self, result: TestResult, runtests: RunTests): self.add_junit(xml_data) def get_coverage_results(self) -> trace.CoverageResults: - counts = {loc: 1 for loc in frozenset(self. covered_lines)} + counts = {loc: 1 for loc in self.covered_lines} return trace.CoverageResults(counts=counts) def need_rerun(self): From d4f5b6bbd6870d012ed7eac2faa103dd5a2ec40d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Mon, 6 Nov 2023 20:16:29 +0100 Subject: [PATCH 4/6] Turn off sys.monitoring hooks for tests marked with `@no_tracing` --- Lib/test/support/__init__.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index de7db70275441a..a3301868cc28b0 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1082,18 +1082,30 @@ def check_impl_detail(**guards): def no_tracing(func): """Decorator to temporarily turn off tracing for the duration of a test.""" - if not hasattr(sys, 'gettrace'): - return func - else: + trace_wrapper = func + if hasattr(sys, 'gettrace'): + @functools.wraps(func) + def trace_wrapper(*args, **kwargs): + original_trace = sys.gettrace() + try: + sys.settrace(None) + return func(*args, **kwargs) + finally: + sys.settrace(original_trace) + + coverage_wrapper = trace_wrapper + if 'test.cov' in sys.modules: # -Xpresite=test.cov used + cov = sys.monitoring.COVERAGE_ID @functools.wraps(func) - def wrapper(*args, **kwargs): - original_trace = sys.gettrace() + def coverage_wrapper(*args, **kwargs): + original_events = sys.monitoring.get_events(cov) try: - sys.settrace(None) - return func(*args, **kwargs) + sys.monitoring.set_events(cov, 0) + return trace_wrapper(*args, **kwargs) finally: - sys.settrace(original_trace) - return wrapper + sys.monitoring.set_events(cov, original_events) + + return coverage_wrapper def refcount_test(test): From 3a0688bfab2b88bcb47b65cbb5218e5ae793a006 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Mon, 6 Nov 2023 22:35:15 +0100 Subject: [PATCH 5/6] Make patchcheck happy --- Lib/test/support/__init__.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 33ef5e3f53689e..9f11850afb2f29 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1084,14 +1084,14 @@ def no_tracing(func): """Decorator to temporarily turn off tracing for the duration of a test.""" trace_wrapper = func if hasattr(sys, 'gettrace'): - @functools.wraps(func) - def trace_wrapper(*args, **kwargs): - original_trace = sys.gettrace() - try: - sys.settrace(None) - return func(*args, **kwargs) - finally: - sys.settrace(original_trace) + @functools.wraps(func) + def trace_wrapper(*args, **kwargs): + original_trace = sys.gettrace() + try: + sys.settrace(None) + return func(*args, **kwargs) + finally: + sys.settrace(original_trace) coverage_wrapper = trace_wrapper if 'test.cov' in sys.modules: # -Xpresite=test.cov used From aa31295615f4c0b676ef7e2641a6ec243527a1d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Fri, 10 Nov 2023 13:42:38 +0100 Subject: [PATCH 6/6] Add improvements after review by Victor and Serhiy --- Doc/library/trace.rst | 6 +++-- Lib/test/cov.py | 40 ++++++++++++++++++++++++++++------ Lib/test/libregrtest/main.py | 9 ++++---- Lib/test/libregrtest/worker.py | 2 +- Lib/trace.py | 2 +- 5 files changed, 43 insertions(+), 16 deletions(-) diff --git a/Doc/library/trace.rst b/Doc/library/trace.rst index b547b99c28ccde..8854905e192b45 100644 --- a/Doc/library/trace.rst +++ b/Doc/library/trace.rst @@ -188,7 +188,7 @@ Programmatic Interface Merge in data from another :class:`CoverageResults` object. .. method:: write_results(show_missing=True, summary=False, coverdir=None,\ - ignore_missing_files=False) + *, ignore_missing_files=False) Write coverage results. Set *show_missing* to show lines that had no hits. Set *summary* to include in the output the coverage summary per @@ -196,10 +196,12 @@ Programmatic Interface result files will be output. If ``None``, the results for each source file are placed in its directory. - If *ignore_missing_files* is True, coverage counts for files that no + If *ignore_missing_files* is ``True``, coverage counts for files that no longer exist are silently ignored. Otherwise, a missing file will raise a :exc:`FileNotFoundError`. + .. versionchanged:: 3.13 + Added *ignore_missing_files* parameter. A simple example demonstrating the use of the programmatic interface:: diff --git a/Lib/test/cov.py b/Lib/test/cov.py index 29c6f84ce3a95e..e4699c7afe174a 100644 --- a/Lib/test/cov.py +++ b/Lib/test/cov.py @@ -1,22 +1,48 @@ -"""A minimal hook for gathering line coverage of the standard library.""" +"""A minimal hook for gathering line coverage of the standard library. + +Designed to be used with -Xpresite= which means: +* it installs itself on import +* it's not imported as `__main__` so can't use the ifmain idiom +* it can't import anything besides `sys` to avoid tainting gathered coverage +* filenames are not normalized + +To get gathered coverage back, look for 'test.cov' in `sys.modules` +instead of importing directly. That way you can determine if the module +was already in use. + +If you need to disable the hook, call the `disable()` function. +""" import sys mon = sys.monitoring -mon.use_tool_id(mon.COVERAGE_ID, "regrtest coverage") FileName = str LineNo = int Location = tuple[FileName, LineNo] -COVERAGE: set[Location] = set() -# `types` and `typing` not imported to avoid invalid coverage +coverage: set[Location] = set() + + +# `types` and `typing` aren't imported to avoid invalid coverage def add_line( code: "types.CodeType", lineno: int, ) -> "typing.Literal[sys.monitoring.DISABLE]": - COVERAGE.add((code.co_filename, lineno)) + coverage.add((code.co_filename, lineno)) return mon.DISABLE -mon.register_callback(mon.COVERAGE_ID, mon.events.LINE, add_line) -mon.set_events(mon.COVERAGE_ID, mon.events.LINE) + +def enable(): + mon.use_tool_id(mon.COVERAGE_ID, "regrtest coverage") + mon.register_callback(mon.COVERAGE_ID, mon.events.LINE, add_line) + mon.set_events(mon.COVERAGE_ID, mon.events.LINE) + + +def disable(): + mon.set_events(mon.COVERAGE_ID, 0) + mon.register_callback(mon.COVERAGE_ID, mon.events.LINE, None) + mon.free_tool_id(mon.COVERAGE_ID) + + +enable() diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 296e1b46317860..86428945a6def2 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -295,6 +295,7 @@ def run_test( namespace = dict(locals()) tracer.runctx(cmd, globals=globals(), locals=namespace) result = namespace['result'] + result.covered_lines = list(tracer.counts) else: result = run_single_test(test_name, runtests) @@ -302,7 +303,7 @@ def run_test( return result - def run_tests_sequentially(self, runtests) -> trace.CoverageResults | None: + def run_tests_sequentially(self, runtests) -> None: if self.coverage: tracer = trace.Trace(trace=False, count=True) else: @@ -351,8 +352,6 @@ def run_tests_sequentially(self, runtests) -> trace.CoverageResults | None: if previous_test: print(previous_test) - return tracer.results() if tracer else None - def get_state(self): state = self.results.get_state(self.fail_env_changed) if self.first_state: @@ -461,10 +460,10 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: try: if self.num_workers: self._run_tests_mp(runtests, self.num_workers) - coverage = self.results.get_coverage_results() else: - coverage = self.run_tests_sequentially(runtests) + self.run_tests_sequentially(runtests) + coverage = self.results.get_coverage_results() self.display_result(runtests) if self.want_rerun and self.results.need_rerun(): diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index 615e2211656dda..b3bb0b7f34a060 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -91,7 +91,7 @@ def worker_process(worker_json: StrJSON) -> NoReturn: result = run_single_test(test_name, runtests) if runtests.coverage: if "test.cov" in sys.modules: # imported by -Xpresite= - result.covered_lines = list(sys.modules["test.cov"].COVERAGE) + result.covered_lines = list(sys.modules["test.cov"].coverage) elif not Py_DEBUG: print( "Gathering coverage in worker processes requires --with-pydebug", diff --git a/Lib/trace.py b/Lib/trace.py index 73e353e836da2c..7cb6f897634b14 100755 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -202,7 +202,7 @@ def update(self, other): for key in other_callers: callers[key] = 1 - def write_results(self, show_missing=True, summary=False, coverdir=None, + def write_results(self, show_missing=True, summary=False, coverdir=None, *, ignore_missing_files=False): """ Write the coverage results.