diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 4b1d987a6ce..2a8aa4de1c3 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -35,6 +35,7 @@ Test fixtures for use by clients are available for each release on the [Github r - ✨ Add a "slow" pytest marker, in order to be able to limit the filled tests until release ([#562](https://github.com/ethereum/execution-spec-tests/pull/562)). - ✨ Add a CLI tool that generates blockchain tests as Python from a transaction hash ([#470](https://github.com/ethereum/execution-spec-tests/pull/470), [#576](https://github.com/ethereum/execution-spec-tests/pull/576)). - ✨ Add more Transaction and Block exceptions from existing ethereum/tests repo ([#572](https://github.com/ethereum/execution-spec-tests/pull/572)). +- ✨ Add "description" and "url" fields containing test case documentation and a source code permalink to fixtures during `fill` and use them in `consume`-generated Hive test reports ([#579](https://github.com/ethereum/execution-spec-tests/pull/579)). ### 🔧 EVM Tools diff --git a/docs/gen_test_case_reference.py b/docs/gen_test_case_reference.py index a624baa7936..387b89861c0 100644 --- a/docs/gen_test_case_reference.py +++ b/docs/gen_test_case_reference.py @@ -16,9 +16,12 @@ import mkdocs_gen_files import pytest -from git import Repo from ethereum_test_forks import get_development_forks, get_forks +from ethereum_test_tools.utility.versioning import ( + generate_github_url, + get_current_commit_hash_or_tag, +) logger = logging.getLogger("mkdocs") @@ -206,44 +209,6 @@ def run_collect_only(test_path: Path = source_directory) -> Tuple[str, str]: return f'fill {" ".join(collect_only_args)}', collect_only_output -def generate_github_url(file_path, branch_or_commit_or_tag="main"): - """ - Generate a link to a source file in Github. - """ - base_url = "https://github.com" - username = "ethereum" - repository = "execution-spec-tests" - if re.match( - r"^v[0-9]{1,2}\.[0-9]{1,3}\.[0-9]{1,3}(a[0-9]+|b[0-9]+|rc[0-9]+)?$", - branch_or_commit_or_tag, - ): - return f"{base_url}/{username}/{repository}/tree/{branch_or_commit_or_tag}/{file_path}" - else: - return f"{base_url}/{username}/{repository}/blob/{branch_or_commit_or_tag}/{file_path}" - - -def get_current_commit_hash_or_tag(repo_path="."): - """ - Get the latest commit hash or tag from the clone where doc is being built. - """ - repo = Repo(repo_path) - try: - # Get the tag that points to the current commit - current_tag = next((tag for tag in repo.tags if tag.commit == repo.head.commit)) - return current_tag.name - except StopIteration: - # If there are no tags that point to the current commit, return the commit hash - return repo.head.commit.hexsha - - -def get_current_commit_hash(repo_path="."): - """ - Get the latest commit hash from the clone where doc is being built. - """ - repo = Repo(repo_path) - return repo.head.commit.hexsha - - COMMIT_HASH_OR_TAG = get_current_commit_hash_or_tag() diff --git a/setup.cfg b/setup.cfg index 659c91453e2..a296760b6e1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,6 +28,7 @@ install_requires = hive.py@git+https://github.com/danceratopz/hive.py@chore/setup.cfg/move-mypy-deps-to-lint-extras setuptools types-setuptools + gitpython>=3.1.31,<4 PyJWT>=2.3.0,<3 tenacity>8.2.0,<9 bidict>=0.23,<1 @@ -87,7 +88,6 @@ lint = docs = cairosvg>=2.7.0,<3 # required for social plugin (material) - gitpython>=3.1.31,<4 mike>=1.1.2,<2 mkdocs>=1.4.3,<2 mkdocs-gen-files>=0.5.0,<1 diff --git a/src/ethereum_test_tools/spec/base/base_test.py b/src/ethereum_test_tools/spec/base/base_test.py index 5001508b442..fbd7021b87a 100644 --- a/src/ethereum_test_tools/spec/base/base_test.py +++ b/src/ethereum_test_tools/spec/base/base_test.py @@ -100,6 +100,8 @@ def json_dict_with_info(self, hash_only: bool = False) -> Dict[str, Any]: def fill_info( self, t8n: TransitionTool, + fixture_description: str, + fixture_source_url: str, ref_spec: ReferenceSpec | None, ): """ @@ -108,6 +110,8 @@ def fill_info( if "comment" not in self.info: self.info["comment"] = "`execution-spec-tests` generated test" self.info["filling-transition-tool"] = t8n.version() + self.info["description"] = fixture_description + self.info["url"] = fixture_source_url if ref_spec is not None: ref_spec.write_info(self.info) diff --git a/src/ethereum_test_tools/utility/__init__.py b/src/ethereum_test_tools/utility/__init__.py new file mode 100644 index 00000000000..90717b92579 --- /dev/null +++ b/src/ethereum_test_tools/utility/__init__.py @@ -0,0 +1,3 @@ +""" +Sub-package for utility functions and classes. +""" diff --git a/src/ethereum_test_tools/utility/versioning.py b/src/ethereum_test_tools/utility/versioning.py new file mode 100644 index 00000000000..99aec4709fe --- /dev/null +++ b/src/ethereum_test_tools/utility/versioning.py @@ -0,0 +1,44 @@ +""" +Utility module with helper functions for versioning. +""" + +import re + +from git import InvalidGitRepositoryError, Repo # type: ignore + + +def get_current_commit_hash_or_tag(repo_path="."): + """ + Get the latest commit hash or tag from the clone where doc is being built. + """ + try: + repo = Repo(repo_path) + # Try to get the current tag that points to the current commit + current_tag = next((tag for tag in repo.tags if tag.commit == repo.head.commit), None) + # Return the commit hash if no such tag exits + return current_tag.name if current_tag else repo.head.commit.hexsha + except InvalidGitRepositoryError: + # This hack is necessary for our framework tests. We use the pytester/tempdir fixtures + # to execute pytest within a pytest session (for top-level tests of our pytest plugins). + # The pytester fixture executes these tests in a temporary directory, which is not a git + # repository; this is a workaround to stop these tests failing. + # + # Tried monkeypatching the pytest plugin tests, but it didn't play well with pytester. + return "Not a git repository; this should only be seen in framework tests." + + +def generate_github_url(file_path, branch_or_commit_or_tag="main", line_number=""): + """ + Generate a permalink to a source file in Github. + """ + base_url = "https://github.com" + username = "ethereum" + repository = "execution-spec-tests" + if line_number: + line_number = f"#L{line_number}" + release_tag_regex = r"^v[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(a[0-9]+|b[0-9]+|rc[0-9]+)?$" + tree_or_blob = "tree" if re.match(release_tag_regex, branch_or_commit_or_tag) else "blob" + return ( + f"{base_url}/{username}/{repository}/{tree_or_blob}/" + f"{branch_or_commit_or_tag}/{file_path}{line_number}" + ) diff --git a/src/pytest_plugins/consume/simulator_common.py b/src/pytest_plugins/consume/simulator_common.py index e8f4b7d2f73..c0b2a8348ab 100644 --- a/src/pytest_plugins/consume/simulator_common.py +++ b/src/pytest_plugins/consume/simulator_common.py @@ -36,3 +36,18 @@ def fixture(fixture_source: JsonSource, test_case: TestCase) -> Fixture: fixtures = BlockchainFixtures.from_file(Path(fixture_source) / test_case.json_path) fixture = fixtures[test_case.id] return fixture + + +@pytest.fixture(scope="function") +def fixture_description(fixture: Fixture, test_case: TestCase) -> str: + """ + Return the description of the current test case. + """ + description = f"Test id: {test_case.id}" + if "url" in fixture.info: + description += f"\n\nTest source: {fixture.info['url']}" + if "description" not in fixture.info: + description += "\n\nNo description field provided in the fixture's 'info' section." + else: + description += f"\n\n{fixture.info['description']}" + return description diff --git a/src/pytest_plugins/pytest_hive/pytest_hive.py b/src/pytest_plugins/pytest_hive/pytest_hive.py index 28537675d51..6fcca2482dc 100644 --- a/src/pytest_plugins/pytest_hive/pytest_hive.py +++ b/src/pytest_plugins/pytest_hive/pytest_hive.py @@ -107,11 +107,17 @@ def hive_test(request, test_suite: HiveTestSuite): """ Propagate the pytest test case and its result to the hive server. """ - test_parameter_string = request.node.nodeid.split("[")[-1].rstrip("]") # test fixture name + try: + fixture_description = request.getfixturevalue("fixture_description") + except pytest.FixtureLookupError: + pytest.exit( + "Error: The 'fixture_description' fixture has not been defined by the simulator " + "or pytest plugin using this plugin!" + ) + test_parameter_string = request.node.nodeid # consume pytest test id test: HiveTest = test_suite.start_test( - # TODO: pass test case documentation when available name=test_parameter_string, - description="TODO: This should come from the '_info' field.", + description=fixture_description, ) yield test try: diff --git a/src/pytest_plugins/test_filler/test_filler.py b/src/pytest_plugins/test_filler/test_filler.py index 9c44d353aa1..f25eb08f5b1 100644 --- a/src/pytest_plugins/test_filler/test_filler.py +++ b/src/pytest_plugins/test_filler/test_filler.py @@ -22,6 +22,10 @@ get_forks_with_solc_support, ) from ethereum_test_tools import SPEC_TYPES, BaseTest, FixtureCollector, TestInfo, Yul +from ethereum_test_tools.utility.versioning import ( + generate_github_url, + get_current_commit_hash_or_tag, +) from evm_transition_tool import FixtureFormats, TransitionTool from pytest_plugins.spec_version_checker.spec_version_checker import EIPSpecTestItem @@ -564,6 +568,38 @@ def node_to_test_info(node) -> TestInfo: ) +@pytest.fixture(scope="function") +def fixture_source_url(request): + """ + Returns the URL to the fixture source. + """ + function_line_number = request.function.__code__.co_firstlineno + module_relative_path = os.path.relpath(request.module.__file__) + hash_or_tag = get_current_commit_hash_or_tag() + github_url = generate_github_url( + module_relative_path, branch_or_commit_or_tag=hash_or_tag, line_number=function_line_number + ) + return github_url + + +@pytest.fixture(scope="function") +def fixture_description(request): + """Fixture to extract and combine docstrings from the test class and the test function.""" + description_unavailable = ( + "No description available - add a docstring to the python test class or function." + ) + test_class_doc = f"Test class documentation:\n{request.cls.__doc__}" if request.cls else "" + test_function_doc = ( + f"Test function documentation:\n{request.function.__doc__}" + if request.function.__doc__ + else "" + ) + if not test_class_doc and not test_function_doc: + return description_unavailable + combined_docstring = f"{test_class_doc}\n\n{test_function_doc}".strip() + return combined_docstring + + def base_test_parametrizer(cls: Type[BaseTest]): """ Generates a pytest.fixture for a given BaseTest subclass. @@ -584,6 +620,8 @@ def base_test_parametrizer_func( eips, dump_dir_parameter_level, fixture_collector, + fixture_description, + fixture_source_url, ): """ Fixture used to instantiate an auto-fillable BaseTest object from within @@ -608,7 +646,12 @@ def __init__(self, *args, **kwargs): fixture_format=fixture_format, eips=eips, ) - fixture.fill_info(t8n, reference_spec) + fixture.fill_info( + t8n, + fixture_description, + fixture_source_url=fixture_source_url, + ref_spec=reference_spec, + ) fixture_path = fixture_collector.add_fixture( node_to_test_info(request.node), diff --git a/whitelist.txt b/whitelist.txt index 1b1d0a3e3c0..d4c60e2e1cb 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -128,6 +128,7 @@ extcodehash extcodesize F00 filesystem +firstlineno fn fname forkchoice @@ -250,6 +251,7 @@ parseable pathlib pdb perf +permalink petersburg pformat png @@ -420,6 +422,7 @@ makepyfile makereport metafunc modifyitems +monkeypatching nodeid noop oog @@ -453,6 +456,7 @@ substring substrings tf teardown +tempdir testdir teststatus tmpdir