diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index e5a69363d2b..3b5626831b5 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -23,6 +23,7 @@ Test fixtures for use by clients are available for each release on the [Github r - ✨ `fill` command now supports parameter `--evm-code-type` that can be (currently) set to `legacy` or `eof_v1` to force all test smart contracts to deployed in normal or in EOF containers ([#610](https://github.com/ethereum/execution-spec-tests/pull/610)). - 🐞 Fixed fixture index generation on EOF tests ([#728](https://github.com/ethereum/execution-spec-tests/pull/728)). - 🐞 Fixes consume genesis mismatch exception for hive based simulators ([#734](https://github.com/ethereum/execution-spec-tests/pull/734)). +- ✨ Adds reproducible consume commands to hiveview ([#717](https://github.com/ethereum/execution-spec-tests/pull/717)). ### 🔧 EVM Tools diff --git a/src/pytest_plugins/consume/consume.py b/src/pytest_plugins/consume/consume.py index 3a950843c74..4466eda3239 100644 --- a/src/pytest_plugins/consume/consume.py +++ b/src/pytest_plugins/consume/consume.py @@ -70,7 +70,7 @@ def download_and_extract(url: str, base_directory: Path) -> Path: with tarfile.open(archive_path, "r:gz") as tar: tar.extractall(path=extract_to) - return extract_to + return extract_to / "fixtures" def pytest_addoption(parser): # noqa: D103 @@ -87,6 +87,15 @@ def pytest_addoption(parser): # noqa: D103 f"'{default_input_directory()}'." ), ) + consume_group.addoption( + "--latest", + action="store_true", + dest="latest_source", + default=False, + help=( + "The latest EEST development JSON test fixtures. Cannot be used alongside `--input`." + ), + ) consume_group.addoption( "--fork", action="store", @@ -123,11 +132,22 @@ def pytest_configure(config): # noqa: D103 called before the pytest-html plugin's pytest_configure to ensure that it uses the modified `htmlpath` option. """ + input_flag = any(arg.startswith("--input") for arg in config.invocation_params.args) + latest_flag = config.getoption("latest_source") + + if input_flag and latest_flag: + pytest.exit("Cannot use both `--input` and `--latest`, please select one input flag.") + input_source = config.getoption("fixture_source") - if input_source == "stdin": + + if input_flag and input_source == "stdin": config.test_cases = TestCases.from_stream(sys.stdin) return + if latest_flag: + release_base_url = "https://github.com/ethereum/execution-spec-tests/releases" + input_source = f"{release_base_url}/latest/download/fixtures_develop.tar.gz" + if is_url(input_source): cached_downloads_directory.mkdir(parents=True, exist_ok=True) input_source = download_and_extract(input_source, cached_downloads_directory) @@ -144,9 +164,9 @@ def pytest_configure(config): # noqa: D103 index_file = input_source / ".meta" / "index.json" if not index_file.exists(): rich.print(f"Generating index file [bold cyan]{index_file}[/]...") - generate_fixtures_index( - input_source, quiet_mode=False, force_flag=False, disable_infer_format=False - ) + generate_fixtures_index( + input_source, quiet_mode=False, force_flag=False, disable_infer_format=False + ) config.test_cases = TestCases.from_index_file(index_file) if config.option.collectonly: diff --git a/src/pytest_plugins/consume/hive_simulators/conftest.py b/src/pytest_plugins/consume/hive_simulators/conftest.py index 2428548c0e4..7632b30b320 100644 --- a/src/pytest_plugins/consume/hive_simulators/conftest.py +++ b/src/pytest_plugins/consume/hive_simulators/conftest.py @@ -4,7 +4,7 @@ import io import json -from typing import Generator, cast +from typing import Generator, List, cast import pytest import rich @@ -28,12 +28,47 @@ def eth_rpc(client: Client) -> EthRPC: return EthRPC(ip=client.ip) +@pytest.fixture(scope="function") +def hive_consume_command( + test_suite_name: str, + client_type: ClientType, + test_case: TestCaseIndexFile | TestCaseStream, +) -> str: + """ + Command to run the test within hive. + """ + return ( + f"./hive --sim ethereum/{test_suite_name} " + f"--client-file configs/develop.yaml " + f"--client {client_type.name} " + f'--sim.limit "{test_case.id}"' + ) + + +@pytest.fixture(scope="function") +def eest_consume_commands( + test_suite_name: str, + client_type: ClientType, + test_case: TestCaseIndexFile | TestCaseStream, +) -> List[str]: + """ + Commands to run the test within EEST using a hive dev back-end. + """ + hive_dev = f"./hive --dev --client-file configs/develop.yaml --client {client_type.name}" + consume = f'consume {test_suite_name.split("-")[-1]} -v --latest -k "{test_case.id}"' + return [hive_dev, consume] + + @pytest.fixture(scope="function") def fixture_description( - blockchain_fixture: BlockchainFixtureCommon, test_case: TestCaseIndexFile | TestCaseStream + blockchain_fixture: BlockchainFixtureCommon, + test_case: TestCaseIndexFile | TestCaseStream, + hive_consume_command: str, + eest_consume_commands: List[str], ) -> str: """ Create the description of the current blockchain fixture test case. + Includes reproducible commands to re-run the test case against the target client. """ description = f"Test id: {test_case.id}" if "url" in blockchain_fixture.info: @@ -42,6 +77,16 @@ def fixture_description( description += "\n\nNo description field provided in the fixture's 'info' section." else: description += f"\n\n{blockchain_fixture.info['description']}" + description += ( + f"\n\nCommand to reproduce entirely in hive:" f"\n{hive_consume_command}" + ) + eest_commands = "\n".join( + f"{i+1}. {cmd}" for i, cmd in enumerate(eest_consume_commands) + ) + description += ( + "\n\nCommands to reproduce within EEST using a hive dev back-end:" f"\n{eest_commands}" + ) + description = description.replace("\n", "
") return description diff --git a/src/pytest_plugins/consume/hive_simulators/rlp/conftest.py b/src/pytest_plugins/consume/hive_simulators/rlp/conftest.py index ef278e05af9..d908346aa00 100644 --- a/src/pytest_plugins/consume/hive_simulators/rlp/conftest.py +++ b/src/pytest_plugins/consume/hive_simulators/rlp/conftest.py @@ -22,7 +22,7 @@ def test_suite_name() -> str: """ The name of the hive test suite used in this simulator. """ - return "eest-rlp" + return "eest-block-rlp" @pytest.fixture(scope="module") diff --git a/src/pytest_plugins/filler/filler.py b/src/pytest_plugins/filler/filler.py index 7f93b4f5481..5ea02fe6335 100644 --- a/src/pytest_plugins/filler/filler.py +++ b/src/pytest_plugins/filler/filler.py @@ -691,7 +691,7 @@ def fixture_collector( f.write(str(fixture_collector_count)) if generate_index and fixture_collector_count == 0: generate_fixtures_index( - output_dir, quiet_mode=True, force_flag=True, disable_infer_format=False + output_dir, quiet_mode=True, force_flag=False, disable_infer_format=False ) diff --git a/whitelist.txt b/whitelist.txt index 64ea044cc2f..3c9953123a7 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -189,6 +189,7 @@ hexary HexNumber hexsha hexbytes +hiveview homebrew html htmlpath