feat(consume): add reproducible consume/hive commands (#717)

* feat(consume): add reproducible hive/consume commands.

* feat(consume): fix command formatting.

* feat(consume): add latest input option, more formatting.

* chore(consume): remove opt. hive command flags.

* chore(consume): use list comp. for code derivation.

* feat(consume): update latest flag option.

* docs: update changelog.

* chore(consume): fix index.json for urls.

* chore(consume): align --sim.limit and -k.

* Update src/pytest_plugins/consume/hive_simulators/conftest.py

---------

Co-authored-by: Mario Vega <marioevz@gmail.com>

docs/CHANGELOG.md

@@ -23,6 +23,7 @@ Test fixtures for use by clients are available for each release on the [Github r
 - ✨ `fill` command now supports parameter `--evm-code-type` that can be (currently) set to `legacy` or `eof_v1` to force all test smart contracts to deployed in normal or in EOF containers ([#610](https://github.com/ethereum/execution-spec-tests/pull/610)).
 - 🐞 Fixed fixture index generation on EOF tests ([#728](https://github.com/ethereum/execution-spec-tests/pull/728)).
 - 🐞 Fixes consume genesis mismatch exception for hive based simulators ([#734](https://github.com/ethereum/execution-spec-tests/pull/734)).
+- ✨ Adds reproducible consume commands to hiveview ([#717](https://github.com/ethereum/execution-spec-tests/pull/717)).
 
 ### 🔧 EVM Tools
 

src/pytest_plugins/consume/consume.py

@@ -70,7 +70,7 @@ def download_and_extract(url: str, base_directory: Path) -> Path:
     with tarfile.open(archive_path, "r:gz") as tar:
         tar.extractall(path=extract_to)
 
-    return extract_to
+    return extract_to / "fixtures"
 
 
 def pytest_addoption(parser):  # noqa: D103
@@ -87,6 +87,15 @@ def pytest_addoption(parser):  # noqa: D103
             f"'{default_input_directory()}'."
         ),
     )
+    consume_group.addoption(
+        "--latest",
+        action="store_true",
+        dest="latest_source",
+        default=False,
+        help=(
+            "The latest EEST development JSON test fixtures. Cannot be used alongside `--input`."
+        ),
+    )
     consume_group.addoption(
         "--fork",
         action="store",
@@ -123,11 +132,22 @@ def pytest_configure(config):  # noqa: D103
     called before the pytest-html plugin's pytest_configure to ensure that
     it uses the modified `htmlpath` option.
     """
+    input_flag = any(arg.startswith("--input") for arg in config.invocation_params.args)
+    latest_flag = config.getoption("latest_source")
+
+    if input_flag and latest_flag:
+        pytest.exit("Cannot use both `--input` and `--latest`, please select one input flag.")
+
     input_source = config.getoption("fixture_source")
-    if input_source == "stdin":
+
+    if input_flag and input_source == "stdin":
         config.test_cases = TestCases.from_stream(sys.stdin)
         return
 
+    if latest_flag:
+        release_base_url = "https://github.com/ethereum/execution-spec-tests/releases"
+        input_source = f"{release_base_url}/latest/download/fixtures_develop.tar.gz"
+
     if is_url(input_source):
         cached_downloads_directory.mkdir(parents=True, exist_ok=True)
         input_source = download_and_extract(input_source, cached_downloads_directory)
@@ -144,9 +164,9 @@ def pytest_configure(config):  # noqa: D103
     index_file = input_source / ".meta" / "index.json"
     if not index_file.exists():
         rich.print(f"Generating index file [bold cyan]{index_file}[/]...")
-    generate_fixtures_index(
-        input_source, quiet_mode=False, force_flag=False, disable_infer_format=False
-    )
+        generate_fixtures_index(
+            input_source, quiet_mode=False, force_flag=False, disable_infer_format=False
+        )
     config.test_cases = TestCases.from_index_file(index_file)
 
     if config.option.collectonly:

src/pytest_plugins/consume/hive_simulators/conftest.py

@@ -4,7 +4,7 @@
 
 import io
 import json
-from typing import Generator, cast
+from typing import Generator, List, cast
 
 import pytest
 import rich
@@ -28,12 +28,47 @@ def eth_rpc(client: Client) -> EthRPC:
     return EthRPC(ip=client.ip)
 
 
+@pytest.fixture(scope="function")
+def hive_consume_command(
+    test_suite_name: str,
+    client_type: ClientType,
+    test_case: TestCaseIndexFile | TestCaseStream,
+) -> str:
+    """
+    Command to run the test within hive.
+    """
+    return (
+        f"./hive --sim ethereum/{test_suite_name} "
+        f"--client-file configs/develop.yaml "
+        f"--client {client_type.name} "
+        f'--sim.limit "{test_case.id}"'
+    )
+
+
+@pytest.fixture(scope="function")
+def eest_consume_commands(
+    test_suite_name: str,
+    client_type: ClientType,
+    test_case: TestCaseIndexFile | TestCaseStream,
+) -> List[str]:
+    """
+    Commands to run the test within EEST using a hive dev back-end.
+    """
+    hive_dev = f"./hive --dev --client-file configs/develop.yaml --client {client_type.name}"
+    consume = f'consume {test_suite_name.split("-")[-1]} -v --latest -k "{test_case.id}"'
+    return [hive_dev, consume]
+
+
 @pytest.fixture(scope="function")
 def fixture_description(
-    blockchain_fixture: BlockchainFixtureCommon, test_case: TestCaseIndexFile | TestCaseStream
+    blockchain_fixture: BlockchainFixtureCommon,
+    test_case: TestCaseIndexFile | TestCaseStream,
+    hive_consume_command: str,
+    eest_consume_commands: List[str],
 ) -> str:
     """
     Create the description of the current blockchain fixture test case.
+    Includes reproducible commands to re-run the test case against the target client.
     """
     description = f"Test id: {test_case.id}"
     if "url" in blockchain_fixture.info:
@@ -42,6 +77,16 @@ def fixture_description(
         description += "\n\nNo description field provided in the fixture's 'info' section."
     else:
         description += f"\n\n{blockchain_fixture.info['description']}"
+    description += (
+        f"\n\nCommand to reproduce entirely in hive:" f"\n<code>{hive_consume_command}</code>"
+    )
+    eest_commands = "\n".join(
+        f"{i+1}. <code>{cmd}</code>" for i, cmd in enumerate(eest_consume_commands)
+    )
+    description += (
+        "\n\nCommands to reproduce within EEST using a hive dev back-end:" f"\n{eest_commands}"
+    )
+    description = description.replace("\n", "<br/>")
     return description
 
 

src/pytest_plugins/consume/hive_simulators/rlp/conftest.py

@@ -22,7 +22,7 @@ def test_suite_name() -> str:
     """
     The name of the hive test suite used in this simulator.
     """
-    return "eest-rlp"
+    return "eest-block-rlp"
 
 
 @pytest.fixture(scope="module")

src/pytest_plugins/filler/filler.py

@@ -691,7 +691,7 @@ def fixture_collector(
                 f.write(str(fixture_collector_count))
     if generate_index and fixture_collector_count == 0:
         generate_fixtures_index(
-            output_dir, quiet_mode=True, force_flag=True, disable_infer_format=False
+            output_dir, quiet_mode=True, force_flag=False, disable_infer_format=False
         )
 
 

whitelist.txt

@@ -189,6 +189,7 @@ hexary
 HexNumber
 hexsha
 hexbytes
+hiveview
 homebrew
 html
 htmlpath