feat(fw): use a metadata folder within fixtures (#721)

* feat(fw): use .meta folder within fixtures.

* chore(consume|fw): tweaks and test updates.

* docs: changelog.

* fix(plugins/filler): Add quotes to fill parameters when it uses spaces

---------

Co-authored-by: Mario Vega <marioevz@gmail.com>

docs/CHANGELOG.md

@@ -13,6 +13,7 @@ Test fixtures for use by clients are available for each release on the [Github r
 - 🐞 Fixed consume hive commands from spawning different hive test suites during the same test execution when using xdist ([#712](https://github.com/ethereum/execution-spec-tests/pull/712)).
 - ✨ `consume hive` command is now available to run all types of hive tests ([#712](https://github.com/ethereum/execution-spec-tests/pull/712)).
 - ✨ Generated fixtures now contain the test index `index.json` by default ([#716](https://github.com/ethereum/execution-spec-tests/pull/716)).
+- ✨ A metadata folder `.meta/` now stores all fixture metadata files by default ([#721](https://github.com/ethereum/execution-spec-tests/pull/721)).
 
 ### 🔧 EVM Tools
 

src/cli/gen_index.py

@@ -134,7 +134,7 @@ def generate_fixtures_index(
     if not quiet_mode:
         total_files = count_json_files_exclude_index(input_path)
 
-    output_file = Path(f"{input_path}/index.json")
+    output_file = Path(f"{input_path}/.meta/index.json")
     try:
         root_hash = HashableItem.from_folder(folder_path=input_path).hash()
     except (KeyError, TypeError):

src/cli/tests/test_pytest_fill_command.py

@@ -78,11 +78,11 @@ def fill_args(self):
         return ["-k", "test_dup and state_test-DUP16", "--fork", "Frontier"]
 
     @pytest.fixture()
-    def default_html_report_filename(self):
+    def default_html_report_file_path(self):
         """
-        The default filename for fill's pytest html report.
+        The default file path for fill's pytest html report.
         """
-        return pytest_plugins.filler.filler.default_html_report_filename()
+        return pytest_plugins.filler.filler.default_html_report_file_path()
 
     @pytest.fixture(scope="function")
     def temp_dir(self) -> Generator[Path, None, None]:  # noqa: D102
@@ -113,12 +113,12 @@ def test_fill_default_output_options(
         runner,
         temp_dir,
         fill_args,
-        default_html_report_filename,
+        default_html_report_file_path,
     ):
         """
         Test default pytest html behavior: Neither `--html` or `--output` is specified.
         """
-        default_html_path = temp_dir / default_html_report_filename
+        default_html_path = temp_dir / default_html_report_file_path
         result = runner.invoke(fill, fill_args)
         assert result.exit_code == pytest.ExitCode.OK
         assert default_html_path.exists()
@@ -128,12 +128,12 @@ def test_fill_no_html_option(
         runner,
         temp_dir,
         fill_args,
-        default_html_report_filename,
+        default_html_report_file_path,
     ):
         """
         Test pytest html report is disabled with the `--no-html` flag.
         """
-        default_html_path = temp_dir / default_html_report_filename
+        default_html_path = temp_dir / default_html_report_file_path
         fill_args += ["--no-html"]
         result = runner.invoke(fill, fill_args)
         assert result.exit_code == pytest.ExitCode.OK
@@ -159,13 +159,13 @@ def test_fill_output_option(
         runner,
         temp_dir,
         fill_args,
-        default_html_report_filename,
+        default_html_report_file_path,
     ):
         """
         Tests pytest html report generation with only the `--output` flag.
         """
         output_dir = temp_dir / "non_default_output_dir"
-        non_default_html_path = output_dir / default_html_report_filename
+        non_default_html_path = output_dir / default_html_report_file_path
         fill_args += ["--output", str(output_dir)]
         result = runner.invoke(fill, fill_args)
         assert result.exit_code == pytest.ExitCode.OK

src/pytest_plugins/consume/consume.py

@@ -30,12 +30,12 @@ def default_input_directory() -> str:
     return "./fixtures"
 
 
-def default_html_report_filename() -> str:
+def default_html_report_file_path() -> str:
     """
-    The default file to store the generated HTML test report. Defined as a
+    The default filepath to store the generated HTML test report. Defined as a
     function to allow for easier testing.
     """
-    return "report_consume.html"
+    return ".meta/report_consume.html"
 
 
 def is_url(string: str) -> bool:
@@ -141,20 +141,20 @@ def pytest_configure(config):  # noqa: D103
             f"Specified fixture directory '{input_source}' does not contain any JSON files."
         )
 
-    index_file = input_source / "index.json"
+    index_file = input_source / ".meta" / "index.json"
     if not index_file.exists():
         rich.print(f"Generating index file [bold cyan]{index_file}[/]...")
     generate_fixtures_index(
-        Path(input_source), quiet_mode=False, force_flag=False, disable_infer_format=False
+        input_source, quiet_mode=False, force_flag=False, disable_infer_format=False
     )
-    config.test_cases = TestCases.from_index_file(Path(input_source) / "index.json")
+    config.test_cases = TestCases.from_index_file(index_file)
 
     if config.option.collectonly:
         return
     if not config.getoption("disable_html") and config.getoption("htmlpath") is None:
         # generate an html report by default, unless explicitly disabled
         config.option.htmlpath = os.path.join(
-            config.getoption("fixture_source"), default_html_report_filename()
+            config.getoption("fixture_source"), default_html_report_file_path()
         )
 
 

src/pytest_plugins/filler/filler.py

@@ -46,12 +46,12 @@ def default_output_directory() -> str:
     return "./fixtures"
 
 
-def default_html_report_filename() -> str:
+def default_html_report_file_path() -> str:
     """
-    The default file to store the generated HTML test report. Defined as a
+    The default file path to store the generated HTML test report. Defined as a
     function to allow for easier testing.
     """
-    return "report_fill.html"
+    return ".meta/report_fill.html"
 
 
 def strip_output_tarball_suffix(output: Path) -> Path:
@@ -235,7 +235,7 @@ def pytest_configure(config):
         # generate an html report by default, unless explicitly disabled
         config.option.htmlpath = (
             strip_output_tarball_suffix(config.getoption("output"))
-            / default_html_report_filename()
+            / default_html_report_file_path()
         )
     # Instantiate the transition tool here to check that the binary path/trace option is valid.
     # This ensures we only raise an error once, if appropriate, instead of for every test.
@@ -263,7 +263,11 @@ def pytest_configure(config):
         "t8n": t8n.version(),
         "solc": str(config.solc_version),
     }
-    command_line_args = "fill " + " ".join(config.invocation_params.args)
+    args = ["fill"] + [str(arg) for arg in config.invocation_params.args]
+    for i in range(len(args)):
+        if " " in args[i]:
+            args[i] = f'"{args[i]}"'
+    command_line_args = " ".join(args)
     config.stash[metadata_key]["Command-line args"] = f"<code>{command_line_args}</code>"
 
 
@@ -492,8 +496,18 @@ def output_dir(request: pytest.FixtureRequest, is_output_tarball: bool) -> Path:
     return output
 
 
+@pytest.fixture(scope="session")
+def output_metadata_dir(output_dir: Path) -> Path:
+    """
+    Returns the metadata directory to store fixture meta files.
+    """
+    return output_dir / ".meta"
+
+
 @pytest.fixture(scope="session", autouse=True)
-def create_properties_file(request: pytest.FixtureRequest, output_dir: Path) -> None:
+def create_properties_file(
+    request: pytest.FixtureRequest, output_dir: Path, output_metadata_dir: Path
+) -> None:
     """
     Creates an ini file with fixture build properties in the fixture output
     directory.
@@ -502,6 +516,8 @@ def create_properties_file(request: pytest.FixtureRequest, output_dir: Path) ->
         return
     if not output_dir.exists():
         output_dir.mkdir(parents=True)
+    if not output_metadata_dir.exists():
+        output_metadata_dir.mkdir(parents=True)
 
     fixture_properties = {
         "timestamp": datetime.datetime.now().isoformat(),
@@ -530,7 +546,7 @@ def create_properties_file(request: pytest.FixtureRequest, output_dir: Path) ->
             warnings.warn(f"Fixtures ini file: Skipping metadata key {key} with value {val}.")
     config["environment"] = environment_properties
 
-    ini_filename = output_dir / "fixtures.ini"
+    ini_filename = output_metadata_dir / "fixtures.ini"
     with open(ini_filename, "w") as f:
         f.write("; This file describes fixture build properties\n\n")
         config.write(f)
@@ -619,7 +635,7 @@ def fixture_collector(
     if do_fixture_verification:
         fixture_collector.verify_fixture_files(evm_fixture_verification)
     generate_fixtures_index(
-        output_dir, quiet_mode=False, force_flag=False, disable_infer_format=False
+        output_dir, quiet_mode=True, force_flag=True, disable_infer_format=False
     )
 
 

src/pytest_plugins/filler/tests/test_filler.py

@@ -524,43 +524,39 @@ def test_fixture_output_based_on_command_line_args(
     assert output_dir.exists()
 
     all_files = get_all_files_in_directory(output_dir)
+    meta_dir = os.path.join(output_dir, ".meta")
+    assert os.path.exists(meta_dir), f"The directory {meta_dir} does not exist"
 
-    expected_fixtures_ini_filename = "fixtures.ini"
-    expected_fixtures_index_filename = "index.json"
+    expected_ini_file = "fixtures.ini"
+    expected_index_file = "index.json"
 
     ini_file = None
     index_file = None
     for file in all_files:
-        if file.name == expected_fixtures_ini_filename:
+        if file.name == expected_ini_file:
             ini_file = file
-        elif file.name == expected_fixtures_index_filename:
+        elif file.name == expected_index_file:
             index_file = file
 
-    assert ini_file is not None, f"No {expected_fixtures_ini_filename} file was written"
-    assert index_file is not None, f"No {expected_fixtures_index_filename} file was written"
-
-    all_files = [
-        file
-        for file in all_files
-        if file.name
-        not in {
-            expected_fixtures_ini_filename,
-            expected_fixtures_index_filename,
-        }
+    all_fixtures = [
+        file for file in all_files if file.name not in {expected_ini_file, expected_index_file}
     ]
-
     for fixture_file, fixture_count in zip(expected_fixture_files, expected_fixture_counts):
-        assert fixture_file.exists()
-        assert fixture_count == count_keys_in_fixture(fixture_file)
+        assert fixture_file.exists(), f"{fixture_file} does not exist"
+        assert fixture_count == count_keys_in_fixture(
+            fixture_file
+        ), f"Fixture count mismatch for {fixture_file}"
 
-    assert set(all_files) == set(
+    assert set(all_fixtures) == set(
         expected_fixture_files
-    ), f"Unexpected files in directory: {set(all_files) - set(expected_fixture_files)}"
+    ), f"Unexpected files in directory: {set(all_fixtures) - set(expected_fixture_files)}"
 
-    assert ini_file is not None, f"No {expected_fixtures_ini_filename} file was written"
+    assert ini_file is not None, f"No {expected_ini_file} file was found in {meta_dir}"
     config = configparser.ConfigParser()
     config.read(ini_file)
 
+    assert index_file is not None, f"No {expected_index_file} file was found in {meta_dir}"
+
     properties = {key: value for key, value in config.items("fixtures")}
     assert "timestamp" in properties
     timestamp = datetime.fromisoformat(properties["timestamp"])