feat(tests): Add multiple exception support to EOF tests

docs/CHANGELOG.md

@@ -25,6 +25,7 @@ Test fixtures for use by clients are available for each release on the [Github r
 - 🐞 Fixed fixture index generation on EOF tests ([#728](https://github.com/ethereum/execution-spec-tests/pull/728)).
 - 🐞 Fixes consume genesis mismatch exception for hive based simulators ([#734](https://github.com/ethereum/execution-spec-tests/pull/734)).
 - ✨ Adds reproducible consume commands to hiveview ([#717](https://github.com/ethereum/execution-spec-tests/pull/717)).
+- 💥 Added multiple exceptions to the EOF fixture format ([#759](https://github.com/ethereum/execution-spec-tests/pull/759)).
 - ✨ Added optional parameter to all `with_all_*` markers to specify a lambda function that filters the parametrized values ([#739](https://github.com/ethereum/execution-spec-tests/pull/739)).
 - ✨ Added [`extend_with_defaults` utility function](https://ethereum.github.io/execution-spec-tests/main/writing_tests/writing_a_new_test/#ethereum_test_tools.utility.pytest.extend_with_defaults), which helps extend test case parameter sets with default values. `@pytest.mark.parametrize` ([#739](https://github.com/ethereum/execution-spec-tests/pull/739)).
 - ✨ Added `Container.Init` to `ethereum_test_types.EOF.V1` package, which allows generation of an EOF init container more easily ([#739](https://github.com/ethereum/execution-spec-tests/pull/739)).
@@ -37,6 +38,10 @@ Test fixtures for use by clients are available for each release on the [Github r
 - ✨ Releases for feature eip7692 now include both Cancun and Prague based tests in the same release, in files `fixtures_eip7692.tar.gz` and `fixtures_eip7692-prague.tar.gz` respectively ([#743](https://github.com/ethereum/execution-spec-tests/pull/743)).
 - 🔀 Simplify Python project configuration and consolidate it into `pyproject.toml` ([#764](https://github.com/ethereum/execution-spec-tests/pull/764)).
 
+### 💥 Breaking Change
+
+- The EOF fixture format contained in `eof_tests` may now contain multiple exceptions in the `"exception"` field in the form of a pipe (`|`) separated string ([#759](https://github.com/ethereum/execution-spec-tests/pull/759)).
+
 ## [v3.0.0](https://github.com/ethereum/execution-spec-tests/releases/tag/v3.0.0) - 2024-07-22
 
 ### 🧪 Test Cases

src/ethereum_test_exceptions/__init__.py

@@ -8,6 +8,7 @@
     BlockException,
     BlockExceptionInstanceOrList,
     EOFException,
+    EOFExceptionInstanceOrList,
     ExceptionInstanceOrList,
     TransactionException,
     TransactionExceptionInstanceOrList,
@@ -16,10 +17,11 @@
 __all__ = [
     "BlockException",
     "BlockExceptionInstanceOrList",
-    "EngineAPIError",
     "EOFException",
+    "EOFExceptionInstanceOrList",
+    "EngineAPIError",
+    "EvmoneExceptionMapper",
     "ExceptionInstanceOrList",
     "TransactionException",
     "TransactionExceptionInstanceOrList",
-    "EvmoneExceptionMapper",
 ]

src/ethereum_test_exceptions/evmone_exceptions.py

@@ -85,6 +85,9 @@ class EvmoneExceptionMapper:
         ),
         ExceptionMessage(EOFException.STACK_HEIGHT_MISMATCH, "err: stack_height_mismatch"),
         ExceptionMessage(EOFException.TOO_MANY_CONTAINERS, "err: too_many_container_sections"),
+        ExceptionMessage(
+            EOFException.INVALID_CODE_SECTION_INDEX, "err: invalid_code_section_index"
+        ),
     )
 
     def __init__(self) -> None:

src/ethereum_test_exceptions/exceptions.py

@@ -730,6 +730,14 @@ class EOFException(ExceptionBase):
     """
     EOF container header has too many sub-containers.
     """
+    INVALID_CODE_SECTION_INDEX = auto()
+    """
+    CALLF Operation referes to a non-existent code section
+    """
+    UNEXPECTED_HEADER_KIND = auto()
+    """
+    Header parsing encounterd a section kind it wasn't expecting
+    """
 
 
 """
@@ -753,3 +761,9 @@ class EOFException(ExceptionBase):
     BeforeValidator(from_pipe_str),
     PlainSerializer(to_pipe_str),
 ]
+
+EOFExceptionInstanceOrList = Annotated[
+    List[EOFException] | EOFException,
+    BeforeValidator(from_pipe_str),
+    PlainSerializer(to_pipe_str),
+]

src/ethereum_test_fixtures/eof.py

@@ -8,7 +8,7 @@
 from pydantic import Field
 
 from ethereum_test_base_types import Bytes, CamelModel, Number
-from ethereum_test_exceptions import EOFException
+from ethereum_test_exceptions.exceptions import EOFExceptionInstanceOrList
 from ethereum_test_types.eof.v1 import ContainerKind
 
 from .base import BaseFixture
@@ -20,7 +20,7 @@ class Result(CamelModel):
     Result for a single fork in a fixture.
     """
 
-    exception: EOFException | None = None
+    exception: EOFExceptionInstanceOrList | None = None
     valid: bool = Field(..., alias="result")
 
     def model_post_init(self, __context: Any) -> None:

src/ethereum_test_specs/eof.py

@@ -12,7 +12,8 @@
 from pydantic import Field, model_validator
 
 from ethereum_test_base_types import Account, Bytes
-from ethereum_test_exceptions import EOFException, EvmoneExceptionMapper
+from ethereum_test_exceptions import EvmoneExceptionMapper
+from ethereum_test_exceptions.exceptions import EOFExceptionInstanceOrList, to_pipe_str
 from ethereum_test_fixtures import BaseFixture, FixtureFormats
 from ethereum_test_fixtures.eof import Fixture, Result, Vector
 from ethereum_test_forks import Fork
@@ -136,7 +137,7 @@ class EOFTest(BaseTest):
     """
 
     data: Bytes
-    expect_exception: EOFException | None = None
+    expect_exception: EOFExceptionInstanceOrList | None = None
     container_kind: ContainerKind | None = None
 
     supported_fixture_formats: ClassVar[List[FixtureFormats]] = [
@@ -237,21 +238,23 @@ def verify_result(self, result: CompletedProcess, expected_result: Result, code:
                 raise UnexpectedEOFException(
                     code=code, got=f"{actual_exception} ({actual_message})"
                 )
-
-        expected_exception = expected_result.exception
-        expected_message = parser.exception_to_message(expected_exception)
-
-        if "OK" in actual_message:
-            raise ExpectedEOFException(
-                code=code, expected=f"{expected_exception} ({expected_message})"
-            )
-
-        if expected_exception != actual_exception:
-            raise EOFExceptionMismatch(
-                code=code,
-                expected=f"{expected_exception} ({expected_message})",
-                got=f"{actual_exception} ({actual_message})",
-            )
+        else:
+            expected_string = to_pipe_str(expected_result.exception)
+            print(expected_string)
+            print(actual_exception)
+            if "OK" in actual_message:
+                raise ExpectedEOFException(
+                    code=code,
+                    expected=f"{expected_string}",
+                )
+            elif actual_exception in expected_result.exception:
+                return
+            else:
+                raise EOFExceptionMismatch(
+                    code=code,
+                    expected=f"{expected_string}",
+                    got=f"{actual_exception} ({actual_message})",
+                )
 
     def generate(
         self,

src/ethereum_test_types/eof/v1/__init__.py

@@ -17,7 +17,7 @@
 from ethereum_test_base_types import Bytes
 from ethereum_test_base_types.conversions import BytesConvertible
 from ethereum_test_base_types.pydantic import CopyValidateModel
-from ethereum_test_exceptions import EOFException
+from ethereum_test_exceptions.exceptions import EOFExceptionInstanceOrList
 from ethereum_test_vm import Bytecode
 from ethereum_test_vm import Opcodes as Op
 
@@ -365,7 +365,7 @@ class Container(CopyValidateModel):
     Body: type section first, all code sections, data section(s), last
                 container sections
     """
-    validity_error: EOFException | str | None = None
+    validity_error: EOFExceptionInstanceOrList | str | None = None
     """
     Optional error expected for the container.