Improvements to codebase

README.md

@@ -43,18 +43,21 @@ Optionally, instruction context messages may also be left in the original Protob
 To run the test suite, use the following command:
 
 ```sh
-solana-test-suite run-tests --input-dir <input_dir> --solana-target <solana_target.so> --target <firedancer> [--target <target_2> ...] --output-dir <log_output_dir> --num-processes <num_processes> --chunk-size <chunk_size> [--randomize-output-buffer]
+solana-test-suite run-tests --input-dir <input_dir> --solana-target <solana_target.so> --target <firedancer.so> [--target <target_2> ...] --output-dir <log_output_dir> --num-processes <num_processes> --chunk-size <chunk_size> [--randomize-output-buffer]
 ```
 
+You can provide both `InstrContext` and `InstrFixture` within `--input-dir` - parsing is taken care of depending on the file extension `.bin` for `InstrContext` and `.fix` for `InstrFixture`.
+
 | Argument        | Description                                                                                         |
 |-----------------|-----------------------------------------------------------------------------------------------------|
-| `--input-dir`   | Input directory containing instruction context messages |
+| `--input-dir`   | Input directory containing instruction context or fixture messages |
 | `--solana-target` | Path to Solana Agave shared object (.so) target file            |
 | `--target`      | Additional shared object (.so) target file paths  |
 | `--output-dir`  | Log output directory for test results |
 | `--num-processes`  | Number of processes to use |
 | `--randomize-output-buffer`| Randomizes bytes in output buffer before shared library execution                                                        |
 | `--chunk-size`  | Number of test results per log file |
+| `--verbose`   | Verbose output: log failed test cases |
 
 **Note:** Each `.so` target file name should be unique.
 
@@ -111,16 +114,22 @@ solana-test-suite minimize-tests --input-dir <input_dir> --solana-target <solana
 Create full test fixtures containing both instruction context and effects. Effects are computed by running instruction context through `--solana-target`. Fixtures with `None` values for instruction context/effects are not included.
 
 ```sh
-solana-test-suite create-fixtures --input-dir <input_dir> --solana-target <solana_target.so> --output-dir <fixtures_output_dir> --num-processes <num_processes> [--readable]
+solana-test-suite create-fixtures --input-dir <input_dir> --solana-target <solana_target.so> --target <firedancer.so> [--target <target_2> ...] --output-dir <fixtures_output_dir> --num-processes <num_processes> [--readable] [--keep-passing] [--group-by-program]
 ```
 
+You have an additional option to produce fixtures for only passing test cases (makes it easier to produce fixtures from larger batches of new-passing mismatches).
+
+
 | Argument        | Description                                                                                         |
 |-----------------|-----------------------------------------------------------------------------------------------------|
 | `--input-dir`   | Input directory containing instruction context messages |
 | `--solana-target` | Path to Solana Agave shared object (.so) target file            |
+| `--target`  | Shared object (.so) target file paths (pairs with `--keep-passing`)
 | `--output-dir`  | Instruction fixtures dumping directory |
 | `--num-processes`  | Number of processes to use |
 | `--readable` | Output fixtures in human-readable format |
+| `--keep-passing` | Only keep passing test cases |
+| `--group-by-program` | Group fixture output by program type |
 
 
 ### Create Instruction Context from Fixtures
@@ -137,23 +146,6 @@ solana-test-suite instr-from-fixtures --input-dir <input_dir> --solana-target <s
 | `--output-dir`  | Output directory for instr contexts |
 | `--num-processes`  | Number of processes to use |
 
-### Validation
-
-Used to detect potential memory corruption issues / inconsistent outputs. The program will run each supplied library `num-iteration` times on the entire test suite. Use the following:
-
-```sh
-solana-test-suite check-consistency --input-dir <input_dir> --target <firedancer> [--target <target_2> ...] --output-dir <log_output_dir> --num-iterations <num_iterations> --num-processes <num_processes> [--randomize-output-buffer]
-```
-
-| Argument                   | Description                                                                                                              |
-|----------------------------|--------------------------------------------------------------------------------------------------------------------------|
-| `--input-dir`              | Input directory containing instruction context messages                                       |
-| `--target`                 | Additional shared object (.so) target file paths                                                                         |
-| `--output-dir`             | Log output directory for test results                                                                                    |
-| `--num-iterations`         | Number of consistency iterations to run for each library                                                                 |
-| `--num-processes`          | Number of processes to use                                                                                               |
-| `--randomize-output-buffer`| Randomizes bytes in output buffer before shared library execution                                                        |
-
 
 ### Uninstalling
 

src/test_suite/constants.py

@@ -3,3 +3,17 @@
 
 # Output buffer size
 OUTPUT_BUFFER_SIZE = 100 * 1024 * 1024
+
+# Native program mappings
+NATIVE_PROGRAM_MAPPING = {
+    "11111111111111111111111111111111": "system",
+    "Config1111111111111111111111111111111111111": "config",
+    "ComputeBudget111111111111111111111111111111": "compute-budget",
+    "Stake11111111111111111111111111111111111111": "stake",
+    "Vote111111111111111111111111111111111111111": "vote",
+    "AddressLookupTab1e1111111111111111111111111": "address-lookup-table",
+    "BPFLoader1111111111111111111111111111111111": "bpf-loader-v1",
+    "BPFLoader2111111111111111111111111111111111": "bpf-loader-v2",
+    "BPFLoaderUpgradeab1e11111111111111111111111": "bpf-loader-upgradeable-v1",
+    "LoaderV411111111111111111111111111111111111": "bpf-loader-v4",
+}

src/test_suite/fixture_utils.py

@@ -1,57 +1,59 @@
+import fd58
 from test_suite.codec_utils import encode_input, encode_output
-from test_suite.multiprocessing_utils import prune_execution_result
+from test_suite.constants import NATIVE_PROGRAM_MAPPING
+from test_suite.multiprocessing_utils import (
+    build_test_results,
+    read_instr,
+    process_single_test_case,
+    prune_execution_result,
+)
 import test_suite.globals as globals
 import test_suite.invoke_pb2 as pb
 from google.protobuf import text_format
 from pathlib import Path
 
 
-def create_fixture(
-    file_serialized_instruction_context: tuple[str, dict],
-    file_serialized_instruction_effects: tuple[str, dict[str, str | None]],
-) -> tuple[str, str | None]:
+def create_fixture(test_file: Path) -> int:
     """
     Create instruction fixture for an instruction context and effects.
 
     Args:
-        - file_serialized_instruction_context (tuple[str, str]): Tuple of file stem and serialized instruction context.
-        - file_serialized_instruction_effects (tuple[str, dict[str, str | None]]): Tuple of file stem and dictionary of target library names and serialized instruction effects.
+        - test_file (Path): Path to the file containing serialized instruction contexts
 
     Returns:
-        - tuple[str, str | None]: Tuple of file stem and instruction fixture.
+        - int: 1 on success, 0 on failure
     """
+    serialized_instr_context = read_instr(test_file)
+    results = process_single_test_case(serialized_instr_context)
+    pruned_results = prune_execution_result(serialized_instr_context, results)
 
-    file_stem, serialized_instruction_context = file_serialized_instruction_context
-    file_stem_2, serialized_instruction_effects = file_serialized_instruction_effects
-
-    assert file_stem == file_stem_2, f"{file_stem} != {file_stem_2}"
-
-    # Both instruction context and instruction effects should not be None
-    if serialized_instruction_context is None or serialized_instruction_effects is None:
-        return file_stem, None
+    # This is only relevant when you gather results for multiple targets
+    if globals.only_keep_passing:
+        status, _ = build_test_results(pruned_results)
+        if status != 1:
+            return 0
 
-    _, targets_to_serialized_pruned_instruction_effects = prune_execution_result(
-        file_serialized_instruction_context, file_serialized_instruction_effects
-    )
+    if pruned_results is None:
+        return 0
 
-    pruned_instruction_effects = targets_to_serialized_pruned_instruction_effects[
-        globals.solana_shared_library
-    ]
+    serialized_instr_effects = pruned_results[globals.solana_shared_library]
 
-    if pruned_instruction_effects is None:
-        return file_stem, None
+    if serialized_instr_context is None or serialized_instr_effects is None:
+        return 0
 
     # Create instruction fixture
     instr_context = pb.InstrContext()
-    instr_context.ParseFromString(serialized_instruction_context)
+    instr_context.ParseFromString(serialized_instr_context)
     instr_effects = pb.InstrEffects()
-    instr_effects.ParseFromString(pruned_instruction_effects)
+    instr_effects.ParseFromString(serialized_instr_effects)
 
     fixture = pb.InstrFixture()
     fixture.input.MergeFrom(instr_context)
     fixture.output.MergeFrom(instr_effects)
 
-    return file_stem, fixture.SerializeToString(deterministic=True)
+    return write_fixture_to_disk(
+        test_file.stem, fixture.SerializeToString(deterministic=True)
+    )
 
 
 def write_fixture_to_disk(file_stem: str, serialized_instruction_fixture: str) -> int:
@@ -61,14 +63,22 @@ def write_fixture_to_disk(file_stem: str, serialized_instruction_fixture: str) -
 
     Args:
         - file_stem (str): File stem
-        - serialized_instruction_fixture (str): Serialized instruction fixture
 
     Returns:
         - int: 0 on failure, 1 on success
     """
     if serialized_instruction_fixture is None:
         return 0
 
+    output_dir = globals.output_dir
+
+    if globals.organize_fixture_dir:
+        instr_fixture = pb.InstrFixture()
+        instr_fixture.ParseFromString(serialized_instruction_fixture)
+        program_type = get_program_type(instr_fixture)
+        output_dir = output_dir / program_type
+        output_dir.mkdir(parents=True, exist_ok=True)
+
     if globals.readable:
         # Deserialize fixture
         instr_fixture = pb.InstrFixture()
@@ -86,12 +96,12 @@ def write_fixture_to_disk(file_stem: str, serialized_instruction_fixture: str) -
         instr_fixture.input.CopyFrom(instr_context)
         instr_fixture.output.CopyFrom(instr_effects)
 
-        with open(globals.output_dir / (file_stem + ".fix.txt"), "w") as f:
+        with open(output_dir / (file_stem + ".fix.txt"), "w") as f:
             f.write(
                 text_format.MessageToString(instr_fixture, print_unknown_fields=False)
             )
     else:
-        with open(f"{globals.output_dir}/{file_stem}.fix", "wb") as f:
+        with open(output_dir / (file_stem + ".fix"), "wb") as f:
             f.write(serialized_instruction_fixture)
 
     return 1
@@ -118,3 +128,30 @@ def extract_instr_context_from_fixture(fixture_file: Path):
         return 0
 
     return 1
+
+
+def get_program_type(instr_fixture: pb.InstrFixture) -> str:
+    """
+    Get the program type based on the program / loader id.
+
+    Args:
+        - fixture (pb.InstrFixture): Instruction fixture
+
+    Returns:
+        - str | None: Program type (unknown if not found)
+    """
+    # Check if the program type can be deduced from program_id
+    program_id = fd58.enc32(instr_fixture.input.program_id).decode()
+
+    program_type = NATIVE_PROGRAM_MAPPING.get(program_id, None)
+    if program_type:
+        return program_type
+
+    # Use the program_id owner instead (loader_id may not be reliable)
+    for account_state in instr_fixture.input.accounts:
+        if account_state.address == instr_fixture.input.program_id:
+            return NATIVE_PROGRAM_MAPPING.get(
+                fd58.enc32(account_state.owner).decode(), "unknown"
+            )
+
+    return "unknown"

src/test_suite/globals.py

@@ -21,3 +21,9 @@
 
 # (For fixtures) Whether to output in human-readable format
 readable = False
+
+# (For fixtures) Whether to organize fixtures by program type
+organize_fixture_dir = False
+
+# (For fixtures) Whether to only keep passing tests
+only_keep_passing = False

src/test_suite/minimize_utils.py

@@ -2,7 +2,7 @@
 import test_suite.invoke_pb2 as pb
 import test_suite.globals as globals
 from test_suite.multiprocessing_utils import (
-    generate_test_case,
+    read_instr,
     process_instruction,
 )
 
@@ -18,7 +18,7 @@ def minimize_single_test_case(test_file: Path) -> int:
     Returns:
         int: 0 on failure, 1 on success
     """
-    _, serialized_instruction_context = generate_test_case(test_file)
+    _, serialized_instruction_context = read_instr(test_file)
 
     # Skip if input is invalid
     if serialized_instruction_context is None: