diff --git a/commands.md b/commands.md index 973b0cf..2bbf314 100644 --- a/commands.md +++ b/commands.md @@ -69,11 +69,13 @@ $ solana-test-suite debug-mismatches [OPTIONS] **Options**: * `-s, --solana-target PATH`: Solana (or ground truth) shared object (.so) target file path [default: impl/lib/libsolfuzz_agave_v2.0.so] +* `-h, --default-harness-type TEXT`: Harness type to use for Context protobufs [default: InstrHarness] * `-t, --target PATH`: Shared object (.so) target file paths (pairs with --keep-passing). Targets must have required function entrypoints defined [default: impl/lib/libsolfuzz_firedancer.so] * `-o, --output-dir PATH`: Output directory for messages [default: debug_mismatch] * `-u, --repro-urls TEXT`: Comma-delimited list of FuzzCorp mismatch links * `-s, --section-names TEXT`: Comma-delimited list of FuzzCorp section names * `-f, --fuzzcorp-url TEXT`: Comma-delimited list of FuzzCorp section names [default: https://api.dev.fuzzcorp.asymmetric.re/uglyweb/firedancer-io/solfuzz/bugs/] +* `-l, --log-level INTEGER`: FD logging level [default: 5] * `--help`: Show this message and exit. ## `solana-test-suite debug-non-repros` @@ -96,6 +98,7 @@ $ solana-test-suite debug-non-repros [OPTIONS] * `-u, --repro-urls TEXT`: Comma-delimited list of FuzzCorp mismatch links * `-s, --section-names TEXT`: Comma-delimited list of FuzzCorp section names * `-f, --fuzzcorp-url TEXT`: Comma-delimited list of FuzzCorp section names [default: https://api.dev.fuzzcorp.asymmetric.re/uglyweb/firedancer-io/solfuzz/bugs/] +* `-l, --log-level INTEGER`: FD logging level [default: 5] * `--help`: Show this message and exit. ## `solana-test-suite decode-protobufs` diff --git a/src/test_suite/multiprocessing_utils.py b/src/test_suite/multiprocessing_utils.py index 96cfed3..72d7dcc 100644 --- a/src/test_suite/multiprocessing_utils.py +++ b/src/test_suite/multiprocessing_utils.py @@ -14,7 +14,7 @@ def process_target( - harness_ctx: HarnessCtx, library: ctypes.CDLL, serialized_instruction_context: str + harness_ctx: HarnessCtx, library: ctypes.CDLL, context: ContextType ) -> invoke_pb.InstrEffects | None: """ Process an instruction through a provided shared library and return the result. @@ -26,6 +26,11 @@ def process_target( Returns: - invoke_pb.InstrEffects | None: Result of instruction execution. """ + + serialized_instruction_context = context.SerializeToString(deterministic=True) + if serialized_instruction_context is None: + return None + # Prepare input data and output buffers in_data = serialized_instruction_context in_ptr = (ctypes.c_uint8 * len(in_data))(*in_data) @@ -60,22 +65,6 @@ def process_target( return output_object -def read_context_serialized(harness_ctx: HarnessCtx, test_file: Path) -> str | None: - """ - Reads in test files and generates a serialized Context Protobuf message for a test case. - - Args: - - test_file (Path): Path to the instruction context message. - - Returns: - - str | None: Serialized instruction context, or None if reading failed. - """ - - # Serialize instruction context to string (pickleable) - ctx = read_context(harness_ctx, test_file) - return ctx.SerializeToString(deterministic=True) if ctx else None - - def extract_metadata(fixture_file: Path) -> str | None: """ Extracts metadata from a fixture file. @@ -143,25 +132,6 @@ def read_context(harness_ctx: HarnessCtx, test_file: Path) -> message.Message | return context -def read_fixture_serialized(fixture_file: Path) -> str | None: - """ - Same as read_instr, but for InstrFixture protobuf messages. - - DOES NOT SUPPORT HUMAN READABLE MESSAGES!!! - - Args: - - fixture_file (Path): Path to the instruction fixture message. - - Returns: - - str | None: Serialized instruction fixture, or None if reading failed. - """ - fixture = read_fixture(fixture_file) - if fixture is None: - return None - # Serialize instruction fixture to string (pickleable) - return fixture.SerializeToString(deterministic=True) - - def read_fixture(fixture_file: Path) -> message.Message | None: """ Reads in test files and generates an Fixture Protobuf object for a test case. @@ -209,10 +179,12 @@ def decode_single_test_case(test_file: Path) -> int: if test_file.suffix == ".fix": fn_entrypoint = extract_metadata(test_file).fn_entrypoint harness_ctx = ENTRYPOINT_HARNESS_MAP[fn_entrypoint] - serialized_protobuf = read_fixture_serialized(test_file) + fixture = read_fixture(test_file) + serialized_protobuf = fixture.SerializeToString(deterministic=True) else: harness_ctx = globals.default_harness_ctx - serialized_protobuf = read_context_serialized(harness_ctx, test_file) + context = read_context(harness_ctx, test_file) + serialized_protobuf = context.SerializeToString(deterministic=True) # Skip if input is invalid if serialized_protobuf is None: @@ -252,9 +224,6 @@ def process_single_test_case( - dict[str, str | None] | None: Dictionary of target library names and instruction effects. """ # Mark as skipped if instruction context doesn't exist - serialized_instruction_context = context.SerializeToString(deterministic=True) - if serialized_instruction_context is None: - return None # Execute test case on each target library results = {} @@ -262,7 +231,7 @@ def process_single_test_case( instruction_effects = process_target( harness_ctx, globals.target_libraries[target], - serialized_instruction_context, + context, ) result = ( instruction_effects.SerializeToString(deterministic=True) @@ -378,18 +347,6 @@ def initialize_process_output_buffers(randomize_output_buffer=False): ) -def serialize_context(harness_ctx: HarnessCtx, file: Path) -> str | None: - if file.suffix == ".fix": - fixture = harness_ctx.fixture_type() - fixture.ParseFromString(file.open("rb").read()) - serialized_instr_context = fixture.input.SerializeToString(deterministic=True) - else: - serialized_instr_context = read_context_serialized(harness_ctx, file) - - assert serialized_instr_context is not None, f"Unable to read {file.name}" - return serialized_instr_context - - def run_test(test_file: Path) -> tuple[str, int, dict | None]: """ Runs a single test from start to finish. diff --git a/src/test_suite/test_suite.py b/src/test_suite/test_suite.py index 856835b..dcd85e5 100644 --- a/src/test_suite/test_suite.py +++ b/src/test_suite/test_suite.py @@ -22,7 +22,6 @@ process_target, run_test, read_context, - serialize_context, ) import test_suite.globals as globals from test_suite.util import set_ld_preload_asan @@ -98,13 +97,14 @@ def execute( if file.suffix == ".fix": fn_entrypoint = extract_metadata(file).fn_entrypoint harness_ctx = ENTRYPOINT_HARNESS_MAP[fn_entrypoint] + context = read_fixture(file).input else: harness_ctx = HARNESS_MAP[default_harness_ctx] + context = read_context(harness_ctx, file) # Execute and cleanup - context = read_context(harness_ctx, file) start = time.time() - effects = process_target(harness_ctx, lib, serialize_context(harness_ctx, file)) + effects = process_target(harness_ctx, lib, context) end = time.time() print(f"Total time taken for {file}: {(end - start) * 1000} ms\n------------") @@ -259,6 +259,7 @@ def create_fixtures( test_cases = [input] if input.is_file() else list(input.iterdir()) num_test_cases = len(test_cases) + globals.default_harness_ctx = HARNESS_MAP[default_harness_ctx] # Generate the test cases in parallel from files on disk @@ -453,7 +454,7 @@ def run_tests( failed += 1 failed_tests.append(file_stem) if save_failures: - failed_protobufs = list(file_or_dir.glob(f"{file_stem}*")) + failed_protobufs = list(input.glob(f"{file_stem}*")) for failed_protobuf in failed_protobufs: shutil.copy(failed_protobuf, failed_protobufs_dir) @@ -545,6 +546,12 @@ def debug_mismatches( "-s", help="Solana (or ground truth) shared object (.so) target file path", ), + default_harness_ctx: str = typer.Option( + "InstrHarness", + "--default-harness-type", + "-h", + help=f"Harness type to use for Context protobufs", + ), shared_libraries: List[Path] = typer.Option( [Path(os.getenv("FIREDANCER_TARGET", "impl/lib/libsolfuzz_firedancer.so"))], "--target", @@ -573,7 +580,25 @@ def debug_mismatches( "-f", help="Comma-delimited list of FuzzCorp section names", ), + log_level: int = typer.Option( + 5, + "--log-level", + "-l", + help="FD logging level", + ), ): + globals.output_dir = output_dir + + if globals.output_dir.exists(): + shutil.rmtree(globals.output_dir) + globals.output_dir.mkdir(parents=True, exist_ok=True) + + globals.inputs_dir = globals.output_dir / "inputs" + + if globals.inputs_dir.exists(): + shutil.rmtree(globals.inputs_dir) + globals.inputs_dir.mkdir(parents=True, exist_ok=True) + fuzzcorp_cookie = os.getenv("FUZZCORP_COOKIE") repro_urls_list = repro_urls.split(",") if repro_urls else [] section_names_list = section_names.split(",") if section_names else [] @@ -621,18 +646,6 @@ def debug_mismatches( ].strip() custom_data_urls.append(custom_url) - globals.output_dir = output_dir - - if globals.output_dir.exists(): - shutil.rmtree(globals.output_dir) - globals.output_dir.mkdir(parents=True, exist_ok=True) - - globals.inputs_dir = globals.output_dir / "inputs" - - if globals.inputs_dir.exists(): - shutil.rmtree(globals.inputs_dir) - globals.inputs_dir.mkdir(parents=True, exist_ok=True) - for url in custom_data_urls: zip_name = url.split("/")[-1] result = subprocess.run( @@ -648,15 +661,16 @@ def debug_mismatches( ) result = subprocess.run( - f"mv {globals.output_dir}/repro_custom/*ctx {globals.inputs_dir}", + f"mv {globals.output_dir}/repro_custom/*.fix {globals.inputs_dir}", shell=True, capture_output=True, text=True, ) run_tests( - file_or_dir=globals.inputs_dir, + input=globals.inputs_dir, reference_shared_library=reference_shared_library, + default_harness_ctx=default_harness_ctx, shared_libraries=shared_libraries, output_dir=globals.output_dir / "test_results", num_processes=4, @@ -666,6 +680,7 @@ def debug_mismatches( consensus_mode=False, failures_only=False, save_failures=True, + log_level=log_level, ) @@ -711,6 +726,12 @@ def debug_non_repros( "-f", help="Comma-delimited list of FuzzCorp section names", ), + log_level: int = typer.Option( + 5, + "--log-level", + "-l", + help="FD logging level", + ), ): fuzzcorp_cookie = os.getenv("FUZZCORP_COOKIE") repro_urls_list = repro_urls.split(",") if repro_urls else [] @@ -775,7 +796,7 @@ def debug_non_repros( ) run_tests( - file_or_dir=globals.inputs_dir, + input=globals.inputs_dir, reference_shared_library=reference_shared_library, shared_libraries=shared_libraries, output_dir=globals.output_dir / "test_results", @@ -786,6 +807,7 @@ def debug_non_repros( consensus_mode=False, failures_only=False, save_failures=True, + log_level=log_level, ) @@ -795,7 +817,7 @@ def debug_non_repros( """ ) def regenerate_fixtures( - input_path: Path = typer.Option( + input: Path = typer.Option( Path("corpus8"), "--input", "-i", @@ -844,7 +866,7 @@ def regenerate_fixtures( globals.target_libraries[shared_library] = lib initialize_process_output_buffers() - test_cases = list(input_path.iterdir()) if input_path.is_dir() else [input_path] + test_cases = list(input.iterdir()) if input.is_dir() else [input] num_regenerated = 0 for file in test_cases: @@ -971,7 +993,7 @@ def get_harness_type_for_folder(src, regenerate_folder): ) if folder_harness_type in ["CpiHarness"]: regenerate_fixtures( - input_path=Path(source_folder), + input=Path(source_folder), shared_library=stubbed_shared_library, output_dir=Path(output_folder), dry_run=False, @@ -981,7 +1003,7 @@ def get_harness_type_for_folder(src, regenerate_folder): shutil.copytree(source_folder, output_folder, dirs_exist_ok=True) else: regenerate_fixtures( - input_path=Path(source_folder), + input=Path(source_folder), shared_library=shared_library, output_dir=Path(output_folder), dry_run=False,