diff --git a/commands.md b/commands.md index ff565cf..017a107 100644 --- a/commands.md +++ b/commands.md @@ -24,6 +24,7 @@ $ solana-test-suite [OPTIONS] COMMAND [ARGS]... * `exec-instr`: Execute InstrContext message(s) and print... * `instr-from-fixtures`: Extract InstrContext messages from fixtures. * `list-harness-types`: List harness types available for use. +* `regenerate-all-fixtures`: Regenerate all fixtures in provided... * `regenerate-fixtures`: Regenerate InstrFixture messages by... * `run-tests`: Run tests on a set of targets with a... @@ -43,7 +44,7 @@ $ solana-test-suite create-fixtures [OPTIONS] **Options**: * `-i, --input-dir PATH`: Either a file or directory containing InstrContext messages [default: corpus8] -* `-s, --solana-target PATH`: Solana (or ground truth) shared object (.so) target file path [default: impl/lib/libsolfuzz_agave_v2.0.so] +* `-s, --solana-target PATH`: Solana (or ground truth) shared object (.so) target file path [default: /home/kbhargava/repos/solfuzz-agave/target/release/libsolfuzz_agave.so] * `-t, --target PATH`: Shared object (.so) target file paths (pairs with --keep-passing). Targets must have sol_compat_instr_execute_v1 defined * `-o, --output-dir PATH`: Output directory for fixtures [default: test_fixtures] * `-p, --num-processes INTEGER`: Number of processes to use [default: 4] @@ -81,12 +82,12 @@ $ solana-test-suite debug-mismatches [OPTIONS] **Options**: -* `-s, --solana-target PATH`: Solana (or ground truth) shared object (.so) target file path [default: impl/lib/libsolfuzz_agave_v2.0.so] -* `-t, --target PATH`: Shared object (.so) target file paths (pairs with --keep-passing). Targets must have sol_compat_instr_execute_v1 defined [default: impl/lib/libsolfuzz_firedancer.so] +* `-s, --solana-target PATH`: Solana (or ground truth) shared object (.so) target file path [default: /home/kbhargava/repos/solfuzz-agave/target/release/libsolfuzz_agave.so] +* `-t, --target PATH`: Shared object (.so) target file paths (pairs with --keep-passing). Targets must have sol_compat_instr_execute_v1 defined [default: /home/kbhargava/repos/firedancer/build/native/gcc/lib/libfd_exec_sol_compat.so] * `-o, --output-dir PATH`: Output directory for InstrContext messages [default: debug_mismatch] * `-u, --repro-urls TEXT`: Comma-delimited list of FuzzCorp mismatch links * `-s, --section-names TEXT`: Comma-delimited list of FuzzCorp section names -* `-f, --fuzzcorp-url TEXT`: Comma-delimited list of FuzzCorp section names +* `-f, --fuzzcorp-url TEXT`: Comma-delimited list of FuzzCorp section names [default: https://api.dev.fuzzcorp.asymmetric.re/uglyweb/firedancer-io/solfuzz/bugs/] * `--help`: Show this message and exit. ## `solana-test-suite debug-non-repros` @@ -103,12 +104,12 @@ $ solana-test-suite debug-non-repros [OPTIONS] **Options**: -* `-s, --solana-target PATH`: Solana (or ground truth) shared object (.so) target file path [default: impl/lib/libsolfuzz_agave_v2.0.so] -* `-t, --target PATH`: Shared object (.so) target file paths (pairs with --keep-passing). Targets must have sol_compat_instr_execute_v1 defined [default: impl/lib/libsolfuzz_firedancer.so] +* `-s, --solana-target PATH`: Solana (or ground truth) shared object (.so) target file path [default: /home/kbhargava/repos/solfuzz-agave/target/release/libsolfuzz_agave.so] +* `-t, --target PATH`: Shared object (.so) target file paths (pairs with --keep-passing). Targets must have sol_compat_instr_execute_v1 defined [default: /home/kbhargava/repos/firedancer/build/native/gcc/lib/libfd_exec_sol_compat.so] * `-o, --output-dir PATH`: Output directory for InstrContext messages [default: debug_mismatch] * `-u, --repro-urls TEXT`: Comma-delimited list of FuzzCorp mismatch links * `-s, --section-names TEXT`: Comma-delimited list of FuzzCorp section names -* `-f, --fuzzcorp-url TEXT`: Comma-delimited list of FuzzCorp section names +* `-f, --fuzzcorp-url TEXT`: Comma-delimited list of FuzzCorp section names [default: https://api.dev.fuzzcorp.asymmetric.re/uglyweb/firedancer-io/solfuzz/bugs/] * `--help`: Show this message and exit. ## `solana-test-suite decode-protobuf` @@ -176,6 +177,24 @@ $ solana-test-suite list-harness-types [OPTIONS] * `--help`: Show this message and exit. +## `solana-test-suite regenerate-all-fixtures` + +Regenerate all fixtures in provided test-vectors folder + +**Usage**: + +```console +$ solana-test-suite regenerate-all-fixtures [OPTIONS] +``` + +**Options**: + +* `-i, --input-dir PATH`: Input test-vectors directory [default: corpus8] +* `-o, --output-dir PATH`: Output directory for regenerated fixtures [default: /tmp/regenerated_fixtures] +* `-t, --target PATH`: Shared object (.so) target file path to execute [default: /home/kbhargava/repos/solfuzz-agave/target/release/libsolfuzz_agave.so] +* `-s, --stubbed-target PATH`: Stubbed shared object (.so) target file path to execute [default: /home/kbhargava/repos/solfuzz-agave/target/x86_64-unknown-linux-gnu/release/libsolfuzz_agave_stubbed.so] +* `--help`: Show this message and exit. + ## `solana-test-suite regenerate-fixtures` Regenerate InstrFixture messages by @@ -190,7 +209,7 @@ $ solana-test-suite regenerate-fixtures [OPTIONS] **Options**: * `-i, --input-dir PATH`: Either a file or directory containing InstrFixture messages [default: corpus8] -* `-t, --target PATH`: Shared object (.so) target file path to execute [default: impl/lib/libsolfuzz_firedancer.so] +* `-t, --target PATH`: Shared object (.so) target file path to execute [default: /home/kbhargava/repos/solfuzz-agave/target/release/libsolfuzz_agave.so] * `-o, --output-dir PATH`: Output directory for regenerated fixtures [default: regenerated_fixtures] * `-d, --dry-run`: Only print the fixtures that would be regenerated * `-a, --all-fixtures`: Regenerate all fixtures, regardless of FeatureSet compatibility. Will apply minimum compatible features. @@ -212,8 +231,8 @@ $ solana-test-suite run-tests [OPTIONS] **Options**: * `-i, --input PATH`: Single input file or input directory containing InstrContext or InstrFixture messages [default: corpus8] -* `-s, --solana-target PATH`: Solana (or ground truth) shared object (.so) target file path [default: impl/lib/libsolfuzz_agave_v2.0.so] -* `-t, --target PATH`: Shared object (.so) target file paths [default: impl/lib/libsolfuzz_firedancer.so] +* `-s, --solana-target PATH`: Solana (or ground truth) shared object (.so) target file path [default: /home/kbhargava/repos/solfuzz-agave/target/release/libsolfuzz_agave.so] +* `-t, --target PATH`: Shared object (.so) target file paths [default: /home/kbhargava/repos/firedancer/build/native/gcc/lib/libfd_exec_sol_compat.so] * `-o, --output-dir PATH`: Output directory for test results [default: test_results] * `-p, --num-processes INTEGER`: Number of processes to use [default: 4] * `-r, --randomize-output-buffer`: Randomizes bytes in output buffer before shared library execution diff --git a/src/test_suite/test_suite.py b/src/test_suite/test_suite.py index da05e33..f5590c5 100644 --- a/src/test_suite/test_suite.py +++ b/src/test_suite/test_suite.py @@ -3,6 +3,7 @@ import typer from collections import defaultdict import ctypes +from glob import glob from multiprocessing import Pool from pathlib import Path import subprocess @@ -803,7 +804,7 @@ def regenerate_fixtures( help=f"Either a file or directory containing {globals.harness_ctx.fixture_type.__name__} messages", ), shared_library: Path = typer.Option( - Path(os.getenv("FIREDANCER_TARGET", "impl/lib/libsolfuzz_firedancer.so")), + Path(os.getenv("SOLFUZZ_TARGET", "impl/lib/libsolfuzz_agave_v2.0.so")), "--target", "-t", help="Shared object (.so) target file path to execute", @@ -885,5 +886,105 @@ def regenerate_fixtures( print(f"Regenerated {num_regenerated} fixtures") +@app.command( + help=f""" + Regenerate all fixtures in provided test-vectors folder + """ +) +def regenerate_all_fixtures( + test_vectors: Path = typer.Option( + Path("corpus8"), + "--input-dir", + "-i", + help=f"Input test-vectors directory", + ), + output_dir: Path = typer.Option( + Path("/tmp/regenerated_fixtures"), + "--output-dir", + "-o", + help="Output directory for regenerated fixtures", + ), + shared_library: Path = typer.Option( + Path(os.getenv("SOLFUZZ_TARGET", "impl/lib/libsolfuzz_agave_v2.0.so")), + "--target", + "-t", + help="Shared object (.so) target file path to execute", + ), + stubbed_shared_library: Path = typer.Option( + Path(os.getenv("SOLFUZZ_STUBBED_TARGET", "impl/lib/libsolfuzz_firedancer.so")), + "--stubbed-target", + "-s", + help="Stubbed shared object (.so) target file path to execute", + ), +): + globals.output_dir = output_dir + + if output_dir.exists(): + shutil.rmtree(globals.output_dir) + globals.output_dir.mkdir(parents=True, exist_ok=True) + + def copy_files_excluding_fixture_files(src, dst): + regenerate_folders = set() + fixtures_folders = glob(str(src) + "/*/fixtures*") + for root, dirs, files in os.walk(src): + src_dir = os.path.join(src, os.path.relpath(root, src)) + dest_dir = os.path.join(dst, os.path.relpath(root, src)) + + if not os.path.exists(dest_dir): + os.makedirs(dest_dir) + + if not any( + root.startswith(fixture_folder) for fixture_folder in fixtures_folders + ): + for file in files: + src_file = os.path.join(root, file) + dst_file = os.path.join(dest_dir, file) + shutil.copy2(src_file, dst_file) + else: + if files: + regenerate_folders.add(src_dir) + return regenerate_folders + + def get_harness_type_for_folder(src, regenerate_folder): + relative_path = os.path.relpath(regenerate_folder, src) + fuzz_folder = relative_path.split(os.sep)[0] + capitalized_name = "".join(word.capitalize() for word in fuzz_folder.split("_")) + harness_name = capitalized_name + "Harness" + return harness_name + + regenerate_folders = copy_files_excluding_fixture_files(test_vectors, output_dir) + for source_folder in regenerate_folders: + globals.target_libraries = {} + output_folder = os.path.join( + output_dir, os.path.relpath(source_folder, test_vectors) + ) + folder_harness_type = get_harness_type_for_folder(test_vectors, source_folder) + os.environ["HARNESS_TYPE"] = folder_harness_type + globals.harness_ctx = eval(folder_harness_type) + print( + f"Regenerating fixtures for {source_folder} with harness type {folder_harness_type}" + ) + if folder_harness_type in ["CpiHarness"]: + regenerate_fixtures( + input_path=Path(source_folder), + shared_library=stubbed_shared_library, + output_dir=Path(output_folder), + dry_run=False, + all_fixtures=True, + ) + elif folder_harness_type in ["ElfLoaderHarness"]: + shutil.copytree(source_folder, output_folder, dirs_exist_ok=True) + else: + regenerate_fixtures( + input_path=Path(source_folder), + shared_library=shared_library, + output_dir=Path(output_folder), + dry_run=False, + all_fixtures=True, + ) + + print(f"Regenerated fixtures from {test_vectors} to {output_dir}") + + if __name__ == "__main__": app()