From 71256801f492204bdf2f938bd60a4f84885f1a96 Mon Sep 17 00:00:00 2001 From: lsankar4033 Date: Thu, 13 Aug 2020 13:39:02 -0700 Subject: [PATCH] Setup ping test --- lib/fixtures.py | 4 --- lib/instance_configs.py | 1 + lib/runner.py | 32 ++++++++++++++++------ steth.py | 59 ++++++++++++++++++++--------------------- tests/reqresp/ping.py | 7 +++-- 5 files changed, 57 insertions(+), 46 deletions(-) diff --git a/lib/fixtures.py b/lib/fixtures.py index b2b50a9..f75e568 100644 --- a/lib/fixtures.py +++ b/lib/fixtures.py @@ -26,10 +26,6 @@ def extract_fixtures(clients_to_test=SUPPORTED_CLIENTS) -> List[Fixture]: def setup_fixture(fixture: Fixture): for instance in fixture.instances: - if instance.client not in SUPPORTED_CLIENTS: - raise ValueError( - f"can't start instance: client {instance_config.client} not supported yet") - start_instance(instance) diff --git a/lib/instance_configs.py b/lib/instance_configs.py index 46a84de..545a9d3 100644 --- a/lib/instance_configs.py +++ b/lib/instance_configs.py @@ -3,6 +3,7 @@ DEFAULT_BEACON_STATE_PATH = 'ssz/default.ssz' DEFAULT_ENR = { 'enr': 'enr:-LK4QJCIZoViytOOmAzAbdOJODwQ36PhjXwvXlTFTloTzpawVpvPRmtrM6UZdPmOGck5yPZ9AbgmwyZnE3jm4jX0Yx0Bh2F0dG5ldHOIAAAAAAAAAACEZXRoMpBGMkSJAAAAAf__________gmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQOnBq2PcxFfkFACZvJz91cd-UKaTPtLv7zYJSJyAtq60YN0Y3CCIyiDdWRwgiMp', + 'enr_teku': 'enr:-KG4QF9w4w5DORl2-AtVaqK6n-VF3e-2p5fm4uqDLwlR8cQ4MVk2yJ68YkEUTASbbJJ123CKYDDY1KdfBWOuVrmsZtUChGV0aDKQGK5MywAAAAH__________4JpZIJ2NIJpcIR_AAABiXNlY3AyNTZrMaEDpwatj3MRX5BQAmbyc_dXHflCmkz7S7-82CUicgLautGDdGNwgiMog3VkcIIjKA', 'private_key': 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', 'udp': 9001, 'tcp': 9000, diff --git a/lib/runner.py b/lib/runner.py index 6bc1bb2..9d3d3c3 100644 --- a/lib/runner.py +++ b/lib/runner.py @@ -6,7 +6,6 @@ from colored import fg from lib.console import ConsoleWriter -from lib.instance_configs import DEFAULT_ARGS TESTS_DIR = 'tests' @@ -42,12 +41,29 @@ def run_module(module_str, args): return (return_code, logs) -def test_matches_filter(test, test_filter): - return test_filter is None or test == test_filter +def file_matches_filter(file, file_filter): + return file_filter is None or file == file_filter -def run_all_tests(cw=ConsoleWriter(None, None), test_filter=None): - for test_file in all_test_files(): - if test_matches_filter(test_file, test_filter): - cw = cw._replace(test=file_to_module(test_file)) +def run_test_files(fixture_name, files, args): + print(fixture_name) - run_module(file_to_module(test_file), DEFAULT_ARGS) + failures = {} + for file in files: + module = file_to_module(file) + (return_code, logs) = run_module(module, {**args, 'client': fixture_name}) + + print(f'\t{module} {return_code_to_status(return_code)}') + if return_code != 0: + failures[module] = (return_code, logs) + + if len(failures) > 0: + for module, (return_code, logs) in failures.items(): + print('') + print(f'\t{module} {return_code_to_status(return_code)}') + + for log in logs: + print(f'\t{log}') + + return 1 + + return 0 diff --git a/steth.py b/steth.py index 792f4c1..43c5548 100755 --- a/steth.py +++ b/steth.py @@ -5,13 +5,13 @@ import yaml from sclients import SUPPORTED_CLIENTS, stop_instance -from lib.console import ConsoleWriter from lib.fixtures import extract_fixtures, setup_fixture, teardown_fixture -from lib.runner import run_all_tests -from lib.logging_tests import run_logging_tests +from lib.instance_configs import DEFAULT_ARGS +from lib.runner import run_test_files, all_test_files, file_matches_filter +from lib.logging_tests import all_logging_test_files -def run_start_fixture(args): +def run_start_fixture_cmd(args): client = args.client cw = ConsoleWriter(None, None) @@ -23,49 +23,48 @@ def run_start_fixture(args): setup_fixture(fixture) -def run_stop_fixture(args): +def run_stop_fixture_cmd(args): cw = ConsoleWriter(None, None) for client in SUPPORTED_CLIENTS: cw.info(f'stopping client {client}') stop_instance(client) -def run_test(args): +def run_test_cmd(args): clients = SUPPORTED_CLIENTS if args.client is None else [args.client] reuse_clients = args.reuse - test_filter = args.only - - cw = ConsoleWriter(None, None) + file_filter = args.only + test_files = [file for file in all_test_files() if file_matches_filter(file, file_filter)] fixtures = extract_fixtures(clients) + failed = False for fixture in fixtures: - cw = cw._replace(fixture=fixture.name) - - if reuse_clients: - run_all_tests(cw, test_filter) + if not reuse_clients: + setup_fixture(fixture) - else: - try: - cw.info('setting up fixture') - setup_fixture(fixture) + try: + return_code = run_test_files(fixture.name, test_files, DEFAULT_ARGS) + if return_code != 0: + failed = True - except ValueError as e: - cw.info(f'skipping fixture: {e}') - continue - - try: - run_all_tests(cw, test_filter) + except Exception as e: + print(f'Exception while running tests: {e}') + failed = True - finally: - cw.info('tearing down fixture') + finally: + if not reuse_clients: teardown_fixture(fixture) + if failed: + exit(1) def run_logging_test(args): + test_files = all_logging_test_files() + failed = False for client in SUPPORTED_CLIENTS: - return_code= run_logging_tests(client) + return_code= run_test_files(client, test_files, {}) if return_code != 0: failed = True @@ -82,17 +81,17 @@ def run_logging_test(args): start = fixture_sub.add_parser('start') start.add_argument( 'client', help=f'client to start. can only start 1 at a time until dynamic ENRs implemented. possible values: {SUPPORTED_CLIENTS}') - start.set_defaults(func=run_start_fixture) + start.set_defaults(func=run_start_fixture_cmd) stop = fixture_sub.add_parser('stop') - stop.set_defaults(func=run_stop_fixture) + stop.set_defaults(func=run_stop_fixture_cmd) test = steth_sub.add_parser('test', help='Run stethoscope unit tests') test.add_argument('-c', '--client', - help='run test(s) for a specific client. possible values: {SUPPORTED_CLIENTS}') + help=f'run test(s) for a specific client. possible values: {SUPPORTED_CLIENTS}') test.add_argument('-o', '--only', help='run specific tests by name') test.add_argument('-r', '--reuse', default=False, action='store_true', help='reuse running fixtures') - test.set_defaults(func=run_test) + test.set_defaults(func=run_test_cmd) test = steth_sub.add_parser('logging_test', help='Display example logs from ./steth.py test. Useful for testing CI integration') test.set_defaults(func=run_logging_test) diff --git a/tests/reqresp/ping.py b/tests/reqresp/ping.py index 855fa60..28e1727 100644 --- a/tests/reqresp/ping.py +++ b/tests/reqresp/ping.py @@ -1,6 +1,4 @@ -import trio from eth2spec.utils.ssz.ssz_typing import uint64 -from pyrum import SubprocessConn, Rumor from sclients import connect_rumor from ..utils import parse_response, with_rumor @@ -8,7 +6,7 @@ @with_rumor async def run(rumor, args): - peer_id = await connect_rumor(rumor, args['enr']) + peer_id = await connect_rumor(rumor, args['client'], args['enr']) return_code = 0 for i in range(5): @@ -22,6 +20,7 @@ async def run(rumor, args): pong = uint64.decode_bytes(bytes.fromhex(resp_data)) if not isinstance(pong, int) or pong < 0: + # TODO: add logs return_code = 1 - return return_code + return (return_code, [])