From b889ff1ef59110e70f7a935e7aa96d81cb722019 Mon Sep 17 00:00:00 2001 From: lsankar4033 Date: Wed, 5 Aug 2020 18:26:56 -0700 Subject: [PATCH] Surgically remove the notion of suites --- lib/console.py | 12 +++---- lib/fixtures.py | 56 +++++++++++++++--------------- lib/runner.py | 75 +++++++++++++++++------------------------ scripts/reqresp/ping.py | 1 - scripts/utils.py | 2 +- steth.py | 31 +++++------------ 6 files changed, 72 insertions(+), 105 deletions(-) diff --git a/lib/console.py b/lib/console.py index 08a17c9..97346c3 100644 --- a/lib/console.py +++ b/lib/console.py @@ -3,25 +3,21 @@ class ConsoleWriter(NamedTuple): - suite: str fixture: str test: str def _prefix(self): - if self.suite is None and self.fixture is None and self.test is None: + if self.fixture is None and self.test is None: return '' - elif self.fixture is None and self.test is None: - return f'{self.suite}' - elif self.fixture is None: - return f'{self.suite} -- {self.test}' + return f'{self.test}' elif self.test is None: - return f'{self.suite}[{self.fixture}]' + return f'{self.fixture}' else: - return f'{self.suite}[{self.fixture}] -- {self.test}' + return f'{self.fixture} -- {self.test}' def info(self, s): print(f'{self._prefix()} {s}') diff --git a/lib/fixtures.py b/lib/fixtures.py index e9946fb..0a314f1 100644 --- a/lib/fixtures.py +++ b/lib/fixtures.py @@ -7,33 +7,35 @@ from lib.types import Fixture - -def extract_fixtures(config, clients_to_test=SUPPORTED_CLIENTS) -> List[Fixture]: - # turn a yml config into a list of fixtures. if any 'all' in the file, do 1:1 for every one (rather than all - # pairwise combos) - if any((i['client'] == 'all' for i in config['instances'])): - fixtures = [] - - for client in SUPPORTED_CLIENTS: - if client not in clients_to_test: - continue - - # convert 'all' instances to $client - def convert(ic): return ic._replace(client=client) if ic.client == 'all' else ic - - fixture = Fixture(client, [convert(InstanceConfig.from_dict(i)) for i in config['instances']]) - fixtures.append(fixture) - - return fixtures - - # if any clients in the fixture are not among those specified in clients_to_test, return no fixtures - elif any((i['client'] not in clients_to_test for i in config['instances'])): - return [] - - else: - fixture = Fixture(None, [InstanceConfig.from_dict(i) for i in config['instances']]) - return [fixture] - +DEFAULT_INSTANCE_CONFIG = { + 'beacon_state_path': 'ssz/single_client_genesis.ssz', + 'enr': { + 'enr': 'enr:-LK4QJCIZoViytOOmAzAbdOJODwQ36PhjXwvXlTFTloTzpawVpvPRmtrM6UZdPmOGck5yPZ9AbgmwyZnE3jm4jX0Yx0Bh2F0dG5ldHOIAAAAAAAAAACEZXRoMpBGMkSJAAAAAf__________gmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQOnBq2PcxFfkFACZvJz91cd-UKaTPtLv7zYJSJyAtq60YN0Y3CCIyiDdWRwgiMp', + 'private_key': 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', + 'udp': 9001, + 'tcp': 9000, + 'id': 'v4', + 'ip': '127.0.0.1', + 'attnets': '0x0000000000000000', + 'eth2': '0x4632448900000001ffffffffffffffff' + } +} + +def extract_fixtures(clients_to_test=SUPPORTED_CLIENTS) -> List[Fixture]: + fixtures = [] + for client in SUPPORTED_CLIENTS: + if client not in clients_to_test: + continue + + instance_config = { + **DEFAULT_INSTANCE_CONFIG, + 'client': client + } + + fixture = Fixture(client, [InstanceConfig.from_dict(instance_config)]) + fixtures.append(fixture) + + return fixtures def setup_fixture(fixture: Fixture): for instance in fixture.instances: diff --git a/lib/runner.py b/lib/runner.py index d454026..8232341 100644 --- a/lib/runner.py +++ b/lib/runner.py @@ -1,66 +1,51 @@ import importlib +import os import subprocess import trio from lib.console import ConsoleWriter +TESTS_DIR = 'scripts' -def script_to_module(script): - return script.replace('/', '.')[0:-3] - - -def script_to_test(script): - remove_prefix_suffix = script[8:len(script) - 3] - return remove_prefix_suffix.replace('/', '.') - - -def run_script(script, args, cw): - script_list = ['python3', '-m', script_to_module(script)] - arg_list = [item for sublist in [['--' + k, v] for k, v in args.items()] for item in sublist] +def all_test_files(): + tests = [] + for root, dirs, files in os.walk(TESTS_DIR): + # skip files in top-level + if root == 'scripts': + continue - output = subprocess.run( - script_list + arg_list, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True - ) + tests.extend([f'{root}/{f}' for f in files if f.endswith('.py')]) - if len(output.stderr) > 0: - cw.fail('FAILED') - cw.info(output.stderr) + return tests - else: - cw.success('SUCCESS') +def file_to_module(script): + return script.replace('/', '.')[0:-3] -def run_module(module, args, cw): - module = importlib.import_module(module) +# TODO: add args back in +def run_module(module, cw): + print(f'Running module: {module}') + #module = importlib.import_module(module) - if not hasattr(module, 'run'): - cw.fail(f'module {module} does not have a run method') - return + #if not hasattr(module, 'run'): + #cw.fail(f'module {module} does not have a run method') + #return - return_code = trio.run(module.run, args) - if return_code == 0: - cw.success('SUCCESS') + #return_code = trio.run(module.run, args) + #if return_code == 0: + #cw.success('SUCCESS') - else: - cw.fail('FAILED') + #else: + #cw.fail('FAILED') def test_matches_filter(test, test_filter): - return test_filter is None or test == test_filter or test.startswith(test_filter) - - -def run_test_config(test_config, cw=ConsoleWriter(None, None, None), test_filter=None): - tests = test_config['test_suite'] - - for test in tests: - if test_matches_filter(test['name'], test_filter): - script = test['script'] - args = test['args'] + return test_filter is None or test == test_filter - cw = cw._replace(test=script_to_test(script)) +def run_all_tests(cw=ConsoleWriter(None, None), test_filter=None): + for test_file in all_test_files(): + if test_matches_filter(test_file, test_filter): + cw = cw._replace(test=file_to_module(test_file)) - run_module(script_to_module(script), args, cw) + run_module(file_to_module(test_file), cw) diff --git a/scripts/reqresp/ping.py b/scripts/reqresp/ping.py index aa4386c..855fa60 100644 --- a/scripts/reqresp/ping.py +++ b/scripts/reqresp/ping.py @@ -3,7 +3,6 @@ from pyrum import SubprocessConn, Rumor from sclients import connect_rumor -from lib.console import ConsoleWriter from ..utils import parse_response, with_rumor diff --git a/scripts/utils.py b/scripts/utils.py index ccc59ff..47f7cc3 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -56,7 +56,7 @@ def parse_response(resp): class TestBase: - def __init__(self, args, cw=ConsoleWriter(None, None, None)): + def __init__(self, args, cw=ConsoleWriter(None, None)): self.args = args self.cw = cw self.return_code = 0 diff --git a/steth.py b/steth.py index 10dba16..f466d95 100755 --- a/steth.py +++ b/steth.py @@ -7,25 +7,14 @@ from lib.console import ConsoleWriter from lib.fixtures import extract_fixtures, setup_fixture, teardown_fixture -from lib.runner import run_test_config - -SUITES_DIR = 'suites' - - -def load_suite_config(suite): - suite_file = f'{SUITES_DIR}/{suite}.yml' - with open(suite_file, 'r') as f: - config = yaml.load(f, Loader=yaml.Loader) - return config +from lib.runner import run_all_tests def run_start_fixture(args): client = args.client - suite = args.suite - cw = ConsoleWriter(suite, None, None) + cw = ConsoleWriter(None, None) - config = load_suite_config(suite) - fixtures = extract_fixtures(config, [client]) + fixtures = extract_fixtures([client]) for fixture in fixtures: cw = cw._replace(fixture=fixture.name) @@ -34,7 +23,7 @@ def run_start_fixture(args): def run_stop_fixture(args): - cw = ConsoleWriter(None, None, None) + cw = ConsoleWriter(None, None) for client in SUPPORTED_CLIENTS: cw.info(f'stopping client {client}') stop_instance(client) @@ -42,20 +31,18 @@ def run_stop_fixture(args): def run_test(args): clients = SUPPORTED_CLIENTS if args.client is None else [args.client] - suite = args.suite reuse_clients = args.reuse test_filter = args.only - cw = ConsoleWriter(suite, None, None) + cw = ConsoleWriter(None, None) - config = load_suite_config(suite) - fixtures = extract_fixtures(config, clients) + fixtures = extract_fixtures(clients) for fixture in fixtures: cw = cw._replace(fixture=fixture.name) if reuse_clients: - run_test_config(config, cw, test_filter) + run_all_tests(cw, test_filter) else: try: @@ -67,7 +54,7 @@ def run_test(args): continue try: - run_test_config(config, cw, test_filter) + run_all_tests(cw, test_filter) finally: cw.info('tearing down fixture') @@ -82,7 +69,6 @@ def run_test(args): fixture_sub = fixture.add_subparsers() start = fixture_sub.add_parser('start') - start.add_argument('-s', '--suite', help='suite file to parse fixtures from', required=True) start.add_argument( 'client', help=f'client to start. can only start 1 at a time until dynamic ENRs implemented. possible values: {SUPPORTED_CLIENTS}') start.set_defaults(func=run_start_fixture) @@ -95,7 +81,6 @@ def run_test(args): help='run test(s) for a specific client. possible values: {SUPPORTED_CLIENTS}') test.add_argument('-o', '--only', help='run specific tests by name') test.add_argument('-r', '--reuse', default=False, action='store_true', help='reuse running fixtures') - test.add_argument('-s', '--suite', help='suite file to parse fixtures from', required=True) test.set_defaults(func=run_test) args = steth.parse_args()