Surgically remove the notion of suites

lib/console.py

@@ -3,25 +3,21 @@
 
 
 class ConsoleWriter(NamedTuple):
-    suite: str
     fixture: str
     test: str
 
     def _prefix(self):
-        if self.suite is None and self.fixture is None and self.test is None:
+        if self.fixture is None and self.test is None:
             return ''
 
-        elif self.fixture is None and self.test is None:
-            return f'{self.suite}'
-
         elif self.fixture is None:
-            return f'{self.suite} -- {self.test}'
+            return f'{self.test}'
 
         elif self.test is None:
-            return f'{self.suite}[{self.fixture}]'
+            return f'{self.fixture}'
 
         else:
-            return f'{self.suite}[{self.fixture}] -- {self.test}'
+            return f'{self.fixture} -- {self.test}'
 
     def info(self, s):
         print(f'{self._prefix()} {s}')

lib/fixtures.py

@@ -7,33 +7,35 @@
 
 from lib.types import Fixture
 
-
-def extract_fixtures(config, clients_to_test=SUPPORTED_CLIENTS) -> List[Fixture]:
-    # turn a yml config into a list of fixtures. if any 'all' in the file, do 1:1 for every one (rather than all
-    # pairwise combos)
-    if any((i['client'] == 'all' for i in config['instances'])):
-        fixtures = []
-
-        for client in SUPPORTED_CLIENTS:
-            if client not in clients_to_test:
-                continue
-
-            # convert 'all' instances to $client
-            def convert(ic): return ic._replace(client=client) if ic.client == 'all' else ic
-
-            fixture = Fixture(client, [convert(InstanceConfig.from_dict(i)) for i in config['instances']])
-            fixtures.append(fixture)
-
-        return fixtures
-
-    # if any clients in the fixture are not among those specified in clients_to_test, return no fixtures
-    elif any((i['client'] not in clients_to_test for i in config['instances'])):
-        return []
-
-    else:
-        fixture = Fixture(None, [InstanceConfig.from_dict(i) for i in config['instances']])
-        return [fixture]
-
+DEFAULT_INSTANCE_CONFIG = {
+    'beacon_state_path': 'ssz/single_client_genesis.ssz',
+    'enr': {
+        'enr': 'enr:-LK4QJCIZoViytOOmAzAbdOJODwQ36PhjXwvXlTFTloTzpawVpvPRmtrM6UZdPmOGck5yPZ9AbgmwyZnE3jm4jX0Yx0Bh2F0dG5ldHOIAAAAAAAAAACEZXRoMpBGMkSJAAAAAf__________gmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQOnBq2PcxFfkFACZvJz91cd-UKaTPtLv7zYJSJyAtq60YN0Y3CCIyiDdWRwgiMp',
+        'private_key': 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',
+        'udp': 9001,
+        'tcp': 9000,
+        'id': 'v4',
+        'ip': '127.0.0.1',
+        'attnets': '0x0000000000000000',
+        'eth2': '0x4632448900000001ffffffffffffffff'
+    }
+}
+
+def extract_fixtures(clients_to_test=SUPPORTED_CLIENTS) -> List[Fixture]:
+    fixtures = []
+    for client in SUPPORTED_CLIENTS:
+        if client not in clients_to_test:
+            continue
+
+        instance_config = {
+            **DEFAULT_INSTANCE_CONFIG,
+            'client': client
+        }
+
+        fixture = Fixture(client, [InstanceConfig.from_dict(instance_config)])
+        fixtures.append(fixture)
+
+    return fixtures
 
 def setup_fixture(fixture: Fixture):
     for instance in fixture.instances:

lib/runner.py

@@ -1,66 +1,51 @@
 import importlib
+import os
 import subprocess
 
 import trio
 
 from lib.console import ConsoleWriter
 
+TESTS_DIR = 'scripts'
 
-def script_to_module(script):
-    return script.replace('/', '.')[0:-3]
-
-
-def script_to_test(script):
-    remove_prefix_suffix = script[8:len(script) - 3]
-    return remove_prefix_suffix.replace('/', '.')
-
-
-def run_script(script, args, cw):
-    script_list = ['python3', '-m', script_to_module(script)]
-    arg_list = [item for sublist in [['--' + k, v] for k, v in args.items()] for item in sublist]
+def all_test_files():
+    tests = []
+    for root, dirs, files in os.walk(TESTS_DIR):
+        # skip files in top-level
+        if root == 'scripts':
+            continue
 
-    output = subprocess.run(
-        script_list + arg_list,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        text=True
-    )
+        tests.extend([f'{root}/{f}' for f in files if f.endswith('.py')])
 
-    if len(output.stderr) > 0:
-        cw.fail('FAILED')
-        cw.info(output.stderr)
+    return tests
 
-    else:
-        cw.success('SUCCESS')
+def file_to_module(script):
+    return script.replace('/', '.')[0:-3]
 
 
-def run_module(module, args, cw):
-    module = importlib.import_module(module)
+# TODO: add args back in
+def run_module(module, cw):
+    print(f'Running module: {module}')
+    #module = importlib.import_module(module)
 
-    if not hasattr(module, 'run'):
-        cw.fail(f'module {module} does not have a run method')
-        return
+    #if not hasattr(module, 'run'):
+        #cw.fail(f'module {module} does not have a run method')
+        #return
 
-    return_code = trio.run(module.run, args)
-    if return_code == 0:
-        cw.success('SUCCESS')
+    #return_code = trio.run(module.run, args)
+    #if return_code == 0:
+        #cw.success('SUCCESS')
 
-    else:
-        cw.fail('FAILED')
+    #else:
+        #cw.fail('FAILED')
 
 
 def test_matches_filter(test, test_filter):
-    return test_filter is None or test == test_filter or test.startswith(test_filter)
-
-
-def run_test_config(test_config, cw=ConsoleWriter(None, None, None), test_filter=None):
-    tests = test_config['test_suite']
-
-    for test in tests:
-        if test_matches_filter(test['name'], test_filter):
-            script = test['script']
-            args = test['args']
+    return test_filter is None or test == test_filter
 
-            cw = cw._replace(test=script_to_test(script))
+def run_all_tests(cw=ConsoleWriter(None, None), test_filter=None):
+    for test_file in all_test_files():
+        if test_matches_filter(test_file, test_filter):
+            cw = cw._replace(test=file_to_module(test_file))
 
-            run_module(script_to_module(script), args, cw)
+            run_module(file_to_module(test_file), cw)

scripts/reqresp/ping.py

@@ -3,7 +3,6 @@
 from pyrum import SubprocessConn, Rumor
 from sclients import connect_rumor
 
-from lib.console import ConsoleWriter
 from ..utils import parse_response, with_rumor
 
 

scripts/utils.py

@@ -56,7 +56,7 @@ def parse_response(resp):
 
 class TestBase:
 
-    def __init__(self, args, cw=ConsoleWriter(None, None, None)):
+    def __init__(self, args, cw=ConsoleWriter(None, None)):
         self.args = args
         self.cw = cw
         self.return_code = 0

steth.py

@@ -7,25 +7,14 @@
 
 from lib.console import ConsoleWriter
 from lib.fixtures import extract_fixtures, setup_fixture, teardown_fixture
-from lib.runner import run_test_config
-
-SUITES_DIR = 'suites'
-
-
-def load_suite_config(suite):
-    suite_file = f'{SUITES_DIR}/{suite}.yml'
-    with open(suite_file, 'r') as f:
-        config = yaml.load(f, Loader=yaml.Loader)
-    return config
+from lib.runner import run_all_tests
 
 
 def run_start_fixture(args):
     client = args.client
-    suite = args.suite
-    cw = ConsoleWriter(suite, None, None)
+    cw = ConsoleWriter(None, None)
 
-    config = load_suite_config(suite)
-    fixtures = extract_fixtures(config, [client])
+    fixtures = extract_fixtures([client])
 
     for fixture in fixtures:
         cw = cw._replace(fixture=fixture.name)
@@ -34,28 +23,26 @@ def run_start_fixture(args):
 
 
 def run_stop_fixture(args):
-    cw = ConsoleWriter(None, None, None)
+    cw = ConsoleWriter(None, None)
     for client in SUPPORTED_CLIENTS:
         cw.info(f'stopping client {client}')
         stop_instance(client)
 
 
 def run_test(args):
     clients = SUPPORTED_CLIENTS if args.client is None else [args.client]
-    suite = args.suite
 
     reuse_clients = args.reuse
     test_filter = args.only
 
-    cw = ConsoleWriter(suite, None, None)
+    cw = ConsoleWriter(None, None)
 
-    config = load_suite_config(suite)
-    fixtures = extract_fixtures(config, clients)
+    fixtures = extract_fixtures(clients)
     for fixture in fixtures:
         cw = cw._replace(fixture=fixture.name)
 
         if reuse_clients:
-            run_test_config(config, cw, test_filter)
+            run_all_tests(cw, test_filter)
 
         else:
             try:
@@ -67,7 +54,7 @@ def run_test(args):
                 continue
 
             try:
-                run_test_config(config, cw, test_filter)
+                run_all_tests(cw, test_filter)
 
             finally:
                 cw.info('tearing down fixture')
@@ -82,7 +69,6 @@ def run_test(args):
     fixture_sub = fixture.add_subparsers()
 
     start = fixture_sub.add_parser('start')
-    start.add_argument('-s', '--suite', help='suite file to parse fixtures from', required=True)
     start.add_argument(
         'client', help=f'client to start. can only start 1 at a time until dynamic ENRs implemented. possible values: {SUPPORTED_CLIENTS}')
     start.set_defaults(func=run_start_fixture)
@@ -95,7 +81,6 @@ def run_test(args):
                       help='run test(s) for a specific client. possible values: {SUPPORTED_CLIENTS}')
     test.add_argument('-o', '--only', help='run specific tests by name')
     test.add_argument('-r', '--reuse', default=False, action='store_true', help='reuse running fixtures')
-    test.add_argument('-s', '--suite', help='suite file to parse fixtures from', required=True)
     test.set_defaults(func=run_test)
 
     args = steth.parse_args()