Setup ping test

lib/fixtures.py

@@ -26,10 +26,6 @@ def extract_fixtures(clients_to_test=SUPPORTED_CLIENTS) -> List[Fixture]:
 
 def setup_fixture(fixture: Fixture):
     for instance in fixture.instances:
-        if instance.client not in SUPPORTED_CLIENTS:
-            raise ValueError(
-                f"can't start instance: client {instance_config.client} not supported yet")
-
         start_instance(instance)
 
 

lib/instance_configs.py

@@ -3,6 +3,7 @@
 DEFAULT_BEACON_STATE_PATH = 'ssz/default.ssz'
 DEFAULT_ENR =  {
     'enr': 'enr:-LK4QJCIZoViytOOmAzAbdOJODwQ36PhjXwvXlTFTloTzpawVpvPRmtrM6UZdPmOGck5yPZ9AbgmwyZnE3jm4jX0Yx0Bh2F0dG5ldHOIAAAAAAAAAACEZXRoMpBGMkSJAAAAAf__________gmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQOnBq2PcxFfkFACZvJz91cd-UKaTPtLv7zYJSJyAtq60YN0Y3CCIyiDdWRwgiMp',
+    'enr_teku': 'enr:-KG4QF9w4w5DORl2-AtVaqK6n-VF3e-2p5fm4uqDLwlR8cQ4MVk2yJ68YkEUTASbbJJ123CKYDDY1KdfBWOuVrmsZtUChGV0aDKQGK5MywAAAAH__________4JpZIJ2NIJpcIR_AAABiXNlY3AyNTZrMaEDpwatj3MRX5BQAmbyc_dXHflCmkz7S7-82CUicgLautGDdGNwgiMog3VkcIIjKA',
     'private_key': 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',
     'udp': 9001,
     'tcp': 9000,

lib/runner.py

@@ -6,7 +6,6 @@
 from colored import fg
 
 from lib.console import ConsoleWriter
-from lib.instance_configs import DEFAULT_ARGS
 
 TESTS_DIR = 'tests'
 
@@ -42,12 +41,29 @@ def run_module(module_str, args):
 
     return (return_code, logs)
 
-def test_matches_filter(test, test_filter):
-    return test_filter is None or test == test_filter
+def file_matches_filter(file, file_filter):
+    return file_filter is None or file == file_filter
 
-def run_all_tests(cw=ConsoleWriter(None, None), test_filter=None):
-    for test_file in all_test_files():
-        if test_matches_filter(test_file, test_filter):
-            cw = cw._replace(test=file_to_module(test_file))
+def run_test_files(fixture_name, files, args):
+    print(fixture_name)
 
-            run_module(file_to_module(test_file), DEFAULT_ARGS)
+    failures = {}
+    for file in files:
+        module = file_to_module(file)
+        (return_code, logs) = run_module(module, {**args, 'client': fixture_name})
+
+        print(f'\t{module} {return_code_to_status(return_code)}')
+        if return_code != 0:
+            failures[module] = (return_code, logs)
+
+    if len(failures) > 0:
+        for module, (return_code, logs) in failures.items():
+            print('')
+            print(f'\t{module} {return_code_to_status(return_code)}')
+
+            for log in logs:
+                print(f'\t{log}')
+
+        return 1
+
+    return 0

steth.py

@@ -5,13 +5,13 @@
 import yaml
 from sclients import SUPPORTED_CLIENTS, stop_instance
 
-from lib.console import ConsoleWriter
 from lib.fixtures import extract_fixtures, setup_fixture, teardown_fixture
-from lib.runner import run_all_tests
-from lib.logging_tests import run_logging_tests
+from lib.instance_configs import DEFAULT_ARGS
+from lib.runner import run_test_files, all_test_files, file_matches_filter
+from lib.logging_tests import all_logging_test_files
 
 
-def run_start_fixture(args):
+def run_start_fixture_cmd(args):
     client = args.client
     cw = ConsoleWriter(None, None)
 
@@ -23,49 +23,48 @@ def run_start_fixture(args):
         setup_fixture(fixture)
 
 
-def run_stop_fixture(args):
+def run_stop_fixture_cmd(args):
     cw = ConsoleWriter(None, None)
     for client in SUPPORTED_CLIENTS:
         cw.info(f'stopping client {client}')
         stop_instance(client)
 
 
-def run_test(args):
+def run_test_cmd(args):
     clients = SUPPORTED_CLIENTS if args.client is None else [args.client]
 
     reuse_clients = args.reuse
-    test_filter = args.only
-
-    cw = ConsoleWriter(None, None)
+    file_filter = args.only
 
+    test_files = [file for file in all_test_files() if file_matches_filter(file, file_filter)]
     fixtures = extract_fixtures(clients)
+    failed = False
     for fixture in fixtures:
-        cw = cw._replace(fixture=fixture.name)
-
-        if reuse_clients:
-            run_all_tests(cw, test_filter)
+        if not reuse_clients:
+            setup_fixture(fixture)
 
-        else:
-            try:
-                cw.info('setting up fixture')
-                setup_fixture(fixture)
+        try:
+            return_code = run_test_files(fixture.name, test_files, DEFAULT_ARGS)
+            if return_code != 0:
+                failed = True
 
-            except ValueError as e:
-                cw.info(f'skipping fixture: {e}')
-                continue
-
-            try:
-                run_all_tests(cw, test_filter)
+        except Exception as e:
+            print(f'Exception while running tests: {e}')
+            failed = True
 
-            finally:
-                cw.info('tearing down fixture')
+        finally:
+            if not reuse_clients:
                 teardown_fixture(fixture)
+    if failed:
+        exit(1)
 
 
 def run_logging_test(args):
+    test_files = all_logging_test_files()
+
     failed = False
     for client in SUPPORTED_CLIENTS:
-        return_code= run_logging_tests(client)
+        return_code= run_test_files(client, test_files, {})
         if return_code != 0:
             failed = True
 
@@ -82,17 +81,17 @@ def run_logging_test(args):
     start = fixture_sub.add_parser('start')
     start.add_argument(
         'client', help=f'client to start. can only start 1 at a time until dynamic ENRs implemented. possible values: {SUPPORTED_CLIENTS}')
-    start.set_defaults(func=run_start_fixture)
+    start.set_defaults(func=run_start_fixture_cmd)
 
     stop = fixture_sub.add_parser('stop')
-    stop.set_defaults(func=run_stop_fixture)
+    stop.set_defaults(func=run_stop_fixture_cmd)
 
     test = steth_sub.add_parser('test', help='Run stethoscope unit tests')
     test.add_argument('-c', '--client',
-                      help='run test(s) for a specific client. possible values: {SUPPORTED_CLIENTS}')
+                      help=f'run test(s) for a specific client. possible values: {SUPPORTED_CLIENTS}')
     test.add_argument('-o', '--only', help='run specific tests by name')
     test.add_argument('-r', '--reuse', default=False, action='store_true', help='reuse running fixtures')
-    test.set_defaults(func=run_test)
+    test.set_defaults(func=run_test_cmd)
 
     test = steth_sub.add_parser('logging_test', help='Display example logs from ./steth.py test. Useful for testing CI integration')
     test.set_defaults(func=run_logging_test)

tests/reqresp/ping.py

@@ -1,14 +1,12 @@
-import trio
 from eth2spec.utils.ssz.ssz_typing import uint64
-from pyrum import SubprocessConn, Rumor
 from sclients import connect_rumor
 
 from ..utils import parse_response, with_rumor
 
 
 @with_rumor
 async def run(rumor, args):
-    peer_id = await connect_rumor(rumor, args['enr'])
+    peer_id = await connect_rumor(rumor, args['client'], args['enr'])
 
     return_code = 0
     for i in range(5):
@@ -22,6 +20,7 @@ async def run(rumor, args):
         pong = uint64.decode_bytes(bytes.fromhex(resp_data))
 
         if not isinstance(pong, int) or pong < 0:
+            # TODO: add logs
             return_code = 1
 
-    return return_code
+    return (return_code, [])