logging_test with TestGroup

lib/logging_tests.py

@@ -1,38 +1,27 @@
 import os
+from typing import List
 
+from sclients import SUPPORTED_CLIENTS
+
+from lib.beacon_state import BeaconState
 from lib.runner import run_module, file_to_module, return_code_to_status
+from lib.test_groups import get_test_groups, TestGroup
 
 LOGGING_TESTS_DIR = 'logging_tests'
 
+LOGGING_TEST_FILE_TO_CONFIGS = {
+    'logging_tests/test_success.py': [BeaconState.DEFAULT],
+    'logging_tests/test_fail.py': [BeaconState.DEFAULT],
+}
+
+
+def get_logging_test_groups() -> List[TestGroup]:
+    return get_test_groups(LOGGING_TEST_FILE_TO_CONFIGS, SUPPORTED_CLIENTS, None)
+
 
 def all_logging_test_files():
     tests = []
     for root, dirs, files in os.walk(LOGGING_TESTS_DIR):
         tests.extend([f'{root}/{f}' for f in files if f.endswith('.py')])
 
     return tests
-
-
-def run_logging_tests(client):
-    print(client)
-
-    failures = {}
-    for file in all_logging_test_files():
-        module = file_to_module(file)
-        (return_code, logs) = run_module(module, {})
-
-        print(f'\t{module} {return_code_to_status(return_code)}')
-        if return_code != 0:
-            failures[module] = (return_code, logs)
-
-    if len(failures) > 0:
-        for module, (return_code, logs) in failures.items():
-            print('')
-            print(f'\t{module} {return_code_to_status(return_code)}')
-
-            for log in logs:
-                print(f'\t{log}')
-
-        return 1
-
-    return 0

lib/runner.py

@@ -48,18 +48,17 @@ def file_matches_filter(file, file_filter):
     return file_filter is None or file == file_filter
 
 
-def run_test_files(fixture_name, files, args):
-    print(fixture_name)
-
+def run_test_files(client, files, args):
     failures = {}
     for file in files:
         module = file_to_module(file)
-        (return_code, logs) = run_module(module, {**args, 'client': fixture_name})
+        (return_code, logs) = run_module(module, {**args, 'client': client})
 
         print(f'\t{module} {return_code_to_status(return_code)}')
         if return_code != 0:
             failures[module] = (return_code, logs)
 
+    # TODO: report all errors *at the end* of test run
     if len(failures) > 0:
         print('')
         print('--failures--')

lib/test_groups.py

@@ -18,16 +18,19 @@ class TestGroup(NamedTuple):
     test_files: List[str]
     instance_config: InstanceConfig
 
+    def __str__(self):
+        return f'{self.instance_config.client} - {self.instance_config.beacon_state_path}'
+
 
 # TODO: allow for comma-separated test filter
 def file_matches_filter(file, file_filter):
     return file_filter is None or file == file_filter
 
 
 # NOTE: this file does sorting
-def get_test_groups(clients, test_file_filter) -> List[TestGroup]:
+def get_test_groups(test_file_to_configs, clients, test_file_filter) -> List[TestGroup]:
     client_path_to_test_files = defaultdict(list)
-    for test_file, beacon_states in TEST_FILE_TO_CONFIGS.items():
+    for test_file, beacon_states in test_file_to_configs.items():
         if file_matches_filter(test_file, test_file_filter):
             for client in clients:
                 for beacon_state in beacon_states:

steth.py

@@ -3,19 +3,20 @@
 import argparse
 import os
 import yaml
-from sclients import SUPPORTED_CLIENTS, stop_instance
+from sclients import SUPPORTED_CLIENTS
 
 from lib.fixtures import extract_fixtures, setup_fixture, teardown_fixture
 from lib.instance_configs import DEFAULT_ARGS
 from lib.runner import run_test_files, all_test_files, file_matches_filter
-from lib.logging_tests import all_logging_test_files
+from lib.logging_tests import get_logging_test_groups
 
 
 def run_test_cmd(args):
     clients = SUPPORTED_CLIENTS if args.client is None else [args.client]
 
     file_filter = args.only
 
+    # TODO: use test groups!
     test_files = [file for file in all_test_files() if file_matches_filter(file, file_filter)]
     fixtures = extract_fixtures(clients)
     failed = False
@@ -38,11 +39,13 @@ def run_test_cmd(args):
 
 
 def run_logging_test(args):
-    test_files = all_logging_test_files()
+    test_groups = get_logging_test_groups()
 
     failed = False
-    for client in SUPPORTED_CLIENTS:
-        return_code = run_test_files(client, test_files, {})
+    for test_group in test_groups:
+        print(test_group)
+
+        return_code = run_test_files(test_group.instance_config.client, test_group.test_files, {})
         if return_code != 0:
             failed = True