From c3b4631401e864bd8c1e7653c5d6c65da68c8682 Mon Sep 17 00:00:00 2001
From: lsankar4033 <lsankar4033@gmail.com>
Date: Wed, 19 Aug 2020 18:53:07 -0700
Subject: [PATCH] logging_test with TestGroup

---
 lib/logging_tests.py | 39 ++++++++++++++-------------------------
 lib/runner.py        |  7 +++----
 lib/test_groups.py   |  7 +++++--
 steth.py             | 13 ++++++++-----
 4 files changed, 30 insertions(+), 36 deletions(-)

diff --git a/lib/logging_tests.py b/lib/logging_tests.py
index fb49eec..051eb41 100644
--- a/lib/logging_tests.py
+++ b/lib/logging_tests.py
@@ -1,9 +1,23 @@
 import os
+from typing import List
 
+from sclients import SUPPORTED_CLIENTS
+
+from lib.beacon_state import BeaconState
 from lib.runner import run_module, file_to_module, return_code_to_status
+from lib.test_groups import get_test_groups, TestGroup
 
 LOGGING_TESTS_DIR = 'logging_tests'
 
+LOGGING_TEST_FILE_TO_CONFIGS = {
+    'logging_tests/test_success.py': [BeaconState.DEFAULT],
+    'logging_tests/test_fail.py': [BeaconState.DEFAULT],
+}
+
+
+def get_logging_test_groups() -> List[TestGroup]:
+    return get_test_groups(LOGGING_TEST_FILE_TO_CONFIGS, SUPPORTED_CLIENTS, None)
+
 
 def all_logging_test_files():
     tests = []
@@ -11,28 +25,3 @@ def all_logging_test_files():
         tests.extend([f'{root}/{f}' for f in files if f.endswith('.py')])
 
     return tests
-
-
-def run_logging_tests(client):
-    print(client)
-
-    failures = {}
-    for file in all_logging_test_files():
-        module = file_to_module(file)
-        (return_code, logs) = run_module(module, {})
-
-        print(f'\t{module} {return_code_to_status(return_code)}')
-        if return_code != 0:
-            failures[module] = (return_code, logs)
-
-    if len(failures) > 0:
-        for module, (return_code, logs) in failures.items():
-            print('')
-            print(f'\t{module} {return_code_to_status(return_code)}')
-
-            for log in logs:
-                print(f'\t{log}')
-
-        return 1
-
-    return 0
diff --git a/lib/runner.py b/lib/runner.py
index a710796..e479d61 100644
--- a/lib/runner.py
+++ b/lib/runner.py
@@ -48,18 +48,17 @@ def file_matches_filter(file, file_filter):
     return file_filter is None or file == file_filter
 
 
-def run_test_files(fixture_name, files, args):
-    print(fixture_name)
-
+def run_test_files(client, files, args):
     failures = {}
     for file in files:
         module = file_to_module(file)
-        (return_code, logs) = run_module(module, {**args, 'client': fixture_name})
+        (return_code, logs) = run_module(module, {**args, 'client': client})
 
         print(f'\t{module} {return_code_to_status(return_code)}')
         if return_code != 0:
             failures[module] = (return_code, logs)
 
+    # TODO: report all errors *at the end* of test run
     if len(failures) > 0:
         print('')
         print('--failures--')
diff --git a/lib/test_groups.py b/lib/test_groups.py
index c44fe1d..6c1b814 100644
--- a/lib/test_groups.py
+++ b/lib/test_groups.py
@@ -18,6 +18,9 @@ class TestGroup(NamedTuple):
     test_files: List[str]
     instance_config: InstanceConfig
 
+    def __str__(self):
+        return f'{self.instance_config.client} - {self.instance_config.beacon_state_path}'
+
 
 # TODO: allow for comma-separated test filter
 def file_matches_filter(file, file_filter):
@@ -25,9 +28,9 @@ def file_matches_filter(file, file_filter):
 
 
 # NOTE: this file does sorting
-def get_test_groups(clients, test_file_filter) -> List[TestGroup]:
+def get_test_groups(test_file_to_configs, clients, test_file_filter) -> List[TestGroup]:
     client_path_to_test_files = defaultdict(list)
-    for test_file, beacon_states in TEST_FILE_TO_CONFIGS.items():
+    for test_file, beacon_states in test_file_to_configs.items():
         if file_matches_filter(test_file, test_file_filter):
             for client in clients:
                 for beacon_state in beacon_states:
diff --git a/steth.py b/steth.py
index 0745646..bbf5456 100755
--- a/steth.py
+++ b/steth.py
@@ -3,12 +3,12 @@
 import argparse
 import os
 import yaml
-from sclients import SUPPORTED_CLIENTS, stop_instance
+from sclients import SUPPORTED_CLIENTS
 
 from lib.fixtures import extract_fixtures, setup_fixture, teardown_fixture
 from lib.instance_configs import DEFAULT_ARGS
 from lib.runner import run_test_files, all_test_files, file_matches_filter
-from lib.logging_tests import all_logging_test_files
+from lib.logging_tests import get_logging_test_groups
 
 
 def run_test_cmd(args):
@@ -16,6 +16,7 @@ def run_test_cmd(args):
 
     file_filter = args.only
 
+    # TODO: use test groups!
     test_files = [file for file in all_test_files() if file_matches_filter(file, file_filter)]
     fixtures = extract_fixtures(clients)
     failed = False
@@ -38,11 +39,13 @@ def run_test_cmd(args):
 
 
 def run_logging_test(args):
-    test_files = all_logging_test_files()
+    test_groups = get_logging_test_groups()
 
     failed = False
-    for client in SUPPORTED_CLIENTS:
-        return_code = run_test_files(client, test_files, {})
+    for test_group in test_groups:
+        print(test_group)
+
+        return_code = run_test_files(test_group.instance_config.client, test_group.test_files, {})
         if return_code != 0:
             failed = True