Logging test for CI

.travis.yml

@@ -10,4 +10,4 @@ before_install:
   - pip install -U pip
 
 script:
-  - ./steth.py fixture start lighthouse && docker logs lighthouse && ./steth.py fixture stop
+  - ./steth.py logging_test

lib/logging_tests.py

@@ -1,6 +1,6 @@
 import os
 
-from lib.runner import run_module, file_to_module
+from lib.runner import run_module, file_to_module, return_code_to_status
 
 LOGGING_TESTS_DIR = 'logging_tests'
 
@@ -13,10 +13,25 @@ def all_logging_test_files():
 
 
 def run_logging_tests(client):
-    # print client, then call run_module for each test. collect all failures and print them at the end
     print(client)
+
+    failures = {}
     for file in all_logging_test_files():
-        # TODO: collect failure
-        run_module(file_to_module(file), {})
+        module = file_to_module(file)
+        (return_code, logs) = run_module(module, {})
+
+        print(f'\t{module} {return_code_to_status(return_code)}')
+        if return_code != 0:
+            failures[module] = (return_code, logs)
+
+    if len(failures) > 0:
+        for module, (return_code, logs) in failures.items():
+            print('')
+            print(f'\t{module} {return_code_to_status(return_code)}')
+
+            for log in logs:
+                print(f'\t{log}')
+
+        return 1
 
-    # TODO: print out all failures + logs
+    return 0

lib/runner.py

@@ -35,12 +35,12 @@ def run_module(module_str, args):
     module = importlib.import_module(module_str)
 
     if not hasattr(module, 'run'):
-        cw.fail(f'module {module} does not have a run method')
+        print(f'\t{module} does not have a run method')
         return
 
-    return_code = trio.run(module.run, args)
+    return_code, logs = trio.run(module.run, args)
 
-    print(f'\t{module_str} {return_code_to_status(return_code)}')
+    return (return_code, logs)
 
 def test_matches_filter(test, test_filter):
     return test_filter is None or test == test_filter

logging_tests/test_fail.py

@@ -1,2 +1,2 @@
 async def run(args):
-    return 1
+    return (1, ['error 1', 'error 2'])

logging_tests/test_success.py

@@ -1,2 +1,2 @@
 async def run(args):
-    return 0
+    return (0, [])

steth.py

@@ -63,9 +63,14 @@ def run_test(args):
 
 
 def run_logging_test(args):
+    failed = False
     for client in SUPPORTED_CLIENTS:
-        run_logging_tests(client)
+        return_code= run_logging_tests(client)
+        if return_code != 0:
+            failed = True
 
+    if failed:
+        exit(1)
 
 if __name__ == '__main__':
     steth = argparse.ArgumentParser(description='Stethoscope tool for running multi-client Eth2 scenarios')