From 71f777fbc75bb2f37e88930b0383d62143a42ffc Mon Sep 17 00:00:00 2001
From: lsankar4033 <lsankar4033@gmail.com>
Date: Sun, 9 Aug 2020 15:56:09 -0700
Subject: [PATCH] Logging test for CI

---
 .travis.yml                   |  2 +-
 lib/logging_tests.py          | 25 ++++++++++++++++++++-----
 lib/runner.py                 |  6 +++---
 logging_tests/test_fail.py    |  2 +-
 logging_tests/test_success.py |  2 +-
 steth.py                      |  7 ++++++-
 6 files changed, 32 insertions(+), 12 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 2daca94..c7ca1ab 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,4 +10,4 @@ before_install:
   - pip install -U pip
 
 script:
-  - ./steth.py fixture start lighthouse && docker logs lighthouse && ./steth.py fixture stop
+  - ./steth.py logging_test
diff --git a/lib/logging_tests.py b/lib/logging_tests.py
index 3576340..eee5701 100644
--- a/lib/logging_tests.py
+++ b/lib/logging_tests.py
@@ -1,6 +1,6 @@
 import os
 
-from lib.runner import run_module, file_to_module
+from lib.runner import run_module, file_to_module, return_code_to_status
 
 LOGGING_TESTS_DIR = 'logging_tests'
 
@@ -13,10 +13,25 @@ def all_logging_test_files():
 
 
 def run_logging_tests(client):
-    # print client, then call run_module for each test. collect all failures and print them at the end
     print(client)
+
+    failures = {}
     for file in all_logging_test_files():
-        # TODO: collect failure
-        run_module(file_to_module(file), {})
+        module = file_to_module(file)
+        (return_code, logs) = run_module(module, {})
+
+        print(f'\t{module} {return_code_to_status(return_code)}')
+        if return_code != 0:
+            failures[module] = (return_code, logs)
+
+    if len(failures) > 0:
+        for module, (return_code, logs) in failures.items():
+            print('')
+            print(f'\t{module} {return_code_to_status(return_code)}')
+
+            for log in logs:
+                print(f'\t{log}')
+
+        return 1
 
-    # TODO: print out all failures + logs
+    return 0
diff --git a/lib/runner.py b/lib/runner.py
index 251c7b2..6bc1bb2 100644
--- a/lib/runner.py
+++ b/lib/runner.py
@@ -35,12 +35,12 @@ def run_module(module_str, args):
     module = importlib.import_module(module_str)
 
     if not hasattr(module, 'run'):
-        cw.fail(f'module {module} does not have a run method')
+        print(f'\t{module} does not have a run method')
         return
 
-    return_code = trio.run(module.run, args)
+    return_code, logs = trio.run(module.run, args)
 
-    print(f'\t{module_str} {return_code_to_status(return_code)}')
+    return (return_code, logs)
 
 def test_matches_filter(test, test_filter):
     return test_filter is None or test == test_filter
diff --git a/logging_tests/test_fail.py b/logging_tests/test_fail.py
index 8fff890..94c98fe 100644
--- a/logging_tests/test_fail.py
+++ b/logging_tests/test_fail.py
@@ -1,2 +1,2 @@
 async def run(args):
-    return 1
+    return (1, ['error 1', 'error 2'])
diff --git a/logging_tests/test_success.py b/logging_tests/test_success.py
index bf0cbd2..67a4499 100644
--- a/logging_tests/test_success.py
+++ b/logging_tests/test_success.py
@@ -1,2 +1,2 @@
 async def run(args):
-    return 0
+    return (0, [])
diff --git a/steth.py b/steth.py
index c9dcff3..792f4c1 100755
--- a/steth.py
+++ b/steth.py
@@ -63,9 +63,14 @@ def run_test(args):
 
 
 def run_logging_test(args):
+    failed = False
     for client in SUPPORTED_CLIENTS:
-        run_logging_tests(client)
+        return_code= run_logging_tests(client)
+        if return_code != 0:
+            failed = True
 
+    if failed:
+        exit(1)
 
 if __name__ == '__main__':
     steth = argparse.ArgumentParser(description='Stethoscope tool for running multi-client Eth2 scenarios')