From eec4db47d645ad067f2aced3433408196c7d53d6 Mon Sep 17 00:00:00 2001 From: lsankar4033 Date: Fri, 7 Aug 2020 12:50:14 -0700 Subject: [PATCH 1/5] Temporary .travis.yml for testing --- .travis.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..ab795ed --- /dev/null +++ b/.travis.yml @@ -0,0 +1,13 @@ +language: python + +python: + - 3.8 + +services: + - docker + +before_install: + - pip install -U pip + +script: + - ./steth.py --help From 75743ddadca8556172c659c10061840bec7273da Mon Sep 17 00:00:00 2001 From: lsankar4033 Date: Fri, 7 Aug 2020 13:00:43 -0700 Subject: [PATCH 2/5] Docker+TravisCI test --- .travis.yml | 2 +- ssz/{single_client_genesis.ssz => default.ssz} | Bin 2 files changed, 1 insertion(+), 1 deletion(-) rename ssz/{single_client_genesis.ssz => default.ssz} (100%) diff --git a/.travis.yml b/.travis.yml index ab795ed..2daca94 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,4 +10,4 @@ before_install: - pip install -U pip script: - - ./steth.py --help + - ./steth.py fixture start lighthouse && docker logs lighthouse && ./steth.py fixture stop diff --git a/ssz/single_client_genesis.ssz b/ssz/default.ssz similarity index 100% rename from ssz/single_client_genesis.ssz rename to ssz/default.ssz From 1623489979d0437190654ad3eaf81a4b29b196fc Mon Sep 17 00:00:00 2001 From: lsankar4033 Date: Fri, 7 Aug 2020 14:08:54 -0700 Subject: [PATCH 3/5] Placeholder logging_tests command --- logging_tests/test_fail.py | 8 ++++++++ logging_tests/test_success.py | 8 ++++++++ steth.py | 8 ++++++++ 3 files changed, 24 insertions(+) create mode 100644 logging_tests/test_fail.py create mode 100644 logging_tests/test_success.py diff --git a/logging_tests/test_fail.py b/logging_tests/test_fail.py new file mode 100644 index 0000000..3fc8ff3 --- /dev/null +++ b/logging_tests/test_fail.py @@ -0,0 +1,8 @@ +import trio +from pyrum import SubprocessConn, Rumor + +from .utils import * + +@with_rumor +async def run(rumor, args): + return 1 diff --git a/logging_tests/test_success.py b/logging_tests/test_success.py new file mode 100644 index 0000000..24de356 --- /dev/null +++ b/logging_tests/test_success.py @@ -0,0 +1,8 @@ +import trio +from pyrum import SubprocessConn, Rumor + +from .utils import with_rumor + +@with_rumor +async def run(rumor, args): + return 0 diff --git a/steth.py b/steth.py index f466d95..fd53686 100755 --- a/steth.py +++ b/steth.py @@ -61,6 +61,11 @@ def run_test(args): teardown_fixture(fixture) +def run_logging_test(args): + for client in SUPPORTED_CLIENTS: + print(f'Testing client {client}') + + if __name__ == '__main__': steth = argparse.ArgumentParser(description='Stethoscope tool for running multi-client Eth2 scenarios') steth_sub = steth.add_subparsers() @@ -83,5 +88,8 @@ def run_test(args): test.add_argument('-r', '--reuse', default=False, action='store_true', help='reuse running fixtures') test.set_defaults(func=run_test) + test = steth_sub.add_parser('logging_test', help='Display example logs from ./steth.py test. Useful for testing CI integration') + test.set_defaults(func=run_logging_test) + args = steth.parse_args() args.func(args) From fc377b55df5b21922ca7ebe12a96678c70b3d323 Mon Sep 17 00:00:00 2001 From: lsankar4033 Date: Sun, 9 Aug 2020 14:27:57 -0700 Subject: [PATCH 4/5] Add logging_tests command w/o error collection --- lib/logging_tests.py | 22 ++++++++++++++++++++++ lib/runner.py | 27 ++++++++++++++------------- logging_tests/test_fail.py | 8 +------- logging_tests/test_success.py | 8 +------- steth.py | 3 ++- 5 files changed, 40 insertions(+), 28 deletions(-) create mode 100644 lib/logging_tests.py diff --git a/lib/logging_tests.py b/lib/logging_tests.py new file mode 100644 index 0000000..3576340 --- /dev/null +++ b/lib/logging_tests.py @@ -0,0 +1,22 @@ +import os + +from lib.runner import run_module, file_to_module + +LOGGING_TESTS_DIR = 'logging_tests' + +def all_logging_test_files(): + tests = [] + for root, dirs, files in os.walk(LOGGING_TESTS_DIR): + tests.extend([f'{root}/{f}' for f in files if f.endswith('.py')]) + + return tests + + +def run_logging_tests(client): + # print client, then call run_module for each test. collect all failures and print them at the end + print(client) + for file in all_logging_test_files(): + # TODO: collect failure + run_module(file_to_module(file), {}) + + # TODO: print out all failures + logs diff --git a/lib/runner.py b/lib/runner.py index 7a659d1..251c7b2 100644 --- a/lib/runner.py +++ b/lib/runner.py @@ -3,6 +3,7 @@ import subprocess import trio +from colored import fg from lib.console import ConsoleWriter from lib.instance_configs import DEFAULT_ARGS @@ -23,23 +24,23 @@ def all_test_files(): def file_to_module(script): return script.replace('/', '.')[0:-3] +def return_code_to_status(return_code): + if return_code == 0: + return f"{fg('green')}\u2713{fg('white')}" -# TODO: add args back in -def run_module(module, args, cw): - print(f'Running module: {module} with args {args}') - #module = importlib.import_module(module) + else: + return f"{fg('red')}\u2717{fg('white')}" - #if not hasattr(module, 'run'): - #cw.fail(f'module {module} does not have a run method') - #return +def run_module(module_str, args): + module = importlib.import_module(module_str) - #return_code = trio.run(module.run, args) - #if return_code == 0: - #cw.success('SUCCESS') + if not hasattr(module, 'run'): + cw.fail(f'module {module} does not have a run method') + return - #else: - #cw.fail('FAILED') + return_code = trio.run(module.run, args) + print(f'\t{module_str} {return_code_to_status(return_code)}') def test_matches_filter(test, test_filter): return test_filter is None or test == test_filter @@ -49,4 +50,4 @@ def run_all_tests(cw=ConsoleWriter(None, None), test_filter=None): if test_matches_filter(test_file, test_filter): cw = cw._replace(test=file_to_module(test_file)) - run_module(file_to_module(test_file), DEFAULT_ARGS, cw) + run_module(file_to_module(test_file), DEFAULT_ARGS) diff --git a/logging_tests/test_fail.py b/logging_tests/test_fail.py index 3fc8ff3..8fff890 100644 --- a/logging_tests/test_fail.py +++ b/logging_tests/test_fail.py @@ -1,8 +1,2 @@ -import trio -from pyrum import SubprocessConn, Rumor - -from .utils import * - -@with_rumor -async def run(rumor, args): +async def run(args): return 1 diff --git a/logging_tests/test_success.py b/logging_tests/test_success.py index 24de356..bf0cbd2 100644 --- a/logging_tests/test_success.py +++ b/logging_tests/test_success.py @@ -1,8 +1,2 @@ -import trio -from pyrum import SubprocessConn, Rumor - -from .utils import with_rumor - -@with_rumor -async def run(rumor, args): +async def run(args): return 0 diff --git a/steth.py b/steth.py index fd53686..c9dcff3 100755 --- a/steth.py +++ b/steth.py @@ -8,6 +8,7 @@ from lib.console import ConsoleWriter from lib.fixtures import extract_fixtures, setup_fixture, teardown_fixture from lib.runner import run_all_tests +from lib.logging_tests import run_logging_tests def run_start_fixture(args): @@ -63,7 +64,7 @@ def run_test(args): def run_logging_test(args): for client in SUPPORTED_CLIENTS: - print(f'Testing client {client}') + run_logging_tests(client) if __name__ == '__main__': From 71f777fbc75bb2f37e88930b0383d62143a42ffc Mon Sep 17 00:00:00 2001 From: lsankar4033 Date: Sun, 9 Aug 2020 15:56:09 -0700 Subject: [PATCH 5/5] Logging test for CI --- .travis.yml | 2 +- lib/logging_tests.py | 25 ++++++++++++++++++++----- lib/runner.py | 6 +++--- logging_tests/test_fail.py | 2 +- logging_tests/test_success.py | 2 +- steth.py | 7 ++++++- 6 files changed, 32 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2daca94..c7ca1ab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,4 +10,4 @@ before_install: - pip install -U pip script: - - ./steth.py fixture start lighthouse && docker logs lighthouse && ./steth.py fixture stop + - ./steth.py logging_test diff --git a/lib/logging_tests.py b/lib/logging_tests.py index 3576340..eee5701 100644 --- a/lib/logging_tests.py +++ b/lib/logging_tests.py @@ -1,6 +1,6 @@ import os -from lib.runner import run_module, file_to_module +from lib.runner import run_module, file_to_module, return_code_to_status LOGGING_TESTS_DIR = 'logging_tests' @@ -13,10 +13,25 @@ def all_logging_test_files(): def run_logging_tests(client): - # print client, then call run_module for each test. collect all failures and print them at the end print(client) + + failures = {} for file in all_logging_test_files(): - # TODO: collect failure - run_module(file_to_module(file), {}) + module = file_to_module(file) + (return_code, logs) = run_module(module, {}) + + print(f'\t{module} {return_code_to_status(return_code)}') + if return_code != 0: + failures[module] = (return_code, logs) + + if len(failures) > 0: + for module, (return_code, logs) in failures.items(): + print('') + print(f'\t{module} {return_code_to_status(return_code)}') + + for log in logs: + print(f'\t{log}') + + return 1 - # TODO: print out all failures + logs + return 0 diff --git a/lib/runner.py b/lib/runner.py index 251c7b2..6bc1bb2 100644 --- a/lib/runner.py +++ b/lib/runner.py @@ -35,12 +35,12 @@ def run_module(module_str, args): module = importlib.import_module(module_str) if not hasattr(module, 'run'): - cw.fail(f'module {module} does not have a run method') + print(f'\t{module} does not have a run method') return - return_code = trio.run(module.run, args) + return_code, logs = trio.run(module.run, args) - print(f'\t{module_str} {return_code_to_status(return_code)}') + return (return_code, logs) def test_matches_filter(test, test_filter): return test_filter is None or test == test_filter diff --git a/logging_tests/test_fail.py b/logging_tests/test_fail.py index 8fff890..94c98fe 100644 --- a/logging_tests/test_fail.py +++ b/logging_tests/test_fail.py @@ -1,2 +1,2 @@ async def run(args): - return 1 + return (1, ['error 1', 'error 2']) diff --git a/logging_tests/test_success.py b/logging_tests/test_success.py index bf0cbd2..67a4499 100644 --- a/logging_tests/test_success.py +++ b/logging_tests/test_success.py @@ -1,2 +1,2 @@ async def run(args): - return 0 + return (0, []) diff --git a/steth.py b/steth.py index c9dcff3..792f4c1 100755 --- a/steth.py +++ b/steth.py @@ -63,9 +63,14 @@ def run_test(args): def run_logging_test(args): + failed = False for client in SUPPORTED_CLIENTS: - run_logging_tests(client) + return_code= run_logging_tests(client) + if return_code != 0: + failed = True + if failed: + exit(1) if __name__ == '__main__': steth = argparse.ArgumentParser(description='Stethoscope tool for running multi-client Eth2 scenarios')