From c17733218950d633714060d2dfc2b1aad5e8b450 Mon Sep 17 00:00:00 2001 From: Michel Hidalgo <michel@ekumenlabs.com> Date: Tue, 13 Aug 2019 17:07:38 -0300 Subject: [PATCH 01/13] Enable launch test discovery in pytest. Signed-off-by: Michel Hidalgo <michel@ekumenlabs.com> --- launch_testing/launch_testing/pytest/hooks.py | 114 ++++++++++++++++++ .../launch_testing/pytest/hookspecs.py | 21 ++++ launch_testing/setup.py | 3 +- 3 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 launch_testing/launch_testing/pytest/hooks.py create mode 100644 launch_testing/launch_testing/pytest/hookspecs.py diff --git a/launch_testing/launch_testing/pytest/hooks.py b/launch_testing/launch_testing/pytest/hooks.py new file mode 100644 index 000000000..7e6dc20cf --- /dev/null +++ b/launch_testing/launch_testing/pytest/hooks.py @@ -0,0 +1,114 @@ +# Copyright 2019 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from ..loader import LoadTestsFromPythonModule +from ..test_runner import LaunchTestRunner + + +class LaunchTestFailure(Exception): + + def __init__(self, message, results): + super().__init__() + self.message = message + self.results = results + + def __str__(self): + return self.message + + +class LaunchTestItem(pytest.Item): + + def __init__(self, name, parent, test_runs): + super().__init__(name, parent) + self.test_runs = test_runs + + def runtest(self): + launch_args = sum(( + args_set for args_set in self.config.getoption('--launch-args') + ), []) + runner = self.ihook.pytest_launch_test_makerunner( + test_runs=self.test_runs, + launch_args=launch_args, + debug=self.config.getoption('verbose') + ) + runner.validate() + results_per_run = runner.run() + if any(not result.wasSuccessful() for result in results_per_run.values()): + raise LaunchTestFailure( + message=self.name + ' failed', + results=results_per_run + ) + + def repr_failure(self, excinfo): + if isinstance(excinfo.value, LaunchTestFailure): + return '\n'.join({ + '{} failed at {}.{}'.format( + str(test_run), + type(test_case).__name__, + test_case._testMethodName + ) + for test_run, test_result in excinfo.value.results.items() + for test_case, _ in (test_result.errors + test_result.failures) + if not test_result.wasSuccessful() + }) + + def reportinfo(self): + return self.fspath, 0, 'launch tests: {}'.format(self.name) + + +class LaunchTestModule(pytest.File): + + def collect(self): + module = self.fspath.pyimport() + yield LaunchTestItem( + name=module.__name__, parent=self, + test_runs=LoadTestsFromPythonModule( + module, name=module.__name__ + ) + ) + + +def _is_launch_test(path): + try: + return hasattr(path.pyimport(), 'generate_test_description') + except SyntaxError: + return False + + +def pytest_pycollect_makemodule(path, parent): + if _is_launch_test(path): + return LaunchTestModule(path, parent) + elif path.basename == '__init__.py': + return pytest.Package(path, parent) + return pytest.Module(path, parent) + + +def pytest_addhooks(pluginmanager): + import launch_testing.pytest.hookspecs as hookspecs + pluginmanager.add_hookspecs(hookspecs) + + +def pytest_addoption(parser): + parser.addoption( + '--launch-args', action='append', nargs='*', + default=[], help='One or more Launch test arguments' + ) + + +def pytest_launch_test_makerunner(test_runs, launch_args, debug): + return LaunchTestRunner( + test_runs=test_runs, launch_file_arguments=launch_args, debug=debug + ) diff --git a/launch_testing/launch_testing/pytest/hookspecs.py b/launch_testing/launch_testing/pytest/hookspecs.py new file mode 100644 index 000000000..736da0211 --- /dev/null +++ b/launch_testing/launch_testing/pytest/hookspecs.py @@ -0,0 +1,21 @@ +# Copyright 2019 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + + +@pytest.hookspec(firstresult=True) +def pytest_launch_test_makerunner(test_runs, launch_args, debug): + """Py.test hook for launch tests' runner construction.""" + pass diff --git a/launch_testing/setup.py b/launch_testing/setup.py index bac74a3ae..06b49a430 100644 --- a/launch_testing/setup.py +++ b/launch_testing/setup.py @@ -14,7 +14,8 @@ ('share/launch_testing/examples', glob.glob('examples/[!_]**')), ], entry_points={ - 'console_scripts': ['launch_test=launch_testing.launch_test:main'] + 'console_scripts': ['launch_test=launch_testing.launch_test:main'], + 'pytest11': ['launch = launch_testing.pytest.hooks'], }, install_requires=['setuptools'], zip_safe=True, From 67d53fe9058ab590bcac33418e6013bb2539f99e Mon Sep 17 00:00:00 2001 From: Michel Hidalgo <michel@ekumenlabs.com> Date: Fri, 16 Aug 2019 10:24:23 -0300 Subject: [PATCH 02/13] Test examples using pytest. Signed-off-by: Michel Hidalgo <michel@ekumenlabs.com> --- launch_testing/pytest.ini | 2 + .../launch_testing}/examples/README.md | 23 +++++----- .../examples/args_launch_test.py} | 0 .../examples/context_launch_test.py} | 0 .../examples/good_proc_launch_test.py} | 2 +- .../examples/parameters_launch_test.py} | 0 .../examples/terminating_proc_launch_test.py} | 0 .../test/launch_testing/test_examples.py | 45 ------------------- 8 files changed, 15 insertions(+), 57 deletions(-) rename launch_testing/{ => test/launch_testing}/examples/README.md (76%) rename launch_testing/{examples/args.test.py => test/launch_testing/examples/args_launch_test.py} (100%) rename launch_testing/{examples/example_test_context.test.py => test/launch_testing/examples/context_launch_test.py} (100%) rename launch_testing/{examples/good_proc.test.py => test/launch_testing/examples/good_proc_launch_test.py} (97%) rename launch_testing/{examples/parameters.test.py => test/launch_testing/examples/parameters_launch_test.py} (100%) rename launch_testing/{examples/terminating_proc.test.py => test/launch_testing/examples/terminating_proc_launch_test.py} (100%) delete mode 100644 launch_testing/test/launch_testing/test_examples.py diff --git a/launch_testing/pytest.ini b/launch_testing/pytest.ini index 0535da1e7..5419594c1 100644 --- a/launch_testing/pytest.ini +++ b/launch_testing/pytest.ini @@ -1,3 +1,5 @@ [pytest] # Set testpaths, otherwise pytest finds 'tests' in the examples directory testpaths = test +# Add arguments for launch tests +addopts = --launch-args dut_arg:=test diff --git a/launch_testing/examples/README.md b/launch_testing/test/launch_testing/examples/README.md similarity index 76% rename from launch_testing/examples/README.md rename to launch_testing/test/launch_testing/examples/README.md index 7332a1e0e..ecf20caa5 100644 --- a/launch_testing/examples/README.md +++ b/launch_testing/test/launch_testing/examples/README.md @@ -1,12 +1,12 @@ # Examples -## `good_proc.test.py` +## `good_proc_launch_test.py` Usage: ```sh -launch_test examples/good_proc.test.py +launch_test good_proc_launch_test.py ``` -This test checks a process called good_proc (source found in the [example_processes folder](../example_processes)). +This test checks a process called good_proc (source found in the [example_processes folder](../../../example_processes)). good_proc is a simple python process that prints "Loop 1, Loop2, etc. every second until it's terminated with ctrl+c. The test will launch the process, wait for a few loops to complete by monitoring stdout, then terminate the process and run some post-shutdown checks. @@ -18,24 +18,25 @@ After shutdown, we run a similar test that checks more output, and also checks t order of the output. `test_out_of_order` demonstrates that the `assertSequentialStdout` context manager is able to detect out of order stdout. -## `terminating_proc.test.py` + +## `terminating_proc_launch_test.py` Usage: ```sh -launch_test examples/terminating_proc.test.py +launch_test terminating_proc_launch_test.py ``` -This test checks proper functionality of the _terminating\_proc_ example (source found in the [example\_processes\ folder](../example\_processes)). +This test checks proper functionality of the _terminating\_proc_ example (source found in the [example_processes folder](../../../example_processes)). -## `args.test.py` +## `args_launch_test.py` Usage to view the arguments: ```sh -launch_test examples/args.test.py --show-args +launch_test args_launch_test.py --show-args ``` Usage to run the test: ```sh -launch_test examples/args.test.py dut_arg:=hey +launch_test args_launch_test.py dut_arg:=hey ``` This example shows how to pass arguments into a launch test. The arguments are made avilable in the launch description via a launch.substitutions.LaunchConfiguration. The arguments are made @@ -43,11 +44,11 @@ available to the test cases via a self.test_args dictionary This example will fail if no arguments are passed. -## `example_test_context.test.py` +## `context_launch_test.py` Usage: ```sh -launch_test examples/example_test_context.test.py +launch_test context_launch_test.py ``` This example shows how the `generate_test_description` function can return a tuple where the second item is a dictionary of objects that will be injected into the individual test cases. Tests that diff --git a/launch_testing/examples/args.test.py b/launch_testing/test/launch_testing/examples/args_launch_test.py similarity index 100% rename from launch_testing/examples/args.test.py rename to launch_testing/test/launch_testing/examples/args_launch_test.py diff --git a/launch_testing/examples/example_test_context.test.py b/launch_testing/test/launch_testing/examples/context_launch_test.py similarity index 100% rename from launch_testing/examples/example_test_context.test.py rename to launch_testing/test/launch_testing/examples/context_launch_test.py diff --git a/launch_testing/examples/good_proc.test.py b/launch_testing/test/launch_testing/examples/good_proc_launch_test.py similarity index 97% rename from launch_testing/examples/good_proc.test.py rename to launch_testing/test/launch_testing/examples/good_proc_launch_test.py index c7eb38b83..a72aa9dc6 100644 --- a/launch_testing/examples/good_proc.test.py +++ b/launch_testing/test/launch_testing/examples/good_proc_launch_test.py @@ -86,7 +86,7 @@ def test_full_output(self): def test_out_of_order(self): # This demonstrates that we notice out-of-order IO - with self.assertRaisesRegexp(AssertionError, "'Loop 2' not found"): + with self.assertRaisesRegex(AssertionError, "'Loop 2' not found"): with assertSequentialStdout(self.proc_output, dut_process) as cm: cm.assertInStdout('Loop 1') cm.assertInStdout('Loop 3') diff --git a/launch_testing/examples/parameters.test.py b/launch_testing/test/launch_testing/examples/parameters_launch_test.py similarity index 100% rename from launch_testing/examples/parameters.test.py rename to launch_testing/test/launch_testing/examples/parameters_launch_test.py diff --git a/launch_testing/examples/terminating_proc.test.py b/launch_testing/test/launch_testing/examples/terminating_proc_launch_test.py similarity index 100% rename from launch_testing/examples/terminating_proc.test.py rename to launch_testing/test/launch_testing/examples/terminating_proc_launch_test.py diff --git a/launch_testing/test/launch_testing/test_examples.py b/launch_testing/test/launch_testing/test_examples.py deleted file mode 100644 index 2abc131d7..000000000 --- a/launch_testing/test/launch_testing/test_examples.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2019 Apex.AI, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import glob -import os -import subprocess - -import ament_index_python - -import pytest - - -testdata = glob.glob( - os.path.join( - ament_index_python.get_package_share_directory('launch_testing'), - 'examples', - '*.test.py' - ) -) - - -# This test will automatically run for any *.test.py file in the examples folder and expect -# it to pass -@pytest.mark.parametrize('example_path', testdata, ids=[os.path.basename(d) for d in testdata]) -def test_examples(example_path): - - proc = ['launch_test', example_path] - - # The args.test.py example is a little special - it is required to run with args - # or else it will fail. Hopefully this is the only example we need to special-case - if 'args.test.py' in example_path: - proc.append('dut_arg:=foobarbaz') - - assert 0 == subprocess.run(args=proc).returncode From 94922bc02b6e7a9b83b456b0dfe1d2e4921bdfda Mon Sep 17 00:00:00 2001 From: Michel Hidalgo <michel@ekumenlabs.com> Date: Fri, 16 Aug 2019 12:14:36 -0300 Subject: [PATCH 03/13] Enable downstream customization of launch tests execution. Signed-off-by: Michel Hidalgo <michel@ekumenlabs.com> --- .../launch_testing/pytest/__init__.py | 0 launch_testing/launch_testing/pytest/hooks.py | 48 +++++++++++-------- .../launch_testing/pytest/hookspecs.py | 4 +- 3 files changed, 31 insertions(+), 21 deletions(-) create mode 100644 launch_testing/launch_testing/pytest/__init__.py diff --git a/launch_testing/launch_testing/pytest/__init__.py b/launch_testing/launch_testing/pytest/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/launch_testing/launch_testing/pytest/hooks.py b/launch_testing/launch_testing/pytest/hooks.py index 7e6dc20cf..2c56b42f9 100644 --- a/launch_testing/launch_testing/pytest/hooks.py +++ b/launch_testing/launch_testing/pytest/hooks.py @@ -31,30 +31,34 @@ def __str__(self): class LaunchTestItem(pytest.Item): - def __init__(self, name, parent, test_runs): + def __init__(self, name, parent, test_runs, runner_cls=LaunchTestRunner): super().__init__(name, parent) self.test_runs = test_runs + self.runner_cls = runner_cls def runtest(self): launch_args = sum(( args_set for args_set in self.config.getoption('--launch-args') ), []) - runner = self.ihook.pytest_launch_test_makerunner( + runner = self.runner_cls( test_runs=self.test_runs, - launch_args=launch_args, + launch_file_arguments=launch_args, debug=self.config.getoption('verbose') ) - runner.validate() + try: + runner.validate() + except Exception as e: + raise LaunchTestFailure(message=str(e), results=[]) + results_per_run = runner.run() if any(not result.wasSuccessful() for result in results_per_run.values()): raise LaunchTestFailure( - message=self.name + ' failed', - results=results_per_run + message='some test cases have failed', results=results_per_run ) def repr_failure(self, excinfo): if isinstance(excinfo.value, LaunchTestFailure): - return '\n'.join({ + return excinfo.value.message + ':\n' + '\n'.join({ '{} failed at {}.{}'.format( str(test_run), type(test_case).__name__, @@ -71,9 +75,12 @@ def reportinfo(self): class LaunchTestModule(pytest.File): + def makeitem(self, *args, **kwargs): + return LaunchTestItem(*args, **kwargs) + def collect(self): module = self.fspath.pyimport() - yield LaunchTestItem( + yield self.makeitem( name=module.__name__, parent=self, test_runs=LoadTestsFromPythonModule( module, name=module.__name__ @@ -81,21 +88,30 @@ def collect(self): ) -def _is_launch_test(path): +def find_launch_test_entrypoint(path): try: - return hasattr(path.pyimport(), 'generate_test_description') + return getattr(path.pyimport(), 'generate_test_description', None) except SyntaxError: - return False + return None def pytest_pycollect_makemodule(path, parent): - if _is_launch_test(path): - return LaunchTestModule(path, parent) + entrypoint = find_launch_test_entrypoint(path) + if entrypoint is not None: + ihook = parent.session.gethookproxy(path) + return ihook.pytest_launch_collect_makemodule( + path=path, parent=parent, entrypoint=entrypoint + ) elif path.basename == '__init__.py': return pytest.Package(path, parent) return pytest.Module(path, parent) +@pytest.hookimpl(trylast=True) +def pytest_launch_collect_makemodule(path, parent, entrypoint): + return LaunchTestModule(path, parent) + + def pytest_addhooks(pluginmanager): import launch_testing.pytest.hookspecs as hookspecs pluginmanager.add_hookspecs(hookspecs) @@ -106,9 +122,3 @@ def pytest_addoption(parser): '--launch-args', action='append', nargs='*', default=[], help='One or more Launch test arguments' ) - - -def pytest_launch_test_makerunner(test_runs, launch_args, debug): - return LaunchTestRunner( - test_runs=test_runs, launch_file_arguments=launch_args, debug=debug - ) diff --git a/launch_testing/launch_testing/pytest/hookspecs.py b/launch_testing/launch_testing/pytest/hookspecs.py index 736da0211..89249d5ea 100644 --- a/launch_testing/launch_testing/pytest/hookspecs.py +++ b/launch_testing/launch_testing/pytest/hookspecs.py @@ -16,6 +16,6 @@ @pytest.hookspec(firstresult=True) -def pytest_launch_test_makerunner(test_runs, launch_args, debug): - """Py.test hook for launch tests' runner construction.""" +def pytest_launch_collect_makemodule(path, parent, entrypoint): + """Make launch test module appropriate for the found test entrypoint.""" pass From 8014b4c94a5cded069bca06d4f031398f6c76a4b Mon Sep 17 00:00:00 2001 From: Michel Hidalgo <michel@ekumenlabs.com> Date: Wed, 21 Aug 2019 12:29:38 -0300 Subject: [PATCH 04/13] Move launch testing examples README content to the root README. Signed-off-by: Michel Hidalgo <michel@ekumenlabs.com> --- launch_testing/README.md | 64 +++++++++++++++++++ .../test/launch_testing/examples/README.md | 55 ---------------- 2 files changed, 64 insertions(+), 55 deletions(-) delete mode 100644 launch_testing/test/launch_testing/examples/README.md diff --git a/launch_testing/README.md b/launch_testing/README.md index b26ffcbbc..417673e4e 100644 --- a/launch_testing/README.md +++ b/launch_testing/README.md @@ -179,3 +179,67 @@ add_launch_test( ARGS "arg1:=foo" ) ``` + +## Examples + +### `good_proc_launch_test.py` + +Usage: + +```sh +launch_test test/launch_testing/examples/good_proc_launch_test.py +``` + +This test checks a process called good_proc (source found in the [example_processes folder](example_processes)). +good_proc is a simple python process that prints "Loop 1, Loop2, etc. every second until it's terminated with ctrl+c. +The test will launch the process, wait for a few loops to complete by monitoring stdout, then terminate the process +and run some post-shutdown checks. + +The pre-shutdown tests check that "Loop 1, Loop 2, Loop 3, Loop 4" +are all printed to stdout. Once this test finishes, the process under test is shut down + +After shutdown, we run a similar test that checks more output, and also checks the +order of the output. `test_out_of_order` demonstrates that the `assertSequentialStdout` +context manager is able to detect out of order stdout. + +### `terminating_proc_launch_test.py` + +Usage: + +```sh +launch_test test/launch_testing/examples/terminating_proc_launch_test.py +``` + +This test checks proper functionality of the _terminating\_proc_ example (source found in the [example_processes folder](example_processes)). + +### `args_launch_test.py` + +Usage to view the arguments: + +```sh +launch_test test/launch_testing/examples/args_launch_test.py --show-args +``` + +Usage to run the test: + +```sh +launch_test test/launch_testing/examples/args_launch_test.py dut_arg:=hey +``` + +This example shows how to pass arguments into a launch test. The arguments are made avilable +in the launch description via a launch.substitutions.LaunchConfiguration. The arguments are made +available to the test cases via a self.test_args dictionary + +This example will fail if no arguments are passed. + +### `context_launch_test.py` + +Usage: + +```sh +launch_test test/launch_testing/examples/context_launch_test.py +``` + +This example shows how the `generate_test_description` function can return a tuple where the second +item is a dictionary of objects that will be injected into the individual test cases. Tests that +wish to use elements of the test context can add arguments with names matching the keys of the dictionary. diff --git a/launch_testing/test/launch_testing/examples/README.md b/launch_testing/test/launch_testing/examples/README.md deleted file mode 100644 index ecf20caa5..000000000 --- a/launch_testing/test/launch_testing/examples/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# Examples - -## `good_proc_launch_test.py` - -Usage: -```sh -launch_test good_proc_launch_test.py -``` -This test checks a process called good_proc (source found in the [example_processes folder](../../../example_processes)). -good_proc is a simple python process that prints "Loop 1, Loop2, etc. every second until it's terminated with ctrl+c. -The test will launch the process, wait for a few loops to complete by monitoring stdout, then terminate the process -and run some post-shutdown checks. - -The pre-shutdown tests check that "Loop 1, Loop 2, Loop 3, Loop 4" -are all printed to stdout. Once this test finishes, the process under test is shut down - -After shutdown, we run a similar test that checks more output, and also checks the -order of the output. `test_out_of_order` demonstrates that the `assertSequentialStdout` -context manager is able to detect out of order stdout. - - -## `terminating_proc_launch_test.py` - -Usage: -```sh -launch_test terminating_proc_launch_test.py -``` - -This test checks proper functionality of the _terminating\_proc_ example (source found in the [example_processes folder](../../../example_processes)). - -## `args_launch_test.py` - -Usage to view the arguments: -```sh -launch_test args_launch_test.py --show-args -``` -Usage to run the test: -```sh -launch_test args_launch_test.py dut_arg:=hey -``` -This example shows how to pass arguments into a launch test. The arguments are made avilable -in the launch description via a launch.substitutions.LaunchConfiguration. The arguments are made -available to the test cases via a self.test_args dictionary - -This example will fail if no arguments are passed. - -## `context_launch_test.py` - -Usage: -```sh -launch_test context_launch_test.py -``` -This example shows how the `generate_test_description` function can return a tuple where the second -item is a dictionary of objects that will be injected into the individual test cases. Tests that -wish to use elements of the test context can add arguments with names matching the keys of the dictionary. From 44917596b72ea7d105147e6d3f57af8f10d01446 Mon Sep 17 00:00:00 2001 From: Michel Hidalgo <michel@ekumenlabs.com> Date: Wed, 21 Aug 2019 12:43:41 -0300 Subject: [PATCH 05/13] Mark launch tests explicitly. Signed-off-by: Michel Hidalgo <michel@ekumenlabs.com> --- launch_testing/launch_testing/pytest/hooks.py | 16 +++++++++++++--- .../launch_testing/examples/args_launch_test.py | 4 ++++ .../examples/context_launch_test.py | 3 +++ .../examples/good_proc_launch_test.py | 3 +++ .../examples/parameters_launch_test.py | 3 +++ .../examples/terminating_proc_launch_test.py | 3 +++ 6 files changed, 29 insertions(+), 3 deletions(-) diff --git a/launch_testing/launch_testing/pytest/hooks.py b/launch_testing/launch_testing/pytest/hooks.py index 2c56b42f9..cd0756a83 100644 --- a/launch_testing/launch_testing/pytest/hooks.py +++ b/launch_testing/launch_testing/pytest/hooks.py @@ -99,17 +99,21 @@ def pytest_pycollect_makemodule(path, parent): entrypoint = find_launch_test_entrypoint(path) if entrypoint is not None: ihook = parent.session.gethookproxy(path) - return ihook.pytest_launch_collect_makemodule( + module = ihook.pytest_launch_collect_makemodule( path=path, parent=parent, entrypoint=entrypoint ) - elif path.basename == '__init__.py': + if module is not None: + return module + if path.basename == '__init__.py': return pytest.Package(path, parent) return pytest.Module(path, parent) @pytest.hookimpl(trylast=True) def pytest_launch_collect_makemodule(path, parent, entrypoint): - return LaunchTestModule(path, parent) + marks = getattr(entrypoint, 'pytestmark', []) + if marks and any(m.name == 'launch_test' for m in marks): + return LaunchTestModule(path, parent) def pytest_addhooks(pluginmanager): @@ -122,3 +126,9 @@ def pytest_addoption(parser): '--launch-args', action='append', nargs='*', default=[], help='One or more Launch test arguments' ) + + +def pytest_configure(config): + config.addinivalue_line( + 'markers', 'launch_test: mark a generate_test_description function as a launch test entrypoint' + ) diff --git a/launch_testing/test/launch_testing/examples/args_launch_test.py b/launch_testing/test/launch_testing/examples/args_launch_test.py index 23492aecf..891efeeda 100644 --- a/launch_testing/test/launch_testing/examples/args_launch_test.py +++ b/launch_testing/test/launch_testing/examples/args_launch_test.py @@ -25,6 +25,9 @@ import launch_testing import launch_testing.util +import pytest + + dut_process = launch.actions.ExecuteProcess( cmd=[ sys.executable, @@ -40,6 +43,7 @@ ) +@pytest.mark.launch_test def generate_test_description(ready_fn): return launch.LaunchDescription([ diff --git a/launch_testing/test/launch_testing/examples/context_launch_test.py b/launch_testing/test/launch_testing/examples/context_launch_test.py index df4df3558..6adafc853 100644 --- a/launch_testing/test/launch_testing/examples/context_launch_test.py +++ b/launch_testing/test/launch_testing/examples/context_launch_test.py @@ -24,6 +24,8 @@ import launch_testing from launch_testing.asserts import assertSequentialStdout +import pytest + def get_test_process_action(): TEST_PROC_PATH = os.path.join( @@ -42,6 +44,7 @@ def get_test_process_action(): # This launch description shows the prefered way to let the tests access launch actions. By # adding them to the test context, it's not necessary to scope them at the module level like in # the good_proc.test.py example +@pytest.mark.launch_test def generate_test_description(ready_fn): dut_process = get_test_process_action() diff --git a/launch_testing/test/launch_testing/examples/good_proc_launch_test.py b/launch_testing/test/launch_testing/examples/good_proc_launch_test.py index a72aa9dc6..215dffeb1 100644 --- a/launch_testing/test/launch_testing/examples/good_proc_launch_test.py +++ b/launch_testing/test/launch_testing/examples/good_proc_launch_test.py @@ -24,6 +24,8 @@ import launch_testing from launch_testing.asserts import assertSequentialStdout +import pytest + TEST_PROC_PATH = os.path.join( ament_index_python.get_package_prefix('launch_testing'), @@ -41,6 +43,7 @@ ) +@pytest.mark.launch_test def generate_test_description(ready_fn): return launch.LaunchDescription([ diff --git a/launch_testing/test/launch_testing/examples/parameters_launch_test.py b/launch_testing/test/launch_testing/examples/parameters_launch_test.py index 69fefa62d..4596e81e9 100644 --- a/launch_testing/test/launch_testing/examples/parameters_launch_test.py +++ b/launch_testing/test/launch_testing/examples/parameters_launch_test.py @@ -23,7 +23,10 @@ import launch_testing import launch_testing.util +import pytest + +@pytest.mark.launch_test @launch_testing.parametrize('arg_param', ['thing=On', 'thing=Off', 'flag1']) def generate_test_description(arg_param, ready_fn): diff --git a/launch_testing/test/launch_testing/examples/terminating_proc_launch_test.py b/launch_testing/test/launch_testing/examples/terminating_proc_launch_test.py index a6d78331c..cbcadebec 100644 --- a/launch_testing/test/launch_testing/examples/terminating_proc_launch_test.py +++ b/launch_testing/test/launch_testing/examples/terminating_proc_launch_test.py @@ -25,6 +25,8 @@ import launch_testing.asserts import launch_testing.tools +import pytest + def get_test_process_action(*, args=[]): test_proc_path = os.path.join( @@ -40,6 +42,7 @@ def get_test_process_action(*, args=[]): ) +@pytest.mark.launch_test def generate_test_description(ready_fn): return launch.LaunchDescription([ launch_testing.util.KeepAliveProc(), From 5be3345e9373b85456a1d578b2b6d8b58475e10c Mon Sep 17 00:00:00 2001 From: Michel Hidalgo <michel@ekumenlabs.com> Date: Wed, 21 Aug 2019 13:28:20 -0300 Subject: [PATCH 06/13] Refactor parameterization support to play nice with pytest markers. Signed-off-by: Michel Hidalgo <michel@ekumenlabs.com> --- launch_testing/launch_testing/loader.py | 11 ++++++----- launch_testing/launch_testing/parametrize.py | 20 +++++++------------- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/launch_testing/launch_testing/loader.py b/launch_testing/launch_testing/loader.py index 2aeedd75f..ada17597a 100644 --- a/launch_testing/launch_testing/loader.py +++ b/launch_testing/launch_testing/loader.py @@ -106,11 +106,12 @@ def _format_params(self): def LoadTestsFromPythonModule(module, *, name='launch_tests'): - - if hasattr(module.generate_test_description, '__parametrized__'): - normalized_test_description_func = module.generate_test_description + if not hasattr(module.generate_test_description, '__parametrized__'): + normalized_test_description_func = ( + lambda: [(module.generate_test_description, {})] + ) else: - normalized_test_description_func = [(module.generate_test_description, {})] + normalized_test_description_func = module.generate_test_description # If our test description is parameterized, we'll load a set of tests for each # individual launch @@ -119,7 +120,7 @@ def LoadTestsFromPythonModule(module, *, name='launch_tests'): args, PreShutdownTestLoader().loadTestsFromModule(module), PostShutdownTestLoader().loadTestsFromModule(module)) - for description, args in normalized_test_description_func] + for description, args in normalized_test_description_func()] def PreShutdownTestLoader(): diff --git a/launch_testing/launch_testing/parametrize.py b/launch_testing/launch_testing/parametrize.py index ec4b2e137..a90abb1aa 100644 --- a/launch_testing/launch_testing/parametrize.py +++ b/launch_testing/launch_testing/parametrize.py @@ -35,22 +35,16 @@ def parametrize(argnames, argvalues): argnames = [x.strip() for x in argnames.split(',') if x.strip()] argvalues = [_normalize_to_tuple(x) for x in argvalues] - class decorator: - - def __init__(self, func): - setattr(self, '__parametrized__', True) - self.__calls = [] - + def _decorator(func): + @functools.wraps(func) + def _wrapped(): for val in argvalues: partial_args = dict(zip(argnames, val)) partial = functools.partial(func, **partial_args) functools.update_wrapper(partial, func) - self.__calls.append( - (partial, partial_args) - ) - - def __iter__(self): - return iter(self.__calls) + yield partial, partial_args + setattr(_wrapped, '__parametrized__', True) + return _wrapped - return decorator + return _decorator From e7cc2b93d7319a1647a6fc0548849867d788d060 Mon Sep 17 00:00:00 2001 From: Michel Hidalgo <michel@ekumenlabs.com> Date: Wed, 21 Aug 2019 13:28:50 -0300 Subject: [PATCH 07/13] Improve pytest output on launch test failure. Signed-off-by: Michel Hidalgo <michel@ekumenlabs.com> --- launch_testing/launch_testing/pytest/hooks.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/launch_testing/launch_testing/pytest/hooks.py b/launch_testing/launch_testing/pytest/hooks.py index cd0756a83..87fe7358c 100644 --- a/launch_testing/launch_testing/pytest/hooks.py +++ b/launch_testing/launch_testing/pytest/hooks.py @@ -45,12 +45,10 @@ def runtest(self): launch_file_arguments=launch_args, debug=self.config.getoption('verbose') ) - try: - runner.validate() - except Exception as e: - raise LaunchTestFailure(message=str(e), results=[]) + runner.validate() results_per_run = runner.run() + if any(not result.wasSuccessful() for result in results_per_run.values()): raise LaunchTestFailure( message='some test cases have failed', results=results_per_run @@ -67,7 +65,8 @@ def repr_failure(self, excinfo): for test_run, test_result in excinfo.value.results.items() for test_case, _ in (test_result.errors + test_result.failures) if not test_result.wasSuccessful() - }) + }) if excinfo.value.results else '' + return super().repr_failure(excinfo) def reportinfo(self): return self.fspath, 0, 'launch tests: {}'.format(self.name) From b658591a2c4d027511ada0d27ec843fd2426d5a0 Mon Sep 17 00:00:00 2001 From: Michel Hidalgo <michel@ekumenlabs.com> Date: Wed, 21 Aug 2019 13:30:07 -0300 Subject: [PATCH 08/13] Please flake8 Signed-off-by: Michel Hidalgo <michel@ekumenlabs.com> --- launch_testing/launch_testing/pytest/hooks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/launch_testing/launch_testing/pytest/hooks.py b/launch_testing/launch_testing/pytest/hooks.py index 87fe7358c..796ba83df 100644 --- a/launch_testing/launch_testing/pytest/hooks.py +++ b/launch_testing/launch_testing/pytest/hooks.py @@ -129,5 +129,6 @@ def pytest_addoption(parser): def pytest_configure(config): config.addinivalue_line( - 'markers', 'launch_test: mark a generate_test_description function as a launch test entrypoint' + 'markers', + 'launch_test: mark a generate_test_description function as a launch test entrypoint' ) From d15a47ce7c53194b8f68750a1799e74e15d4e940 Mon Sep 17 00:00:00 2001 From: Michel Hidalgo <michel@ekumenlabs.com> Date: Wed, 21 Aug 2019 14:10:18 -0300 Subject: [PATCH 09/13] Fix failing launch_testing tests. Signed-off-by: Michel Hidalgo <michel@ekumenlabs.com> --- launch_testing/setup.py | 2 +- .../test/launch_testing/test_parametrize_decorator.py | 8 ++++---- .../test/launch_testing/test_print_arguments.py | 4 ++-- launch_testing/test/launch_testing/test_xml_output.py | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/launch_testing/setup.py b/launch_testing/setup.py index 06b49a430..c5f00d0e1 100644 --- a/launch_testing/setup.py +++ b/launch_testing/setup.py @@ -11,7 +11,7 @@ data_files=[ ('share/ament_index/resource_index/packages', ['resource/launch_testing']), ('lib/launch_testing', glob.glob('example_processes/**')), - ('share/launch_testing/examples', glob.glob('examples/[!_]**')), + ('share/launch_testing/examples', glob.glob('test/launch_testing/examples/[!_]**')), ], entry_points={ 'console_scripts': ['launch_test=launch_testing.launch_test:main'], diff --git a/launch_testing/test/launch_testing/test_parametrize_decorator.py b/launch_testing/test/launch_testing/test_parametrize_decorator.py index 4cae428cc..8774dca02 100644 --- a/launch_testing/test/launch_testing/test_parametrize_decorator.py +++ b/launch_testing/test/launch_testing/test_parametrize_decorator.py @@ -32,7 +32,7 @@ def test_binding_arguments(): def fake_test_description(val): results.append(val) - for func, params in fake_test_description: + for func, params in fake_test_description(): func() assert results == [1, 2, 3] @@ -46,7 +46,7 @@ def test_binding_one_tuples(): def fake_test_description(val): results.append(val) - for func, params in fake_test_description: + for func, params in fake_test_description(): func() assert results == [1, 2, 3] @@ -60,7 +60,7 @@ def test_partial_binding(): def fake_test_description(val, arg): results.append((val, arg)) - for index, (func, params) in enumerate(fake_test_description): + for index, (func, params) in enumerate(fake_test_description()): func(arg=index) assert results == [('x', 0), ('y', 1), ('z', 2)] @@ -74,7 +74,7 @@ def test_multiple_args(): def fake_test_description(arg_1, arg_2): results.append((arg_1, arg_2)) - for index, (func, params) in enumerate(fake_test_description): + for index, (func, params) in enumerate(fake_test_description()): func() assert results == [(5, 10), (15, 20), (25, 30)] diff --git a/launch_testing/test/launch_testing/test_print_arguments.py b/launch_testing/test/launch_testing/test_print_arguments.py index b542dc12a..f643e03ec 100644 --- a/launch_testing/test/launch_testing/test_print_arguments.py +++ b/launch_testing/test/launch_testing/test_print_arguments.py @@ -25,7 +25,7 @@ def test_print_args(): testpath = os.path.join( ament_index_python.get_package_share_directory('launch_testing'), 'examples', - 'args.test.py', + 'args_launch_test.py', ) completed_process = subprocess.run( @@ -49,7 +49,7 @@ def test_no_args_to_print(): testpath = os.path.join( ament_index_python.get_package_share_directory('launch_testing'), 'examples', - 'good_proc.test.py', + 'good_proc_launch_test.py', ) completed_process = subprocess.run( diff --git a/launch_testing/test/launch_testing/test_xml_output.py b/launch_testing/test/launch_testing/test_xml_output.py index 0e859989c..3ff57f049 100644 --- a/launch_testing/test/launch_testing/test_xml_output.py +++ b/launch_testing/test/launch_testing/test_xml_output.py @@ -39,7 +39,7 @@ def setUpClass(cls): path = os.path.join( ament_index_python.get_package_share_directory('launch_testing'), 'examples', - 'good_proc.test.py' + 'good_proc_launch_test.py' ) assert 0 == subprocess.run( @@ -65,7 +65,7 @@ def test_pre_and_post(self): # Expecting an element called '{package}.{test_base_name}.launch_tests' since this # was not parametrized self.assertEqual( - test_suite.attrib['name'], 'test_xml_output.good_proc.test.launch_tests' + test_suite.attrib['name'], 'test_xml_output.good_proc_launch_test.launch_tests' ) # Drilling down a little further, we expect the class names to show up in the testcase From b4b56a23f9f8c6a9b8d917307f4055f759dddf6b Mon Sep 17 00:00:00 2001 From: Michel Hidalgo <michel@ekumenlabs.com> Date: Wed, 21 Aug 2019 14:51:36 -0300 Subject: [PATCH 10/13] Prevent pytest from performing any assertion rewriting outside the launch_testing plugin. Signed-off-by: Michel Hidalgo <michel@ekumenlabs.com> --- launch_testing/launch_testing/asserts/assert_exit_codes.py | 7 +++++++ launch_testing/launch_testing/asserts/assert_output.py | 7 +++++++ .../launch_testing/asserts/assert_sequential_output.py | 7 +++++++ launch_testing/launch_testing/io_handler.py | 7 +++++++ launch_testing/launch_testing/proc_info_handler.py | 7 +++++++ 5 files changed, 35 insertions(+) diff --git a/launch_testing/launch_testing/asserts/assert_exit_codes.py b/launch_testing/launch_testing/asserts/assert_exit_codes.py index 69425904c..0aee59772 100644 --- a/launch_testing/launch_testing/asserts/assert_exit_codes.py +++ b/launch_testing/launch_testing/asserts/assert_exit_codes.py @@ -12,6 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +A module providing exit code assertions. + +PYTEST_DONT_REWRITE +""" + + import os from ..util import resolveProcesses diff --git a/launch_testing/launch_testing/asserts/assert_output.py b/launch_testing/launch_testing/asserts/assert_output.py index 5d58a5c85..3d3147140 100644 --- a/launch_testing/launch_testing/asserts/assert_output.py +++ b/launch_testing/launch_testing/asserts/assert_output.py @@ -12,6 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +A module providing process output assertions. + +PYTEST_DONT_REWRITE +""" + + import os from osrf_pycommon.terminal_color import remove_ansi_escape_senquences diff --git a/launch_testing/launch_testing/asserts/assert_sequential_output.py b/launch_testing/launch_testing/asserts/assert_sequential_output.py index 2f66ae37d..801992b30 100644 --- a/launch_testing/launch_testing/asserts/assert_sequential_output.py +++ b/launch_testing/launch_testing/asserts/assert_sequential_output.py @@ -12,6 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +A module providing process output sequence assertions. + +PYTEST_DONT_REWRITE +""" + + from contextlib import contextmanager from ..util import resolveProcesses diff --git a/launch_testing/launch_testing/io_handler.py b/launch_testing/launch_testing/io_handler.py index cf4d180e0..2f0d00590 100644 --- a/launch_testing/launch_testing/io_handler.py +++ b/launch_testing/launch_testing/io_handler.py @@ -12,6 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +A module providing process IO capturing classes. + +PYTEST_DONT_REWRITE +""" + + import threading from .asserts.assert_output import assertInStdout diff --git a/launch_testing/launch_testing/proc_info_handler.py b/launch_testing/launch_testing/proc_info_handler.py index 026a91e8e..56b533913 100644 --- a/launch_testing/launch_testing/proc_info_handler.py +++ b/launch_testing/launch_testing/proc_info_handler.py @@ -12,6 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +A module providing process info capturing classes. + +PYTEST_DONT_REWRITE +""" + + import threading from launch.actions import ExecuteProcess # noqa from launch.events.process import ProcessExited From 0705777c90b01c4b253ba5f03c464ef09ff220bb Mon Sep 17 00:00:00 2001 From: Michel Hidalgo <michel@ekumenlabs.com> Date: Thu, 22 Aug 2019 15:14:42 -0300 Subject: [PATCH 11/13] Change explicit setattr for regular assignment. Signed-off-by: Michel Hidalgo <michel@ekumenlabs.com> --- launch_testing/launch_testing/parametrize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launch_testing/launch_testing/parametrize.py b/launch_testing/launch_testing/parametrize.py index a90abb1aa..04c0f794c 100644 --- a/launch_testing/launch_testing/parametrize.py +++ b/launch_testing/launch_testing/parametrize.py @@ -44,7 +44,7 @@ def _wrapped(): partial = functools.partial(func, **partial_args) functools.update_wrapper(partial, func) yield partial, partial_args - setattr(_wrapped, '__parametrized__', True) + _wrapped.__parametrized__ = True return _wrapped return _decorator From c7e6f8043a9643ac2f3a6f2206c7186011dff32e Mon Sep 17 00:00:00 2001 From: Michel Hidalgo <michel@ekumenlabs.com> Date: Thu, 22 Aug 2019 15:18:28 -0300 Subject: [PATCH 12/13] Better document PYTEST_DONT_REWRITE docstring. Signed-off-by: Michel Hidalgo <michel@ekumenlabs.com> --- launch_testing/launch_testing/asserts/assert_exit_codes.py | 4 +++- launch_testing/launch_testing/asserts/assert_output.py | 4 +++- .../launch_testing/asserts/assert_sequential_output.py | 4 +++- launch_testing/launch_testing/io_handler.py | 4 +++- launch_testing/launch_testing/proc_info_handler.py | 4 +++- 5 files changed, 15 insertions(+), 5 deletions(-) diff --git a/launch_testing/launch_testing/asserts/assert_exit_codes.py b/launch_testing/launch_testing/asserts/assert_exit_codes.py index 0aee59772..5d3e73cd4 100644 --- a/launch_testing/launch_testing/asserts/assert_exit_codes.py +++ b/launch_testing/launch_testing/asserts/assert_exit_codes.py @@ -15,7 +15,9 @@ """ A module providing exit code assertions. -PYTEST_DONT_REWRITE +To prevent pytest from rewriting this module assertions, please PYTEST_DONT_REWRITE. +See https://docs.pytest.org/en/latest/assert.html#disabling-assert-rewriting for +further reference. """ diff --git a/launch_testing/launch_testing/asserts/assert_output.py b/launch_testing/launch_testing/asserts/assert_output.py index 3d3147140..b85190bc7 100644 --- a/launch_testing/launch_testing/asserts/assert_output.py +++ b/launch_testing/launch_testing/asserts/assert_output.py @@ -15,7 +15,9 @@ """ A module providing process output assertions. -PYTEST_DONT_REWRITE +To prevent pytest from rewriting this module assertions, please PYTEST_DONT_REWRITE. +See https://docs.pytest.org/en/latest/assert.html#disabling-assert-rewriting for +further reference. """ diff --git a/launch_testing/launch_testing/asserts/assert_sequential_output.py b/launch_testing/launch_testing/asserts/assert_sequential_output.py index 801992b30..891f1718f 100644 --- a/launch_testing/launch_testing/asserts/assert_sequential_output.py +++ b/launch_testing/launch_testing/asserts/assert_sequential_output.py @@ -15,7 +15,9 @@ """ A module providing process output sequence assertions. -PYTEST_DONT_REWRITE +To prevent pytest from rewriting this module assertions, please PYTEST_DONT_REWRITE. +See https://docs.pytest.org/en/latest/assert.html#disabling-assert-rewriting for +further reference. """ diff --git a/launch_testing/launch_testing/io_handler.py b/launch_testing/launch_testing/io_handler.py index 2f0d00590..cc000b5b4 100644 --- a/launch_testing/launch_testing/io_handler.py +++ b/launch_testing/launch_testing/io_handler.py @@ -15,7 +15,9 @@ """ A module providing process IO capturing classes. -PYTEST_DONT_REWRITE +To prevent pytest from rewriting this module assertions, please PYTEST_DONT_REWRITE. +See https://docs.pytest.org/en/latest/assert.html#disabling-assert-rewriting for +further reference. """ diff --git a/launch_testing/launch_testing/proc_info_handler.py b/launch_testing/launch_testing/proc_info_handler.py index 56b533913..6abf2c652 100644 --- a/launch_testing/launch_testing/proc_info_handler.py +++ b/launch_testing/launch_testing/proc_info_handler.py @@ -15,7 +15,9 @@ """ A module providing process info capturing classes. -PYTEST_DONT_REWRITE +To prevent pytest from rewriting this module assertions, please PYTEST_DONT_REWRITE. +See https://docs.pytest.org/en/latest/assert.html#disabling-assert-rewriting for +further reference. """ From dfa47c488b01c74051288b980e1ded18b61f2c74 Mon Sep 17 00:00:00 2001 From: Michel Hidalgo <michel@ekumenlabs.com> Date: Thu, 22 Aug 2019 17:00:53 -0300 Subject: [PATCH 13/13] Please flake8 Signed-off-by: Michel Hidalgo <michel@ekumenlabs.com> --- launch_testing/launch_testing/parametrize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launch_testing/launch_testing/parametrize.py b/launch_testing/launch_testing/parametrize.py index 04c0f794c..38782af61 100644 --- a/launch_testing/launch_testing/parametrize.py +++ b/launch_testing/launch_testing/parametrize.py @@ -44,7 +44,7 @@ def _wrapped(): partial = functools.partial(func, **partial_args) functools.update_wrapper(partial, func) yield partial, partial_args - _wrapped.__parametrized__ = True + _wrapped.__parametrized__ = True return _wrapped return _decorator