diff --git a/Jenkinsfile b/Jenkinsfile index 4daeb27cef..14953f6927 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -133,7 +133,8 @@ pipeline { stage('HITL tests') { steps { script { - docker_run("HITL tests", 35, 'PANDAS_JUNGLE=23002d000851393038373731 PANDAS_EXCLUDE="1d0002000c51303136383232 2f002e000c51303136383232" ./tests/hitl/test.sh') + docker_run("parallel tests", 5, 'PANDAS_JUNGLE=23002d000851393038373731 PANDAS_EXCLUDE="1d0002000c51303136383232 2f002e000c51303136383232" ./tests/hitl/run_parallel_tests.sh') + docker_run("serial tests", 9, 'PANDAS_JUNGLE=23002d000851393038373731 PANDAS_EXCLUDE="1d0002000c51303136383232 2f002e000c51303136383232" ./tests/hitl/run_serial_tests.sh') } } } diff --git a/requirements.txt b/requirements.txt index 6693a0af83..41c5226aa2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,8 @@ hexdump>=3.3 pycryptodome==3.9.8 tqdm>=4.14.0 pytest +pytest-mock +pytest-xdist pytest-timeouts parameterized requests @@ -14,5 +16,4 @@ pre-commit==2.13.0 scons==4.4.0 flaky spidev -pytest-mock termcolor diff --git a/tests/hitl/conftest.py b/tests/hitl/conftest.py index d825c60e30..c337013f05 100644 --- a/tests/hitl/conftest.py +++ b/tests/hitl/conftest.py @@ -1,15 +1,14 @@ -import concurrent.futures import os -import time import pytest +import concurrent.futures -from panda import Panda, PandaDFU +from panda import Panda, PandaDFU, PandaJungle from panda.tests.hitl.helpers import clear_can_buffers -NO_JUNGLE = os.environ.get("NO_JUNGLE", "0") == "1" -if not NO_JUNGLE: - from panda import PandaJungle - +# needed to get output when using xdist +if "DEBUG" in os.environ: + import sys + sys.stdout = sys.stderr SPEED_NORMAL = 500 SPEED_GMLAN = 33.3 @@ -17,10 +16,15 @@ JUNGLE_SERIAL = os.getenv("PANDAS_JUNGLE") +NO_JUNGLE = os.environ.get("NO_JUNGLE", "0") == "1" PANDAS_EXCLUDE = os.getenv("PANDAS_EXCLUDE", "").strip().split(" ") -PARTIAL_TESTS = os.environ.get("PARTIAL_TESTS", "0") == "1" HW_TYPES = os.environ.get("HW_TYPES", None) +PARALLEL = "PARALLEL" in os.environ +NON_PARALLEL = "NON_PARALLEL" in os.environ +if PARALLEL: + NO_JUNGLE = True + class PandaGroup: H7 = (Panda.HW_TYPE_RED_PANDA, Panda.HW_TYPE_RED_PANDA_V2, Panda.HW_TYPE_TRES) GEN2 = (Panda.HW_TYPE_BLACK_PANDA, Panda.HW_TYPE_UNO, Panda.HW_TYPE_DOS) + H7 @@ -43,7 +47,7 @@ def init_all_pandas(): for serial in Panda.list(): if serial not in PANDAS_EXCLUDE: - with Panda(serial=serial) as p: + with Panda(serial=serial, claim=False) as p: ptype = bytes(p.get_type()) if ptype in PandaGroup.TESTED: _all_pandas[serial] = ptype @@ -54,7 +58,7 @@ def init_all_pandas(): print(f"{len(_all_pandas)} total pandas") init_all_pandas() -_all_panda_serials = list(_all_pandas.keys()) +_all_panda_serials = list(sorted(_all_pandas.keys())) def init_jungle(): @@ -86,6 +90,7 @@ def pytest_configure(config): "markers", "expected_logs(amount, ...): mark test to expect a certain amount of panda logs" ) +@pytest.hookimpl(tryfirst=True) def pytest_collection_modifyitems(items): for item in items: if item.get_closest_marker('execution_timeout') is None: @@ -94,6 +99,16 @@ def pytest_collection_modifyitems(items): item.add_marker(pytest.mark.setup_timeout(20)) item.add_marker(pytest.mark.teardown_timeout(20)) + # xdist grouping by panda + serial = item.name.split("serial=")[1].split(",")[0] + assert len(serial) == 24 + item.add_marker(pytest.mark.xdist_group(serial)) + + needs_jungle = "panda_jungle" in item.fixturenames + if PARALLEL and needs_jungle: + item.add_marker(pytest.mark.skip(reason="no jungle tests in PARALLEL mode")) + elif NON_PARALLEL and not needs_jungle: + item.add_marker(pytest.mark.skip(reason="only running jungle tests")) def pytest_make_parametrize_id(config, val, argname): if val in _all_pandas: @@ -103,12 +118,12 @@ def pytest_make_parametrize_id(config, val, argname): return None -@pytest.fixture(name='panda_jungle') +@pytest.fixture(name='panda_jungle', scope='function') def fixture_panda_jungle(request): init_jungle() return _panda_jungle -@pytest.fixture(name='p') +@pytest.fixture(name='p', scope='function') def func_fixture_panda(request, module_panda): p = module_panda @@ -201,12 +216,6 @@ def fixture_panda_setup(request): # Initialize jungle init_jungle() - # wait for all pandas to come up - for _ in range(50): - if set(_all_panda_serials).issubset(set(Panda.list())): - break - time.sleep(0.1) - # Connect to pandas def cnnct(s): if s == panda_serial: @@ -221,7 +230,7 @@ def cnnct(s): clear_can_buffers(p) p.set_power_save(False) return p - else: + elif not PARALLEL: with Panda(serial=s) as p: p.reset(reconnect=False) return None diff --git a/tests/hitl/run_parallel_tests.sh b/tests/hitl/run_parallel_tests.sh new file mode 100755 index 0000000000..b6b79d99f6 --- /dev/null +++ b/tests/hitl/run_parallel_tests.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +cd $DIR + +# n = number of pandas tested +PARALLEL=1 pytest --durations=0 *.py -n 5 --dist loadgroup -x diff --git a/tests/hitl/test.sh b/tests/hitl/run_serial_tests.sh similarity index 67% rename from tests/hitl/test.sh rename to tests/hitl/run_serial_tests.sh index 7382ac5988..31270f044c 100755 --- a/tests/hitl/test.sh +++ b/tests/hitl/run_serial_tests.sh @@ -4,4 +4,4 @@ set -e DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" cd $DIR -pytest --durations=0 --maxfail=1 *.py +NON_PARALLEL=1 pytest --durations=0 *.py -x