From d60372fdf2cb5cf12ebf1215cb45ce0720ba0cf6 Mon Sep 17 00:00:00 2001 From: Jeremy Retailleau Date: Sat, 12 Oct 2024 17:52:56 -0700 Subject: [PATCH] Add custom properties support and simplify test property handling. - Add "PROPERTIES" option to pytest_discover_tests function to allow passing custom properties; - Refactor to use lua-style long bracket syntax for serializing environment and custom properties, consolidating them within a single set_tests_properties command; - Revert to using Pytest_ROOT instead of CMAKE_PREFIX_PATH in the integration document; - Add tests and update documentation. --- .github/workflows/test.yml | 2 +- cmake/FindPytest.cmake | 32 +++-- cmake/PytestAddTests.cmake | 119 +++++++++--------- doc/api_reference.rst | 14 +++ doc/integration.rst | 4 +- doc/release/release_notes.rst | 19 +-- doc/tutorial.rst | 9 +- example/test/CMakeLists.txt | 2 + test/01-modify-name/CMakeLists.txt | 2 +- .../compare_discovered_tests.cmake | 4 +- test/02-library-path/CMakeLists.txt | 2 +- test/02-library-path/test_path.py | 1 - test/03-python-path/CMakeLists.txt | 2 +- test/03-python-path/test_path.py | 4 +- test/04-environment/CMakeLists.txt | 11 +- test/04-environment/test_env.py | 17 ++- test/05-properties/CMakeLists.txt | 59 +++++++++ test/05-properties/check_test_property.cmake | 60 +++++++++ test/05-properties/test_property.py | 2 + test/CMakeLists.txt | 11 ++ 20 files changed, 268 insertions(+), 108 deletions(-) create mode 100644 test/05-properties/CMakeLists.txt create mode 100644 test/05-properties/check_test_property.cmake create mode 100644 test/05-properties/test_property.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 893bf67..896d6b0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -61,7 +61,7 @@ jobs: - name: Build and Run Unit Tests shell: bash run: | - cmake -D "CMAKE_BUILD_TYPE=Release" -S ./test -B ./test/build + cmake -S ./test -B ./test/build cmake --build ./test/build - name: Configure Example diff --git a/cmake/FindPytest.cmake b/cmake/FindPytest.cmake index 276c746..3e20bd9 100644 --- a/cmake/FindPytest.cmake +++ b/cmake/FindPytest.cmake @@ -3,9 +3,9 @@ # This module defines the following imported targets: # Pytest::Pytest # -# It also exposes the 'pytest_discover_tests' function which adds ctest -# for each pytest tests. The "BUNDLE_PYTHON_TESTS" environment variable -# can be used to run all discovered tests all together. +# It also exposes the 'pytest_discover_tests' function, which adds CTest +# test for each Pytest test. The "BUNDLE_PYTHON_TESTS" environment variable +# can be used to run all discovered tests together. # # Usage: # find_package(Pytest) @@ -53,14 +53,15 @@ if (Pytest_FOUND AND NOT TARGET Pytest::Pytest) PROPERTIES IMPORTED_LOCATION "${PYTEST_EXECUTABLE}") + # Function to discover pytest tests and add them to CTest. function(pytest_discover_tests NAME) cmake_parse_arguments( PARSE_ARGV 1 "" "STRIP_PARAM_BRACKETS;INCLUDE_FILE_PATH;BUNDLE_TESTS" "WORKING_DIRECTORY;TRIM_FROM_NAME;TRIM_FROM_FULL_NAME" - "LIBRARY_PATH_PREPEND;PYTHON_PATH_PREPEND;ENVIRONMENT;DEPENDS" + "LIBRARY_PATH_PREPEND;PYTHON_PATH_PREPEND;ENVIRONMENT;PROPERTIES;DEPENDS" ) - # Identify library path environment name depending on the platform. + # Set platform-specific library path environment variable. if (CMAKE_SYSTEM_NAME STREQUAL Windows) set(LIBRARY_ENV_NAME PATH) elseif(CMAKE_SYSTEM_NAME STREQUAL Darwin) @@ -69,11 +70,11 @@ if (Pytest_FOUND AND NOT TARGET Pytest::Pytest) set(LIBRARY_ENV_NAME LD_LIBRARY_PATH) endif() - # Sanitize all paths for CMake. + # Convert paths to CMake-friendly format. cmake_path(CONVERT "$ENV{${LIBRARY_ENV_NAME}}" TO_CMAKE_PATH_LIST LIBRARY_PATH) cmake_path(CONVERT "$ENV{PYTHONPATH}" TO_CMAKE_PATH_LIST PYTHON_PATH) - # Prepend input path to environment variables. + # Prepend specified paths to the library and Python paths. if (_LIBRARY_PATH_PREPEND) list(REVERSE _LIBRARY_PATH_PREPEND) foreach (_path ${_LIBRARY_PATH_PREPEND}) @@ -88,7 +89,7 @@ if (Pytest_FOUND AND NOT TARGET Pytest::Pytest) endforeach() endif() - # Default working directory to current build path if none is provided. + # Set default working directory if none is specified. if (NOT _WORKING_DIRECTORY) set(_WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) endif() @@ -100,17 +101,11 @@ if (Pytest_FOUND AND NOT TARGET Pytest::Pytest) set(_BUNDLE_TESTS $ENV{BUNDLE_PYTHON_TESTS}) endif() - # Serialize environment if necessary. - set(ENCODED_ENVIRONMENT "") - foreach(env ${_ENVIRONMENT}) - string(REPLACE [[\]] [\\]] env ${env}) - string(REPLACE [[;]] [\\;]] env ${env}) - list(APPEND ENCODED_ENVIRONMENT ${env}) - endforeach() - + # Define file paths for generated CMake include files. set(_include_file "${CMAKE_CURRENT_BINARY_DIR}/${NAME}_include.cmake") set(_tests_file "${CMAKE_CURRENT_BINARY_DIR}/${NAME}_tests.cmake") + # Create a custom target to run the tests. add_custom_target( ${NAME} ALL VERBATIM BYPRODUCTS "${_tests_file}" @@ -127,7 +122,8 @@ if (Pytest_FOUND AND NOT TARGET Pytest::Pytest) -D "STRIP_PARAM_BRACKETS=${_STRIP_PARAM_BRACKETS}" -D "INCLUDE_FILE_PATH=${_INCLUDE_FILE_PATH}" -D "WORKING_DIRECTORY=${_WORKING_DIRECTORY}" - -D "ENVIRONMENT=${ENCODED_ENVIRONMENT}" + -D "ENVIRONMENT=${_ENVIRONMENT}" + -D "TEST_PROPERTIES=${_PROPERTIES}" -D "CTEST_FILE=${_tests_file}" -P "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/PytestAddTests.cmake") @@ -139,7 +135,7 @@ if (Pytest_FOUND AND NOT TARGET Pytest::Pytest) "endif()\n" ) - # Add discovered tests to directory TEST_INCLUDE_FILES + # Register the include file to be processed for tests. set_property(DIRECTORY APPEND PROPERTY TEST_INCLUDE_FILES "${_include_file}") diff --git a/cmake/PytestAddTests.cmake b/cmake/PytestAddTests.cmake index d28b449..f789c10 100644 --- a/cmake/PytestAddTests.cmake +++ b/cmake/PytestAddTests.cmake @@ -3,54 +3,62 @@ cmake_minimum_required(VERSION 3.20...3.30) if(CMAKE_SCRIPT_MODE_FILE) - # Set Cmake test file to execute each test. + # Initialize content for the CMake test file. set(_content "") + # Convert library and Python paths to native format. cmake_path(CONVERT "${LIBRARY_PATH}" TO_NATIVE_PATH_LIST LIBRARY_PATH) cmake_path(CONVERT "${PYTHON_PATH}" TO_NATIVE_PATH_LIST PYTHON_PATH) # Serialize path values separated by semicolons (required on Windows). - macro(encode_value VARIABLE_NAME) - string(REPLACE [[\]] [[\\]] ${VARIABLE_NAME} "${${VARIABLE_NAME}}") - string(REPLACE [[;]] [[\\;]] ${VARIABLE_NAME} "${${VARIABLE_NAME}}") - endmacro() - - encode_value(LIBRARY_PATH) - encode_value(PYTHON_PATH) - - if (BUNDLE_TESTS) + string(REPLACE [[;]] [[\\;]] LIBRARY_PATH "${LIBRARY_PATH}") + string(REPLACE [[;]] [[\\;]] PYTHON_PATH "${PYTHON_PATH}") + + # Set up the encoded environment with required paths. + set(ENCODED_ENVIRONMENT + "${LIBRARY_ENV_NAME}=${LIBRARY_PATH}" + "PYTHONPATH=${PYTHON_PATH}" + ) + + # Serialize additional environment variables if any are provided. + foreach(env ${ENVIRONMENT}) + string(REPLACE [[;]] [[\\;]] env "${env}") + list(APPEND ENCODED_ENVIRONMENT "${env}") + endforeach() + + # Macro to create individual tests with optional test properties. + macro(create_test NAME IDENTIFIER) string(APPEND _content - "add_test(\n" - " \"${TEST_GROUP_NAME}\"\n" - " \"${PYTEST_EXECUTABLE}\" \"${WORKING_DIRECTORY}\"\n" - ")\n" - "set_tests_properties(\n" - " \"${TEST_GROUP_NAME}\" PROPERTIES\n" - " ENVIRONMENT \"${LIBRARY_ENV_NAME}=${LIBRARY_PATH}\"\n" - ")\n" - "set_tests_properties(\n" - " \"${TEST_GROUP_NAME}\"\n" - " APPEND PROPERTIES\n" - " ENVIRONMENT \"PYTHONPATH=${PYTHON_PATH}\"\n" - ")\n" + "add_test(\"${NAME}\" \"${PYTEST_EXECUTABLE}\" \"${IDENTIFIER}\")\n" ) - foreach(env ${ENVIRONMENT}) - string(APPEND _content - "set_tests_properties(\n" - " \"${TEST_GROUP_NAME}\"\n" - " APPEND PROPERTIES\n" - " ENVIRONMENT ${env}\n" - ")\n" - ) + # Prepare the properties for the test, including the environment settings. + set(args "PROPERTIES ENVIRONMENT [==[${ENCODED_ENVIRONMENT}]==]") + + # Append any additional properties, escaping complex characters if necessary. + foreach(property ${TEST_PROPERTIES}) + if(property MATCHES "[^-./:a-zA-Z0-9_]") + string(APPEND args " [==[${property}]==]") + else() + string(APPEND args " ${property}") + endif() endforeach() + # Append the test properties to the content. + string(APPEND _content "set_tests_properties(\"${NAME}\" ${args})\n") + endmacro() + + # If tests are bundled together, create a single test group. + if (BUNDLE_TESTS) + create_test("${TEST_GROUP_NAME}" "${WORKING_DIRECTORY}") + else() - # Set environment for collecting tests. + # Set environment variables for collecting tests. set(ENV{${LIBRARY_ENV_NAME}} "${LIBRARY_PATH}") set(ENV{PYTHONPATH} "${PYTHON_PATH}") set(ENV{PYTHONWARNINGS} "ignore") + # Collect tests. execute_process( COMMAND "${PYTEST_EXECUTABLE}" --collect-only -q @@ -61,6 +69,7 @@ if(CMAKE_SCRIPT_MODE_FILE) WORKING_DIRECTORY ${WORKING_DIRECTORY} ) + # Check for errors during test collection. string(REGEX MATCH "=+ ERRORS =+(.*)" _error "${_output_lines}") if (_error) @@ -68,84 +77,72 @@ if(CMAKE_SCRIPT_MODE_FILE) message(FATAL_ERROR "An error occurred during the collection of Python tests.") endif() - # Convert output into list. + # Convert the collected output into a list of lines. string(REPLACE [[;]] [[\;]] _output_lines "${_output_lines}") string(REPLACE "\n" ";" _output_lines "${_output_lines}") + # Regex pattern to identify pytest test identifiers. set(test_pattern "([^:]+)\.py(::([^:]+))?::([^:]+)") - foreach (line ${_output_lines}) + # Iterate through each line to identify and process tests. + foreach(line ${_output_lines}) string(REGEX MATCHALL ${test_pattern} matching "${line}") - # Ignore lines not identified as a test. + # Skip lines that are not identified as tests. if (NOT matching) continue() endif() + # Extract file, class, and function names from the test pattern. set(_file ${CMAKE_MATCH_1}) set(_class ${CMAKE_MATCH_3}) set(_func ${CMAKE_MATCH_4}) + # Optionally trim parts of the class or function name. if (TRIM_FROM_NAME) string(REGEX REPLACE "${TRIM_FROM_NAME}" "" _class "${_class}") string(REGEX REPLACE "${TRIM_FROM_NAME}" "" _func "${_func}") endif() + # Form the test name using class and function. if (_class) set(test_name "${_class}.${_func}") else() set(test_name "${_func}") endif() + # Optionally strip parameter brackets from the test name. if (STRIP_PARAM_BRACKETS) string(REGEX REPLACE "\\[(.+)\\]$" ".\\1" test_name "${test_name}") endif() + # Optionally include the file path in the test name. if (INCLUDE_FILE_PATH) cmake_path(CONVERT "${_file}" TO_CMAKE_PATH_LIST _file) string(REGEX REPLACE "/" "." _file "${_file}") set(test_name "${_file}.${test_name}") endif() + # Optionally trim parts of the full test name. if (TRIM_FROM_FULL_NAME) string(REGEX REPLACE "${TRIM_FROM_FULL_NAME}" "" test_name "${test_name}") endif() + # Prefix the test name with the test group name. set(test_name "${TEST_GROUP_NAME}.${test_name}") set(test_case "${WORKING_DIRECTORY}/${line}") - string(APPEND _content - "add_test(\n" - " \"${test_name}\"\n" - " \"${PYTEST_EXECUTABLE}\" \"${test_case}\"\n" - ")\n" - "set_tests_properties(\n" - " \"${test_name}\" PROPERTIES\n" - " ENVIRONMENT \"${LIBRARY_ENV_NAME}=${LIBRARY_PATH}\"\n" - ")\n" - "set_tests_properties(\n" - " \"${test_name}\"\n" - " APPEND PROPERTIES\n" - " ENVIRONMENT \"PYTHONPATH=${PYTHON_PATH}\"\n" - ")\n" - ) - - foreach(env ${ENVIRONMENT}) - string(APPEND _content - "set_tests_properties(\n" - " \"${test_name}\"\n" - " APPEND PROPERTIES\n" - " ENVIRONMENT ${env}\n" - ")\n" - ) - endforeach() - + # Create the test for CTest. + create_test("${test_name}" "${test_case}") endforeach() + # Warn if no tests were discovered. if(NOT _content) message(WARNING "No Python tests have been discovered.") endif() endif() + # Write the generated test content to the specified CTest file. file(WRITE ${CTEST_FILE} ${_content}) + endif() diff --git a/doc/api_reference.rst b/doc/api_reference.rst index 7cbfb19..30635b7 100644 --- a/doc/api_reference.rst +++ b/doc/api_reference.rst @@ -18,6 +18,7 @@ API Reference [LIBRARY_PATH_PREPEND path1 path2...] [PYTHON_PATH_PREPEND path1 path2...] [ENVIRONMENT env1 env2...] + [PROPERTIES prop1 prop2...] [DEPENDS target1 target2...] [INCLUDE_FILE_PATH] [STRIP_PARAM_BRACKETS] @@ -129,6 +130,19 @@ API Reference "ENV_VAR3=VALUE3" ) + * ``PROPERTIES`` + + List of custom `test properties + `_ + to apply for all generated tests:: + + pytest_discover_tests( + ... + PROPERTIES + LABELS "python;unit" + TIMEOUT 120 + ) + * ``DEPENDS`` List of dependent targets that need to be executed before running diff --git a/doc/integration.rst b/doc/integration.rst index 604fcc2..106fe32 100644 --- a/doc/integration.rst +++ b/doc/integration.rst @@ -43,10 +43,10 @@ able to discover the newly installed configuration automatically using its option should not be set to False. When using a Python virtual environment, or if Python is installed in a -non-standard location, the :envvar:`Pytest_ROOT` environment variable +non-standard location, the :envvar:`CMAKE_PREFIX_PATH` environment variable (or :term:`CMake` option) can be used to guide the discovery process:: - cmake -S . -B ./build -D "Pytest_ROOT=/path/to/python/prefix" + cmake -S . -B ./build -D "CMAKE_PREFIX_PATH=/path/to/python/prefix" This is also necessary when installing the configuration in the `Python user directory diff --git a/doc/release/release_notes.rst b/doc/release/release_notes.rst index 9e49094..d6c22d4 100644 --- a/doc/release/release_notes.rst +++ b/doc/release/release_notes.rst @@ -4,22 +4,27 @@ Release Notes ************* +.. release:: Upcoming + + .. change:: new + + Added ``PROPERTIES`` option to the :func:`pytest_discover_tests` + function, providing custom `test properties + `_ + for all generated tests. + .. release:: 0.10.0 :date: 2024-10-11 .. change:: new Added ``INCLUDE_FILE_PATH`` option to the :func:`pytest_discover_tests` - function use the file path to compute the test identifier. - - .. seealso:: :ref:`tutorial/function` + function, allowing the file path to be included in the test identifier. .. change:: new Added ``TRIM_FROM_FULL_NAME`` option to the :func:`pytest_discover_tests` - function trim parts of the full test name generated. - - .. seealso:: :ref:`tutorial/function` + function, enabling parts of the full test name to be trimmed. .. change:: fixed @@ -40,8 +45,6 @@ Release Notes Added ``STRIP_PARAM_BRACKETS`` option to the :func:`pytest_discover_tests` function to strip square brackets used for :term:`parametrizing tests`. - .. seealso:: :ref:`tutorial/function` - .. release:: 0.8.4 :date: 2024-10-06 diff --git a/doc/tutorial.rst b/doc/tutorial.rst index 1ca641d..5728ed4 100644 --- a/doc/tutorial.rst +++ b/doc/tutorial.rst @@ -14,8 +14,10 @@ Using the target Let's consider a project that wraps C++ logic with Python bindings. We need to add a :file:`CMakeLists.txt` configuration file to include Python tests within -the same directory. The :term:`Pytest` command can easily be implemented using -the :term:`add_test` function: +the same directory. + +The :term:`Pytest` command can easily be implemented using the :term:`add_test` +function: .. code-block:: cmake @@ -176,6 +178,9 @@ follows: Start 4: PythonTest.subfolder.greet_michael 4/4 Test #4: PythonTest.subfolder.greet_michael ... Passed 0.54 sec +You can also define custom environment variables and test properties using the +``ENVIRONMENT`` and ``PROPERTIES`` options, respectively. + It is also possible to regroup all tests under one :term:`CTest` test, as was the case when :ref:`using the target `. This can be useful during development to ensure that the tests run faster, especially diff --git a/example/test/CMakeLists.txt b/example/test/CMakeLists.txt index c7aef14..94c93ec 100644 --- a/example/test/CMakeLists.txt +++ b/example/test/CMakeLists.txt @@ -12,6 +12,8 @@ pytest_discover_tests( ENVIRONMENT "DEFAULT_LANGUAGE=en" "FOO_SETTINGS_FILE=${CMAKE_CURRENT_SOURCE_DIR}/resource/foo.txt" + PROPERTIES + TIMEOUT 120 ) if (WIN32) diff --git a/test/01-modify-name/CMakeLists.txt b/test/01-modify-name/CMakeLists.txt index b747819..35e092f 100644 --- a/test/01-modify-name/CMakeLists.txt +++ b/test/01-modify-name/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.20) project(TestModifyName) -find_package(Pytest 4.6.11 REQUIRED) +find_package(Pytest REQUIRED) enable_testing() diff --git a/test/01-modify-name/compare_discovered_tests.cmake b/test/01-modify-name/compare_discovered_tests.cmake index 52eaffd..799bc8f 100644 --- a/test/01-modify-name/compare_discovered_tests.cmake +++ b/test/01-modify-name/compare_discovered_tests.cmake @@ -1,7 +1,9 @@ cmake_minimum_required(VERSION 3.20) execute_process( - COMMAND ${CMAKE_CTEST_COMMAND} --show-only + COMMAND ${CMAKE_CTEST_COMMAND} + --show-only + -R "${TEST_PREFIX}" WORKING_DIRECTORY ${CMAKE_BINARY_DIR} OUTPUT_VARIABLE test_output ) diff --git a/test/02-library-path/CMakeLists.txt b/test/02-library-path/CMakeLists.txt index 2295334..2dcabd3 100644 --- a/test/02-library-path/CMakeLists.txt +++ b/test/02-library-path/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.20) project(TestLibraryPath) -find_package(Pytest 4.6.11 REQUIRED) +find_package(Pytest REQUIRED) enable_testing() diff --git a/test/02-library-path/test_path.py b/test/02-library-path/test_path.py index 73a016e..82b7c32 100644 --- a/test/02-library-path/test_path.py +++ b/test/02-library-path/test_path.py @@ -1,6 +1,5 @@ import os import platform -import pytest PREPENDED_PATH = "@EXPECTED@" diff --git a/test/03-python-path/CMakeLists.txt b/test/03-python-path/CMakeLists.txt index ae78f3f..a5f070e 100644 --- a/test/03-python-path/CMakeLists.txt +++ b/test/03-python-path/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.20) project(TestPythonPath) -find_package(Pytest 4.6.11 REQUIRED) +find_package(Pytest REQUIRED) enable_testing() diff --git a/test/03-python-path/test_path.py b/test/03-python-path/test_path.py index 4f731d0..57e46a6 100644 --- a/test/03-python-path/test_path.py +++ b/test/03-python-path/test_path.py @@ -1,6 +1,4 @@ import os -import platform -import pytest PREPENDED_PATH = "@EXPECTED@" @@ -9,4 +7,4 @@ def test_path(): current_path = os.environ.get("PYTHONPATH", "") assert current_path == PREPENDED_PATH or current_path.startswith(PREPENDED_PATH + os.pathsep), \ - f"{env_var} does not start with {PREPENDED_PATH}, found: {current_path}" + f"PYTHONPATH does not start with {PREPENDED_PATH}, found: {current_path}" diff --git a/test/04-environment/CMakeLists.txt b/test/04-environment/CMakeLists.txt index 531cbee..65f0dce 100644 --- a/test/04-environment/CMakeLists.txt +++ b/test/04-environment/CMakeLists.txt @@ -2,14 +2,17 @@ cmake_minimum_required(VERSION 3.20) project(TestEnvironment) -find_package(Pytest 4.6.11 REQUIRED) +find_package(Pytest REQUIRED) enable_testing() pytest_discover_tests( TestEnvironment ENVIRONMENT - "ENV_VAR1=VALUE1" - "ENV_VAR2=VALUE2" - "ENV_VAR3=PATH1:PATH2:PATH3" + "KEY1=VALUE1" + "KEY2=VALUE2" + "KEY3=PATH1:PATH2:PATH3" + "KEY4=PATH1\;PATH2\;PATH3" + "KEY5=C:\\Path\\To\\Dir1\;C:\\Path\\To\\Dir2" + "K3Y4=SPECIAL$VALUE!@#%^&*" ) diff --git a/test/04-environment/test_env.py b/test/04-environment/test_env.py index 482056e..79f7533 100644 --- a/test/04-environment/test_env.py +++ b/test/04-environment/test_env.py @@ -1,8 +1,17 @@ import os -import pytest +import platform def test_env(): """Ensure that the environment variables have been set.""" - assert os.environ.get("ENV_VAR1", "") == "VALUE1" - assert os.environ.get("ENV_VAR2", "") == "VALUE2" - assert os.environ.get("ENV_VAR3", "") == "PATH1:PATH2:PATH3" + assert os.environ.get("KEY1", "") == "VALUE1" + assert os.environ.get("KEY2", "") == "VALUE2" + assert os.environ.get("KEY3", "") == "PATH1:PATH2:PATH3" + + if platform.system() == "Windows": + assert os.environ.get("KEY4", "") == "PATH1;PATH2;PATH3" + assert os.environ.get("KEY5", "") == r"C:\Path\To\Dir1;C:\Path\To\Dir2" + else: + assert os.environ.get("KEY4", "") == "PATH1\;PATH2\;PATH3" + assert os.environ.get("KEY5", "") == r"C:\Path\To\Dir1\;C:\Path\To\Dir2" + + assert os.environ.get("K3Y4", "") == "SPECIAL$VALUE!@#%^&*" diff --git a/test/05-properties/CMakeLists.txt b/test/05-properties/CMakeLists.txt new file mode 100644 index 0000000..9ffd7c1 --- /dev/null +++ b/test/05-properties/CMakeLists.txt @@ -0,0 +1,59 @@ +cmake_minimum_required(VERSION 3.20) + +project(TestProperties) + +find_package(Pytest REQUIRED) + +enable_testing() + +pytest_discover_tests( + TestProperties.Timeout + PROPERTIES + TIMEOUT 120 +) +add_test(NAME TestProperties.Validate.Timeout + COMMAND ${CMAKE_COMMAND} + -D "TEST_PREFIX=TestProperties.Timeout" + -D "TEST_PROPERTY=TIMEOUT" + -D "EXPECTED_VALUE=120.0" + -P ${CMAKE_CURRENT_LIST_DIR}/check_test_property.cmake +) + +pytest_discover_tests( + TestProperties.Labels + PROPERTIES + LABELS "foo\;bar" +) +add_test(NAME TestProperties.Validate.Labels + COMMAND ${CMAKE_COMMAND} + -D "TEST_PREFIX=TestProperties.Labels" + -D "TEST_PROPERTY=LABELS" + -D "EXPECTED_VALUE=[ \"bar\", \"foo\" ]" + -P ${CMAKE_CURRENT_LIST_DIR}/check_test_property.cmake +) + +pytest_discover_tests( + TestProperties.Disabled + PROPERTIES + DISABLED ON +) +add_test(NAME TestProperties.Validate.Disabled + COMMAND ${CMAKE_COMMAND} + -D "TEST_PREFIX=TestProperties.Disabled" + -D "TEST_PROPERTY=DISABLED" + -D "EXPECTED_VALUE=ON" + -P ${CMAKE_CURRENT_LIST_DIR}/check_test_property.cmake +) + +pytest_discover_tests( + TestProperties.SkipReturnCode + PROPERTIES + SKIP_RETURN_CODE 77 +) +add_test(NAME TestProperties.Validate.SkipReturnCode + COMMAND ${CMAKE_COMMAND} + -D "TEST_PREFIX=TestProperties.SkipReturnCode" + -D "TEST_PROPERTY=SKIP_RETURN_CODE" + -D "EXPECTED_VALUE=77" + -P ${CMAKE_CURRENT_LIST_DIR}/check_test_property.cmake +) diff --git a/test/05-properties/check_test_property.cmake b/test/05-properties/check_test_property.cmake new file mode 100644 index 0000000..1c750d1 --- /dev/null +++ b/test/05-properties/check_test_property.cmake @@ -0,0 +1,60 @@ +cmake_minimum_required(VERSION 3.20) + +execute_process( + COMMAND ${CMAKE_CTEST_COMMAND} + --show-only=json-v1 + -R "${TEST_PREFIX}" + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + OUTPUT_VARIABLE test_output + OUTPUT_STRIP_TRAILING_WHITESPACE +) + +# Trailing slashes in Windows environment paths produce '\;', leading to JSON +# parsing issues. Replace '\;' with ';' to resolve. +string(REPLACE [[\;]] [[;]] test_output "${test_output}") + +string(JSON num_tests LENGTH ${test_output} tests) + +if (num_tests EQUAL 0) + message(FATAL_ERROR "'${TEST_PREFIX}': No tests discovered.") +endif() + +string(JSON tests_array GET ${test_output} tests) +math(EXPR last_test_index "${num_tests}-1") + +foreach (i RANGE 0 ${last_test_index}) + string(JSON test_name GET ${tests_array} ${i} name) + string(JSON num_properties LENGTH ${tests_array} ${i} properties) + + if (num_properties EQUAL 0) + message(FATAL_ERROR "Test '${test_name}' does not have properties.") + endif() + + string(JSON properties GET ${tests_array} ${i} properties) + math(EXPR last_property_index "${num_properties}-1") + + set(property_found 0) + + foreach (j RANGE 0 ${last_property_index}) + string(JSON property_name GET ${properties} ${j} name) + string(JSON property_value GET ${properties} ${j} value) + message("${property_name} = ${property_value}") + + if (property_name STREQUAL "${TEST_PROPERTY}") + set(property_found 1) + + if (NOT "${property_value}" STREQUAL "${EXPECTED_VALUE}") + message(FATAL_ERROR + "Test '${test_name}' does not have ${TEST_PROPERTY} set to ${EXPECTED_VALUE}.\n" + "Found: ${property_value}" + ) + endif() + endif() + endforeach() + + if(NOT property_found) + message(FATAL_ERROR + "Test '${test_name}' does not have ${TEST_PROPERTY} property." + ) + endif() +endforeach() \ No newline at end of file diff --git a/test/05-properties/test_property.py b/test/05-properties/test_property.py new file mode 100644 index 0000000..ca4655d --- /dev/null +++ b/test/05-properties/test_property.py @@ -0,0 +1,2 @@ +def test_addition(): + assert 1 + 1 == 2 diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 84ec74a..4732ec5 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -41,3 +41,14 @@ ExternalProject_Add( INSTALL_COMMAND "" TEST_COMMAND ${CMAKE_CTEST_COMMAND} -C Release -VV ) + +ExternalProject_Add( + TestProperties + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/05-properties + BINARY_DIR ${CMAKE_BINARY_DIR}/_deps/05-properties + BUILD_COMMAND ${CMAKE_COMMAND} --build + INSTALL_COMMAND "" + TEST_COMMAND ${CMAKE_CTEST_COMMAND} + -C Release -VV + -R TestProperties.Validate +)