From 23239f2e65404b0609d6ebda0199ee3ba1db2bec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20Dom=C3=ADnguez=20L=C3=B3pez?= <116071334+Mario-DL@users.noreply.github.com> Date: Thu, 9 May 2024 14:36:53 +0200 Subject: [PATCH] Fix Python Installation version in Github CI. Address failing system tests environment issues (#4759) * Refs #20965: Install a fix python version in ubuntu and windows cis Signed-off-by: Mario Dominguez * Refs #20965: Decide Python environment depending on the version Signed-off-by: Mario Dominguez * Refs #20965: Prevent Qt installation from setting up python in Ubuntu CI Signed-off-by: Mario Dominguez * Refs #20965: Add some comments Signed-off-by: Mario Dominguez --------- Signed-off-by: Mario Dominguez (cherry picked from commit 3b2e781f3133460cfcce3a316d82ac6abed06302) # Conflicts: # test/system/tools/fastdds/CMakeLists.txt --- .github/workflows/reusable-ubuntu-ci.yml | 8 + .github/workflows/reusable-windows-ci.yml | 5 + test/system/tools/fastdds/CMakeLists.txt | 204 ++++++++++++++++++++++ tools/fastdds/fastdds.bat | 11 +- tools/fastdds/ros-discovery.bat | 11 +- 5 files changed, 237 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable-ubuntu-ci.yml b/.github/workflows/reusable-ubuntu-ci.yml index c6a7738bdd6..a6a48070fad 100644 --- a/.github/workflows/reusable-ubuntu-ci.yml +++ b/.github/workflows/reusable-ubuntu-ci.yml @@ -56,6 +56,11 @@ jobs: path: src/fastrtps ref: ${{ inputs.fastdds-branch }} + - name: Install Fix Python version + uses: eProsima/eProsima-CI/external/setup-python@v0 + with: + python-version: '3.11' + - name: Get minimum supported version of CMake uses: eProsima/eProsima-CI/external/get-cmake@v0 with: @@ -114,12 +119,15 @@ jobs: ref: ${{ steps.get_shapes_demo_branch.outputs.deduced_branch }} # Required for Shapes Demo + # Do not setup python as it will internally modify the pythonLocation env variable + # and we want to use a fix version - name: Install Qt uses: jurplel/install-qt-action@v2.13.0 with: version: '5.15.2' dir: '${{ github.workspace }}/qt_installation/' modules: 'qtcharts' + setup-python: 'false' - name: Colcon build continue-on-error: false diff --git a/.github/workflows/reusable-windows-ci.yml b/.github/workflows/reusable-windows-ci.yml index 2583c0586dc..ebc79427190 100644 --- a/.github/workflows/reusable-windows-ci.yml +++ b/.github/workflows/reusable-windows-ci.yml @@ -56,6 +56,11 @@ jobs: submodules: true ref: ${{ inputs.fastdds_branch }} + - name: Install Fix Python version + uses: eProsima/eProsima-CI/external/setup-python@v0 + with: + python-version: '3.11' + - name: Get minimum supported version of CMake uses: eProsima/eProsima-CI/external/get-cmake@v0 with: diff --git a/test/system/tools/fastdds/CMakeLists.txt b/test/system/tools/fastdds/CMakeLists.txt index bdc578a3eec..14e4811f46f 100644 --- a/test/system/tools/fastdds/CMakeLists.txt +++ b/test/system/tools/fastdds/CMakeLists.txt @@ -20,6 +20,10 @@ find_package(PythonInterp 3 REQUIRED) if(PYTHONINTERP_FOUND) + # Set environment for tests + set(TEST_ENVIRONMENT + "PYTHON_VERSION=${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}") + set(TESTS test_fastdds_installed test_fastdds_discovery @@ -43,6 +47,7 @@ if(PYTHONINTERP_FOUND) PROPERTY LABELS "NoMemoryCheck" ) +<<<<<<< HEAD endforeach() endif() @@ -169,3 +174,202 @@ endif() # Define test add_test(NAME xml.validate COMMAND ${env_fast_command} xml validate xmldocuments) +======= + # Set environment + set_property( + TEST system.tools.fastdds.${TEST} + PROPERTY ENVIRONMENT ${TEST_ENVIRONMENT} + ) + + endforeach() + + ############################################################################### + # XML GENERAL validation + ############################################################################### + message(STATUS "Configuring CLI xml validation...") + + # Configure command based on the OS running + if (MSVC) + set(env_fast_command "fastdds.bat") # WINDOWS + else() + set(env_fast_command "fastdds") # POSIX + endif() + + # Copy the examples validation files over to the build directory + file(GLOB_RECURSE XML_EXAMPLE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/../../../../examples/*_profile.xml) + # for each xml file detected + foreach(XML_EXAMPLE_FILE_COMPLETE_PATH ${XML_EXAMPLE_FILES}) + # obtain the file name + get_filename_component(XML_EXAMPLE_FILE ${XML_EXAMPLE_FILE_COMPLETE_PATH} NAME_WE) + # copy the file from src to build folders + configure_file( + ${XML_EXAMPLE_FILE_COMPLETE_PATH} # from full src path + ${CMAKE_CURRENT_BINARY_DIR}/xmldocuments/examples/${XML_EXAMPLE_FILE}.xml # to relative build path + COPYONLY) + endforeach() + + # Copy the blackbox tests validation files over to the build directory + file(GLOB_RECURSE XML_BLACKBOX_FILES ${CMAKE_CURRENT_SOURCE_DIR}/../../../blackbox/*_profile.xml) + # for each xml file detected + foreach(XML_BLACKBOX_FILE_COMPLETE_PATH ${XML_BLACKBOX_FILES}) + # obtain the file name + get_filename_component(XML_BLACKBOX_FILE ${XML_BLACKBOX_FILE_COMPLETE_PATH} NAME_WE) + # copy the file from src to build folders + configure_file( + ${XML_BLACKBOX_FILE_COMPLETE_PATH} # from full src path + ${CMAKE_CURRENT_BINARY_DIR}/xmldocuments/test/blackbox/${XML_BLACKBOX_FILE}.xml # to relative build path + COPYONLY) + endforeach() + + # Copy the unit tests validation files over to the build directory + file(GLOB_RECURSE XML_UNITTEST_FILES ${CMAKE_CURRENT_SOURCE_DIR}/../../../unittest/*_profile.xml) + # for each xml file detected + foreach(XML_UNITTEST_FILE_COMPLETE_PATH ${XML_UNITTEST_FILES}) + # obtain the file name + get_filename_component(XML_UNITTEST_FILE ${XML_UNITTEST_FILE_COMPLETE_PATH} NAME_WE) + # copy the file from src to build folders + configure_file( + ${XML_UNITTEST_FILE_COMPLETE_PATH} # from full src path + ${CMAKE_CURRENT_BINARY_DIR}/xmldocuments/test/unittest/${XML_UNITTEST_FILE}.xml # to relative build path + COPYONLY) + endforeach() + + # Copy the system tests validation files over to the build directory + file(GLOB_RECURSE XML_SYSTEM_FILES ${CMAKE_CURRENT_SOURCE_DIR}/../*_profile.xml) + # for each xml file detected + foreach(XML_SYSTEM_FILE_COMPLETE_PATH ${XML_SYSTEM_FILES}) + # obtain the file name + get_filename_component(XML_SYSTEM_FILE ${XML_SYSTEM_FILE_COMPLETE_PATH} NAME_WE) + # copy the file from src to build folders + configure_file( + ${XML_SYSTEM_FILE_COMPLETE_PATH} # from full src path + ${CMAKE_CURRENT_BINARY_DIR}/xmldocuments/test/system/${XML_SYSTEM_FILE}.xml # to relative build path + COPYONLY) + endforeach() + + # Copy the comunication tests validation files over to the build directory + file(GLOB_RECURSE XML_COMMUNICATION_FILES ${CMAKE_CURRENT_SOURCE_DIR}/../../../communication/*_profile.xml) + # for each xml file detected + foreach(XML_COMMUNICATION_FILE_COMPLETE_PATH ${XML_COMMUNICATION_FILES}) + # obtain the file name + get_filename_component(XML_COMMUNICATION_FILE ${XML_COMMUNICATION_FILE_COMPLETE_PATH} NAME_WE) + # copy the file from src to build folders + configure_file( + ${XML_COMMUNICATION_FILE_COMPLETE_PATH} # from full src path + ${CMAKE_CURRENT_BINARY_DIR}/xmldocuments/test/communication/${XML_COMMUNICATION_FILE}.xml # to relative build path + COPYONLY) + endforeach() + + # Copy the dds tests validation files over to the build directory + file(GLOB_RECURSE XML_DDS_TEST_FILES ${CMAKE_CURRENT_SOURCE_DIR}/../../../dds/*_profile.xml) + # for each xml file detected + foreach(XML_DDS_TEST_FILE_COMPLETE_PATH ${XML_DDS_TEST_FILES}) + # obtain the file name + get_filename_component(XML_DDS_TEST_FILE ${XML_DDS_TEST_FILE_COMPLETE_PATH} NAME_WE) + # copy the file from src to build folders + configure_file( + ${XML_DDS_TEST_FILE_COMPLETE_PATH} # from full src path + ${CMAKE_CURRENT_BINARY_DIR}/xmldocuments/test/dds/${XML_DDS_TEST_FILE}.xml # to relative build path + COPYONLY) + endforeach() + + if(PERFORMANCE_TESTS) + # Copy the performance tests validation files over to the build directory + file(GLOB_RECURSE XML_PERFORMANCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/../../../performance/*_profile.xml) + # for each xml file detected + foreach(XML_PERFORMANCE_FILE_COMPLETE_PATH ${XML_PERFORMANCE_FILES}) + # obtain the file name + get_filename_component(XML_PERFORMANCE_FILE ${XML_PERFORMANCE_FILE_COMPLETE_PATH} NAME_WE) + # copy the file from src to build folders + configure_file( + ${XML_PERFORMANCE_FILE_COMPLETE_PATH} # from full src path + ${CMAKE_CURRENT_BINARY_DIR}/xmldocuments/test/performance/${XML_PERFORMANCE_FILE}.xml # to relative build path + COPYONLY) + endforeach() + endif() + + if(PROFILING_TESTS) + # Copy the profiling tests validation files over to the build directory + file(GLOB_RECURSE XML_PROFILING_FILES ${CMAKE_CURRENT_SOURCE_DIR}/../../../profiling/*_profile.xml) + # for each xml file detected + foreach(XML_PROFILING_FILE_COMPLETE_PATH ${XML_PROFILING_FILES}) + # obtain the file name + get_filename_component(XML_PROFILING_FILE ${XML_PROFILING_FILE_COMPLETE_PATH} NAME_WE) + # copy the file from src to build folders + configure_file( + ${XML_PROFILING_FILE_COMPLETE_PATH} # from full src path + ${CMAKE_CURRENT_BINARY_DIR}/xmldocuments/test/profiling/${XML_PROFILING_FILE}.xml # to relative build path + COPYONLY) + endforeach() + endif() + + # Define test + add_test(NAME xml.validate COMMAND ${env_fast_command} xml validate xmldocuments) + + set_property( + TEST xml.validate + PROPERTY ENVIRONMENT ${TEST_ENVIRONMENT} + ) + + ############################################################################### + # XML Static discovery validation + ############################################################################### + + # Copy the examples validation files over to the build directory + file(GLOB_RECURSE STATIC_DISC_XML_EXAMPLE ${CMAKE_CURRENT_SOURCE_DIR}/../../../../examples/*_static_disc.xml) + + # for each xml file detected in examples folder + foreach(STATIC_DISC_XML_EXAMPLE_COMPLETE_PATH ${STATIC_DISC_XML_EXAMPLE}) + # obtain the file name + get_filename_component(STATIC_DISC_XML_EXAMPLE ${STATIC_DISC_XML_EXAMPLE_COMPLETE_PATH} NAME_WE) + # copy the file from src to build folders + configure_file( + ${STATIC_DISC_XML_EXAMPLE_COMPLETE_PATH} # from full src path + ${CMAKE_CURRENT_BINARY_DIR}/xml_static_disc_docs/examples/${STATIC_DISC_XML_EXAMPLE}.xml # to relative build path + COPYONLY) + endforeach() + + # Copy the test validation files over to the build directory + file(GLOB_RECURSE STATIC_DISC_XML_TEST ${CMAKE_CURRENT_SOURCE_DIR}/../*_static_disc.xml) + + # for each xml file detected in test folder + foreach(STATIC_DISC_XML_TEST_COMPLETE_PATH ${STATIC_DISC_XML_TEST}) + # obtain the file name + get_filename_component(STATIC_DISC_XML_TEST ${STATIC_DISC_XML_TEST_COMPLETE_PATH} NAME_WE) + # copy the file from src to build folders + configure_file( + ${STATIC_DISC_XML_TEST_COMPLETE_PATH} # from full src path + ${CMAKE_CURRENT_BINARY_DIR}/xml_static_disc_docs/test/system/${STATIC_DISC_XML_TEST}.xml # to relative build path + COPYONLY) + endforeach() + + # Copy the sampled test files over to the build directory + file(GLOB_RECURSE STATIC_DISC_XML_SAMPLED_TEST ${CMAKE_CURRENT_SOURCE_DIR}/../../../blackbox/*_static_disc.xml.in) + + # set the expected CMake variables for the samples + set(TOPIC_RANDOM_NUMBER "123") + set(W_UNICAST_PORT_RANDOM_NUMBER "456") + set(R_UNICAST_PORT_RANDOM_NUMBER "654") + set(MULTICAST_PORT_RANDOM_NUMBER "789") + + # for each xml file detected in examples folder + foreach(STATIC_DISC_XML_SAMPLED_TEST_COMPLETE_PATH ${STATIC_DISC_XML_SAMPLED_TEST}) + # obtain the file name + get_filename_component(STATIC_DISC_XML_SAMPLED_TEST ${STATIC_DISC_XML_SAMPLED_TEST_COMPLETE_PATH} NAME_WE) + # copy the file from src to build folders + configure_file( + ${STATIC_DISC_XML_SAMPLED_TEST_COMPLETE_PATH} # from full src path + ${CMAKE_CURRENT_BINARY_DIR}/xml_static_disc_docs/test/blackbox/${STATIC_DISC_XML_SAMPLED_TEST}.xml # to relative build path + ) + endforeach() + + # Define test + add_test(NAME xml.static_disc_validate COMMAND ${env_fast_command} xml validate xml_static_disc_docs -x ${CMAKE_SOURCE_DIR}/resources/xsd/fastdds_static_discovery.xsd) + + set_property( + TEST xml.static_disc_validate + PROPERTY ENVIRONMENT ${TEST_ENVIRONMENT} + ) + +endif() +>>>>>>> 3b2e781f3 (Fix Python Installation version in Github CI. Address failing system tests environment issues (#4759)) diff --git a/tools/fastdds/fastdds.bat b/tools/fastdds/fastdds.bat index 9bbcc0d6aa0..d796ffcae29 100644 --- a/tools/fastdds/fastdds.bat +++ b/tools/fastdds/fastdds.bat @@ -20,5 +20,14 @@ if not %ERRORLEVEL%==0 ( exit /B 65 ) ) + +:: Python version in the form "Major.Minor" +if %PYTHON_VERSION%=="" ( + echo error retrieving python version. Please, make sure python is installed and accessible. + exit /B 65 + ) +) + :: Use launcher to profit from shebang hints on fastdds.py -py "%dir%\..\tools\fastdds\fastdds.py" %args% +:: Select the correct python version to source the appropriate paths +py -%PYTHON_VERSION% "%dir%\..\tools\fastdds\fastdds.py" %args% diff --git a/tools/fastdds/ros-discovery.bat b/tools/fastdds/ros-discovery.bat index ac1875e8195..26fee3900cf 100644 --- a/tools/fastdds/ros-discovery.bat +++ b/tools/fastdds/ros-discovery.bat @@ -20,5 +20,14 @@ if not %ERRORLEVEL%==0 ( exit /B 65 ) ) + +:: Python version in the form "Major.Minor" +if %PYTHON_VERSION%=="" ( + echo error retrieving python version. Please, make sure python is installed and accessible. + exit /B 65 + ) +) + :: Use launcher to profit from shebang hints on fastdds.py -py "%dir%\..\tools\fastdds\fastdds.py" discovery %args% +:: Select the correct python version to source the appropriate paths +py -%PYTHON_VERSION% "%dir%\..\tools\fastdds\fastdds.py" discovery %args%