From 5084eb3dd2096dc5bc7ad60daa718297c2acd4dc Mon Sep 17 00:00:00 2001 From: Miguel Company Date: Tue, 17 Oct 2023 10:47:50 +0200 Subject: [PATCH 01/10] Refs #19727. Preparing new approach based on colcon. Signed-off-by: Miguel Company --- .github/workflows/config/tsan_colcon.meta | 23 +++ .github/workflows/thread-sanitizer-v2.yaml | 163 +++++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100644 .github/workflows/config/tsan_colcon.meta create mode 100644 .github/workflows/thread-sanitizer-v2.yaml diff --git a/.github/workflows/config/tsan_colcon.meta b/.github/workflows/config/tsan_colcon.meta new file mode 100644 index 00000000000..a7d46a38cc1 --- /dev/null +++ b/.github/workflows/config/tsan_colcon.meta @@ -0,0 +1,23 @@ +{ + "names": + { + "fastrtps": + { + "cmake-args": + [ + "-DCMAKE_BUILD_TYPE=Debug", + "-DEPROSIMA_BUILD_TESTS=ON", + "-DGTEST_INDIVIDUAL=ON", + "-DRTPS_API_TESTS=ON", + "-DFASTRTPS_API_TESTS=OFF", + "-DFASTDDS_PIM_API_TESTS=ON", + "-DPERFORMANCE_TESTS=ON", + "-DSECURITY=ON", + "-DFASTDDS_STATISTICS=ON", + "-DCMAKE_C_FLAGS='-fsanitize=thread -O2 -g -fno-omit-frame-pointer'", + "-DCMAKE_CXX_FLAGS='-fsanitize=thread -O2 -g -fno-omit-frame-pointer'" + ] + } + + } +} diff --git a/.github/workflows/thread-sanitizer-v2.yaml b/.github/workflows/thread-sanitizer-v2.yaml new file mode 100644 index 00000000000..242e07432b7 --- /dev/null +++ b/.github/workflows/thread-sanitizer-v2.yaml @@ -0,0 +1,163 @@ +name: Thread Sanitizer analysis v2 + +on: + workflow_dispatch: + inputs: + fastdds_branch: + description: 'Branch or tag of Fast DDS repository (https://github.com/eProsima/Fast-DDS)' + required: true + default: 'master' + pull_request: + push: + branches: + - master + schedule: + - cron: '0 1 * * *' + +jobs: + + tsan-test: + + if: ${{ !(contains(github.event.pull_request.labels.*.name, 'no-test') || + contains(github.event.pull_request.labels.*.name, 'skip-ci')) }} + + runs-on: ubuntu-22.04 + + env: + FASTDDS_BRANCH: ${{ github.head_ref || github.event.inputs.fastdds_branch || 'master' }} + TSAN_OPTIONS: second_deadlock_stack=1 history_size=7 memory_limit_mb=5000 + # GCC 11.3 (Ubuntu Jammy default) produces several false positives regarding timed synchronization protocols + # These issues were fixed in GCC 12 so we upgrade to that version. + CC: gcc-12 + CXX: g++-12 + + steps: + - uses: eProsima/eProsima-CI/ubuntu/install_apt_packages@v0 + with: + packages: wget + + - uses: eProsima/eProsima-CI/ubuntu/install_python_packages@v0 + with: + packages: vcstool + + - uses: eProsima/eProsima-CI/ubuntu/get_file_from_repo@v0 + with: + source_repository: eProsima/Fast-DDS + source_repository_branch: ${{ env.FASTDDS_BRANCH }} + file_name: fastrtps.repos + file_result: fastrtps.repos + + - name: Fetch Fast DDS & dependencies + uses: eProsima/eProsima-CI/ubuntu/vcs_import@v0 + with: + vcs_repos_file: fastrtps.repos + destination_workspace: src + + - name: Checkout Fast DDS branch + run: | + cd ./src/fastrtps + git checkout ${{ env.FASTDDS_BRANCH }} + + - name: Install apt packages + uses: ./src/fastrtps/.github/actions/install-apt-packages + + - name: Install Python packages + uses: ./src/fastrtps/.github/actions/install-python-packages + + - name: Install GTest + uses: eProsima/eProsima-CI/ubuntu/install_gtest@v0 + with: + version: v1.12.1 + + - name: Build workspace + run: | + cat src/fastrtps/.github/workflows/config/tsan_colcon.meta + colcon build \ + --event-handlers=console_direct+ \ + --metas src/fastrtps/.github/workflows/config/tsan_colcon.meta + + - name: Run tests Fast DDS + run: | + source install/setup.bash && \ + colcon test \ + --packages-select fastrtps \ + --event-handlers=console_direct+ \ + --return-code-on-test-failure \ + --ctest-args \ + --label-exclude xfail \ + --timeout 300 + continue-on-error: true + + - name: Upload Logs + uses: actions/upload-artifact@v3 + with: + name: tsan-logs + path: log/ + if: always() + + - name: Process sanitizer reports + id: report_summary + shell: pwsh + continue-on-error: true + run: | + # Create a dir for the exports + $exports = New-Item -ItemType Directory -Path ./ws/exports + # Move to the reports dir + pushd ./ws/build/fastdds/reports + # Install the report parser module + Find-Module -Repository PSGallery -Name SanReportParser | Install-Module -Scope CurrentUser -Force + # Parse the report files + $rp = Show-Tsan (gci) + # filter duplicates + $rp = $rp | group md5hash | % { $_.group[0] } + # Export raw data + $rp | Export-CliXML (Join-Path $exports all_reports.xml) + # Group the reports by issue + $g = $rp | group fuzzhash + # Split up deadlocks and race reports + $gd = $rp | ? type -match dead | group fuzzhash + $gr = $rp | ? type -match race | group fuzzhash + # Simplified deadlock summary (only one representative report and tests associated) + $sd = $gd | Sort-Object count -desc | select @{l="fuzzhash";e="name"}, count, ` + @{l="échantillon";e={$_.group[0].report}}, @{l="tests"; ` + e={$_.group.file | sls "(.*)\.\d+$" | % { $_.Matches.Groups[1].Value } | Sort-Object | get-unique}} + # Simplified race summary (only one representative report and tests associated) + $sr = $gr | Sort-Object count -desc | select @{l="fuzzhash";e="name"}, count, ` + @{l="échantillon";e={$_.group[0].report}}, @{l="tests"; ` + e={$_.group.file | sls "(.*)\.\d+$" | % { $_.Matches.Groups[1].Value } | Sort-Object | get-unique}} + # Export simplified summaries + $sd, $sr | Export-Clixml (Join-Path $exports summary_data.xml) + # Export CSV summary of frequencies + $sd | select fuzzhash, count | Export-CSV -Path (Join-Path $exports deadlocks.csv) + $sr | select fuzzhash, count | Export-CSV -Path (Join-Path $exports races.csv) + # Export test mappings + & {$sd; $sr} | Sort-Object { [int]$_.fuzzhash} | % { $fuzzy=$_.fuzzhash;$_.tests | + select @{l="fuzzhash";e={$fuzzy}}, @{l="test";e={$_}}} | + Export-Csv (Join-Path $exports issue_test_map.csv) + # Keep a file per issue + $dir = New-Item -ItemType Directory -Path (Join-Path $exports reports) + & {$sd; $sr} | % { $_.échantillon | Out-File (Join-Path $dir "$($_.fuzzhash).tsan") } + # Create a summary table + @{Type="Deadlock";Failed=$sd.count;Hashes=$sd.fuzzhash}, + @{Type="Data race";Failed=$sr.count;Hashes=$sr.fuzzhash} | + % { $_.Summary = $_.Hashes | select -First 5 | Join-String -Separator ", " + if ($_.Hashes.count -gt 5 ) { + $_.Summary += ", ..." }; $_ } | + % { [PSCustomObject]$_} | + New-MDTable -Columns ([ordered]@{Failed=$null;Type=$null;Summary=$null}) | + Out-File $Env:GITHUB_STEP_SUMMARY + # The step fails on new deadlocks + $LASTEXITCODE=$sd.count + + - name: Archive sanitation results + uses: actions/upload-artifact@v3 + with: + name: sanitation-report + path: /home/runner/work/ws/exports/* + + - name: Check on failures + if: ${{ steps.report_summary.outcome == 'failure' }} + shell: pwsh + run: | + Write-Host ${{ steps.report_summary.outcome }} + exit 1 From 6ce077c8283ee846b484da5124a44267b58d925f Mon Sep 17 00:00:00 2001 From: Miguel Company Date: Tue, 17 Oct 2023 10:57:53 +0200 Subject: [PATCH 02/10] Refs #19727. Cancel in-progress jobs. Signed-off-by: Miguel Company --- .github/workflows/thread-sanitizer-v2.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/thread-sanitizer-v2.yaml b/.github/workflows/thread-sanitizer-v2.yaml index 242e07432b7..ad7a8b38f89 100644 --- a/.github/workflows/thread-sanitizer-v2.yaml +++ b/.github/workflows/thread-sanitizer-v2.yaml @@ -14,6 +14,10 @@ on: schedule: - cron: '0 1 * * *' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: tsan-test: From cfe589d6a5ab052f7805ae4a4482fa2cec282700 Mon Sep 17 00:00:00 2001 From: Miguel Company Date: Tue, 17 Oct 2023 11:08:00 +0200 Subject: [PATCH 03/10] Refs #19727. Build all packages with CMAKE_BUILD_TYPE=Debug. Signed-off-by: Miguel Company --- .github/workflows/config/tsan_colcon.meta | 2 +- .github/workflows/thread-sanitizer-v2.yaml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/config/tsan_colcon.meta b/.github/workflows/config/tsan_colcon.meta index a7d46a38cc1..3ed6fd5c6c4 100644 --- a/.github/workflows/config/tsan_colcon.meta +++ b/.github/workflows/config/tsan_colcon.meta @@ -5,7 +5,7 @@ { "cmake-args": [ - "-DCMAKE_BUILD_TYPE=Debug", + "-DCOMPILE_EXAMPLES=OFF", "-DEPROSIMA_BUILD_TESTS=ON", "-DGTEST_INDIVIDUAL=ON", "-DRTPS_API_TESTS=ON", diff --git a/.github/workflows/thread-sanitizer-v2.yaml b/.github/workflows/thread-sanitizer-v2.yaml index ad7a8b38f89..fc459087f2f 100644 --- a/.github/workflows/thread-sanitizer-v2.yaml +++ b/.github/workflows/thread-sanitizer-v2.yaml @@ -78,7 +78,8 @@ jobs: cat src/fastrtps/.github/workflows/config/tsan_colcon.meta colcon build \ --event-handlers=console_direct+ \ - --metas src/fastrtps/.github/workflows/config/tsan_colcon.meta + --metas src/fastrtps/.github/workflows/config/tsan_colcon.meta \ + --cmake-args -DCMAKE_BUILD_TYPE=Debug - name: Run tests Fast DDS run: | From cb030e9702966e511a0100fa809eb1b90bf1d880 Mon Sep 17 00:00:00 2001 From: Miguel Company Date: Tue, 17 Oct 2023 16:07:06 +0200 Subject: [PATCH 04/10] Refs #19727. Change ctest args. Signed-off-by: Miguel Company --- .github/workflows/thread-sanitizer-v2.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/thread-sanitizer-v2.yaml b/.github/workflows/thread-sanitizer-v2.yaml index fc459087f2f..b7ae237f60e 100644 --- a/.github/workflows/thread-sanitizer-v2.yaml +++ b/.github/workflows/thread-sanitizer-v2.yaml @@ -89,8 +89,8 @@ jobs: --event-handlers=console_direct+ \ --return-code-on-test-failure \ --ctest-args \ - --label-exclude xfail \ - --timeout 300 + --timeout 30 \ + -V -E DDSSQLFilterValueTests continue-on-error: true - name: Upload Logs From fa3eb62b6e42739ee4c36b4cd1f53c24e75f419d Mon Sep 17 00:00:00 2001 From: Miguel Company Date: Tue, 17 Oct 2023 16:07:45 +0200 Subject: [PATCH 05/10] Refs #19727. Fix report steps. Signed-off-by: Miguel Company --- .github/workflows/thread-sanitizer-v2.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/thread-sanitizer-v2.yaml b/.github/workflows/thread-sanitizer-v2.yaml index b7ae237f60e..1c02fc93947 100644 --- a/.github/workflows/thread-sanitizer-v2.yaml +++ b/.github/workflows/thread-sanitizer-v2.yaml @@ -106,13 +106,13 @@ jobs: continue-on-error: true run: | # Create a dir for the exports - $exports = New-Item -ItemType Directory -Path ./ws/exports + $exports = New-Item -ItemType Directory -Path ./exports # Move to the reports dir - pushd ./ws/build/fastdds/reports + pushd ./log/latest_test/fastrtps # Install the report parser module Find-Module -Repository PSGallery -Name SanReportParser | Install-Module -Scope CurrentUser -Force # Parse the report files - $rp = Show-Tsan (gci) + $rp = Show-Tsan -Path ./stdout_stderr.log # filter duplicates $rp = $rp | group md5hash | % { $_.group[0] } # Export raw data @@ -158,7 +158,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: sanitation-report - path: /home/runner/work/ws/exports/* + path: exports/* - name: Check on failures if: ${{ steps.report_summary.outcome == 'failure' }} From cc929df4dce9271604cd6fd51c4b25dc10ab542d Mon Sep 17 00:00:00 2001 From: Miguel Company Date: Fri, 20 Oct 2023 16:02:07 +0200 Subject: [PATCH 06/10] Refs #19727. Move changes back into old workflow. Signed-off-by: Miguel Company --- .github/workflows/thread-sanitizer.yaml | 204 ++++++++++-------------- 1 file changed, 84 insertions(+), 120 deletions(-) diff --git a/.github/workflows/thread-sanitizer.yaml b/.github/workflows/thread-sanitizer.yaml index 7c402fffa2f..299350bb597 100644 --- a/.github/workflows/thread-sanitizer.yaml +++ b/.github/workflows/thread-sanitizer.yaml @@ -2,14 +2,10 @@ name: Thread Sanitizer analysis on: workflow_dispatch: inputs: - optional_cmake_args: - description: 'Optional CMake Compilation Flags' - required: false - type: string - optional_ctest_args: - description: 'Optional CTest Testing Flags' - required: false - type: string + fastdds_branch: + description: 'Branch or tag of Fast DDS repository (https://github.com/eProsima/Fast-DDS)' + required: true + default: 'master' push: branches: @@ -27,139 +23,106 @@ on: - '**.txt' - '!**/CMakeLists.txt' -env: - TSAN_OPTIONS: second_deadlock_stack=1 history_size=7 memory_limit_mb=5000 - # ubuntu host doesn't have an environment SHELL value (docker and vm images does)??? - SHELL: /bin/bash - # GCC 11.3 (Ubuntu Jammy default) produces several false positives regarding timed synchronization protocols - # These issues were fixed in GCC 12 so we upgrade to that version. - CC: gcc-12 - CXX: g++-12 - -defaults: - run: - shell: pwsh - working-directory: /home/runner/work +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: + ubuntu-sanitizer-run: name: Sanitizer Evaluation + if: ${{ !(contains(github.event.pull_request.labels.*.name, 'no-test') || contains(github.event.pull_request.labels.*.name, 'skip-ci')) }} + runs-on: ubuntu-22.04 + + env: + FASTDDS_BRANCH: ${{ github.head_ref || github.event.inputs.fastdds_branch || 'master' }} + TSAN_OPTIONS: second_deadlock_stack=1 history_size=7 memory_limit_mb=5000 + # GCC 11.3 (Ubuntu Jammy default) produces several false positives regarding timed synchronization protocols + # These issues were fixed in GCC 12 so we upgrade to that version. + CC: gcc-12 + CXX: g++-12 + steps: - - name: Download the sanitizers cmake module and apply some fixes - run: | - git config --global user.email "dummy@mail.com" - git config --global user.name "dummy" - cd (New-Item -ItemType Directory -Path ./ws) - git clone https://github.com/arsenm/sanitizers-cmake - cd sanitizers-cmake - git remote add efork https://github.com/MiguelBarro/sanitizers-cmake.git - git fetch efork - git merge --no-edit efork/bugfix/cmake_updates - - - name: Download googletest and make it sanitizer aware - run: | - cd ws - git clone --branch release-1.11.0 https://github.com/google/googletest.git - cd googletest - git remote add efork https://github.com/MiguelBarro/googletest.git - git fetch efork - git merge --no-edit efork/bugfix/add_sanitizer_support - - - name: Build & install googletest - run: | - cd ws - cmake -DCMAKE_MODULE_PATH="$(pwd)/sanitizers-cmake/cmake" -DCMAKE_INSTALL_PREFIX="./install" ` - -DBUILD_GMOCK=ON -DSANITIZE_THREAD=ON -DCMAKE_BUILD_TYPE=Debug ` - -DCMAKE_VERBOSE_MAKEFILE=ON -B ./build/googletest ./googletest - cmake --build ./build/googletest --target install --verbose + - uses: eProsima/eProsima-CI/ubuntu/install_apt_packages@v0 + with: + packages: wget - - name: Download Fast-CDR and make it sanitizer aware - run: | - cd ws - git clone --branch 1.1.x https://github.com/eProsima/Fast-CDR.git - cd Fast-CDR - git remote add efork https://github.com/MiguelBarro/Fast-CDR.git - git fetch efork - git merge --no-edit efork/bugfix/add_sanitizer_support - - - name: Build & install Fast-CDR - run: | - cd ws - cmake -DCMAKE_INSTALL_PREFIX="./install" -DCMAKE_PREFIX_PATH="./install" ` - -DCMAKE_MODULE_PATH="$(pwd)/sanitizers-cmake/cmake" -DCMAKE_VERBOSE_MAKEFILE=ON ` - -DSANITIZE_THREAD=ON -DCMAKE_BUILD_TYPE=Debug -B ./build/fastcdr ./Fast-CDR - cmake --build ./build/fastcdr --target install --verbose + - uses: eProsima/eProsima-CI/ubuntu/install_python_packages@v0 + with: + packages: vcstool - - name: Download foonathan/memory and make it sanitizer aware - run: | - cd ws - git clone https://github.com/foonathan/memory.git - cd memory - git remote add efork https://github.com/MiguelBarro/memory.git - git fetch efork - git merge --no-edit efork/bugfix/add_sanitizer_support - - - name: Build & install foonathan/memory - run: | - cd ws - cmake -DCMAKE_MODULE_PATH="$(pwd)/sanitizers-cmake/cmake" -DCMAKE_VERBOSE_MAKEFILE=ON -DSANITIZE_THREAD=ON ` - -DCMAKE_INSTALL_PREFIX="./install" -DFOONATHAN_MEMORY_BUILD_TOOLS=ON -DCMAKE_POSITION_INDEPENDENT_CODE=ON ` - -DFOONATHAN_MEMORY_BUILD_TESTS=ON -DFOONATHAN_MEMORY_BUILD_EXAMPLES=ON -DFOONATHAN_MEMORY_BUILD_TOOLS=ON ` - -DCMAKE_BUILD_TYPE=Debug -B ./build/memory ./memory - cmake --build ./build/memory --target install --verbose - - - name: Download Fast-DDS and make it sanitizer aware + - uses: eProsima/eProsima-CI/ubuntu/get_file_from_repo@v0 + with: + source_repository: eProsima/Fast-DDS + source_repository_branch: ${{ env.FASTDDS_BRANCH }} + file_name: fastrtps.repos + file_result: fastrtps.repos + + - name: Fetch Fast DDS & dependencies + uses: eProsima/eProsima-CI/ubuntu/vcs_import@v0 + with: + vcs_repos_file: fastrtps.repos + destination_workspace: src + + - name: Checkout Fast DDS branch run: | - cd ws - git clone https://github.com/eProsima/Fast-DDS.git - cd Fast-DDS - git fetch origin ${{ github.ref }}:merge-branch - git checkout merge-branch - echo "Modified files against destination branch" - git diff --name-only HEAD~1 - git remote add efork https://github.com/MiguelBarro/Fast-DDS.git - git fetch efork - git merge --no-edit efork/bugfix/add_sanitizer_support - - - name: Build & install Fast-DDS + cd ./src/fastrtps + git checkout ${{ env.FASTDDS_BRANCH }} + + - name: Install apt packages + uses: ./src/fastrtps/.github/actions/install-apt-packages + + - name: Install Python packages + uses: ./src/fastrtps/.github/actions/install-python-packages + + - name: Install GTest + uses: eProsima/eProsima-CI/ubuntu/install_gtest@v0 + with: + version: v1.12.1 + + - name: Build workspace run: | - cd ws - cmake -DCMAKE_PREFIX_PATH="$(pwd)/install" -DCMAKE_INSTALL_PREFIX='./install' ` - -DCMAKE_MODULE_PATH="$(pwd)/sanitizers-cmake/cmake" -DCMAKE_VERBOSE_MAKEFILE=ON -DSANITIZE_THREAD=ON ` - -DGTEST_INDIVIDUAL=ON -DFASTDDS_STATISTICS=ON -DTHIRDPARTY=ON ` - -DCOMPILE_EXAMPLES=OFF -DEPROSIMA_BUILD_TESTS=ON -DSECURITY=ON ` - -DRTPS_API_TESTS=ON -DFASTRTPS_API_TESTS=ON -DFASTDDS_PIM_API_TESTS=ON ` - -DCMAKE_BUILD_TYPE=Debug ${{ inputs.optional_cmake_args }} -B ./build/fastdds ./Fast-DDS - # Create a symlink to the reports folder - $report_dir = gci -Path ./build/fastdds -Attributes D | - ? Name -like "$(Get-Date -Format 'MMMM-dd-yyyy')*" - $report_dir = Join-Path $report_dir tsan | gi - echo "report folder is: $report_dir" - New-Item -ItemType SymbolicLink -Target $report_dir -Path ./build/fastdds/reports - cmake --build ./build/fastdds --target install -j 2 --verbose - - - name: Run Fast-DDS test & generate sanitizer reports + cat src/fastrtps/.github/workflows/config/tsan_colcon.meta + colcon build \ + --event-handlers=console_direct+ \ + --metas src/fastrtps/.github/workflows/config/tsan_colcon.meta \ + --cmake-args -DCMAKE_BUILD_TYPE=Debug + + - name: Run tests Fast DDS run: | - cd ws/build/fastdds - # Exclude DDSSQLFilterValueTests tests to prevent hitting GH Actions resource limits - ctest --timeout 30 -j 2 -V -E DDSSQLFilterValueTests ${{ inputs.optional_ctest_args }} - # ctest fails if sanitizer reports are present - $LASTEXITCODE=0 + source install/setup.bash && \ + colcon test \ + --packages-select fastrtps \ + --event-handlers=console_direct+ \ + --return-code-on-test-failure \ + --ctest-args \ + --timeout 30 \ + -V -E DDSSQLFilterValueTests + continue-on-error: true + + - name: Upload Logs + uses: actions/upload-artifact@v3 + with: + name: tsan-logs + path: log/ + if: always() - name: Process sanitizer reports id: report_summary + shell: pwsh continue-on-error: true run: | # Create a dir for the exports - $exports = New-Item -ItemType Directory -Path ./ws/exports + $exports = New-Item -ItemType Directory -Path ./exports # Move to the reports dir - pushd ./ws/build/fastdds/reports + pushd ./log/latest_test/fastrtps # Install the report parser module Find-Module -Repository PSGallery -Name SanReportParser | Install-Module -Scope CurrentUser -Force # Parse the report files - $rp = Show-Tsan (gci) + $rp = Show-Tsan -Path ./stdout_stderr.log # filter duplicates $rp = $rp | group md5hash | % { $_.group[0] } # Export raw data @@ -205,10 +168,11 @@ jobs: uses: actions/upload-artifact@v3 with: name: sanitation-report - path: /home/runner/work/ws/exports/* + path: exports/* - name: Check on failures if: ${{ steps.report_summary.outcome == 'failure' }} + shell: pwsh run: | Write-Host ${{ steps.report_summary.outcome }} exit 1 From 92256ae8a9f5b95a51a97e4c73a6a538623b6403 Mon Sep 17 00:00:00 2001 From: Miguel Company Date: Fri, 20 Oct 2023 16:08:47 +0200 Subject: [PATCH 07/10] Refs #19727. Removed v2 workflow. Signed-off-by: Miguel Company --- .github/workflows/thread-sanitizer-v2.yaml | 168 --------------------- 1 file changed, 168 deletions(-) delete mode 100644 .github/workflows/thread-sanitizer-v2.yaml diff --git a/.github/workflows/thread-sanitizer-v2.yaml b/.github/workflows/thread-sanitizer-v2.yaml deleted file mode 100644 index 1c02fc93947..00000000000 --- a/.github/workflows/thread-sanitizer-v2.yaml +++ /dev/null @@ -1,168 +0,0 @@ -name: Thread Sanitizer analysis v2 - -on: - workflow_dispatch: - inputs: - fastdds_branch: - description: 'Branch or tag of Fast DDS repository (https://github.com/eProsima/Fast-DDS)' - required: true - default: 'master' - pull_request: - push: - branches: - - master - schedule: - - cron: '0 1 * * *' - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - - tsan-test: - - if: ${{ !(contains(github.event.pull_request.labels.*.name, 'no-test') || - contains(github.event.pull_request.labels.*.name, 'skip-ci')) }} - - runs-on: ubuntu-22.04 - - env: - FASTDDS_BRANCH: ${{ github.head_ref || github.event.inputs.fastdds_branch || 'master' }} - TSAN_OPTIONS: second_deadlock_stack=1 history_size=7 memory_limit_mb=5000 - # GCC 11.3 (Ubuntu Jammy default) produces several false positives regarding timed synchronization protocols - # These issues were fixed in GCC 12 so we upgrade to that version. - CC: gcc-12 - CXX: g++-12 - - steps: - - uses: eProsima/eProsima-CI/ubuntu/install_apt_packages@v0 - with: - packages: wget - - - uses: eProsima/eProsima-CI/ubuntu/install_python_packages@v0 - with: - packages: vcstool - - - uses: eProsima/eProsima-CI/ubuntu/get_file_from_repo@v0 - with: - source_repository: eProsima/Fast-DDS - source_repository_branch: ${{ env.FASTDDS_BRANCH }} - file_name: fastrtps.repos - file_result: fastrtps.repos - - - name: Fetch Fast DDS & dependencies - uses: eProsima/eProsima-CI/ubuntu/vcs_import@v0 - with: - vcs_repos_file: fastrtps.repos - destination_workspace: src - - - name: Checkout Fast DDS branch - run: | - cd ./src/fastrtps - git checkout ${{ env.FASTDDS_BRANCH }} - - - name: Install apt packages - uses: ./src/fastrtps/.github/actions/install-apt-packages - - - name: Install Python packages - uses: ./src/fastrtps/.github/actions/install-python-packages - - - name: Install GTest - uses: eProsima/eProsima-CI/ubuntu/install_gtest@v0 - with: - version: v1.12.1 - - - name: Build workspace - run: | - cat src/fastrtps/.github/workflows/config/tsan_colcon.meta - colcon build \ - --event-handlers=console_direct+ \ - --metas src/fastrtps/.github/workflows/config/tsan_colcon.meta \ - --cmake-args -DCMAKE_BUILD_TYPE=Debug - - - name: Run tests Fast DDS - run: | - source install/setup.bash && \ - colcon test \ - --packages-select fastrtps \ - --event-handlers=console_direct+ \ - --return-code-on-test-failure \ - --ctest-args \ - --timeout 30 \ - -V -E DDSSQLFilterValueTests - continue-on-error: true - - - name: Upload Logs - uses: actions/upload-artifact@v3 - with: - name: tsan-logs - path: log/ - if: always() - - - name: Process sanitizer reports - id: report_summary - shell: pwsh - continue-on-error: true - run: | - # Create a dir for the exports - $exports = New-Item -ItemType Directory -Path ./exports - # Move to the reports dir - pushd ./log/latest_test/fastrtps - # Install the report parser module - Find-Module -Repository PSGallery -Name SanReportParser | Install-Module -Scope CurrentUser -Force - # Parse the report files - $rp = Show-Tsan -Path ./stdout_stderr.log - # filter duplicates - $rp = $rp | group md5hash | % { $_.group[0] } - # Export raw data - $rp | Export-CliXML (Join-Path $exports all_reports.xml) - # Group the reports by issue - $g = $rp | group fuzzhash - # Split up deadlocks and race reports - $gd = $rp | ? type -match dead | group fuzzhash - $gr = $rp | ? type -match race | group fuzzhash - # Simplified deadlock summary (only one representative report and tests associated) - $sd = $gd | Sort-Object count -desc | select @{l="fuzzhash";e="name"}, count, ` - @{l="échantillon";e={$_.group[0].report}}, @{l="tests"; ` - e={$_.group.file | sls "(.*)\.\d+$" | % { $_.Matches.Groups[1].Value } | Sort-Object | get-unique}} - # Simplified race summary (only one representative report and tests associated) - $sr = $gr | Sort-Object count -desc | select @{l="fuzzhash";e="name"}, count, ` - @{l="échantillon";e={$_.group[0].report}}, @{l="tests"; ` - e={$_.group.file | sls "(.*)\.\d+$" | % { $_.Matches.Groups[1].Value } | Sort-Object | get-unique}} - # Export simplified summaries - $sd, $sr | Export-Clixml (Join-Path $exports summary_data.xml) - # Export CSV summary of frequencies - $sd | select fuzzhash, count | Export-CSV -Path (Join-Path $exports deadlocks.csv) - $sr | select fuzzhash, count | Export-CSV -Path (Join-Path $exports races.csv) - # Export test mappings - & {$sd; $sr} | Sort-Object { [int]$_.fuzzhash} | % { $fuzzy=$_.fuzzhash;$_.tests | - select @{l="fuzzhash";e={$fuzzy}}, @{l="test";e={$_}}} | - Export-Csv (Join-Path $exports issue_test_map.csv) - # Keep a file per issue - $dir = New-Item -ItemType Directory -Path (Join-Path $exports reports) - & {$sd; $sr} | % { $_.échantillon | Out-File (Join-Path $dir "$($_.fuzzhash).tsan") } - # Create a summary table - @{Type="Deadlock";Failed=$sd.count;Hashes=$sd.fuzzhash}, - @{Type="Data race";Failed=$sr.count;Hashes=$sr.fuzzhash} | - % { $_.Summary = $_.Hashes | select -First 5 | Join-String -Separator ", " - if ($_.Hashes.count -gt 5 ) { - $_.Summary += ", ..." }; $_ } | - % { [PSCustomObject]$_} | - New-MDTable -Columns ([ordered]@{Failed=$null;Type=$null;Summary=$null}) | - Out-File $Env:GITHUB_STEP_SUMMARY - # The step fails on new deadlocks - $LASTEXITCODE=$sd.count - - - name: Archive sanitation results - uses: actions/upload-artifact@v3 - with: - name: sanitation-report - path: exports/* - - - name: Check on failures - if: ${{ steps.report_summary.outcome == 'failure' }} - shell: pwsh - run: | - Write-Host ${{ steps.report_summary.outcome }} - exit 1 From a156a745182177f4333e29bafec3345b96aab995 Mon Sep 17 00:00:00 2001 From: Miguel Company Date: Mon, 23 Oct 2023 07:37:13 +0200 Subject: [PATCH 08/10] Refs #19727. Optional cmake and ctest arguments on dispatch inputs. Signed-off-by: Miguel Company --- .github/workflows/thread-sanitizer.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/thread-sanitizer.yaml b/.github/workflows/thread-sanitizer.yaml index 299350bb597..2c4b6432aab 100644 --- a/.github/workflows/thread-sanitizer.yaml +++ b/.github/workflows/thread-sanitizer.yaml @@ -6,6 +6,14 @@ on: description: 'Branch or tag of Fast DDS repository (https://github.com/eProsima/Fast-DDS)' required: true default: 'master' + optional_cmake_args: + description: 'Optional CMake Compilation Flags' + required: false + type: string + optional_ctest_args: + description: 'Optional CTest Testing Flags' + required: false + type: string push: branches: @@ -89,7 +97,7 @@ jobs: colcon build \ --event-handlers=console_direct+ \ --metas src/fastrtps/.github/workflows/config/tsan_colcon.meta \ - --cmake-args -DCMAKE_BUILD_TYPE=Debug + --cmake-args -DCMAKE_BUILD_TYPE=Debug ${{ inputs.optional_cmake_args }} - name: Run tests Fast DDS run: | @@ -100,7 +108,7 @@ jobs: --return-code-on-test-failure \ --ctest-args \ --timeout 30 \ - -V -E DDSSQLFilterValueTests + -V -E DDSSQLFilterValueTests ${{ inputs.optional_ctest_args }} continue-on-error: true - name: Upload Logs From 58c192a50d6faea25014bee1f431641a3b0b35d8 Mon Sep 17 00:00:00 2001 From: Miguel Company Date: Mon, 23 Oct 2023 07:39:37 +0200 Subject: [PATCH 09/10] Refs #19727. Add build flags to fastcdr. Signed-off-by: Miguel Company --- .github/workflows/config/tsan_colcon.meta | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/config/tsan_colcon.meta b/.github/workflows/config/tsan_colcon.meta index 3ed6fd5c6c4..3b45525dec8 100644 --- a/.github/workflows/config/tsan_colcon.meta +++ b/.github/workflows/config/tsan_colcon.meta @@ -1,6 +1,14 @@ { "names": { + "fastcdr": + { + "cmake-args": + [ + "-DCMAKE_C_FLAGS='-fsanitize=thread -O2 -g -fno-omit-frame-pointer'", + "-DCMAKE_CXX_FLAGS='-fsanitize=thread -O2 -g -fno-omit-frame-pointer'" + ] + }, "fastrtps": { "cmake-args": From 2e660346d2bc940987c4ebf85cc12cfb13f1f890 Mon Sep 17 00:00:00 2001 From: Miguel Company Date: Mon, 23 Oct 2023 12:40:51 +0200 Subject: [PATCH 10/10] Refs #19727. Change google test version. Signed-off-by: Miguel Company Co-authored-by: Eduardo Ponz Segrelles --- .github/workflows/thread-sanitizer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/thread-sanitizer.yaml b/.github/workflows/thread-sanitizer.yaml index 2c4b6432aab..712e001d4dc 100644 --- a/.github/workflows/thread-sanitizer.yaml +++ b/.github/workflows/thread-sanitizer.yaml @@ -89,7 +89,7 @@ jobs: - name: Install GTest uses: eProsima/eProsima-CI/ubuntu/install_gtest@v0 with: - version: v1.12.1 + version: release-1.11.0 - name: Build workspace run: |