From 9fa0363dfee202f1ddd01c65e2360e645adf6cd0 Mon Sep 17 00:00:00 2001 From: Gregor Olenik Date: Mon, 13 Jan 2025 14:49:21 +0100 Subject: [PATCH 01/15] execute benchmarks on aws --- .github/workflows/build_on_aws.yaml | 28 +++++- benchmarks/CMakeLists.txt | 29 +++---- benchmarks/fields/bench_fields.cpp | 130 ++++++++++++++-------------- 3 files changed, 102 insertions(+), 85 deletions(-) diff --git a/.github/workflows/build_on_aws.yaml b/.github/workflows/build_on_aws.yaml index d0937c2e6..a4eb2dd87 100644 --- a/.github/workflows/build_on_aws.yaml +++ b/.github/workflows/build_on_aws.yaml @@ -121,11 +121,37 @@ jobs: module load libfabric-aws module load cmake ctest --preset ${{matrix.preset}} + benchmark-on-aws: + name: Build on aws + needs: [start-runner, build-on-aws] # required to start the main job when the runner is ready + runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner + steps: + - name: Build NeoFOAM + shell: bash -i {0} + run: | + export HOME=/share/ec2-user + module load clang/16 + module load libfabric-aws + module spider libfabric-aws + module load cmake + cmake --version + CC=clang \ + CXX=clang++ \ + cmake --preset production \ + -DNEOFOAM_ENABLE_MPI_WITH_THREAD_SUPPORT=OFF \ + -DNEOFOAM_BUILD_BENCHMARKS=ON \ + -DKokkos_ENABLE_CUDA=ON + cmake --build --preset production + - name: Execute Benchmarks + shell: bash -i {0} + run: | + cmake --build --preset production --target execute_benchmarks + stop-runner: name: Stop self-hosted EC2 runner needs: - start-runner # required to get output from the start-runner job - - build-on-aws # required to wait when the main job is done + - benchmark-on-aws # required to wait when the main job is done runs-on: ubuntu-latest permissions: id-token: write diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 071ef3180..b1c4bbb56 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -1,30 +1,21 @@ # SPDX-License-Identifier: Unlicense -# SPDX-FileCopyrightText: 2023 NeoFOAM authors +# SPDX-FileCopyrightText: 2023-25 NeoFOAM authors add_subdirectory(fields) add_custom_command( OUTPUT ${PROJECT_BINARY_DIR}/benchmarks/fields.xml - COMMAND ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/bench_fields -r XML > fields.xml + COMMAND ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/bench_fields -r json COMMENT "Execute benchmarks") -find_package( - Python3 - COMPONENTS Interpreter - REQUIRED) +# find_package( Python3 COMPONENTS Interpreter REQUIRED) -add_custom_command( - OUTPUT ${PROJECT_BINARY_DIR}/benchmarks/fields.png - COMMAND Python3::Interpreter ${PROJECT_SOURCE_DIR}/scripts/plotBenchmarks.py - ${PROJECT_BINARY_DIR}/benchmarks/fields.xml - COMMENT "Plot benchmark results") +# add_custom_command( OUTPUT ${PROJECT_BINARY_DIR}/benchmarks/fields.png COMMAND +# Python3::Interpreter ${PROJECT_SOURCE_DIR}/scripts/plotBenchmarks.py +# ${PROJECT_BINARY_DIR}/benchmarks/fields.xml COMMENT "Plot benchmark results") -add_custom_target( - execute_benchmarks - DEPENDS ${PROJECT_BINARY_DIR}/benchmarks/fields.xml - COMMENT "execute benchmarks") +add_custom_target(execute_benchmarks # DEPENDS ${PROJECT_BINARY_DIR}/benchmarks/fields.xml + COMMENT "execute benchmarks") -add_custom_target( - execute_plot_benchmark - DEPENDS ${PROJECT_BINARY_DIR}/benchmarks/fields.png - COMMENT "plot benchmark results") +# add_custom_target( execute_plot_benchmark DEPENDS ${PROJECT_BINARY_DIR}/benchmarks/fields.png +# COMMENT "plot benchmark results") diff --git a/benchmarks/fields/bench_fields.cpp b/benchmarks/fields/bench_fields.cpp index e7f61be37..c92bb535f 100644 --- a/benchmarks/fields/bench_fields.cpp +++ b/benchmarks/fields/bench_fields.cpp @@ -40,8 +40,7 @@ int main(int argc, char* argv[]) TEST_CASE("Vector addition [benchmark]") { - auto size = - GENERATE(8, 64, 512, 4096, 32768, 262144, 1048576, 1048576 * 4, 1048576 * 16, 1048576 * 64); + auto size = GENERATE(8, 64, 512, 4096, 32768); CAPTURE(size); // Capture the value of size @@ -56,70 +55,71 @@ TEST_CASE("Vector addition [benchmark]") BENCHMARK("Field addition") { return (cpuC = cpuA + cpuB); }; } - -{ - NeoFOAM::CPUExecutor ompExec {}; - NeoFOAM::Field ompA(ompExec, size); - NeoFOAM::fill(ompA, 1.0); - NeoFOAM::Field ompB(ompExec, size); - NeoFOAM::fill(ompB, 2.0); - NeoFOAM::Field ompC(ompExec, size); - NeoFOAM::fill(ompC, 0.0); - - BENCHMARK("Field addition") { return (ompC = ompA + ompB); }; -} - -{ - NeoFOAM::GPUExecutor gpuExec {}; - NeoFOAM::Field gpuA(gpuExec, size); - NeoFOAM::fill(gpuA, 1.0); - NeoFOAM::Field gpuB(gpuExec, size); - NeoFOAM::fill(gpuB, 2.0); - NeoFOAM::Field gpuC(gpuExec, size); - NeoFOAM::fill(gpuC, 0.0); - - BENCHMARK("Field addition") - { - gpuC = gpuA + gpuB; - return Kokkos::fence(); - }; } -{ - NeoFOAM::GPUExecutor gpuExec {}; - NeoFOAM::Field gpuA(gpuExec, size); - NeoFOAM::fill(gpuA, 1.0); - NeoFOAM::Field gpuB(gpuExec, size); - NeoFOAM::fill(gpuB, 2.0); - NeoFOAM::Field gpuC(gpuExec, size); - NeoFOAM::fill(gpuC, 0.0); - - auto sGpuB = gpuB.span(); - auto sGpuC = gpuC.span(); - BENCHMARK("Field addition no allocation") - { - gpuA.apply(KOKKOS_LAMBDA(const NeoFOAM::size_t i) { return sGpuB[i] + sGpuC[i]; }); - return Kokkos::fence(); - // return GPUa; - }; -} - -{ - NeoFOAM::CPUExecutor ompExec {}; - NeoFOAM::Field ompA(ompExec, size); - NeoFOAM::fill(ompA, 1.0); - NeoFOAM::Field ompB(ompExec, size); - NeoFOAM::fill(ompB, 2.0); - NeoFOAM::Field ompC(ompExec, size); - NeoFOAM::fill(ompC, 0.0); - - auto sompB = ompB.span(); - auto sompC = ompC.span(); - BENCHMARK("Field addition no allocation") - { - ompA.apply(KOKKOS_LAMBDA(const NeoFOAM::size_t i) { return sompB[i] + sompC[i]; }); - }; -} -} +// { +// NeoFOAM::CPUExecutor ompExec {}; +// NeoFOAM::Field ompA(ompExec, size); +// NeoFOAM::fill(ompA, 1.0); +// NeoFOAM::Field ompB(ompExec, size); +// NeoFOAM::fill(ompB, 2.0); +// NeoFOAM::Field ompC(ompExec, size); +// NeoFOAM::fill(ompC, 0.0); + +// BENCHMARK("Field addition") { return (ompC = ompA + ompB); }; +// } + +// { +// NeoFOAM::GPUExecutor gpuExec {}; +// NeoFOAM::Field gpuA(gpuExec, size); +// NeoFOAM::fill(gpuA, 1.0); +// NeoFOAM::Field gpuB(gpuExec, size); +// NeoFOAM::fill(gpuB, 2.0); +// NeoFOAM::Field gpuC(gpuExec, size); +// NeoFOAM::fill(gpuC, 0.0); + +// BENCHMARK("Field addition") +// { +// gpuC = gpuA + gpuB; +// return Kokkos::fence(); +// }; +// } + +// { +// NeoFOAM::GPUExecutor gpuExec {}; +// NeoFOAM::Field gpuA(gpuExec, size); +// NeoFOAM::fill(gpuA, 1.0); +// NeoFOAM::Field gpuB(gpuExec, size); +// NeoFOAM::fill(gpuB, 2.0); +// NeoFOAM::Field gpuC(gpuExec, size); +// NeoFOAM::fill(gpuC, 0.0); + +// auto sGpuB = gpuB.span(); +// auto sGpuC = gpuC.span(); +// BENCHMARK("Field addition no allocation") +// { +// gpuA.apply(KOKKOS_LAMBDA(const NeoFOAM::size_t i) { return sGpuB[i] + sGpuC[i]; }); +// return Kokkos::fence(); +// // return GPUa; +// }; +// } + +// { +// NeoFOAM::CPUExecutor ompExec {}; +// NeoFOAM::Field ompA(ompExec, size); +// NeoFOAM::fill(ompA, 1.0); +// NeoFOAM::Field ompB(ompExec, size); +// NeoFOAM::fill(ompB, 2.0); +// NeoFOAM::Field ompC(ompExec, size); +// NeoFOAM::fill(ompC, 0.0); + +// auto sompB = ompB.span(); +// auto sompC = ompC.span(); +// BENCHMARK("Field addition no allocation") +// { +// ompA.apply(KOKKOS_LAMBDA(const NeoFOAM::size_t i) { return sompB[i] + sompC[i]; }); +// }; +// } +// } ; } From 5fcef0aea029c63f607fb270238effaaf7060b84 Mon Sep 17 00:00:00 2001 From: Gregor Olenik Date: Mon, 13 Jan 2025 18:18:09 +0100 Subject: [PATCH 02/15] output benchmark results --- benchmarks/CMakeLists.txt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index b1c4bbb56..1d250455d 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -5,7 +5,7 @@ add_subdirectory(fields) add_custom_command( OUTPUT ${PROJECT_BINARY_DIR}/benchmarks/fields.xml - COMMAND ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/bench_fields -r json + COMMAND ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/bench_fields -r xml > fields.xml COMMENT "Execute benchmarks") # find_package( Python3 COMPONENTS Interpreter REQUIRED) @@ -14,8 +14,10 @@ add_custom_command( # Python3::Interpreter ${PROJECT_SOURCE_DIR}/scripts/plotBenchmarks.py # ${PROJECT_BINARY_DIR}/benchmarks/fields.xml COMMENT "Plot benchmark results") -add_custom_target(execute_benchmarks # DEPENDS ${PROJECT_BINARY_DIR}/benchmarks/fields.xml - COMMENT "execute benchmarks") +add_custom_target( + execute_benchmarks + DEPENDS ${PROJECT_BINARY_DIR}/benchmarks/fields.xml + COMMENT "execute benchmarks") # add_custom_target( execute_plot_benchmark DEPENDS ${PROJECT_BINARY_DIR}/benchmarks/fields.png # COMMENT "plot benchmark results") From f98e03ea9b842893f084e52086f96d0e09ea1b0b Mon Sep 17 00:00:00 2001 From: Gregor Olenik Date: Mon, 13 Jan 2025 19:51:11 +0100 Subject: [PATCH 03/15] move workflow execution step --- .github/workflows/build_on_aws.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build_on_aws.yaml b/.github/workflows/build_on_aws.yaml index a4eb2dd87..ee954ee32 100644 --- a/.github/workflows/build_on_aws.yaml +++ b/.github/workflows/build_on_aws.yaml @@ -142,9 +142,6 @@ jobs: -DNEOFOAM_BUILD_BENCHMARKS=ON \ -DKokkos_ENABLE_CUDA=ON cmake --build --preset production - - name: Execute Benchmarks - shell: bash -i {0} - run: | cmake --build --preset production --target execute_benchmarks stop-runner: From 24eb505e78f1af35182681daabae3353b9611142 Mon Sep 17 00:00:00 2001 From: Gregor Olenik Date: Mon, 13 Jan 2025 20:41:01 +0100 Subject: [PATCH 04/15] push to BenchmarkData repo --- .github/workflows/build_on_aws.yaml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_on_aws.yaml b/.github/workflows/build_on_aws.yaml index ee954ee32..659d45d86 100644 --- a/.github/workflows/build_on_aws.yaml +++ b/.github/workflows/build_on_aws.yaml @@ -122,7 +122,7 @@ jobs: module load cmake ctest --preset ${{matrix.preset}} benchmark-on-aws: - name: Build on aws + name: Benchmark on aws needs: [start-runner, build-on-aws] # required to start the main job when the runner is ready runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner steps: @@ -134,6 +134,7 @@ jobs: module load libfabric-aws module spider libfabric-aws module load cmake + pip install yq cmake --version CC=clang \ CXX=clang++ \ @@ -143,7 +144,20 @@ jobs: -DKokkos_ENABLE_CUDA=ON cmake --build --preset production cmake --build --preset production --target execute_benchmarks - + mkdir ${{ github.ref_name }} + cat build/develop/benchmarks/fields.xml | yq -p xml -o json > ${{ github.ref_name }}/fields.json + cat ${{ github.ref_name }}/fields.json + lscpu > ${{ github.ref_name }}/lscpu.log + - name: Push Benchmark Data + uses: cpina/github-action-push-to-another-repository@main + env: + API_TOKEN_GITHUB: ${{ secrets.API_TOKEN_GITHUB }} + with: + source-directory: ${{ github.ref_name }} + destination-github-username: github-actions + destination-repository-name: 'NeoFOAM-BenchmarkData' + user-email: github-actions@github.com + target-branch: main stop-runner: name: Stop self-hosted EC2 runner needs: From 9426d55c9bb8db5b1ab5ec40e2546269cdd993c7 Mon Sep 17 00:00:00 2001 From: Gregor Olenik Date: Mon, 13 Jan 2025 21:14:54 +0100 Subject: [PATCH 05/15] push to BenchmarkData repo --- .github/workflows/build_on_aws.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_on_aws.yaml b/.github/workflows/build_on_aws.yaml index 659d45d86..c6a1a8a6b 100644 --- a/.github/workflows/build_on_aws.yaml +++ b/.github/workflows/build_on_aws.yaml @@ -144,17 +144,17 @@ jobs: -DKokkos_ENABLE_CUDA=ON cmake --build --preset production cmake --build --preset production --target execute_benchmarks - mkdir ${{ github.ref_name }} - cat build/develop/benchmarks/fields.xml | yq -p xml -o json > ${{ github.ref_name }}/fields.json - cat ${{ github.ref_name }}/fields.json - lscpu > ${{ github.ref_name }}/lscpu.log + mkdir -p ${{github.event.number}} + cat build/production/benchmarks/fields.xml | yq -p xml -o json > ${{github.event.number}}/fields.json + cat ${{github.event.number}}/fields.json + lscpu > ${{github.event.number}}/lscpu.log - name: Push Benchmark Data uses: cpina/github-action-push-to-another-repository@main env: API_TOKEN_GITHUB: ${{ secrets.API_TOKEN_GITHUB }} with: - source-directory: ${{ github.ref_name }} - destination-github-username: github-actions + source-directory: ${{github.event.number}} + destination-github-username: 'exasim-project' destination-repository-name: 'NeoFOAM-BenchmarkData' user-email: github-actions@github.com target-branch: main From 997c580046de509273c0529d0bb6f4597e682f01 Mon Sep 17 00:00:00 2001 From: Gregor Olenik Date: Tue, 14 Jan 2025 10:27:54 +0100 Subject: [PATCH 06/15] add target directory --- .github/workflows/build_on_aws.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_on_aws.yaml b/.github/workflows/build_on_aws.yaml index c6a1a8a6b..7e58eee4c 100644 --- a/.github/workflows/build_on_aws.yaml +++ b/.github/workflows/build_on_aws.yaml @@ -156,6 +156,7 @@ jobs: source-directory: ${{github.event.number}} destination-github-username: 'exasim-project' destination-repository-name: 'NeoFOAM-BenchmarkData' + target-directory: ${{github.event.number}} user-email: github-actions@github.com target-branch: main stop-runner: From b98b10e2f1748262f87d4ab38de920b0bbcfd46e Mon Sep 17 00:00:00 2001 From: Gregor Olenik Date: Tue, 14 Jan 2025 13:40:08 +0100 Subject: [PATCH 07/15] change and use presets for profiling/benchmarking --- .github/workflows/build_on_aws.yaml | 13 +-- CMakeLists.txt | 1 + CMakePresets.json | 4 +- benchmarks/CMakeLists.txt | 42 ++++++---- benchmarks/fields/CMakeLists.txt | 14 +--- benchmarks/fields/bench_fields.cpp | 125 ---------------------------- benchmarks/fields/field.cpp | 82 ++++++++++++++++++ 7 files changed, 115 insertions(+), 166 deletions(-) delete mode 100644 benchmarks/fields/bench_fields.cpp create mode 100644 benchmarks/fields/field.cpp diff --git a/.github/workflows/build_on_aws.yaml b/.github/workflows/build_on_aws.yaml index 7e58eee4c..b90b83cb1 100644 --- a/.github/workflows/build_on_aws.yaml +++ b/.github/workflows/build_on_aws.yaml @@ -134,19 +134,14 @@ jobs: module load libfabric-aws module spider libfabric-aws module load cmake - pip install yq cmake --version CC=clang \ CXX=clang++ \ - cmake --preset production \ - -DNEOFOAM_ENABLE_MPI_WITH_THREAD_SUPPORT=OFF \ - -DNEOFOAM_BUILD_BENCHMARKS=ON \ - -DKokkos_ENABLE_CUDA=ON - cmake --build --preset production - cmake --build --preset production --target execute_benchmarks + cmake --preset profiling + cmake --build --preset profiling + cmake --build --preset profiling --target execute_benchmarks mkdir -p ${{github.event.number}} - cat build/production/benchmarks/fields.xml | yq -p xml -o json > ${{github.event.number}}/fields.json - cat ${{github.event.number}}/fields.json + cp build/production/benchmarks/fields.xml ${{github.event.number}}/fields.xml lscpu > ${{github.event.number}}/lscpu.log - name: Push Benchmark Data uses: cpina/github-action-push-to-another-repository@main diff --git a/CMakeLists.txt b/CMakeLists.txt index 2ad2245e8..378fac1f4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -113,6 +113,7 @@ if(NEOFOAM_BUILD_TESTS) endif() if(NEOFOAM_BUILD_BENCHMARKS) + enable_testing() add_subdirectory(benchmarks) endif() diff --git a/CMakePresets.json b/CMakePresets.json index 25a42e3f1..69bf55aa6 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -37,7 +37,7 @@ "CMAKE_BUILD_TYPE": "Debug", "NEOFOAM_DEVEL_TOOLS": "ON", "NEOFOAM_BUILD_TESTS": "ON", - "NEOFOAM_BUILD_BENCHMARKS": "ON", + "NEOFOAM_BUILD_BENCHMARKS": "OFF", "NEOFOAM_ENABLE_WARNINGS": "ON", "Kokkos_ENABLE_DEBUG": "ON", "Kokkos_ENABLE_DEBUG_BOUNDS_CHECK": "ON", @@ -58,7 +58,7 @@ "description": "Configuration for profiling use", "cacheVariables": { "CMAKE_BUILD_TYPE": "RelWithDebInfo", - "NEOFOAM_BUILD_TESTS": "ON", + "NEOFOAM_BUILD_TESTS": "OFF", "NEOFOAM_BUILD_BENCHMARKS": "ON", "NEOFOAM_ENABLE_WARNINGS": "OFF", "CMAKE_CXX_FLAGS": "-fno-omit-frame-pointer $env{CXXFLAGS}" diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 1d250455d..ed62acb97 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -1,23 +1,31 @@ # SPDX-License-Identifier: Unlicense # SPDX-FileCopyrightText: 2023-25 NeoFOAM authors -add_subdirectory(fields) - -add_custom_command( - OUTPUT ${PROJECT_BINARY_DIR}/benchmarks/fields.xml - COMMAND ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/bench_fields -r xml > fields.xml - COMMENT "Execute benchmarks") +function(neofoam_benchmark TEST) -# find_package( Python3 COMPONENTS Interpreter REQUIRED) + add_executable(bench_${TEST} "${TEST}.cpp") + target_link_libraries(bench_${TEST} PRIVATE Catch2::Catch2 NeoFOAM) -# add_custom_command( OUTPUT ${PROJECT_BINARY_DIR}/benchmarks/fields.png COMMAND -# Python3::Interpreter ${PROJECT_SOURCE_DIR}/scripts/plotBenchmarks.py -# ${PROJECT_BINARY_DIR}/benchmarks/fields.xml COMMENT "Plot benchmark results") + if(WIN32) + set_target_properties( + bench_${TEST} + PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/$<0:> + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/$<0:> + ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/$<0:>) + else() + set_property(TARGET bench_${TEST} PROPERTY RUNTIME_OUTPUT_DIRECTORY + ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks) + endif() + if(NOT DEFINED "neofoam_COMMAND") + set(neofoam_COMMAND bench_${TEST}) + endif() + if(NOT DEFINED "neofoam_WORKING_DIRECTORY") + set(neofoam_WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks) + endif() + add_test( + NAME bench_${TEST} + COMMAND ${neofoam_COMMAND} + WORKING_DIRECTORY ${neofoam_WORKING_DIRECTORY}) +endfunction() -add_custom_target( - execute_benchmarks - DEPENDS ${PROJECT_BINARY_DIR}/benchmarks/fields.xml - COMMENT "execute benchmarks") - -# add_custom_target( execute_plot_benchmark DEPENDS ${PROJECT_BINARY_DIR}/benchmarks/fields.png -# COMMENT "plot benchmark results") +add_subdirectory(fields) diff --git a/benchmarks/fields/CMakeLists.txt b/benchmarks/fields/CMakeLists.txt index f83d15f36..aede4c87c 100644 --- a/benchmarks/fields/CMakeLists.txt +++ b/benchmarks/fields/CMakeLists.txt @@ -1,16 +1,4 @@ # SPDX-License-Identifier: Unlicense # SPDX-FileCopyrightText: 2023 NeoFOAM authors -add_executable(bench_fields "bench_fields.cpp") -target_link_libraries(bench_fields PRIVATE Catch2::Catch2 NeoFOAM Kokkos::kokkos) - -if(WIN32) - set_target_properties( - bench_fields - PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/$<0:> - LIBRARY_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/$<0:> - ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/$<0:>) -else() - set_property(TARGET bench_fields PROPERTY RUNTIME_OUTPUT_DIRECTORY - ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks) -endif() +neofoam_benchmark(field) diff --git a/benchmarks/fields/bench_fields.cpp b/benchmarks/fields/bench_fields.cpp deleted file mode 100644 index c92bb535f..000000000 --- a/benchmarks/fields/bench_fields.cpp +++ /dev/null @@ -1,125 +0,0 @@ -// SPDX-License-Identifier: MIT -// SPDX-FileCopyrightText: 2023 NeoFOAM authors - -#define CATCH_CONFIG_RUNNER // Define this before including catch.hpp to create - // a custom main -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "NeoFOAM/fields/field.hpp" -#include "NeoFOAM/fields/fieldTypeDefs.hpp" - - -int main(int argc, char* argv[]) -{ - // Initialize Catch2 - Kokkos::initialize(argc, argv); - Catch::Session session; - - // Specify command line options - int returnCode = session.applyCommandLine(argc, argv); - if (returnCode != 0) // Indicates a command line error - return returnCode; - - int result = session.run(); - - // Run benchmarks if there are any - Kokkos::finalize(); - - return result; -} - -TEST_CASE("Vector addition [benchmark]") -{ - - auto size = GENERATE(8, 64, 512, 4096, 32768); - - CAPTURE(size); // Capture the value of size - - // capture the value of size as section name - DYNAMIC_SECTION("" << size) {{NeoFOAM::SerialExecutor cpuExec {}; - NeoFOAM::Field cpuA(cpuExec, size); - NeoFOAM::fill(cpuA, 1.0); - NeoFOAM::Field cpuB(cpuExec, size); - NeoFOAM::fill(cpuB, 2.0); - NeoFOAM::Field cpuC(cpuExec, size); - NeoFOAM::fill(cpuC, 0.0); - - BENCHMARK("Field addition") { return (cpuC = cpuA + cpuB); }; -} -} - -// { -// NeoFOAM::CPUExecutor ompExec {}; -// NeoFOAM::Field ompA(ompExec, size); -// NeoFOAM::fill(ompA, 1.0); -// NeoFOAM::Field ompB(ompExec, size); -// NeoFOAM::fill(ompB, 2.0); -// NeoFOAM::Field ompC(ompExec, size); -// NeoFOAM::fill(ompC, 0.0); - -// BENCHMARK("Field addition") { return (ompC = ompA + ompB); }; -// } - -// { -// NeoFOAM::GPUExecutor gpuExec {}; -// NeoFOAM::Field gpuA(gpuExec, size); -// NeoFOAM::fill(gpuA, 1.0); -// NeoFOAM::Field gpuB(gpuExec, size); -// NeoFOAM::fill(gpuB, 2.0); -// NeoFOAM::Field gpuC(gpuExec, size); -// NeoFOAM::fill(gpuC, 0.0); - -// BENCHMARK("Field addition") -// { -// gpuC = gpuA + gpuB; -// return Kokkos::fence(); -// }; -// } - -// { -// NeoFOAM::GPUExecutor gpuExec {}; -// NeoFOAM::Field gpuA(gpuExec, size); -// NeoFOAM::fill(gpuA, 1.0); -// NeoFOAM::Field gpuB(gpuExec, size); -// NeoFOAM::fill(gpuB, 2.0); -// NeoFOAM::Field gpuC(gpuExec, size); -// NeoFOAM::fill(gpuC, 0.0); - -// auto sGpuB = gpuB.span(); -// auto sGpuC = gpuC.span(); -// BENCHMARK("Field addition no allocation") -// { -// gpuA.apply(KOKKOS_LAMBDA(const NeoFOAM::size_t i) { return sGpuB[i] + sGpuC[i]; }); -// return Kokkos::fence(); -// // return GPUa; -// }; -// } - -// { -// NeoFOAM::CPUExecutor ompExec {}; -// NeoFOAM::Field ompA(ompExec, size); -// NeoFOAM::fill(ompA, 1.0); -// NeoFOAM::Field ompB(ompExec, size); -// NeoFOAM::fill(ompB, 2.0); -// NeoFOAM::Field ompC(ompExec, size); -// NeoFOAM::fill(ompC, 0.0); - -// auto sompB = ompB.span(); -// auto sompC = ompC.span(); -// BENCHMARK("Field addition no allocation") -// { -// ompA.apply(KOKKOS_LAMBDA(const NeoFOAM::size_t i) { return sompB[i] + sompC[i]; }); -// }; -// } -// } -; -} diff --git a/benchmarks/fields/field.cpp b/benchmarks/fields/field.cpp new file mode 100644 index 000000000..1babf692c --- /dev/null +++ b/benchmarks/fields/field.cpp @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2023 NeoFOAM authors + +#define CATCH_CONFIG_RUNNER // Define this before including catch.hpp to create + // a custom main +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "NeoFOAM/fields/field.hpp" +#include "NeoFOAM/fields/fieldTypeDefs.hpp" + + +int main(int argc, char* argv[]) +{ + // Initialize Catch2 + Kokkos::initialize(argc, argv); + Catch::Session session; + + // Specify command line options + int returnCode = session.applyCommandLine(argc, argv); + if (returnCode != 0) // Indicates a command line error + return returnCode; + + int result = session.run(); + + // Run benchmarks if there are any + Kokkos::finalize(); + + return result; +} + +TEST_CASE("[bench] fields") +{ + + auto size = GENERATE(1 << 16, 1 << 17, 1 << 18, 1 << 19, 1 << 20); + + NeoFOAM::Executor exec = GENERATE( + NeoFOAM::Executor(NeoFOAM::SerialExecutor {}), + NeoFOAM::Executor(NeoFOAM::CPUExecutor {}), + NeoFOAM::Executor(NeoFOAM::GPUExecutor {}) + ); + + CAPTURE(size); // Capture the value of size + + std::string execName = std::visit([](auto e) { return e.name(); }, exec); + + // capture the value of size as section name + DYNAMIC_SECTION("" << size) + { + { + NeoFOAM::Field cpuA(exec, size); + NeoFOAM::fill(cpuA, 1.0); + NeoFOAM::Field cpuB(exec, size); + NeoFOAM::fill(cpuB, 2.0); + NeoFOAM::Field cpuC(exec, size); + NeoFOAM::fill(cpuC, 0.0); + + BENCHMARK("Field::addition on " + execName) { return (cpuC = cpuA + cpuB); }; + } + { + NeoFOAM::Field cpuA(exec, size); + NeoFOAM::fill(cpuA, 1.0); + NeoFOAM::Field cpuB(exec, size); + NeoFOAM::fill(cpuB, 2.0); + NeoFOAM::Field cpuC(exec, size); + NeoFOAM::fill(cpuC, 0.0); + + BENCHMARK("Field::multiplication on " + execName) + { + return (cpuC = cpuA * cpuB); + }; + } + } +} From 578bf79f2ddac672051c74b1edc4ae3a137de604 Mon Sep 17 00:00:00 2001 From: Gregor Olenik Date: Tue, 14 Jan 2025 14:54:27 +0100 Subject: [PATCH 08/15] add divOperator benchmark, fix copy results --- .github/workflows/build_on_aws.yaml | 4 +- benchmarks/CMakeLists.txt | 4 +- .../cellCentred/operator/CMakeLists.txt | 4 + .../cellCentred/operator/divOperator.cpp | 74 +++++++++++++++++++ 4 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 benchmarks/finiteVolume/cellCentred/operator/CMakeLists.txt create mode 100644 benchmarks/finiteVolume/cellCentred/operator/divOperator.cpp diff --git a/.github/workflows/build_on_aws.yaml b/.github/workflows/build_on_aws.yaml index b90b83cb1..1030995c0 100644 --- a/.github/workflows/build_on_aws.yaml +++ b/.github/workflows/build_on_aws.yaml @@ -139,9 +139,9 @@ jobs: CXX=clang++ \ cmake --preset profiling cmake --build --preset profiling - cmake --build --preset profiling --target execute_benchmarks + ctest --preset profiling mkdir -p ${{github.event.number}} - cp build/production/benchmarks/fields.xml ${{github.event.number}}/fields.xml + cp build/profiling/bin/benchmarks/*.xml ${{github.event.number}}/ lscpu > ${{github.event.number}}/lscpu.log - name: Push Benchmark Data uses: cpina/github-action-push-to-another-repository@main diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index ed62acb97..cf28c66d4 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -19,13 +19,15 @@ function(neofoam_benchmark TEST) if(NOT DEFINED "neofoam_COMMAND") set(neofoam_COMMAND bench_${TEST}) endif() + if(NOT DEFINED "neofoam_WORKING_DIRECTORY") set(neofoam_WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks) endif() add_test( NAME bench_${TEST} - COMMAND ${neofoam_COMMAND} + COMMAND sh -c "./${neofoam_COMMAND} -r xml > ${TEST}.xml" WORKING_DIRECTORY ${neofoam_WORKING_DIRECTORY}) endfunction() add_subdirectory(fields) +add_subdirectory(finiteVolume/cellCentred/operator) diff --git a/benchmarks/finiteVolume/cellCentred/operator/CMakeLists.txt b/benchmarks/finiteVolume/cellCentred/operator/CMakeLists.txt new file mode 100644 index 000000000..ae6508a61 --- /dev/null +++ b/benchmarks/finiteVolume/cellCentred/operator/CMakeLists.txt @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: Unlicense +# SPDX-FileCopyrightText: 2025 NeoFOAM authors + +neofoam_benchmark(divOperator) diff --git a/benchmarks/finiteVolume/cellCentred/operator/divOperator.cpp b/benchmarks/finiteVolume/cellCentred/operator/divOperator.cpp new file mode 100644 index 000000000..803ea8937 --- /dev/null +++ b/benchmarks/finiteVolume/cellCentred/operator/divOperator.cpp @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2023 NeoFOAM authors + +#define CATCH_CONFIG_RUNNER // Define this before including catch.hpp to create + // a custom main +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "NeoFOAM/NeoFOAM.hpp" + +using Operator = NeoFOAM::dsl::Operator; + +int main(int argc, char* argv[]) +{ + // Initialize Catch2 + Kokkos::initialize(argc, argv); + Catch::Session session; + + // Specify command line options + int returnCode = session.applyCommandLine(argc, argv); + if (returnCode != 0) // Indicates a command line error + return returnCode; + + int result = session.run(); + + // Run benchmarks if there are any + Kokkos::finalize(); + + return result; +} + +TEST_CASE("divOperator", "[bench]") +{ + + auto size = GENERATE(1 << 16, 1 << 17, 1 << 18, 1 << 19, 1 << 20); + CAPTURE(size); // Capture the value of size + + NeoFOAM::Executor exec = GENERATE( + NeoFOAM::Executor(NeoFOAM::SerialExecutor {}), + NeoFOAM::Executor(NeoFOAM::CPUExecutor {}), + NeoFOAM::Executor(NeoFOAM::GPUExecutor {}) + ); + + std::string execName = std::visit([](auto e) { return e.name(); }, exec); + NeoFOAM::UnstructuredMesh mesh = NeoFOAM::create1DUniformMesh(exec, size); + auto surfaceBCs = fvcc::createCalculatedBCs>(mesh); + fvcc::SurfaceField faceFlux(exec, "sf", mesh, surfaceBCs); + NeoFOAM::fill(faceFlux.internalField(), 1.0); + + auto volumeBCs = fvcc::createCalculatedBCs>(mesh); + fvcc::VolumeField phi(exec, "vf", mesh, volumeBCs); + fvcc::VolumeField divPhi(exec, "divPhi", mesh, volumeBCs); + NeoFOAM::fill(phi.internalField(), 1.0); + + // capture the value of size as section name + DYNAMIC_SECTION("" << size) + { + { + NeoFOAM::Input input = + NeoFOAM::TokenList({std::string("Gauss"), std::string("linear")}); + auto op = fvcc::DivOperator(Operator::Type::Explicit, faceFlux, phi, input); + + BENCHMARK("DivOperator::addition on " + execName) { return (op.div(divPhi)); }; + } + } +} From f9dba6b21e44d2c5f5632cc9e677bfe32a5a5a5a Mon Sep 17 00:00:00 2001 From: Gregor Olenik Date: Tue, 14 Jan 2025 15:47:14 +0100 Subject: [PATCH 09/15] add aws instance name --- .github/workflows/build_on_aws.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_on_aws.yaml b/.github/workflows/build_on_aws.yaml index 1030995c0..3558222db 100644 --- a/.github/workflows/build_on_aws.yaml +++ b/.github/workflows/build_on_aws.yaml @@ -140,9 +140,9 @@ jobs: cmake --preset profiling cmake --build --preset profiling ctest --preset profiling - mkdir -p ${{github.event.number}} - cp build/profiling/bin/benchmarks/*.xml ${{github.event.number}}/ - lscpu > ${{github.event.number}}/lscpu.log + mkdir -p ${{github.event.number}}/gdnxlarge + cp build/profiling/bin/benchmarks/*.xml ${{github.event.number}}/gdnxlarge + lscpu > ${{github.event.number}}/gdnxlarge/lscpu.log - name: Push Benchmark Data uses: cpina/github-action-push-to-another-repository@main env: @@ -151,7 +151,7 @@ jobs: source-directory: ${{github.event.number}} destination-github-username: 'exasim-project' destination-repository-name: 'NeoFOAM-BenchmarkData' - target-directory: ${{github.event.number}} + target-directory: ${{github.event.number}}/gdnxlarge user-email: github-actions@github.com target-branch: main stop-runner: From 5e6458b191a1f87fb24e4c40532755c7d416dc92 Mon Sep 17 00:00:00 2001 From: Gregor Olenik Date: Tue, 14 Jan 2025 22:16:53 +0100 Subject: [PATCH 10/15] refactor catch parser + field.cpp --- benchmarks/fields/field.cpp | 59 ++++++++-------- scripts/catch2json.py | 56 +++++++++++++++ scripts/plotBenchmarks.py | 132 ------------------------------------ 3 files changed, 87 insertions(+), 160 deletions(-) create mode 100644 scripts/catch2json.py delete mode 100644 scripts/plotBenchmarks.py diff --git a/benchmarks/fields/field.cpp b/benchmarks/fields/field.cpp index 1babf692c..1cb09d0d1 100644 --- a/benchmarks/fields/field.cpp +++ b/benchmarks/fields/field.cpp @@ -37,46 +37,49 @@ int main(int argc, char* argv[]) return result; } -TEST_CASE("[bench] fields") +TEST_CASE("Field::addition", "[bench]") { - auto size = GENERATE(1 << 16, 1 << 17, 1 << 18, 1 << 19, 1 << 20); - NeoFOAM::Executor exec = GENERATE( NeoFOAM::Executor(NeoFOAM::SerialExecutor {}), NeoFOAM::Executor(NeoFOAM::CPUExecutor {}), NeoFOAM::Executor(NeoFOAM::GPUExecutor {}) ); + std::string execName = std::visit([](auto e) { return e.name(); }, exec); - CAPTURE(size); // Capture the value of size + DYNAMIC_SECTION("" << size) + { + NeoFOAM::Field cpuA(exec, size); + NeoFOAM::fill(cpuA, 1.0); + NeoFOAM::Field cpuB(exec, size); + NeoFOAM::fill(cpuB, 2.0); + NeoFOAM::Field cpuC(exec, size); + NeoFOAM::fill(cpuC, 0.0); + + BENCHMARK(std::string(execName)) { return (cpuC = cpuA + cpuB); }; + } +} +TEST_CASE("Field::multiplication", "[bench]") +{ + auto size = GENERATE(1 << 16, 1 << 17, 1 << 18, 1 << 19, 1 << 20); + + NeoFOAM::Executor exec = GENERATE( + NeoFOAM::Executor(NeoFOAM::SerialExecutor {}), + NeoFOAM::Executor(NeoFOAM::CPUExecutor {}), + NeoFOAM::Executor(NeoFOAM::GPUExecutor {}) + ); std::string execName = std::visit([](auto e) { return e.name(); }, exec); - // capture the value of size as section name DYNAMIC_SECTION("" << size) { - { - NeoFOAM::Field cpuA(exec, size); - NeoFOAM::fill(cpuA, 1.0); - NeoFOAM::Field cpuB(exec, size); - NeoFOAM::fill(cpuB, 2.0); - NeoFOAM::Field cpuC(exec, size); - NeoFOAM::fill(cpuC, 0.0); - - BENCHMARK("Field::addition on " + execName) { return (cpuC = cpuA + cpuB); }; - } - { - NeoFOAM::Field cpuA(exec, size); - NeoFOAM::fill(cpuA, 1.0); - NeoFOAM::Field cpuB(exec, size); - NeoFOAM::fill(cpuB, 2.0); - NeoFOAM::Field cpuC(exec, size); - NeoFOAM::fill(cpuC, 0.0); - - BENCHMARK("Field::multiplication on " + execName) - { - return (cpuC = cpuA * cpuB); - }; - } + NeoFOAM::Field cpuA(exec, size); + NeoFOAM::fill(cpuA, 1.0); + NeoFOAM::Field cpuB(exec, size); + NeoFOAM::fill(cpuB, 2.0); + NeoFOAM::Field cpuC(exec, size); + NeoFOAM::fill(cpuC, 0.0); + + BENCHMARK(std::string(execName)) { return (cpuC = cpuA * cpuB); }; } } diff --git a/scripts/catch2json.py b/scripts/catch2json.py new file mode 100644 index 000000000..9980b0937 --- /dev/null +++ b/scripts/catch2json.py @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: 2023-2025 NeoFOAM authors + +import xmltodict +import json +import sys +import os + + +def parse_xml_dict(d): + """takes the catch2 xml dict, performs clean-up and returns a + list of records""" + data = d["Catch2TestRun"]["TestCase"] + if isinstance(data, dict): + data = [data] + records = [] + for cases in data: + test_case = cases["@name"] + for d in cases["Section"]: + size = d["@name"] + res = {} + for k, v in d["BenchmarkResults"].items(): + # print(k, v) + if k == "@name": + res["executor"] = v + if k == "mean": + res["mean"] = v["@value"] + continue + if k == "standardDeviation": + res["standardDeviation"] = v["@value"] + continue + if k.startswith("@"): + continue + res["size"] = size + res["test_case"] = test_case + records.append(res) + return records + + +def main(): + _, _, files = next(os.walk(".")) + for xml_file in files: + if not xml_file.endswith("xml"): + continue + try: + with open(xml_file, "r") as fh: + d = xmltodict.parse(fh.read()) + res = parse_xml_dict(d) + with open(xml_file.replace("xml", "json"), "w") as outfile: + json.dump(res, outfile) + except Exception as e: + print(e) + + +if __name__ == "__main__": + main() diff --git a/scripts/plotBenchmarks.py b/scripts/plotBenchmarks.py deleted file mode 100644 index 86ea10fcf..000000000 --- a/scripts/plotBenchmarks.py +++ /dev/null @@ -1,132 +0,0 @@ -# SPDX-License-Identifier: MIT -# SPDX-FileCopyrightText: 2023 NeoFOAM authors - -import xml.etree.ElementTree as ET -import pandas as pd -import matplotlib.pyplot as plt -import seaborn as sns -import sys - -xml_file = sys.argv[1] - -# Parse the XML file -tree = ET.parse(xml_file) - -# Get the root element -root = tree.getroot() - - -# %% -def extract_benchmarks(xml_file): - tree = ET.parse(xml_file) - root = tree.getroot() - testcase_name = "" - section_names = [] - - for testcase in root.iter("TestCase"): - testcase_name = testcase.attrib["name"] - print(f"Test case: {testcase_name}") - - for section in testcase.iter("Section"): - section_name = section.attrib["name"] - print(f" Section: {section_name}") - - for benchmark in section.iter("BenchmarkResults"): - for result in benchmark.iter("Result"): - name = result.attrib["name"] - value = result.attrib["value"] - print(f" Benchmark {name}: {value}") - - -extract_benchmarks(xml_file) - - -# %% -def getBenchmarkData(element): - for bRes in element: - if bRes.tag == "mean": - meanValue = bRes.attrib["value"] - lowerBound = bRes.attrib["lowerBound"] - upperBound = bRes.attrib["upperBound"] - # if bRes.tag == 'mean': - # meanValue = bRes.attrib['value'] - return meanValue, lowerBound, upperBound - - -def process_element(f, element, testcase_name, section_names, depth=0): - indent_str = " " * depth - if element.tag == "TestCase": - testcase_name = element.attrib["name"] - elif element.tag == "Section": - section_names[depth] = element.attrib["name"] - elif element.tag == "BenchmarkResults": - bName = element.attrib["name"] - bValues = getBenchmarkData(element) - f.write( - f'{testcase_name},{",".join(section_names)},{bName},{",".join(bValues)}\n' - ) - - for child in element: - if element.tag == "Section": - process_element(f, child, testcase_name, section_names, depth + 1) - else: - process_element(f, child, testcase_name, section_names, depth + 0) - - -def nSections(element, depth=0): - max_depth = max(depth, 0) - for child in element: - if element.tag == "Section": - max_depth = max(max_depth, nSections(child, depth + 1)) - else: - max_depth = max(max_depth, nSections(child, depth + 0)) - - return max_depth - - -def extract_benchmarks(xml_file, csv_file): - tree = ET.parse(xml_file) - root = tree.getroot() - nSec = nSections(root) - section_names = [""] * nSec - testcase_name = "" - with open(csv_file, "w") as f: - process_element(f, root, testcase_name, section_names) - - -def main(xml_file): - extract_benchmarks(xml_file, "bench_blas.csv") - - df = pd.read_csv( - "bench_blas.csv", - names=[ - "Testcase", - "Vector Elements", - "Benchmark", - "runTime [ms]", - "lowerBound", - "upperBound", - ], - ) - print(df) - df["Vector Elements"] = pd.to_numeric(df["Vector Elements"], errors="coerce") - df["runTime [ms]"] /= 1e6 - - sns.lineplot( - data=df, - x="Vector Elements", - y="runTime [ms]", - hue="Benchmark", - style="Benchmark", - markers=True, - ) - plt.xscale("log") - plt.yscale("log") - plt.savefig("fields.png") - plt.tight_layout() - plt.show() - # %% - - -if __name__ == "__main__": - main(sys.argv[1]) From 5cf9d84491318b9ee4456bbc30b47566f289a0d1 Mon Sep 17 00:00:00 2001 From: Gregor Olenik Date: Wed, 15 Jan 2025 09:46:19 +0100 Subject: [PATCH 11/15] convert to json --- .github/workflows/build_on_aws.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_on_aws.yaml b/.github/workflows/build_on_aws.yaml index 3558222db..af6bbc766 100644 --- a/.github/workflows/build_on_aws.yaml +++ b/.github/workflows/build_on_aws.yaml @@ -141,7 +141,10 @@ jobs: cmake --build --preset profiling ctest --preset profiling mkdir -p ${{github.event.number}}/gdnxlarge - cp build/profiling/bin/benchmarks/*.xml ${{github.event.number}}/gdnxlarge + cd build/profiling/bin/benchmarks + python3 -m pip install xmltodict + python3 ../../../../scripts/catch2json.py + cp build/profiling/bin/benchmarks/*.json ${{github.event.number}}/gdnxlarge lscpu > ${{github.event.number}}/gdnxlarge/lscpu.log - name: Push Benchmark Data uses: cpina/github-action-push-to-another-repository@main From b3c698082d35e9144ae20e7a6a62d2183f9f2854 Mon Sep 17 00:00:00 2001 From: Gregor Olenik Date: Wed, 15 Jan 2025 09:47:22 +0100 Subject: [PATCH 12/15] clean-up --- benchmarks/fields/field.cpp | 4 +--- .../cellCentred/operator/divOperator.cpp | 12 ++++-------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/benchmarks/fields/field.cpp b/benchmarks/fields/field.cpp index 1cb09d0d1..400457a22 100644 --- a/benchmarks/fields/field.cpp +++ b/benchmarks/fields/field.cpp @@ -14,9 +14,7 @@ #include #include -#include "NeoFOAM/fields/field.hpp" -#include "NeoFOAM/fields/fieldTypeDefs.hpp" - +#include "NeoFOAM/NeoFOAM.hpp" int main(int argc, char* argv[]) { diff --git a/benchmarks/finiteVolume/cellCentred/operator/divOperator.cpp b/benchmarks/finiteVolume/cellCentred/operator/divOperator.cpp index 803ea8937..8c0d64f43 100644 --- a/benchmarks/finiteVolume/cellCentred/operator/divOperator.cpp +++ b/benchmarks/finiteVolume/cellCentred/operator/divOperator.cpp @@ -37,11 +37,10 @@ int main(int argc, char* argv[]) return result; } -TEST_CASE("divOperator", "[bench]") +TEST_CASE("DivOperator::div", "[bench]") { auto size = GENERATE(1 << 16, 1 << 17, 1 << 18, 1 << 19, 1 << 20); - CAPTURE(size); // Capture the value of size NeoFOAM::Executor exec = GENERATE( NeoFOAM::Executor(NeoFOAM::SerialExecutor {}), @@ -63,12 +62,9 @@ TEST_CASE("divOperator", "[bench]") // capture the value of size as section name DYNAMIC_SECTION("" << size) { - { - NeoFOAM::Input input = - NeoFOAM::TokenList({std::string("Gauss"), std::string("linear")}); - auto op = fvcc::DivOperator(Operator::Type::Explicit, faceFlux, phi, input); + NeoFOAM::Input input = NeoFOAM::TokenList({std::string("Gauss"), std::string("linear")}); + auto op = fvcc::DivOperator(Operator::Type::Explicit, faceFlux, phi, input); - BENCHMARK("DivOperator::addition on " + execName) { return (op.div(divPhi)); }; - } + BENCHMARK(std::string(execName)) { return (op.div(divPhi)); }; } } From 9bf6b13e5072ae2eb1fc72798a6f96863ded40ba Mon Sep 17 00:00:00 2001 From: Gregor Olenik Date: Wed, 15 Jan 2025 09:52:42 +0100 Subject: [PATCH 13/15] move catch_main.hpp --- .github/workflows/build_on_aws.yaml | 7 +++-- benchmarks/catch_main.hpp | 31 +++++++++++++++++++ benchmarks/fields/field.cpp | 30 +----------------- .../cellCentred/operator/divOperator.cpp | 31 +------------------ 4 files changed, 37 insertions(+), 62 deletions(-) create mode 100644 benchmarks/catch_main.hpp diff --git a/.github/workflows/build_on_aws.yaml b/.github/workflows/build_on_aws.yaml index af6bbc766..b7ae917da 100644 --- a/.github/workflows/build_on_aws.yaml +++ b/.github/workflows/build_on_aws.yaml @@ -140,12 +140,13 @@ jobs: cmake --preset profiling cmake --build --preset profiling ctest --preset profiling - mkdir -p ${{github.event.number}}/gdnxlarge + mkdir -p ${{github.event.number}}/ cd build/profiling/bin/benchmarks python3 -m pip install xmltodict python3 ../../../../scripts/catch2json.py - cp build/profiling/bin/benchmarks/*.json ${{github.event.number}}/gdnxlarge - lscpu > ${{github.event.number}}/gdnxlarge/lscpu.log + cd ../../../.. + cp build/profiling/bin/benchmarks/*.json ${{github.event.number}}/ + lscpu > ${{github.event.number}}/lscpu.log - name: Push Benchmark Data uses: cpina/github-action-push-to-another-repository@main env: diff --git a/benchmarks/catch_main.hpp b/benchmarks/catch_main.hpp new file mode 100644 index 000000000..1d913d290 --- /dev/null +++ b/benchmarks/catch_main.hpp @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2025 NeoFOAM authors +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +int main(int argc, char* argv[]) +{ + // Initialize Catch2 + Kokkos::initialize(argc, argv); + Catch::Session session; + + // Specify command line options + int returnCode = session.applyCommandLine(argc, argv); + if (returnCode != 0) // Indicates a command line error + return returnCode; + + int result = session.run(); + + // Run benchmarks if there are any + Kokkos::finalize(); + + return result; +} diff --git a/benchmarks/fields/field.cpp b/benchmarks/fields/field.cpp index 400457a22..9ec0ca7da 100644 --- a/benchmarks/fields/field.cpp +++ b/benchmarks/fields/field.cpp @@ -3,38 +3,10 @@ #define CATCH_CONFIG_RUNNER // Define this before including catch.hpp to create // a custom main -#include -#include - -#include -#include -#include -#include -#include -#include -#include +#include "../catch_main.hpp" #include "NeoFOAM/NeoFOAM.hpp" -int main(int argc, char* argv[]) -{ - // Initialize Catch2 - Kokkos::initialize(argc, argv); - Catch::Session session; - - // Specify command line options - int returnCode = session.applyCommandLine(argc, argv); - if (returnCode != 0) // Indicates a command line error - return returnCode; - - int result = session.run(); - - // Run benchmarks if there are any - Kokkos::finalize(); - - return result; -} - TEST_CASE("Field::addition", "[bench]") { auto size = GENERATE(1 << 16, 1 << 17, 1 << 18, 1 << 19, 1 << 20); diff --git a/benchmarks/finiteVolume/cellCentred/operator/divOperator.cpp b/benchmarks/finiteVolume/cellCentred/operator/divOperator.cpp index 8c0d64f43..3d0d2e522 100644 --- a/benchmarks/finiteVolume/cellCentred/operator/divOperator.cpp +++ b/benchmarks/finiteVolume/cellCentred/operator/divOperator.cpp @@ -3,43 +3,14 @@ #define CATCH_CONFIG_RUNNER // Define this before including catch.hpp to create // a custom main -#include -#include - -#include -#include -#include -#include -#include -#include -#include #include "NeoFOAM/NeoFOAM.hpp" +#include "../../../catch_main.hpp" using Operator = NeoFOAM::dsl::Operator; -int main(int argc, char* argv[]) -{ - // Initialize Catch2 - Kokkos::initialize(argc, argv); - Catch::Session session; - - // Specify command line options - int returnCode = session.applyCommandLine(argc, argv); - if (returnCode != 0) // Indicates a command line error - return returnCode; - - int result = session.run(); - - // Run benchmarks if there are any - Kokkos::finalize(); - - return result; -} - TEST_CASE("DivOperator::div", "[bench]") { - auto size = GENERATE(1 << 16, 1 << 17, 1 << 18, 1 << 19, 1 << 20); NeoFOAM::Executor exec = GENERATE( From 5a6abb5feba69fb45115fdc32460a3769bef94e4 Mon Sep 17 00:00:00 2001 From: Gregor Olenik Date: Fri, 17 Jan 2025 13:53:48 +0100 Subject: [PATCH 14/15] add Kokkos::ScopeGuard --- benchmarks/catch_main.hpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/benchmarks/catch_main.hpp b/benchmarks/catch_main.hpp index 1d913d290..e6739016e 100644 --- a/benchmarks/catch_main.hpp +++ b/benchmarks/catch_main.hpp @@ -14,7 +14,7 @@ int main(int argc, char* argv[]) { // Initialize Catch2 - Kokkos::initialize(argc, argv); + Kokkos::ScopeGuard guard(argc, argv); Catch::Session session; // Specify command line options @@ -24,8 +24,5 @@ int main(int argc, char* argv[]) int result = session.run(); - // Run benchmarks if there are any - Kokkos::finalize(); - return result; } From 422794852e01ae766caf787ea82a30bf244e59de Mon Sep 17 00:00:00 2001 From: Gregor Olenik Date: Mon, 20 Jan 2025 10:09:44 +0100 Subject: [PATCH 15/15] clean-up --- benchmarks/CMakeLists.txt | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index cf28c66d4..886cf9a80 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -1,31 +1,28 @@ # SPDX-License-Identifier: Unlicense # SPDX-FileCopyrightText: 2023-25 NeoFOAM authors -function(neofoam_benchmark TEST) +function(neofoam_benchmark BENCH) - add_executable(bench_${TEST} "${TEST}.cpp") - target_link_libraries(bench_${TEST} PRIVATE Catch2::Catch2 NeoFOAM) + add_executable(bench_${BENCH} "${BENCH}.cpp") + target_link_libraries(bench_${BENCH} PRIVATE Catch2::Catch2 NeoFOAM) if(WIN32) set_target_properties( - bench_${TEST} - PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/$<0:> - LIBRARY_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/$<0:> - ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/$<0:>) + bench_${BENCH} + PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/ + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/ + ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/) else() - set_property(TARGET bench_${TEST} PROPERTY RUNTIME_OUTPUT_DIRECTORY - ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks) - endif() - if(NOT DEFINED "neofoam_COMMAND") - set(neofoam_COMMAND bench_${TEST}) + set_property(TARGET bench_${BENCH} PROPERTY RUNTIME_OUTPUT_DIRECTORY + ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks) endif() if(NOT DEFINED "neofoam_WORKING_DIRECTORY") set(neofoam_WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks) endif() add_test( - NAME bench_${TEST} - COMMAND sh -c "./${neofoam_COMMAND} -r xml > ${TEST}.xml" + NAME bench_${BENCH} + COMMAND sh -c "./bench_${BENCH} -r xml > ${BENCH}.xml" WORKING_DIRECTORY ${neofoam_WORKING_DIRECTORY}) endfunction()