Merge pull request Execute benchmarks on aws #241 see #241

Execute benchmarks on aws

.github/workflows/build_on_aws.yaml

@@ -121,11 +121,48 @@ jobs:
           module load libfabric-aws
           module load cmake
           ctest --preset ${{matrix.preset}}
+  benchmark-on-aws:
+    name: Benchmark on aws
+    needs: [start-runner, build-on-aws] # required to start the main job when the runner is ready
+    runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
+    steps:
+      - name: Build NeoFOAM
+        shell: bash -i {0}
+        run: |
+          export HOME=/share/ec2-user
+          module load clang/16
+          module load libfabric-aws
+          module spider libfabric-aws
+          module load cmake
+          cmake --version
+          CC=clang \
+          CXX=clang++ \
+          cmake --preset profiling
+          cmake --build --preset profiling
+          ctest --preset profiling
+          mkdir -p ${{github.event.number}}/
+          cd build/profiling/bin/benchmarks
+          python3 -m pip install xmltodict
+          python3 ../../../../scripts/catch2json.py
+          cd ../../../..
+          cp build/profiling/bin/benchmarks/*.json ${{github.event.number}}/
+          lscpu > ${{github.event.number}}/lscpu.log
+      - name: Push Benchmark Data
+        uses: cpina/github-action-push-to-another-repository@main
+        env:
+          API_TOKEN_GITHUB: ${{ secrets.API_TOKEN_GITHUB }}
+        with:
+          source-directory: ${{github.event.number}}
+          destination-github-username: 'exasim-project'
+          destination-repository-name: 'NeoFOAM-BenchmarkData'
+          target-directory: ${{github.event.number}}/gdnxlarge
+          user-email: github-actions@github.com
+          target-branch: main
   stop-runner:
     name: Stop self-hosted EC2 runner
     needs:
       - start-runner # required to get output from the start-runner job
-      - build-on-aws # required to wait when the main job is done
+      - benchmark-on-aws # required to wait when the main job is done
     runs-on: ubuntu-latest
     permissions:
       id-token: write

CMakeLists.txt

@@ -113,6 +113,7 @@ if(NEOFOAM_BUILD_TESTS)
 endif()
 
 if(NEOFOAM_BUILD_BENCHMARKS)
+  enable_testing()
   add_subdirectory(benchmarks)
 endif()
 

CMakePresets.json

@@ -37,7 +37,7 @@
         "CMAKE_BUILD_TYPE": "Debug",
         "NEOFOAM_DEVEL_TOOLS": "ON",
         "NEOFOAM_BUILD_TESTS": "ON",
-        "NEOFOAM_BUILD_BENCHMARKS": "ON",
+        "NEOFOAM_BUILD_BENCHMARKS": "OFF",
         "NEOFOAM_ENABLE_WARNINGS": "ON",
         "Kokkos_ENABLE_DEBUG": "ON",
         "Kokkos_ENABLE_DEBUG_BOUNDS_CHECK": "ON",
@@ -58,7 +58,7 @@
       "description": "Configuration for profiling use",
       "cacheVariables": {
         "CMAKE_BUILD_TYPE": "RelWithDebInfo",
-        "NEOFOAM_BUILD_TESTS": "ON",
+        "NEOFOAM_BUILD_TESTS": "OFF",
         "NEOFOAM_BUILD_BENCHMARKS": "ON",
         "NEOFOAM_ENABLE_WARNINGS": "OFF",
         "CMAKE_CXX_FLAGS": "-fno-omit-frame-pointer $env{CXXFLAGS}"

benchmarks/CMakeLists.txt

@@ -1,30 +1,30 @@
 # SPDX-License-Identifier: Unlicense
-# SPDX-FileCopyrightText: 2023 NeoFOAM authors
+# SPDX-FileCopyrightText: 2023-25 NeoFOAM authors
 
-add_subdirectory(fields)
-
-add_custom_command(
-  OUTPUT ${PROJECT_BINARY_DIR}/benchmarks/fields.xml
-  COMMAND ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/bench_fields -r XML > fields.xml
-  COMMENT "Execute benchmarks")
+function(neofoam_benchmark BENCH)
 
-find_package(
-  Python3
-  COMPONENTS Interpreter
-  REQUIRED)
+  add_executable(bench_${BENCH} "${BENCH}.cpp")
+  target_link_libraries(bench_${BENCH} PRIVATE Catch2::Catch2 NeoFOAM)
 
-add_custom_command(
-  OUTPUT ${PROJECT_BINARY_DIR}/benchmarks/fields.png
-  COMMAND Python3::Interpreter ${PROJECT_SOURCE_DIR}/scripts/plotBenchmarks.py
-          ${PROJECT_BINARY_DIR}/benchmarks/fields.xml
-  COMMENT "Plot benchmark results")
+  if(WIN32)
+    set_target_properties(
+      bench_${BENCH}
+      PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/
+                 LIBRARY_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/
+                 ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/)
+  else()
+    set_property(TARGET bench_${BENCH} PROPERTY RUNTIME_OUTPUT_DIRECTORY
+                                                ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks)
+  endif()
 
-add_custom_target(
-  execute_benchmarks
-  DEPENDS ${PROJECT_BINARY_DIR}/benchmarks/fields.xml
-  COMMENT "execute benchmarks")
+  if(NOT DEFINED "neofoam_WORKING_DIRECTORY")
+    set(neofoam_WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks)
+  endif()
+  add_test(
+    NAME bench_${BENCH}
+    COMMAND sh -c "./bench_${BENCH} -r xml > ${BENCH}.xml"
+    WORKING_DIRECTORY ${neofoam_WORKING_DIRECTORY})
+endfunction()
 
-add_custom_target(
-  execute_plot_benchmark
-  DEPENDS ${PROJECT_BINARY_DIR}/benchmarks/fields.png
-  COMMENT "plot benchmark results")
+add_subdirectory(fields)
+add_subdirectory(finiteVolume/cellCentred/operator)

benchmarks/catch_main.hpp

@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: MIT
+// SPDX-FileCopyrightText: 2025 NeoFOAM authors
+#pragma once
+
+#include <Kokkos_Core.hpp>
+#include <catch2/benchmark/catch_benchmark.hpp>
+#include <catch2/catch_session.hpp>
+#include <catch2/catch_test_macros.hpp>
+#include <catch2/catch_test_case_info.hpp>
+#include <catch2/generators/catch_generators_all.hpp>
+#include <catch2/reporters/catch_reporter_registrars.hpp>
+#include <catch2/reporters/catch_reporter_streaming_base.hpp>
+
+int main(int argc, char* argv[])
+{
+    // Initialize Catch2
+    Kokkos::ScopeGuard guard(argc, argv);
+    Catch::Session session;
+
+    // Specify command line options
+    int returnCode = session.applyCommandLine(argc, argv);
+    if (returnCode != 0) // Indicates a command line error
+        return returnCode;
+
+    int result = session.run();
+
+    return result;
+}

benchmarks/fields/CMakeLists.txt

@@ -1,16 +1,4 @@
 # SPDX-License-Identifier: Unlicense
 # SPDX-FileCopyrightText: 2023 NeoFOAM authors
 
-add_executable(bench_fields "bench_fields.cpp")
-target_link_libraries(bench_fields PRIVATE Catch2::Catch2 NeoFOAM Kokkos::kokkos)
-
-if(WIN32)
-  set_target_properties(
-    bench_fields
-    PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/$<0:>
-               LIBRARY_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/$<0:>
-               ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/$<0:>)
-else()
-  set_property(TARGET bench_fields PROPERTY RUNTIME_OUTPUT_DIRECTORY
-                                            ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks)
-endif()
+neofoam_benchmark(field)

benchmarks/fields/field.cpp

@@ -0,0 +1,55 @@
+// SPDX-License-Identifier: MIT
+// SPDX-FileCopyrightText: 2023 NeoFOAM authors
+
+#define CATCH_CONFIG_RUNNER // Define this before including catch.hpp to create
+                            // a custom main
+
+#include "../catch_main.hpp"
+#include "NeoFOAM/NeoFOAM.hpp"
+
+TEST_CASE("Field<scalar>::addition", "[bench]")
+{
+    auto size = GENERATE(1 << 16, 1 << 17, 1 << 18, 1 << 19, 1 << 20);
+    NeoFOAM::Executor exec = GENERATE(
+        NeoFOAM::Executor(NeoFOAM::SerialExecutor {}),
+        NeoFOAM::Executor(NeoFOAM::CPUExecutor {}),
+        NeoFOAM::Executor(NeoFOAM::GPUExecutor {})
+    );
+    std::string execName = std::visit([](auto e) { return e.name(); }, exec);
+
+    DYNAMIC_SECTION("" << size)
+    {
+        NeoFOAM::Field<NeoFOAM::scalar> cpuA(exec, size);
+        NeoFOAM::fill(cpuA, 1.0);
+        NeoFOAM::Field<NeoFOAM::scalar> cpuB(exec, size);
+        NeoFOAM::fill(cpuB, 2.0);
+        NeoFOAM::Field<NeoFOAM::scalar> cpuC(exec, size);
+        NeoFOAM::fill(cpuC, 0.0);
+
+        BENCHMARK(std::string(execName)) { return (cpuC = cpuA + cpuB); };
+    }
+}
+
+TEST_CASE("Field<scalar>::multiplication", "[bench]")
+{
+    auto size = GENERATE(1 << 16, 1 << 17, 1 << 18, 1 << 19, 1 << 20);
+
+    NeoFOAM::Executor exec = GENERATE(
+        NeoFOAM::Executor(NeoFOAM::SerialExecutor {}),
+        NeoFOAM::Executor(NeoFOAM::CPUExecutor {}),
+        NeoFOAM::Executor(NeoFOAM::GPUExecutor {})
+    );
+    std::string execName = std::visit([](auto e) { return e.name(); }, exec);
+
+    DYNAMIC_SECTION("" << size)
+    {
+        NeoFOAM::Field<NeoFOAM::scalar> cpuA(exec, size);
+        NeoFOAM::fill(cpuA, 1.0);
+        NeoFOAM::Field<NeoFOAM::scalar> cpuB(exec, size);
+        NeoFOAM::fill(cpuB, 2.0);
+        NeoFOAM::Field<NeoFOAM::scalar> cpuC(exec, size);
+        NeoFOAM::fill(cpuC, 0.0);
+
+        BENCHMARK(std::string(execName)) { return (cpuC = cpuA * cpuB); };
+    }
+}

benchmarks/finiteVolume/cellCentred/operator/CMakeLists.txt

@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: Unlicense
+# SPDX-FileCopyrightText: 2025 NeoFOAM authors
+
+neofoam_benchmark(divOperator)