diff --git a/.github/workflows/build_on_aws.yaml b/.github/workflows/build_on_aws.yaml index d0937c2e6..b7ae917da 100644 --- a/.github/workflows/build_on_aws.yaml +++ b/.github/workflows/build_on_aws.yaml @@ -121,11 +121,48 @@ jobs: module load libfabric-aws module load cmake ctest --preset ${{matrix.preset}} + benchmark-on-aws: + name: Benchmark on aws + needs: [start-runner, build-on-aws] # required to start the main job when the runner is ready + runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner + steps: + - name: Build NeoFOAM + shell: bash -i {0} + run: | + export HOME=/share/ec2-user + module load clang/16 + module load libfabric-aws + module spider libfabric-aws + module load cmake + cmake --version + CC=clang \ + CXX=clang++ \ + cmake --preset profiling + cmake --build --preset profiling + ctest --preset profiling + mkdir -p ${{github.event.number}}/ + cd build/profiling/bin/benchmarks + python3 -m pip install xmltodict + python3 ../../../../scripts/catch2json.py + cd ../../../.. + cp build/profiling/bin/benchmarks/*.json ${{github.event.number}}/ + lscpu > ${{github.event.number}}/lscpu.log + - name: Push Benchmark Data + uses: cpina/github-action-push-to-another-repository@main + env: + API_TOKEN_GITHUB: ${{ secrets.API_TOKEN_GITHUB }} + with: + source-directory: ${{github.event.number}} + destination-github-username: 'exasim-project' + destination-repository-name: 'NeoFOAM-BenchmarkData' + target-directory: ${{github.event.number}}/gdnxlarge + user-email: github-actions@github.com + target-branch: main stop-runner: name: Stop self-hosted EC2 runner needs: - start-runner # required to get output from the start-runner job - - build-on-aws # required to wait when the main job is done + - benchmark-on-aws # required to wait when the main job is done runs-on: ubuntu-latest permissions: id-token: write diff --git a/CMakeLists.txt b/CMakeLists.txt index 2ad2245e8..378fac1f4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -113,6 +113,7 @@ if(NEOFOAM_BUILD_TESTS) endif() if(NEOFOAM_BUILD_BENCHMARKS) + enable_testing() add_subdirectory(benchmarks) endif() diff --git a/CMakePresets.json b/CMakePresets.json index 25a42e3f1..69bf55aa6 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -37,7 +37,7 @@ "CMAKE_BUILD_TYPE": "Debug", "NEOFOAM_DEVEL_TOOLS": "ON", "NEOFOAM_BUILD_TESTS": "ON", - "NEOFOAM_BUILD_BENCHMARKS": "ON", + "NEOFOAM_BUILD_BENCHMARKS": "OFF", "NEOFOAM_ENABLE_WARNINGS": "ON", "Kokkos_ENABLE_DEBUG": "ON", "Kokkos_ENABLE_DEBUG_BOUNDS_CHECK": "ON", @@ -58,7 +58,7 @@ "description": "Configuration for profiling use", "cacheVariables": { "CMAKE_BUILD_TYPE": "RelWithDebInfo", - "NEOFOAM_BUILD_TESTS": "ON", + "NEOFOAM_BUILD_TESTS": "OFF", "NEOFOAM_BUILD_BENCHMARKS": "ON", "NEOFOAM_ENABLE_WARNINGS": "OFF", "CMAKE_CXX_FLAGS": "-fno-omit-frame-pointer $env{CXXFLAGS}" diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 071ef3180..886cf9a80 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -1,30 +1,30 @@ # SPDX-License-Identifier: Unlicense -# SPDX-FileCopyrightText: 2023 NeoFOAM authors +# SPDX-FileCopyrightText: 2023-25 NeoFOAM authors -add_subdirectory(fields) - -add_custom_command( - OUTPUT ${PROJECT_BINARY_DIR}/benchmarks/fields.xml - COMMAND ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/bench_fields -r XML > fields.xml - COMMENT "Execute benchmarks") +function(neofoam_benchmark BENCH) -find_package( - Python3 - COMPONENTS Interpreter - REQUIRED) + add_executable(bench_${BENCH} "${BENCH}.cpp") + target_link_libraries(bench_${BENCH} PRIVATE Catch2::Catch2 NeoFOAM) -add_custom_command( - OUTPUT ${PROJECT_BINARY_DIR}/benchmarks/fields.png - COMMAND Python3::Interpreter ${PROJECT_SOURCE_DIR}/scripts/plotBenchmarks.py - ${PROJECT_BINARY_DIR}/benchmarks/fields.xml - COMMENT "Plot benchmark results") + if(WIN32) + set_target_properties( + bench_${BENCH} + PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/ + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/ + ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/) + else() + set_property(TARGET bench_${BENCH} PROPERTY RUNTIME_OUTPUT_DIRECTORY + ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks) + endif() -add_custom_target( - execute_benchmarks - DEPENDS ${PROJECT_BINARY_DIR}/benchmarks/fields.xml - COMMENT "execute benchmarks") + if(NOT DEFINED "neofoam_WORKING_DIRECTORY") + set(neofoam_WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks) + endif() + add_test( + NAME bench_${BENCH} + COMMAND sh -c "./bench_${BENCH} -r xml > ${BENCH}.xml" + WORKING_DIRECTORY ${neofoam_WORKING_DIRECTORY}) +endfunction() -add_custom_target( - execute_plot_benchmark - DEPENDS ${PROJECT_BINARY_DIR}/benchmarks/fields.png - COMMENT "plot benchmark results") +add_subdirectory(fields) +add_subdirectory(finiteVolume/cellCentred/operator) diff --git a/benchmarks/catch_main.hpp b/benchmarks/catch_main.hpp new file mode 100644 index 000000000..e6739016e --- /dev/null +++ b/benchmarks/catch_main.hpp @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2025 NeoFOAM authors +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +int main(int argc, char* argv[]) +{ + // Initialize Catch2 + Kokkos::ScopeGuard guard(argc, argv); + Catch::Session session; + + // Specify command line options + int returnCode = session.applyCommandLine(argc, argv); + if (returnCode != 0) // Indicates a command line error + return returnCode; + + int result = session.run(); + + return result; +} diff --git a/benchmarks/fields/CMakeLists.txt b/benchmarks/fields/CMakeLists.txt index f83d15f36..aede4c87c 100644 --- a/benchmarks/fields/CMakeLists.txt +++ b/benchmarks/fields/CMakeLists.txt @@ -1,16 +1,4 @@ # SPDX-License-Identifier: Unlicense # SPDX-FileCopyrightText: 2023 NeoFOAM authors -add_executable(bench_fields "bench_fields.cpp") -target_link_libraries(bench_fields PRIVATE Catch2::Catch2 NeoFOAM Kokkos::kokkos) - -if(WIN32) - set_target_properties( - bench_fields - PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/$<0:> - LIBRARY_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/$<0:> - ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks/$<0:>) -else() - set_property(TARGET bench_fields PROPERTY RUNTIME_OUTPUT_DIRECTORY - ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/benchmarks) -endif() +neofoam_benchmark(field) diff --git a/benchmarks/fields/bench_fields.cpp b/benchmarks/fields/bench_fields.cpp deleted file mode 100644 index e7f61be37..000000000 --- a/benchmarks/fields/bench_fields.cpp +++ /dev/null @@ -1,125 +0,0 @@ -// SPDX-License-Identifier: MIT -// SPDX-FileCopyrightText: 2023 NeoFOAM authors - -#define CATCH_CONFIG_RUNNER // Define this before including catch.hpp to create - // a custom main -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "NeoFOAM/fields/field.hpp" -#include "NeoFOAM/fields/fieldTypeDefs.hpp" - - -int main(int argc, char* argv[]) -{ - // Initialize Catch2 - Kokkos::initialize(argc, argv); - Catch::Session session; - - // Specify command line options - int returnCode = session.applyCommandLine(argc, argv); - if (returnCode != 0) // Indicates a command line error - return returnCode; - - int result = session.run(); - - // Run benchmarks if there are any - Kokkos::finalize(); - - return result; -} - -TEST_CASE("Vector addition [benchmark]") -{ - - auto size = - GENERATE(8, 64, 512, 4096, 32768, 262144, 1048576, 1048576 * 4, 1048576 * 16, 1048576 * 64); - - CAPTURE(size); // Capture the value of size - - // capture the value of size as section name - DYNAMIC_SECTION("" << size) {{NeoFOAM::SerialExecutor cpuExec {}; - NeoFOAM::Field cpuA(cpuExec, size); - NeoFOAM::fill(cpuA, 1.0); - NeoFOAM::Field cpuB(cpuExec, size); - NeoFOAM::fill(cpuB, 2.0); - NeoFOAM::Field cpuC(cpuExec, size); - NeoFOAM::fill(cpuC, 0.0); - - BENCHMARK("Field addition") { return (cpuC = cpuA + cpuB); }; -} - -{ - NeoFOAM::CPUExecutor ompExec {}; - NeoFOAM::Field ompA(ompExec, size); - NeoFOAM::fill(ompA, 1.0); - NeoFOAM::Field ompB(ompExec, size); - NeoFOAM::fill(ompB, 2.0); - NeoFOAM::Field ompC(ompExec, size); - NeoFOAM::fill(ompC, 0.0); - - BENCHMARK("Field addition") { return (ompC = ompA + ompB); }; -} - -{ - NeoFOAM::GPUExecutor gpuExec {}; - NeoFOAM::Field gpuA(gpuExec, size); - NeoFOAM::fill(gpuA, 1.0); - NeoFOAM::Field gpuB(gpuExec, size); - NeoFOAM::fill(gpuB, 2.0); - NeoFOAM::Field gpuC(gpuExec, size); - NeoFOAM::fill(gpuC, 0.0); - - BENCHMARK("Field addition") - { - gpuC = gpuA + gpuB; - return Kokkos::fence(); - }; -} - -{ - NeoFOAM::GPUExecutor gpuExec {}; - NeoFOAM::Field gpuA(gpuExec, size); - NeoFOAM::fill(gpuA, 1.0); - NeoFOAM::Field gpuB(gpuExec, size); - NeoFOAM::fill(gpuB, 2.0); - NeoFOAM::Field gpuC(gpuExec, size); - NeoFOAM::fill(gpuC, 0.0); - - auto sGpuB = gpuB.span(); - auto sGpuC = gpuC.span(); - BENCHMARK("Field addition no allocation") - { - gpuA.apply(KOKKOS_LAMBDA(const NeoFOAM::size_t i) { return sGpuB[i] + sGpuC[i]; }); - return Kokkos::fence(); - // return GPUa; - }; -} - -{ - NeoFOAM::CPUExecutor ompExec {}; - NeoFOAM::Field ompA(ompExec, size); - NeoFOAM::fill(ompA, 1.0); - NeoFOAM::Field ompB(ompExec, size); - NeoFOAM::fill(ompB, 2.0); - NeoFOAM::Field ompC(ompExec, size); - NeoFOAM::fill(ompC, 0.0); - - auto sompB = ompB.span(); - auto sompC = ompC.span(); - BENCHMARK("Field addition no allocation") - { - ompA.apply(KOKKOS_LAMBDA(const NeoFOAM::size_t i) { return sompB[i] + sompC[i]; }); - }; -} -} -; -} diff --git a/benchmarks/fields/field.cpp b/benchmarks/fields/field.cpp new file mode 100644 index 000000000..9ec0ca7da --- /dev/null +++ b/benchmarks/fields/field.cpp @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2023 NeoFOAM authors + +#define CATCH_CONFIG_RUNNER // Define this before including catch.hpp to create + // a custom main + +#include "../catch_main.hpp" +#include "NeoFOAM/NeoFOAM.hpp" + +TEST_CASE("Field::addition", "[bench]") +{ + auto size = GENERATE(1 << 16, 1 << 17, 1 << 18, 1 << 19, 1 << 20); + NeoFOAM::Executor exec = GENERATE( + NeoFOAM::Executor(NeoFOAM::SerialExecutor {}), + NeoFOAM::Executor(NeoFOAM::CPUExecutor {}), + NeoFOAM::Executor(NeoFOAM::GPUExecutor {}) + ); + std::string execName = std::visit([](auto e) { return e.name(); }, exec); + + DYNAMIC_SECTION("" << size) + { + NeoFOAM::Field cpuA(exec, size); + NeoFOAM::fill(cpuA, 1.0); + NeoFOAM::Field cpuB(exec, size); + NeoFOAM::fill(cpuB, 2.0); + NeoFOAM::Field cpuC(exec, size); + NeoFOAM::fill(cpuC, 0.0); + + BENCHMARK(std::string(execName)) { return (cpuC = cpuA + cpuB); }; + } +} + +TEST_CASE("Field::multiplication", "[bench]") +{ + auto size = GENERATE(1 << 16, 1 << 17, 1 << 18, 1 << 19, 1 << 20); + + NeoFOAM::Executor exec = GENERATE( + NeoFOAM::Executor(NeoFOAM::SerialExecutor {}), + NeoFOAM::Executor(NeoFOAM::CPUExecutor {}), + NeoFOAM::Executor(NeoFOAM::GPUExecutor {}) + ); + std::string execName = std::visit([](auto e) { return e.name(); }, exec); + + DYNAMIC_SECTION("" << size) + { + NeoFOAM::Field cpuA(exec, size); + NeoFOAM::fill(cpuA, 1.0); + NeoFOAM::Field cpuB(exec, size); + NeoFOAM::fill(cpuB, 2.0); + NeoFOAM::Field cpuC(exec, size); + NeoFOAM::fill(cpuC, 0.0); + + BENCHMARK(std::string(execName)) { return (cpuC = cpuA * cpuB); }; + } +} diff --git a/benchmarks/finiteVolume/cellCentred/operator/CMakeLists.txt b/benchmarks/finiteVolume/cellCentred/operator/CMakeLists.txt new file mode 100644 index 000000000..ae6508a61 --- /dev/null +++ b/benchmarks/finiteVolume/cellCentred/operator/CMakeLists.txt @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: Unlicense +# SPDX-FileCopyrightText: 2025 NeoFOAM authors + +neofoam_benchmark(divOperator) diff --git a/benchmarks/finiteVolume/cellCentred/operator/divOperator.cpp b/benchmarks/finiteVolume/cellCentred/operator/divOperator.cpp new file mode 100644 index 000000000..3d0d2e522 --- /dev/null +++ b/benchmarks/finiteVolume/cellCentred/operator/divOperator.cpp @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2023 NeoFOAM authors + +#define CATCH_CONFIG_RUNNER // Define this before including catch.hpp to create + // a custom main + +#include "NeoFOAM/NeoFOAM.hpp" +#include "../../../catch_main.hpp" + +using Operator = NeoFOAM::dsl::Operator; + +TEST_CASE("DivOperator::div", "[bench]") +{ + auto size = GENERATE(1 << 16, 1 << 17, 1 << 18, 1 << 19, 1 << 20); + + NeoFOAM::Executor exec = GENERATE( + NeoFOAM::Executor(NeoFOAM::SerialExecutor {}), + NeoFOAM::Executor(NeoFOAM::CPUExecutor {}), + NeoFOAM::Executor(NeoFOAM::GPUExecutor {}) + ); + + std::string execName = std::visit([](auto e) { return e.name(); }, exec); + NeoFOAM::UnstructuredMesh mesh = NeoFOAM::create1DUniformMesh(exec, size); + auto surfaceBCs = fvcc::createCalculatedBCs>(mesh); + fvcc::SurfaceField faceFlux(exec, "sf", mesh, surfaceBCs); + NeoFOAM::fill(faceFlux.internalField(), 1.0); + + auto volumeBCs = fvcc::createCalculatedBCs>(mesh); + fvcc::VolumeField phi(exec, "vf", mesh, volumeBCs); + fvcc::VolumeField divPhi(exec, "divPhi", mesh, volumeBCs); + NeoFOAM::fill(phi.internalField(), 1.0); + + // capture the value of size as section name + DYNAMIC_SECTION("" << size) + { + NeoFOAM::Input input = NeoFOAM::TokenList({std::string("Gauss"), std::string("linear")}); + auto op = fvcc::DivOperator(Operator::Type::Explicit, faceFlux, phi, input); + + BENCHMARK(std::string(execName)) { return (op.div(divPhi)); }; + } +} diff --git a/scripts/catch2json.py b/scripts/catch2json.py new file mode 100644 index 000000000..9980b0937 --- /dev/null +++ b/scripts/catch2json.py @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: 2023-2025 NeoFOAM authors + +import xmltodict +import json +import sys +import os + + +def parse_xml_dict(d): + """takes the catch2 xml dict, performs clean-up and returns a + list of records""" + data = d["Catch2TestRun"]["TestCase"] + if isinstance(data, dict): + data = [data] + records = [] + for cases in data: + test_case = cases["@name"] + for d in cases["Section"]: + size = d["@name"] + res = {} + for k, v in d["BenchmarkResults"].items(): + # print(k, v) + if k == "@name": + res["executor"] = v + if k == "mean": + res["mean"] = v["@value"] + continue + if k == "standardDeviation": + res["standardDeviation"] = v["@value"] + continue + if k.startswith("@"): + continue + res["size"] = size + res["test_case"] = test_case + records.append(res) + return records + + +def main(): + _, _, files = next(os.walk(".")) + for xml_file in files: + if not xml_file.endswith("xml"): + continue + try: + with open(xml_file, "r") as fh: + d = xmltodict.parse(fh.read()) + res = parse_xml_dict(d) + with open(xml_file.replace("xml", "json"), "w") as outfile: + json.dump(res, outfile) + except Exception as e: + print(e) + + +if __name__ == "__main__": + main() diff --git a/scripts/plotBenchmarks.py b/scripts/plotBenchmarks.py deleted file mode 100644 index 86ea10fcf..000000000 --- a/scripts/plotBenchmarks.py +++ /dev/null @@ -1,132 +0,0 @@ -# SPDX-License-Identifier: MIT -# SPDX-FileCopyrightText: 2023 NeoFOAM authors - -import xml.etree.ElementTree as ET -import pandas as pd -import matplotlib.pyplot as plt -import seaborn as sns -import sys - -xml_file = sys.argv[1] - -# Parse the XML file -tree = ET.parse(xml_file) - -# Get the root element -root = tree.getroot() - - -# %% -def extract_benchmarks(xml_file): - tree = ET.parse(xml_file) - root = tree.getroot() - testcase_name = "" - section_names = [] - - for testcase in root.iter("TestCase"): - testcase_name = testcase.attrib["name"] - print(f"Test case: {testcase_name}") - - for section in testcase.iter("Section"): - section_name = section.attrib["name"] - print(f" Section: {section_name}") - - for benchmark in section.iter("BenchmarkResults"): - for result in benchmark.iter("Result"): - name = result.attrib["name"] - value = result.attrib["value"] - print(f" Benchmark {name}: {value}") - - -extract_benchmarks(xml_file) - - -# %% -def getBenchmarkData(element): - for bRes in element: - if bRes.tag == "mean": - meanValue = bRes.attrib["value"] - lowerBound = bRes.attrib["lowerBound"] - upperBound = bRes.attrib["upperBound"] - # if bRes.tag == 'mean': - # meanValue = bRes.attrib['value'] - return meanValue, lowerBound, upperBound - - -def process_element(f, element, testcase_name, section_names, depth=0): - indent_str = " " * depth - if element.tag == "TestCase": - testcase_name = element.attrib["name"] - elif element.tag == "Section": - section_names[depth] = element.attrib["name"] - elif element.tag == "BenchmarkResults": - bName = element.attrib["name"] - bValues = getBenchmarkData(element) - f.write( - f'{testcase_name},{",".join(section_names)},{bName},{",".join(bValues)}\n' - ) - - for child in element: - if element.tag == "Section": - process_element(f, child, testcase_name, section_names, depth + 1) - else: - process_element(f, child, testcase_name, section_names, depth + 0) - - -def nSections(element, depth=0): - max_depth = max(depth, 0) - for child in element: - if element.tag == "Section": - max_depth = max(max_depth, nSections(child, depth + 1)) - else: - max_depth = max(max_depth, nSections(child, depth + 0)) - - return max_depth - - -def extract_benchmarks(xml_file, csv_file): - tree = ET.parse(xml_file) - root = tree.getroot() - nSec = nSections(root) - section_names = [""] * nSec - testcase_name = "" - with open(csv_file, "w") as f: - process_element(f, root, testcase_name, section_names) - - -def main(xml_file): - extract_benchmarks(xml_file, "bench_blas.csv") - - df = pd.read_csv( - "bench_blas.csv", - names=[ - "Testcase", - "Vector Elements", - "Benchmark", - "runTime [ms]", - "lowerBound", - "upperBound", - ], - ) - print(df) - df["Vector Elements"] = pd.to_numeric(df["Vector Elements"], errors="coerce") - df["runTime [ms]"] /= 1e6 - - sns.lineplot( - data=df, - x="Vector Elements", - y="runTime [ms]", - hue="Benchmark", - style="Benchmark", - markers=True, - ) - plt.xscale("log") - plt.yscale("log") - plt.savefig("fields.png") - plt.tight_layout() - plt.show() - # %% - - -if __name__ == "__main__": - main(sys.argv[1])