From c2fbf0e69202157c04dc0b730732c7a0409c2c36 Mon Sep 17 00:00:00 2001 From: Spiros Tsalikis Date: Wed, 8 Nov 2023 10:33:48 -0500 Subject: [PATCH 1/6] Change install directory of Gray scott files --- examples/simulations/gray-scott-struct/CMakeLists.txt | 6 +++--- examples/simulations/gray-scott/CMakeLists.txt | 6 +++--- examples/simulations/gray-scott/ReadMe.md | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/simulations/gray-scott-struct/CMakeLists.txt b/examples/simulations/gray-scott-struct/CMakeLists.txt index a892b757dd..4753ec9bc3 100644 --- a/examples/simulations/gray-scott-struct/CMakeLists.txt +++ b/examples/simulations/gray-scott-struct/CMakeLists.txt @@ -43,11 +43,11 @@ if(ADIOS2_HAVE_MPI) "simulation/settings-inline.json" "plot/decomp.py" "plot/gsplot.py" "plot/pdfplot.py" "ReadMe.md" - DESTINATION ${CMAKE_INSTALL_PREFIX}/share/adios2/examples/simulations/gray-scott-struct) + DESTINATION ${CMAKE_INSTALL_PREFIX}/adios2/gray-scott-struct) - install(DIRECTORY "catalyst" DESTINATION ${CMAKE_INSTALL_PREFIX}/share/adios2/examples/simulations/gray-scott-struct) + install(DIRECTORY "catalyst" DESTINATION ${CMAKE_INSTALL_PREFIX}/adios2/gray-scott-struct) - install(PROGRAMS "cleanup.sh" DESTINATION ${CMAKE_INSTALL_PREFIX}/share/adios2/examples/simulations/gray-scott-struct) + install(PROGRAMS "cleanup.sh" DESTINATION ${CMAKE_INSTALL_PREFIX}/adios2/gray-scott-struct) find_package(VTK QUIET) if(VTK_FOUND) diff --git a/examples/simulations/gray-scott/CMakeLists.txt b/examples/simulations/gray-scott/CMakeLists.txt index 7fed3add60..e5c932e5d4 100644 --- a/examples/simulations/gray-scott/CMakeLists.txt +++ b/examples/simulations/gray-scott/CMakeLists.txt @@ -43,11 +43,11 @@ if(ADIOS2_HAVE_MPI) "simulation/settings-inline.json" "plot/decomp.py" "plot/gsplot.py" "plot/pdfplot.py" "ReadMe.md" - DESTINATION ${CMAKE_INSTALL_PREFIX}/share/adios2/examples/simulations/gray-scott) + DESTINATION ${CMAKE_INSTALL_PREFIX}/adios2/gray-scott) - install(DIRECTORY "catalyst" DESTINATION ${CMAKE_INSTALL_PREFIX}/share/adios2/examples/simulations/gray-scott) + install(DIRECTORY "catalyst" DESTINATION ${CMAKE_INSTALL_PREFIX}/adios2/gray-scott) - install(PROGRAMS "cleanup.sh" DESTINATION ${CMAKE_INSTALL_PREFIX}/share/adios2/examples/simulations/gray-scott) + install(PROGRAMS "cleanup.sh" DESTINATION ${CMAKE_INSTALL_PREFIX}/adios2/gray-scott) find_package(VTK QUIET) if(VTK_FOUND) diff --git a/examples/simulations/gray-scott/ReadMe.md b/examples/simulations/gray-scott/ReadMe.md index db615b1f12..ff64216c27 100644 --- a/examples/simulations/gray-scott/ReadMe.md +++ b/examples/simulations/gray-scott/ReadMe.md @@ -17,7 +17,7 @@ care of this aspect). From a scratch directory copy the config files from your installation of adios2: ``` -$ cp -r /share/adios2/examples/simulations/gray-scott . +$ cp -r /adios2/gray-scott . $ cd gray-scott $ mpirun -n 4 adios2_simulations_gray-scott settings-files.json ======================================== From 247e2f3d006187e368bc5b5e6a5a160435c1266b Mon Sep 17 00:00:00 2001 From: Vicente Adolfo Bolea Sanchez Date: Mon, 13 Nov 2023 12:54:23 -0500 Subject: [PATCH 2/6] ci: add shellcheck coverage to source and testing --- .shellcheck_exclude_paths | 15 +++++++++++++++ scripts/ci/scripts/run-shellcheck.sh | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.shellcheck_exclude_paths b/.shellcheck_exclude_paths index 1ef95d56a2..e84d523737 100644 --- a/.shellcheck_exclude_paths +++ b/.shellcheck_exclude_paths @@ -15,3 +15,18 @@ scripts/developer/setup.sh scripts/docker/setup-user.sh scripts/runconf/runconf.sh scripts/runconf/runconf_olcf.sh +testing/contract/lammps/build.sh +testing/contract/lammps/config.sh +testing/contract/lammps/install.sh +testing/contract/lammps/setup.sh +testing/contract/lammps/test.sh +testing/contract/scorpio/build.sh +testing/contract/scorpio/config.sh +testing/contract/scorpio/install.sh +testing/contract/scorpio/setup.sh +testing/contract/scorpio/test.sh +testing/contract/tau/build.sh +testing/contract/tau/config.sh +testing/contract/tau/install.sh +testing/contract/tau/setup.sh +testing/contract/tau/test.sh diff --git a/scripts/ci/scripts/run-shellcheck.sh b/scripts/ci/scripts/run-shellcheck.sh index c987a5bea0..1dc735fc8b 100755 --- a/scripts/ci/scripts/run-shellcheck.sh +++ b/scripts/ci/scripts/run-shellcheck.sh @@ -15,7 +15,7 @@ fi # Give me a sorted list of the project scripts found_scripts="$({ - find scripts -regextype posix-extended -iregex '.*\.(sh|bash)' -print; + find scripts source testing -regextype posix-extended -iregex '.*\.(sh|bash)' -print; grep -rnlE -e '#!/(/usr)?/bin/(bash|sh)' -e '#!(/usr)?/bin/env\s+(bash|sh)' scripts; } | sort -u)" From 4fab35aacfa8eeb18db7633333a018a32ee25c53 Mon Sep 17 00:00:00 2001 From: Spiros Tsalikis Date: Tue, 14 Nov 2023 11:13:29 -0500 Subject: [PATCH 3/6] Change install directory of Gray scott files again The directory should be in share --- examples/simulations/gray-scott-struct/CMakeLists.txt | 6 +++--- examples/simulations/gray-scott/CMakeLists.txt | 6 +++--- examples/simulations/gray-scott/ReadMe.md | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/simulations/gray-scott-struct/CMakeLists.txt b/examples/simulations/gray-scott-struct/CMakeLists.txt index 4753ec9bc3..8538e6bd07 100644 --- a/examples/simulations/gray-scott-struct/CMakeLists.txt +++ b/examples/simulations/gray-scott-struct/CMakeLists.txt @@ -43,11 +43,11 @@ if(ADIOS2_HAVE_MPI) "simulation/settings-inline.json" "plot/decomp.py" "plot/gsplot.py" "plot/pdfplot.py" "ReadMe.md" - DESTINATION ${CMAKE_INSTALL_PREFIX}/adios2/gray-scott-struct) + DESTINATION ${CMAKE_INSTALL_PREFIX}/share/adios2/gray-scott-struct) - install(DIRECTORY "catalyst" DESTINATION ${CMAKE_INSTALL_PREFIX}/adios2/gray-scott-struct) + install(DIRECTORY "catalyst" DESTINATION ${CMAKE_INSTALL_PREFIX}/share/adios2/gray-scott-struct) - install(PROGRAMS "cleanup.sh" DESTINATION ${CMAKE_INSTALL_PREFIX}/adios2/gray-scott-struct) + install(PROGRAMS "cleanup.sh" DESTINATION ${CMAKE_INSTALL_PREFIX}/share/adios2/gray-scott-struct) find_package(VTK QUIET) if(VTK_FOUND) diff --git a/examples/simulations/gray-scott/CMakeLists.txt b/examples/simulations/gray-scott/CMakeLists.txt index e5c932e5d4..e58acd4144 100644 --- a/examples/simulations/gray-scott/CMakeLists.txt +++ b/examples/simulations/gray-scott/CMakeLists.txt @@ -43,11 +43,11 @@ if(ADIOS2_HAVE_MPI) "simulation/settings-inline.json" "plot/decomp.py" "plot/gsplot.py" "plot/pdfplot.py" "ReadMe.md" - DESTINATION ${CMAKE_INSTALL_PREFIX}/adios2/gray-scott) + DESTINATION ${CMAKE_INSTALL_PREFIX}/share/adios2/gray-scott) - install(DIRECTORY "catalyst" DESTINATION ${CMAKE_INSTALL_PREFIX}/adios2/gray-scott) + install(DIRECTORY "catalyst" DESTINATION ${CMAKE_INSTALL_PREFIX}/share/adios2/gray-scott) - install(PROGRAMS "cleanup.sh" DESTINATION ${CMAKE_INSTALL_PREFIX}/adios2/gray-scott) + install(PROGRAMS "cleanup.sh" DESTINATION ${CMAKE_INSTALL_PREFIX}/share/adios2/gray-scott) find_package(VTK QUIET) if(VTK_FOUND) diff --git a/examples/simulations/gray-scott/ReadMe.md b/examples/simulations/gray-scott/ReadMe.md index ff64216c27..4fca5b4348 100644 --- a/examples/simulations/gray-scott/ReadMe.md +++ b/examples/simulations/gray-scott/ReadMe.md @@ -17,7 +17,7 @@ care of this aspect). From a scratch directory copy the config files from your installation of adios2: ``` -$ cp -r /adios2/gray-scott . +$ cp -r /share/adios2/gray-scott . $ cd gray-scott $ mpirun -n 4 adios2_simulations_gray-scott settings-files.json ======================================== From 92bd27f843a9ca16723a43d0b4f000029d550b9b Mon Sep 17 00:00:00 2001 From: anagainaru Date: Tue, 14 Nov 2023 15:42:53 -0500 Subject: [PATCH 4/6] Changes to MallocV to take into consideration the memory space of a variable --- source/adios2/toolkit/format/buffer/malloc/MallocV.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/source/adios2/toolkit/format/buffer/malloc/MallocV.cpp b/source/adios2/toolkit/format/buffer/malloc/MallocV.cpp index 90359071fb..efe28d79bd 100644 --- a/source/adios2/toolkit/format/buffer/malloc/MallocV.cpp +++ b/source/adios2/toolkit/format/buffer/malloc/MallocV.cpp @@ -7,6 +7,7 @@ */ #include "MallocV.h" +#include "adios2/helper/adiosFunctions.h" #include "adios2/toolkit/format/buffer/BufferV.h" #include @@ -74,7 +75,12 @@ size_t MallocV::AddToVec(const size_t size, const void *buf, size_t align, bool m_InternalBlock = (char *)realloc(m_InternalBlock, NewSize); m_AllocatedSize = NewSize; } - memcpy(m_InternalBlock + m_internalPos, buf, size); +#ifdef ADIOS2_HAVE_GPU_SUPPORT + if (MemSpace == MemorySpace::GPU) + helper::CopyFromGPUToBuffer(m_InternalBlock, m_internalPos, buf, MemSpace, size); +#endif + if (MemSpace == MemorySpace::Host) + memcpy(m_InternalBlock + m_internalPos, buf, size); if (DataV.size() && !DataV.back().External && (m_internalPos == (DataV.back().Offset + DataV.back().Size))) From b26cfbf528e20557467d7195c17dafee62da1b43 Mon Sep 17 00:00:00 2001 From: anagainaru Date: Tue, 14 Nov 2023 17:08:51 -0500 Subject: [PATCH 5/6] Example using Kokkos buffers with SST --- examples/hello/CMakeLists.txt | 3 + examples/hello/sstKokkos/CMakeLists.txt | 37 ++++++++ examples/hello/sstKokkos/sstReaderKokkos.cpp | 88 ++++++++++++++++++ examples/hello/sstKokkos/sstWriterKokkos.cpp | 96 ++++++++++++++++++++ 4 files changed, 224 insertions(+) create mode 100644 examples/hello/sstKokkos/CMakeLists.txt create mode 100644 examples/hello/sstKokkos/sstReaderKokkos.cpp create mode 100644 examples/hello/sstKokkos/sstWriterKokkos.cpp diff --git a/examples/hello/CMakeLists.txt b/examples/hello/CMakeLists.txt index 778cebf970..4d0d447a77 100644 --- a/examples/hello/CMakeLists.txt +++ b/examples/hello/CMakeLists.txt @@ -25,6 +25,9 @@ if(ADIOS2_HAVE_Kokkos_HIP) endif() if(ADIOS2_HAVE_Kokkos) add_subdirectory(bpStepsWriteReadKokkos) + if(ADIOS2_HAVE_SST) + add_subdirectory(sstKokkos) + endif() endif() add_subdirectory(bpThreadWrite) diff --git a/examples/hello/sstKokkos/CMakeLists.txt b/examples/hello/sstKokkos/CMakeLists.txt new file mode 100644 index 0000000000..4074583bb7 --- /dev/null +++ b/examples/hello/sstKokkos/CMakeLists.txt @@ -0,0 +1,37 @@ +#-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- # +#Distributed under the OSI - approved Apache License, Version 2.0. See +#accompanying file Copyright.txt for details. +#-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- # + +cmake_minimum_required(VERSION 3.12) +project(ADIOS2HelloSSTKokkosExample) + +#CXX Compiler settings only in for this example +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +if (NOT TARGET adios2_core) + set(_components CXX) + + find_package(Kokkos 3.7 QUIET) + if (Kokkos_FOUND AND DEFINED Kokkos_CXX_COMPILER) + set(CMAKE_CXX_COMPILER "${Kokkos_CXX_COMPILER}") + endif() + + find_package(ADIOS2 REQUIRED COMPONENTS ${_components}) +else() + if (DEFINED Kokkos_CXX_COMPILER) + set(CMAKE_CXX_COMPILER "${Kokkos_CXX_COMPILER}") + endif() +endif() + +if (ADIOS2_HAVE_Kokkos) + add_executable(adios2_hello_sstWriterKokkos sstWriterKokkos.cpp) + add_executable(adios2_hello_sstReaderKokkos sstReaderKokkos.cpp) + kokkos_compilation(SOURCE sstWriterKokkos.cpp) + kokkos_compilation(SOURCE sstReaderKokkos.cpp) + target_link_libraries(adios2_hello_sstWriterKokkos adios2::cxx11 Kokkos::kokkos) + install(TARGETS adios2_hello_sstWriterKokkos RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + target_link_libraries(adios2_hello_sstReaderKokkos adios2::cxx11 Kokkos::kokkos) + install(TARGETS adios2_hello_sstReaderKokkos RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +endif() diff --git a/examples/hello/sstKokkos/sstReaderKokkos.cpp b/examples/hello/sstKokkos/sstReaderKokkos.cpp new file mode 100644 index 0000000000..6dd6f9eed8 --- /dev/null +++ b/examples/hello/sstKokkos/sstReaderKokkos.cpp @@ -0,0 +1,88 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * sstReaderKokkos.cpp Simple example of reading bpFloats through ADIOS2 SST + * engine with multiple simulations steps for every IO step using Kokkos + */ +#include +#include +#include + +#include +#include + +#include + +template +int BPRead(adios2::ADIOS &adios, const std::string fname, const size_t Nx, const size_t Ny, + const size_t nSteps, const std::string engine) +{ + adios2::IO io = adios.DeclareIO("ReadIO"); + io.SetEngine(engine); + + ExecSpace exe_space; + std::cout << "Read on memory space: " << exe_space.name() << std::endl; + + adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + + unsigned int step = 0; + bool correctValues = true; + Kokkos::View gpuSimData("simBuffer", Nx, Ny); + for (; bpReader.BeginStep() == adios2::StepStatus::OK; ++step) + { + auto data = io.InquireVariable("bpFloats"); + const adios2::Dims start{0, 0}; + const adios2::Dims count{Nx, Ny}; + const adios2::Box sel(start, count); + data.SetSelection(sel); + + // var.SetMemorySpace(adios2::MemorySpace::GPU); + bpReader.Get(data, gpuSimData); + bpReader.EndStep(); + + auto cpuData = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, gpuSimData); + if (cpuData(0, 0) != step * 10) + { + std::cout << "Value mismatch at step " << step << std::endl; + correctValues = false; + break; + } + } + if (correctValues) + std::cout << "Read " << step << " steps successfully" << std::endl; + + bpReader.Close(); + return 0; +} + +int main(int argc, char **argv) +{ + const std::string engine = argv[1] ? argv[1] : "SST"; + std::cout << "Using engine " << engine << std::endl; + const size_t Nx = 600, Ny = 100, nSteps = 2; + const std::string memorySpace = "Device"; + + const std::string filename = engine + "StepsWriteReadKokkos"; + Kokkos::initialize(argc, argv); + { + adios2::ADIOS adios; + + std::cout << "Using engine " << engine << std::endl; + if (memorySpace == "Device") + { + using mem_space = Kokkos::DefaultExecutionSpace::memory_space; + std::cout << "Memory space: DefaultMemorySpace" << std::endl; + BPRead(adios, filename + "_DD.bp", Nx, Ny, + nSteps, engine); + } + else + { + std::cout << "Memory space: HostSpace" << std::endl; + BPRead(adios, filename + "_HH.bp", Nx, Ny, nSteps, + engine); + } + } + Kokkos::finalize(); + return 0; +} diff --git a/examples/hello/sstKokkos/sstWriterKokkos.cpp b/examples/hello/sstKokkos/sstWriterKokkos.cpp new file mode 100644 index 0000000000..5354d7e228 --- /dev/null +++ b/examples/hello/sstKokkos/sstWriterKokkos.cpp @@ -0,0 +1,96 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * sstWriterKokkos.cpp Simple example of writing bpFloats through ADIOS2 SST + * engine with multiple simulations steps for every IO step using Kokkos + */ +#include +#include +#include + +#include +#include + +#include + +template +int BPWrite(adios2::ADIOS &adios, const std::string fname, const size_t Nx, const size_t Ny, + const size_t nSteps, const std::string engine) +{ + // Initialize the simulation data + Kokkos::View gpuSimData("simBuffer", Nx, Ny); + static_assert(Kokkos::SpaceAccessibility::accessible, ""); + Kokkos::parallel_for( + "initBuffer", Kokkos::RangePolicy(0, Nx), KOKKOS_LAMBDA(int i) { + for (int j = 0; j < Ny; j++) + gpuSimData(i, j) = static_cast(i); + }); + Kokkos::fence(); + + adios2::IO io = adios.DeclareIO("WriteIO"); + io.SetEngine(engine); + + const adios2::Dims shape{Nx, Ny}; + const adios2::Dims start{0, 0}; + const adios2::Dims count{Nx, Ny}; + auto data = io.DefineVariable("bpFloats", shape, start, count); + + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); + + // Simulation steps + for (int step = 0; step < nSteps; ++step) + { + adios2::Box sel({0, 0}, {Nx, Ny}); + data.SetSelection(sel); + + bpWriter.BeginStep(); + // var.SetMemorySpace(adios2::MemorySpace::GPU); + bpWriter.Put(data, gpuSimData); + bpWriter.EndStep(); + + // Update values in the simulation data + Kokkos::parallel_for( + "updateBuffer", Kokkos::RangePolicy(0, Nx), KOKKOS_LAMBDA(int i) { + for (int j = 0; j < Ny; j++) + gpuSimData(i, j) += 10; + }); + Kokkos::fence(); + } + + bpWriter.Close(); + ExecSpace exe_space; + std::cout << "Done writing on memory space: " << exe_space.name() << std::endl; + return 0; +} + +int main(int argc, char **argv) +{ + const std::string engine = argv[1] ? argv[1] : "SST"; + std::cout << "Using engine " << engine << std::endl; + const size_t Nx = 600, Ny = 100, nSteps = 2; + const std::string memorySpace = "Device"; + + const std::string filename = engine + "StepsWriteReadKokkos"; + Kokkos::initialize(argc, argv); + { + adios2::ADIOS adios; + + std::cout << "Using engine " << engine << std::endl; + if (memorySpace == "Device") + { + using mem_space = Kokkos::DefaultExecutionSpace::memory_space; + std::cout << "Memory space: DefaultMemorySpace" << std::endl; + BPWrite(adios, filename + "_DD.bp", Nx, Ny, + nSteps, engine); + } + else + { + std::cout << "Memory space: HostSpace" << std::endl; + BPWrite(adios, filename + "_HH.bp", Nx, Ny, nSteps, + engine); + } + } + Kokkos::finalize(); + return 0; +} From 92ebfed0a747a3a92bed469bd909f925dd44bc0c Mon Sep 17 00:00:00 2001 From: Vicente Adolfo Bolea Sanchez Date: Tue, 14 Nov 2023 16:41:47 -0500 Subject: [PATCH 6/6] ci: set openmpi and openmp params --- scripts/ci/cmake/ci-el8-icc-ompi.cmake | 7 ------- scripts/ci/cmake/ci-el8-oneapi-ompi.cmake | 7 ------- .../ci/cmake/ci-ubuntu20.04-clang10-ompi.cmake | 7 ------- .../ci/cmake/ci-ubuntu20.04-clang6-ompi.cmake | 7 ------- .../ci-ubuntu20.04-clang6-static-ompi.cmake | 7 ------- .../ci/cmake/ci-ubuntu20.04-gcc10-ompi.cmake | 7 ------- .../ci/cmake/ci-ubuntu20.04-gcc11-ompi.cmake | 7 ------- .../ci/cmake/ci-ubuntu20.04-gcc8-ompi.cmake | 7 ------- .../ci-ubuntu20.04-gcc8-static-ompi.cmake | 7 ------- .../ci/cmake/ci-ubuntu20.04-gcc9-ompi.cmake | 7 ------- scripts/ci/gh-actions/run.sh | 18 ++++++++++++++---- 11 files changed, 14 insertions(+), 74 deletions(-) diff --git a/scripts/ci/cmake/ci-el8-icc-ompi.cmake b/scripts/ci/cmake/ci-el8-icc-ompi.cmake index c191d79c26..6f8803e44d 100644 --- a/scripts/ci/cmake/ci-el8-icc-ompi.cmake +++ b/scripts/ci/cmake/ci-el8-icc-ompi.cmake @@ -1,7 +1,3 @@ -include(ProcessorCount) -ProcessorCount(NCPUS) -math(EXPR N2CPUS "${NCPUS}*2") - set(ENV{CC} icc) set(ENV{CXX} icpc) set(ENV{FC} ifort) @@ -28,9 +24,6 @@ CMAKE_C_FLAGS:STRING=-Wall -diag-disable=10441 CMAKE_C_FLAGS_DEBUG:STRING=-g -O0 CMAKE_CXX_FLAGS:STRING=-Wall -diag-disable=10441 CMAKE_CXX_FLAGS_DEBUG:STRING=-g -O0 - -MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe -MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS} ") set(CTEST_CMAKE_GENERATOR "Unix Makefiles") diff --git a/scripts/ci/cmake/ci-el8-oneapi-ompi.cmake b/scripts/ci/cmake/ci-el8-oneapi-ompi.cmake index 7a637d1888..8a1c5e5fa9 100644 --- a/scripts/ci/cmake/ci-el8-oneapi-ompi.cmake +++ b/scripts/ci/cmake/ci-el8-oneapi-ompi.cmake @@ -1,7 +1,3 @@ -include(ProcessorCount) -ProcessorCount(NCPUS) -math(EXPR N2CPUS "${NCPUS}*2") - set(ENV{CC} icx) set(ENV{CXX} icpx) set(ENV{FC} ifort) # oneapi fortran compiler currently has issues @@ -29,9 +25,6 @@ CMAKE_C_FLAGS_DEBUG:STRING=-g -O0 CMAKE_CXX_FLAGS:STRING=-Wall CMAKE_CXX_FLAGS_DEBUG:STRING=-g -O0 CMAKE_Fortran_FLAGS:STRING=-W1 - -MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe -MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS} ") set(CTEST_CMAKE_GENERATOR "Unix Makefiles") diff --git a/scripts/ci/cmake/ci-ubuntu20.04-clang10-ompi.cmake b/scripts/ci/cmake/ci-ubuntu20.04-clang10-ompi.cmake index e62ed9b57a..447c304316 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-clang10-ompi.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-clang10-ompi.cmake @@ -1,7 +1,3 @@ -include(ProcessorCount) -ProcessorCount(NCPUS) -math(EXPR N2CPUS "${NCPUS}*2") - set(ENV{CC} clang-10) set(ENV{CXX} clang++-10) set(ENV{FC} gfortran-11) @@ -35,9 +31,6 @@ CMAKE_CXX_COMPILER_LAUNCHER=ccache CMAKE_C_FLAGS:STRING=-Wall CMAKE_CXX_FLAGS:STRING=-Wall CMAKE_Fortran_FLAGS:STRING=-Wall - -MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe -MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS} ") set(CTEST_CMAKE_GENERATOR "Ninja") diff --git a/scripts/ci/cmake/ci-ubuntu20.04-clang6-ompi.cmake b/scripts/ci/cmake/ci-ubuntu20.04-clang6-ompi.cmake index 2bdf2c410d..bfe89a699c 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-clang6-ompi.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-clang6-ompi.cmake @@ -1,7 +1,3 @@ -include(ProcessorCount) -ProcessorCount(NCPUS) -math(EXPR N2CPUS "${NCPUS}*2") - set(ENV{CC} clang-6.0) set(ENV{CXX} clang++-6.0) set(ENV{FC} gfortran-11) @@ -36,9 +32,6 @@ CMAKE_CXX_COMPILER_LAUNCHER=ccache CMAKE_C_FLAGS:STRING=-Wall CMAKE_CXX_FLAGS:STRING=-Wall CMAKE_Fortran_FLAGS:STRING=-Wall - -MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe -MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS} ") # We have a dedicated build for this setup without MPI diff --git a/scripts/ci/cmake/ci-ubuntu20.04-clang6-static-ompi.cmake b/scripts/ci/cmake/ci-ubuntu20.04-clang6-static-ompi.cmake index 4b338b9c89..71c0f9710e 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-clang6-static-ompi.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-clang6-static-ompi.cmake @@ -1,7 +1,3 @@ -include(ProcessorCount) -ProcessorCount(NCPUS) -math(EXPR N2CPUS "${NCPUS}*2") - set(ENV{CC} clang-6.0) set(ENV{CXX} clang++-6.0) set(ENV{FC} gfortran-11) @@ -37,9 +33,6 @@ CMAKE_CXX_COMPILER_LAUNCHER=ccache CMAKE_C_FLAGS:STRING=-Wall CMAKE_CXX_FLAGS:STRING=-Wall CMAKE_Fortran_FLAGS:STRING=-Wall - -MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe -MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS} ") set(CTEST_CMAKE_GENERATOR "Ninja") diff --git a/scripts/ci/cmake/ci-ubuntu20.04-gcc10-ompi.cmake b/scripts/ci/cmake/ci-ubuntu20.04-gcc10-ompi.cmake index 0f80809145..6463d8f72a 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-gcc10-ompi.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-gcc10-ompi.cmake @@ -1,7 +1,3 @@ -include(ProcessorCount) -ProcessorCount(NCPUS) -math(EXPR N2CPUS "${NCPUS}*2") - set(ENV{CC} gcc) set(ENV{CXX} g++) set(ENV{FC} gfortran) @@ -35,9 +31,6 @@ CMAKE_CXX_COMPILER_LAUNCHER=ccache CMAKE_C_FLAGS:STRING=-Wall CMAKE_CXX_FLAGS:STRING=-Wall CMAKE_Fortran_FLAGS:STRING=-Wall - -MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe -MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS} ") # We have a dedicated build for this setup without MPI diff --git a/scripts/ci/cmake/ci-ubuntu20.04-gcc11-ompi.cmake b/scripts/ci/cmake/ci-ubuntu20.04-gcc11-ompi.cmake index 70cd4b3552..a686bb0c87 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-gcc11-ompi.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-gcc11-ompi.cmake @@ -1,7 +1,3 @@ -include(ProcessorCount) -ProcessorCount(NCPUS) -math(EXPR N2CPUS "${NCPUS}*2") - set(ENV{CC} gcc) set(ENV{CXX} g++) set(ENV{FC} gfortran) @@ -35,9 +31,6 @@ CMAKE_CXX_COMPILER_LAUNCHER=ccache CMAKE_C_FLAGS:STRING=-Wall CMAKE_CXX_FLAGS:STRING=-Wall CMAKE_Fortran_FLAGS:STRING=-Wall - -MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe -MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS} ") set(CTEST_CMAKE_GENERATOR "Ninja") diff --git a/scripts/ci/cmake/ci-ubuntu20.04-gcc8-ompi.cmake b/scripts/ci/cmake/ci-ubuntu20.04-gcc8-ompi.cmake index 70cd4b3552..a686bb0c87 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-gcc8-ompi.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-gcc8-ompi.cmake @@ -1,7 +1,3 @@ -include(ProcessorCount) -ProcessorCount(NCPUS) -math(EXPR N2CPUS "${NCPUS}*2") - set(ENV{CC} gcc) set(ENV{CXX} g++) set(ENV{FC} gfortran) @@ -35,9 +31,6 @@ CMAKE_CXX_COMPILER_LAUNCHER=ccache CMAKE_C_FLAGS:STRING=-Wall CMAKE_CXX_FLAGS:STRING=-Wall CMAKE_Fortran_FLAGS:STRING=-Wall - -MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe -MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS} ") set(CTEST_CMAKE_GENERATOR "Ninja") diff --git a/scripts/ci/cmake/ci-ubuntu20.04-gcc8-static-ompi.cmake b/scripts/ci/cmake/ci-ubuntu20.04-gcc8-static-ompi.cmake index 9f4b991bc8..88ed69ff7a 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-gcc8-static-ompi.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-gcc8-static-ompi.cmake @@ -1,7 +1,3 @@ -include(ProcessorCount) -ProcessorCount(NCPUS) -math(EXPR N2CPUS "${NCPUS}*2") - set(ENV{CC} gcc) set(ENV{CXX} g++) set(ENV{FC} gfortran) @@ -37,9 +33,6 @@ CMAKE_CXX_COMPILER_LAUNCHER=ccache CMAKE_C_FLAGS:STRING=-Wall CMAKE_CXX_FLAGS:STRING=-Wall CMAKE_Fortran_FLAGS:STRING=-Wall - -MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe -MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS} ") set(CTEST_CMAKE_GENERATOR "Ninja") diff --git a/scripts/ci/cmake/ci-ubuntu20.04-gcc9-ompi.cmake b/scripts/ci/cmake/ci-ubuntu20.04-gcc9-ompi.cmake index 70cd4b3552..a686bb0c87 100644 --- a/scripts/ci/cmake/ci-ubuntu20.04-gcc9-ompi.cmake +++ b/scripts/ci/cmake/ci-ubuntu20.04-gcc9-ompi.cmake @@ -1,7 +1,3 @@ -include(ProcessorCount) -ProcessorCount(NCPUS) -math(EXPR N2CPUS "${NCPUS}*2") - set(ENV{CC} gcc) set(ENV{CXX} g++) set(ENV{FC} gfortran) @@ -35,9 +31,6 @@ CMAKE_CXX_COMPILER_LAUNCHER=ccache CMAKE_C_FLAGS:STRING=-Wall CMAKE_CXX_FLAGS:STRING=-Wall CMAKE_Fortran_FLAGS:STRING=-Wall - -MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe -MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS} ") set(CTEST_CMAKE_GENERATOR "Ninja") diff --git a/scripts/ci/gh-actions/run.sh b/scripts/ci/gh-actions/run.sh index 86d12a4b49..5dcfcca7aa 100755 --- a/scripts/ci/gh-actions/run.sh +++ b/scripts/ci/gh-actions/run.sh @@ -72,13 +72,20 @@ mkdir -p "${TMPDIR}" # OpenMPI specific setup and workarounds if [[ "${GH_YML_MATRIX_PARALLEL}" =~ ompi && "${GH_YML_BASE_OS}" != "Windows" ]] then - # Quiet some warnings from OpenMPI - export OMPI_MCA_btl_base_warn_component_unused=0 - export OMPI_MCA_btl_vader_single_copy_mechanism=none + # Enable run as root + export OMPI_ALLOW_RUN_AS_ROOT=1 + export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 # Enable overscription in OpenMPI - export OMPI_MCA_rmaps_base_oversubscribe=1 + export OMPI_MCA_rmaps_base_oversubscribe=true export OMPI_MCA_hwloc_base_binding_policy=none + + # Only use loop interface + export OMPI_MCA_btl_tcp_if_include=lo + + # Quiet some warnings from OpenMPI + export OMPI_MCA_btl_base_warn_component_unused=0 + export OMPI_MCA_btl_vader_single_copy_mechanism=none fi if [[ "${GH_YML_MATRIX_PARALLEL}" =~ ompi ]] @@ -89,6 +96,9 @@ fi # Make sure staging tests use localhost export ADIOS2_IP=127.0.0.1 +# We already paralelize with mpi (MGARD uses OMP) +export OMP_NUM_THREADS=1 + # Load any additional setup scripts if [ -f gha/scripts/ci/setup-run/ci-${GH_YML_JOBNAME}.sh ] then