CMake: add HIP support

.github/workflows/dependencies/dependencies_hip.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+#
+# Copyright 2020 The AMReX Community
+#
+# License: BSD-3-Clause-LBNL
+# Authors: Axel Huebl
+
+# search recursive inside a folder if a file contains tabs
+#
+# @result 0 if no files are found, else 1
+#
+
+set -eu -o pipefail
+
+# Ref.: https://rocmdocs.amd.com/en/latest/Installation_Guide/Installation-Guide.html#ubuntu
+wget -q -O - http://repo.radeon.com/rocm/rocm.gpg.key \
+  | sudo apt-key add -
+echo 'deb [arch=amd64] http://repo.radeon.com/rocm/apt/debian/ xenial main' \
+  | sudo tee /etc/apt/sources.list.d/rocm.list
+
+echo 'export PATH=$PATH:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin' \
+  | sudo tee -a /etc/profile.d/rocm.sh
+# we should not need to export HIP_PATH=/opt/rocm/hip with those installs
+
+sudo apt-get update
+
+# Ref.: https://rocmdocs.amd.com/en/latest/Installation_Guide/Installation-Guide.html#installing-development-packages-for-cross-compilation
+# meta-package: rocm-dkms
+# OpenCL: rocm-opencl
+# other: rocm-dev rocm-utils
+sudo apt-get install -y --no-install-recommends \
+    build-essential \
+    gfortran        \
+    libnuma-dev     \
+    libopenmpi-dev  \
+    openmpi-bin     \
+    rocm-dev
+

.github/workflows/linux.yml

@@ -179,6 +179,29 @@ jobs:
             -DCMAKE_Fortran_COMPILER=$(which gfortran)
         make -j 2 tutorials
 
+  tutorials-hip:
+    name: HIP ROCm@3.7 GFortran@9.3 C++17 [tutorials]
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Dependencies
+      run: .github/workflows/dependencies/dependencies_hip.sh
+    - name: Build & Install
+      run: |
+        source /etc/profile.d/rocm.sh
+        hipcc --version
+        mkdir build
+        cd build
+        cmake ..                                           \
+            -DCMAKE_VERBOSE_MAKEFILE=ON                    \
+            -DENABLE_TUTORIALS=ON                          \
+            -DENABLE_PARTICLES=ON                          \
+            -DENABLE_HIP=ON                                \
+            -DCMAKE_C_COMPILER=$(which hipcc)              \
+            -DCMAKE_CXX_COMPILER=$(which hipcc)            \
+            -DCMAKE_Fortran_COMPILER=$(which gfortran)
+        make -j 2 tutorials
+
   # Build 1D libamrex with configure
   configure-1d:
     name: GNU@7.5 Release [configure 1D]

Src/CMakeLists.txt

@@ -1,5 +1,5 @@
 #
-# Fist, define the library we want to add
+# First, define the library we want to add
 # Think of this as a constructor for object "amrex"
 # In the following, we will use setters to change
 # the properties of this object, like setting the sources,
@@ -178,6 +178,13 @@ if (ENABLE_CUDA)
    setup_target_for_cuda_compilation( amrex )
 endif ()
 
+#
+# If ENABLE_HIP, target 'amrex' requires a special treatment
+#
+if (ENABLE_HIP)
+   setup_target_for_hip_compilation( amrex )
+endif ()
+
 
 #
 # Install amrex  -- Export

Tools/CMake/AMReXParallelBackends.cmake

@@ -64,3 +64,38 @@ if (ENABLE_DPCPP)
    include(AMReXSYCL)
    target_link_libraries(amrex PUBLIC SYCL)
 endif ()
+
+
+#
+#
+# HIP
+#
+#
+if (ENABLE_HIP)
+   if(NOT DEFINED HIP_PATH)
+      if(NOT DEFINED ENV{HIP_PATH})
+         set(HIP_PATH "/opt/rocm/hip" CACHE PATH "Path to which HIP has been installed")
+      else()
+         set(HIP_PATH $ENV{HIP_PATH} CACHE PATH "Path to which HIP has been installed")
+      endif()
+   endif()
+
+   set(CMAKE_MODULE_PATH "${HIP_PATH}/cmake" ${CMAKE_MODULE_PATH})
+
+   find_package(HIP REQUIRED)
+   if("${HIP_COMPILER}" STREQUAL "hcc")
+      message(FATAL_ERROR "Using (deprecated) HCC compiler: please update ROCm")
+   endif()
+
+   if(HIP_FOUND)
+      message(STATUS "Found HIP: ${HIP_VERSION}")
+      message(STATUS "HIP: Platform=${HIP_PLATFORM} Compiler=${HIP_COMPILER}")
+   else()
+      message(FATAL_ERROR "Could not find HIP."
+         " Ensure that HIP is either installed in /opt/rocm/hip or the variable HIP_PATH is set to point to the right location.")
+   endif()
+
+   # Let's put the defines here for the time being
+   target_compile_definitions( amrex PUBLIC AMREX_USE_HIP AMREX_HIP_PLATFORM=${HIP_PLATFORM} )
+
+endif ()

Tools/CMake/AMReXTargetHelpers.cmake

@@ -122,3 +122,40 @@ function (setup_target_for_cuda_compilation _target)
       )
    set_cpp_sources_to_cuda_language(${_target})
 endfunction ()
+
+
+#
+# Create target 'amrex-hip' by "cloning" target 'amrex' and
+# adding HIP specific settings
+#
+function (setup_target_for_hip_compilation _target)
+
+   # Make C++ files with HIP_SOURCE_PROPERTY_FORMAT
+   # This will trigger HIP compilation of those files
+   get_target_property(_sources ${_target} SOURCES)
+   list(FILTER _sources INCLUDE REGEX "\\.cpp")
+   set_source_files_properties(${_sources} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT 1)
+
+   # Separate the sources from the options
+   # HIP_GET_SOURCES_AND_OPTIONS(_sources _cmake_options _hipcc_options _hcc_options _nvcc_options ${ARGN})
+   set(_cmake_options)
+   set(_hipcc_options)
+   set(_hcc_options)
+   set(_nvcc_options)
+   hip_prepare_target_commands(${_target}
+      OBJ _generated_files _source_files ${_sources} ${_cmake_options}
+      HIPCC_OPTIONS ${_hipcc_options}
+      HCC_OPTIONS ${_hcc_options}
+      NVCC_OPTIONS ${_nvcc_options})
+
+   if (_source_files)
+      list(REMOVE_ITEM _sources ${_source_files})
+   endif ()
+
+   # overwrite sources of _target with "new" sources
+   set_target_properties(${_target} PROPERTIES SOURCES "${_generated_files};${_sources}")
+
+   # set linker language
+   set_target_properties(${_target} PROPERTIES LINKER_LANGUAGE ${HIP_C_OR_CXX})
+
+endfunction ()

Tools/CMake/AMReX_Options.cmake

@@ -129,6 +129,10 @@ cmake_dependent_option( ENABLE_CUDA "Enable GPU support via CUDA" OFF
    "NOT ENABLE_DPCPP" OFF)
 print_option( ENABLE_CUDA )
 
+cmake_dependent_option( ENABLE_HIP "Enable GPU support via HIP" OFF
+   "NOT ENABLE_DPCPP;NOT ENABLE_CUDA" OFF)
+print_option( ENABLE_HIP )
+
 option( ENABLE_ACC  "Enable GPU support via OpenACC" OFF )
 print_option( ENABLE_ACC )