More robust build-catalogue

CMakeLists.txt

@@ -77,10 +77,8 @@ option(ARB_WITH_ASSERTIONS "enable arb_assert() assertions in code" OFF)
 # NeuroML support library:
 #----------------------------------------------------------
 
-option(ARB_WITH_NEUROML  "build NeuroML support library" OFF)
+option(ARB_WITH_NEUROML "build NeuroML support library" OFF)
 
-set(ARB_LIBXML "" CACHE STRING "path to XML library to link, use with care")
-mark_as_advanced(ARB_LIBXML)
 #----------------------------------------------------------
 # Python front end for Arbor:
 #----------------------------------------------------------

arborio/CMakeLists.txt

@@ -12,10 +12,6 @@ if(ARB_WITH_NEUROML)
             nml_parse_morphology.cpp
             xml.cpp
             xmlwrap.cpp)
-    if(NOT ${ARB_LIBXML} STREQUAL "")
-        set(LIBXML2_LIBRARY ${ARB_LIBXML})
-    endif()
-    message("-lxml = ${ARB_LIBXML}")
     find_package(LibXml2 REQUIRED)
 endif()
 

doc/install/python.rst

@@ -7,9 +7,7 @@ Arbor's Python API will be the most convenient interface for most users.
 
 .. note::
     Arbor requires Python version 3.6 and later. It is advised that you update `pip` as well.
-
-.. note::
-   We assume ``pip`` is used to install Arbor. See later for using ``setup.py``.
+    We strongly encourage ``pip`` is used to install Arbor.
 
 Getting Arbor
 -------------
@@ -74,78 +72,97 @@ Every time you make changes to the code, you'll have to repeat the second step.
 Advanced options
 ^^^^^^^^^^^^^^^^
 
-.. note:: When installing via ``setup.py install`` directly instead of using
-   ``pip install .``, you will have to install the build-time dependencies
-   listed in ``pyproject.toml`` on your own.
+By default Arbor is installed with multi-threading enabled. To enable more
+advanced forms of parallelism and other features, Arbor comes with a few
+compilation options. These are of the form ``-D<KEY>=<VALUE>``, must be
+separated from the actual ``pip`` invocation by a double dash ``--`` and can be
+used on both local (``pip3 install ./arbor``) and remote (``pip3 install
+arbor``) copies of Arbor.
+
+.. Note::
+
+   ``pip`` compiles the Arbor C++ library and wrapper, as well as dependencies
+    you might not have had installed yet (e.g. ``numpy``). It may take a few
+    minutes. Pass the ``--verbose`` flag to pip to see the individual steps
+    being performed if you are concerned that progress is halting.
+
+    If you had Arbor installed already, you may need to remove it first before
+    you can (re)compile it with the flags you need.
+
+The following flags can be used to configure the installation:
+
+* ``-DARB_WITH_NEUROML=<ON|OFF>``: Enable support for NeuroML2 morphologies,
+  requires ``libxml2`` library. Default ``OFF``
+* ``-DARB_WITH__MPI=<ON|OFF>``: Enable MPI support, requires MPI library.
+  Default ``OF``.
+* ``-DARB_GPU=<none|cuda|cuda-clang|hip>``: Enable GPU support for NVIDIA GPUs
+  with nvcc using ``cuda``, or with clang using ``cuda-clang`` (both require
+  cudaruntime). Enable GPU support for AMD GPUs with hipcc using ``hip``. By
+  default set to ``none``, which disables GPU support.
+* ``-DARB_VECTORIZE=<ON|OFF>``: Enable vectorization. The architecture argument,
+  documented below, may also have to be set appropriately to generated
+  vectorized code. See :ref:`install-architecture` for details.
+* ``-DARB_ARCH=<native|*>``: CPU micro-architecture to target. The advised
+  default is ``native``. See `here
+  <https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html>`_ for a full list of
+  options.
+
+Finally, some flags must be be passed after a second set of ``--``
 
-By default Arbor is installed with multi-threading enabled. To enable more advanced forms of parallelism,
-Arbor comes with a few compilation options. These can be used on both local (``pip3 install ./arbor``) and
-remote (``pip3 install arbor``) copies of Arbor. Below we assume you are working off a local copy.
+* ``-j N``: Use ``N`` processes for compilation, used to speed up builds.
 
-The following optional flags can be used to configure the installation:
+.. note::
+
+   There are more, advanced flags that can be set. We are using ``sk-build`` and
+   ``CMake`` under the hood, so all flags and options valid in ``CMake`` can be
+   used in this fashion.
+
+   Detailed instructions on how to install using CMake are in the :ref:`Python
+   configuration <install-python>` section of the :ref:`installation guide
+   <in_build_install>`. CMake is recommended if you need more control over
+   compilation and installation, plan to use Arbor with C++, or if you are
+   integrating with package managers such as Spack and EasyBuild.
 
-* ``--mpi``: Enable MPI support (requires MPI library).
-* ``--gpu``: Enable GPU support for NVIDIA GPUs with nvcc using ``cuda``, or with clang using ``cuda-clang`` (both require cudaruntime).
-  Enable GPU support for AMD GPUs with hipcc using ``hip``. By default set to ``none``, which disables gpu support.
-* ``--vec``: Enable vectorization. The ``--arch`` argument, documented below, may also have to be set appropriately to generated vectorized code.
-  See :ref:`install-architecture` for details.
-* ``--arch``: CPU micro-architecture to target. The advised default is ``native``.
-  See `here <https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html>`_ for a full list of options.
-* ``-j``: Specify the amount of concurrent jobs for building the project with for faster build times on multicore systems. By default set to ``2``.
+In the examples below we assume you are working off a local copy.
 
 **Vanilla install** with no additional features enabled:
 
 .. code-block:: bash
 
-    pip3 install arbor
+    pip3 install ./arbor
 
 **With MPI support**. This might require loading an MPI module or setting the ``CC`` and ``CXX``
 :ref:`environment variables <install-mpi>`:
 
 .. code-block:: bash
 
-    pip3 install --install-option='--mpi' ./arbor
+    pip3 install ./arbor -- -DARB_WITH_MPI=ON
 
 **Compile with** :ref:`vectorization <install-vectorize>` on a system with a SkyLake
 :ref:`architecture <install-architecture>`:
 
 .. code-block:: bash
 
-    pip3 install --install-option='--vec' --install-option='--arch=skylake' arbor
+    pip3 install ./arbor -- -DARB_VECTORIZE=ON -DARB_ARCH=skylake
 
 **Enable NVIDIA GPUs (compiled with nvcc)**. This requires the :ref:`CUDA toolkit <install-gpu>`:
 
 .. code-block:: bash
 
-    pip3 install --install-option='--gpu=cuda' ./arbor
+    pip3 install ./arbor -- -DARB_GPU=cuda
 
 **Enable NVIDIA GPUs (compiled with clang)**. This also requires the :ref:`CUDA toolkit <install-gpu>`:
 
 .. code-block:: bash
 
-    pip3 install --install-option='--gpu=cuda-clang' ./arbor
+    pip3 install ./arbor -- -DARB_GPU=cuda-clang
 
 **Enable AMD GPUs (compiled with hipcc)**. This requires setting the ``CC`` and ``CXX``
 :ref:`environment variables <install-gpu>`
 
 .. code-block:: bash
 
-    pip3 install --install-option='--gpu=hip' ./arbor
-
-.. Note::
-    Setuptools compiles the Arbor C++ library and wrapper, as well as dependencies you did not have installed
-    yet (e.g. `numpy`). It may take a few minutes. Pass the ``--verbose`` flag to pip
-    to see the individual steps being performed if you are concerned that progress
-    is halting.
-
-    If you had Arbor installed already, you may need to remove it first before you can (re)compile
-    it with the flags you need.
-
-.. Note::
-    Detailed instructions on how to install using CMake are in the
-    :ref:`Python configuration <install-python>` section of the :ref:`installation guide <in_build_install>`.
-    CMake is recommended if you need more control over compilation and installation, plan to use Arbor with C++,
-    or if you are integrating with package managers such as Spack and EasyBuild.
+    pip3 install ./arbor -- -DARB_GPU=hip
 
 Dependencies
 ^^^^^^^^^^^^
@@ -158,8 +175,8 @@ with MPI support would add the following to its requirements:
 
 .. code-block:: python
 
-    arbor >= 0.3 --install-option='--gpu=cuda' \
-                 --install-option='--mpi'
+    TODO: Does this work?
+    arbor >= 0.3 -- -DARB_GPU='cuda' -DARB_WITH_MPI=ON
 
 Note on performance
 -------------------

doc/internals/extending_catalogues.rst

@@ -12,7 +12,7 @@ all likelihood interested in the former.
    If you are coming from NEURON and looking for the equivalent of
    ``nrnivmodl``, please read on :ref:`here <mechanisms_dynamic>`.
 
-   Following this path is for developers rather than end-users.
+   Following the below instructions is for developers rather than end-users.
 
 This requires a copy of the Arbor source tree and the compiler toolchain used to
 build Arbor in addition to the installed library. Following these steps will
@@ -21,7 +21,10 @@ produce a catalogue of the same level of integration as the built-in catalogues
 
 1. Go to the Arbor source tree.
 2. Create a new directory under *mechanisms*.
-3. Add your .mod files.
+
+   1. Add any ``.mod`` files you wish to integrate.
+   2. Add any raw C++ files to be included in the catalogue.
+
 4. Edit *mechanisms/CMakeLists.txt* to add a definition like this (example from
    *default* catalogue)
 
@@ -31,8 +34,8 @@ produce a catalogue of the same level of integration as the built-in catalogues
        NAME default                                                   # Name of your catalogue
        SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/default"                  # Directory name (added above)
        OUTPUT "CAT_DEFAULT_SOURCES"                                   # Variable name to store C++ files into (see below)
-       MECHS exp2syn expsyn expsyn_stdp hh kamt kdrmt nax nernst pas  # Space separated list of mechanisms w/o .mod suffix.
-       EXTRA                                                          # Not needed, see below.
+       MOD exp2syn expsyn expsyn_stdp hh kamt kdrmt nax nernst pas    # Space separated list of NMODL mechanism names
+       CXX                                                            # Space separated list of raw C++ mechanism names
        PREFIX "${PROJECT_SOURCE_DIR}/mechanisms"                      # where does 'generate_catalogue' live, do not change
        STANDALONE FALSE                                               # build as shared object, must be OFF
        VERBOSE OFF)                                                   # Print debug info at configuration time
@@ -50,9 +53,10 @@ produce a catalogue of the same level of integration as the built-in catalogues
 6. Add a ``global_NAME_catalogue`` function in ``mechcat.hpp`` and ``mechcat.cpp``
 7. Bind this function in ``python/mechanisms.cpp``.
 
-All steps can be more or less copied from the surrounding code.
+All steps can be directly adapted from the surrounding code.
 
 .. note::
 
-   If you are sure of your understanding you can write mechanisms in C/C++
-   directly against Arbor's ABI. See :ref:`here <mechanisms_dynamic>` for more.
+   If you have special requirements, you can write mechanisms in C/C++ directly
+   against Arbor's ABI. These need to adhere to the calling convention of the
+   ABI. See :ref:`here <abi_raw>` for more.

doc/internals/mechanism_abi.rst

@@ -457,6 +457,7 @@ supported. The naming scheme is shown in the example below
 
 Writing Mechanisms Directly Against the ABI
 -------------------------------------------
+.. _abi_raw:
 
 .. warning::
 
@@ -489,19 +490,16 @@ For this to compile, the following must be upheld:
 - For each mechanism ``M`` these files must be present
 
   - ``M.hpp`` must define the mechanism metadata and declare the used interfaces.
-
-  - ``M_cpu.cpp`` must define the CPU interface.
+  - ``M_cpu.cpp`` must define the CPU interface. (You can disable this for raw
+    mechanisms by passing ``-C``.)
 
   - If GPU support is used
 
     - ``M_gpu.cpp`` must define the GPU interface.
-
     - ``M_gpu.cu``  must define CUDA kernels.
 
 - The interface names must adhere to the chosen catalogue name, eg here ``make_arb_my_catalogue_B_interface_multicore();``
 
   - names may only contain alphanumeric characters and underscores.
-
   - names must not contain multiple successive underscores.
-
-  - in general, a valid C variable name should be used.
+  - in general, a valid ``C++`` variable name should be used.

mechanisms/BuildModules.cmake

@@ -58,7 +58,7 @@ function(build_modules)
 endfunction()
 
 function("make_catalogue")
-  cmake_parse_arguments(MK_CAT "" "NAME;SOURCES;OUTPUT;PREFIX;STANDALONE;VERBOSE" "CXX_FLAGS_TARGET;MECHS;EXTRA" ${ARGN})
+  cmake_parse_arguments(MK_CAT "" "NAME;SOURCES;OUTPUT;PREFIX;STANDALONE;VERBOSE" "CXX_FLAGS_TARGET;MOD;CXX" ${ARGN})
   set(MK_CAT_OUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated/${MK_CAT_NAME}")
 
   # Need to set ARB_WITH_EXTERNAL_MODCC *and* modcc
@@ -69,8 +69,8 @@ function("make_catalogue")
 
   if(MK_CAT_VERBOSE)
     message("Catalogue name:       ${MK_CAT_NAME}")
-    message("Catalogue mechanisms: ${MK_CAT_MECHS}")
-    message("Extra cxx files:      ${MK_CAT_EXTRA}")
+    message("Catalogue mechanisms: ${MK_CAT_MOD}")
+    message("Extra cxx files:      ${MK_CAT_CXX}")
     message("Catalogue sources:    ${MK_CAT_SOURCES}")
     message("Catalogue output:     ${MK_CAT_OUT_DIR}")
     message("Build as standalone:  ${MK_CAT_STANDALONE}")
@@ -87,7 +87,7 @@ function("make_catalogue")
   endif()
 
   build_modules(
-    ${MK_CAT_MECHS}
+    ${MK_CAT_MOD}
     SOURCE_DIR "${MK_CAT_SOURCES}"
     DEST_DIR "${MK_CAT_OUT_DIR}"
     ${external_modcc} # NB: expands to 'MODCC <binary>' to add an optional argument
@@ -103,21 +103,21 @@ function("make_catalogue")
 
   add_custom_command(
     OUTPUT  ${catalogue_${MK_CAT_NAME}_source}
-    COMMAND ${MK_CAT_PREFIX}/generate_catalogue ${catalogue_${MK_CAT_NAME}_options} ${MK_CAT_MECHS} ${MK_CAT_EXTRA}
+    COMMAND ${MK_CAT_PREFIX}/generate_catalogue ${catalogue_${MK_CAT_NAME}_options} ${MK_CAT_MOD} ${MK_CAT_CXX}
     COMMENT "Building catalogue ${MK_CAT_NAME}"
     DEPENDS ${MK_CAT_PREFIX}/generate_catalogue)
 
   add_custom_target(${MK_CAT_NAME}_catalogue_cpp_target DEPENDS ${catalogue_${MK_CAT_NAME}_source})
   add_dependencies(build_catalogue_${MK_CAT_NAME}_mods ${MK_CAT_NAME}_catalogue_cpp_target)
   add_dependencies(build_all_mods build_catalogue_${MK_CAT_NAME}_mods)
 
-  foreach(mech ${MK_CAT_MECHS})
+  foreach(mech ${MK_CAT_MOD})
     list(APPEND catalogue_${MK_CAT_NAME}_source ${MK_CAT_OUT_DIR}/${mech}_cpu.cpp)
     if(ARB_WITH_GPU)
       list(APPEND catalogue_${MK_CAT_NAME}_source ${MK_CAT_OUT_DIR}/${mech}_gpu.cpp ${MK_CAT_OUT_DIR}/${mech}_gpu.cu)
     endif()
   endforeach()
-  foreach(mech ${MK_CAT_EXTRA})
+  foreach(mech ${MK_CAT_CXX})
     list(APPEND catalogue_${MK_CAT_NAME}_source ${MK_CAT_OUT_DIR}/${mech}_cpu.cpp)
     if(ARB_WITH_GPU)
       list(APPEND catalogue_${MK_CAT_NAME}_source ${MK_CAT_OUT_DIR}/${mech}_gpu.cpp ${MK_CAT_OUT_DIR}/${mech}_gpu.cu)

mechanisms/CMakeLists.txt

@@ -5,8 +5,8 @@ make_catalogue(
   NAME bbp
   SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/bbp"
   OUTPUT "CAT_BBP_SOURCES"
-  MECHS CaDynamics_E2 Ca_HVA Ca_LVAst Ih Im K_Pst K_Tst Nap_Et2 NaTa_t NaTs2_t SK_E2 SKv3_1
-  EXTRA
+  MOD CaDynamics_E2 Ca_HVA Ca_LVAst Ih Im K_Pst K_Tst Nap_Et2 NaTa_t NaTs2_t SK_E2 SKv3_1
+  CXX
   PREFIX "${PROJECT_SOURCE_DIR}/mechanisms"
   CXX_FLAGS_TARGET "${ARB_CXX_FLAGS_TARGET_FULL}"
   STANDALONE FALSE
@@ -16,8 +16,8 @@ make_catalogue(
   NAME allen
   SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/allen"
   OUTPUT "CAT_ALLEN_SOURCES"
-  MECHS CaDynamics Ca_HVA Ca_LVA Ih Im Im_v2 K_P K_T Kd Kv2like Kv3_1 NaTa NaTs NaV Nap SK
-  EXTRA
+  MOD CaDynamics Ca_HVA Ca_LVA Ih Im Im_v2 K_P K_T Kd Kv2like Kv3_1 NaTa NaTs NaV Nap SK
+  CXX
   PREFIX "${PROJECT_SOURCE_DIR}/mechanisms"
   CXX_FLAGS_TARGET "${ARB_CXX_FLAGS_TARGET_FULL}"
   STANDALONE FALSE
@@ -27,8 +27,8 @@ make_catalogue(
   NAME default
   SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/default"
   OUTPUT "CAT_DEFAULT_SOURCES"
-  MECHS exp2syn expsyn expsyn_stdp hh kamt kdrmt nax nernst pas gj
-  EXTRA
+  MOD exp2syn expsyn expsyn_stdp hh kamt kdrmt nax nernst pas gj
+  CXX
   PREFIX "${PROJECT_SOURCE_DIR}/mechanisms"
   CXX_FLAGS_TARGET "${ARB_CXX_FLAGS_TARGET_FULL}"
   STANDALONE FALSE

pyproject.toml

@@ -1,2 +1,3 @@
 [build-system]
 requires = ["setuptools", "wheel", "scikit-build", "cmake", "ninja"]
+description-file = "python/readme.md"