Wait all fix

.gitlab-ci.yml

@@ -9,7 +9,12 @@ stages:
 
 perlmutter-no-cache-build:
     stage: build
-    when: manual
+    rules:
+    - if: '$METRICS == null'
+      when: manual
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     allow_failure: false
     tags:
         - perlmutter
@@ -32,7 +37,12 @@ perlmutter-no-cache-build:
 
 perlmutter-cache-build:
     stage: build
-    when: manual
+    rules:
+    - if: '$METRICS == null'
+      when: manual
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     allow_failure: false
     tags:
         - perlmutter
@@ -60,6 +70,9 @@ perlmutter-cache-build:
 
 perlmutter-no-cache-parallel-pdc:
     stage: test
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -80,6 +93,9 @@ perlmutter-no-cache-parallel-pdc:
 
 perlmutter-no-cache-parallel-obj:
     stage: test
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -101,6 +117,9 @@ perlmutter-no-cache-parallel-obj:
 
 perlmutter-no-cache-parallel-cont:
     stage: test
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -122,6 +141,9 @@ perlmutter-no-cache-parallel-cont:
 
 perlmutter-no-cache-parallel-prop:
     stage: test
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -143,6 +165,9 @@ perlmutter-no-cache-parallel-prop:
 
 perlmutter-no-cache-parallel-region:
     stage: test
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -164,6 +189,9 @@ perlmutter-no-cache-parallel-region:
 
 perlmutter-no-cache-parallel-region-all:
     stage: test
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -190,6 +218,9 @@ perlmutter-no-cache-parallel-region-all:
 
 perlmutter-cache-parallel-pdc:
     stage: test
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -210,6 +241,9 @@ perlmutter-cache-parallel-pdc:
 
 perlmutter-cache-parallel-obj:
     stage: test
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -231,6 +265,9 @@ perlmutter-cache-parallel-obj:
 
 perlmutter-cache-parallel-cont:
     stage: test
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -252,6 +289,9 @@ perlmutter-cache-parallel-cont:
 
 perlmutter-cache-parallel-prop:
     stage: test
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -273,6 +313,9 @@ perlmutter-cache-parallel-prop:
 
 perlmutter-cache-parallel-region:
     stage: test
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -294,6 +337,9 @@ perlmutter-cache-parallel-region:
 
 perlmutter-cache-parallel-region-all:
     stage: test
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     rules:
     - if: '$METRICS == null'
     needs:
@@ -313,26 +359,54 @@ perlmutter-cache-parallel-region-all:
         - ctest -L parallel_region_transfer_all
         - rm -rf ${PDC_TMPDIR} ${PDC_DATA_LOC}
 
+perlmutter-metrics-build:
+    stage: build
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
+    rules:
+        - if: '$METRICS == "true"'
+    allow_failure: false
+    tags:
+        - perlmutter
+    variables:
+        SCHEDULER_PARAMETERS: "-A m2621 --qos=debug --constraint=cpu --tasks-per-node=64 -N 1 -t 00:30:00"
+        SUPERCOMPUTER: "perlmutter"
+        MERCURY_DIR: "/global/cfs/cdirs/m2621/pdc-perlmutter/mercury/install"
+    script:
+        - module load libfabric/1.15.2.0
+        - module list
+        - mkdir -p ${PDC_BUILD_PATH}/perlmutter/metrics
+        - cd ${PDC_BUILD_PATH}/perlmutter/metrics
+        - cmake ../../.. -DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DPDC_SERVER_CACHE=ON -DBUILD_TESTING=ON -DCMAKE_INSTALL_PREFIX=$PDC_DIR -DPDC_ENABLE_MPI=ON -DMERCURY_DIR=$MERCURY_DIR -DCMAKE_C_COMPILER=cc -DMPI_RUN_CMD="srun -A m2621 --qos=debug --constraint=cpu --tasks-per-node=64" -DCMAKE_INSTALL_PREFIX=${PDC_INSTALL_PATH}/perlmutter/metrics
+        - make -j
+        - make install
+    artifacts:
+        paths:
+            - ${PDC_BUILD_PATH}/perlmutter/metrics
+            - ${PDC_INSTALL_PATH}/perlmutter/metrics
+
 perlmutter-metrics:
     stage: metrics
+    id_tokens:
+        SITE_ID_TOKEN:
+            aud: https://software.nersc.gov/
     rules:
         - if: '$METRICS == "true"'
     needs:
-        - perlmutter-cache-build
+        - perlmutter-metrics-build
     tags:
         - perlmutter
     variables:
-        PDC_N_NODES: 64
-        PDC_N_CLIENTS: 127
-        SCHEDULER_PARAMETERS: "-A m2621 --qos=regular --constraint=cpu --tasks-per-node=${PDC_N_CLIENTS} -N ${PDC_N_NODES} -t 00:30:00"
+        SCHEDULER_PARAMETERS: "-A m2621 --qos=${PDC_QUEUE} --constraint=cpu --tasks-per-node=${PDC_N_CLIENTS} -N ${PDC_N_NODES} -t 00:30:00"
         SUPERCOMPUTER: "perlmutter"
         MERCURY_DIR: "/global/cfs/cdirs/m2621/pdc-perlmutter/mercury/install"
         PDC_TMPDIR: "${PDC_BUILD_PATH}/pdc-tmp-metrics"
         PDC_DATA_LOC: "${PDC_BUILD_PATH}/pdc-data-metrics"
         PDC_CLIENT_LOOKUP: "NONE"
-        PDC_SERVER: "${PDC_BUILD_PATH}/perlmutter/cache/bin/pdc_server.exe"
-        PDC_SERVER_CLOSE: "${PDC_BUILD_PATH}/perlmutter/cache/bin/close_server"
-        PDC_CLIENT: "${PDC_BUILD_PATH}/perlmutter/cache/bin/vpicio_mts"
+        PDC_SERVER: "${PDC_BUILD_PATH}/perlmutter/metrics/bin/pdc_server.exe"
+        PDC_SERVER_CLOSE: "${PDC_BUILD_PATH}/perlmutter/metrics/bin/close_server"
+        PDC_CLIENT: "${PDC_BUILD_PATH}/perlmutter/metrics/bin/vpicio_mts"
         PDC_JOB_OUTPUT: "pdc-metrics.log"
     script: 
         - hostname

CMakeLists.txt

@@ -380,7 +380,7 @@ endif()
 #-----------------------------------------------------------------------------
 # SERVER CACHE option
 #-----------------------------------------------------------------------------
-option(PDC_SERVER_CACHE "Enable Server Caching." OFF)
+option(PDC_SERVER_CACHE "Enable Server Caching." ON)
 if(PDC_SERVER_CACHE)
   set(PDC_SERVER_CACHE 1)
   set(PDC_SERVER_CACHE_MAX_GB "3" CACHE STRING "Max GB for server cache")

docs/source/getting_started.rst

@@ -125,11 +125,8 @@ Install libfabric
 
 .. attention::
 
-	If you're installing PDC on MacOS, you need to make sure you enable ``sockets``:
-
-	.. code-block: Bash
-
-		./configure CFLAG=-O2 --enable-sockets=yes --enable-tcp=yes --enable-udp=yes --enable-rxm=yes
+	When installing on MacOS, make sure to enable ``sockets`` with the following configure command:
+	``./configure CFLAG=-O2 --enable-sockets=yes --enable-tcp=yes --enable-udp=yes --enable-rxm=yes``
 
 
 Install Mercury
@@ -156,17 +153,13 @@ Install Mercury
 	echo 'export PATH=$MERCURY_DIR/include:$MERCURY_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh
 
 .. note::
+
 	``CC=mpicc`` may need to be changed to the corresponding compiler in your system, e.g. ``-DCMAKE_C_COMPILER=cc`` or ``-DCMAKE_C_COMPILER=gcc``.
 	Make sure the ctest passes. PDC may not work without passing all the tests of Mercury.
 
 .. attention::
 
-	If you're installing PDC on MacOS, for the tests to work you need to specify the protocol used by Mercury:
-
-	.. code-block: Bash
-
-		cmake -DCMAKE_INSTALL_PREFIX=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc -DBUILD_SHARED_LIBS=ON \
-	      -DBUILD_TESTING=ON -DNA_USE_OFI=ON -DNA_USE_SM=OFF -DNA_OFI_TESTING_PROTOCOL=sockets
+	When installing on MacOS, specify the ``sockets`` protocol used by Mercury by replacing the cmake command from ``-DNA_OFI_TESTING_PROTOCOL=tcp`` to ``-DNA_OFI_TESTING_PROTOCOL=sockets``
 
 
 Install PDC
@@ -197,11 +190,7 @@ Install PDC
 	If you are trying to compile PDC on MacOS, ``LibUUID`` needs to be installed on your MacOS first. Simple use ``brew install ossp-uuid`` to install it.
 	If you are trying to compile PDC on Linux, you should also make sure ``LibUUID`` is installed on your system. If not, you can install it with ``sudo apt-get install uuid-dev`` on Ubuntu or ``yum install libuuid-devel`` on CentOS.
 
-	In MacOS you also need to export the following environment variable so PDC (i.e., Mercury) uses the ``socket`` protocol, the only one supported in MacOS:
-
-	.. code-block: Bash
-
-		export HG_TRANSPORT="sockets"
+	In MacOS you also need to export the following environment variable so PDC (i.e., Mercury) uses the ``socket`` protocol, the only one supported in MacOS: ``export HG_TRANSPORT="sockets"``
 
 
 Test Your PDC Installation
@@ -219,6 +208,7 @@ You can also specify a timeout (e.g., 2 minutes) for the tests by specifying the
 	ctest --timeout 120
 
 .. note::
+
 	If you are using PDC on an HPC system, e.g. Perlmutter@NERSC, ``ctest`` should be run on a compute node, you can submit an interactive job on Perlmutter: ``salloc --nodes 1 --qos interactive --time 01:00:00 --constraint cpu --account=mxxxx``
 
 
@@ -232,23 +222,21 @@ But before adding any of your function, we need to enable the Julia support firs
 Prerequisite
 ^^^^^^^^^^^^
 
-Make sure you have Julia-lang installed. You can check with your system administrator to see if you already have Julia-lang installed. If not, you can either ask your system administrator to install it for you or you can install it yourself if permitted. On macOS, the best way to install Julia is via `Homebrew https://brew.sh`_. You may also refer to `Julia Download Page https://julialang.org/downloads/`_ for instructions on installing Julia.
-Once you installed Julia, you can set `JULIA_HOME` to be where Julia-lang is installed.
+Make sure you have Julia-lang installed. You can check with your system administrator to see if you already have Julia-lang installed. If not, you can either ask your system administrator to install it for you or you can install it yourself if permitted. On macOS, the best way to install Julia is via `Homebrew <https://brew.sh>`_. You may also refer to `Julia Download Page <https://julialang.org/downloads/>`_ for instructions on installing Julia.
+Once you installed Julia, you can set ``JULIA_HOME`` to be where Julia-lang is installed.
 
 .. code-block:: Bash
-	export JULIA_HOME=/path/to/julia/install/directory
 
+	export JULIA_HOME=/path/to/julia/install/directory
 
-.. note:: Note on perlmutter:
-	You can easily perform `module load julia` to load the Julia-lang environment. Then, you can do the following to set `$JULIA_HOME`:
+.. note::
 
-	.. code-block:: Bash
-		export JULIA_HOME=$(dirname $(dirname $(which julia)))
+	On Perlmutter You can ``module load julia`` to load the Julia-lang module and set ``$JULIA_HOME`` with: ``export JULIA_HOME=$(dirname $(dirname $(which julia)))``
 
 
 Enabling Julia Support for PDC Tests
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Once the Prerequisite is satisfied, you can enable Julia support by adding `--DPDC_ENABLE_JULIA_SUPPORT=ON` to your cmake command and re-run it. 
+Once the Prerequisite is satisfied, you can enable Julia support by adding ``--DPDC_ENABLE_JULIA_SUPPORT=ON`` to your cmake command and re-run it. 
 Then you can compile your PDC project with Julia support. 
 
 Now, see Developer Notes to know how you can add your own Julia functions to enhance your test cases in PDC. 
@@ -257,9 +245,10 @@ Now, see Developer Notes to know how you can add your own Julia functions to enh
 Build PDC in a Docker Container
 ---------------------------------------------------
 Simply run the following command from the project root directory to build PDC in a Docker container:
+
 .. code-block:: Bash
-	.docker/run_dev_base.sh
 
+	.docker/run_dev_base.sh
 
 Build PDC in Github Codespace
 ---------------------------------------------------

examples/Makefile

@@ -4,7 +4,7 @@ CFLAGS = -O2 -Wall -Wextra
 INCLUDES = -I. -I$(PDC_DIR)/include
 LDFLAGS = -Wl,-rpath,$(PDC_DIR)/lib -L$(PDC_DIR)/lib
 
-LIBS = -lpdc
+LIBS = -lpdc -lpdc_commons
 
 .c.o:
 	$(CC) $(CFLAGS) $(INCLUDES) -c $<