From fc920975e77ca4616d208020f1fcd55eaa5369b8 Mon Sep 17 00:00:00 2001
From: edwardhartnett <edwardjameshartnett@gmail.com>
Date: Mon, 9 Sep 2019 05:19:33 -0600
Subject: [PATCH 1/7] adding new ncint async test

---
 tests/ncint/Makefile.am       |   3 +-
 tests/ncint/run_tests.sh      |   2 +-
 tests/ncint/tst_async_multi.c | 157 ++++++++++++++++++++++++++++++++++
 tests/ncint/tst_pio_async.c   |   3 +
 tests/ncint/tst_pio_udf.c     |   3 +
 5 files changed, 166 insertions(+), 2 deletions(-)
 create mode 100644 tests/ncint/tst_async_multi.c

diff --git a/tests/ncint/Makefile.am b/tests/ncint/Makefile.am
index 4de93c50c49..d4b98126e3c 100644
--- a/tests/ncint/Makefile.am
+++ b/tests/ncint/Makefile.am
@@ -8,10 +8,11 @@ AM_CPPFLAGS = -I$(top_srcdir)/src/clib
 LDADD = ${top_builddir}/src/clib/libpioc.la
 
 # Build the test for make check.
-check_PROGRAMS = tst_pio_udf tst_pio_async
+check_PROGRAMS = tst_pio_udf tst_pio_async tst_async_multi
 
 tst_pio_udf_SOURCES = tst_pio_udf.c pio_err_macros.h
 tst_pio_async_SOURCES = tst_pio_async.c pio_err_macros.h
+tst_async_multi_SOURCES = tst_async_multi.c pio_err_macros.h
 
 if RUN_TESTS
 # Tests will run from a bash script.
diff --git a/tests/ncint/run_tests.sh b/tests/ncint/run_tests.sh
index 1cbfebc6633..c21727ce557 100755
--- a/tests/ncint/run_tests.sh
+++ b/tests/ncint/run_tests.sh
@@ -10,7 +10,7 @@ trap exit INT TERM
 
 printf 'running PIO tests...\n'
 
-PIO_TESTS='tst_pio_udf tst_pio_async'
+PIO_TESTS='tst_pio_udf tst_pio_async tst_async_multi'
 
 success1=true
 success2=true
diff --git a/tests/ncint/tst_async_multi.c b/tests/ncint/tst_async_multi.c
new file mode 100644
index 00000000000..57318742269
--- /dev/null
+++ b/tests/ncint/tst_async_multi.c
@@ -0,0 +1,157 @@
+/* Test netcdf integration layer.
+
+   Ed Hartnett
+*/
+
+#include "config.h"
+#include <pio.h>
+#include "pio_err_macros.h"
+
+#define FILE_NAME "tst_async_multi.nc"
+#define VAR_NAME "data_var"
+#define DIM_NAME_UNLIMITED "dim_unlimited"
+#define DIM_NAME_X "dim_x"
+#define DIM_NAME_Y "dim_y"
+#define DIM_LEN_X 3
+#define DIM_LEN_Y 4
+#define NDIM2 2
+#define NDIM3 3
+
+extern NC_Dispatch NCINT_dispatcher;
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+int
+main(int argc, char **argv)
+{
+    int my_rank;
+    int ntasks;
+
+    /* Initialize MPI. */
+    if (MPI_Init(&argc, &argv)) PERR;
+
+    /* Learn my rank and the total number of processors. */
+    if (MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)) PERR;
+    if (MPI_Comm_size(MPI_COMM_WORLD, &ntasks)) PERR;
+
+    if (!my_rank)
+        printf("\n*** Testing netCDF integration layer.\n");
+    if (!my_rank)
+        printf("*** testing simple async use of netCDF integration layer...");
+    {
+        int ncid, ioid;
+        int dimid[NDIM3], varid;
+        int dimlen[NDIM3] = {NC_UNLIMITED, DIM_LEN_X, DIM_LEN_Y};
+        int iosysid;
+        size_t elements_per_pe;
+        size_t *compdof; /* The decomposition mapping. */
+        int *my_data;
+        int *data_in;
+        int num_procs2[COMPONENT_COUNT] = {3};
+        int num_io_procs = 1;
+        int i;
+
+        /* Turn on logging for PIO library. */
+        /* PIOc_set_log_level(4); */
+        /* if (!my_rank) */
+        /*     nc_set_log_level(3); */
+
+        /* Initialize the intracomm. The IO task will not return from
+         * this call until the PIOc_finalize() is called by the
+         * compute tasks. */
+        if (nc_def_async(MPI_COMM_WORLD, num_io_procs, NULL, COMPONENT_COUNT,
+                         num_procs2, NULL, NULL, NULL, PIO_REARR_BOX, &iosysid))
+            PERR;
+
+        if (my_rank)
+        {
+            /* Create a file with a 3D record var. */
+            if (nc_create(FILE_NAME, NC_PIO|NC_NETCDF4, &ncid)) PERR;
+            if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR;
+            if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR;
+            if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR;
+            if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM3, dimid, &varid)) PERR;
+            if (nc_enddef(ncid)) PERR;
+
+            /* Calculate a decomposition for distributed arrays. */
+            elements_per_pe = DIM_LEN_X * DIM_LEN_Y / (ntasks - num_io_procs);
+            /* printf("my_rank %d elements_per_pe %ld\n", my_rank, elements_per_pe); */
+            if (!(compdof = malloc(elements_per_pe * sizeof(size_t))))
+                PERR;
+            for (i = 0; i < elements_per_pe; i++)
+            {
+                compdof[i] = (my_rank - num_io_procs) * elements_per_pe + i;
+                /* printf("my_rank %d compdof[%d]=%ld\n", my_rank, i, compdof[i]); */
+            }
+
+            /* Create the PIO decomposition for this test. */
+            if (nc_def_decomp(iosysid, PIO_INT, NDIM2, &dimlen[1], elements_per_pe,
+                              compdof, &ioid, 1, NULL, NULL)) PERR;
+            free(compdof);
+
+            /* Create some data on this processor. */
+            if (!(my_data = malloc(elements_per_pe * sizeof(int)))) PERR;
+            for (i = 0; i < elements_per_pe; i++)
+                my_data[i] = my_rank * 10 + i;
+
+            /* Write some data with distributed arrays. */
+            if (nc_put_vard_int(ncid, varid, ioid, 0, my_data)) PERR;
+            if (nc_close(ncid)) PERR;
+
+            /* Reopen the file using netCDF integration. */
+            {
+                int ndims, nvars, ngatts, unlimdimid;
+                nc_type xtype_in;
+                char var_name_in[NC_MAX_NAME + 1];
+                char dim_name_in[NC_MAX_NAME + 1];
+                int natts_in;
+                int dimids_in[NDIM3];
+                size_t dim_len_in;
+
+                /* Open the file. */
+                if (nc_open(FILE_NAME, NC_PIO, &ncid)) PERR;
+
+                /* Check the file. */
+                if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) PERR;
+                if (ndims != 3 || nvars != 1 || ngatts != 0 ||
+                    unlimdimid != 0) PERR;
+                if (nc_inq_var(ncid, 0, var_name_in, &xtype_in, &ndims,
+                               dimids_in, &natts_in)) PERR;
+                if (strcmp(var_name_in, VAR_NAME) || xtype_in != NC_INT || ndims != NDIM3
+                    || dimids_in[0] != 0 || dimids_in[1] != 1 || dimids_in[2] != 2 ||
+                    natts_in != 0) PERR;
+                if (nc_inq_dim(ncid, 0, dim_name_in, &dim_len_in)) PERR;
+                if (strcmp(dim_name_in, DIM_NAME_UNLIMITED) || dim_len_in != 1) PERR;
+                if (nc_inq_dim(ncid, 1, dim_name_in, &dim_len_in)) PERR;
+                if (strcmp(dim_name_in, DIM_NAME_X) || dim_len_in != DIM_LEN_X) PERR;
+                if (nc_inq_dim(ncid, 2, dim_name_in, &dim_len_in)) PERR;
+                if (strcmp(dim_name_in, DIM_NAME_Y) || dim_len_in != DIM_LEN_Y) PERR;
+
+                /* Read distributed arrays. */
+                if (!(data_in = malloc(elements_per_pe * sizeof(int)))) PERR;
+                if (nc_get_vard_int(ncid, varid, ioid, 0, data_in)) PERR;
+
+                /* Check results. */
+                for (i = 0; i < elements_per_pe; i++)
+                    if (data_in[i] != my_data[i]) PERR;
+
+                /* Close file. */
+                if (nc_close(ncid)) PERR;
+
+                /* Free resources. */
+                free(data_in);
+            }
+
+            free(my_data);
+            if (nc_free_decomp(ioid)) PERR;
+            if (nc_free_iosystem(iosysid)) PERR;
+        }
+    }
+    if (!my_rank)
+        PSUMMARIZE_ERR;
+
+    /* Finalize MPI. */
+    MPI_Finalize();
+    PFINAL_RESULTS;
+}
diff --git a/tests/ncint/tst_pio_async.c b/tests/ncint/tst_pio_async.c
index 795a887c84e..23f09537b63 100644
--- a/tests/ncint/tst_pio_async.c
+++ b/tests/ncint/tst_pio_async.c
@@ -1,5 +1,8 @@
 /* Test netcdf integration layer.
 
+   This is a very simple test of async mode in PIO, using the netCDF
+   integration layer.
+
    Ed Hartnett
 */
 
diff --git a/tests/ncint/tst_pio_udf.c b/tests/ncint/tst_pio_udf.c
index 2c452197330..2e40fe4b04b 100644
--- a/tests/ncint/tst_pio_udf.c
+++ b/tests/ncint/tst_pio_udf.c
@@ -1,5 +1,8 @@
 /* Test netcdf integration layer.
 
+   This is a very simple test for PIO in intercomm mode, using the
+   netCDF integration layer.
+
    Ed Hartnett
 */
 

From 94cdb68ee52d840bd762e921a1c0ce69b0712c49 Mon Sep 17 00:00:00 2001
From: edwardhartnett <edwardjameshartnett@gmail.com>
Date: Mon, 9 Sep 2019 05:29:53 -0600
Subject: [PATCH 2/7] moving code to function so that multiple computation
 units can be handled

---
 tests/ncint/tst_async_multi.c | 194 +++++++++++++++++++---------------
 1 file changed, 106 insertions(+), 88 deletions(-)

diff --git a/tests/ncint/tst_async_multi.c b/tests/ncint/tst_async_multi.c
index 57318742269..0307fc44b50 100644
--- a/tests/ncint/tst_async_multi.c
+++ b/tests/ncint/tst_async_multi.c
@@ -1,5 +1,8 @@
 /* Test netcdf integration layer.
 
+   This tests that multiple computation units can work in async mode,
+   using the netCDF integration layer.
+
    Ed Hartnett
 */
 
@@ -7,7 +10,7 @@
 #include <pio.h>
 #include "pio_err_macros.h"
 
-#define FILE_NAME "tst_async_multi.nc"
+#define TEST_NAME "tst_async_multi"
 #define VAR_NAME "data_var"
 #define DIM_NAME_UNLIMITED "dim_unlimited"
 #define DIM_NAME_X "dim_x"
@@ -22,6 +25,105 @@ extern NC_Dispatch NCINT_dispatcher;
 /* Number of computational components to create. */
 #define COMPONENT_COUNT 1
 
+/* Create a file with one 2D var. */
+int
+create_file_0(int my_rank, int ntasks, int num_io_procs, int iosysid)
+{
+    int ncid, ioid;
+    int dimid[NDIM3], varid;
+    int dimlen[NDIM3] = {NC_UNLIMITED, DIM_LEN_X, DIM_LEN_Y};
+    size_t elements_per_pe;
+    size_t *compdof; /* The decomposition mapping. */
+    int *my_data;
+    int *data_in;
+    char file_name[NC_MAX_NAME + 1];
+    int i;
+
+    /* Create a file with a 3D record var. */
+    sprintf(file_name, "%s_component_%d.nc", TEST_NAME, 0);
+    if (nc_create(file_name, NC_PIO|NC_NETCDF4, &ncid)) PERR;
+    if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR;
+    if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR;
+    if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR;
+    if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM3, dimid, &varid)) PERR;
+    if (nc_enddef(ncid)) PERR;
+
+    /* Calculate a decomposition for distributed arrays. */
+    elements_per_pe = DIM_LEN_X * DIM_LEN_Y / (ntasks - num_io_procs);
+    /* printf("my_rank %d elements_per_pe %ld\n", my_rank, elements_per_pe); */
+    if (!(compdof = malloc(elements_per_pe * sizeof(size_t))))
+        PERR;
+    for (i = 0; i < elements_per_pe; i++)
+    {
+        compdof[i] = (my_rank - num_io_procs) * elements_per_pe + i;
+        /* printf("my_rank %d compdof[%d]=%ld\n", my_rank, i, compdof[i]); */
+    }
+
+    /* Create the PIO decomposition for this test. */
+    if (nc_def_decomp(iosysid, PIO_INT, NDIM2, &dimlen[1], elements_per_pe,
+                      compdof, &ioid, 1, NULL, NULL)) PERR;
+    free(compdof);
+
+    /* Create some data on this processor. */
+    if (!(my_data = malloc(elements_per_pe * sizeof(int)))) PERR;
+    for (i = 0; i < elements_per_pe; i++)
+        my_data[i] = my_rank * 10 + i;
+
+    /* Write some data with distributed arrays. */
+    if (nc_put_vard_int(ncid, varid, ioid, 0, my_data)) PERR;
+    if (nc_close(ncid)) PERR;
+
+    /* Reopen the file using netCDF integration. */
+    {
+        int ndims, nvars, ngatts, unlimdimid;
+        nc_type xtype_in;
+        char var_name_in[NC_MAX_NAME + 1];
+        char dim_name_in[NC_MAX_NAME + 1];
+        int natts_in;
+        int dimids_in[NDIM3];
+        size_t dim_len_in;
+
+        /* Open the file. */
+        if (nc_open(file_name, NC_PIO, &ncid)) PERR;
+
+        /* Check the file. */
+        if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) PERR;
+        if (ndims != 3 || nvars != 1 || ngatts != 0 ||
+            unlimdimid != 0) PERR;
+        if (nc_inq_var(ncid, 0, var_name_in, &xtype_in, &ndims,
+                       dimids_in, &natts_in)) PERR;
+        if (strcmp(var_name_in, VAR_NAME) || xtype_in != NC_INT || ndims != NDIM3
+            || dimids_in[0] != 0 || dimids_in[1] != 1 || dimids_in[2] != 2 ||
+            natts_in != 0) PERR;
+        if (nc_inq_dim(ncid, 0, dim_name_in, &dim_len_in)) PERR;
+        if (strcmp(dim_name_in, DIM_NAME_UNLIMITED) || dim_len_in != 1) PERR;
+        if (nc_inq_dim(ncid, 1, dim_name_in, &dim_len_in)) PERR;
+        if (strcmp(dim_name_in, DIM_NAME_X) || dim_len_in != DIM_LEN_X) PERR;
+        if (nc_inq_dim(ncid, 2, dim_name_in, &dim_len_in)) PERR;
+        if (strcmp(dim_name_in, DIM_NAME_Y) || dim_len_in != DIM_LEN_Y) PERR;
+
+        /* Read distributed arrays. */
+        if (!(data_in = malloc(elements_per_pe * sizeof(int)))) PERR;
+        if (nc_get_vard_int(ncid, varid, ioid, 0, data_in)) PERR;
+
+        /* Check results. */
+        for (i = 0; i < elements_per_pe; i++)
+            if (data_in[i] != my_data[i]) PERR;
+
+        /* Close file. */
+        if (nc_close(ncid)) PERR;
+
+        /* Free resources. */
+        free(data_in);
+    }
+
+    /* Release resources. */
+    free(my_data);
+    if (nc_free_decomp(ioid)) PERR;
+
+    return 0;
+}
+
 int
 main(int argc, char **argv)
 {
@@ -40,17 +142,9 @@ main(int argc, char **argv)
     if (!my_rank)
         printf("*** testing simple async use of netCDF integration layer...");
     {
-        int ncid, ioid;
-        int dimid[NDIM3], varid;
-        int dimlen[NDIM3] = {NC_UNLIMITED, DIM_LEN_X, DIM_LEN_Y};
         int iosysid;
-        size_t elements_per_pe;
-        size_t *compdof; /* The decomposition mapping. */
-        int *my_data;
-        int *data_in;
         int num_procs2[COMPONENT_COUNT] = {3};
         int num_io_procs = 1;
-        int i;
 
         /* Turn on logging for PIO library. */
         /* PIOc_set_log_level(4); */
@@ -66,85 +160,9 @@ main(int argc, char **argv)
 
         if (my_rank)
         {
-            /* Create a file with a 3D record var. */
-            if (nc_create(FILE_NAME, NC_PIO|NC_NETCDF4, &ncid)) PERR;
-            if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR;
-            if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR;
-            if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR;
-            if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM3, dimid, &varid)) PERR;
-            if (nc_enddef(ncid)) PERR;
-
-            /* Calculate a decomposition for distributed arrays. */
-            elements_per_pe = DIM_LEN_X * DIM_LEN_Y / (ntasks - num_io_procs);
-            /* printf("my_rank %d elements_per_pe %ld\n", my_rank, elements_per_pe); */
-            if (!(compdof = malloc(elements_per_pe * sizeof(size_t))))
-                PERR;
-            for (i = 0; i < elements_per_pe; i++)
-            {
-                compdof[i] = (my_rank - num_io_procs) * elements_per_pe + i;
-                /* printf("my_rank %d compdof[%d]=%ld\n", my_rank, i, compdof[i]); */
-            }
-
-            /* Create the PIO decomposition for this test. */
-            if (nc_def_decomp(iosysid, PIO_INT, NDIM2, &dimlen[1], elements_per_pe,
-                              compdof, &ioid, 1, NULL, NULL)) PERR;
-            free(compdof);
-
-            /* Create some data on this processor. */
-            if (!(my_data = malloc(elements_per_pe * sizeof(int)))) PERR;
-            for (i = 0; i < elements_per_pe; i++)
-                my_data[i] = my_rank * 10 + i;
-
-            /* Write some data with distributed arrays. */
-            if (nc_put_vard_int(ncid, varid, ioid, 0, my_data)) PERR;
-            if (nc_close(ncid)) PERR;
-
-            /* Reopen the file using netCDF integration. */
-            {
-                int ndims, nvars, ngatts, unlimdimid;
-                nc_type xtype_in;
-                char var_name_in[NC_MAX_NAME + 1];
-                char dim_name_in[NC_MAX_NAME + 1];
-                int natts_in;
-                int dimids_in[NDIM3];
-                size_t dim_len_in;
-
-                /* Open the file. */
-                if (nc_open(FILE_NAME, NC_PIO, &ncid)) PERR;
-
-                /* Check the file. */
-                if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) PERR;
-                if (ndims != 3 || nvars != 1 || ngatts != 0 ||
-                    unlimdimid != 0) PERR;
-                if (nc_inq_var(ncid, 0, var_name_in, &xtype_in, &ndims,
-                               dimids_in, &natts_in)) PERR;
-                if (strcmp(var_name_in, VAR_NAME) || xtype_in != NC_INT || ndims != NDIM3
-                    || dimids_in[0] != 0 || dimids_in[1] != 1 || dimids_in[2] != 2 ||
-                    natts_in != 0) PERR;
-                if (nc_inq_dim(ncid, 0, dim_name_in, &dim_len_in)) PERR;
-                if (strcmp(dim_name_in, DIM_NAME_UNLIMITED) || dim_len_in != 1) PERR;
-                if (nc_inq_dim(ncid, 1, dim_name_in, &dim_len_in)) PERR;
-                if (strcmp(dim_name_in, DIM_NAME_X) || dim_len_in != DIM_LEN_X) PERR;
-                if (nc_inq_dim(ncid, 2, dim_name_in, &dim_len_in)) PERR;
-                if (strcmp(dim_name_in, DIM_NAME_Y) || dim_len_in != DIM_LEN_Y) PERR;
-
-                /* Read distributed arrays. */
-                if (!(data_in = malloc(elements_per_pe * sizeof(int)))) PERR;
-                if (nc_get_vard_int(ncid, varid, ioid, 0, data_in)) PERR;
-
-                /* Check results. */
-                for (i = 0; i < elements_per_pe; i++)
-                    if (data_in[i] != my_data[i]) PERR;
-
-                /* Close file. */
-                if (nc_close(ncid)) PERR;
-
-                /* Free resources. */
-                free(data_in);
-            }
-
-            free(my_data);
-            if (nc_free_decomp(ioid)) PERR;
+            /* Create a file, write some data, and check it. */
+            if (create_file_0(my_rank, ntasks, num_io_procs, iosysid)) PERR;
+
             if (nc_free_iosystem(iosysid)) PERR;
         }
     }

From d6097d97f4be96f64539a38812e737f0b5999f48 Mon Sep 17 00:00:00 2001
From: edwardhartnett <edwardjameshartnett@gmail.com>
Date: Mon, 9 Sep 2019 05:31:26 -0600
Subject: [PATCH 3/7] moving code to function so that multiple computation
 units can be handled

---
 tests/ncint/tst_async_multi.c | 102 +++++++++++++++++++++++++++++++++-
 1 file changed, 101 insertions(+), 1 deletion(-)

diff --git a/tests/ncint/tst_async_multi.c b/tests/ncint/tst_async_multi.c
index 0307fc44b50..f548c8b34e0 100644
--- a/tests/ncint/tst_async_multi.c
+++ b/tests/ncint/tst_async_multi.c
@@ -25,7 +25,7 @@ extern NC_Dispatch NCINT_dispatcher;
 /* Number of computational components to create. */
 #define COMPONENT_COUNT 1
 
-/* Create a file with one 2D var. */
+/* Create a file with one 3D var. */
 int
 create_file_0(int my_rank, int ntasks, int num_io_procs, int iosysid)
 {
@@ -124,6 +124,105 @@ create_file_0(int my_rank, int ntasks, int num_io_procs, int iosysid)
     return 0;
 }
 
+/* Create a file with one 3D var. */
+int
+create_file_1(int my_rank, int ntasks, int num_io_procs, int iosysid)
+{
+    int ncid, ioid;
+    int dimid[NDIM3], varid;
+    int dimlen[NDIM3] = {NC_UNLIMITED, DIM_LEN_X, DIM_LEN_Y};
+    size_t elements_per_pe;
+    size_t *compdof; /* The decomposition mapping. */
+    int *my_data;
+    int *data_in;
+    char file_name[NC_MAX_NAME + 1];
+    int i;
+
+    /* Create a file with a 3D record var. */
+    sprintf(file_name, "%s_component_%d.nc", TEST_NAME, 1);
+    if (nc_create(file_name, NC_PIO|NC_NETCDF4, &ncid)) PERR;
+    if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR;
+    if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR;
+    if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR;
+    if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM3, dimid, &varid)) PERR;
+    if (nc_enddef(ncid)) PERR;
+
+    /* Calculate a decomposition for distributed arrays. */
+    elements_per_pe = DIM_LEN_X * DIM_LEN_Y / (ntasks - num_io_procs);
+    /* printf("my_rank %d elements_per_pe %ld\n", my_rank, elements_per_pe); */
+    if (!(compdof = malloc(elements_per_pe * sizeof(size_t))))
+        PERR;
+    for (i = 0; i < elements_per_pe; i++)
+    {
+        compdof[i] = (my_rank - num_io_procs) * elements_per_pe + i;
+        /* printf("my_rank %d compdof[%d]=%ld\n", my_rank, i, compdof[i]); */
+    }
+
+    /* Create the PIO decomposition for this test. */
+    if (nc_def_decomp(iosysid, PIO_INT, NDIM2, &dimlen[1], elements_per_pe,
+                      compdof, &ioid, 1, NULL, NULL)) PERR;
+    free(compdof);
+
+    /* Create some data on this processor. */
+    if (!(my_data = malloc(elements_per_pe * sizeof(int)))) PERR;
+    for (i = 0; i < elements_per_pe; i++)
+        my_data[i] = my_rank * 10 + i;
+
+    /* Write some data with distributed arrays. */
+    if (nc_put_vard_int(ncid, varid, ioid, 0, my_data)) PERR;
+    if (nc_close(ncid)) PERR;
+
+    /* Reopen the file using netCDF integration. */
+    {
+        int ndims, nvars, ngatts, unlimdimid;
+        nc_type xtype_in;
+        char var_name_in[NC_MAX_NAME + 1];
+        char dim_name_in[NC_MAX_NAME + 1];
+        int natts_in;
+        int dimids_in[NDIM3];
+        size_t dim_len_in;
+
+        /* Open the file. */
+        if (nc_open(file_name, NC_PIO, &ncid)) PERR;
+
+        /* Check the file. */
+        if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) PERR;
+        if (ndims != 3 || nvars != 1 || ngatts != 0 ||
+            unlimdimid != 0) PERR;
+        if (nc_inq_var(ncid, 0, var_name_in, &xtype_in, &ndims,
+                       dimids_in, &natts_in)) PERR;
+        if (strcmp(var_name_in, VAR_NAME) || xtype_in != NC_INT || ndims != NDIM3
+            || dimids_in[0] != 0 || dimids_in[1] != 1 || dimids_in[2] != 2 ||
+            natts_in != 0) PERR;
+        if (nc_inq_dim(ncid, 0, dim_name_in, &dim_len_in)) PERR;
+        if (strcmp(dim_name_in, DIM_NAME_UNLIMITED) || dim_len_in != 1) PERR;
+        if (nc_inq_dim(ncid, 1, dim_name_in, &dim_len_in)) PERR;
+        if (strcmp(dim_name_in, DIM_NAME_X) || dim_len_in != DIM_LEN_X) PERR;
+        if (nc_inq_dim(ncid, 2, dim_name_in, &dim_len_in)) PERR;
+        if (strcmp(dim_name_in, DIM_NAME_Y) || dim_len_in != DIM_LEN_Y) PERR;
+
+        /* Read distributed arrays. */
+        if (!(data_in = malloc(elements_per_pe * sizeof(int)))) PERR;
+        if (nc_get_vard_int(ncid, varid, ioid, 0, data_in)) PERR;
+
+        /* Check results. */
+        for (i = 0; i < elements_per_pe; i++)
+            if (data_in[i] != my_data[i]) PERR;
+
+        /* Close file. */
+        if (nc_close(ncid)) PERR;
+
+        /* Free resources. */
+        free(data_in);
+    }
+
+    /* Release resources. */
+    free(my_data);
+    if (nc_free_decomp(ioid)) PERR;
+
+    return 0;
+}
+
 int
 main(int argc, char **argv)
 {
@@ -162,6 +261,7 @@ main(int argc, char **argv)
         {
             /* Create a file, write some data, and check it. */
             if (create_file_0(my_rank, ntasks, num_io_procs, iosysid)) PERR;
+            if (create_file_1(my_rank, ntasks, num_io_procs, iosysid)) PERR;
 
             if (nc_free_iosystem(iosysid)) PERR;
         }

From 85ec39cd825a71f1b5e56a12e68446a26d796823 Mon Sep 17 00:00:00 2001
From: edwardhartnett <edwardjameshartnett@gmail.com>
Date: Mon, 9 Sep 2019 05:54:39 -0600
Subject: [PATCH 4/7] fixed ncid issue for PIO opening second file

---
 src/clib/pioc_support.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c
index 2a59e07fbb9..8598cfff18e 100644
--- a/src/clib/pioc_support.c
+++ b/src/clib/pioc_support.c
@@ -2789,6 +2789,7 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename,
         /* The ncid was assigned on the computational
          * processors. Change the ncid to one that I/O and
          * computational components can agree on. */
+        file->pio_ncid = pio_next_ncid++;
         if ((ierr = nc4_file_change_ncid(*ncidp, file->pio_ncid)))
             return pio_err(NULL, file, ierr, __FILE__, __LINE__);
         file->pio_ncid = file->pio_ncid << ID_SHIFT;

From f19543590a57afc09f47da15477c5a22e8516cd4 Mon Sep 17 00:00:00 2001
From: edwardhartnett <edwardjameshartnett@gmail.com>
Date: Mon, 9 Sep 2019 10:51:45 -0600
Subject: [PATCH 5/7] now passing diosysid when doing a create file in async
 mode

---
 src/clib/pio_msg.c            |  13 +++-
 src/clib/pioc_support.c       |  12 +++-
 tests/ncint/tst_async_multi.c | 131 +++++-----------------------------
 3 files changed, 39 insertions(+), 117 deletions(-)

diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c
index b7ad399ca3b..57a3ce9c36b 100644
--- a/src/clib/pio_msg.c
+++ b/src/clib/pio_msg.c
@@ -174,6 +174,7 @@ int create_file_handler(iosystem_desc_t *ios)
     int mode;
     int use_ext_ncid;
     char ncidp_present;
+    int iosysid;
     int mpierr;
 
     PLOG((1, "create_file_handler comproot = %d", ios->comproot));
@@ -200,15 +201,21 @@ int create_file_handler(iosystem_desc_t *ios)
     if (ncidp_present)
         if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
             return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     PLOG((1, "create_file_handler len %d filename %s iotype %d mode %d "
-          "use_ext_ncid %d ncidp_present %d ncid %d", len, filename, iotype, mode,
-          use_ext_ncid, ncidp_present, ncid));
+          "use_ext_ncid %d ncidp_present %d ncid %d iosysid %d", len,
+          filename, iotype, mode, use_ext_ncid, ncidp_present, ncid,
+          iosysid));
 
     /* Call the create file function. */
     if (use_ext_ncid)
     {
 #ifdef NETCDF_INTEGRATION
-        PLOG((2, "about to call nc_create()"));
+        /* Set the IO system ID. */
+        nc_set_iosystem(iosysid);
+
+        PLOG((2, "about to call nc_create() have set iosysid to %d", iosysid));
         nc_create(filename, mode|NC_UDF0, &ncid);
 #endif /* NETCDF_INTEGRATION */
     }
diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c
index 8598cfff18e..81f8da8eff6 100644
--- a/src/clib/pioc_support.c
+++ b/src/clib/pioc_support.c
@@ -42,9 +42,15 @@ extern int pio_next_ncid;
 /** The default error handler used when iosystem cannot be located. */
 extern int default_error_handler;
 
+#ifdef NETCDF_INTEGRATION
+/* This is used as the default iosysid for the netcdf integration
+ * code. */
+extern int diosysid;
+
 /** This prototype from netCDF is required for netCDF integration to
  * work. */
 int nc4_file_change_ncid(int ncid, unsigned short new_ncid_index);
+#endif /* NETCDF_INTEGRATION */
 
 /**
  * Start the PIO timer.
@@ -2033,9 +2039,11 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename,
             if (ncidp_present)
                 if (!mpierr)
                     mpierr = MPI_Bcast(ncidp, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&diosysid, 1, MPI_INT, ios->compmaster, ios->intercomm);
             PLOG((2, "len %d filename %s iotype %d mode %d use_ext_ncid %d "
-                  "ncidp_present %d", len, filename, file->iotype, mode,
-                  use_ext_ncid, ncidp_present));
+                  "ncidp_present %d diosysid %d", len, filename, file->iotype, mode,
+                  use_ext_ncid, ncidp_present, diosysid));
         }
 
         /* Handle MPI errors. */
diff --git a/tests/ncint/tst_async_multi.c b/tests/ncint/tst_async_multi.c
index f548c8b34e0..cc3f636bd9d 100644
--- a/tests/ncint/tst_async_multi.c
+++ b/tests/ncint/tst_async_multi.c
@@ -23,11 +23,12 @@
 extern NC_Dispatch NCINT_dispatcher;
 
 /* Number of computational components to create. */
-#define COMPONENT_COUNT 1
+#define COMPONENT_COUNT 2
 
 /* Create a file with one 3D var. */
 int
-create_file_0(int my_rank, int ntasks, int num_io_procs, int iosysid)
+create_file(int file_num, int my_rank, int ntasks, int num_io_procs,
+            int iosysid)
 {
     int ncid, ioid;
     int dimid[NDIM3], varid;
@@ -40,106 +41,7 @@ create_file_0(int my_rank, int ntasks, int num_io_procs, int iosysid)
     int i;
 
     /* Create a file with a 3D record var. */
-    sprintf(file_name, "%s_component_%d.nc", TEST_NAME, 0);
-    if (nc_create(file_name, NC_PIO|NC_NETCDF4, &ncid)) PERR;
-    if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR;
-    if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR;
-    if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR;
-    if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM3, dimid, &varid)) PERR;
-    if (nc_enddef(ncid)) PERR;
-
-    /* Calculate a decomposition for distributed arrays. */
-    elements_per_pe = DIM_LEN_X * DIM_LEN_Y / (ntasks - num_io_procs);
-    /* printf("my_rank %d elements_per_pe %ld\n", my_rank, elements_per_pe); */
-    if (!(compdof = malloc(elements_per_pe * sizeof(size_t))))
-        PERR;
-    for (i = 0; i < elements_per_pe; i++)
-    {
-        compdof[i] = (my_rank - num_io_procs) * elements_per_pe + i;
-        /* printf("my_rank %d compdof[%d]=%ld\n", my_rank, i, compdof[i]); */
-    }
-
-    /* Create the PIO decomposition for this test. */
-    if (nc_def_decomp(iosysid, PIO_INT, NDIM2, &dimlen[1], elements_per_pe,
-                      compdof, &ioid, 1, NULL, NULL)) PERR;
-    free(compdof);
-
-    /* Create some data on this processor. */
-    if (!(my_data = malloc(elements_per_pe * sizeof(int)))) PERR;
-    for (i = 0; i < elements_per_pe; i++)
-        my_data[i] = my_rank * 10 + i;
-
-    /* Write some data with distributed arrays. */
-    if (nc_put_vard_int(ncid, varid, ioid, 0, my_data)) PERR;
-    if (nc_close(ncid)) PERR;
-
-    /* Reopen the file using netCDF integration. */
-    {
-        int ndims, nvars, ngatts, unlimdimid;
-        nc_type xtype_in;
-        char var_name_in[NC_MAX_NAME + 1];
-        char dim_name_in[NC_MAX_NAME + 1];
-        int natts_in;
-        int dimids_in[NDIM3];
-        size_t dim_len_in;
-
-        /* Open the file. */
-        if (nc_open(file_name, NC_PIO, &ncid)) PERR;
-
-        /* Check the file. */
-        if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) PERR;
-        if (ndims != 3 || nvars != 1 || ngatts != 0 ||
-            unlimdimid != 0) PERR;
-        if (nc_inq_var(ncid, 0, var_name_in, &xtype_in, &ndims,
-                       dimids_in, &natts_in)) PERR;
-        if (strcmp(var_name_in, VAR_NAME) || xtype_in != NC_INT || ndims != NDIM3
-            || dimids_in[0] != 0 || dimids_in[1] != 1 || dimids_in[2] != 2 ||
-            natts_in != 0) PERR;
-        if (nc_inq_dim(ncid, 0, dim_name_in, &dim_len_in)) PERR;
-        if (strcmp(dim_name_in, DIM_NAME_UNLIMITED) || dim_len_in != 1) PERR;
-        if (nc_inq_dim(ncid, 1, dim_name_in, &dim_len_in)) PERR;
-        if (strcmp(dim_name_in, DIM_NAME_X) || dim_len_in != DIM_LEN_X) PERR;
-        if (nc_inq_dim(ncid, 2, dim_name_in, &dim_len_in)) PERR;
-        if (strcmp(dim_name_in, DIM_NAME_Y) || dim_len_in != DIM_LEN_Y) PERR;
-
-        /* Read distributed arrays. */
-        if (!(data_in = malloc(elements_per_pe * sizeof(int)))) PERR;
-        if (nc_get_vard_int(ncid, varid, ioid, 0, data_in)) PERR;
-
-        /* Check results. */
-        for (i = 0; i < elements_per_pe; i++)
-            if (data_in[i] != my_data[i]) PERR;
-
-        /* Close file. */
-        if (nc_close(ncid)) PERR;
-
-        /* Free resources. */
-        free(data_in);
-    }
-
-    /* Release resources. */
-    free(my_data);
-    if (nc_free_decomp(ioid)) PERR;
-
-    return 0;
-}
-
-/* Create a file with one 3D var. */
-int
-create_file_1(int my_rank, int ntasks, int num_io_procs, int iosysid)
-{
-    int ncid, ioid;
-    int dimid[NDIM3], varid;
-    int dimlen[NDIM3] = {NC_UNLIMITED, DIM_LEN_X, DIM_LEN_Y};
-    size_t elements_per_pe;
-    size_t *compdof; /* The decomposition mapping. */
-    int *my_data;
-    int *data_in;
-    char file_name[NC_MAX_NAME + 1];
-    int i;
-
-    /* Create a file with a 3D record var. */
-    sprintf(file_name, "%s_component_%d.nc", TEST_NAME, 1);
+    sprintf(file_name, "%s_file_%d.nc", TEST_NAME, file_num);
     if (nc_create(file_name, NC_PIO|NC_NETCDF4, &ncid)) PERR;
     if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR;
     if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR;
@@ -241,30 +143,35 @@ main(int argc, char **argv)
     if (!my_rank)
         printf("*** testing simple async use of netCDF integration layer...");
     {
-        int iosysid;
-        int num_procs2[COMPONENT_COUNT] = {3};
+        int iosysid[COMPONENT_COUNT];
+        int num_procs2[COMPONENT_COUNT] = {1, 2};
         int num_io_procs = 1;
 
         /* Turn on logging for PIO library. */
-        /* PIOc_set_log_level(4); */
-        /* if (!my_rank) */
-        /*     nc_set_log_level(3); */
+        PIOc_set_log_level(4);
+        if (!my_rank)
+            nc_set_log_level(3);
 
         /* Initialize the intracomm. The IO task will not return from
          * this call until the PIOc_finalize() is called by the
          * compute tasks. */
         if (nc_def_async(MPI_COMM_WORLD, num_io_procs, NULL, COMPONENT_COUNT,
-                         num_procs2, NULL, NULL, NULL, PIO_REARR_BOX, &iosysid))
+                         num_procs2, NULL, NULL, NULL, PIO_REARR_BOX, iosysid))
             PERR;
 
-        if (my_rank)
+        if (my_rank == 1)
         {
             /* Create a file, write some data, and check it. */
-            if (create_file_0(my_rank, ntasks, num_io_procs, iosysid)) PERR;
-            if (create_file_1(my_rank, ntasks, num_io_procs, iosysid)) PERR;
+            if (create_file(0, my_rank, ntasks, num_io_procs, iosysid[0])) PERR;
+            if (create_file(1, my_rank, ntasks, num_io_procs, iosysid[0])) PERR;
 
-            if (nc_free_iosystem(iosysid)) PERR;
+            if (nc_free_iosystem(iosysid[0])) PERR;
         }
+        else if (my_rank > 1)
+        {
+            if (nc_free_iosystem(iosysid[1])) PERR;
+        }
+
     }
     if (!my_rank)
         PSUMMARIZE_ERR;

From 0d6cbc7565ba2901453adb71ff13cdb562aadd5b Mon Sep 17 00:00:00 2001
From: edwardhartnett <edwardjameshartnett@gmail.com>
Date: Mon, 9 Sep 2019 11:10:38 -0600
Subject: [PATCH 6/7] now netcdf integration works with multiple computational
 components with async

---
 src/clib/pio_msg.c            | 15 +++++++++++----
 src/clib/pioc_support.c       |  2 ++
 tests/ncint/tst_async_multi.c |  6 +++---
 3 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c
index 57a3ce9c36b..57460cd8a9e 100644
--- a/src/clib/pio_msg.c
+++ b/src/clib/pio_msg.c
@@ -215,7 +215,7 @@ int create_file_handler(iosystem_desc_t *ios)
         /* Set the IO system ID. */
         nc_set_iosystem(iosysid);
 
-        PLOG((2, "about to call nc_create() have set iosysid to %d", iosysid));
+        PLOG((2, "about to call nc_create() having set iosysid to %d", iosysid));
         nc_create(filename, mode|NC_UDF0, &ncid);
 #endif /* NETCDF_INTEGRATION */
     }
@@ -1977,6 +1977,7 @@ int open_file_handler(iosystem_desc_t *ios)
     int iotype;
     int mode;
     int use_ext_ncid;
+    int iosysid;
     int mpierr;
 
     PLOG((1, "open_file_handler comproot = %d", ios->comproot));
@@ -1999,16 +2000,22 @@ int open_file_handler(iosystem_desc_t *ios)
         return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&use_ext_ncid, 1, MPI_INT, 0, ios->intercomm)))
         return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
-    PLOG((2, "len %d filename %s iotype %d mode %d use_ext_ncid %d",
-          len, filename, iotype, mode, use_ext_ncid));
+    PLOG((2, "len %d filename %s iotype %d mode %d use_ext_ncid %d iosysid %d",
+          len, filename, iotype, mode, use_ext_ncid, iosysid));
 
     /* Call the open file function. Errors are handled within
      * function, so return code can be ignored. */
     if (use_ext_ncid)
     {
 #ifdef NETCDF_INTEGRATION
-        PLOG((2, "about to call nc_create()"));
+        /* Set the IO system ID. */
+        nc_set_iosystem(iosysid);
+
+        PLOG((2, "about to call nc_create() having set iosysid to %d",
+              iosysid));
         nc_open(filename, mode|NC_UDF0, &ncid);
 #endif /* NETCDF_INTEGRATION */
     }
diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c
index 81f8da8eff6..9e5fc1f5e22 100644
--- a/src/clib/pioc_support.c
+++ b/src/clib/pioc_support.c
@@ -2598,6 +2598,8 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename,
                 mpierr = MPI_Bcast(&mode, 1, MPI_INT, ios->compmaster, ios->intercomm);
             if (!mpierr)
                 mpierr = MPI_Bcast(&use_ext_ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&diosysid, 1, MPI_INT, ios->compmaster, ios->intercomm);
         }
 
         /* Handle MPI errors. */
diff --git a/tests/ncint/tst_async_multi.c b/tests/ncint/tst_async_multi.c
index cc3f636bd9d..576b05d1659 100644
--- a/tests/ncint/tst_async_multi.c
+++ b/tests/ncint/tst_async_multi.c
@@ -148,9 +148,9 @@ main(int argc, char **argv)
         int num_io_procs = 1;
 
         /* Turn on logging for PIO library. */
-        PIOc_set_log_level(4);
-        if (!my_rank)
-            nc_set_log_level(3);
+        /* PIOc_set_log_level(4); */
+        /* if (!my_rank) */
+        /*     nc_set_log_level(3); */
 
         /* Initialize the intracomm. The IO task will not return from
          * this call until the PIOc_finalize() is called by the

From a8b8a42af4648b2f4a6cdc46987088afe4b8f133 Mon Sep 17 00:00:00 2001
From: edwardhartnett <edwardjameshartnett@gmail.com>
Date: Mon, 9 Sep 2019 14:56:28 -0600
Subject: [PATCH 7/7] only send iosysid for create/open when netcdf integration
 is in use

---
 src/clib/pio_msg.c      | 8 ++++++--
 src/clib/pioc_support.c | 8 ++++++--
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c
index 57460cd8a9e..6ffb616a7c3 100644
--- a/src/clib/pio_msg.c
+++ b/src/clib/pio_msg.c
@@ -174,7 +174,7 @@ int create_file_handler(iosystem_desc_t *ios)
     int mode;
     int use_ext_ncid;
     char ncidp_present;
-    int iosysid;
+    int iosysid = 0;
     int mpierr;
 
     PLOG((1, "create_file_handler comproot = %d", ios->comproot));
@@ -201,8 +201,10 @@ int create_file_handler(iosystem_desc_t *ios)
     if (ncidp_present)
         if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
             return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+#ifdef NETCDF_INTEGRATION
     if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm)))
         return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+#endif /* NETCDF_INTEGRATION */
     PLOG((1, "create_file_handler len %d filename %s iotype %d mode %d "
           "use_ext_ncid %d ncidp_present %d ncid %d iosysid %d", len,
           filename, iotype, mode, use_ext_ncid, ncidp_present, ncid,
@@ -1977,7 +1979,7 @@ int open_file_handler(iosystem_desc_t *ios)
     int iotype;
     int mode;
     int use_ext_ncid;
-    int iosysid;
+    int iosysid = 0;
     int mpierr;
 
     PLOG((1, "open_file_handler comproot = %d", ios->comproot));
@@ -2000,8 +2002,10 @@ int open_file_handler(iosystem_desc_t *ios)
         return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&use_ext_ncid, 1, MPI_INT, 0, ios->intercomm)))
         return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+#ifdef NETCDF_INTEGRATION
     if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm)))
         return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+#endif /* NETCDF_INTEGRATION */
 
     PLOG((2, "len %d filename %s iotype %d mode %d use_ext_ncid %d iosysid %d",
           len, filename, iotype, mode, use_ext_ncid, iosysid));
diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c
index 9e5fc1f5e22..fb3f58f2e80 100644
--- a/src/clib/pioc_support.c
+++ b/src/clib/pioc_support.c
@@ -2039,11 +2039,13 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename,
             if (ncidp_present)
                 if (!mpierr)
                     mpierr = MPI_Bcast(ncidp, 1, MPI_INT, ios->compmaster, ios->intercomm);
+#ifdef NETCDF_INTEGRATION
             if (!mpierr)
                 mpierr = MPI_Bcast(&diosysid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+#endif /* NETCDF_INTEGRATION */
             PLOG((2, "len %d filename %s iotype %d mode %d use_ext_ncid %d "
-                  "ncidp_present %d diosysid %d", len, filename, file->iotype, mode,
-                  use_ext_ncid, ncidp_present, diosysid));
+                  "ncidp_present %d", len, filename, file->iotype, mode,
+                  use_ext_ncid, ncidp_present));
         }
 
         /* Handle MPI errors. */
@@ -2598,8 +2600,10 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename,
                 mpierr = MPI_Bcast(&mode, 1, MPI_INT, ios->compmaster, ios->intercomm);
             if (!mpierr)
                 mpierr = MPI_Bcast(&use_ext_ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+#ifdef NETCDF_INTEGRATION
             if (!mpierr)
                 mpierr = MPI_Bcast(&diosysid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+#endif /* NETCDF_INTEGRATION */
         }
 
         /* Handle MPI errors. */