Merge pull request ESMCI#1593 from NCAR/ejh_ncint_async

got netcdf integration working with PIO async mode for one computational component

src/clib/pio_msg.c

@@ -175,6 +175,7 @@ int create_file_handler(iosystem_desc_t *ios)
     int use_ext_ncid;
     char ncidp_present;
     int mpierr;
+    int ret;
 
     PLOG((1, "create_file_handler comproot = %d", ios->comproot));
     assert(ios);
@@ -205,8 +206,19 @@ int create_file_handler(iosystem_desc_t *ios)
           use_ext_ncid, ncidp_present, ncid));
 
     /* Call the create file function. */
-    PIOc_createfile_int(ios->iosysid, &ncid, &iotype, filename, mode,
-                        use_ext_ncid);
+    if (use_ext_ncid)
+    {
+#ifdef NETCDF_INTEGRATION
+        PLOG((2, "about to call nc_create()"));
+        nc_create(filename, mode|NC_UDF0, &ncid);
+#endif /* NETCDF_INTEGRATION */
+    }
+    else
+    {
+        PLOG((2, "about to call PIOc_createfile_int()"));
+        PIOc_createfile_int(ios->iosysid, &ncid, &iotype, filename, mode,
+                            use_ext_ncid);
+    }
 
     PLOG((1, "create_file_handler succeeded!"));
     return PIO_NOERR;

src/clib/pioc.c

@@ -19,8 +19,12 @@ extern int event_num[2][NUM_EVENTS];
 #endif /* USE_MPE */
 
 #ifdef NETCDF_INTEGRATION
-/* Have we initialized? */
+/* Have we initialized the netcdf integration code? */
 extern int ncint_initialized;
+
+/* This is used as the default iosysid for the netcdf integration
+ * code. */
+extern int diosysid;
 #endif /* NETCDF_INTEGRATION */
 
 /**
@@ -1736,6 +1740,16 @@ PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list,
         /* Add this id to the list of PIO iosystem ids. */
         iosysidp[cmp] = pio_add_to_iosystem_list(my_iosys);
         PLOG((2, "new iosys ID added to iosystem_list iosysidp[%d] = %d", cmp, iosysidp[cmp]));
+
+#ifdef NETCDF_INTEGRATION
+        if (in_io || in_cmp)
+        {
+            /* Remember the io system id. */
+            diosysid = iosysidp[cmp];
+            PLOG((3, "diosysid = %d", iosysidp[cmp]));
+        }
+#endif /* NETCDF_INTEGRATION */
+
     } /* next computational component */
 
     /* Now call the function from which the IO tasks will not return

src/clib/pioc_support.c

@@ -14,6 +14,9 @@
 /** This is used with text decomposition files. */
 #define VERSNO 2001
 
+/** Used to shift file index to first two bytes of ncid. */
+#define ID_SHIFT 16
+
 /** In decomposition files, backtraces are included. This is the max
  * number of trace levels that will be used. */
 #define MAX_BACKTRACE 10
@@ -39,6 +42,8 @@ extern int pio_next_ncid;
 /** The default error handler used when iosystem cannot be located. */
 extern int default_error_handler;
 
+int nc4_file_change_ncid(int ncid, unsigned short new_ncid_index);
+
 /**
  * Start the PIO timer.
  *
@@ -1930,8 +1935,12 @@ PIOc_writemap_from_f90(const char *file, int ndims, const int *gdims,
  * @param iosysid A defined pio system ID, obtained from
  * PIOc_Init_Intracomm() or PIOc_InitAsync().
  * @param ncidp A pointer that gets the ncid of the newly created
- * file. For NetCDF integration, this contains the ncid assigned by
- * the netCDF layer, which is used instead of a PIO-generated ncid.
+ * file. This is the PIO ncid. Within PIO, the file will have a
+ * different ID, the file->fh. When netCDF integration is used, the
+ * PIO ncid is also stored in the netcdf-c internal file list, and the
+ * PIO code is called by the netcdf-c dispatch code. In this case,
+ * there are two ncids for the file, the PIO ncid, and the file->fh
+ * ncid.
  * @param iotype A pointer to a pio output format. Must be one of
  * PIO_IOTYPE_PNETCDF, PIO_IOTYPE_NETCDF, PIO_IOTYPE_NETCDF4C, or
  * PIO_IOTYPE_NETCDF4P.
@@ -1969,8 +1978,8 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename,
     if (!iotype_is_valid(*iotype))
         return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
 
-    PLOG((1, "PIOc_createfile_int iosysid = %d iotype = %d filename = %s mode = %d",
-          iosysid, *iotype, filename, mode));
+    PLOG((1, "PIOc_createfile_int iosysid %d iotype %d filename %s mode %d "
+          "use_ext_ncid %d", iosysid, *iotype, filename, mode, use_ext_ncid));
 
     /* Allocate space for the file info. */
     if (!(file = calloc(sizeof(file_desc_t), 1)))
@@ -1998,8 +2007,8 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename,
         if (!ios->ioproc)
         {
             int msg = PIO_MSG_CREATE_FILE;
-            char ncidp_present = ncidp ? 1 : 0;
             size_t len = strlen(filename);
+            char ncidp_present = ncidp ? 1 : 0;
 
             /* Send the message to the message handler. */
             PLOG((3, "msg %d ios->union_comm %d MPI_COMM_NULL %d", msg, ios->union_comm, MPI_COMM_NULL));
@@ -2067,6 +2076,7 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename,
             break;
 #endif
         }
+        PLOG((3, "create call complete file->fh %d", file->fh));
     }
 
     /* Broadcast and check the return code. */
@@ -2096,28 +2106,29 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename,
         PLOG((3, "createfile bcast pio_next_ncid %d", pio_next_ncid));
     }
 
-    /* With the netCDF integration layer, the ncid is assigned for PIO
-     * by the netCDF dispatch layer code. So it is passed in. In
-     * normal PIO operation, the ncid is generated here. */
+    /* Assign the PIO ncid. */
+    file->pio_ncid = pio_next_ncid++;
+
+    /* With the netCDF integration layer, we must override the ncid
+     * generated on the computation processors, with the ncid
+     * generated by the I/O processors (which know about all open
+     * files). In normal PIO operation, the ncid is generated here. */
+#ifdef NETCDF_INTEGRATION
     if (use_ext_ncid)
     {
-        /* Use the ncid passed in from the netCDF dispatch code. */
-        file->pio_ncid = *ncidp;
-
-        /* To prevent PIO from reusing the same ncid, if someone
-         * starting mingling netcdf integration PIO and regular PIO
-         * code. */
-        pio_next_ncid = file->pio_ncid + 1;
+        /* The ncid was assigned on the computational
+         * processors. Change the ncid to one that I/O and
+         * computational components can agree on. */
+        if ((ierr = nc4_file_change_ncid(*ncidp, file->pio_ncid)))
+            return pio_err(NULL, file, ierr, __FILE__, __LINE__);
+        file->pio_ncid = file->pio_ncid << ID_SHIFT;
+        PLOG((2, "changed ncid to file->pio_ncid = %d", file->pio_ncid));
     }
-    else
-    {
-        /* Assign the PIO ncid. */
-        file->pio_ncid = pio_next_ncid++;
-        PLOG((2, "file->fh = %d file->pio_ncid = %d", file->fh, file->pio_ncid));
+#endif /* NETCDF_INTEGRATION */
+    PLOG((2, "file->fh = %d file->pio_ncid = %d", file->fh, file->pio_ncid));
 
-        /* Return the ncid to the caller. */
-        *ncidp = file->pio_ncid;
-    }
+    /* Return the ncid to the caller. */
+    *ncidp = file->pio_ncid;
 
     /* Add the struct with this files info to the global list of
      * open files. */

tests/fncint/Makefile.am

@@ -23,10 +23,10 @@ check_PROGRAMS = ftst_pio ftst_pio_orig tst_c_pio
 ftst_pio_SOURCES = ftst_pio.f90
 ftst_pio_orig_SOURCES = ftst_pio_orig.f90
 
-if RUN_TESTS
-# Tests will run from a bash script.
-TESTS = run_tests.sh
-endif # RUN_TESTS
+# if RUN_TESTS
+# # Tests will run from a bash script.
+# TESTS = run_tests.sh
+# endif # RUN_TESTS
 
 # Distribute the test script.
 EXTRA_DIST = run_tests.sh

tests/ncint/Makefile.am

@@ -18,11 +18,6 @@ if RUN_TESTS
 TESTS = run_tests.sh
 endif # RUN_TESTS
 
-# if RUN_TESTS
-# # Tests will run from a bash script.
-# TESTS = run_tests.sh
-# endif # RUN_TESTS
-
 # Distribute the test script.
 EXTRA_DIST = run_tests.sh
 

tests/ncint/run_tests.sh

@@ -10,7 +10,7 @@ trap exit INT TERM
 
 printf 'running PIO tests...\n'
 
-PIO_TESTS='tst_pio_udf'
+PIO_TESTS='tst_pio_udf tst_pio_async'
 
 success1=true
 success2=true

tests/ncint/tst_pio_async.c

@@ -54,6 +54,8 @@ main(int argc, char **argv)
 
         /* Turn on logging for PIO library. */
         PIOc_set_log_level(3);
+        if (!my_rank)
+            nc_set_log_level(3);
 
         /* Initialize the intracomm. */
         if (nc_def_async(MPI_COMM_WORLD, num_io_procs, NULL, COMPONENT_COUNT,
@@ -63,23 +65,23 @@ main(int argc, char **argv)
         if (my_rank)
         {
             /* Create a file with a 3D record var. */
-            if (nc_create(FILE_NAME, NC_UDF0, &ncid)) PERR;
+            if (nc_create(FILE_NAME, NC_UDF0|NC_NETCDF4, &ncid)) PERR;
             /* if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR; */
             /* if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR; */
             /* if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR; */
             /* if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM3, dimid, &varid)) PERR; */
 
             /* Calculate a decomposition for distributed arrays. */
-            elements_per_pe = DIM_LEN_X * DIM_LEN_Y / ntasks;
-            if (!(compdof = malloc(elements_per_pe * sizeof(size_t))))
-                PERR;
-            for (i = 0; i < elements_per_pe; i++)
-                compdof[i] = my_rank * elements_per_pe + i;
+            /* elements_per_pe = DIM_LEN_X * DIM_LEN_Y / ntasks; */
+            /* if (!(compdof = malloc(elements_per_pe * sizeof(size_t)))) */
+            /*     PERR; */
+            /* for (i = 0; i < elements_per_pe; i++) */
+            /*     compdof[i] = my_rank * elements_per_pe + i; */
 
-            /* Create the PIO decomposition for this test. */
-            if (nc_def_decomp(iosysid, PIO_INT, NDIM2, &dimlen[1], elements_per_pe,
-                              compdof, &ioid, 1, NULL, NULL)) PERR;
-            free(compdof);
+            /* /\* Create the PIO decomposition for this test. *\/ */
+            /* if (nc_def_decomp(iosysid, PIO_INT, NDIM2, &dimlen[1], elements_per_pe, */
+            /*                   compdof, &ioid, 1, NULL, NULL)) PERR; */
+            /* free(compdof); */
 
             /* /\* Create some data on this processor. *\/ */
             /* if (!(my_data = malloc(elements_per_pe * sizeof(int)))) PERR; */
@@ -88,7 +90,8 @@ main(int argc, char **argv)
 
             /* /\* Write some data with distributed arrays. *\/ */
             /* if (nc_put_vard_int(ncid, varid, ioid, 0, my_data)) PERR; */
-            /* if (nc_close(ncid)) PERR; */
+            if (nc_enddef(ncid)) PERR;
+            if (nc_close(ncid)) PERR;
 
             /* /\* Check that our user-defined format has been added. *\/ */
             /* if (nc_inq_user_format(NC_UDF0, &disp_in, NULL)) PERR; */
@@ -106,12 +109,12 @@ main(int argc, char **argv)
             /*     if (data_in[i] != my_data[i]) PERR; */
 
             /* Close file. */
-            if (nc_close(ncid)) PERR;
+            /* if (nc_close(ncid)) PERR; */
 
             /* /\* Free resources. *\/ */
             /* free(data_in); */
             /* free(my_data); */
-            if (nc_free_decomp(ioid)) PERR;
+            /* if (nc_free_decomp(ioid)) PERR; */
             if (nc_free_iosystem(iosysid)) PERR;
         }
     }

tests/ncint/tst_pio_udf.c

@@ -61,7 +61,7 @@ main(int argc, char **argv)
         int i;
 
         /* Turn on logging for PIO library. */
-        /* PIOc_set_log_level(3); */
+        PIOc_set_log_level(3);
 
         /* Initialize the intracomm. */
         if (nc_def_iosystemm(MPI_COMM_WORLD, 1, 1, 0, 0, &iosysid)) PERR;