Merge pull request ESMCI#1595 from NCAR/ejh_more_ncint

more netcdf integration, plus new test

examples/c/darray_async.c

@@ -249,11 +249,11 @@ int main(int argc, char* argv[])
         ERR(ret);
 
 
-    /* The rest of the code executes on computation tasks only. As
-     * PIO functions are called on the computation tasks, the
-     * async system will call them on the IO task. When the
-     * computation tasks call PIO_finalize(), the IO task will get
-     * a message to shut itself down. */
+    /* The rest of the code executes on computation tasks only. As PIO
+     * functions are called on the computation tasks, the async system
+     * will call them on the IO task. When the computation tasks call
+     * PIO_finalize(), the IO task will get a message to shut itself
+     * down. */
     if (comp_task)
     {
         /* PIO_Offset elements_per_pe; /\* Array elements per processing unit. *\/ */

src/clib/pio.h

@@ -311,7 +311,8 @@ typedef struct io_desc_t
      * everywhere (false) */
     bool needsfill;
 
-    /** If the map is not monotonically increasing we will need to sort it. */
+    /** If the map is not monotonically increasing we will need to
+     * sort it. */
     bool needssort;
 
     /** The maximum number of bytes of this iodesc before flushing. */

src/clib/pio_darray.c

@@ -903,6 +903,9 @@ PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen,
     pio_start_mpe_log(DARRAY_READ);
 #endif /* USE_MPE */
 
+    PLOG((1, "PIOc_read_darray ncid %d varid %d ioid %d arraylen %ld ",
+          ncid, varid, ioid, arraylen));
+
     /* Get the file info. */
     if ((ierr = pio_get_file(ncid, &file)))
         return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
@@ -942,12 +945,15 @@ PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen,
         return pio_err(NULL, NULL, PIO_EBADIOTYPE, __FILE__, __LINE__);
     }
 
+    /* If the map is not monotonically increasing we will need to sort
+     * it. */
+    PLOG((3, "iodesc->needssort %d", iodesc->needssort));
     if (iodesc->needssort)
     {
-        if (!(tmparray = malloc(iodesc->piotype_size*iodesc->maplen)))
+        if (!(tmparray = malloc(iodesc->piotype_size * iodesc->maplen)))
             return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
-        for(int m=0; m<iodesc->maplen;m++)
-            ((int *) array)[m] = -1;
+        for (int m = 0; m < iodesc->maplen; m++)
+            ((int *)array)[m] = -1;
     }
     else
         tmparray = array;
@@ -956,6 +962,7 @@ PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen,
     if ((ierr = rearrange_io2comp(ios, iodesc, iobuf, tmparray)))
         return pio_err(ios, file, ierr, __FILE__, __LINE__);
 
+    /* If we need to sort the map, do it. */
     if (iodesc->needssort)
     {
         pio_sorted_copy(tmparray, array, iodesc, 1, 1);
@@ -970,5 +977,7 @@ PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen,
     pio_stop_mpe_log(DARRAY_READ, __func__);
 #endif /* USE_MPE */
 
+    PLOG((2, "done with PIOc_read_darray()"));
+
     return PIO_NOERR;
 }

src/clib/pio_darray_int.c

@@ -1714,6 +1714,7 @@ pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid,
         return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
 #endif /* TIMING */
 
+    PLOG((2, "pio_read_darray_nc_serial complete ierr %d", ierr));
     return PIO_NOERR;
 }
 

src/clib/pio_msg.c

@@ -1970,6 +1970,7 @@ int open_file_handler(iosystem_desc_t *ios)
     int len;
     int iotype;
     int mode;
+    int use_ext_ncid;
     int mpierr;
 
     PLOG((1, "open_file_handler comproot = %d", ios->comproot));
@@ -1990,13 +1991,26 @@ int open_file_handler(iosystem_desc_t *ios)
         return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&mode, 1, MPI_INT, 0, ios->intercomm)))
         return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&use_ext_ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
-    PLOG((2, "open_file_handler got parameters len = %d filename = %s iotype = %d mode = %d",
-          len, filename, iotype, mode));
+    PLOG((2, "len %d filename %s iotype %d mode %d use_ext_ncid %d",
+          len, filename, iotype, mode, use_ext_ncid));
 
     /* Call the open file function. Errors are handled within
      * function, so return code can be ignored. */
-    PIOc_openfile_retry(ios->iosysid, &ncid, &iotype, filename, mode, 0, 0);
+    if (use_ext_ncid)
+    {
+#ifdef NETCDF_INTEGRATION
+        PLOG((2, "about to call nc_create()"));
+        nc_open(filename, mode|NC_UDF0, &ncid);
+#endif /* NETCDF_INTEGRATION */
+    }
+    else
+    {
+        PIOc_openfile_retry(ios->iosysid, &ncid, &iotype, filename, mode, 0,
+                            use_ext_ncid);
+    }
 
     return PIO_NOERR;
 }

src/clib/pio_nc.c

@@ -926,8 +926,10 @@ PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp,
                 char my_name[NC_MAX_NAME + 1];
                 nc_type my_xtype;
                 int my_ndims = 0, my_dimids[ndims], my_natts = 0;
-                ierr = nc_inq_var(file->fh, varid, my_name, &my_xtype, &my_ndims, my_dimids, &my_natts);
-                PLOG((3, "my_name = %s my_xtype = %d my_ndims = %d my_natts = %d",  my_name, my_xtype, my_ndims, my_natts));
+                ierr = nc_inq_var(file->fh, varid, my_name, &my_xtype, &my_ndims, my_dimids,
+                                  &my_natts);
+                PLOG((3, "my_name = %s my_xtype = %d my_ndims = %d my_natts = %d",  my_name,
+                      my_xtype, my_ndims, my_natts));
                 if (!ierr)
                 {
                     if (name)

src/clib/pio_rearrange.c

@@ -450,8 +450,9 @@ define_iodesc_datatypes(iosystem_desc_t *ios, io_desc_t *iodesc)
     int ret; /* Return value. */
 
     pioassert(ios && iodesc, "invalid input", __FILE__, __LINE__);
-    PLOG((1, "define_iodesc_datatypes ios->ioproc = %d iodesc->rtype is %sNULL, iodesc->nrecvs",
-          ios->ioproc, iodesc->rtype ? "not " : "", iodesc->nrecvs));
+    PLOG((1, "define_iodesc_datatypes ios->ioproc = %d iodesc->rtype is %sNULL, "
+          "iodesc->nrecvs %d", ios->ioproc, iodesc->rtype ? "not " : "",
+          iodesc->nrecvs));
 
     /* Set up the to transfer data to and from the IO tasks. */
     if (ios->ioproc)
@@ -1014,6 +1015,7 @@ rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf,
 
     /* Check inputs. */
     pioassert(ios && iodesc, "invalid input", __FILE__, __LINE__);
+    PLOG((2, "rearrange_io2comp iodesc->rearranger %d", iodesc->rearranger));
 
 #ifdef TIMING
     /* Start timer if desired. */
@@ -1033,11 +1035,11 @@ rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf,
         mycomm = iodesc->subset_comm;
         niotasks = 1;
     }
-    PLOG((3, "niotasks = %d", niotasks));
 
     /* Get the size of this communicator. */
     if ((mpierr = MPI_Comm_size(mycomm, &ntasks)))
         return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    PLOG((3, "niotasks %d ntasks %d", niotasks, ntasks));
 
     /* Define the MPI data types that will be used for this
      * io_desc_t. */
@@ -1106,7 +1108,8 @@ rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf,
         }
     }
 
-    /* Data in sbuf on the ionodes is sent to rbuf on the compute nodes */
+    /* Data in sbuf on the ionodes is sent to rbuf on the compute
+     * nodes. */
     if ((ret = pio_swapm(sbuf, sendcounts, sdispls, sendtypes, rbuf, recvcounts,
                          rdispls, recvtypes, mycomm, &iodesc->rearr_opts.io2comp)))
         return pio_err(ios, NULL, ret, __FILE__, __LINE__);

src/clib/pioc_support.c

@@ -2573,6 +2573,8 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename,
                 mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm);
             if (!mpierr)
                 mpierr = MPI_Bcast(&mode, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&use_ext_ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
         }
 
         /* Handle MPI errors. */
@@ -2750,7 +2752,19 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename,
     /* With the netCDF integration layer, the ncid is assigned for PIO
      * by the netCDF dispatch layer code. So it is passed in. In
      * normal PIO operation, the ncid is generated here. */
-    if (!use_ext_ncid)
+#ifdef NETCDF_INTEGRATION
+    if (use_ext_ncid)
+    {
+        /* The ncid was assigned on the computational
+         * processors. Change the ncid to one that I/O and
+         * computational components can agree on. */
+        if ((ierr = nc4_file_change_ncid(*ncidp, file->pio_ncid)))
+            return pio_err(NULL, file, ierr, __FILE__, __LINE__);
+        file->pio_ncid = file->pio_ncid << ID_SHIFT;
+        PLOG((2, "changed ncid to file->pio_ncid = %d", file->pio_ncid));
+    }
+    else
+#endif /* NETCDF_INTEGRATION */
     {
         /* Create the ncid that the user will see. This is necessary
          * because otherwise ncids will be reused if files are opened
@@ -2760,16 +2774,6 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename,
         /* Return the PIO ncid to the user. */
         *ncidp = file->pio_ncid;
     }
-    else
-    {
-        /* Use the ncid passed in from the netCDF dispatch code. */
-        file->pio_ncid = *ncidp;
-
-        /* To prevent PIO from reusing the same ncid, if someone
-         * starting mingling netcdf integration PIO and regular PIO
-         * code. */
-        pio_next_ncid = file->pio_ncid + 1;
-    }
 
     /* Add this file to the list of currently open files. */
     pio_add_to_file_list(file);

src/ncint/ncint_pio.c

@@ -85,6 +85,10 @@ nc_def_async(MPI_Comm world, int num_io_procs, int *io_proc_list,
     if ((ret = PIO_NCINT_initialize()))
         return ret;
 
+    /* Change error handling so we can test inval parameters. */
+    if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
+        return ret;
+
     /* Call the PIOc_ function to initialize the intracomm. */
     if ((ret = PIOc_init_async(world, num_io_procs, io_proc_list,
                                component_count, num_procs_per_comp, proc_list,

tests/cunit/CMakeLists.txt

@@ -109,6 +109,8 @@ if (NOT PIO_USE_MPISERIAL)
     target_link_libraries (test_async_multi2 pioc)
     add_executable (test_async_manyproc EXCLUDE_FROM_ALL test_async_manyproc.c test_common.c)
     target_link_libraries (test_async_manyproc pioc)
+    add_executable (test_async_1d EXCLUDE_FROM_ALL test_async_1d.c)
+    target_link_libraries (test_async_1d pioc)
   endif ()
 endif ()
 add_executable (test_spmd EXCLUDE_FROM_ALL test_spmd.c test_common.c)
@@ -140,6 +142,7 @@ if(PIO_USE_MALLOC)
   add_dependencies (tests test_async_multicomp)
   add_dependencies (tests test_async_multi2)
   add_dependencies (tests test_async_manyproc)
+  add_dependencies (tests test_async_1d)
 endif ()
 
 # Test Timeout in seconds.

tests/cunit/Makefile.am

@@ -19,7 +19,7 @@ test_decomp_uneven test_decomps test_rearr test_darray_async_simple	\
 test_darray_async test_darray_async_many test_darray_2sync		\
 test_async_multicomp test_async_multi2 test_async_manyproc		\
 test_darray_fill test_decomp_frame test_perf2 test_async_perf		\
-test_darray_vard
+test_darray_vard test_async_1d
 
 if RUN_TESTS
 # Tests will run from a bash script.
@@ -64,6 +64,7 @@ test_decomp_frame_SOURCES = test_decomp_frame.c test_common.c pio_tests.h
 test_perf2_SOURCES = test_perf2.c test_common.c pio_tests.h
 test_async_perf_SOURCES = test_async_perf.c test_common.c pio_tests.h
 test_darray_vard_SOURCES = test_darray_vard.c test_common.c pio_tests.h
+test_async_1d_SOURCES = test_async_1d.c pio_tests.h
 
 # Distribute the test script.
 EXTRA_DIST = run_tests.sh CMakeLists.txt test_darray_frame.c

tests/cunit/run_tests.sh

@@ -17,7 +17,7 @@ PIO_TESTS='test_intercomm2 test_async_mpi test_spmd test_rearr test_async_simple
 'test_darray_multivar test_darray_multivar2 test_darray_multivar3 test_darray_1d '\
 'test_darray_3d test_decomp_uneven test_decomps test_darray_async_simple '\
 'test_darray_async test_darray_async_many test_darray_2sync test_async_multicomp '\
-'test_darray_fill test_darray_vard'
+'test_darray_fill test_darray_vard test_async_1d'
 
 success1=true
 success2=true