Merge pull request #6 from NCAR/master

Updating my fork

doc/source/Installing.txt

@@ -20,7 +20,7 @@ The PIO code is currently stored on github at <https://github.com/PARALLELIO/Par
 
 ### Dependencies ###
 
-PIO can use NetCDF (version 4.3.3+) and/or PnetCDF (version 1.6.0+) for I/O.
+PIO can use NetCDF (version 4.3.3+) and/or PnetCDF (version 1.6.1+) for I/O.
 Ideally, the NetCDF version should be built with MPI, which requires that it
 be linked with an MPI-enabled version of HDF5.  Optionally, NetCDF can be 
 built with DAP support, which introduces a dependency on CURL.  Additionally,

doc/source/mach_walkthrough.txt

@@ -44,7 +44,7 @@ Modules required for installation depend on your prefered compiler. Issue the co
    %> module load git/2.3.0<br>
    %> module load cmake/3.0.2<br>
    %> module load netcdf-mpi/4.3.3.1<br>
-   %> module load pnetcdf/1.6.0<br>
+   %> module load pnetcdf/1.6.1<br>
 
 + GNU
 
@@ -54,7 +54,7 @@ Modules required for installation depend on your prefered compiler. Issue the co
    %> module load git/2.3.0<br>
    %> module load cmake/3.0.2<br>
    %> module load netcdf-mpi/4.3.3.1<br>
-   %> module load pnetcdf/1.6.0<br>
+   %> module load pnetcdf/1.6.1<br>
 
 + PGI
 
@@ -64,7 +64,7 @@ Modules required for installation depend on your prefered compiler. Issue the co
    %> module load git/2.3.0<br>
    %> module load cmake/3.0.2<br>
    %> module load netcdf-mpi/4.3.3.1<br>
-   %> module load pnetcdf/1.6.0<br>
+   %> module load pnetcdf/1.6.1<br>
 
 <li>Environment Variables
 
@@ -103,7 +103,7 @@ Modules required for installation depend on your prefered compiler. Issue the co
   %> module load cmake/3.0.0<br>
   %> module load cray-hdf5-parallel/1.8.14<br>
   %> module load cray-netcdf-hdf5parallel/4.3.3.1<br>
-  %> module load cray-parallel-netcdf/1.6.0<br>
+  %> module load cray-parallel-netcdf/1.6.1<br>
 
 + GNU
 
@@ -117,7 +117,7 @@ Modules required for installation depend on your prefered compiler. Issue the co
   %> module load cmake/3.0.0<br>
   %> module load cray-hdf5-parallel/1.8.14<br>
   %> module load cray-netcdf-hdf5parallel/4.3.3.1<br>
-  %> module load cray-parallel-netcdf/1.6.0<br>
+  %> module load cray-parallel-netcdf/1.6.1<br>
 
 + Cray
 
@@ -132,7 +132,7 @@ Modules required for installation depend on your prefered compiler. Issue the co
   %> module load cmake/3.0.0<br>
   %> module load cray-hdf5-parallel/1.8.14<br>
   %> module load cray-netcdf-hdf5parallel/4.3.3.1<br>
-  %> module load cray-parallel-netcdf/1.6.0<br>
+  %> module load cray-parallel-netcdf/1.6.1<br>
 
 <li>Environment Variables
 
@@ -178,7 +178,7 @@ And then set the following environment variables to add in the rest of the libra
   %> setenv LIBZ /soft/libraries/alcf/current/xl/ZLIB<br>
   %> setenv HDF5 /soft/libraries/hdf5/1.8.14/cnk-xl/V1R2M2-20150213<br>
   %> setenv NETCDF /soft/libraries/netcdf/4.3.3-f4.4.1/cnk-xl/V1R2M2-20150213<br>
-  %> setenv PNETCDF /soft/libraries/pnetcdf/1.6.0/cnk-xl/V1R2M2-20150213<br>
+  %> setenv PNETCDF /soft/libraries/pnetcdf/1.6.1/cnk-xl/V1R2M2-20150213<br>
   %> setenv CC /soft/compilers/wrappers/xl/mpixlc_r<br>
   %> setenv FC /soft/compilers/wrappers/xl/mpixlf90_r<br>
 
@@ -209,7 +209,7 @@ Modules required for installation depend on your prefered compiler. Issue the co
   %> module load cmake<br>
   %> module load cray-hdf5-parallel/1.8.14<br>
   %> module load cray-netcdf-hdf5parallel/4.3.3.1<br>
-  %> module load cray-parallel-netcdf/1.6.0<br>
+  %> module load cray-parallel-netcdf/1.6.1<br>
 
 + PGI
 
@@ -219,7 +219,7 @@ Modules required for installation depend on your prefered compiler. Issue the co
   %> module load cmake<br>
   %> module load cray-hdf5-parallel/1.8.14<br>
   %> module load cray-netcdf-hdf5parallel/4.3.3.1<br>
-  %> module load cray-parallel-netcdf/1.6.0<br>
+  %> module load cray-parallel-netcdf/1.6.1<br>
 
 <li>Environment Variables
 
@@ -477,5 +477,5 @@ variables to these tests will increase the time significantly.
 
 </ol>
 
-_Last updated: 11-18-2015_
+_Last updated: 05-16-2016_
 */

examples/c/example2.c

@@ -48,8 +48,8 @@
  * responsibilty for writing and reading them will be spread between
  * all the processors used to run this example. */
 /**@{*/
-#define X_DIM_LEN 400
-#define Y_DIM_LEN 400
+#define X_DIM_LEN 20
+#define Y_DIM_LEN 30
 /**@}*/
 
 /** The number of timesteps of data to write. */

examples/c/valsupp_example1.supp

@@ -0,0 +1,15 @@
+{
+   cond_jump_1
+   Memcheck:Cond
+   fun:MPIC_Waitall
+   fun:MPIR_Alltoallw_intra
+   fun:MPIR_Alltoallw
+   fun:MPIR_Alltoallw_impl
+   fun:PMPI_Alltoallw
+   fun:pio_swapm
+   fun:rearrange_comp2io
+   fun:PIOc_write_darray_multi
+   fun:flush_buffer
+   fun:PIOc_sync
+   fun:main
+}

src/clib/pio.h

@@ -139,43 +139,89 @@ typedef struct io_desc_t
 */
 typedef struct iosystem_desc_t 
 {
+  /** The ID of this iosystem_desc_t. */
   int iosysid;
+
+  /** This is an MPI intra communicator that includes all the tasks in
+   * both the IO and the computation communicators. */
   MPI_Comm union_comm;
+
+  /** This is an MPI intra communicator that includes all the tasks
+   * involved in IO. */
   MPI_Comm io_comm;
+
+  /** This is an MPI intra communicator that includes all the tasks
+   * involved in computation. */
   MPI_Comm comp_comm;
+
+  /** This is an MPI inter communicator between IO communicator and
+   * computation communicator. */
   MPI_Comm intercomm;
+
+  /** This is a copy (but not an MPI copy) of either the comp (for
+   * non-async) or the union (for async) communicator. */
   MPI_Comm my_comm;
 
   /** This MPI group contains the processors involved in
-   * computation. It is created in PIOc_Init_Intracomm(), and freed my
-   * PIO_finalize(). */
+   * computation. */
   MPI_Group compgroup;
 
-  /** This MPI group contains the processors involved in I/O. It is
-   * created in PIOc_Init_Intracomm(), and freed my PIOc_finalize(). */
+  /** This MPI group contains the processors involved in I/O. */
   MPI_Group iogroup;
-
+
+  /** The number of tasks in the IO communicator. */
   int num_iotasks;
+
+  /** The number of tasks in the computation communicator. */
   int num_comptasks;
 
+  /** Rank of this task in the union communicator. */
   int union_rank;
+
+  /** The rank of this process in the computation communicator, or -1
+   * if this process is not part of the computation communicator. */
   int comp_rank;
+
+  /** The rank of this process in the IO communicator, or -1 if this
+   * process is not part of the IO communicator. */
   int io_rank;
 
-  bool iomaster;
-  bool compmaster;
+  /** Set to MPI_ROOT if this task is the master of IO communicator, 0
+   * otherwise. */
+  int iomaster;
 
+  /** Set to MPI_ROOT if this task is the master of comp communicator, 0
+   * otherwise. */
+  int compmaster;
+
+  /** Rank of IO root task (which is rank 0 in io_comm) in the union
+   * communicator. */
   int ioroot;
+
+  /** Rank of computation root task (which is rank 0 in
+   * comm_comms[cmp]) in the union communicator. */
   int comproot;
+
+  /** An array of the ranks of all IO tasks within the union
+   * communicator. */
   int *ioranks;
 
+  /** Controls handling errors. */
   int error_handler;
+
+  /** ??? */
   int default_rearranger;
 
+  /** True if asynchronous interface is in use. */
   bool async_interface;
+
+  /** True if this task is a member of the IO communicator. */
   bool ioproc;
-
+
+  /** MPI Info object. */
   MPI_Info info;
+
+  /** Pointer to the next iosystem_desc_t in the list. */
   struct iosystem_desc_t *next;
 } iosystem_desc_t;
 
@@ -491,7 +537,6 @@ int PIOc_set_blocksize(const int newblocksize);
   int PIOc_put_var_short (int ncid, int varid, const short *op) ;
   int PIOc_get_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], char *buf) ;
   int PIOc_put_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const int *op)  ;
-  int PIOc_put_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const int *op)  ;
 
   int PIOc_put_var1_ushort (int ncid, int varid, const PIO_Offset index[], const unsigned short *op); 
   int PIOc_put_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const char *op);  

src/clib/pio_file.c

@@ -61,11 +61,12 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype,
     if(ios->comp_rank==0) 
       mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
     len = strlen(filename);
-    mpierr = MPI_Bcast((void *) filename,len, MPI_CHAR, ios->compmaster, ios->intercomm);
-    mpierr = MPI_Bcast(&(file->iotype), 1, MPI_INT,  ios->compmaster, ios->intercomm);
-    mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT,  ios->compmaster, ios->intercomm);
+    mpierr = MPI_Bcast(&len, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+    mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+    mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+    mpierr = MPI_Bcast(&file->mode, 1, MPI_INT,  ios->compmaster, ios->intercomm);
   }
-  
+
   if(ios->ioproc){
 
     switch(file->iotype){
@@ -113,7 +114,7 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype,
     // If we failed to open a file due to an incompatible type of NetCDF, try it 
     // once with just plain old basic NetCDF
 #ifdef _NETCDF
-    if(ierr == NC_ENOTNC && (file->iotype != PIO_IOTYPE_NETCDF)) {
+    if((ierr == NC_ENOTNC || ierr == NC_EINVAL) && (file->iotype != PIO_IOTYPE_NETCDF)) {
         if(ios->iomaster) printf("PIO2 pio_file.c retry NETCDF\n"); 
 	// reset ierr on all tasks
 	ierr = PIO_NOERR;
@@ -130,8 +131,10 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype,
 
   ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
 
-  if(ierr==PIO_NOERR){
-    mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT,  ios->ioroot, ios->union_comm);
+  if (!ierr) {
+    mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->ioroot, ios->union_comm);
+    mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->ioroot, ios->union_comm);
+    *ncidp = file->fh;
     pio_add_to_file_list(file);
     *ncidp = file->fh;
   }
@@ -199,10 +202,11 @@ int PIOc_createfile(const int iosysid, int *ncidp,  int *iotype,
 
   if(ios->async_interface && ! ios->ioproc){
     if(ios->comp_rank==0) 
-      mpierr = MPI_Send( &msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+      mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
     len = strlen(filename);
-    mpierr = MPI_Bcast((void *) filename,len, MPI_CHAR, ios->compmaster, ios->intercomm);
-    mpierr = MPI_Bcast(&(file->iotype), 1, MPI_INT,  ios->compmaster, ios->intercomm);
+    mpierr = MPI_Bcast(&len, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+    mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+    mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT,  ios->compmaster, ios->intercomm);
     mpierr = MPI_Bcast(&file->mode, 1, MPI_INT,  ios->compmaster, ios->intercomm);
   }
 
@@ -248,8 +252,10 @@ int PIOc_createfile(const int iosysid, int *ncidp,  int *iotype,
   ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
 
   if(ierr == PIO_NOERR){
-    mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT,  ios->ioroot, ios->union_comm);
+    mpierr = MPI_Bcast(&file->mode, 1, MPI_INT,  ios->ioroot, ios->union_comm);
     file->mode = file->mode | PIO_WRITE;  // This flag is implied by netcdf create functions but we need to know if its set
+    mpierr = MPI_Bcast(&file->fh, 1, MPI_INT,  ios->ioroot, ios->union_comm);
+    *ncidp = file->fh;
     pio_add_to_file_list(file);
     *ncidp = file->fh;
   }
@@ -279,14 +285,20 @@ int PIOc_closefile(int ncid)
   if(file == NULL)
     return PIO_EBADID;
   ios = file->iosystem;
-  msg = 0;
+  msg = PIO_MSG_CLOSE_FILE;
   if((file->mode & PIO_WRITE)){
     PIOc_sync(ncid);
   }
-  if(ios->async_interface && ! ios->ioproc){
-    if(ios->comp_rank==0) 
-      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
-    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+
+  /* If async is in use and this is a comp tasks, then the compmaster
+   * sends a msg to the pio_msg_handler running on the IO master and
+   * waiting for a message. Then broadcast the ncid over the intercomm
+   * to the IO tasks. */
+  if(ios->async_interface && !ios->ioproc){
+    if(ios->comp_rank==0) {
+      mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+    }
+    mpierr = MPI_Bcast(&(file->fh), 1, MPI_INT, ios->compmaster, ios->intercomm);
   }
 
   if(ios->ioproc){
@@ -325,7 +337,6 @@ int PIOc_closefile(int ncid)
 
   int iret =  pio_delete_file_from_list(ncid);
 
-
   return ierr;
 }
 
@@ -342,19 +353,22 @@ int PIOc_deletefile(const int iosysid, const char filename[])
   int mpierr;
   int chkerr;
   iosystem_desc_t *ios;
+  size_t len;
 
   ierr = PIO_NOERR;
   ios = pio_get_iosystem_from_id(iosysid);
 
   if(ios == NULL)
     return PIO_EBADID;
 
-  msg = 0;
+  msg = PIO_MSG_DELETE_FILE;
 
   if(ios->async_interface && ! ios->ioproc){
     if(ios->comp_rank==0) 
       mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
-    //    mpierr = MPI_Bcast(iosysid,1, MPI_INT, ios->compmaster, ios->intercomm);
+    len = strlen(filename);
+    mpierr = MPI_Bcast(&len, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+    mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
   }
   // The barriers are needed to assure that no task is trying to operate on the file while it is being deleted.
   if(ios->ioproc){
@@ -371,8 +385,6 @@ int PIOc_deletefile(const int iosysid, const char filename[])
   }
   //   Special case - always broadcast the return from the  
   MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm);
-
-
 
   return ierr;
 }
@@ -405,9 +417,9 @@ int PIOc_sync (int ncid)
   msg = PIO_MSG_SYNC;
 
   if(ios->async_interface && ! ios->ioproc){
-    if(ios->compmaster) 
+    if(ios->comp_rank == 0) 
       mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
-    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
   }
 
   if((file->mode & PIO_WRITE)){