diff --git a/doc/source/Installing.txt b/doc/source/Installing.txt
index 0c0020d39ac0..6f2811522488 100644
--- a/doc/source/Installing.txt
+++ b/doc/source/Installing.txt
@@ -20,7 +20,7 @@ The PIO code is currently stored on github at <https://github.com/PARALLELIO/Par
 
 ### Dependencies ###
 
-PIO can use NetCDF (version 4.3.3+) and/or PnetCDF (version 1.6.0+) for I/O.
+PIO can use NetCDF (version 4.3.3+) and/or PnetCDF (version 1.6.1+) for I/O.
 Ideally, the NetCDF version should be built with MPI, which requires that it
 be linked with an MPI-enabled version of HDF5.  Optionally, NetCDF can be 
 built with DAP support, which introduces a dependency on CURL.  Additionally,
diff --git a/doc/source/mach_walkthrough.txt b/doc/source/mach_walkthrough.txt
index 7741c44e8e4c..a208e132956c 100644
--- a/doc/source/mach_walkthrough.txt
+++ b/doc/source/mach_walkthrough.txt
@@ -44,7 +44,7 @@ Modules required for installation depend on your prefered compiler. Issue the co
    %> module load git/2.3.0<br>
    %> module load cmake/3.0.2<br>
    %> module load netcdf-mpi/4.3.3.1<br>
-   %> module load pnetcdf/1.6.0<br>
+   %> module load pnetcdf/1.6.1<br>
 
 + GNU
 
@@ -54,7 +54,7 @@ Modules required for installation depend on your prefered compiler. Issue the co
    %> module load git/2.3.0<br>
    %> module load cmake/3.0.2<br>
    %> module load netcdf-mpi/4.3.3.1<br>
-   %> module load pnetcdf/1.6.0<br>
+   %> module load pnetcdf/1.6.1<br>
 
 + PGI
 
@@ -64,7 +64,7 @@ Modules required for installation depend on your prefered compiler. Issue the co
    %> module load git/2.3.0<br>
    %> module load cmake/3.0.2<br>
    %> module load netcdf-mpi/4.3.3.1<br>
-   %> module load pnetcdf/1.6.0<br>
+   %> module load pnetcdf/1.6.1<br>
 
 <li>Environment Variables
 
@@ -103,7 +103,7 @@ Modules required for installation depend on your prefered compiler. Issue the co
   %> module load cmake/3.0.0<br>
   %> module load cray-hdf5-parallel/1.8.14<br>
   %> module load cray-netcdf-hdf5parallel/4.3.3.1<br>
-  %> module load cray-parallel-netcdf/1.6.0<br>
+  %> module load cray-parallel-netcdf/1.6.1<br>
 
 + GNU
 
@@ -117,7 +117,7 @@ Modules required for installation depend on your prefered compiler. Issue the co
   %> module load cmake/3.0.0<br>
   %> module load cray-hdf5-parallel/1.8.14<br>
   %> module load cray-netcdf-hdf5parallel/4.3.3.1<br>
-  %> module load cray-parallel-netcdf/1.6.0<br>
+  %> module load cray-parallel-netcdf/1.6.1<br>
 
 + Cray
 
@@ -132,7 +132,7 @@ Modules required for installation depend on your prefered compiler. Issue the co
   %> module load cmake/3.0.0<br>
   %> module load cray-hdf5-parallel/1.8.14<br>
   %> module load cray-netcdf-hdf5parallel/4.3.3.1<br>
-  %> module load cray-parallel-netcdf/1.6.0<br>
+  %> module load cray-parallel-netcdf/1.6.1<br>
 
 <li>Environment Variables
 
@@ -178,7 +178,7 @@ And then set the following environment variables to add in the rest of the libra
   %> setenv LIBZ /soft/libraries/alcf/current/xl/ZLIB<br>
   %> setenv HDF5 /soft/libraries/hdf5/1.8.14/cnk-xl/V1R2M2-20150213<br>
   %> setenv NETCDF /soft/libraries/netcdf/4.3.3-f4.4.1/cnk-xl/V1R2M2-20150213<br>
-  %> setenv PNETCDF /soft/libraries/pnetcdf/1.6.0/cnk-xl/V1R2M2-20150213<br>
+  %> setenv PNETCDF /soft/libraries/pnetcdf/1.6.1/cnk-xl/V1R2M2-20150213<br>
   %> setenv CC /soft/compilers/wrappers/xl/mpixlc_r<br>
   %> setenv FC /soft/compilers/wrappers/xl/mpixlf90_r<br>
 
@@ -209,7 +209,7 @@ Modules required for installation depend on your prefered compiler. Issue the co
   %> module load cmake<br>
   %> module load cray-hdf5-parallel/1.8.14<br>
   %> module load cray-netcdf-hdf5parallel/4.3.3.1<br>
-  %> module load cray-parallel-netcdf/1.6.0<br>
+  %> module load cray-parallel-netcdf/1.6.1<br>
 
 + PGI
 
@@ -219,7 +219,7 @@ Modules required for installation depend on your prefered compiler. Issue the co
   %> module load cmake<br>
   %> module load cray-hdf5-parallel/1.8.14<br>
   %> module load cray-netcdf-hdf5parallel/4.3.3.1<br>
-  %> module load cray-parallel-netcdf/1.6.0<br>
+  %> module load cray-parallel-netcdf/1.6.1<br>
 
 <li>Environment Variables
 
@@ -477,5 +477,5 @@ variables to these tests will increase the time significantly.
 
 </ol>
 
-_Last updated: 11-18-2015_
+_Last updated: 05-16-2016_
 */
diff --git a/examples/c/example2.c b/examples/c/example2.c
index f4b75d075b74..d86af67774c0 100644
--- a/examples/c/example2.c
+++ b/examples/c/example2.c
@@ -48,8 +48,8 @@
  * responsibilty for writing and reading them will be spread between
  * all the processors used to run this example. */
 /**@{*/
-#define X_DIM_LEN 400
-#define Y_DIM_LEN 400
+#define X_DIM_LEN 20
+#define Y_DIM_LEN 30
 /**@}*/
 
 /** The number of timesteps of data to write. */
diff --git a/examples/c/valsupp_example1.supp b/examples/c/valsupp_example1.supp
new file mode 100644
index 000000000000..63f3e073836d
--- /dev/null
+++ b/examples/c/valsupp_example1.supp
@@ -0,0 +1,15 @@
+{
+   cond_jump_1
+   Memcheck:Cond
+   fun:MPIC_Waitall
+   fun:MPIR_Alltoallw_intra
+   fun:MPIR_Alltoallw
+   fun:MPIR_Alltoallw_impl
+   fun:PMPI_Alltoallw
+   fun:pio_swapm
+   fun:rearrange_comp2io
+   fun:PIOc_write_darray_multi
+   fun:flush_buffer
+   fun:PIOc_sync
+   fun:main
+}
\ No newline at end of file
diff --git a/src/clib/pio.h b/src/clib/pio.h
index 5b8446bf21ef..5185a133b0e5 100644
--- a/src/clib/pio.h
+++ b/src/clib/pio.h
@@ -139,43 +139,89 @@ typedef struct io_desc_t
 */
 typedef struct iosystem_desc_t 
 {
+  /** The ID of this iosystem_desc_t. */
   int iosysid;
+
+  /** This is an MPI intra communicator that includes all the tasks in
+   * both the IO and the computation communicators. */
   MPI_Comm union_comm;
+
+  /** This is an MPI intra communicator that includes all the tasks
+   * involved in IO. */
   MPI_Comm io_comm;
+
+  /** This is an MPI intra communicator that includes all the tasks
+   * involved in computation. */
   MPI_Comm comp_comm;
+
+  /** This is an MPI inter communicator between IO communicator and
+   * computation communicator. */
   MPI_Comm intercomm;
+
+  /** This is a copy (but not an MPI copy) of either the comp (for
+   * non-async) or the union (for async) communicator. */
   MPI_Comm my_comm;
 
   /** This MPI group contains the processors involved in
-   * computation. It is created in PIOc_Init_Intracomm(), and freed my
-   * PIO_finalize(). */
+   * computation. */
   MPI_Group compgroup;
     
-  /** This MPI group contains the processors involved in I/O. It is
-   * created in PIOc_Init_Intracomm(), and freed my PIOc_finalize(). */
+  /** This MPI group contains the processors involved in I/O. */
   MPI_Group iogroup;
-  
+
+  /** The number of tasks in the IO communicator. */
   int num_iotasks;
+
+  /** The number of tasks in the computation communicator. */
   int num_comptasks;
 
+  /** Rank of this task in the union communicator. */
   int union_rank;
+
+  /** The rank of this process in the computation communicator, or -1
+   * if this process is not part of the computation communicator. */
   int comp_rank;
+
+  /** The rank of this process in the IO communicator, or -1 if this
+   * process is not part of the IO communicator. */
   int io_rank;
 
-  bool iomaster;
-  bool compmaster;
+  /** Set to MPI_ROOT if this task is the master of IO communicator, 0
+   * otherwise. */
+  int iomaster;
 
+  /** Set to MPI_ROOT if this task is the master of comp communicator, 0
+   * otherwise. */
+  int compmaster;
+
+  /** Rank of IO root task (which is rank 0 in io_comm) in the union
+   * communicator. */
   int ioroot;
+
+  /** Rank of computation root task (which is rank 0 in
+   * comm_comms[cmp]) in the union communicator. */
   int comproot;
+
+  /** An array of the ranks of all IO tasks within the union
+   * communicator. */
   int *ioranks;
 
+  /** Controls handling errors. */
   int error_handler;
+
+  /** ??? */
   int default_rearranger;
 
+  /** True if asynchronous interface is in use. */
   bool async_interface;
+
+  /** True if this task is a member of the IO communicator. */
   bool ioproc;
-  
+
+  /** MPI Info object. */
   MPI_Info info;
+
+  /** Pointer to the next iosystem_desc_t in the list. */
   struct iosystem_desc_t *next;
 } iosystem_desc_t;
 
@@ -491,7 +537,6 @@ int PIOc_set_blocksize(const int newblocksize);
   int PIOc_put_var_short (int ncid, int varid, const short *op) ;
   int PIOc_get_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], char *buf) ;
   int PIOc_put_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const int *op)  ;
-  int PIOc_put_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const int *op)  ;
 
   int PIOc_put_var1_ushort (int ncid, int varid, const PIO_Offset index[], const unsigned short *op); 
   int PIOc_put_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const char *op);  
diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c
index 38cf64a8e2da..461c8d2b4ca3 100644
--- a/src/clib/pio_file.c
+++ b/src/clib/pio_file.c
@@ -61,11 +61,12 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype,
     if(ios->comp_rank==0) 
       mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
     len = strlen(filename);
-    mpierr = MPI_Bcast((void *) filename,len, MPI_CHAR, ios->compmaster, ios->intercomm);
-    mpierr = MPI_Bcast(&(file->iotype), 1, MPI_INT,  ios->compmaster, ios->intercomm);
-    mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT,  ios->compmaster, ios->intercomm);
+    mpierr = MPI_Bcast(&len, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+    mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+    mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+    mpierr = MPI_Bcast(&file->mode, 1, MPI_INT,  ios->compmaster, ios->intercomm);
   }
-  
+
   if(ios->ioproc){
 
     switch(file->iotype){
@@ -113,7 +114,7 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype,
     // If we failed to open a file due to an incompatible type of NetCDF, try it 
     // once with just plain old basic NetCDF
 #ifdef _NETCDF
-    if(ierr == NC_ENOTNC && (file->iotype != PIO_IOTYPE_NETCDF)) {
+    if((ierr == NC_ENOTNC || ierr == NC_EINVAL) && (file->iotype != PIO_IOTYPE_NETCDF)) {
         if(ios->iomaster) printf("PIO2 pio_file.c retry NETCDF\n"); 
 	// reset ierr on all tasks
 	ierr = PIO_NOERR;
@@ -130,8 +131,10 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype,
 
   ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
 
-  if(ierr==PIO_NOERR){
-    mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT,  ios->ioroot, ios->union_comm);
+  if (!ierr) {
+    mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->ioroot, ios->union_comm);
+    mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->ioroot, ios->union_comm);
+    *ncidp = file->fh;
     pio_add_to_file_list(file);
     *ncidp = file->fh;
   }
@@ -199,10 +202,11 @@ int PIOc_createfile(const int iosysid, int *ncidp,  int *iotype,
 
   if(ios->async_interface && ! ios->ioproc){
     if(ios->comp_rank==0) 
-      mpierr = MPI_Send( &msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+      mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
     len = strlen(filename);
-    mpierr = MPI_Bcast((void *) filename,len, MPI_CHAR, ios->compmaster, ios->intercomm);
-    mpierr = MPI_Bcast(&(file->iotype), 1, MPI_INT,  ios->compmaster, ios->intercomm);
+    mpierr = MPI_Bcast(&len, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+    mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+    mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT,  ios->compmaster, ios->intercomm);
     mpierr = MPI_Bcast(&file->mode, 1, MPI_INT,  ios->compmaster, ios->intercomm);
   }
   
@@ -248,8 +252,10 @@ int PIOc_createfile(const int iosysid, int *ncidp,  int *iotype,
   ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
 
   if(ierr == PIO_NOERR){
-    mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT,  ios->ioroot, ios->union_comm);
+    mpierr = MPI_Bcast(&file->mode, 1, MPI_INT,  ios->ioroot, ios->union_comm);
     file->mode = file->mode | PIO_WRITE;  // This flag is implied by netcdf create functions but we need to know if its set
+    mpierr = MPI_Bcast(&file->fh, 1, MPI_INT,  ios->ioroot, ios->union_comm);
+    *ncidp = file->fh;
     pio_add_to_file_list(file);
     *ncidp = file->fh;
   }
@@ -279,14 +285,20 @@ int PIOc_closefile(int ncid)
   if(file == NULL)
     return PIO_EBADID;
   ios = file->iosystem;
-  msg = 0;
+  msg = PIO_MSG_CLOSE_FILE;
   if((file->mode & PIO_WRITE)){
     PIOc_sync(ncid);
   }
-  if(ios->async_interface && ! ios->ioproc){
-    if(ios->comp_rank==0) 
-      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
-    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+
+  /* If async is in use and this is a comp tasks, then the compmaster
+   * sends a msg to the pio_msg_handler running on the IO master and
+   * waiting for a message. Then broadcast the ncid over the intercomm
+   * to the IO tasks. */
+  if(ios->async_interface && !ios->ioproc){
+    if(ios->comp_rank==0) {
+      mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+    }
+    mpierr = MPI_Bcast(&(file->fh), 1, MPI_INT, ios->compmaster, ios->intercomm);
   }
 
   if(ios->ioproc){
@@ -325,7 +337,6 @@ int PIOc_closefile(int ncid)
 
   int iret =  pio_delete_file_from_list(ncid);
 
-
   return ierr;
 }
 
@@ -342,6 +353,7 @@ int PIOc_deletefile(const int iosysid, const char filename[])
   int mpierr;
   int chkerr;
   iosystem_desc_t *ios;
+  size_t len;
 
   ierr = PIO_NOERR;
   ios = pio_get_iosystem_from_id(iosysid);
@@ -349,12 +361,14 @@ int PIOc_deletefile(const int iosysid, const char filename[])
   if(ios == NULL)
     return PIO_EBADID;
 
-  msg = 0;
+  msg = PIO_MSG_DELETE_FILE;
 
   if(ios->async_interface && ! ios->ioproc){
     if(ios->comp_rank==0) 
       mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
-    //    mpierr = MPI_Bcast(iosysid,1, MPI_INT, ios->compmaster, ios->intercomm);
+    len = strlen(filename);
+    mpierr = MPI_Bcast(&len, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+    mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
   }
   // The barriers are needed to assure that no task is trying to operate on the file while it is being deleted.
   if(ios->ioproc){
@@ -371,8 +385,6 @@ int PIOc_deletefile(const int iosysid, const char filename[])
   }
   //   Special case - always broadcast the return from the  
   MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm);
-  
-
 
   return ierr;
 }
@@ -405,9 +417,9 @@ int PIOc_sync (int ncid)
   msg = PIO_MSG_SYNC;
 
   if(ios->async_interface && ! ios->ioproc){
-    if(ios->compmaster) 
+    if(ios->comp_rank == 0) 
       mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
-    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
   }
 
   if((file->mode & PIO_WRITE)){
diff --git a/src/clib/pio_get_nc_async.c b/src/clib/pio_get_nc_async.c
new file mode 100644
index 000000000000..0da690aaf216
--- /dev/null
+++ b/src/clib/pio_get_nc_async.c
@@ -0,0 +1,4988 @@
+#include <pio.h>
+#include <pio_internal.h>
+
+int PIOc_get_var1_schar (int ncid, int varid, const PIO_Offset index[], signed char *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VAR1_SCHAR;
+  ibuftype = MPI_CHAR;
+  ibufcnt = 1;
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_var1_schar(file->fh, varid, (size_t *) index,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_var1_schar(file->fh, varid, (size_t *) index,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_var1_schar(file->fh, varid, index,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_var1_schar_all(file->fh, varid, index,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_vars_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned long long *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARS_ULONGLONG;
+  ibuftype = MPI_UNSIGNED_LONG_LONG;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i]/stride[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_vars_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_vars_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_vars_ulonglong(file->fh, varid, start, count, stride,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_vars_ulonglong_all(file->fh, varid, start, count, stride,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned char *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARM_UCHAR;
+  ibuftype = MPI_UNSIGNED_CHAR;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i]/stride[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_varm_uchar(file->fh, varid, start, count, stride, imap,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_varm_uchar_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_varm_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], signed char *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARM_SCHAR;
+  ibuftype = MPI_CHAR;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i]/stride[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_varm_schar(file->fh, varid, start, count, stride, imap,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_varm_schar_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_vars_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], short *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARS_SHORT;
+  ibuftype = MPI_SHORT;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i]/stride[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_vars_short(file->fh, varid, start, count, stride,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_vars_short_all(file->fh, varid, start, count, stride,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_var_double (int ncid, int varid, double *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VAR_DOUBLE;
+  ibuftype = MPI_DOUBLE;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  int dimid[ndims];
+  PIO_Offset dimsize;
+  ibufcnt = 1;
+  PIOc_inq_vardimid(file->fh, varid, dimid);
+  for(int i=0;i<ndims;i++){
+    PIOc_inq_dimlen(file->fh, dimid[i], &dimsize);
+    ibufcnt *= dimsize;
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_var_double(file->fh, varid,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_var_double(file->fh, varid,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_var_double(file->fh, varid,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_var_double_all(file->fh, varid,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_vara_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], double *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARA_DOUBLE;
+  ibuftype = MPI_DOUBLE;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_vara_double(file->fh, varid, (size_t *) start, (size_t *) count,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_vara_double(file->fh, varid, (size_t *) start, (size_t *) count,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_vara_double(file->fh, varid, start, count,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_vara_double_all(file->fh, varid, start, count,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_var_int (int ncid, int varid, int *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VAR_INT;
+  ibuftype = MPI_INT;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  int dimid[ndims];
+  PIO_Offset dimsize;
+  ibufcnt = 1;
+  PIOc_inq_vardimid(file->fh, varid, dimid);
+  for(int i=0;i<ndims;i++){
+    PIOc_inq_dimlen(file->fh, dimid[i], &dimsize);
+    ibufcnt *= dimsize;
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_var_int(file->fh, varid,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_var_int(file->fh, varid,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_var_int(file->fh, varid,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_var_int_all(file->fh, varid,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_var_ushort (int ncid, int varid, unsigned short *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VAR_USHORT;
+  ibuftype = MPI_UNSIGNED_SHORT;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  int dimid[ndims];
+  PIO_Offset dimsize;
+  ibufcnt = 1;
+  PIOc_inq_vardimid(file->fh, varid, dimid);
+  for(int i=0;i<ndims;i++){
+    PIOc_inq_dimlen(file->fh, dimid[i], &dimsize);
+    ibufcnt *= dimsize;
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_var_ushort(file->fh, varid,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_var_ushort(file->fh, varid,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_var_ushort(file->fh, varid,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_var_ushort_all(file->fh, varid,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], char *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARA_TEXT;
+  ibuftype = MPI_CHAR;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_vara_text(file->fh, varid, (size_t *) start, (size_t *) count,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_vara_text(file->fh, varid, (size_t *) start, (size_t *) count,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_vara_text(file->fh, varid, start, count,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_vara_text_all(file->fh, varid, start, count,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], int *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARA_INT;
+  ibuftype = MPI_INT;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_vara_int(file->fh, varid, (size_t *) start, (size_t *) count,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_vara_int(file->fh, varid, (size_t *) start, (size_t *) count,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_vara_int(file->fh, varid, start, count,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_vara_int_all(file->fh, varid, start, count,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_var1_float (int ncid, int varid, const PIO_Offset index[], float *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VAR1_FLOAT;
+  ibuftype = MPI_FLOAT;
+  ibufcnt = 1;
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_var1_float(file->fh, varid, (size_t *) index,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_var1_float(file->fh, varid, (size_t *) index,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_var1_float(file->fh, varid, index,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_var1_float_all(file->fh, varid, index,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_var1_short (int ncid, int varid, const PIO_Offset index[], short *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VAR1_SHORT;
+  ibuftype = MPI_SHORT;
+  ibufcnt = 1;
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_var1_short(file->fh, varid, (size_t *) index,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_var1_short(file->fh, varid, (size_t *) index,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_var1_short(file->fh, varid, index,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_var1_short_all(file->fh, varid, index,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_vars_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], int *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARS_INT;
+  ibuftype = MPI_INT;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i]/stride[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_vars_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_vars_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_vars_int(file->fh, varid, start, count, stride,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_vars_int_all(file->fh, varid, start, count, stride,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_var_text (int ncid, int varid, char *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VAR_TEXT;
+  ibuftype = MPI_CHAR;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  int dimid[ndims];
+  PIO_Offset dimsize;
+  ibufcnt = 1;
+  PIOc_inq_vardimid(file->fh, varid, dimid);
+  for(int i=0;i<ndims;i++){
+    PIOc_inq_dimlen(file->fh, dimid[i], &dimsize);
+    ibufcnt *= dimsize;
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_var_text(file->fh, varid,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_var_text(file->fh, varid,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_var_text(file->fh, varid,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_var_text_all(file->fh, varid,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_varm_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], double *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARM_DOUBLE;
+  ibuftype = MPI_DOUBLE;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i]/stride[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_varm_double(file->fh, varid, start, count, stride, imap,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_varm_double_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_vars_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], signed char *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARS_SCHAR;
+  ibuftype = MPI_CHAR;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i]/stride[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_vars_schar(file->fh, varid, start, count, stride,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_vars_schar_all(file->fh, varid, start, count, stride,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_vara_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned short *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARA_USHORT;
+  ibuftype = MPI_UNSIGNED_SHORT;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_vara_ushort(file->fh, varid, start, count,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_vara_ushort_all(file->fh, varid, start, count,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_var1_ushort (int ncid, int varid, const PIO_Offset index[], unsigned short *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VAR1_USHORT;
+  ibuftype = MPI_UNSIGNED_SHORT;
+  ibufcnt = 1;
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_var1_ushort(file->fh, varid, (size_t *) index,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_var1_ushort(file->fh, varid, (size_t *) index,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_var1_ushort(file->fh, varid, index,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_var1_ushort_all(file->fh, varid, index,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_var_float (int ncid, int varid, float *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VAR_FLOAT;
+  ibuftype = MPI_FLOAT;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  int dimid[ndims];
+  PIO_Offset dimsize;
+  ibufcnt = 1;
+  PIOc_inq_vardimid(file->fh, varid, dimid);
+  for(int i=0;i<ndims;i++){
+    PIOc_inq_dimlen(file->fh, dimid[i], &dimsize);
+    ibufcnt *= dimsize;
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_var_float(file->fh, varid,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_var_float(file->fh, varid,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_var_float(file->fh, varid,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_var_float_all(file->fh, varid,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_vars_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned char *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARS_UCHAR;
+  ibuftype = MPI_UNSIGNED_CHAR;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i]/stride[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_vars_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_vars_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_vars_uchar(file->fh, varid, start, count, stride,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_vars_uchar_all(file->fh, varid, start, count, stride,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_var (int ncid, int varid, void *buf, PIO_Offset bufcount, MPI_Datatype buftype) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VAR;
+  ibufcnt = bufcount;
+  ibuftype = buftype;
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_var(file->fh, varid,   buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_var(file->fh, varid,   buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_var(file->fh, varid, buf, bufcount, buftype);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_var_all(file->fh, varid, buf, bufcount, buftype);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_var1_longlong (int ncid, int varid, const PIO_Offset index[], long long *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VAR1_LONGLONG;
+  ibuftype = MPI_LONG_LONG;
+  ibufcnt = 1;
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_var1_longlong(file->fh, varid, (size_t *) index,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_var1_longlong(file->fh, varid, (size_t *) index,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_var1_longlong(file->fh, varid, index,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_var1_longlong_all(file->fh, varid, index,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_vars_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned short *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARS_USHORT;
+  ibuftype = MPI_UNSIGNED_SHORT;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i]/stride[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_vars_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_vars_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_vars_ushort(file->fh, varid, start, count, stride,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_vars_ushort_all(file->fh, varid, start, count, stride,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_var_long (int ncid, int varid, long *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VAR_LONG;
+  ibuftype = MPI_LONG;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  int dimid[ndims];
+  PIO_Offset dimsize;
+  ibufcnt = 1;
+  PIOc_inq_vardimid(file->fh, varid, dimid);
+  for(int i=0;i<ndims;i++){
+    PIOc_inq_dimlen(file->fh, dimid[i], &dimsize);
+    ibufcnt *= dimsize;
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_var_long(file->fh, varid,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_var_long(file->fh, varid,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_var_long(file->fh, varid,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_var_long_all(file->fh, varid,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_var1_double (int ncid, int varid, const PIO_Offset index[], double *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VAR1_DOUBLE;
+  ibuftype = MPI_DOUBLE;
+  ibufcnt = 1;
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_var1_double(file->fh, varid, (size_t *) index,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_var1_double(file->fh, varid, (size_t *) index,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_var1_double(file->fh, varid, index,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_var1_double_all(file->fh, varid, index,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_vara_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned int *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARA_UINT;
+  ibuftype = MPI_UNSIGNED;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_vara_uint(file->fh, varid, start, count,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_vara_uint_all(file->fh, varid, start, count,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_vars_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], long long *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARS_LONGLONG;
+  ibuftype = MPI_LONG_LONG;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i]/stride[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_vars_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_vars_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_vars_longlong(file->fh, varid, start, count, stride,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_vars_longlong_all(file->fh, varid, start, count, stride,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_var_longlong (int ncid, int varid, long long *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VAR_LONGLONG;
+  ibuftype = MPI_LONG_LONG;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  int dimid[ndims];
+  PIO_Offset dimsize;
+  ibufcnt = 1;
+  PIOc_inq_vardimid(file->fh, varid, dimid);
+  for(int i=0;i<ndims;i++){
+    PIOc_inq_dimlen(file->fh, dimid[i], &dimsize);
+    ibufcnt *= dimsize;
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_var_longlong(file->fh, varid,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_var_longlong(file->fh, varid,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_var_longlong(file->fh, varid,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_var_longlong_all(file->fh, varid,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_vara_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], short *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARA_SHORT;
+  ibuftype = MPI_SHORT;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_vara_short(file->fh, varid, (size_t *) start, (size_t *) count,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_vara_short(file->fh, varid, (size_t *) start, (size_t *) count,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_vara_short(file->fh, varid, start, count,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_vara_short_all(file->fh, varid, start, count,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_vara_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], long *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARA_LONG;
+  ibuftype = MPI_LONG;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_vara_long(file->fh, varid, (size_t *) start, (size_t *) count,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_vara_long(file->fh, varid, (size_t *) start, (size_t *) count,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_vara_long(file->fh, varid, start, count,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_vara_long_all(file->fh, varid, start, count,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_var1_int (int ncid, int varid, const PIO_Offset index[], int *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VAR1_INT;
+  ibuftype = MPI_INT;
+  ibufcnt = 1;
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_var1_int(file->fh, varid, (size_t *) index,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_var1_int(file->fh, varid, (size_t *) index,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_var1_int(file->fh, varid, index,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_var1_int_all(file->fh, varid, index,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_var1_ulonglong (int ncid, int varid, const PIO_Offset index[], unsigned long long *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VAR1_ULONGLONG;
+  ibuftype = MPI_UNSIGNED_LONG_LONG;
+  ibufcnt = 1;
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_var1_ulonglong(file->fh, varid, (size_t *) index,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_var1_ulonglong(file->fh, varid, (size_t *) index,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_var1_ulonglong(file->fh, varid, index,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_var1_ulonglong_all(file->fh, varid, index,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_var_uchar (int ncid, int varid, unsigned char *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VAR_UCHAR;
+  ibuftype = MPI_UNSIGNED_CHAR;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  int dimid[ndims];
+  PIO_Offset dimsize;
+  ibufcnt = 1;
+  PIOc_inq_vardimid(file->fh, varid, dimid);
+  for(int i=0;i<ndims;i++){
+    PIOc_inq_dimlen(file->fh, dimid[i], &dimsize);
+    ibufcnt *= dimsize;
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_var_uchar(file->fh, varid,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_var_uchar(file->fh, varid,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_var_uchar(file->fh, varid,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_var_uchar_all(file->fh, varid,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_vara_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned char *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARA_UCHAR;
+  ibuftype = MPI_UNSIGNED_CHAR;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_vara_uchar(file->fh, varid, (size_t *) start, (size_t *) count,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_vara_uchar(file->fh, varid, (size_t *) start, (size_t *) count,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_vara_uchar(file->fh, varid, start, count,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_vara_uchar_all(file->fh, varid, start, count,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_vars_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], float *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARS_FLOAT;
+  ibuftype = MPI_FLOAT;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i]/stride[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_vars_float(file->fh, varid, start, count, stride,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_vars_float_all(file->fh, varid, start, count, stride,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_vars_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], long *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARS_LONG;
+  ibuftype = MPI_LONG;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i]/stride[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_vars_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_vars_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_vars_long(file->fh, varid, start, count, stride,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_vars_long_all(file->fh, varid, start, count, stride,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_var1 (int ncid, int varid, const PIO_Offset index[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VAR1;
+  ibufcnt = bufcount;
+  ibuftype = buftype;
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_var1(file->fh, varid, (size_t *) index,   buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_var1(file->fh, varid, (size_t *) index,   buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_var1(file->fh, varid, index, buf, bufcount, buftype);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_var1_all(file->fh, varid, index, buf, bufcount, buftype);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_var_uint (int ncid, int varid, unsigned int *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VAR_UINT;
+  ibuftype = MPI_UNSIGNED;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  int dimid[ndims];
+  PIO_Offset dimsize;
+  ibufcnt = 1;
+  PIOc_inq_vardimid(file->fh, varid, dimid);
+  for(int i=0;i<ndims;i++){
+    PIOc_inq_dimlen(file->fh, dimid[i], &dimsize);
+    ibufcnt *= dimsize;
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_var_uint(file->fh, varid,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_var_uint(file->fh, varid,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_var_uint(file->fh, varid,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_var_uint_all(file->fh, varid,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_vara (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARA;
+  ibufcnt = bufcount;
+  ibuftype = buftype;
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_vara(file->fh, varid, (size_t *) start, (size_t *) count,   buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_vara(file->fh, varid, (size_t *) start, (size_t *) count,   buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_vara(file->fh, varid, start, count, buf, bufcount, buftype);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_vara_all(file->fh, varid, start, count, buf, bufcount, buftype);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_vara_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], signed char *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARA_SCHAR;
+  ibuftype = MPI_CHAR;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_vara_schar(file->fh, varid, start, count,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_vara_schar_all(file->fh, varid, start, count,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_var1_uint (int ncid, int varid, const PIO_Offset index[], unsigned int *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VAR1_UINT;
+  ibuftype = MPI_UNSIGNED;
+  ibufcnt = 1;
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_var1_uint(file->fh, varid, (size_t *) index,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_var1_uint(file->fh, varid, (size_t *) index,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_var1_uint(file->fh, varid, index,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_var1_uint_all(file->fh, varid, index,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_vars_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned int *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARS_UINT;
+  ibuftype = MPI_UNSIGNED;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i]/stride[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_vars_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_vars_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_vars_uint(file->fh, varid, start, count, stride,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_vars_uint_all(file->fh, varid, start, count, stride,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_vara_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], float *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARA_FLOAT;
+  ibuftype = MPI_FLOAT;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_vara_float(file->fh, varid, (size_t *) start, (size_t *) count,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_vara_float(file->fh, varid, (size_t *) start, (size_t *) count,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_vara_float(file->fh, varid, start, count,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_vara_float_all(file->fh, varid, start, count,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], char *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARM_TEXT;
+  ibuftype = MPI_CHAR;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i]/stride[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_varm_text(file->fh, varid, start, count, stride, imap,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_varm_text_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_var1_text (int ncid, int varid, const PIO_Offset index[], char *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VAR1_TEXT;
+  ibuftype = MPI_CHAR;
+  ibufcnt = 1;
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_var1_text(file->fh, varid, (size_t *) index,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_var1_text(file->fh, varid, (size_t *) index,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_var1_text(file->fh, varid, index,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_var1_text_all(file->fh, varid, index,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], int *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARM_INT;
+  ibuftype = MPI_INT;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i]/stride[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_varm_int(file->fh, varid, start, count, stride, imap,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_varm_int_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned int *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARM_UINT;
+  ibuftype = MPI_UNSIGNED;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i]/stride[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_varm_uint(file->fh, varid, start, count, stride, imap,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_varm_uint_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARM;
+  ibufcnt = bufcount;
+  ibuftype = buftype;
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,   buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,   buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_varm(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_varm_all(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_vars_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], double *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARS_DOUBLE;
+  ibuftype = MPI_DOUBLE;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i]/stride[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_vars_double(file->fh, varid, start, count, stride,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_vars_double_all(file->fh, varid, start, count, stride,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_vara_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], long long *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARA_LONGLONG;
+  ibuftype = MPI_LONG_LONG;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_vara_longlong(file->fh, varid, start, count,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_vara_longlong_all(file->fh, varid, start, count,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_var_ulonglong (int ncid, int varid, unsigned long long *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VAR_ULONGLONG;
+  ibuftype = MPI_UNSIGNED_LONG_LONG;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  int dimid[ndims];
+  PIO_Offset dimsize;
+  ibufcnt = 1;
+  PIOc_inq_vardimid(file->fh, varid, dimid);
+  for(int i=0;i<ndims;i++){
+    PIOc_inq_dimlen(file->fh, dimid[i], &dimsize);
+    ibufcnt *= dimsize;
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_var_ulonglong(file->fh, varid,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_var_ulonglong(file->fh, varid,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_var_ulonglong(file->fh, varid,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_var_ulonglong_all(file->fh, varid,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_vara_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned long long *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARA_ULONGLONG;
+  ibuftype = MPI_UNSIGNED_LONG_LONG;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_vara_ulonglong(file->fh, varid, start, count,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_vara_ulonglong_all(file->fh, varid, start, count,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_var_short (int ncid, int varid, short *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VAR_SHORT;
+  ibuftype = MPI_SHORT;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  int dimid[ndims];
+  PIO_Offset dimsize;
+  ibufcnt = 1;
+  PIOc_inq_vardimid(file->fh, varid, dimid);
+  for(int i=0;i<ndims;i++){
+    PIOc_inq_dimlen(file->fh, dimid[i], &dimsize);
+    ibufcnt *= dimsize;
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_var_short(file->fh, varid,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_var_short(file->fh, varid,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_var_short(file->fh, varid,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_var_short_all(file->fh, varid,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_varm_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], float *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARM_FLOAT;
+  ibuftype = MPI_FLOAT;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i]/stride[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_varm_float(file->fh, varid, start, count, stride, imap,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_varm_float_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_var1_long (int ncid, int varid, const PIO_Offset index[], long *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VAR1_LONG;
+  ibuftype = MPI_LONG;
+  ibufcnt = 1;
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_var1_long(file->fh, varid, (size_t *) index,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_var1_long(file->fh, varid, (size_t *) index,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_var1_long(file->fh, varid, index,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_var1_long_all(file->fh, varid, index,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], long *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARM_LONG;
+  ibuftype = MPI_LONG;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i]/stride[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_varm_long(file->fh, varid, start, count, stride, imap,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_varm_long_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_varm_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned short *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARM_USHORT;
+  ibuftype = MPI_UNSIGNED_SHORT;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i]/stride[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_varm_ushort(file->fh, varid, start, count, stride, imap,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_varm_ushort_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_varm_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], long long *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARM_LONGLONG;
+  ibuftype = MPI_LONG_LONG;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i]/stride[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_varm_longlong(file->fh, varid, start, count, stride, imap,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_varm_longlong_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_vars_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], char *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARS_TEXT;
+  ibuftype = MPI_CHAR;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i]/stride[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_vars_text(file->fh, varid, start, count, stride,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_vars_text_all(file->fh, varid, start, count, stride,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_var1_uchar (int ncid, int varid, const PIO_Offset index[], unsigned char *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VAR1_UCHAR;
+  ibuftype = MPI_UNSIGNED_CHAR;
+  ibufcnt = 1;
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_var1_uchar(file->fh, varid, (size_t *) index,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_var1_uchar(file->fh, varid, (size_t *) index,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_var1_uchar(file->fh, varid, index,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_var1_uchar_all(file->fh, varid, index,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_vars (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARS;
+  ibufcnt = bufcount;
+  ibuftype = buftype;
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,   buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,   buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_vars(file->fh, varid, start, count, stride, buf, bufcount, buftype);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_vars_all(file->fh, varid, start, count, stride, buf, bufcount, buftype);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_varm_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], short *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARM_SHORT;
+  ibuftype = MPI_SHORT;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i]/stride[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_varm_short(file->fh, varid, start, count, stride, imap,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_varm_short_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned long long *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VARM_ULONGLONG;
+  ibuftype = MPI_UNSIGNED_LONG_LONG;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  ibufcnt = 1;
+  for(int i=0;i<ndims;i++){
+    ibufcnt *= count[i]/stride[i];
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_varm_ulonglong(file->fh, varid, start, count, stride, imap,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_varm_ulonglong_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
+int PIOc_get_var_schar (int ncid, int varid, signed char *buf) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  MPI_Datatype ibuftype;
+  int ndims;
+  int ibufcnt;
+  bool bcast = false;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_GET_VAR_SCHAR;
+  ibuftype = MPI_CHAR;
+  ierr = PIOc_inq_varndims(file->fh, varid, &ndims);
+  int dimid[ndims];
+  PIO_Offset dimsize;
+  ibufcnt = 1;
+  PIOc_inq_vardimid(file->fh, varid, dimid);
+  for(int i=0;i<ndims;i++){
+    PIOc_inq_dimlen(file->fh, dimid[i], &dimsize);
+    ibufcnt *= dimsize;
+  }
+  ierr = PIO_NOERR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_get_var_schar(file->fh, varid,  buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      bcast = true;
+      if(ios->iomaster){
+	ierr = nc_get_var_schar(file->fh, varid,  buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+        ncmpi_begin_indep_data(file->fh);
+        if(ios->iomaster){
+      ierr = ncmpi_get_var_schar(file->fh, varid,  buf);;
+        };
+         ncmpi_end_indep_data(file->fh);
+        bcast=true;
+#else
+      ierr = ncmpi_get_var_schar_all(file->fh, varid,  buf);;
+#endif
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  if(ios->async_interface || bcast ||  
+     (ios->num_iotasks < ios->num_comptasks)){
+    MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+  }
+
+  return ierr;
+}
+
diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h
index f7a3c9ddf220..740b8fdf1a6d 100644
--- a/src/clib/pio_internal.h
+++ b/src/clib/pio_internal.h
@@ -158,6 +158,8 @@ void flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk);
 }
 #endif
 
+/** These are the messages that can be sent over the intercomm when
+ * async is being used. */
 enum PIO_MSG{
   PIO_MSG_OPEN_FILE,
   PIO_MSG_CREATE_FILE,
@@ -365,7 +367,15 @@ enum PIO_MSG{
   PIO_MSG_SET_CHUNK_CACHE,
   PIO_MSG_GET_CHUNK_CACHE,
   PIO_MSG_SET_VAR_CHUNK_CACHE,
-  PIO_MSG_GET_VAR_CHUNK_CACHE
+  PIO_MSG_GET_VAR_CHUNK_CACHE,
+  PIO_MSG_INITDECOMP_DOF,
+  PIO_MSG_WRITEDARRAY,
+  PIO_MSG_READDARRAY,
+  PIO_MSG_SETERRORHANDLING,
+  PIO_MSG_FREEDECOMP,
+  PIO_MSG_CLOSE_FILE,
+  PIO_MSG_DELETE_FILE,
+  PIO_MSG_EXIT
 };
 
 #endif
diff --git a/src/clib/pio_lists.c b/src/clib/pio_lists.c
index 511918b52b53..efa0868c4441 100644
--- a/src/clib/pio_lists.c
+++ b/src/clib/pio_lists.c
@@ -9,37 +9,34 @@ static iosystem_desc_t *pio_iosystem_list=NULL;
 static file_desc_t *pio_file_list = NULL;
 static file_desc_t *current_file=NULL;
 
+/** Add a new entry to the global list of open files. 
+ *
+ * @param file pointer to the file_desc_t struct for the new file.
+*/
 void pio_add_to_file_list(file_desc_t *file)
 {
   file_desc_t *cfile;
-  int cnt=-1;
-  // on iotasks the fh returned from netcdf should be unique, on non-iotasks we
-  // need to generate a unique fh, we do this with cnt, its a negative index 
 
+  /* This file will be at the end of the list, and have no next. */
   file->next = NULL;
+
+  /* Get a pointer to the global list of files. */
   cfile = pio_file_list;
+
+  /* Keep a global pointer to the current file. */
   current_file = file;
-  if(cfile==NULL){
-    pio_file_list = file;
-  }else{
-    cnt =  min(cnt,cfile->fh-1);
-    while(cfile->next != NULL)
-      {
-	cfile=cfile->next;
-	cnt =  min(cnt,cfile->fh-1);
-      }
-    cfile->next = file;
-  }
-  if(! file->iosystem->ioproc || ((file->iotype != PIO_IOTYPE_PNETCDF &&
-				   file->iotype != PIO_IOTYPE_NETCDF4P)  && 
-				  file->iosystem->io_rank>0)) 
-    file->fh = cnt;
-  
-  cfile = pio_file_list;
 
+  /* If there is nothing in the list, then file will be the first
+   * entry. Otherwise, move to end of the list. */
+  if (!cfile) 
+      pio_file_list = file;
+  else
+  {
+      while (cfile->next)
+	  cfile = cfile->next;
+      cfile->next = file;
+  }
 }
-
-
      
 file_desc_t *pio_get_file_from_id(int ncid)
 {
diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c
new file mode 100644
index 000000000000..9e9f04a45db7
--- /dev/null
+++ b/src/clib/pio_nc_async.c
@@ -0,0 +1,4534 @@
+/**
+ * @file   
+ * PIO interfaces to
+ * [NetCDF](http://www.unidata.ucar.edu/software/netcdf/docs/modules.html)
+ * support functions
+
+ *  This file provides an interface to the
+ *  [NetCDF](http://www.unidata.ucar.edu/software/netcdf/docs/modules.html)
+ *  support functions.  Each subroutine calls the underlying netcdf or
+ *  pnetcdf or netcdf4 functions from the appropriate subset of mpi
+ *  tasks (io_comm). Each routine must be called collectively from
+ *  union_comm.
+ *  
+ * @author Jim Edwards (jedwards@ucar.edu), Ed Hartnett
+ * @date     Feburary 2014, April 2016
+ */
+
+#include <pio.h>
+#include <pio_internal.h>
+
+/** Internal function to call nc_inq. Call this function only in I/O
+ * tasks. */
+int
+pio_io_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp,
+	   int *unlimdimidp)
+{
+    file_desc_t *file;
+    char *errstr = NULL;
+    int ierr = PIO_NOERR;
+
+    int my_rank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);    
+    printf("%d pio_io_inq ncid = %d\n", my_rank, ncid);
+
+    if (!(file = pio_get_file_from_id(ncid)))
+	return PIO_EBADID;
+    
+    switch (file->iotype)
+    {
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+	ierr = nc_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp);
+	break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+	printf("%d pio_io_inq file->iosystem->io_rank = %d\n", my_rank, file->iosystem->io_rank);
+	if (!file->iosystem->io_rank)
+	{
+	    /* Should not be necessary to do this - nc_inq should
+	     * handle null pointers. This has been reported as a bug
+	     * to netCDF developers. */
+	    int tmp_ndims, tmp_var, tmp_gatts, tmp_unl;
+	    if (!ndimsp)
+		ndimsp = &tmp_ndims;
+	    if (!nvarsp)
+		nvarsp = &tmp_var;
+	    if (!ngattsp)
+		ngattsp = &tmp_gatts;
+	    if (!unlimdimidp)
+		unlimdimidp = &tmp_unl;
+	    ierr = nc_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp);
+	}
+	break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+	ierr = ncmpi_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp);
+	break;
+#endif
+    default:
+	ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+
+    if(ierr != PIO_NOERR)
+    {
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_inq
+ * The PIO-C interface for the NetCDF function nc_inq.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @return PIO_NOERR for success, error code otherwise.  See
+ * PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq (int ncid, int *ndimsp, int *nvarsp, int *ngattsp,
+	      int *unlimdimidp) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    int my_rank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);    
+    printf("%d PIOc_inq ncid = %d\n", my_rank, ncid);
+  
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_INQ;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+    if(ios->ioproc){
+	ierr = pio_io_inq(file->fh, ndimsp, nvarsp, ngattsp, unlimdimidp);
+	printf("%d PIOc_inq pio_io_inq returned %d\n", my_rank, ierr);
+	ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    }
+    if(ndimsp != NULL)
+    {
+	printf("%d PIOc_inq Bcast ios->ioroot = %d *ndimsp = %d\n", my_rank, ios->ioroot, *ndimsp);
+	mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm);
+    }
+    if(nvarsp != NULL)
+	mpierr = MPI_Bcast(nvarsp, 1, MPI_INT, ios->ioroot, ios->my_comm);
+    if(ngattsp != NULL)
+	mpierr = MPI_Bcast(ngattsp, 1, MPI_INT, ios->ioroot, ios->my_comm);
+    if(unlimdimidp != NULL)
+	mpierr = MPI_Bcast(unlimdimidp, 1, MPI_INT, ios->ioroot, ios->my_comm);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_inq_dimname
+ * The PIO-C interface for the NetCDF function nc_inq_dimname.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_dimname (int ncid, int dimid, char *name) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_INQ_DIMNAME;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+	mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_inq_dimname(file->fh, dimid, name);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_inq_dimname(file->fh, dimid, name);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_inq_dimname(file->fh, dimid, name);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if (name)
+    {
+	int slen;
+	if(ios->iomaster)
+	    slen = (int) strlen(name) + 1;
+	mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm);
+	mpierr = MPI_Bcast((void *)name, slen, MPI_CHAR, ios->ioroot, ios->my_comm);
+    }
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_put_att_short
+ * The PIO-C interface for the NetCDF function nc_put_att_short.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_put_att_short (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const short *op) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_PUT_ATT_SHORT;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_put_att_short(file->fh, varid, name, xtype, (size_t)len, op);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_put_att_short(file->fh, varid, name, xtype, (size_t)len, op);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_put_att_short(file->fh, varid, name, xtype, len, op);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_rename_dim
+ * The PIO-C interface for the NetCDF function nc_rename_dim.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_rename_dim (int ncid, int dimid, const char *name) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_RENAME_DIM;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_rename_dim(file->fh, dimid, name);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_rename_dim(file->fh, dimid, name);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_rename_dim(file->fh, dimid, name);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_get_att_double
+ * The PIO-C interface for the NetCDF function nc_get_att_double.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_get_att_double (int ncid, int varid, const char *name, double *ip) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_GET_ATT_DOUBLE;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_get_att_double(file->fh, varid, name, ip);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_get_att_double(file->fh, varid, name, ip);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_get_att_double(file->fh, varid, name, ip);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char));
+	sprintf(errstr,"name %s in file %s",name,__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(ierr == PIO_NOERR){
+        PIO_Offset attlen;
+        PIOc_inq_attlen(file->fh, varid, name, &attlen);
+        mpierr = MPI_Bcast(ip , (int) attlen, MPI_DOUBLE, ios->ioroot, ios->my_comm);
+    }
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_set_fill
+ * The PIO-C interface for the NetCDF function nc_set_fill.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_set_fill (int ncid, int fillmode, int *old_modep) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_SET_FILL;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_set_fill(file->fh, fillmode, old_modep);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_set_fill(file->fh, fillmode, old_modep);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_set_fill(file->fh, fillmode, old_modep);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_def_var
+ * The PIO-C interface for the NetCDF function nc_def_var.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param varidp a pointer that will get the variable id 
+ * @return PIO_NOERR for success, error code otherwise. See
+ * PIOc_Set_File_Error_Handling
+ */
+int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims,
+		  const int *dimidsp, int *varidp) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+    int namelen;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_DEF_VAR;
+
+    /* If async is in use, and this is not an IO tasks, then bcast the
+     * function parameters to the intercomm, where it will be read by
+     * the IO root task. */
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+	namelen = strlen(name);
+	printf("bcasting namelen = %d name = %s\n", namelen, name);
+	if (!ios->compmaster)
+	    ios->compmaster = MPI_PROC_NULL;
+	mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+	mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+	mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm);
+	mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm);
+	mpierr = MPI_Bcast((void *)dimidsp, ndims, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+	    if(ios->io_rank==0){
+		ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);
+		if(ierr == PIO_NOERR){
+		    ierr = nc_def_var_deflate(file->fh, *varidp, 0,1,1);
+		}
+	    }
+	    break;
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    mpierr = MPI_Bcast(varidp , 1, MPI_INT, ios->ioroot, ios->my_comm);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_put_att_double
+ * The PIO-C interface for the NetCDF function nc_put_att_double.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_put_att_double (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const double *op) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_PUT_ATT_DOUBLE;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_put_att_double(file->fh, varid, name, xtype, (size_t)len, op);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_put_att_double(file->fh, varid, name, xtype, (size_t)len, op);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_put_att_double(file->fh, varid, name, xtype, len, op);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_inq_dim
+ * The PIO-C interface for the NetCDF function nc_inq_dim.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param lenp a pointer that will get the number of values 
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_dim (int ncid, int dimid, char *name, PIO_Offset *lenp) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_INQ_DIM;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm);
+	mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_inq_dim(file->fh, dimid, name, lenp);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(name)
+    { 
+	int slen;
+	if(ios->iomaster)
+	    slen = (int) strlen(name) + 1;
+	mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm);
+	mpierr = MPI_Bcast((void *)name, slen, MPI_CHAR, ios->ioroot, ios->my_comm);
+    }
+    if(lenp != NULL)
+	mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_get_att_uchar
+ * The PIO-C interface for the NetCDF function nc_get_att_uchar.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_GET_ATT_UCHAR;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_get_att_uchar(file->fh, varid, name, ip);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_get_att_uchar(file->fh, varid, name, ip);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_get_att_uchar(file->fh, varid, name, ip);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char));
+	sprintf(errstr,"name %s in file %s",name,__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(ierr == PIO_NOERR){
+        PIO_Offset attlen;
+        PIOc_inq_attlen(file->fh, varid, name, &attlen);
+        mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_CHAR, ios->ioroot, ios->my_comm);
+    }
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_inq_var_fill
+ * The PIO-C interface for the NetCDF function nc_inq_var_fill.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_var_fill (int ncid, int varid, int *no_fill, void *fill_value) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_INQ_VAR_FILL;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_value);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_value);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_inq_var_fill(file->fh, varid, no_fill, fill_value);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    mpierr = MPI_Bcast(fill_value, 1, MPI_INT, ios->ioroot, ios->my_comm);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_inq_attid
+ * The PIO-C interface for the NetCDF function nc_inq_attid.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param idp a pointer that will get the id of the variable or attribute.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_INQ_ATTID;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_inq_attid(file->fh, varid, name, idp);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_inq_attid(file->fh, varid, name, idp);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_inq_attid(file->fh, varid, name, idp);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_inq_vartype
+ * The PIO-C interface for the NetCDF function nc_inq_vartype.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param xtypep a pointer that will get the type of the attribute.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_vartype (int ncid, int varid, nc_type *xtypep) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_INQ_VARTYPE;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&file->fh,1, MPI_INT, ios->compmaster, ios->intercomm);
+	mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_inq_vartype(file->fh, varid, xtypep);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_inq_vartype(file->fh, varid, xtypep);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_inq_vartype(file->fh, varid, xtypep);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if (xtypep)
+	mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_put_att_schar
+ * The PIO-C interface for the NetCDF function nc_put_att_schar.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const signed char *op) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_PUT_ATT_SCHAR;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_put_att_schar(file->fh, varid, name, xtype, (size_t)len, op);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_put_att_schar(file->fh, varid, name, xtype, (size_t)len, op);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_put_att_schar(file->fh, varid, name, xtype, len, op);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_inq_vardimid
+ * The PIO-C interface for the NetCDF function nc_inq_vardimid.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_vardimid (int ncid, int varid, int *dimidsp) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_INQ_VARDIMID;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm);
+	mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_inq_vardimid(file->fh, varid, dimidsp);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_inq_vardimid(file->fh, varid, dimidsp);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_inq_vardimid(file->fh, varid, dimidsp);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if (!ierr && dimidsp)
+    {
+	int ndims;
+	PIOc_inq_varndims(file->fh, varid, &ndims);
+	mpierr = MPI_Bcast(dimidsp , ndims, MPI_INT, ios->ioroot, ios->my_comm);
+    }
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_get_att_ushort
+ * The PIO-C interface for the NetCDF function nc_get_att_ushort.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short *ip) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_GET_ATT_USHORT;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_get_att_ushort(file->fh, varid, name, ip);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_get_att_ushort(file->fh, varid, name, ip);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_get_att_ushort(file->fh, varid, name, ip);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char));
+	sprintf(errstr,"name %s in file %s",name,__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(ierr == PIO_NOERR){
+        PIO_Offset attlen;
+        PIOc_inq_attlen(file->fh, varid, name, &attlen);
+        mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_SHORT, ios->ioroot, ios->my_comm);
+    }
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_inq_varid
+ * The PIO-C interface for the NetCDF function nc_inq_varid.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param varidp a pointer that will get the variable id 
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_varid (int ncid, const char *name, int *varidp) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_INQ_VARID;
+
+    int my_rank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);    
+    printf("%d PIOc_inq_varid ncid = %d name = %s\n", my_rank, ncid, name);
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	printf("%d PIOc_inq_varid BCast ncid = %d\n", my_rank, ncid);
+	mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm);
+	int namelen;
+	namelen = strlen(name);
+	printf("%d PIOc_inq_varid BCast namelen = %d\n", my_rank, namelen);
+	mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm);
+	printf("%d PIOc_inq_varid BCast name = %s\n", my_rank, name);
+	mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+	printf("%d PIOc_inq_varid BCast done\n", my_rank);
+    }
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_inq_varid(file->fh, name, varidp);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_inq_varid(file->fh, name, varidp);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_inq_varid(file->fh, name, varidp);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if (varidp)
+	mpierr = MPI_Bcast(varidp, 1, MPI_INT, ios->ioroot, ios->my_comm);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_inq_attlen
+ * The PIO-C interface for the NetCDF function nc_inq_attlen.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param lenp a pointer that will get the number of values 
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_attlen (int ncid, int varid, const char *name, PIO_Offset *lenp) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_INQ_ATTLEN;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_inq_attlen(file->fh, varid, name, (size_t *)lenp);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_inq_attlen(file->fh, varid, name, (size_t *)lenp);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_inq_attlen(file->fh, varid, name, lenp);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_inq_atttype
+ * The PIO-C interface for the NetCDF function nc_inq_atttype.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param xtypep a pointer that will get the type of the attribute.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_atttype (int ncid, int varid, const char *name, nc_type *xtypep) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_INQ_ATTTYPE;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_inq_atttype(file->fh, varid, name, xtypep);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_inq_atttype(file->fh, varid, name, xtypep);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_inq_atttype(file->fh, varid, name, xtypep);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    mpierr = MPI_Bcast(xtypep , 1, MPI_INT, ios->ioroot, ios->my_comm);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_rename_var
+ * The PIO-C interface for the NetCDF function nc_rename_var.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_rename_var (int ncid, int varid, const char *name) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_RENAME_VAR;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_rename_var(file->fh, varid, name);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_rename_var(file->fh, varid, name);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_rename_var(file->fh, varid, name);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_inq_natts
+ * The PIO-C interface for the NetCDF function nc_inq_natts.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_natts (int ncid, int *ngattsp) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_INQ_NATTS;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_inq_natts(file->fh, ngattsp);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_inq_natts(file->fh, ngattsp);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_inq_natts(file->fh, ngattsp);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if (ngattsp)
+	mpierr = MPI_Bcast(ngattsp,1, MPI_INT, ios->ioroot, ios->my_comm);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_put_att_ulonglong
+ * The PIO-C interface for the NetCDF function nc_put_att_ulonglong.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_put_att_ulonglong (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned long long *op) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_PUT_ATT_ULONGLONG;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_put_att_ulonglong(file->fh, varid, name, xtype, (size_t)len, op);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_put_att_ulonglong(file->fh, varid, name, xtype, (size_t)len, op);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_put_att_ulonglong(file->fh, varid, name, xtype, len, op);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_inq_var
+ * The PIO-C interface for the NetCDF function nc_inq_var.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param xtypep a pointer that will get the type of the attribute.
+ * @param nattsp a pointer that will get the number of attributes 
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_var (int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp,
+		  int *dimidsp, int *nattsp) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_INQ_VAR;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm);
+	mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if (xtypep)
+	mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm);
+    if (ndimsp)
+    {
+	mpierr = MPI_Bcast(ndimsp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm);
+	file->varlist[varid].ndims = (*ndimsp);
+    }
+    if (nattsp)
+	mpierr = MPI_Bcast(nattsp, 1, MPI_INT, ios->ioroot, ios->my_comm);
+    if (name)
+    { 
+	int slen;
+	if(ios->iomaster)
+	    slen = (int) strlen(name) + 1;
+	mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm);
+	mpierr = MPI_Bcast((void *)name, slen, MPI_CHAR, ios->ioroot, ios->my_comm);
+    }
+    if (dimidsp)
+    {
+	int ndims;
+	PIOc_inq_varndims(file->fh, varid, &ndims);
+	mpierr = MPI_Bcast(dimidsp, ndims, MPI_INT, ios->ioroot, ios->my_comm);
+    }
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_rename_att
+ * The PIO-C interface for the NetCDF function nc_rename_att.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_RENAME_ATT;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_rename_att(file->fh, varid, name, newname);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_rename_att(file->fh, varid, name, newname);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_rename_att(file->fh, varid, name, newname);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_put_att_ushort
+ * The PIO-C interface for the NetCDF function nc_put_att_ushort.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_put_att_ushort (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned short *op) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_PUT_ATT_USHORT;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_put_att_ushort(file->fh, varid, name, xtype, (size_t)len, op);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_put_att_ushort(file->fh, varid, name, xtype, (size_t)len, op);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_put_att_ushort(file->fh, varid, name, xtype, len, op);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_inq_dimid
+ * The PIO-C interface for the NetCDF function nc_inq_dimid.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param idp a pointer that will get the id of the variable or attribute.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_dimid (int ncid, const char *name, int *idp) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_INQ_DIMID;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm);
+	int namelen;
+	if (ios->iomaster)
+	    namelen = strlen(name);
+	mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm);
+	mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_inq_dimid(file->fh, name, idp);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_inq_dimid(file->fh, name, idp);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_inq_dimid(file->fh, name, idp);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if (idp)
+	mpierr = MPI_Bcast(idp, 1, MPI_INT, ios->ioroot, ios->my_comm);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_put_att_text
+ * The PIO-C interface for the NetCDF function nc_put_att_text.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_put_att_text (int ncid, int varid, const char *name, PIO_Offset len, const char *op) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_PUT_ATT_TEXT;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_put_att_text(file->fh, varid, name, (size_t)len, op);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_put_att_text(file->fh, varid, name, (size_t)len, op);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_put_att_text(file->fh, varid, name, len, op);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_get_att_uint
+ * The PIO-C interface for the NetCDF function nc_get_att_uint.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_get_att_uint (int ncid, int varid, const char *name, unsigned int *ip) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_GET_ATT_UINT;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_get_att_uint(file->fh, varid, name, ip);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_get_att_uint(file->fh, varid, name, ip);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_get_att_uint(file->fh, varid, name, ip);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char));
+	sprintf(errstr,"name %s in file %s",name,__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(ierr == PIO_NOERR){
+        PIO_Offset attlen;
+        PIOc_inq_attlen(file->fh, varid, name, &attlen);
+        mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED, ios->ioroot, ios->my_comm);
+    }
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_inq_format
+ * The PIO-C interface for the NetCDF function nc_inq_format.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param formatp a pointer that will get the file format 
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_format (int ncid, int *formatp) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_INQ_FORMAT;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_inq_format(file->fh, formatp);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_inq_format(file->fh, formatp);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_inq_format(file->fh, formatp);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    mpierr = MPI_Bcast(formatp , 1, MPI_INT, ios->ioroot, ios->my_comm);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_get_att_long
+ * The PIO-C interface for the NetCDF function nc_get_att_long.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_get_att_long (int ncid, int varid, const char *name, long *ip) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_GET_ATT_LONG;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_get_att_long(file->fh, varid, name, ip);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_get_att_long(file->fh, varid, name, ip);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_get_att_long(file->fh, varid, name, ip);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char));
+	sprintf(errstr,"name %s in file %s",name,__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(ierr == PIO_NOERR){
+        PIO_Offset attlen;
+        PIOc_inq_attlen(file->fh, varid, name, &attlen);
+        mpierr = MPI_Bcast(ip , (int) attlen, MPI_LONG, ios->ioroot, ios->my_comm);
+    }
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_inq_attname
+ * The PIO-C interface for the NetCDF function nc_inq_attname.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param attnum the attribute ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_attname (int ncid, int varid, int attnum, char *name) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_INQ_ATTNAME;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_inq_attname(file->fh, varid, attnum, name);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_inq_attname(file->fh, varid, attnum, name);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_inq_attname(file->fh, varid, attnum, name);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(name != NULL){
+	int slen;
+	if(ios->iomaster)
+	    slen = (int) strlen(name) + 1;
+	mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm);
+	mpierr = MPI_Bcast((void *)name, slen, MPI_CHAR, ios->ioroot, ios->my_comm);
+    }
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_inq_att
+ * The PIO-C interface for the NetCDF function nc_inq_att.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param xtypep a pointer that will get the type of the attribute.
+ * @param lenp a pointer that will get the number of values 
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, PIO_Offset *lenp) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_INQ_ATT;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_inq_att(file->fh, varid, name, xtypep, lenp);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(xtypep != NULL) mpierr = MPI_Bcast(xtypep , 1, MPI_INT, ios->ioroot, ios->my_comm);
+    if(lenp != NULL) mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_put_att_long
+ * The PIO-C interface for the NetCDF function nc_put_att_long.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_put_att_long (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const long *op) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_PUT_ATT_LONG;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_put_att_long(file->fh, varid, name, xtype, (size_t)len, op);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_put_att_long(file->fh, varid, name, xtype, (size_t)len, op);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_put_att_long(file->fh, varid, name, xtype, len, op);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_inq_unlimdim
+ * The PIO-C interface for the NetCDF function nc_inq_unlimdim.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_unlimdim (int ncid, int *unlimdimidp) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_INQ_UNLIMDIM;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_inq_unlimdim(file->fh, unlimdimidp);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_inq_unlimdim(file->fh, unlimdimidp);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_inq_unlimdim(file->fh, unlimdimidp);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if (unlimdimidp)
+	mpierr = MPI_Bcast(unlimdimidp,1, MPI_INT, ios->ioroot, ios->my_comm);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_get_att_float
+ * The PIO-C interface for the NetCDF function nc_get_att_float.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_get_att_float (int ncid, int varid, const char *name, float *ip) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_GET_ATT_FLOAT;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_get_att_float(file->fh, varid, name, ip);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_get_att_float(file->fh, varid, name, ip);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_get_att_float(file->fh, varid, name, ip);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char));
+	sprintf(errstr,"name %s in file %s",name,__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(ierr == PIO_NOERR){
+        PIO_Offset attlen;
+        PIOc_inq_attlen(file->fh, varid, name, &attlen);
+        mpierr = MPI_Bcast(ip , (int) attlen, MPI_FLOAT, ios->ioroot, ios->my_comm);
+    }
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_inq_ndims
+ * The PIO-C interface for the NetCDF function nc_inq_ndims.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_ndims (int ncid, int *ndimsp) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_INQ_NDIMS;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_inq_ndims(file->fh, ndimsp);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_inq_ndims(file->fh, ndimsp);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_inq_ndims(file->fh, ndimsp);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if (ndimsp)
+	mpierr = MPI_Bcast(ndimsp , 1, MPI_INT, ios->ioroot, ios->my_comm);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_put_att_int
+ * The PIO-C interface for the NetCDF function nc_put_att_int.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_put_att_int (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const int *op) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_PUT_ATT_INT;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm);
+	mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_put_att_int(file->fh, varid, name, xtype, (size_t)len, op);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_put_att_int(file->fh, varid, name, xtype, (size_t)len, op);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_put_att_int(file->fh, varid, name, xtype, len, op);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_inq_nvars
+ * The PIO-C interface for the NetCDF function nc_inq_nvars.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_nvars (int ncid, int *nvarsp) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_INQ_NVARS;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(!ios->comp_rank) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_inq_nvars(file->fh, nvarsp);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_inq_nvars(file->fh, nvarsp);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_inq_nvars(file->fh, nvarsp);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if (nvarsp)
+	mpierr = MPI_Bcast(nvarsp,1, MPI_INT, ios->ioroot, ios->my_comm);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_enddef
+ * The PIO-C interface for the NetCDF function nc_enddef.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_enddef (int ncid) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_ENDDEF;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(!ios->comp_rank) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	printf("PIOc_enddef file->fh = %d\n", file->fh);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_enddef(file->fh);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_enddef(file->fh);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_enddef(file->fh);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_put_att_uchar
+ * The PIO-C interface for the NetCDF function nc_put_att_uchar.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_put_att_uchar (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned char *op) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_PUT_ATT_UCHAR;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_put_att_uchar(file->fh, varid, name, xtype, (size_t)len, op);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_put_att_uchar(file->fh, varid, name, xtype, (size_t)len, op);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_put_att_uchar(file->fh, varid, name, xtype, len, op);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_put_att_longlong
+ * The PIO-C interface for the NetCDF function nc_put_att_longlong.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_put_att_longlong (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const long long *op) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_PUT_ATT_LONGLONG;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_put_att_longlong(file->fh, varid, name, xtype, (size_t)len, op);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_put_att_longlong(file->fh, varid, name, xtype, (size_t)len, op);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_put_att_longlong(file->fh, varid, name, xtype, len, op);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_inq_varnatts
+ * The PIO-C interface for the NetCDF function nc_inq_varnatts.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param nattsp a pointer that will get the number of attributes 
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_varnatts (int ncid, int varid, int *nattsp) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_INQ_VARNATTS;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm);
+	mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_inq_varnatts(file->fh, varid, nattsp);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_inq_varnatts(file->fh, varid, nattsp);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_inq_varnatts(file->fh, varid, nattsp);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if (nattsp)
+	mpierr = MPI_Bcast(nattsp, 1, MPI_INT, ios->ioroot, ios->my_comm);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_get_att_ubyte
+ * The PIO-C interface for the NetCDF function nc_get_att_ubyte.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_get_att_ubyte (int ncid, int varid, const char *name, unsigned char *ip) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_GET_ATT_UBYTE;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_get_att_ubyte(file->fh, varid, name, ip);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_get_att_ubyte(file->fh, varid, name, ip);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_get_att_ubyte(file->fh, varid, name, ip);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char));
+	sprintf(errstr,"name %s in file %s",name,__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(ierr == PIO_NOERR){
+        PIO_Offset attlen;
+        PIOc_inq_attlen(file->fh, varid, name, &attlen);
+        mpierr = MPI_Bcast(ip , (int) attlen, MPI_BYTE, ios->ioroot, ios->my_comm);
+    }
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_get_att_text
+ * The PIO-C interface for the NetCDF function nc_get_att_text.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_get_att_text (int ncid, int varid, const char *name, char *ip) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_GET_ATT_TEXT;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_get_att_text(file->fh, varid, name, ip);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_get_att_text(file->fh, varid, name, ip);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_get_att_text(file->fh, varid, name, ip);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char));
+	sprintf(errstr,"name %s in file %s",name,__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(ierr == PIO_NOERR){
+        PIO_Offset attlen;
+        PIOc_inq_attlen(file->fh, varid, name, &attlen);
+        mpierr = MPI_Bcast(ip , (int) attlen, MPI_CHAR, ios->ioroot, ios->my_comm);
+    }
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_del_att
+ * The PIO-C interface for the NetCDF function nc_del_att.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_del_att (int ncid, int varid, const char *name) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_DEL_ATT;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_del_att(file->fh, varid, name);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_del_att(file->fh, varid, name);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_del_att(file->fh, varid, name);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_inq_dimlen
+ * The PIO-C interface for the NetCDF function nc_inq_dimlen.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param lenp a pointer that will get the number of values 
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_dimlen (int ncid, int dimid, PIO_Offset *lenp) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_INQ_DIMLEN;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+	mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm);    
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_inq_dimlen(file->fh, dimid, (size_t *)lenp);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_inq_dimlen(file->fh, dimid, (size_t *)lenp);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_inq_dimlen(file->fh, dimid, lenp);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if (lenp)
+	mpierr = MPI_Bcast(lenp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_get_att_schar
+ * The PIO-C interface for the NetCDF function nc_get_att_schar.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_get_att_schar (int ncid, int varid, const char *name, signed char *ip) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_GET_ATT_SCHAR;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_get_att_schar(file->fh, varid, name, ip);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_get_att_schar(file->fh, varid, name, ip);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_get_att_schar(file->fh, varid, name, ip);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char));
+	sprintf(errstr,"name %s in file %s",name,__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(ierr == PIO_NOERR){
+        PIO_Offset attlen;
+        PIOc_inq_attlen(file->fh, varid, name, &attlen);
+        mpierr = MPI_Bcast(ip , (int) attlen, MPI_CHAR, ios->ioroot, ios->my_comm);
+    }
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_get_att_ulonglong
+ * The PIO-C interface for the NetCDF function nc_get_att_ulonglong.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_get_att_ulonglong (int ncid, int varid, const char *name, unsigned long long *ip) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_GET_ATT_ULONGLONG;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_get_att_ulonglong(file->fh, varid, name, ip);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_get_att_ulonglong(file->fh, varid, name, ip);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_get_att_ulonglong(file->fh, varid, name, ip);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char));
+	sprintf(errstr,"name %s in file %s",name,__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(ierr == PIO_NOERR){
+        PIO_Offset attlen;
+        PIOc_inq_attlen(file->fh, varid, name, &attlen);
+        mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_LONG_LONG, ios->ioroot, ios->my_comm);
+    }
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_inq_varndims
+ * The PIO-C interface for the NetCDF function nc_inq_varndims.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_varndims (int ncid, int varid, int *ndimsp) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_INQ_VARNDIMS;
+    if(file->varlist[varid].ndims > 0){
+	(*ndimsp) = file->varlist[varid].ndims;
+	return PIO_NOERR;
+    }
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&file->fh,1, MPI_INT, ios->compmaster, ios->intercomm);
+	mpierr = MPI_Bcast(&varid,1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_inq_varndims(file->fh, varid, ndimsp);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_inq_varndims(file->fh, varid, ndimsp);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_inq_varndims(file->fh, varid, ndimsp);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if (ndimsp)
+    {
+	mpierr = MPI_Bcast(ndimsp,1, MPI_INT, ios->ioroot, ios->my_comm);
+	file->varlist[varid].ndims = (*ndimsp);
+    }
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_inq_varname
+ * The PIO-C interface for the NetCDF function nc_inq_varname.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_varname (int ncid, int varid, char *name) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_INQ_VARNAME;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&file->fh,1, MPI_INT, ios->compmaster, ios->intercomm);
+	mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_inq_varname(file->fh, varid, name);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_inq_varname(file->fh, varid, name);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_inq_varname(file->fh, varid, name);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(name != NULL){
+	int slen;
+	if(ios->iomaster)
+	    slen = (int) strlen(name) + 1;
+	mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm);
+	mpierr = MPI_Bcast((void *)name, slen, MPI_CHAR, ios->ioroot, ios->my_comm);
+    }
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_def_dim
+ * The PIO-C interface for the NetCDF function nc_def_dim.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param idp a pointer that will get the id of the variable or attribute.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+    int namelen;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    int my_rank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);    
+    printf("%d PIOc_def_dim ncid = %d name = %s len = %d\n", my_rank,
+	   ncid, name, len);
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_DEF_DIM;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+	namelen = strlen(name);
+	printf("bcasting namelen = %d name = %s len = %d\n", namelen, name, len);
+	if (!ios->compmaster)
+	    ios->compmaster = MPI_PROC_NULL;
+	mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+	mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+	mpierr = MPI_Bcast(&len, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+    }
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_def_dim(file->fh, name, (size_t)len, idp);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_def_dim(file->fh, name, (size_t)len, idp);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_def_dim(file->fh, name, len, idp);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_put_att_uint
+ * The PIO-C interface for the NetCDF function nc_put_att_uint.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_put_att_uint (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned int *op) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_PUT_ATT_UINT;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_put_att_uint(file->fh, varid, name, xtype, (size_t)len, op);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_put_att_uint(file->fh, varid, name, xtype, (size_t)len, op);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_put_att_uint(file->fh, varid, name, xtype, len, op);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_get_att_short
+ * The PIO-C interface for the NetCDF function nc_get_att_short.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_get_att_short (int ncid, int varid, const char *name, short *ip) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_GET_ATT_SHORT;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_get_att_short(file->fh, varid, name, ip);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_get_att_short(file->fh, varid, name, ip);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_get_att_short(file->fh, varid, name, ip);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char));
+	sprintf(errstr,"name %s in file %s",name,__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(ierr == PIO_NOERR){
+        PIO_Offset attlen;
+        PIOc_inq_attlen(file->fh, varid, name, &attlen);
+        mpierr = MPI_Bcast(ip , (int) attlen, MPI_SHORT, ios->ioroot, ios->my_comm);
+    }
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_redef
+ * The PIO-C interface for the NetCDF function nc_redef.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_redef (int ncid) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_REDEF;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_redef(file->fh);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_redef(file->fh);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_redef(file->fh);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_put_att_ubyte
+ * The PIO-C interface for the NetCDF function nc_put_att_ubyte.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_put_att_ubyte (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned char *op) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_PUT_ATT_UBYTE;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_put_att_ubyte(file->fh, varid, name, xtype, (size_t)len, op);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_put_att_ubyte(file->fh, varid, name, xtype, (size_t)len, op);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_put_att_ubyte(file->fh, varid, name, xtype, len, op);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_get_att_int
+ * The PIO-C interface for the NetCDF function nc_get_att_int.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_get_att_int (int ncid, int varid, const char *name, int *ip) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_GET_ATT_INT;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_get_att_int(file->fh, varid, name, ip);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_get_att_int(file->fh, varid, name, ip);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_get_att_int(file->fh, varid, name, ip);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char));
+	sprintf(errstr,"name %s in file %s",name,__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(ierr == PIO_NOERR){
+        PIO_Offset attlen;
+        PIOc_inq_attlen(file->fh, varid, name, &attlen);
+        mpierr = MPI_Bcast(ip , (int) attlen, MPI_INT, ios->ioroot, ios->my_comm);
+    }
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_get_att_longlong
+ * The PIO-C interface for the NetCDF function nc_get_att_longlong.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_get_att_longlong (int ncid, int varid, const char *name, long long *ip) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_GET_ATT_LONGLONG;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_get_att_longlong(file->fh, varid, name, ip);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_get_att_longlong(file->fh, varid, name, ip);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_get_att_longlong(file->fh, varid, name, ip);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char));
+	sprintf(errstr,"name %s in file %s",name,__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(ierr == PIO_NOERR){
+        PIO_Offset attlen;
+        PIOc_inq_attlen(file->fh, varid, name, &attlen);
+        mpierr = MPI_Bcast(ip , (int) attlen, MPI_LONG_LONG, ios->ioroot, ios->my_comm);
+    }
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
+/** 
+ * @ingroup PIOc_put_att_float
+ * The PIO-C interface for the NetCDF function nc_put_att_float.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_put_att_float (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const float *op) 
+{
+    int ierr;
+    int msg;
+    int mpierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    char *errstr;
+
+    errstr = NULL;
+    ierr = PIO_NOERR;
+
+    file = pio_get_file_from_id(ncid);
+    if(file == NULL)
+	return PIO_EBADID;
+    ios = file->iosystem;
+    msg = PIO_MSG_PUT_ATT_FLOAT;
+
+    if(ios->async_interface && ! ios->ioproc){
+	if(ios->compmaster) 
+	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
+    }
+
+
+    if(ios->ioproc){
+	switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+	case PIO_IOTYPE_NETCDF4P:
+	    ierr = nc_put_att_float(file->fh, varid, name, xtype, (size_t)len, op);;
+	    break;
+	case PIO_IOTYPE_NETCDF4C:
+#endif
+	case PIO_IOTYPE_NETCDF:
+	    if(ios->io_rank==0){
+		ierr = nc_put_att_float(file->fh, varid, name, xtype, (size_t)len, op);;
+	    }
+	    break;
+#endif
+#ifdef _PNETCDF
+	case PIO_IOTYPE_PNETCDF:
+	    ierr = ncmpi_put_att_float(file->fh, varid, name, xtype, len, op);;
+	    break;
+#endif
+	default:
+	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+	}
+    }
+
+    if(ierr != PIO_NOERR){
+	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
+	sprintf(errstr,"in file %s",__FILE__);
+    }
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    if(errstr != NULL) free(errstr);
+    return ierr;
+}
+
diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c
new file mode 100644
index 000000000000..c0b621eb903d
--- /dev/null
+++ b/src/clib/pio_put_nc_async.c
@@ -0,0 +1,5073 @@
+#include <pio.h>
+#include <pio_internal.h>
+
+///
+/// PIO interface to nc_put_vars_uchar
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_vars_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned char *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARS_UCHAR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_vars_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_vars_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_vars_uchar(file->fh, varid, start, count, stride, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_vars_ushort
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_vars_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned short *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARS_USHORT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_vars_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_vars_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_vars_ushort(file->fh, varid, start, count, stride, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_vars_ulonglong
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_vars_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned long long *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARS_ULONGLONG;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_vars_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_vars_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_vars_ulonglong(file->fh, varid, start, count, stride, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARM;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,   buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,   buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_varm(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_vars_uint
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_vars_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned int *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARS_UINT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_vars_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_vars_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_vars_uint(file->fh, varid, start, count, stride, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_uchar
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned char *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARM_UCHAR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_varm_uchar(file->fh, varid, start, count, stride, imap, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_var_ushort
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_var_ushort (int ncid, int varid, const unsigned short *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VAR_USHORT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_var_ushort(file->fh, varid, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_var_ushort(file->fh, varid, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_var_ushort(file->fh, varid, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_var1_longlong
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_var1_longlong (int ncid, int varid, const PIO_Offset index[], const long long *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VAR1_LONGLONG;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_var1_longlong(file->fh, varid, (size_t *) index, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_var1_longlong(file->fh, varid, (size_t *) index, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_var1_longlong(file->fh, varid, index, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_vara_uchar
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_vara_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned char *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARA_UCHAR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_vara_uchar(file->fh, varid, (size_t *) start, (size_t *) count, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_vara_uchar(file->fh, varid, (size_t *) start, (size_t *) count, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_vara_uchar(file->fh, varid, start, count, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_short
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_varm_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const short *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARM_SHORT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_varm_short(file->fh, varid, start, count, stride, imap, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_var1_long
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_var1_long (int ncid, int varid, const PIO_Offset index[], const long *ip) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VAR1_LONG;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_var1_long(file->fh, varid, (size_t *) index,  ip);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_var1_long(file->fh, varid, (size_t *) index,  ip);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_var1_long(file->fh, varid, index, ip, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_vars_long
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_vars_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const long *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARS_LONG;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_vars_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_vars_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_vars_long(file->fh, varid, start, count, stride, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_var_short
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_var_short (int ncid, int varid, const short *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VAR_SHORT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_var_short(file->fh, varid, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_var_short(file->fh, varid, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_var_short(file->fh, varid, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_vara_int
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const int *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARA_INT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_vara_int(file->fh, varid, (size_t *) start, (size_t *) count, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_vara_int(file->fh, varid, (size_t *) start, (size_t *) count, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_vara_int(file->fh, varid, start, count, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_var1_ushort
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_var1_ushort (int ncid, int varid, const PIO_Offset index[], const unsigned short *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VAR1_USHORT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_var1_ushort(file->fh, varid, (size_t *) index, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_var1_ushort(file->fh, varid, (size_t *) index, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_var1_ushort(file->fh, varid, index, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_vara_text
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const char *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARA_TEXT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_vara_text(file->fh, varid, (size_t *) start, (size_t *) count, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_vara_text(file->fh, varid, (size_t *) start, (size_t *) count, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_vara_text(file->fh, varid, start, count, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_text
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const char *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARM_TEXT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_varm_text(file->fh, varid, start, count, stride, imap, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_ushort
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_varm_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned short *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARM_USHORT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_varm_ushort(file->fh, varid, start, count, stride, imap, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_var_ulonglong
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_var_ulonglong (int ncid, int varid, const unsigned long long *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VAR_ULONGLONG;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_var_ulonglong(file->fh, varid, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_var_ulonglong(file->fh, varid, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_var_ulonglong(file->fh, varid, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_var_int
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_var_int (int ncid, int varid, const int *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VAR_INT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_var_int(file->fh, varid, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_var_int(file->fh, varid, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_var_int(file->fh, varid, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_var_longlong
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_var_longlong (int ncid, int varid, const long long *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VAR_LONGLONG;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_var_longlong(file->fh, varid, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_var_longlong(file->fh, varid, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_var_longlong(file->fh, varid, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_var_schar
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_var_schar (int ncid, int varid, const signed char *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VAR_SCHAR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_var_schar(file->fh, varid, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_var_schar(file->fh, varid, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_var_schar(file->fh, varid, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_var_uint
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_var_uint (int ncid, int varid, const unsigned int *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VAR_UINT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_var_uint(file->fh, varid, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_var_uint(file->fh, varid, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_var_uint(file->fh, varid, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_var
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_var (int ncid, int varid, const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VAR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_var(file->fh, varid,   buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_var(file->fh, varid,   buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_var(file->fh, varid, buf, bufcount, buftype, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_vara_ushort
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_vara_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned short *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARA_USHORT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_vara_ushort(file->fh, varid, start, count, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_vars_short
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_vars_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const short *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARS_SHORT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_vars_short(file->fh, varid, start, count, stride, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_vara_uint
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_vara_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned int *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARA_UINT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_vara_uint(file->fh, varid, start, count, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_vara_schar
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_vara_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const signed char *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARA_SCHAR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_vara_schar(file->fh, varid, start, count, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_ulonglong
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned long long *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARM_ULONGLONG;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_varm_ulonglong(file->fh, varid, start, count, stride, imap, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_var1_uchar
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_var1_uchar (int ncid, int varid, const PIO_Offset index[], const unsigned char *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VAR1_UCHAR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_var1_uchar(file->fh, varid, (size_t *) index, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_var1_uchar(file->fh, varid, (size_t *) index, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_var1_uchar(file->fh, varid, index, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_int
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const int *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARM_INT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_varm_int(file->fh, varid, start, count, stride, imap, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_vars_schar
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_vars_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const signed char *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARS_SCHAR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_vars_schar(file->fh, varid, start, count, stride, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_var1
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_var1 (int ncid, int varid, const PIO_Offset index[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VAR1;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_var1(file->fh, varid, (size_t *) index,   buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_var1(file->fh, varid, (size_t *) index,   buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_var1(file->fh, varid, index, buf, bufcount, buftype, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_vara_float
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_vara_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const float *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARA_FLOAT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_vara_float(file->fh, varid, (size_t *) start, (size_t *) count, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_vara_float(file->fh, varid, (size_t *) start, (size_t *) count, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_vara_float(file->fh, varid, start, count, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_var1_float
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_var1_float (int ncid, int varid, const PIO_Offset index[], const float *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VAR1_FLOAT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_var1_float(file->fh, varid, (size_t *) index, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_var1_float(file->fh, varid, (size_t *) index, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_var1_float(file->fh, varid, index, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_float
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_varm_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const float *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARM_FLOAT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_varm_float(file->fh, varid, start, count, stride, imap, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_var1_text
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_var1_text (int ncid, int varid, const PIO_Offset index[], const char *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VAR1_TEXT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_var1_text(file->fh, varid, (size_t *) index, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_var1_text(file->fh, varid, (size_t *) index, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_var1_text(file->fh, varid, index, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_vars_text
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_vars_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const char *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARS_TEXT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_vars_text(file->fh, varid, start, count, stride, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_long
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const long *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARM_LONG;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_varm_long(file->fh, varid, start, count, stride, imap, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_vars_double
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_vars_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const double *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARS_DOUBLE;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_vars_double(file->fh, varid, start, count, stride, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_vara_longlong
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_vara_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const long long *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARA_LONGLONG;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_vara_longlong(file->fh, varid, start, count, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_var_double
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_var_double (int ncid, int varid, const double *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VAR_DOUBLE;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_var_double(file->fh, varid, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_var_double(file->fh, varid, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_var_double(file->fh, varid, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_var_float
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_var_float (int ncid, int varid, const float *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VAR_FLOAT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_var_float(file->fh, varid, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_var_float(file->fh, varid, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_var_float(file->fh, varid, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_var1_ulonglong
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_var1_ulonglong (int ncid, int varid, const PIO_Offset index[], const unsigned long long *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VAR1_ULONGLONG;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_var1_ulonglong(file->fh, varid, (size_t *) index, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_var1_ulonglong(file->fh, varid, (size_t *) index, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_var1_ulonglong(file->fh, varid, index, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_uint
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned int *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARM_UINT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_varm_uint(file->fh, varid, start, count, stride, imap, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_var1_uint
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_var1_uint (int ncid, int varid, const PIO_Offset index[], const unsigned int *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VAR1_UINT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_var1_uint(file->fh, varid, (size_t *) index, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_var1_uint(file->fh, varid, (size_t *) index, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_var1_uint(file->fh, varid, index, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_var1_int
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_var1_int (int ncid, int varid, const PIO_Offset index[], const int *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VAR1_INT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_var1_int(file->fh, varid, (size_t *) index, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_var1_int(file->fh, varid, (size_t *) index, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_var1_int(file->fh, varid, index, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_vars_float
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_vars_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const float *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARS_FLOAT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_vars_float(file->fh, varid, start, count, stride, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_vara_short
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_vara_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const short *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARA_SHORT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_vara_short(file->fh, varid, (size_t *) start, (size_t *) count, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_vara_short(file->fh, varid, (size_t *) start, (size_t *) count, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_vara_short(file->fh, varid, start, count, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_var1_schar
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_var1_schar (int ncid, int varid, const PIO_Offset index[], const signed char *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VAR1_SCHAR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_var1_schar(file->fh, varid, (size_t *) index, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_var1_schar(file->fh, varid, (size_t *) index, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_var1_schar(file->fh, varid, index, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_vara_ulonglong
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_vara_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned long long *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARA_ULONGLONG;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_vara_ulonglong(file->fh, varid, start, count, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_double
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_varm_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const double *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARM_DOUBLE;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_varm_double(file->fh, varid, start, count, stride, imap, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_vara
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_vara (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARA;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_vara(file->fh, varid, (size_t *) start, (size_t *) count,   buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_vara(file->fh, varid, (size_t *) start, (size_t *) count,   buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_vara(file->fh, varid, start, count, buf, bufcount, buftype, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_vara_long
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_vara_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const long *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARA_LONG;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_vara_long(file->fh, varid, (size_t *) start, (size_t *) count, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_vara_long(file->fh, varid, (size_t *) start, (size_t *) count, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_vara_long(file->fh, varid, start, count, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_var1_double
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_var1_double (int ncid, int varid, const PIO_Offset index[], const double *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VAR1_DOUBLE;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_var1_double(file->fh, varid, (size_t *) index, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_var1_double(file->fh, varid, (size_t *) index, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_var1_double(file->fh, varid, index, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_schar
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_varm_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const signed char *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARM_SCHAR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_varm_schar(file->fh, varid, start, count, stride, imap, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_var_text
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_var_text (int ncid, int varid, const char *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VAR_TEXT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_var_text(file->fh, varid, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_var_text(file->fh, varid, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_var_text(file->fh, varid, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_vars_int
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_vars_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const int *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARS_INT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_vars_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_vars_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_vars_int(file->fh, varid, start, count, stride, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_var1_short
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_var1_short (int ncid, int varid, const PIO_Offset index[], const short *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VAR1_SHORT;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_var1_short(file->fh, varid, (size_t *) index, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_var1_short(file->fh, varid, (size_t *) index, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_var1_short(file->fh, varid, index, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_vars_longlong
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_vars_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const long long *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARS_LONGLONG;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_vars_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_vars_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_vars_longlong(file->fh, varid, start, count, stride, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_vara_double
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_vara_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const double *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARA_DOUBLE;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_vara_double(file->fh, varid, (size_t *) start, (size_t *) count, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_vara_double(file->fh, varid, (size_t *) start, (size_t *) count, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_vara_double(file->fh, varid, start, count, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_vars
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_vars (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARS;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,   buf);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride,   buf);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_vars(file->fh, varid, start, count, stride, buf, bufcount, buftype, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_var_uchar
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_var_uchar (int ncid, int varid, const unsigned char *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VAR_UCHAR;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_var_uchar(file->fh, varid, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_var_uchar(file->fh, varid, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_var_uchar(file->fh, varid, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_var_long
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_var_long (int ncid, int varid, const long *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VAR_LONG;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_var_long(file->fh, varid, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_var_long(file->fh, varid, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_var_long(file->fh, varid, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_longlong
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.  
+/// 
+/// Refer to the <A HREF="http://www.unidata.ucar.edu/software/netcdf/docs/netcdf_documentation.html"> netcdf documentation. </A>
+///
+int PIOc_put_varm_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const long long *op) 
+{
+  int ierr;
+  int msg;
+  int mpierr;
+  iosystem_desc_t *ios;
+  file_desc_t *file;
+  var_desc_t *vdesc;
+  PIO_Offset usage;
+  int *request;
+
+  ierr = PIO_NOERR;
+
+  file = pio_get_file_from_id(ncid);
+  if(file == NULL)
+    return PIO_EBADID;
+  ios = file->iosystem;
+  msg = PIO_MSG_PUT_VARM_LONGLONG;
+
+  if(ios->async_interface && ! ios->ioproc){
+    if(ios->compmaster) 
+      mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+    mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
+  }
+
+
+  if(ios->ioproc){
+    switch(file->iotype){
+#ifdef _NETCDF
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+      ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+      ierr = nc_put_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+      break;
+    case PIO_IOTYPE_NETCDF4C:
+#endif
+    case PIO_IOTYPE_NETCDF:
+      if(ios->io_rank==0){
+	ierr = nc_put_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+      }
+      break;
+#endif
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+      vdesc = file->varlist + varid;
+
+      if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+	vdesc->request = realloc(vdesc->request, 
+				 sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+      }
+      request = vdesc->request+vdesc->nreqs;
+
+      if(ios->io_rank==0){
+	ierr = ncmpi_bput_varm_longlong(file->fh, varid, start, count, stride, imap, op, request);;
+      }else{
+	*request = PIO_REQ_NULL;
+      }
+      vdesc->nreqs++;
+      flush_output_buffer(file, false, 0);
+      break;
+#endif
+    default:
+      ierr = iotype_error(file->iotype,__FILE__,__LINE__);
+    }
+  }
+
+  ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+  return ierr;
+}
+
diff --git a/src/clib/pioc.c b/src/clib/pioc.c
index eb9a790ffa2f..22d6ea3307f1 100644
--- a/src/clib/pioc.c
+++ b/src/clib/pioc.c
@@ -357,87 +357,107 @@ int PIOc_Init_Intracomm(const MPI_Comm comp_comm,
 			const int base,const int rearr, int *iosysidp)
 {
   iosystem_desc_t *iosys;
-  int ierr;
+  int ierr = PIO_NOERR;
   int ustride;
   int lbase;
+  int mpierr;
+
   iosys = (iosystem_desc_t *) malloc(sizeof(iosystem_desc_t));
 
-  iosys->union_comm = comp_comm;
-  iosys->comp_comm = comp_comm;
-  iosys->my_comm = comp_comm;
-  iosys->io_comm = MPI_COMM_NULL;
-  iosys->intercomm = MPI_COMM_NULL;
-  iosys->error_handler = PIO_INTERNAL_ERROR;
-  iosys->async_interface= false;
-  iosys->compmaster = false;
-  iosys->iomaster = false;
-  iosys->ioproc = false;
-  iosys->default_rearranger = rearr;
-  iosys->num_iotasks = num_iotasks;
-
-  ustride = stride;
- 
-  CheckMPIReturn(MPI_Comm_rank(comp_comm, &(iosys->comp_rank)),__FILE__,__LINE__);
-  CheckMPIReturn(MPI_Comm_size(comp_comm, &(iosys->num_comptasks)),__FILE__,__LINE__);
-  if(iosys->comp_rank==0)
-    iosys->compmaster = true;  
-
-#ifdef BGQxxx
-  lbase = base;
-  determineiotasks(comp_comm, &(iosys->num_iotasks), &lbase, &stride, &rearr, &(iosys->ioproc));
-  if(iosys->comp_rank==0)
-    printf("%s %d %d\n",__FILE__,__LINE__,iosys->num_iotasks);
-  if(iosys->ioproc)
-    printf("%s %d %d\n",__FILE__,__LINE__,iosys->comp_rank);
-    
-#else
-  if((iosys->num_comptasks == 1) && (num_iotasks*ustride > 1)) {
-    // This is a serial run with a bad configuration. Set up a single task.
-    fprintf(stderr, "PIO_TP PIOc_Init_Intracomm reset stride and tasks.\n");
-    iosys->num_iotasks = 1;
-    ustride = 1;
+  /* Copy the computation communicator into union_comm. */
+  mpierr = MPI_Comm_dup(comp_comm, &iosys->union_comm);
+  CheckMPIReturn(mpierr, __FILE__, __LINE__);		
+  if (mpierr)
+      ierr = PIO_EIO;
+
+  /* Copy the computation communicator into comp_comm. */
+  if (!ierr)
+  {
+      mpierr = MPI_Comm_dup(comp_comm, &iosys->comp_comm);
+      CheckMPIReturn(mpierr, __FILE__, __LINE__);		
+      if (mpierr)
+	  ierr = PIO_EIO;
   }
-  if((iosys->num_iotasks < 1) || ((iosys->num_iotasks*ustride) > iosys->num_comptasks)){
-    fprintf(stderr, "PIO_TP PIOc_Init_Intracomm error\n");
-    fprintf(stderr, "num_iotasks=%d, ustride=%d, num_comptasks=%d\n", num_iotasks, ustride, iosys->num_comptasks);
-    return PIO_EBADID;
-  }
-  iosys->ioranks = (int *) calloc(sizeof(int), iosys->num_iotasks);
-  for(int i=0;i< iosys->num_iotasks; i++){
-    iosys->ioranks[i] = (base + i*ustride) % iosys->num_comptasks;
-    if(iosys->ioranks[i] == iosys->comp_rank)
-      iosys->ioproc = true;
-  }
-  iosys->ioroot = iosys->ioranks[0];
-#endif
 
-  CheckMPIReturn(MPI_Info_create(&(iosys->info)),__FILE__,__LINE__);
-  iosys->info = MPI_INFO_NULL;
+  if (!ierr)
+  {
+      iosys->my_comm = iosys->comp_comm;
+      iosys->io_comm = MPI_COMM_NULL;
+      iosys->intercomm = MPI_COMM_NULL;
+      iosys->error_handler = PIO_INTERNAL_ERROR;
+      iosys->async_interface= false;
+      iosys->compmaster = 0;
+      iosys->iomaster = 0;
+      iosys->ioproc = false;
+      iosys->default_rearranger = rearr;
+      iosys->num_iotasks = num_iotasks;
+
+      ustride = stride;
+
+      /* Find MPI rank and number of tasks in comp_comm communicator. */
+      CheckMPIReturn(MPI_Comm_rank(iosys->comp_comm, &(iosys->comp_rank)),__FILE__,__LINE__);
+      CheckMPIReturn(MPI_Comm_size(iosys->comp_comm, &(iosys->num_comptasks)),__FILE__,__LINE__);
+      if(iosys->comp_rank==0)
+	  iosys->compmaster = MPI_ROOT;  
+
+      /* Ensure that settings for number of computation tasks, number
+       * of IO tasks, and the stride are reasonable. */
+      if((iosys->num_comptasks == 1) && (num_iotasks*ustride > 1)) {
+	  // This is a serial run with a bad configuration. Set up a single task.
+	  fprintf(stderr, "PIO_TP PIOc_Init_Intracomm reset stride and tasks.\n");
+	  iosys->num_iotasks = 1;
+	  ustride = 1;
+      }
+      if((iosys->num_iotasks < 1) || ((iosys->num_iotasks*ustride) > iosys->num_comptasks)){
+	  fprintf(stderr, "PIO_TP PIOc_Init_Intracomm error\n");
+	  fprintf(stderr, "num_iotasks=%d, ustride=%d, num_comptasks=%d\n", num_iotasks, ustride, iosys->num_comptasks);
+	  return PIO_EBADID;
+      }
+
+      /* Create an array that holds the ranks of the tasks to be used for IO. */
+      iosys->ioranks = (int *) calloc(sizeof(int), iosys->num_iotasks);
+      for(int i=0;i< iosys->num_iotasks; i++){
+	  iosys->ioranks[i] = (base + i*ustride) % iosys->num_comptasks;
+	  if(iosys->ioranks[i] == iosys->comp_rank)
+	      iosys->ioproc = true;
+      }
+      iosys->ioroot = iosys->ioranks[0];
 
-  if(iosys->comp_rank == iosys->ioranks[0])
-    iosys->iomaster = true;
+      /* Create an MPI info object. */
+      CheckMPIReturn(MPI_Info_create(&(iosys->info)),__FILE__,__LINE__);
+      iosys->info = MPI_INFO_NULL;
 
-  CheckMPIReturn(MPI_Comm_group(comp_comm, &(iosys->compgroup)),__FILE__,__LINE__);
+      if(iosys->comp_rank == iosys->ioranks[0])
+	  iosys->iomaster = MPI_ROOT;
+
+      /* Create a group for the computation tasks. */
+      CheckMPIReturn(MPI_Comm_group(iosys->comp_comm, &(iosys->compgroup)),__FILE__,__LINE__);
 			
-  CheckMPIReturn(MPI_Group_incl(iosys->compgroup, iosys->num_iotasks, iosys->ioranks,
-				&(iosys->iogroup)),__FILE__,__LINE__);
+      /* Create a group for the IO tasks. */
+      CheckMPIReturn(MPI_Group_incl(iosys->compgroup, iosys->num_iotasks, iosys->ioranks,
+				    &(iosys->iogroup)),__FILE__,__LINE__);
 
-  CheckMPIReturn(MPI_Comm_create(comp_comm, iosys->iogroup, &(iosys->io_comm)),__FILE__,__LINE__);
-  if(iosys->ioproc)
-    CheckMPIReturn(MPI_Comm_rank(iosys->io_comm, &(iosys->io_rank)),__FILE__,__LINE__);
-  else
-    iosys->io_rank = -1;
+      /* Create an MPI communicator for the IO tasks. */
+      CheckMPIReturn(MPI_Comm_create(iosys->comp_comm, iosys->iogroup, &(iosys->io_comm)),__FILE__,__LINE__);
 
-  iosys->union_rank = iosys->comp_rank;
+      /* For the tasks that are doing IO, get their rank. */
+      if(iosys->ioproc)
+	  CheckMPIReturn(MPI_Comm_rank(iosys->io_comm, &(iosys->io_rank)),__FILE__,__LINE__);
+      else
+	  iosys->io_rank = -1;
 
-  *iosysidp = pio_add_to_iosystem_list(iosys);
+      iosys->union_rank = iosys->comp_rank;
 
-  pio_get_env();
+      /* Add this iosys struct to the list in the PIO library. */
+      *iosysidp = pio_add_to_iosystem_list(iosys);
 
-  /* allocate buffer space for compute nodes */
-  compute_buffer_init(*iosys);
+      pio_get_env();
 
-  return PIO_NOERR;
+      /* allocate buffer space for compute nodes */
+      compute_buffer_init(*iosys);
+  }
+  
+  return ierr;
 }
 
 /**
@@ -469,8 +489,9 @@ int PIOc_set_hint(const int iosysid, char hint[], const char hintval[])
 
 }
 
-/** @ingroup PIO_finalize
- * @brief Clean up data structures and exit the pio library.
+/** @ingroup PIO_finalize 
+ * Clean up internal data structures, free MPI resources, and exit the
+ * pio library.
  *
  * @param iosysid: the io system ID provided by PIOc_Init_Intracomm().
  *
@@ -480,40 +501,53 @@ int PIOc_set_hint(const int iosysid, char hint[], const char hintval[])
 int PIOc_finalize(const int iosysid)
 {
   iosystem_desc_t *ios, *nios;
+  int msg;
+  int mpierr;
 
   ios = pio_get_iosystem_from_id(iosysid);
   if(ios == NULL)
-    return PIO_EBADID; 
-  /* FIXME: The memory for ioranks is allocated in C only for intracomms
-   * Remove this check once mem allocs for ioranks completely moves to the
-   * C code
-   */ 
-  if(ios->intercomm == MPI_COMM_NULL){
-    if(ios->ioranks != NULL){
-      free(ios->ioranks);
-    }
+    return PIO_EBADID;
+  
+  /* If asynch IO is in use, send the PIO_MSG_EXIT message from the
+   * comp master to the IO processes. */
+  if (ios->async_interface && !ios->comp_rank)
+  {
+    msg = PIO_MSG_EXIT;
+    mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+    CheckMPIReturn(mpierr, __FILE__, __LINE__);		      
   }
 
+  /* Free this memory that was allocated in init_intracomm. */
+  if (ios->ioranks)
+      free(ios->ioranks);
+
+  /* Free the buffer pool. */
   free_cn_buffer_pool(*ios);
 
   /* Free the MPI groups. */
-  MPI_Group_free(&(ios->compgroup));
-  MPI_Group_free(&(ios->iogroup));
+  if (ios->compgroup != MPI_GROUP_NULL)
+    MPI_Group_free(&ios->compgroup);
 
-  /* Free the MPI communicators. */
+  if (ios->iogroup != MPI_GROUP_NULL)
+    MPI_Group_free(&(ios->iogroup));
+
+  /* Free the MPI communicators. my_comm is just a copy (but not an
+   * MPI copy), so does not have to have an MPI_Comm_free() call. */
   if(ios->intercomm != MPI_COMM_NULL){
     MPI_Comm_free(&(ios->intercomm));
   }
   if(ios->io_comm != MPI_COMM_NULL){
     MPI_Comm_free(&(ios->io_comm));
   }
-  /* if(ios->comp_comm != MPI_COMM_NULL){ */
-  /*   MPI_Comm_free(&(ios->comp_comm)); */
-  /* } */
+  if(ios->comp_comm != MPI_COMM_NULL){
+    MPI_Comm_free(&(ios->comp_comm));
+  }
+  if(ios->union_comm != MPI_COMM_NULL){
+    MPI_Comm_free(&(ios->union_comm));
+  }
 
+  /* Delete the iosystem_desc_t data associated with this id. */
   return pio_delete_iosystem_from_list(iosysid);
-
-  
 }
 
 /**