From c854f3498c4d6675826b50e792b6b1a0cbeaf953 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 5 May 2016 07:18:06 -0400 Subject: [PATCH] changes to pioc.c to support async, also some temporary copies of code files for async development --- src/clib/pio_get_nc_async.c | 4988 ++++++++++++++++++++++++++++++++++ src/clib/pio_nc_async.c | 4534 +++++++++++++++++++++++++++++++ src/clib/pio_put_nc_async.c | 5073 +++++++++++++++++++++++++++++++++++ src/clib/pioc.c | 51 +- 4 files changed, 14626 insertions(+), 20 deletions(-) create mode 100644 src/clib/pio_get_nc_async.c create mode 100644 src/clib/pio_nc_async.c create mode 100644 src/clib/pio_put_nc_async.c diff --git a/src/clib/pio_get_nc_async.c b/src/clib/pio_get_nc_async.c new file mode 100644 index 000000000000..0da690aaf216 --- /dev/null +++ b/src/clib/pio_get_nc_async.c @@ -0,0 +1,4988 @@ +#include +#include + +int PIOc_get_var1_schar (int ncid, int varid, const PIO_Offset index[], signed char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_SCHAR; + ibuftype = MPI_CHAR; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_schar(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_schar(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_schar(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var1_schar_all(file->fh, varid, index, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vars_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned long long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_ULONGLONG; + ibuftype = MPI_UNSIGNED_LONG_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_ulonglong(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vars_ulonglong_all(file->fh, varid, start, count, stride, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_UCHAR; + ibuftype = MPI_UNSIGNED_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_uchar(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_uchar_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], signed char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_SCHAR; + ibuftype = MPI_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_schar(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_schar_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vars_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], short *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_SHORT; + ibuftype = MPI_SHORT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_short(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vars_short_all(file->fh, varid, start, count, stride, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var_double (int ncid, int varid, double *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_DOUBLE; + ibuftype = MPI_DOUBLE; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_double(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_double(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_double(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var_double_all(file->fh, varid, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vara_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], double *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_DOUBLE; + ibuftype = MPI_DOUBLE; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_double(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_double(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_double(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vara_double_all(file->fh, varid, start, count, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var_int (int ncid, int varid, int *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_INT; + ibuftype = MPI_INT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_int(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_int(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_int(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var_int_all(file->fh, varid, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var_ushort (int ncid, int varid, unsigned short *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_USHORT; + ibuftype = MPI_UNSIGNED_SHORT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_ushort(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_ushort(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_ushort(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var_ushort_all(file->fh, varid, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_TEXT; + ibuftype = MPI_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_text(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_text(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_text(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vara_text_all(file->fh, varid, start, count, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], int *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_INT; + ibuftype = MPI_INT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_int(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_int(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_int(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vara_int_all(file->fh, varid, start, count, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var1_float (int ncid, int varid, const PIO_Offset index[], float *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_FLOAT; + ibuftype = MPI_FLOAT; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_float(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_float(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_float(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var1_float_all(file->fh, varid, index, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var1_short (int ncid, int varid, const PIO_Offset index[], short *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_SHORT; + ibuftype = MPI_SHORT; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_short(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_short(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_short(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var1_short_all(file->fh, varid, index, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vars_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], int *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_INT; + ibuftype = MPI_INT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_int(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vars_int_all(file->fh, varid, start, count, stride, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var_text (int ncid, int varid, char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_TEXT; + ibuftype = MPI_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_text(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_text(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_text(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var_text_all(file->fh, varid, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], double *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_DOUBLE; + ibuftype = MPI_DOUBLE; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_double(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_double_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vars_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], signed char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_SCHAR; + ibuftype = MPI_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_schar(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vars_schar_all(file->fh, varid, start, count, stride, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vara_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned short *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_USHORT; + ibuftype = MPI_UNSIGNED_SHORT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_ushort(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vara_ushort_all(file->fh, varid, start, count, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var1_ushort (int ncid, int varid, const PIO_Offset index[], unsigned short *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_USHORT; + ibuftype = MPI_UNSIGNED_SHORT; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_ushort(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_ushort(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_ushort(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var1_ushort_all(file->fh, varid, index, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var_float (int ncid, int varid, float *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_FLOAT; + ibuftype = MPI_FLOAT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_float(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_float(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_float(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var_float_all(file->fh, varid, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vars_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_UCHAR; + ibuftype = MPI_UNSIGNED_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_uchar(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vars_uchar_all(file->fh, varid, start, count, stride, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var (int ncid, int varid, void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR; + ibufcnt = bufcount; + ibuftype = buftype; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var(file->fh, varid, buf, bufcount, buftype);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var_all(file->fh, varid, buf, bufcount, buftype);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var1_longlong (int ncid, int varid, const PIO_Offset index[], long long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_LONGLONG; + ibuftype = MPI_LONG_LONG; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_longlong(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_longlong(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_longlong(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var1_longlong_all(file->fh, varid, index, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vars_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned short *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_USHORT; + ibuftype = MPI_UNSIGNED_SHORT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_ushort(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vars_ushort_all(file->fh, varid, start, count, stride, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var_long (int ncid, int varid, long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_LONG; + ibuftype = MPI_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_long(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_long(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_long(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var_long_all(file->fh, varid, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var1_double (int ncid, int varid, const PIO_Offset index[], double *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_DOUBLE; + ibuftype = MPI_DOUBLE; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_double(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_double(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_double(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var1_double_all(file->fh, varid, index, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vara_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned int *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_UINT; + ibuftype = MPI_UNSIGNED; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_uint(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vara_uint_all(file->fh, varid, start, count, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vars_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], long long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_LONGLONG; + ibuftype = MPI_LONG_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_longlong(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vars_longlong_all(file->fh, varid, start, count, stride, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var_longlong (int ncid, int varid, long long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_LONGLONG; + ibuftype = MPI_LONG_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_longlong(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_longlong(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_longlong(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var_longlong_all(file->fh, varid, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vara_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], short *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_SHORT; + ibuftype = MPI_SHORT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_short(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_short(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_short(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vara_short_all(file->fh, varid, start, count, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vara_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_LONG; + ibuftype = MPI_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_long(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_long(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_long(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vara_long_all(file->fh, varid, start, count, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var1_int (int ncid, int varid, const PIO_Offset index[], int *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_INT; + ibuftype = MPI_INT; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_int(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_int(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_int(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var1_int_all(file->fh, varid, index, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var1_ulonglong (int ncid, int varid, const PIO_Offset index[], unsigned long long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_ULONGLONG; + ibuftype = MPI_UNSIGNED_LONG_LONG; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_ulonglong(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_ulonglong(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_ulonglong(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var1_ulonglong_all(file->fh, varid, index, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var_uchar (int ncid, int varid, unsigned char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_UCHAR; + ibuftype = MPI_UNSIGNED_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_uchar(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_uchar(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_uchar(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var_uchar_all(file->fh, varid, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vara_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_UCHAR; + ibuftype = MPI_UNSIGNED_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_uchar(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_uchar(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_uchar(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vara_uchar_all(file->fh, varid, start, count, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vars_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], float *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_FLOAT; + ibuftype = MPI_FLOAT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_float(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vars_float_all(file->fh, varid, start, count, stride, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vars_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_LONG; + ibuftype = MPI_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_long(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vars_long_all(file->fh, varid, start, count, stride, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var1 (int ncid, int varid, const PIO_Offset index[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1; + ibufcnt = bufcount; + ibuftype = buftype; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1(file->fh, varid, index, buf, bufcount, buftype);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var1_all(file->fh, varid, index, buf, bufcount, buftype);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var_uint (int ncid, int varid, unsigned int *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_UINT; + ibuftype = MPI_UNSIGNED; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_uint(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_uint(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_uint(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var_uint_all(file->fh, varid, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vara (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA; + ibufcnt = bufcount; + ibuftype = buftype; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara(file->fh, varid, start, count, buf, bufcount, buftype);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vara_all(file->fh, varid, start, count, buf, bufcount, buftype);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vara_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], signed char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_SCHAR; + ibuftype = MPI_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_schar(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vara_schar_all(file->fh, varid, start, count, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var1_uint (int ncid, int varid, const PIO_Offset index[], unsigned int *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_UINT; + ibuftype = MPI_UNSIGNED; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_uint(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_uint(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_uint(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var1_uint_all(file->fh, varid, index, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vars_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned int *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_UINT; + ibuftype = MPI_UNSIGNED; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_uint(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vars_uint_all(file->fh, varid, start, count, stride, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vara_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], float *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_FLOAT; + ibuftype = MPI_FLOAT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_float(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_float(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_float(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vara_float_all(file->fh, varid, start, count, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_TEXT; + ibuftype = MPI_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_text(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_text_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var1_text (int ncid, int varid, const PIO_Offset index[], char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_TEXT; + ibuftype = MPI_CHAR; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_text(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_text(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_text(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var1_text_all(file->fh, varid, index, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], int *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_INT; + ibuftype = MPI_INT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_int(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_int_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned int *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_UINT; + ibuftype = MPI_UNSIGNED; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_uint(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_uint_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM; + ibufcnt = bufcount; + ibuftype = buftype; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_all(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vars_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], double *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_DOUBLE; + ibuftype = MPI_DOUBLE; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_double(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vars_double_all(file->fh, varid, start, count, stride, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vara_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], long long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_LONGLONG; + ibuftype = MPI_LONG_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_longlong(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vara_longlong_all(file->fh, varid, start, count, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var_ulonglong (int ncid, int varid, unsigned long long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_ULONGLONG; + ibuftype = MPI_UNSIGNED_LONG_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_ulonglong(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_ulonglong(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_ulonglong(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var_ulonglong_all(file->fh, varid, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vara_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned long long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_ULONGLONG; + ibuftype = MPI_UNSIGNED_LONG_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_ulonglong(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vara_ulonglong_all(file->fh, varid, start, count, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var_short (int ncid, int varid, short *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_SHORT; + ibuftype = MPI_SHORT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_short(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_short(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_short(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var_short_all(file->fh, varid, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], float *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_FLOAT; + ibuftype = MPI_FLOAT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_float(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_float_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var1_long (int ncid, int varid, const PIO_Offset index[], long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_LONG; + ibuftype = MPI_LONG; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_long(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_long(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_long(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var1_long_all(file->fh, varid, index, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_LONG; + ibuftype = MPI_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_long(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_long_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned short *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_USHORT; + ibuftype = MPI_UNSIGNED_SHORT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_ushort(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_ushort_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], long long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_LONGLONG; + ibuftype = MPI_LONG_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_longlong(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_longlong_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vars_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_TEXT; + ibuftype = MPI_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_text(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vars_text_all(file->fh, varid, start, count, stride, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var1_uchar (int ncid, int varid, const PIO_Offset index[], unsigned char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_UCHAR; + ibuftype = MPI_UNSIGNED_CHAR; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_uchar(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_uchar(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_uchar(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var1_uchar_all(file->fh, varid, index, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vars (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS; + ibufcnt = bufcount; + ibuftype = buftype; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars(file->fh, varid, start, count, stride, buf, bufcount, buftype);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vars_all(file->fh, varid, start, count, stride, buf, bufcount, buftype);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], short *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_SHORT; + ibuftype = MPI_SHORT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_short(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_short_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned long long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_ULONGLONG; + ibuftype = MPI_UNSIGNED_LONG_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_ulonglong(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_ulonglong_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var_schar (int ncid, int varid, signed char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_SCHAR; + ibuftype = MPI_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_schar(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_schar(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_schar(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var_schar_all(file->fh, varid, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c new file mode 100644 index 000000000000..9e9f04a45db7 --- /dev/null +++ b/src/clib/pio_nc_async.c @@ -0,0 +1,4534 @@ +/** + * @file + * PIO interfaces to + * [NetCDF](http://www.unidata.ucar.edu/software/netcdf/docs/modules.html) + * support functions + + * This file provides an interface to the + * [NetCDF](http://www.unidata.ucar.edu/software/netcdf/docs/modules.html) + * support functions. Each subroutine calls the underlying netcdf or + * pnetcdf or netcdf4 functions from the appropriate subset of mpi + * tasks (io_comm). Each routine must be called collectively from + * union_comm. + * + * @author Jim Edwards (jedwards@ucar.edu), Ed Hartnett + * @date Feburary 2014, April 2016 + */ + +#include +#include + +/** Internal function to call nc_inq. Call this function only in I/O + * tasks. */ +int +pio_io_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, + int *unlimdimidp) +{ + file_desc_t *file; + char *errstr = NULL; + int ierr = PIO_NOERR; + + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d pio_io_inq ncid = %d\n", my_rank, ncid); + + if (!(file = pio_get_file_from_id(ncid))) + return PIO_EBADID; + + switch (file->iotype) + { +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp); + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + printf("%d pio_io_inq file->iosystem->io_rank = %d\n", my_rank, file->iosystem->io_rank); + if (!file->iosystem->io_rank) + { + /* Should not be necessary to do this - nc_inq should + * handle null pointers. This has been reported as a bug + * to netCDF developers. */ + int tmp_ndims, tmp_var, tmp_gatts, tmp_unl; + if (!ndimsp) + ndimsp = &tmp_ndims; + if (!nvarsp) + nvarsp = &tmp_var; + if (!ngattsp) + ngattsp = &tmp_gatts; + if (!unlimdimidp) + unlimdimidp = &tmp_unl; + ierr = nc_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp); + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + + if(ierr != PIO_NOERR) + { + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + + return ierr; +} + +/** + * @ingroup PIOc_inq + * The PIO-C interface for the NetCDF function nc_inq. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See + * PIOc_Set_File_Error_Handling + */ +int PIOc_inq (int ncid, int *ndimsp, int *nvarsp, int *ngattsp, + int *unlimdimidp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d PIOc_inq ncid = %d\n", my_rank, ncid); + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + if(ios->ioproc){ + ierr = pio_io_inq(file->fh, ndimsp, nvarsp, ngattsp, unlimdimidp); + printf("%d PIOc_inq pio_io_inq returned %d\n", my_rank, ierr); + ierr = check_netcdf(file, ierr, errstr,__LINE__); + } + if(ndimsp != NULL) + { + printf("%d PIOc_inq Bcast ios->ioroot = %d *ndimsp = %d\n", my_rank, ios->ioroot, *ndimsp); + mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm); + } + if(nvarsp != NULL) + mpierr = MPI_Bcast(nvarsp, 1, MPI_INT, ios->ioroot, ios->my_comm); + if(ngattsp != NULL) + mpierr = MPI_Bcast(ngattsp, 1, MPI_INT, ios->ioroot, ios->my_comm); + if(unlimdimidp != NULL) + mpierr = MPI_Bcast(unlimdimidp, 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_dimname + * The PIO-C interface for the NetCDF function nc_inq_dimname. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_dimname (int ncid, int dimid, char *name) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_DIMNAME; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_dimname(file->fh, dimid, name);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_dimname(file->fh, dimid, name);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_dimname(file->fh, dimid, name);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (name) + { + int slen; + if(ios->iomaster) + slen = (int) strlen(name) + 1; + mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast((void *)name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_put_att_short + * The PIO-C interface for the NetCDF function nc_put_att_short. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att_short (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const short *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_SHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_short(file->fh, varid, name, xtype, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_short(file->fh, varid, name, xtype, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_short(file->fh, varid, name, xtype, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_rename_dim + * The PIO-C interface for the NetCDF function nc_rename_dim. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_rename_dim (int ncid, int dimid, const char *name) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_RENAME_DIM; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_rename_dim(file->fh, dimid, name);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_rename_dim(file->fh, dimid, name);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_rename_dim(file->fh, dimid, name);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_get_att_double + * The PIO-C interface for the NetCDF function nc_get_att_double. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_double (int ncid, int varid, const char *name, double *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_DOUBLE; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_double(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_double(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_double(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_DOUBLE, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_set_fill + * The PIO-C interface for the NetCDF function nc_set_fill. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_set_fill (int ncid, int fillmode, int *old_modep) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_SET_FILL; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_set_fill(file->fh, fillmode, old_modep);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_set_fill(file->fh, fillmode, old_modep);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_set_fill(file->fh, fillmode, old_modep);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_def_var + * The PIO-C interface for the NetCDF function nc_def_var. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param varidp a pointer that will get the variable id + * @return PIO_NOERR for success, error code otherwise. See + * PIOc_Set_File_Error_Handling + */ +int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, + const int *dimidsp, int *varidp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + int namelen; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_DEF_VAR; + + /* If async is in use, and this is not an IO tasks, then bcast the + * function parameters to the intercomm, where it will be read by + * the IO root task. */ + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + namelen = strlen(name); + printf("bcasting namelen = %d name = %s\n", namelen, name); + if (!ios->compmaster) + ios->compmaster = MPI_PROC_NULL; + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)dimidsp, ndims, MPI_INT, ios->compmaster, ios->intercomm); + } + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);; + break; + case PIO_IOTYPE_NETCDF4C: + if(ios->io_rank==0){ + ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp); + if(ierr == PIO_NOERR){ + ierr = nc_def_var_deflate(file->fh, *varidp, 0,1,1); + } + } + break; +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + mpierr = MPI_Bcast(varidp , 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_put_att_double + * The PIO-C interface for the NetCDF function nc_put_att_double. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att_double (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const double *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_DOUBLE; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_double(file->fh, varid, name, xtype, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_double(file->fh, varid, name, xtype, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_double(file->fh, varid, name, xtype, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_dim + * The PIO-C interface for the NetCDF function nc_inq_dim. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param lenp a pointer that will get the number of values + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_dim (int ncid, int dimid, char *name, PIO_Offset *lenp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_DIM; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_dim(file->fh, dimid, name, lenp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(name) + { + int slen; + if(ios->iomaster) + slen = (int) strlen(name) + 1; + mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast((void *)name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); + } + if(lenp != NULL) + mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_get_att_uchar + * The PIO-C interface for the NetCDF function nc_get_att_uchar. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_UCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_uchar(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_uchar(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_uchar(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_CHAR, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_var_fill + * The PIO-C interface for the NetCDF function nc_inq_var_fill. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_var_fill (int ncid, int varid, int *no_fill, void *fill_value) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_VAR_FILL; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_value);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_value);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_var_fill(file->fh, varid, no_fill, fill_value);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + mpierr = MPI_Bcast(fill_value, 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_attid + * The PIO-C interface for the NetCDF function nc_inq_attid. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param idp a pointer that will get the id of the variable or attribute. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_ATTID; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_attid(file->fh, varid, name, idp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_attid(file->fh, varid, name, idp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_attid(file->fh, varid, name, idp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_vartype + * The PIO-C interface for the NetCDF function nc_inq_vartype. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param xtypep a pointer that will get the type of the attribute. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_vartype (int ncid, int varid, nc_type *xtypep) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_VARTYPE; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&file->fh,1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_vartype(file->fh, varid, xtypep);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_vartype(file->fh, varid, xtypep);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_vartype(file->fh, varid, xtypep);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (xtypep) + mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_put_att_schar + * The PIO-C interface for the NetCDF function nc_put_att_schar. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const signed char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_SCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_schar(file->fh, varid, name, xtype, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_schar(file->fh, varid, name, xtype, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_schar(file->fh, varid, name, xtype, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_vardimid + * The PIO-C interface for the NetCDF function nc_inq_vardimid. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_vardimid (int ncid, int varid, int *dimidsp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_VARDIMID; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_vardimid(file->fh, varid, dimidsp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_vardimid(file->fh, varid, dimidsp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_vardimid(file->fh, varid, dimidsp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (!ierr && dimidsp) + { + int ndims; + PIOc_inq_varndims(file->fh, varid, &ndims); + mpierr = MPI_Bcast(dimidsp , ndims, MPI_INT, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_get_att_ushort + * The PIO-C interface for the NetCDF function nc_get_att_ushort. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_USHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_ushort(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_ushort(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_ushort(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_SHORT, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_varid + * The PIO-C interface for the NetCDF function nc_inq_varid. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param varidp a pointer that will get the variable id + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_varid (int ncid, const char *name, int *varidp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_VARID; + + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d PIOc_inq_varid ncid = %d name = %s\n", my_rank, ncid, name); + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + printf("%d PIOc_inq_varid BCast ncid = %d\n", my_rank, ncid); + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + int namelen; + namelen = strlen(name); + printf("%d PIOc_inq_varid BCast namelen = %d\n", my_rank, namelen); + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + printf("%d PIOc_inq_varid BCast name = %s\n", my_rank, name); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + printf("%d PIOc_inq_varid BCast done\n", my_rank); + } + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_varid(file->fh, name, varidp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_varid(file->fh, name, varidp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_varid(file->fh, name, varidp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (varidp) + mpierr = MPI_Bcast(varidp, 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_attlen + * The PIO-C interface for the NetCDF function nc_inq_attlen. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param lenp a pointer that will get the number of values + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_attlen (int ncid, int varid, const char *name, PIO_Offset *lenp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_ATTLEN; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_attlen(file->fh, varid, name, (size_t *)lenp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_attlen(file->fh, varid, name, (size_t *)lenp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_attlen(file->fh, varid, name, lenp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_atttype + * The PIO-C interface for the NetCDF function nc_inq_atttype. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param xtypep a pointer that will get the type of the attribute. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_atttype (int ncid, int varid, const char *name, nc_type *xtypep) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_ATTTYPE; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_atttype(file->fh, varid, name, xtypep);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_atttype(file->fh, varid, name, xtypep);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_atttype(file->fh, varid, name, xtypep);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + mpierr = MPI_Bcast(xtypep , 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_rename_var + * The PIO-C interface for the NetCDF function nc_rename_var. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_rename_var (int ncid, int varid, const char *name) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_RENAME_VAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_rename_var(file->fh, varid, name);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_rename_var(file->fh, varid, name);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_rename_var(file->fh, varid, name);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_natts + * The PIO-C interface for the NetCDF function nc_inq_natts. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_natts (int ncid, int *ngattsp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_NATTS; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_natts(file->fh, ngattsp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_natts(file->fh, ngattsp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_natts(file->fh, ngattsp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (ngattsp) + mpierr = MPI_Bcast(ngattsp,1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_put_att_ulonglong + * The PIO-C interface for the NetCDF function nc_put_att_ulonglong. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att_ulonglong (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned long long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_ULONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_ulonglong(file->fh, varid, name, xtype, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_ulonglong(file->fh, varid, name, xtype, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_ulonglong(file->fh, varid, name, xtype, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_var + * The PIO-C interface for the NetCDF function nc_inq_var. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param xtypep a pointer that will get the type of the attribute. + * @param nattsp a pointer that will get the number of attributes + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_var (int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, + int *dimidsp, int *nattsp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_VAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (xtypep) + mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm); + if (ndimsp) + { + mpierr = MPI_Bcast(ndimsp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm); + file->varlist[varid].ndims = (*ndimsp); + } + if (nattsp) + mpierr = MPI_Bcast(nattsp, 1, MPI_INT, ios->ioroot, ios->my_comm); + if (name) + { + int slen; + if(ios->iomaster) + slen = (int) strlen(name) + 1; + mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast((void *)name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); + } + if (dimidsp) + { + int ndims; + PIOc_inq_varndims(file->fh, varid, &ndims); + mpierr = MPI_Bcast(dimidsp, ndims, MPI_INT, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_rename_att + * The PIO-C interface for the NetCDF function nc_rename_att. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_RENAME_ATT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_rename_att(file->fh, varid, name, newname);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_rename_att(file->fh, varid, name, newname);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_rename_att(file->fh, varid, name, newname);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_put_att_ushort + * The PIO-C interface for the NetCDF function nc_put_att_ushort. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att_ushort (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned short *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_USHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_ushort(file->fh, varid, name, xtype, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_ushort(file->fh, varid, name, xtype, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_ushort(file->fh, varid, name, xtype, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_dimid + * The PIO-C interface for the NetCDF function nc_inq_dimid. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param idp a pointer that will get the id of the variable or attribute. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_dimid (int ncid, const char *name, int *idp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_DIMID; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + int namelen; + if (ios->iomaster) + namelen = strlen(name); + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_dimid(file->fh, name, idp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_dimid(file->fh, name, idp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_dimid(file->fh, name, idp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (idp) + mpierr = MPI_Bcast(idp, 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_put_att_text + * The PIO-C interface for the NetCDF function nc_put_att_text. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att_text (int ncid, int varid, const char *name, PIO_Offset len, const char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_TEXT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_text(file->fh, varid, name, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_text(file->fh, varid, name, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_text(file->fh, varid, name, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_get_att_uint + * The PIO-C interface for the NetCDF function nc_get_att_uint. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_uint (int ncid, int varid, const char *name, unsigned int *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_UINT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_uint(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_uint(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_uint(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_format + * The PIO-C interface for the NetCDF function nc_inq_format. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param formatp a pointer that will get the file format + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_format (int ncid, int *formatp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_FORMAT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_format(file->fh, formatp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_format(file->fh, formatp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_format(file->fh, formatp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + mpierr = MPI_Bcast(formatp , 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_get_att_long + * The PIO-C interface for the NetCDF function nc_get_att_long. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_long (int ncid, int varid, const char *name, long *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_LONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_long(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_long(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_long(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_LONG, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_attname + * The PIO-C interface for the NetCDF function nc_inq_attname. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param attnum the attribute ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_attname (int ncid, int varid, int attnum, char *name) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_ATTNAME; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_attname(file->fh, varid, attnum, name);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_attname(file->fh, varid, attnum, name);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_attname(file->fh, varid, attnum, name);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(name != NULL){ + int slen; + if(ios->iomaster) + slen = (int) strlen(name) + 1; + mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast((void *)name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_att + * The PIO-C interface for the NetCDF function nc_inq_att. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param xtypep a pointer that will get the type of the attribute. + * @param lenp a pointer that will get the number of values + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, PIO_Offset *lenp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_ATT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_att(file->fh, varid, name, xtypep, lenp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(xtypep != NULL) mpierr = MPI_Bcast(xtypep , 1, MPI_INT, ios->ioroot, ios->my_comm); + if(lenp != NULL) mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_put_att_long + * The PIO-C interface for the NetCDF function nc_put_att_long. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att_long (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_LONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_long(file->fh, varid, name, xtype, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_long(file->fh, varid, name, xtype, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_long(file->fh, varid, name, xtype, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_unlimdim + * The PIO-C interface for the NetCDF function nc_inq_unlimdim. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_unlimdim (int ncid, int *unlimdimidp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_UNLIMDIM; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_unlimdim(file->fh, unlimdimidp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_unlimdim(file->fh, unlimdimidp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_unlimdim(file->fh, unlimdimidp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (unlimdimidp) + mpierr = MPI_Bcast(unlimdimidp,1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_get_att_float + * The PIO-C interface for the NetCDF function nc_get_att_float. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_float (int ncid, int varid, const char *name, float *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_FLOAT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_float(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_float(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_float(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_FLOAT, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_ndims + * The PIO-C interface for the NetCDF function nc_inq_ndims. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_ndims (int ncid, int *ndimsp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_NDIMS; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_ndims(file->fh, ndimsp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_ndims(file->fh, ndimsp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_ndims(file->fh, ndimsp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (ndimsp) + mpierr = MPI_Bcast(ndimsp , 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_put_att_int + * The PIO-C interface for the NetCDF function nc_put_att_int. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att_int (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const int *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_INT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_int(file->fh, varid, name, xtype, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_int(file->fh, varid, name, xtype, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_int(file->fh, varid, name, xtype, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_nvars + * The PIO-C interface for the NetCDF function nc_inq_nvars. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_nvars (int ncid, int *nvarsp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_NVARS; + + if(ios->async_interface && ! ios->ioproc){ + if(!ios->comp_rank) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_nvars(file->fh, nvarsp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_nvars(file->fh, nvarsp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_nvars(file->fh, nvarsp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (nvarsp) + mpierr = MPI_Bcast(nvarsp,1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_enddef + * The PIO-C interface for the NetCDF function nc_enddef. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_enddef (int ncid) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_ENDDEF; + + if(ios->async_interface && ! ios->ioproc){ + if(!ios->comp_rank) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + printf("PIOc_enddef file->fh = %d\n", file->fh); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_enddef(file->fh);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_enddef(file->fh);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_enddef(file->fh);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_put_att_uchar + * The PIO-C interface for the NetCDF function nc_put_att_uchar. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att_uchar (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_UCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_uchar(file->fh, varid, name, xtype, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_uchar(file->fh, varid, name, xtype, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_uchar(file->fh, varid, name, xtype, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_put_att_longlong + * The PIO-C interface for the NetCDF function nc_put_att_longlong. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att_longlong (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const long long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_LONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_longlong(file->fh, varid, name, xtype, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_longlong(file->fh, varid, name, xtype, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_longlong(file->fh, varid, name, xtype, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_varnatts + * The PIO-C interface for the NetCDF function nc_inq_varnatts. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param nattsp a pointer that will get the number of attributes + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_varnatts (int ncid, int varid, int *nattsp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_VARNATTS; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_varnatts(file->fh, varid, nattsp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_varnatts(file->fh, varid, nattsp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_varnatts(file->fh, varid, nattsp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (nattsp) + mpierr = MPI_Bcast(nattsp, 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_get_att_ubyte + * The PIO-C interface for the NetCDF function nc_get_att_ubyte. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_ubyte (int ncid, int varid, const char *name, unsigned char *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_UBYTE; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_ubyte(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_ubyte(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_ubyte(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_BYTE, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_get_att_text + * The PIO-C interface for the NetCDF function nc_get_att_text. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_text (int ncid, int varid, const char *name, char *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_TEXT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_text(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_text(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_text(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_CHAR, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_del_att + * The PIO-C interface for the NetCDF function nc_del_att. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_del_att (int ncid, int varid, const char *name) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_DEL_ATT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_del_att(file->fh, varid, name);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_del_att(file->fh, varid, name);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_del_att(file->fh, varid, name);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_dimlen + * The PIO-C interface for the NetCDF function nc_inq_dimlen. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param lenp a pointer that will get the number of values + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_dimlen (int ncid, int dimid, PIO_Offset *lenp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_DIMLEN; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_dimlen(file->fh, dimid, (size_t *)lenp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_dimlen(file->fh, dimid, (size_t *)lenp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_dimlen(file->fh, dimid, lenp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (lenp) + mpierr = MPI_Bcast(lenp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_get_att_schar + * The PIO-C interface for the NetCDF function nc_get_att_schar. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_schar (int ncid, int varid, const char *name, signed char *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_SCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_schar(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_schar(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_schar(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_CHAR, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_get_att_ulonglong + * The PIO-C interface for the NetCDF function nc_get_att_ulonglong. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_ulonglong (int ncid, int varid, const char *name, unsigned long long *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_ULONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_ulonglong(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_ulonglong(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_ulonglong(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_LONG_LONG, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_varndims + * The PIO-C interface for the NetCDF function nc_inq_varndims. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_varndims (int ncid, int varid, int *ndimsp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_VARNDIMS; + if(file->varlist[varid].ndims > 0){ + (*ndimsp) = file->varlist[varid].ndims; + return PIO_NOERR; + } + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&file->fh,1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid,1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_varndims(file->fh, varid, ndimsp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_varndims(file->fh, varid, ndimsp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_varndims(file->fh, varid, ndimsp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (ndimsp) + { + mpierr = MPI_Bcast(ndimsp,1, MPI_INT, ios->ioroot, ios->my_comm); + file->varlist[varid].ndims = (*ndimsp); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_varname + * The PIO-C interface for the NetCDF function nc_inq_varname. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_varname (int ncid, int varid, char *name) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_VARNAME; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&file->fh,1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_varname(file->fh, varid, name);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_varname(file->fh, varid, name);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_varname(file->fh, varid, name);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(name != NULL){ + int slen; + if(ios->iomaster) + slen = (int) strlen(name) + 1; + mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast((void *)name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_def_dim + * The PIO-C interface for the NetCDF function nc_def_dim. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param idp a pointer that will get the id of the variable or attribute. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + int namelen; + + errstr = NULL; + ierr = PIO_NOERR; + + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d PIOc_def_dim ncid = %d name = %s len = %d\n", my_rank, + ncid, name, len); + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_DEF_DIM; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + namelen = strlen(name); + printf("bcasting namelen = %d name = %s len = %d\n", namelen, name, len); + if (!ios->compmaster) + ios->compmaster = MPI_PROC_NULL; + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_def_dim(file->fh, name, (size_t)len, idp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_def_dim(file->fh, name, (size_t)len, idp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_def_dim(file->fh, name, len, idp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_put_att_uint + * The PIO-C interface for the NetCDF function nc_put_att_uint. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att_uint (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned int *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_UINT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_uint(file->fh, varid, name, xtype, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_uint(file->fh, varid, name, xtype, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_uint(file->fh, varid, name, xtype, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_get_att_short + * The PIO-C interface for the NetCDF function nc_get_att_short. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_short (int ncid, int varid, const char *name, short *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_SHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_short(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_short(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_short(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_SHORT, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_redef + * The PIO-C interface for the NetCDF function nc_redef. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_redef (int ncid) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_REDEF; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_redef(file->fh);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_redef(file->fh);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_redef(file->fh);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_put_att_ubyte + * The PIO-C interface for the NetCDF function nc_put_att_ubyte. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att_ubyte (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_UBYTE; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_ubyte(file->fh, varid, name, xtype, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_ubyte(file->fh, varid, name, xtype, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_ubyte(file->fh, varid, name, xtype, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_get_att_int + * The PIO-C interface for the NetCDF function nc_get_att_int. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_int (int ncid, int varid, const char *name, int *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_INT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_int(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_int(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_int(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_INT, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_get_att_longlong + * The PIO-C interface for the NetCDF function nc_get_att_longlong. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_longlong (int ncid, int varid, const char *name, long long *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_LONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_longlong(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_longlong(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_longlong(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_LONG_LONG, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_put_att_float + * The PIO-C interface for the NetCDF function nc_put_att_float. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att_float (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const float *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_FLOAT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_float(file->fh, varid, name, xtype, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_float(file->fh, varid, name, xtype, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_float(file->fh, varid, name, xtype, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c new file mode 100644 index 000000000000..c0b621eb903d --- /dev/null +++ b/src/clib/pio_put_nc_async.c @@ -0,0 +1,5073 @@ +#include +#include + +/// +/// PIO interface to nc_put_vars_uchar +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vars_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS_UCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars_uchar(file->fh, varid, start, count, stride, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vars_ushort +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vars_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned short *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS_USHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars_ushort(file->fh, varid, start, count, stride, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vars_ulonglong +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vars_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned long long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS_ULONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars_ulonglong(file->fh, varid, start, count, stride, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vars_uint +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vars_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned int *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS_UINT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars_uint(file->fh, varid, start, count, stride, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_uchar +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_UCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_uchar(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var_ushort +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var_ushort (int ncid, int varid, const unsigned short *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR_USHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var_ushort(file->fh, varid, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var_ushort(file->fh, varid, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var_ushort(file->fh, varid, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var1_longlong +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1_longlong (int ncid, int varid, const PIO_Offset index[], const long long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_LONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_longlong(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_longlong(file->fh, varid, (size_t *) index, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_longlong(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vara_uchar +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA_UCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara_uchar(file->fh, varid, (size_t *) start, (size_t *) count, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara_uchar(file->fh, varid, (size_t *) start, (size_t *) count, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara_uchar(file->fh, varid, start, count, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_short +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const short *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_SHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_short(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var1_long +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1_long (int ncid, int varid, const PIO_Offset index[], const long *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_LONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_long(file->fh, varid, (size_t *) index, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_long(file->fh, varid, (size_t *) index, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_long(file->fh, varid, index, ip, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vars_long +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vars_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS_LONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars_long(file->fh, varid, start, count, stride, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var_short +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var_short (int ncid, int varid, const short *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR_SHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var_short(file->fh, varid, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var_short(file->fh, varid, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var_short(file->fh, varid, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vara_int +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const int *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA_INT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara_int(file->fh, varid, (size_t *) start, (size_t *) count, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara_int(file->fh, varid, (size_t *) start, (size_t *) count, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara_int(file->fh, varid, start, count, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var1_ushort +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1_ushort (int ncid, int varid, const PIO_Offset index[], const unsigned short *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_USHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_ushort(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_ushort(file->fh, varid, (size_t *) index, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_ushort(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vara_text +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA_TEXT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara_text(file->fh, varid, (size_t *) start, (size_t *) count, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara_text(file->fh, varid, (size_t *) start, (size_t *) count, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara_text(file->fh, varid, start, count, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_text +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_TEXT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_text(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_ushort +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned short *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_USHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_ushort(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var_ulonglong +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var_ulonglong (int ncid, int varid, const unsigned long long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR_ULONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var_ulonglong(file->fh, varid, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var_ulonglong(file->fh, varid, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var_ulonglong(file->fh, varid, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var_int +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var_int (int ncid, int varid, const int *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR_INT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var_int(file->fh, varid, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var_int(file->fh, varid, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var_int(file->fh, varid, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var_longlong +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var_longlong (int ncid, int varid, const long long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR_LONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var_longlong(file->fh, varid, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var_longlong(file->fh, varid, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var_longlong(file->fh, varid, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var_schar +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var_schar (int ncid, int varid, const signed char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR_SCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var_schar(file->fh, varid, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var_schar(file->fh, varid, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var_schar(file->fh, varid, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var_uint +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var_uint (int ncid, int varid, const unsigned int *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR_UINT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var_uint(file->fh, varid, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var_uint(file->fh, varid, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var_uint(file->fh, varid, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var (int ncid, int varid, const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var(file->fh, varid, buf, bufcount, buftype, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vara_ushort +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned short *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA_USHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara_ushort(file->fh, varid, start, count, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vars_short +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vars_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const short *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS_SHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars_short(file->fh, varid, start, count, stride, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vara_uint +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned int *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA_UINT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara_uint(file->fh, varid, start, count, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vara_schar +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const signed char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA_SCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara_schar(file->fh, varid, start, count, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_ulonglong +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned long long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_ULONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_ulonglong(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var1_uchar +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1_uchar (int ncid, int varid, const PIO_Offset index[], const unsigned char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_UCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_uchar(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_uchar(file->fh, varid, (size_t *) index, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_uchar(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_int +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const int *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_INT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_int(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vars_schar +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vars_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const signed char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS_SCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars_schar(file->fh, varid, start, count, stride, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var1 +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1 (int ncid, int varid, const PIO_Offset index[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1(file->fh, varid, index, buf, bufcount, buftype, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vara_float +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const float *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA_FLOAT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara_float(file->fh, varid, (size_t *) start, (size_t *) count, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara_float(file->fh, varid, (size_t *) start, (size_t *) count, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara_float(file->fh, varid, start, count, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var1_float +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1_float (int ncid, int varid, const PIO_Offset index[], const float *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_FLOAT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_float(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_float(file->fh, varid, (size_t *) index, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_float(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_float +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const float *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_FLOAT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_float(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var1_text +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1_text (int ncid, int varid, const PIO_Offset index[], const char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_TEXT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_text(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_text(file->fh, varid, (size_t *) index, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_text(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vars_text +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vars_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS_TEXT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars_text(file->fh, varid, start, count, stride, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_long +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_LONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_long(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vars_double +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vars_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const double *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS_DOUBLE; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars_double(file->fh, varid, start, count, stride, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vara_longlong +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const long long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA_LONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara_longlong(file->fh, varid, start, count, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var_double +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var_double (int ncid, int varid, const double *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR_DOUBLE; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var_double(file->fh, varid, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var_double(file->fh, varid, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var_double(file->fh, varid, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var_float +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var_float (int ncid, int varid, const float *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR_FLOAT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var_float(file->fh, varid, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var_float(file->fh, varid, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var_float(file->fh, varid, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var1_ulonglong +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1_ulonglong (int ncid, int varid, const PIO_Offset index[], const unsigned long long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_ULONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_ulonglong(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_ulonglong(file->fh, varid, (size_t *) index, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_ulonglong(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_uint +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned int *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_UINT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_uint(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var1_uint +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1_uint (int ncid, int varid, const PIO_Offset index[], const unsigned int *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_UINT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_uint(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_uint(file->fh, varid, (size_t *) index, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_uint(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var1_int +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1_int (int ncid, int varid, const PIO_Offset index[], const int *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_INT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_int(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_int(file->fh, varid, (size_t *) index, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_int(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vars_float +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vars_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const float *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS_FLOAT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars_float(file->fh, varid, start, count, stride, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vara_short +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const short *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA_SHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara_short(file->fh, varid, (size_t *) start, (size_t *) count, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara_short(file->fh, varid, (size_t *) start, (size_t *) count, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara_short(file->fh, varid, start, count, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var1_schar +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1_schar (int ncid, int varid, const PIO_Offset index[], const signed char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_SCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_schar(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_schar(file->fh, varid, (size_t *) index, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_schar(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vara_ulonglong +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned long long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA_ULONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara_ulonglong(file->fh, varid, start, count, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_double +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const double *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_DOUBLE; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_double(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vara +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara(file->fh, varid, start, count, buf, bufcount, buftype, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vara_long +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA_LONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara_long(file->fh, varid, (size_t *) start, (size_t *) count, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara_long(file->fh, varid, (size_t *) start, (size_t *) count, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara_long(file->fh, varid, start, count, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var1_double +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1_double (int ncid, int varid, const PIO_Offset index[], const double *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_DOUBLE; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_double(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_double(file->fh, varid, (size_t *) index, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_double(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_schar +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const signed char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_SCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_schar(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var_text +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var_text (int ncid, int varid, const char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR_TEXT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var_text(file->fh, varid, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var_text(file->fh, varid, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var_text(file->fh, varid, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vars_int +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vars_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const int *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS_INT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars_int(file->fh, varid, start, count, stride, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var1_short +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1_short (int ncid, int varid, const PIO_Offset index[], const short *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_SHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_short(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_short(file->fh, varid, (size_t *) index, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_short(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vars_longlong +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vars_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const long long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS_LONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars_longlong(file->fh, varid, start, count, stride, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vara_double +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const double *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA_DOUBLE; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara_double(file->fh, varid, (size_t *) start, (size_t *) count, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara_double(file->fh, varid, (size_t *) start, (size_t *) count, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara_double(file->fh, varid, start, count, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vars +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vars (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars(file->fh, varid, start, count, stride, buf, bufcount, buftype, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var_uchar +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var_uchar (int ncid, int varid, const unsigned char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR_UCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var_uchar(file->fh, varid, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var_uchar(file->fh, varid, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var_uchar(file->fh, varid, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var_long +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var_long (int ncid, int varid, const long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR_LONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var_long(file->fh, varid, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var_long(file->fh, varid, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var_long(file->fh, varid, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_longlong +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const long long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_LONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_longlong(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + diff --git a/src/clib/pioc.c b/src/clib/pioc.c index e81f0a5e2436..22d6ea3307f1 100644 --- a/src/clib/pioc.c +++ b/src/clib/pioc.c @@ -386,8 +386,8 @@ int PIOc_Init_Intracomm(const MPI_Comm comp_comm, iosys->intercomm = MPI_COMM_NULL; iosys->error_handler = PIO_INTERNAL_ERROR; iosys->async_interface= false; - iosys->compmaster = false; - iosys->iomaster = false; + iosys->compmaster = 0; + iosys->iomaster = 0; iosys->ioproc = false; iosys->default_rearranger = rearr; iosys->num_iotasks = num_iotasks; @@ -398,7 +398,7 @@ int PIOc_Init_Intracomm(const MPI_Comm comp_comm, CheckMPIReturn(MPI_Comm_rank(iosys->comp_comm, &(iosys->comp_rank)),__FILE__,__LINE__); CheckMPIReturn(MPI_Comm_size(iosys->comp_comm, &(iosys->num_comptasks)),__FILE__,__LINE__); if(iosys->comp_rank==0) - iosys->compmaster = true; + iosys->compmaster = MPI_ROOT; /* Ensure that settings for number of computation tasks, number * of IO tasks, and the stride are reasonable. */ @@ -428,7 +428,7 @@ int PIOc_Init_Intracomm(const MPI_Comm comp_comm, iosys->info = MPI_INFO_NULL; if(iosys->comp_rank == iosys->ioranks[0]) - iosys->iomaster = true; + iosys->iomaster = MPI_ROOT; /* Create a group for the computation tasks. */ CheckMPIReturn(MPI_Comm_group(iosys->comp_comm, &(iosys->compgroup)),__FILE__,__LINE__); @@ -489,8 +489,9 @@ int PIOc_set_hint(const int iosysid, char hint[], const char hintval[]) } -/** @ingroup PIO_finalize - * @brief Clean up data structures and exit the pio library. +/** @ingroup PIO_finalize + * Clean up internal data structures, free MPI resources, and exit the + * pio library. * * @param iosysid: the io system ID provided by PIOc_Init_Intracomm(). * @@ -500,27 +501,38 @@ int PIOc_set_hint(const int iosysid, char hint[], const char hintval[]) int PIOc_finalize(const int iosysid) { iosystem_desc_t *ios, *nios; + int msg; + int mpierr; ios = pio_get_iosystem_from_id(iosysid); if(ios == NULL) - return PIO_EBADID; - /* FIXME: The memory for ioranks is allocated in C only for intracomms - * Remove this check once mem allocs for ioranks completely moves to the - * C code - */ - if(ios->intercomm == MPI_COMM_NULL){ - if(ios->ioranks != NULL){ - free(ios->ioranks); - } + return PIO_EBADID; + + /* If asynch IO is in use, send the PIO_MSG_EXIT message from the + * comp master to the IO processes. */ + if (ios->async_interface && !ios->comp_rank) + { + msg = PIO_MSG_EXIT; + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); } + /* Free this memory that was allocated in init_intracomm. */ + if (ios->ioranks) + free(ios->ioranks); + + /* Free the buffer pool. */ free_cn_buffer_pool(*ios); /* Free the MPI groups. */ - MPI_Group_free(&(ios->compgroup)); - MPI_Group_free(&(ios->iogroup)); + if (ios->compgroup != MPI_GROUP_NULL) + MPI_Group_free(&ios->compgroup); + + if (ios->iogroup != MPI_GROUP_NULL) + MPI_Group_free(&(ios->iogroup)); - /* Free the MPI communicators. */ + /* Free the MPI communicators. my_comm is just a copy (but not an + * MPI copy), so does not have to have an MPI_Comm_free() call. */ if(ios->intercomm != MPI_COMM_NULL){ MPI_Comm_free(&(ios->intercomm)); } @@ -534,9 +546,8 @@ int PIOc_finalize(const int iosysid) MPI_Comm_free(&(ios->union_comm)); } + /* Delete the iosystem_desc_t data associated with this id. */ return pio_delete_iosystem_from_list(iosysid); - - } /**