Skip to content

Commit

Permalink
Squashed 'src/externals/pio2/' changes from c84f74c..0f241db
Browse files Browse the repository at this point in the history 
0f241db response to comments
1007a7a cannot predetermin ndims here
99ef07d Merge pull request ESMCI#1241 from NCAR/free_new_allocs
29ed162 free recently allocated vars
fbc3584 Merge pull request ESMCI#1239 from NCAR/dontuse_nc_max
63dee3d Merge pull request ESMCI#1240 from NCAR/limitto2GiB
64f2492 limit to 2GiB due to romio bug
29aee05 dont use NC_MAX values
d831ad3 Merge pull request ESMCI#1231 from mgduda/mpi_type_fix
e996bdb Merge pull request ESMCI#1222 from NCAR/ejh_autoconf_logging
426af22 Partial fix for incorrect type of 'mpi_type' in pioc_support.c
7eb724f added enable-logging option to autotools build

git-subtree-dir: src/externals/pio2
git-subtree-split: 0f241db
  • Loading branch information
@jedwards4b
jedwards4b committed Dec 12, 2017
1 parent f095332 commit a920f03
Show file tree
Hide file tree
Showing 9 changed files with 90 additions and 52 deletions.
13 changes: 11 additions & 2 deletions configure.ac
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,17 @@ AC_ARG_WITH([piobuffersize],
AC_MSG_RESULT([$PIO_BUFFER_SIZE])
AC_DEFINE_UNQUOTED([PIO_BUFFER_SIZE], [$PIO_BUFFER_SIZE], [buffer size for darray data.])

# Need to allow user to set this.
AC_DEFINE([PIO_ENABLE_LOGGING], [1], [log messages from library])
# Does the user want to enable logging?
AC_MSG_CHECKING([whether debug logging is enabled])
AC_ARG_ENABLE([logging],
[AS_HELP_STRING([--enable-logging],
[enable debug logging capability (will negatively impact performance). \
This debugging feature is probably only of interest to PIO developers.])])
test "x$enable_logging" = xyes || enable_logging=no
AC_MSG_RESULT([$enable_logging])
if test "x$enable_logging" = xyes; then
AC_DEFINE([PIO_ENABLE_LOGGING], 1, [If true, turn on logging.])
fi

# NetCDF (at least classic) is required for PIO to build.
AC_DEFINE([_NETCDF], [1], [netCDF classic library available])
Expand Down
6 changes: 3 additions & 3 deletions ctest/runcdash-nwscla-intel.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,14 @@ module unload netcdf
module swap intel intel/17.0.1
module load cmake/3.7.2
module load netcdf-mpi/4.4.1.1
module load pnetcdf/1.8.0
module switch mpt mpt/2.15
module load pnetcdf/1.8.1
module switch mpt mpt/2.16
echo "MODULE LIST..."
module list

export CC=mpicc
export FC=mpif90

export MPI_TYPE_DEPTH=24
export PIO_DASHBOARD_ROOT=/glade/scratch/jedwards/dashboard
export PIO_COMPILER_ID=Intel-`$CC --version | head -n 1 | cut -d' ' -f3`

Expand Down
4 changes: 2 additions & 2 deletions doc/source/mach_walkthrough.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -454,10 +454,10 @@ which causes the build commands to be made visible.
autoreconf -i && LD_LIBRARY_PATH=/usr/local/netcdf-4.4.1_mpich-3.2/lib CC=mpicc CFLAGS='-g' CPPFLAGS='-I/usr/local/netcdf-4.4.1_mpich-3.2/include/ -I/usr/local/pnetcdf-1.8.1_mpich-3.2/include' LDFLAGS='-L/usr/local/netcdf-4.4.1_mpich-3.2/lib -L/usr/local/pnetcdf-1.8.1_mpich-3.2/lib' ./configure && make check
</pre>

<p>To build with the address sanitizer for memory checking (debugging builds only!):
<p>To build with debug logging and the address sanitizer for memory checking (debugging builds only!):

<pre>
autoreconf -i && LD_LIBRARY_PATH=/usr/local/netcdf-4.4.1_mpich-3.2/lib CC=mpicc CFLAGS='-g -fsanitize=address -fno-omit-frame-pointer' CPPFLAGS='-I/usr/local/netcdf-4.4.1_mpich-3.2/include/ -I/usr/local/pnetcdf-1.8.1_mpich-3.2/include' LDFLAGS='-L/usr/local/netcdf-4.4.1_mpich-3.2/lib -L/usr/local/pnetcdf-1.8.1_mpich-3.2/lib' ./configure && make check
autoreconf -i && LD_LIBRARY_PATH=/usr/local/netcdf-4.4.1_mpich-3.2/lib CC=mpicc CFLAGS='-g -fsanitize=address -fno-omit-frame-pointer' CPPFLAGS='-I/usr/local/netcdf-4.4.1_mpich-3.2/include/ -I/usr/local/pnetcdf-1.8.1_mpich-3.2/include' LDFLAGS='-L/usr/local/netcdf-4.4.1_mpich-3.2/lib -L/usr/local/pnetcdf-1.8.1_mpich-3.2/lib' ./configure --enable-logging && make check
</pre>

<li>Building and Running Performance Tests
Expand Down
12 changes: 9 additions & 3 deletions src/clib/pio_darray.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -485,6 +485,7 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
#endif
int mpierr = MPI_SUCCESS; /* Return code from MPI functions. */
int ierr = PIO_NOERR; /* Return code. */
size_t io_data_size; /* potential size of data on io task */

LOG((1, "PIOc_write_darray ncid = %d varid = %d ioid = %d arraylen = %d",
ncid, varid, ioid, arraylen));
Expand Down Expand Up @@ -532,7 +533,7 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
if (fillvalue)
if (memcmp(fillvalue, vdesc->fillvalue, vdesc->pio_type_size))
return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);

/* Move to end of list or the entry that matches this ioid. */
for (wmb = &file->buffer; wmb->next; wmb = wmb->next)
if (wmb->ioid == ioid && wmb->recordvar == vdesc->rec_var)
Expand Down Expand Up @@ -560,13 +561,12 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
}
LOG((2, "wmb->num_arrays = %d arraylen = %d vdesc->mpi_type_size = %d\n",
wmb->num_arrays, arraylen, vdesc->mpi_type_size));

#if PIO_USE_MALLOC
/* Try realloc first and call flush if realloc fails. */
if (arraylen > 0)
{
size_t data_size = (1 + wmb->num_arrays) * arraylen * vdesc->mpi_type_size;

if ((realloc_data = realloc(wmb->data, data_size)))
{
needsflush = 0;
Expand All @@ -588,6 +588,12 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
if (needsflush == 0)
needsflush = (maxfree <= 1.1 * (1 + wmb->num_arrays) * arraylen * vdesc->mpi_type_size);
#endif
/* the limit of data_size < INT_MAX is due to a bug in ROMIO which limits
the size of contiguous data to INT_MAX, a fix has been proposed in
https://github.com/pmodels/mpich/pull/2888 */
io_data_size = (1 + wmb->num_arrays) * iodesc->maxiobuflen * vdesc->mpi_type_size;
if(io_data_size > INT_MAX)
needsflush = 2;

/* Tell all tasks on the computation communicator whether we need
* to flush data. */
Expand Down
45 changes: 27 additions & 18 deletions src/clib/pio_getput_int.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -797,8 +797,6 @@ int PIOc_get_var_tc(int ncid, int varid, nc_type xtype, void *buf)
PIO_Offset *startp = NULL; /* Pointer to start array. */
PIO_Offset *countp = NULL; /* Pointer to count array. */
int ndims; /* The number of dimensions in the variable. */
PIO_Offset my_start[PIO_MAX_DIMS];
PIO_Offset dimlen[PIO_MAX_DIMS];
int ierr; /* Return code from function calls. */

LOG((1, "PIOc_get_var_tc ncid = %d varid = %d xtype = %d", ncid, varid,
Expand All @@ -821,26 +819,33 @@ int PIOc_get_var_tc(int ncid, int varid, nc_type xtype, void *buf)
int dimids[ndims];
if ((ierr = PIOc_inq_vardimid(ncid, varid, dimids)))
return pio_err(ios, file, ierr, __FILE__, __LINE__);
if (!(startp = malloc(ndims * sizeof(PIO_Offset))))
return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
if (!(countp = malloc(ndims * sizeof(PIO_Offset))))
return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);

/* Find the dimension lengths. */
for (int d = 0; d < ndims; d++)
if ((ierr = PIOc_inq_dimlen(ncid, dimids[d], &dimlen[d])))
if ((ierr = PIOc_inq_dimlen(ncid, dimids[d], &countp[d])))
return pio_err(ios, file, ierr, __FILE__, __LINE__);

/* Set up start array. */
for (int d = 0; d < ndims; d++)
{
my_start[d] = 0;
LOG((3, "my_start[%d] = %d dimlen[%d] = %d", d, my_start[d], d,
dimlen[d]));
startp[d] = 0;
LOG((3, "startp[%d] = %d countp[%d] = %d", d, startp[d], d,
countp[d]));
}

/* Set the start/count arrays. */
startp = my_start;
countp = dimlen;
}

return PIOc_get_vars_tc(ncid, varid, startp, countp, NULL, xtype, buf);
ierr = PIOc_get_vars_tc(ncid, varid, startp, countp, NULL, xtype, buf);
if(startp != NULL)
free(startp);
if(countp != NULL)
free(countp);
return ierr;

}

/**
Expand Down Expand Up @@ -1294,8 +1299,6 @@ int PIOc_put_var_tc(int ncid, int varid, nc_type xtype, const void *op)
file_desc_t *file; /* Pointer to file information. */
PIO_Offset *startp = NULL; /* Pointer to start array. */
PIO_Offset *countp = NULL; /* Pointer to count array. */
PIO_Offset start[PIO_MAX_DIMS];
PIO_Offset count[PIO_MAX_DIMS];
int ndims; /* The number of dimensions in the variable. */
int ierr; /* Return code from function calls. */

Expand All @@ -1316,24 +1319,30 @@ int PIOc_put_var_tc(int ncid, int varid, nc_type xtype, const void *op)
if (ndims)
{
int dimid[ndims];
if (!(startp = malloc(ndims * sizeof(PIO_Offset))))
return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
if (!(countp = malloc(ndims * sizeof(PIO_Offset))))
return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);

/* Set up start array. */
for (int d = 0; d < ndims; d++)
start[d] = 0;
startp[d] = 0;

/* Get the dimids for this var. */
if ((ierr = PIOc_inq_vardimid(ncid, varid, dimid)))
return check_netcdf(file, ierr, __FILE__, __LINE__);

/* Count array are the dimlens. */
for (int d = 0; d < ndims; d++)
if ((ierr = PIOc_inq_dimlen(ncid, dimid[d], &count[d])))
if ((ierr = PIOc_inq_dimlen(ncid, dimid[d], &countp[d])))
return pio_err(ios, file, ierr, __FILE__, __LINE__);

/* Set the array pointers. */
startp = start;
countp = count;
}

return PIOc_put_vars_tc(ncid, varid, startp, countp, NULL, xtype, op);
ierr = PIOc_put_vars_tc(ncid, varid, startp, countp, NULL, xtype, op);
if (startp != NULL)
free(startp);
if (countp != NULL)
free(countp);
return ierr;
}
40 changes: 26 additions & 14 deletions src/clib/pio_msg.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,8 @@ int create_file_handler(iosystem_desc_t *ios)

/* Call the create file function. */
PIOc_createfile(ios->iosysid, &ncid, &iotype, filename, mode);



LOG((1, "create_file_handler succeeded!"));
return PIO_NOERR;
}
Expand Down Expand Up @@ -276,7 +277,7 @@ int inq_handler(iosystem_desc_t *ios)

/* Call the inq function to get the values. */
PIOc_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp);

return PIO_NOERR;
}

Expand Down Expand Up @@ -1005,8 +1006,10 @@ int inq_var_handler(iosystem_desc_t *ios)
char name_present, xtype_present, ndims_present, dimids_present, natts_present;
char name[NC_MAX_NAME + 1], *namep = NULL;
nc_type xtype, *xtypep = NULL;
/* no easy way not to hard code this array length */
int dimids[8];
int *ndimsp = NULL, *dimidsp = NULL, *nattsp = NULL;
int ndims, dimids[NC_MAX_DIMS], natts;
int ndims, natts;
int mpierr;

LOG((1, "inq_var_handler"));
Expand Down Expand Up @@ -1040,7 +1043,7 @@ int inq_var_handler(iosystem_desc_t *ios)
if (ndims_present)
ndimsp = &ndims;
if (dimids_present)
dimidsp = dimids;
dimidsp = dimids;
if (natts_present)
nattsp = &natts;

Expand All @@ -1064,9 +1067,10 @@ int inq_var_chunking_handler(iosystem_desc_t *ios)
{
int ncid;
int varid;
int ndims;
char storage_present, chunksizes_present;
int storage, *storagep = NULL;
PIO_Offset chunksizes[NC_MAX_DIMS], *chunksizesp = NULL;
PIO_Offset *chunksizesp = NULL;
int mpierr;

assert(ios);
Expand All @@ -1080,6 +1084,8 @@ int inq_var_chunking_handler(iosystem_desc_t *ios)
return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
if ((mpierr = MPI_Bcast(&storage_present, 1, MPI_CHAR, 0, ios->intercomm)))
return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, 0, ios->intercomm)))
return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
if ((mpierr = MPI_Bcast(&chunksizes_present, 1, MPI_CHAR, 0, ios->intercomm)))
return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
LOG((2,"inq_var_handler ncid = %d varid = %d storage_present = %d chunksizes_present = %d",
Expand All @@ -1089,11 +1095,15 @@ int inq_var_chunking_handler(iosystem_desc_t *ios)
if (storage_present)
storagep = &storage;
if (chunksizes_present)
chunksizesp = chunksizes;
if (!(chunksizesp = malloc(ndims *sizeof(PIO_Offset))))
return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);

/* Call the inq function to get the values. */
PIOc_inq_var_chunking(ncid, varid, storagep, chunksizesp);

if(chunksizes_present)
free(chunksizesp);

return PIO_NOERR;
}

Expand Down Expand Up @@ -1492,7 +1502,7 @@ int def_var_chunking_handler(iosystem_desc_t *ios)
int ndims;
int storage;
char chunksizes_present;
PIO_Offset chunksizes[NC_MAX_DIMS], *chunksizesp = NULL;
PIO_Offset *chunksizesp = NULL;
int mpierr;

assert(ios);
Expand All @@ -1510,19 +1520,21 @@ int def_var_chunking_handler(iosystem_desc_t *ios)
return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
if ((mpierr = MPI_Bcast(&chunksizes_present, 1, MPI_CHAR, 0, ios->intercomm)))
return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
if (chunksizes_present)
if ((mpierr = MPI_Bcast(chunksizes, ndims, MPI_OFFSET, 0, ios->intercomm)))
if (chunksizes_present){
if (!(chunksizesp = malloc(ndims* sizeof(PIO_Offset))))
return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
if ((mpierr = MPI_Bcast(chunksizesp, ndims, MPI_OFFSET, 0, ios->intercomm)))
return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
}
LOG((1, "def_var_chunking_handler got parameters ncid = %d varid = %d storage = %d "
"ndims = %d chunksizes_present = %d", ncid, varid, storage, ndims, chunksizes_present));

/* Set the non-NULL pointers. */
if (chunksizes_present)
chunksizesp = chunksizes;

/* Call the function. */
PIOc_def_var_chunking(ncid, varid, storage, chunksizesp);

if(chunksizes_present)
free(chunksizesp);

LOG((1, "def_var_chunking_handler succeeded!"));
return PIO_NOERR;
}
Expand Down Expand Up @@ -2703,7 +2715,7 @@ int pio_msg_handler2(int io_rank, int component_count, iosystem_desc_t **iosys,
/* If an error was returned by the handler, exit. */
LOG((3, "pio_msg_handler2 ret %d msg %d index %d io_rank %d", ret, msg, index, io_rank));
if (ret)
return pio_err(my_iosys, NULL, ret, __FILE__, __LINE__);
return pio_err(my_iosys, NULL, ret, __FILE__, __LINE__);

/* Listen for another msg from the component whose message we
* just handled. */
Expand Down
2 changes: 2 additions & 0 deletions src/clib/pio_nc4.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -400,6 +400,8 @@ int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, PIO_Offset *chunks
mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
if (!mpierr)
mpierr = MPI_Bcast(&storage_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
if (!mpierr)
mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm);
if (!mpierr)
mpierr = MPI_Bcast(&chunksizes_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
LOG((2, "PIOc_inq_var_chunking ncid = %d varid = %d storage_present = %d chunksizes_present = %d",
Expand Down
10 changes: 5 additions & 5 deletions src/clib/pioc_support.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1937,7 +1937,7 @@ int check_unlim_use(int ncid)
* @author Ed Hartnett
*/
int inq_file_metadata(file_desc_t *file, int ncid, int iotype, int *nvars, int **rec_var,
int **pio_type, int **pio_type_size, int **mpi_type, int **mpi_type_size)
int **pio_type, int **pio_type_size, MPI_Datatype **mpi_type, int **mpi_type_size)
{
int nunlimdims; /* The number of unlimited dimensions. */
int unlimdimid;
Expand Down Expand Up @@ -1966,7 +1966,7 @@ int inq_file_metadata(file_desc_t *file, int ncid, int iotype, int *nvars, int *
return PIO_ENOMEM;
if (!(*pio_type_size = malloc(*nvars * sizeof(int))))
return PIO_ENOMEM;
if (!(*mpi_type = malloc(*nvars * sizeof(int))))
if (!(*mpi_type = malloc(*nvars * sizeof(MPI_Datatype))))
return PIO_ENOMEM;
if (!(*mpi_type_size = malloc(*nvars * sizeof(int))))
return PIO_ENOMEM;
Expand Down Expand Up @@ -2145,7 +2145,7 @@ int PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filena
int *rec_var = NULL;
int *pio_type = NULL;
int *pio_type_size = NULL;
int *mpi_type = NULL;
MPI_Datatype *mpi_type = NULL;
int *mpi_type_size = NULL;
int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */
int ierr = PIO_NOERR; /* Return code from function calls. */
Expand Down Expand Up @@ -2355,7 +2355,7 @@ int PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filena
return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
if (!(pio_type_size = malloc(nvars * sizeof(int))))
return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
if (!(mpi_type = malloc(nvars * sizeof(int))))
if (!(mpi_type = malloc(nvars * sizeof(MPI_Datatype))))
return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
if (!(mpi_type_size = malloc(nvars * sizeof(int))))
return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
Expand All @@ -2368,7 +2368,7 @@ int PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filena
return check_mpi(file, mpierr, __FILE__, __LINE__);
if ((mpierr = MPI_Bcast(pio_type_size, nvars, MPI_INT, ios->ioroot, ios->my_comm)))
return check_mpi(file, mpierr, __FILE__, __LINE__);
if ((mpierr = MPI_Bcast(mpi_type, nvars, MPI_INT, ios->ioroot, ios->my_comm)))
if ((mpierr = MPI_Bcast(mpi_type, nvars*(int)(sizeof(MPI_Datatype)/sizeof(int)), MPI_INT, ios->ioroot, ios->my_comm)))
return check_mpi(file, mpierr, __FILE__, __LINE__);
if ((mpierr = MPI_Bcast(mpi_type_size, nvars, MPI_INT, ios->ioroot, ios->my_comm)))
return check_mpi(file, mpierr, __FILE__, __LINE__);
Expand Down
Loading

0 comments on commit a920f03

Please sign in to comment.