From a920f03930074e4f4e359f8b692111f5a40349c9 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 12 Dec 2017 16:06:22 -0700 Subject: [PATCH] Squashed 'src/externals/pio2/' changes from c84f74c..0f241db 0f241db response to comments 1007a7a cannot predetermin ndims here 99ef07d Merge pull request #1241 from NCAR/free_new_allocs 29ed162 free recently allocated vars fbc3584 Merge pull request #1239 from NCAR/dontuse_nc_max 63dee3d Merge pull request #1240 from NCAR/limitto2GiB 64f2492 limit to 2GiB due to romio bug 29aee05 dont use NC_MAX values d831ad3 Merge pull request #1231 from mgduda/mpi_type_fix e996bdb Merge pull request #1222 from NCAR/ejh_autoconf_logging 426af22 Partial fix for incorrect type of 'mpi_type' in pioc_support.c 7eb724f added enable-logging option to autotools build git-subtree-dir: src/externals/pio2 git-subtree-split: 0f241db88cfee1912a2769a052dba0d2d79f83d5 --- configure.ac | 13 ++++++++-- ctest/runcdash-nwscla-intel.sh | 6 ++--- doc/source/mach_walkthrough.txt | 4 +-- src/clib/pio_darray.c | 12 ++++++--- src/clib/pio_getput_int.c | 45 ++++++++++++++++++++------------- src/clib/pio_msg.c | 40 +++++++++++++++++++---------- src/clib/pio_nc4.c | 2 ++ src/clib/pioc_support.c | 10 ++++---- src/flib/pio_types.F90 | 10 ++++---- 9 files changed, 90 insertions(+), 52 deletions(-) diff --git a/configure.ac b/configure.ac index bfddcde9606..be47a262a60 100644 --- a/configure.ac +++ b/configure.ac @@ -26,8 +26,17 @@ AC_ARG_WITH([piobuffersize], AC_MSG_RESULT([$PIO_BUFFER_SIZE]) AC_DEFINE_UNQUOTED([PIO_BUFFER_SIZE], [$PIO_BUFFER_SIZE], [buffer size for darray data.]) -# Need to allow user to set this. -AC_DEFINE([PIO_ENABLE_LOGGING], [1], [log messages from library]) +# Does the user want to enable logging? +AC_MSG_CHECKING([whether debug logging is enabled]) +AC_ARG_ENABLE([logging], + [AS_HELP_STRING([--enable-logging], + [enable debug logging capability (will negatively impact performance). \ + This debugging feature is probably only of interest to PIO developers.])]) +test "x$enable_logging" = xyes || enable_logging=no +AC_MSG_RESULT([$enable_logging]) +if test "x$enable_logging" = xyes; then + AC_DEFINE([PIO_ENABLE_LOGGING], 1, [If true, turn on logging.]) +fi # NetCDF (at least classic) is required for PIO to build. AC_DEFINE([_NETCDF], [1], [netCDF classic library available]) diff --git a/ctest/runcdash-nwscla-intel.sh b/ctest/runcdash-nwscla-intel.sh index 630ac838085..d7a7f8c9120 100755 --- a/ctest/runcdash-nwscla-intel.sh +++ b/ctest/runcdash-nwscla-intel.sh @@ -14,14 +14,14 @@ module unload netcdf module swap intel intel/17.0.1 module load cmake/3.7.2 module load netcdf-mpi/4.4.1.1 -module load pnetcdf/1.8.0 -module switch mpt mpt/2.15 +module load pnetcdf/1.8.1 +module switch mpt mpt/2.16 echo "MODULE LIST..." module list export CC=mpicc export FC=mpif90 - +export MPI_TYPE_DEPTH=24 export PIO_DASHBOARD_ROOT=/glade/scratch/jedwards/dashboard export PIO_COMPILER_ID=Intel-`$CC --version | head -n 1 | cut -d' ' -f3` diff --git a/doc/source/mach_walkthrough.txt b/doc/source/mach_walkthrough.txt index bae5540da85..05cd8776d65 100644 --- a/doc/source/mach_walkthrough.txt +++ b/doc/source/mach_walkthrough.txt @@ -454,10 +454,10 @@ which causes the build commands to be made visible. autoreconf -i && LD_LIBRARY_PATH=/usr/local/netcdf-4.4.1_mpich-3.2/lib CC=mpicc CFLAGS='-g' CPPFLAGS='-I/usr/local/netcdf-4.4.1_mpich-3.2/include/ -I/usr/local/pnetcdf-1.8.1_mpich-3.2/include' LDFLAGS='-L/usr/local/netcdf-4.4.1_mpich-3.2/lib -L/usr/local/pnetcdf-1.8.1_mpich-3.2/lib' ./configure && make check -

To build with the address sanitizer for memory checking (debugging builds only!): +

To build with debug logging and the address sanitizer for memory checking (debugging builds only!): 

-autoreconf -i && LD_LIBRARY_PATH=/usr/local/netcdf-4.4.1_mpich-3.2/lib CC=mpicc CFLAGS='-g -fsanitize=address -fno-omit-frame-pointer' CPPFLAGS='-I/usr/local/netcdf-4.4.1_mpich-3.2/include/ -I/usr/local/pnetcdf-1.8.1_mpich-3.2/include' LDFLAGS='-L/usr/local/netcdf-4.4.1_mpich-3.2/lib -L/usr/local/pnetcdf-1.8.1_mpich-3.2/lib' ./configure && make check
+autoreconf -i && LD_LIBRARY_PATH=/usr/local/netcdf-4.4.1_mpich-3.2/lib CC=mpicc CFLAGS='-g -fsanitize=address -fno-omit-frame-pointer' CPPFLAGS='-I/usr/local/netcdf-4.4.1_mpich-3.2/include/ -I/usr/local/pnetcdf-1.8.1_mpich-3.2/include' LDFLAGS='-L/usr/local/netcdf-4.4.1_mpich-3.2/lib -L/usr/local/pnetcdf-1.8.1_mpich-3.2/lib' ./configure --enable-logging && make check
  • Building and Running Performance Tests diff --git a/src/clib/pio_darray.c b/src/clib/pio_darray.c index a4fc6e558f7..4f205cede7a 100644 --- a/src/clib/pio_darray.c +++ b/src/clib/pio_darray.c @@ -485,6 +485,7 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void * #endif int mpierr = MPI_SUCCESS; /* Return code from MPI functions. */ int ierr = PIO_NOERR; /* Return code. */ + size_t io_data_size; /* potential size of data on io task */ LOG((1, "PIOc_write_darray ncid = %d varid = %d ioid = %d arraylen = %d", ncid, varid, ioid, arraylen)); @@ -532,7 +533,7 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void * if (fillvalue) if (memcmp(fillvalue, vdesc->fillvalue, vdesc->pio_type_size)) return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__); - + /* Move to end of list or the entry that matches this ioid. */ for (wmb = &file->buffer; wmb->next; wmb = wmb->next) if (wmb->ioid == ioid && wmb->recordvar == vdesc->rec_var) @@ -560,13 +561,12 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void * } LOG((2, "wmb->num_arrays = %d arraylen = %d vdesc->mpi_type_size = %d\n", wmb->num_arrays, arraylen, vdesc->mpi_type_size)); - #if PIO_USE_MALLOC /* Try realloc first and call flush if realloc fails. */ if (arraylen > 0) { size_t data_size = (1 + wmb->num_arrays) * arraylen * vdesc->mpi_type_size; - + if ((realloc_data = realloc(wmb->data, data_size))) { needsflush = 0; @@ -588,6 +588,12 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void * if (needsflush == 0) needsflush = (maxfree <= 1.1 * (1 + wmb->num_arrays) * arraylen * vdesc->mpi_type_size); #endif + /* the limit of data_size < INT_MAX is due to a bug in ROMIO which limits + the size of contiguous data to INT_MAX, a fix has been proposed in + https://github.com/pmodels/mpich/pull/2888 */ + io_data_size = (1 + wmb->num_arrays) * iodesc->maxiobuflen * vdesc->mpi_type_size; + if(io_data_size > INT_MAX) + needsflush = 2; /* Tell all tasks on the computation communicator whether we need * to flush data. */ diff --git a/src/clib/pio_getput_int.c b/src/clib/pio_getput_int.c index 791cce1b1bd..6ae26141d6d 100644 --- a/src/clib/pio_getput_int.c +++ b/src/clib/pio_getput_int.c @@ -797,8 +797,6 @@ int PIOc_get_var_tc(int ncid, int varid, nc_type xtype, void *buf) PIO_Offset *startp = NULL; /* Pointer to start array. */ PIO_Offset *countp = NULL; /* Pointer to count array. */ int ndims; /* The number of dimensions in the variable. */ - PIO_Offset my_start[PIO_MAX_DIMS]; - PIO_Offset dimlen[PIO_MAX_DIMS]; int ierr; /* Return code from function calls. */ LOG((1, "PIOc_get_var_tc ncid = %d varid = %d xtype = %d", ncid, varid, @@ -821,26 +819,33 @@ int PIOc_get_var_tc(int ncid, int varid, nc_type xtype, void *buf) int dimids[ndims]; if ((ierr = PIOc_inq_vardimid(ncid, varid, dimids))) return pio_err(ios, file, ierr, __FILE__, __LINE__); + if (!(startp = malloc(ndims * sizeof(PIO_Offset)))) + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); + if (!(countp = malloc(ndims * sizeof(PIO_Offset)))) + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); /* Find the dimension lengths. */ for (int d = 0; d < ndims; d++) - if ((ierr = PIOc_inq_dimlen(ncid, dimids[d], &dimlen[d]))) + if ((ierr = PIOc_inq_dimlen(ncid, dimids[d], &countp[d]))) return pio_err(ios, file, ierr, __FILE__, __LINE__); /* Set up start array. */ for (int d = 0; d < ndims; d++) { - my_start[d] = 0; - LOG((3, "my_start[%d] = %d dimlen[%d] = %d", d, my_start[d], d, - dimlen[d])); + startp[d] = 0; + LOG((3, "startp[%d] = %d countp[%d] = %d", d, startp[d], d, + countp[d])); } - /* Set the start/count arrays. */ - startp = my_start; - countp = dimlen; } - return PIOc_get_vars_tc(ncid, varid, startp, countp, NULL, xtype, buf); + ierr = PIOc_get_vars_tc(ncid, varid, startp, countp, NULL, xtype, buf); + if(startp != NULL) + free(startp); + if(countp != NULL) + free(countp); + return ierr; + } /** @@ -1294,8 +1299,6 @@ int PIOc_put_var_tc(int ncid, int varid, nc_type xtype, const void *op) file_desc_t *file; /* Pointer to file information. */ PIO_Offset *startp = NULL; /* Pointer to start array. */ PIO_Offset *countp = NULL; /* Pointer to count array. */ - PIO_Offset start[PIO_MAX_DIMS]; - PIO_Offset count[PIO_MAX_DIMS]; int ndims; /* The number of dimensions in the variable. */ int ierr; /* Return code from function calls. */ @@ -1316,10 +1319,14 @@ int PIOc_put_var_tc(int ncid, int varid, nc_type xtype, const void *op) if (ndims) { int dimid[ndims]; + if (!(startp = malloc(ndims * sizeof(PIO_Offset)))) + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); + if (!(countp = malloc(ndims * sizeof(PIO_Offset)))) + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); /* Set up start array. */ for (int d = 0; d < ndims; d++) - start[d] = 0; + startp[d] = 0; /* Get the dimids for this var. */ if ((ierr = PIOc_inq_vardimid(ncid, varid, dimid))) @@ -1327,13 +1334,15 @@ int PIOc_put_var_tc(int ncid, int varid, nc_type xtype, const void *op) /* Count array are the dimlens. */ for (int d = 0; d < ndims; d++) - if ((ierr = PIOc_inq_dimlen(ncid, dimid[d], &count[d]))) + if ((ierr = PIOc_inq_dimlen(ncid, dimid[d], &countp[d]))) return pio_err(ios, file, ierr, __FILE__, __LINE__); - /* Set the array pointers. */ - startp = start; - countp = count; } - return PIOc_put_vars_tc(ncid, varid, startp, countp, NULL, xtype, op); + ierr = PIOc_put_vars_tc(ncid, varid, startp, countp, NULL, xtype, op); + if (startp != NULL) + free(startp); + if (countp != NULL) + free(countp); + return ierr; } diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index c5b10244511..eb5e09d9827 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -190,7 +190,8 @@ int create_file_handler(iosystem_desc_t *ios) /* Call the create file function. */ PIOc_createfile(ios->iosysid, &ncid, &iotype, filename, mode); - + + LOG((1, "create_file_handler succeeded!")); return PIO_NOERR; } @@ -276,7 +277,7 @@ int inq_handler(iosystem_desc_t *ios) /* Call the inq function to get the values. */ PIOc_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp); - + return PIO_NOERR; } @@ -1005,8 +1006,10 @@ int inq_var_handler(iosystem_desc_t *ios) char name_present, xtype_present, ndims_present, dimids_present, natts_present; char name[NC_MAX_NAME + 1], *namep = NULL; nc_type xtype, *xtypep = NULL; + /* no easy way not to hard code this array length */ + int dimids[8]; int *ndimsp = NULL, *dimidsp = NULL, *nattsp = NULL; - int ndims, dimids[NC_MAX_DIMS], natts; + int ndims, natts; int mpierr; LOG((1, "inq_var_handler")); @@ -1040,7 +1043,7 @@ int inq_var_handler(iosystem_desc_t *ios) if (ndims_present) ndimsp = &ndims; if (dimids_present) - dimidsp = dimids; + dimidsp = dimids; if (natts_present) nattsp = &natts; @@ -1064,9 +1067,10 @@ int inq_var_chunking_handler(iosystem_desc_t *ios) { int ncid; int varid; + int ndims; char storage_present, chunksizes_present; int storage, *storagep = NULL; - PIO_Offset chunksizes[NC_MAX_DIMS], *chunksizesp = NULL; + PIO_Offset *chunksizesp = NULL; int mpierr; assert(ios); @@ -1080,6 +1084,8 @@ int inq_var_chunking_handler(iosystem_desc_t *ios) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(&storage_present, 1, MPI_CHAR, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(&chunksizes_present, 1, MPI_CHAR, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); LOG((2,"inq_var_handler ncid = %d varid = %d storage_present = %d chunksizes_present = %d", @@ -1089,11 +1095,15 @@ int inq_var_chunking_handler(iosystem_desc_t *ios) if (storage_present) storagep = &storage; if (chunksizes_present) - chunksizesp = chunksizes; + if (!(chunksizesp = malloc(ndims *sizeof(PIO_Offset)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); /* Call the inq function to get the values. */ PIOc_inq_var_chunking(ncid, varid, storagep, chunksizesp); + if(chunksizes_present) + free(chunksizesp); + return PIO_NOERR; } @@ -1492,7 +1502,7 @@ int def_var_chunking_handler(iosystem_desc_t *ios) int ndims; int storage; char chunksizes_present; - PIO_Offset chunksizes[NC_MAX_DIMS], *chunksizesp = NULL; + PIO_Offset *chunksizesp = NULL; int mpierr; assert(ios); @@ -1510,19 +1520,21 @@ int def_var_chunking_handler(iosystem_desc_t *ios) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(&chunksizes_present, 1, MPI_CHAR, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); - if (chunksizes_present) - if ((mpierr = MPI_Bcast(chunksizes, ndims, MPI_OFFSET, 0, ios->intercomm))) + if (chunksizes_present){ + if (!(chunksizesp = malloc(ndims* sizeof(PIO_Offset)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(chunksizesp, ndims, MPI_OFFSET, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + } LOG((1, "def_var_chunking_handler got parameters ncid = %d varid = %d storage = %d " "ndims = %d chunksizes_present = %d", ncid, varid, storage, ndims, chunksizes_present)); - /* Set the non-NULL pointers. */ - if (chunksizes_present) - chunksizesp = chunksizes; - /* Call the function. */ PIOc_def_var_chunking(ncid, varid, storage, chunksizesp); + if(chunksizes_present) + free(chunksizesp); + LOG((1, "def_var_chunking_handler succeeded!")); return PIO_NOERR; } @@ -2703,7 +2715,7 @@ int pio_msg_handler2(int io_rank, int component_count, iosystem_desc_t **iosys, /* If an error was returned by the handler, exit. */ LOG((3, "pio_msg_handler2 ret %d msg %d index %d io_rank %d", ret, msg, index, io_rank)); if (ret) - return pio_err(my_iosys, NULL, ret, __FILE__, __LINE__); + return pio_err(my_iosys, NULL, ret, __FILE__, __LINE__); /* Listen for another msg from the component whose message we * just handled. */ diff --git a/src/clib/pio_nc4.c b/src/clib/pio_nc4.c index 086bbf699f8..62ceb080c43 100644 --- a/src/clib/pio_nc4.c +++ b/src/clib/pio_nc4.c @@ -400,6 +400,8 @@ int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, PIO_Offset *chunks mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&storage_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&chunksizes_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); LOG((2, "PIOc_inq_var_chunking ncid = %d varid = %d storage_present = %d chunksizes_present = %d", diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c index 5f97a9fad1a..9bee1ec65a2 100644 --- a/src/clib/pioc_support.c +++ b/src/clib/pioc_support.c @@ -1937,7 +1937,7 @@ int check_unlim_use(int ncid) * @author Ed Hartnett */ int inq_file_metadata(file_desc_t *file, int ncid, int iotype, int *nvars, int **rec_var, - int **pio_type, int **pio_type_size, int **mpi_type, int **mpi_type_size) + int **pio_type, int **pio_type_size, MPI_Datatype **mpi_type, int **mpi_type_size) { int nunlimdims; /* The number of unlimited dimensions. */ int unlimdimid; @@ -1966,7 +1966,7 @@ int inq_file_metadata(file_desc_t *file, int ncid, int iotype, int *nvars, int * return PIO_ENOMEM; if (!(*pio_type_size = malloc(*nvars * sizeof(int)))) return PIO_ENOMEM; - if (!(*mpi_type = malloc(*nvars * sizeof(int)))) + if (!(*mpi_type = malloc(*nvars * sizeof(MPI_Datatype)))) return PIO_ENOMEM; if (!(*mpi_type_size = malloc(*nvars * sizeof(int)))) return PIO_ENOMEM; @@ -2145,7 +2145,7 @@ int PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filena int *rec_var = NULL; int *pio_type = NULL; int *pio_type_size = NULL; - int *mpi_type = NULL; + MPI_Datatype *mpi_type = NULL; int *mpi_type_size = NULL; int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ int ierr = PIO_NOERR; /* Return code from function calls. */ @@ -2355,7 +2355,7 @@ int PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filena return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); if (!(pio_type_size = malloc(nvars * sizeof(int)))) return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); - if (!(mpi_type = malloc(nvars * sizeof(int)))) + if (!(mpi_type = malloc(nvars * sizeof(MPI_Datatype)))) return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); if (!(mpi_type_size = malloc(nvars * sizeof(int)))) return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); @@ -2368,7 +2368,7 @@ int PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filena return check_mpi(file, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(pio_type_size, nvars, MPI_INT, ios->ioroot, ios->my_comm))) return check_mpi(file, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Bcast(mpi_type, nvars, MPI_INT, ios->ioroot, ios->my_comm))) + if ((mpierr = MPI_Bcast(mpi_type, nvars*(int)(sizeof(MPI_Datatype)/sizeof(int)), MPI_INT, ios->ioroot, ios->my_comm))) return check_mpi(file, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(mpi_type_size, nvars, MPI_INT, ios->ioroot, ios->my_comm))) return check_mpi(file, mpierr, __FILE__, __LINE__); diff --git a/src/flib/pio_types.F90 b/src/flib/pio_types.F90 index d8bb4a3f019..1e8fb9532e4 100644 --- a/src/flib/pio_types.F90 +++ b/src/flib/pio_types.F90 @@ -186,7 +186,7 @@ module pio_types integer, public, parameter :: PIO_NOCLOBBER = nf_NOclobber integer, public, parameter :: PIO_NOFILL = nf_nofill integer, public, parameter :: PIO_MAX_NAME = nf_max_name - integer, public, parameter :: PIO_MAX_VAR_DIMS = nf_max_var_dims + integer, public, parameter :: PIO_MAX_VAR_DIMS = min(6,nf_max_var_dims) integer, public, parameter :: PIO_64BIT_OFFSET = nf_64bit_offset integer, public, parameter :: PIO_64BIT_DATA = nf_64bit_data integer, public, parameter :: PIO_FILL_INT = nf_fill_int; @@ -209,7 +209,7 @@ module pio_types integer, public, parameter :: PIO_NOCLOBBER = nf_NOclobber integer, public, parameter :: PIO_NOFILL = nf_nofill integer, public, parameter :: PIO_MAX_NAME = nf_max_name - integer, public, parameter :: PIO_MAX_VAR_DIMS = nf_max_var_dims + integer, public, parameter :: PIO_MAX_VAR_DIMS = min(6,nf_max_var_dims) integer, public, parameter :: PIO_64BIT_OFFSET = nf_64bit_offset integer, public, parameter :: PIO_64BIT_DATA = 0 integer, public, parameter :: PIO_FILL_INT = nf_fill_int; @@ -239,7 +239,7 @@ module pio_types !> !! @defgroup PIO_rearr_comm_t PIO_rearr_comm_t -!! @public +!! @public !! @brief The two choices for rearranger communication !! @details !! - PIO_rearr_comm_p2p : Point to point @@ -252,7 +252,7 @@ module pio_types !> !! @defgroup PIO_rearr_comm_dir PIO_rearr_comm_dir -!! @public +!! @public !! @brief The four choices for rearranger communication direction !! @details !! - PIO_rearr_comm_fc_2d_enable : COMM procs to IO procs and vice versa @@ -271,7 +271,7 @@ module pio_types !! @defgroup PIO_rearr_comm_fc_options PIO_rearr_comm_fc_options !! @brief Type that defines the PIO rearranger options !! @details -!! - enable_hs : Enable handshake (true/false) +!! - enable_hs : Enable handshake (true/false) !! - enable_isend : Enable Isends (true/false) !! - max_pend_req : Maximum pending requests (To indicated unlimited !! number of requests use PIO_REARR_COMM_UNLIMITED_PEND_REQ)